From 42985f4f7476de242ae5f07284a8eeed9f756492 Mon Sep 17 00:00:00 2001 From: Mike Date: Fri, 25 Sep 2026 12:07:54 -0500 Subject: [PATCH 1/3] initial refactoring of url selection --- rescript/get_gtdb.py | 19 +++++----- rescript/plugin_setup.py | 12 ++++--- rescript/tests/test_get_gtdb.py | 63 ++++++++++++++++++++++----------- 3 files changed, 60 insertions(+), 34 deletions(-) diff --git a/rescript/get_gtdb.py b/rescript/get_gtdb.py index d74a7e3..d6f7176 100644 --- a/rescript/get_gtdb.py +++ b/rescript/get_gtdb.py @@ -34,32 +34,35 @@ def get_gtdb_data( + db_url: str, version: str = '232.0', domain: str = 'Both', db_type: str = 'SpeciesReps', - url_type: str = 'Primary' ) -> (TSVTaxonomyFormat, DNAFASTAFormat): - queries = _assemble_queries(version=version, + queries = _assemble_queries(db_url=db_url, + version=version, db_type=db_type, - domain=domain, - url_type=url_type) + domain=domain) tax_q, seqs_q = _retrieve_data_from_gtdb(queries) print('\n Saving files...\n') return tax_q, seqs_q -def _assemble_queries(version='232.0', +def _assemble_queries(db_url, + version='232.0', db_type='SpeciesReps', domain='Both', - url_type='Primary'): + ): queries = [] - if url_type == 'Primary': + if db_url == 'australia': base_url = 'https://data.gtdb.ecogenomic.org/releases/' - elif url_type == 'Mirror': + elif db_url == 'world-wide-asia': base_url = 'https://data.ace.uq.edu.au/public/gtdb/data/releases/' + elif db_url == 'world-wide-europe': + base_url = 'https://data.gtdb.aau.ecogenomic.org/releases/' base_version = version.split('.')[0] # ^^ Set `base_version` variable becuase number after the decimal is diff --git a/rescript/plugin_setup.py b/rescript/plugin_setup.py index 87d0720..e619452 100644 --- a/rescript/plugin_setup.py +++ b/rescript/plugin_setup.py @@ -930,7 +930,8 @@ '220.0', '226.0', '232.0']), 'domain': Str % Choices(['Both', 'Bacteria', 'Archaea']), 'db_type': Str % Choices(['All', 'SpeciesReps']), - 'url_type': Str % Choices(['Primary', 'Mirror']) + 'db_url': Str % Choices(['world-wide-europe', + 'world-wide-asia', 'australia']) }, outputs=[('gtdb_taxonomy', FeatureData[Taxonomy]), ('gtdb_sequences', FeatureData[Sequence])], @@ -951,10 +952,11 @@ 'parameter will be ignored as GTDB does not maintain ' 'separate domain-level files for these non-clustered ' 'data.', - 'url_type': 'Toggle download URL. \'Primary\' will download data ' - 'from the primary GTDB URL. \'Mirror\' will dowload data ' - 'from the GTDB data mirror. Use \'Mirror\' if downloads ' - 'from \'Primary\' are slow.'}, + 'db_url': 'Toggle download URL. \'world-wide-europe\' will download ' + 'data from European servers. \'world-wide-asia\' will ' + 'download data from Asian-Pacific servers. and ' + '\'australia\' will download from Australian servers. ' + 'Choose the server that is best for your reagion.'}, output_descriptions={ 'gtdb_taxonomy': 'SSU GTDB reference taxonomy.', 'gtdb_sequences': 'SSU GTDB reference sequences.'}, diff --git a/rescript/tests/test_get_gtdb.py b/rescript/tests/test_get_gtdb.py index c75d9d0..1b40b78 100644 --- a/rescript/tests/test_get_gtdb.py +++ b/rescript/tests/test_get_gtdb.py @@ -48,54 +48,71 @@ def setUp(self): def test_assemble_species_rep_queries(self): # checking v220 as GTDB updated the file format for the "ssu_rep" # FASTA files to 'fna.gz' from their usual 'tar.gz'. - obs_query_urls = _assemble_queries('220.0', 'SpeciesReps', 'Both') + obs_query_urls = _assemble_queries('world-wide-europe', + '220.0', 'SpeciesReps', 'Both') print('obs queries: ', obs_query_urls) exp_query_urls = [('Archaea', - 'https://data.gtdb.ecogenomic.org/releases/' + 'https://data.gtdb.aau.ecogenomic.org/releases/' 'release220/220.0/genomic_files_reps/' 'ar53_ssu_reps_r220.fna.gz'), ('Bacteria', - 'https://data.gtdb.ecogenomic.org/releases/' + 'https://data.gtdb.aau.ecogenomic.org/releases/' 'release220/220.0/genomic_files_reps/' 'bac120_ssu_reps_r220.fna.gz')] print('exp queries: ', exp_query_urls) self.assertEqual(obs_query_urls, exp_query_urls) def test_assemble_species_rep_queries_archaea(self): - obs_query_urls = _assemble_queries('202.0', 'SpeciesReps', 'Archaea') + obs_query_urls = _assemble_queries('world-wide-europe', '202.0', + 'SpeciesReps', 'Archaea') print('obs queries: ', obs_query_urls) exp_query_urls = [('Archaea', - 'https://data.gtdb.ecogenomic.org/releases/' + 'https://data.gtdb.aau.ecogenomic.org/releases/' 'release202/202.0/genomic_files_reps/' 'ar122_ssu_reps_r202.tar.gz')] print('exp queries: ', exp_query_urls) self.assertEqual(obs_query_urls, exp_query_urls) def test_assemble_queries_all_primary(self): - obs_query_urls = _assemble_queries(version='207.0', - db_type='All', - url_type='Primary') + obs_query_urls = _assemble_queries(db_url='world-wide-europe', + version='207.0', + db_type='All') print('obs queries: ', obs_query_urls) exp_query_urls = [('All', - 'https://data.gtdb.ecogenomic.org/releases/' + 'https://data.gtdb.aau.ecogenomic.org/releases/' 'release207/207.0/genomic_files_all/' 'ssu_all_r207.tar.gz')] print('exp queries: ', exp_query_urls) self.assertEqual(obs_query_urls, exp_query_urls) - def test_assemble_queries_all_mirror(self): - obs_query_urls = _assemble_queries(version='207.0', - db_type='All', - url_type='Mirror') + def test_assemble_queries_all_asia(self): + obs_query_urls = _assemble_queries(db_url='world-wide-asia', + version='207.0', + db_type='All') print('obs queries: ', obs_query_urls) - exp_query_urls = [('All', - 'https://data.ace.uq.edu.au/public/gtdb/data/' - 'releases/release207/207.0/genomic_files_all/' - 'ssu_all_r207.tar.gz')] + exp_query_urls = [( + 'All', + 'https://data.ace.uq.edu.au/public/gtdb/data/releases/' + 'release207/207.0/genomic_files_all/' + 'ssu_all_r207.tar.gz')] + print('exp queries: ', exp_query_urls) + self.assertEqual(obs_query_urls, exp_query_urls) + + def test_assemble_queries_all_australia(self): + obs_query_urls = _assemble_queries(db_url='australia', + version='232.0', + db_type='All') + print('obs queries: ', obs_query_urls) + + exp_query_urls = [( + 'All', + 'https://data.gtdb.ecogenomic.org/releases/' + 'release232/232.0/genomic_files_all/' + 'ssu_all_r232.fna.gz')] print('exp queries: ', exp_query_urls) self.assertEqual(obs_query_urls, exp_query_urls) @@ -113,7 +130,8 @@ def _makey_fakey_bact(faking_ignore_this): with patch('rescript.get_gtdb._retrieve_data_from_gtdb', new=_makey_fakey_both): res = rescript.actions.get_gtdb_data( - version='214.1', db_type='SpeciesReps', domain='Both') + db_url='world-wide-europe', version='214.1', + db_type='SpeciesReps', domain='Both') self.assertEqual(str(res[0].type), 'FeatureData[Taxonomy]') self.assertEqual(str(res[1].type), 'FeatureData[Sequence]') @@ -121,7 +139,8 @@ def _makey_fakey_bact(faking_ignore_this): with patch('rescript.get_gtdb._retrieve_data_from_gtdb', new=_makey_fakey_arch): resa = rescript.actions.get_gtdb_data( - version='202.0', domain='Archaea') + db_url='world-wide-europe', version='202.0', + domain='Archaea') self.assertEqual(str(resa[0].type), 'FeatureData[Taxonomy]') self.assertEqual(str(resa[1].type), 'FeatureData[Sequence]') @@ -129,7 +148,8 @@ def _makey_fakey_bact(faking_ignore_this): with patch('rescript.get_gtdb._retrieve_data_from_gtdb', new=_makey_fakey_bact): resb = rescript.actions.get_gtdb_data( - version='207.0', domain='Bacteria') + db_url='world-wide-europe', version='207.0', + domain='Bacteria') self.assertEqual(str(resb[0].type), 'FeatureData[Taxonomy]') self.assertEqual(str(resb[1].type), 'FeatureData[Sequence]') @@ -137,7 +157,8 @@ def _makey_fakey_bact(faking_ignore_this): with patch('rescript.get_gtdb._retrieve_data_from_gtdb', new=_makey_fakey_both): resc = rescript.actions.get_gtdb_data( - version='214.1', db_type='All') + db_url='world-wide-europe', version='214.1', + db_type='All') self.assertEqual(str(resc[0].type), 'FeatureData[Taxonomy]') self.assertEqual(str(resc[1].type), 'FeatureData[Sequence]') From 8b0f597972b350a6fdc4775fc21e8b46c3706ff5 Mon Sep 17 00:00:00 2001 From: Mike Date: Tue, 29 Sep 2026 09:57:17 -0500 Subject: [PATCH 2/3] make world-wide-europe default and fixed help-text typo --- rescript/get_gtdb.py | 2 +- rescript/plugin_setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/rescript/get_gtdb.py b/rescript/get_gtdb.py index d6f7176..fbfc93c 100644 --- a/rescript/get_gtdb.py +++ b/rescript/get_gtdb.py @@ -34,7 +34,7 @@ def get_gtdb_data( - db_url: str, + db_url: str = 'world-wide-europe', version: str = '232.0', domain: str = 'Both', db_type: str = 'SpeciesReps', diff --git a/rescript/plugin_setup.py b/rescript/plugin_setup.py index e619452..7f5bd5e 100644 --- a/rescript/plugin_setup.py +++ b/rescript/plugin_setup.py @@ -956,7 +956,7 @@ 'data from European servers. \'world-wide-asia\' will ' 'download data from Asian-Pacific servers. and ' '\'australia\' will download from Australian servers. ' - 'Choose the server that is best for your reagion.'}, + 'Choose the server that is best for your region.'}, output_descriptions={ 'gtdb_taxonomy': 'SSU GTDB reference taxonomy.', 'gtdb_sequences': 'SSU GTDB reference sequences.'}, From b7d593abbf610aefe7fde1399392e0827f4f2768 Mon Sep 17 00:00:00 2001 From: Mike Date: Tue, 29 Sep 2026 10:13:18 -0500 Subject: [PATCH 3/3] rearranged parameter order in plugin_setup, other cosmetic test code changes --- rescript/get_gtdb.py | 12 ++++----- rescript/tests/test_get_gtdb.py | 44 +++++++++++++++++---------------- 2 files changed, 29 insertions(+), 27 deletions(-) diff --git a/rescript/get_gtdb.py b/rescript/get_gtdb.py index fbfc93c..dc5575c 100644 --- a/rescript/get_gtdb.py +++ b/rescript/get_gtdb.py @@ -34,26 +34,26 @@ def get_gtdb_data( - db_url: str = 'world-wide-europe', version: str = '232.0', domain: str = 'Both', db_type: str = 'SpeciesReps', + db_url: str = 'world-wide-europe' ) -> (TSVTaxonomyFormat, DNAFASTAFormat): - queries = _assemble_queries(db_url=db_url, - version=version, + queries = _assemble_queries(version=version, db_type=db_type, - domain=domain) + domain=domain, + db_url=db_url) tax_q, seqs_q = _retrieve_data_from_gtdb(queries) print('\n Saving files...\n') return tax_q, seqs_q -def _assemble_queries(db_url, - version='232.0', +def _assemble_queries(version='232.0', db_type='SpeciesReps', domain='Both', + db_url='world-wide-europe', ): queries = [] diff --git a/rescript/tests/test_get_gtdb.py b/rescript/tests/test_get_gtdb.py index 1b40b78..a9384b8 100644 --- a/rescript/tests/test_get_gtdb.py +++ b/rescript/tests/test_get_gtdb.py @@ -48,8 +48,8 @@ def setUp(self): def test_assemble_species_rep_queries(self): # checking v220 as GTDB updated the file format for the "ssu_rep" # FASTA files to 'fna.gz' from their usual 'tar.gz'. - obs_query_urls = _assemble_queries('world-wide-europe', - '220.0', 'SpeciesReps', 'Both') + obs_query_urls = _assemble_queries('220.0', 'SpeciesReps', + 'Both', 'world-wide-europe') print('obs queries: ', obs_query_urls) exp_query_urls = [('Archaea', @@ -64,8 +64,10 @@ def test_assemble_species_rep_queries(self): self.assertEqual(obs_query_urls, exp_query_urls) def test_assemble_species_rep_queries_archaea(self): - obs_query_urls = _assemble_queries('world-wide-europe', '202.0', - 'SpeciesReps', 'Archaea') + obs_query_urls = _assemble_queries(version='202.0', + domain='Archaea', + db_type='SpeciesReps', + db_url='world-wide-europe') print('obs queries: ', obs_query_urls) exp_query_urls = [('Archaea', @@ -76,9 +78,9 @@ def test_assemble_species_rep_queries_archaea(self): self.assertEqual(obs_query_urls, exp_query_urls) def test_assemble_queries_all_primary(self): - obs_query_urls = _assemble_queries(db_url='world-wide-europe', - version='207.0', - db_type='All') + obs_query_urls = _assemble_queries(version='207.0', + db_type='All', + db_url='world-wide-europe') print('obs queries: ', obs_query_urls) exp_query_urls = [('All', @@ -89,9 +91,9 @@ def test_assemble_queries_all_primary(self): self.assertEqual(obs_query_urls, exp_query_urls) def test_assemble_queries_all_asia(self): - obs_query_urls = _assemble_queries(db_url='world-wide-asia', - version='207.0', - db_type='All') + obs_query_urls = _assemble_queries(version='207.0', + db_type='All', + db_url='world-wide-asia') print('obs queries: ', obs_query_urls) exp_query_urls = [( @@ -103,9 +105,9 @@ def test_assemble_queries_all_asia(self): self.assertEqual(obs_query_urls, exp_query_urls) def test_assemble_queries_all_australia(self): - obs_query_urls = _assemble_queries(db_url='australia', - version='232.0', - db_type='All') + obs_query_urls = _assemble_queries(version='232.0', + db_type='All', + db_url='australia') print('obs queries: ', obs_query_urls) exp_query_urls = [( @@ -130,8 +132,8 @@ def _makey_fakey_bact(faking_ignore_this): with patch('rescript.get_gtdb._retrieve_data_from_gtdb', new=_makey_fakey_both): res = rescript.actions.get_gtdb_data( - db_url='world-wide-europe', version='214.1', - db_type='SpeciesReps', domain='Both') + version='214.1', db_type='SpeciesReps', + domain='Both', db_url='world-wide-europe') self.assertEqual(str(res[0].type), 'FeatureData[Taxonomy]') self.assertEqual(str(res[1].type), 'FeatureData[Sequence]') @@ -139,8 +141,8 @@ def _makey_fakey_bact(faking_ignore_this): with patch('rescript.get_gtdb._retrieve_data_from_gtdb', new=_makey_fakey_arch): resa = rescript.actions.get_gtdb_data( - db_url='world-wide-europe', version='202.0', - domain='Archaea') + version='202.0', domain='Archaea', + db_url='world-wide-europe') self.assertEqual(str(resa[0].type), 'FeatureData[Taxonomy]') self.assertEqual(str(resa[1].type), 'FeatureData[Sequence]') @@ -148,8 +150,8 @@ def _makey_fakey_bact(faking_ignore_this): with patch('rescript.get_gtdb._retrieve_data_from_gtdb', new=_makey_fakey_bact): resb = rescript.actions.get_gtdb_data( - db_url='world-wide-europe', version='207.0', - domain='Bacteria') + version='207.0', domain='Bacteria', + db_url='world-wide-europe') self.assertEqual(str(resb[0].type), 'FeatureData[Taxonomy]') self.assertEqual(str(resb[1].type), 'FeatureData[Sequence]') @@ -157,8 +159,8 @@ def _makey_fakey_bact(faking_ignore_this): with patch('rescript.get_gtdb._retrieve_data_from_gtdb', new=_makey_fakey_both): resc = rescript.actions.get_gtdb_data( - db_url='world-wide-europe', version='214.1', - db_type='All') + version='214.1', db_type='All', + db_url='world-wide-europe') self.assertEqual(str(resc[0].type), 'FeatureData[Taxonomy]') self.assertEqual(str(resc[1].type), 'FeatureData[Sequence]')