Skip to content

IMP: updating GTDB url selection #271

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mikerobeson merged 3 commits into from
Sep 29, 2026
Merged

IMP: updating GTDB url selection #271

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
42985f4
initial refactoring of url selection
mikerobeson Sep 25, 2026
8b0f597
make world-wide-europe default and fixed help-text typo
mikerobeson Sep 29, 2026
b7d593a
rearranged parameter order in plugin_setup, other cosmetic test code …
mikerobeson Sep 29, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 8 additions & 5 deletions rescript/get_gtdb.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,13 +37,13 @@ def get_gtdb_data(
version: str = '232.0',
domain: str = 'Both',
db_type: str = 'SpeciesReps',
url_type: str = 'Primary'
db_url: str = 'world-wide-europe'
) -> (TSVTaxonomyFormat, DNAFASTAFormat):

queries = _assemble_queries(version=version,
db_type=db_type,
domain=domain,
url_type=url_type)
db_url=db_url)
tax_q, seqs_q = _retrieve_data_from_gtdb(queries)

print('\n Saving files...\n')
Expand All @@ -53,13 +53,16 @@ def get_gtdb_data(
def _assemble_queries(version='232.0',
db_type='SpeciesReps',
domain='Both',
url_type='Primary'):
db_url='world-wide-europe',
):
queries = []

if url_type == 'Primary':
if db_url == 'australia':
base_url = 'https://data.gtdb.ecogenomic.org/releases/'
elif url_type == 'Mirror':
elif db_url == 'world-wide-asia':
base_url = 'https://data.ace.uq.edu.au/public/gtdb/data/releases/'
elif db_url == 'world-wide-europe':
base_url = 'https://data.gtdb.aau.ecogenomic.org/releases/'

base_version = version.split('.')[0]
# ^^ Set `base_version` variable becuase number after the decimal is
Expand Down
12 changes: 7 additions & 5 deletions rescript/plugin_setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -930,7 +930,8 @@
'220.0', '226.0', '232.0']),
'domain': Str % Choices(['Both', 'Bacteria', 'Archaea']),
'db_type': Str % Choices(['All', 'SpeciesReps']),
'url_type': Str % Choices(['Primary', 'Mirror'])
'db_url': Str % Choices(['world-wide-europe',
'world-wide-asia', 'australia'])
},
outputs=[('gtdb_taxonomy', FeatureData[Taxonomy]),
('gtdb_sequences', FeatureData[Sequence])],
Expand All @@ -951,10 +952,11 @@
'parameter will be ignored as GTDB does not maintain '
'separate domain-level files for these non-clustered '
'data.',
'url_type': 'Toggle download URL. \'Primary\' will download data '
'from the primary GTDB URL. \'Mirror\' will dowload data '
'from the GTDB data mirror. Use \'Mirror\' if downloads '
'from \'Primary\' are slow.'},
'db_url': 'Toggle download URL. \'world-wide-europe\' will download '
'data from European servers. \'world-wide-asia\' will '
'download data from Asian-Pacific servers. and '
'\'australia\' will download from Australian servers. '
'Choose the server that is best for your region.'},
output_descriptions={
'gtdb_taxonomy': 'SSU GTDB reference taxonomy.',
'gtdb_sequences': 'SSU GTDB reference sequences.'},
Expand Down
57 changes: 40 additions & 17 deletions rescript/tests/test_get_gtdb.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,26 +48,30 @@ def setUp(self):
def test_assemble_species_rep_queries(self):
# checking v220 as GTDB updated the file format for the "ssu_rep"
# FASTA files to 'fna.gz' from their usual 'tar.gz'.
obs_query_urls = _assemble_queries('220.0', 'SpeciesReps', 'Both')
obs_query_urls = _assemble_queries('220.0', 'SpeciesReps',
'Both', 'world-wide-europe')
print('obs queries: ', obs_query_urls)

exp_query_urls = [('Archaea',
'https://data.gtdb.ecogenomic.org/releases/'
'https://data.gtdb.aau.ecogenomic.org/releases/'
'release220/220.0/genomic_files_reps/'
'ar53_ssu_reps_r220.fna.gz'),
('Bacteria',
'https://data.gtdb.ecogenomic.org/releases/'
'https://data.gtdb.aau.ecogenomic.org/releases/'
'release220/220.0/genomic_files_reps/'
'bac120_ssu_reps_r220.fna.gz')]
print('exp queries: ', exp_query_urls)
self.assertEqual(obs_query_urls, exp_query_urls)

def test_assemble_species_rep_queries_archaea(self):
obs_query_urls = _assemble_queries('202.0', 'SpeciesReps', 'Archaea')
obs_query_urls = _assemble_queries(version='202.0',
domain='Archaea',
db_type='SpeciesReps',
db_url='world-wide-europe')
print('obs queries: ', obs_query_urls)

exp_query_urls = [('Archaea',
'https://data.gtdb.ecogenomic.org/releases/'
'https://data.gtdb.aau.ecogenomic.org/releases/'
'release202/202.0/genomic_files_reps/'
'ar122_ssu_reps_r202.tar.gz')]
print('exp queries: ', exp_query_urls)
Expand All @@ -76,26 +80,41 @@ def test_assemble_species_rep_queries_archaea(self):
def test_assemble_queries_all_primary(self):
obs_query_urls = _assemble_queries(version='207.0',
db_type='All',
url_type='Primary')
db_url='world-wide-europe')
print('obs queries: ', obs_query_urls)

exp_query_urls = [('All',
'https://data.gtdb.ecogenomic.org/releases/'
'https://data.gtdb.aau.ecogenomic.org/releases/'
'release207/207.0/genomic_files_all/'
'ssu_all_r207.tar.gz')]
print('exp queries: ', exp_query_urls)
self.assertEqual(obs_query_urls, exp_query_urls)

def test_assemble_queries_all_mirror(self):
def test_assemble_queries_all_asia(self):
obs_query_urls = _assemble_queries(version='207.0',
db_type='All',
url_type='Mirror')
db_url='world-wide-asia')
print('obs queries: ', obs_query_urls)

exp_query_urls = [('All',
'https://data.ace.uq.edu.au/public/gtdb/data/'
'releases/release207/207.0/genomic_files_all/'
'ssu_all_r207.tar.gz')]
exp_query_urls = [(
'All',
'https://data.ace.uq.edu.au/public/gtdb/data/releases/'
'release207/207.0/genomic_files_all/'
'ssu_all_r207.tar.gz')]
print('exp queries: ', exp_query_urls)
self.assertEqual(obs_query_urls, exp_query_urls)

def test_assemble_queries_all_australia(self):
obs_query_urls = _assemble_queries(version='232.0',
db_type='All',
db_url='australia')
print('obs queries: ', obs_query_urls)

exp_query_urls = [(
'All',
'https://data.gtdb.ecogenomic.org/releases/'
'release232/232.0/genomic_files_all/'
'ssu_all_r232.fna.gz')]
print('exp queries: ', exp_query_urls)
self.assertEqual(obs_query_urls, exp_query_urls)

Expand All @@ -113,31 +132,35 @@ def _makey_fakey_bact(faking_ignore_this):
with patch('rescript.get_gtdb._retrieve_data_from_gtdb',
new=_makey_fakey_both):
res = rescript.actions.get_gtdb_data(
version='214.1', db_type='SpeciesReps', domain='Both')
version='214.1', db_type='SpeciesReps',
domain='Both', db_url='world-wide-europe')
self.assertEqual(str(res[0].type), 'FeatureData[Taxonomy]')
self.assertEqual(str(res[1].type), 'FeatureData[Sequence]')

# just grab archaea domain, and version 202
with patch('rescript.get_gtdb._retrieve_data_from_gtdb',
new=_makey_fakey_arch):
resa = rescript.actions.get_gtdb_data(
version='202.0', domain='Archaea')
version='202.0', domain='Archaea',
db_url='world-wide-europe')
self.assertEqual(str(resa[0].type), 'FeatureData[Taxonomy]')
self.assertEqual(str(resa[1].type), 'FeatureData[Sequence]')

# just grab bacteria domain, and version 207
with patch('rescript.get_gtdb._retrieve_data_from_gtdb',
new=_makey_fakey_bact):
resb = rescript.actions.get_gtdb_data(
version='207.0', domain='Bacteria')
version='207.0', domain='Bacteria',
db_url='world-wide-europe')
self.assertEqual(str(resb[0].type), 'FeatureData[Taxonomy]')
self.assertEqual(str(resb[1].type), 'FeatureData[Sequence]')

# Non-species rep version 214.1)
with patch('rescript.get_gtdb._retrieve_data_from_gtdb',
new=_makey_fakey_both):
resc = rescript.actions.get_gtdb_data(
version='214.1', db_type='All')
version='214.1', db_type='All',
db_url='world-wide-europe')
self.assertEqual(str(resc[0].type), 'FeatureData[Taxonomy]')
self.assertEqual(str(resc[1].type), 'FeatureData[Sequence]')

Expand Down
Loading