From 2eea118264adb48e3b7a46eee6d8dcffd104beba Mon Sep 17 00:00:00 2001 From: Scott Wales Date: Tue, 1 Sep 2020 14:13:13 +1000 Subject: [PATCH 1/2] Don't hide exceptions --- clef/code.py | 55 +++++++++++++++++++++++++--------------------------- 1 file changed, 26 insertions(+), 29 deletions(-) diff --git a/clef/code.py b/clef/code.py index ba69348..bdb3c96 100644 --- a/clef/code.py +++ b/clef/code.py @@ -89,35 +89,32 @@ def matching(session, cols, fixed, project='CMIP5', local=True, latest=True, **k """ results = pd.DataFrame() - try: - # use local search - if local: - msg = "There are no simulations stored locally" - # perform the query for each variable separately and concatenate the results - combs = [dict(zip(kwargs, x)) for x in itertools.product(*kwargs.values())] - for c in combs: - results = results.append(search(session,project=project.upper(), - latest=latest, **c), - ignore_index=True) - # use ESGF search - else: - msg = "There are no simulations currently available on the ESGF nodes" - kwquery = {k:tuple(v) for k,v in kwargs.items()} - kwquery['project']=project.upper() - attrs = ['dataset_id', 'version'] # datetime_start, datetime_stop - attrs.extend( load_vocabularies(project)['attributes']) - query=None - response = esgf_query(query, ','.join(attrs), latest=latest, **kwquery) - # can't create dataframe in one go because many values are unidimensional lists - res_list = [] - for row in response['response']['docs']: - row['version'] = row['dataset_id'].split("|")[0].split(".")[-1], - res_list.append({k:(v[0] if isinstance(v,list) else v) for k,v in row.items()}) - results = pd.DataFrame(res_list) - - except Exception as e: - print('ERROR',str(e)) - return None + + # use local search + if local: + msg = "There are no simulations stored locally" + # perform the query for each variable separately and concatenate the results + combs = [dict(zip(kwargs, x)) for x in itertools.product(*kwargs.values())] + for c in combs: + results = results.append(search(session,project=project.upper(), + latest=latest, **c), + ignore_index=True) + # use ESGF search + else: + msg = "There are no simulations currently available on the ESGF nodes" + kwquery = {k:tuple(v) for k,v in kwargs.items()} + kwquery['project']=project.upper() + attrs = ['dataset_id', 'version'] # datetime_start, datetime_stop + attrs.extend( load_vocabularies(project)['attributes']) + query=None + response = esgf_query(query, ','.join(attrs), latest=latest, **kwquery) + # can't create dataframe in one go because many values are unidimensional lists + res_list = [] + for row in response['response']['docs']: + row['version'] = row['dataset_id'].split("|")[0].split(".")[-1], + res_list.append({k:(v[0] if isinstance(v,list) else v) for k,v in row.items()}) + results = pd.DataFrame(res_list) + # if nothing turned by query print warning and return if len(results.index) == 0: From 0a62f53ae41d2a59949898c3f00244e0b5d9e77f Mon Sep 17 00:00:00 2001 From: Scott Wales Date: Tue, 1 Sep 2020 14:20:36 +1000 Subject: [PATCH 2/2] Update test --- test/test_code.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/test_code.py b/test/test_code.py index 0ac5ec4..49502d7 100644 --- a/test/test_code.py +++ b/test/test_code.py @@ -17,6 +17,7 @@ import pytest from clef.code import and_filter, matching, local_latest, search, stats, ids_df +from clef.exception import ClefException from code_fixtures import * # Tests for the functions defined in code.py @@ -145,9 +146,9 @@ def test_matching(session): 'ensemble':'r1i1p1', 'variable': ['x'], } - # Errors should print a message and return 'None' - r = matching(session, ['variable','experiment'],['model','ensemble'], **facets) - assert r is None + # Errors should raise + with pytest.raises(ClefException): + matching(session, ['variable','experiment'],['model','ensemble'], **facets) def test_ids_df(dids6, results6, dids5, results5):