From c8fa99fa89e0aa7a75142a437ab9937fb7955890 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=BCneyt=20Hizli?= Date: Mon, 31 Oct 2016 15:17:01 +0100 Subject: [PATCH 1/5] created eurostat auto function --- apps/datasetmanager/management/__init__.py | 0 .../management/commands/__init__.py | 0 .../management/commands/_private.py | 0 .../management/commands/eurostat.py | 154 ++++++++++++++++++ 4 files changed, 154 insertions(+) create mode 100644 apps/datasetmanager/management/__init__.py create mode 100644 apps/datasetmanager/management/commands/__init__.py create mode 100644 apps/datasetmanager/management/commands/_private.py create mode 100644 apps/datasetmanager/management/commands/eurostat.py diff --git a/apps/datasetmanager/management/__init__.py b/apps/datasetmanager/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/apps/datasetmanager/management/commands/__init__.py b/apps/datasetmanager/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/apps/datasetmanager/management/commands/_private.py b/apps/datasetmanager/management/commands/_private.py new file mode 100644 index 0000000..e69de29 diff --git a/apps/datasetmanager/management/commands/eurostat.py b/apps/datasetmanager/management/commands/eurostat.py new file mode 100644 index 0000000..f244701 --- /dev/null +++ b/apps/datasetmanager/management/commands/eurostat.py @@ -0,0 +1,154 @@ +from django.core.management.base import BaseCommand, CommandError +from apps.datasetmanager.models import Dataset +from django.db import models +from django.apps import apps as django_apps +from collections import OrderedDict +import json +import requests +import datetime +import pandas + +class Command(BaseCommand): + help = 'Eurostat' + + def handle(self, *args, **options): + for code in args: + data = requests.get("http://ec.europa.eu/eurostat/wdds/rest/data/v2.1/json/en/" + str(code) + "?precision=1") + + if data.status_code == 400: + errorDict = {"result": 400} + errorString = str(errorDict).replace("'", '"') + errorJson = json.loads(errorString) + return Response(errorJson) + elif data.status_code == 416: + errorDict = {"result": 416} + errorString = str(errorDict).replace("'", '"') + errorJson = json.loads(errorString) + return Response(errorJson) + + array = data.json(object_pairs_hook=OrderedDict) + + rowHeaders = array['dimension']['geo']['category']['label'] + + rowHeadersValues = list(rowHeaders.values()) + + rowLen = len(rowHeadersValues) + + colHeaders = array['dimension']['time']['category']['label'] + + colHeadersVals = list(colHeaders.values()) + + colLen = len(colHeadersVals) + 1 + + colHeadersValues = [] + + colHeadersValues.append("") + + for q in range(0, colLen - 1): + colHeadersValues.append(colHeadersVals[q]) + + val = array['value'] + + values = list(val.values()) + + rowArrays = [] + + index = 0 + for row in range(0, rowLen): + colArray = [] + position = row * (colLen - 1) + colArray.append(rowHeadersValues[row]) + for col in range(0, colLen - 1): + if val.get(str(position + col)) is None: + colArray.append("") + else: + colArray.append(values[index]) + index += 1 + + rowArrays.append(colArray) + + Individual = django_apps.get_model("referencepool", "Individual") + individuals_list = Individual.objects.all() + + DataClass = django_apps.get_model("referencepool", "DataClass") + data_class = DataClass.objects.get(title="Country") + + Indicator = django_apps.get_model("indicatorservice", "Indicator") + + try: + new_indicator = Indicator.objects.get(name=array['label']) + except: + new_indicator = Indicator(name=array['label'], description=array['label']) + new_indicator.save() + + + spatials = [] + + + for i in range(0, len(rowHeaders)): + spatial_key = list(rowHeaders.keys())[i] + spatial_value = list(rowHeaders.values())[i] + found = False + for individual in individuals_list: + if spatial_value == individual.title: + spatials.append(individual.id) + found = True + + if found is False: + new_individual = Individual(title=spatial_value, code=spatial_key, data_class=data_class) + new_individual.save() + spatials.append(new_individual.id) + + + resultArray = [] + + dataset_test = Dataset.objects.get(pk=19) + + print("dataset ", type(dataset_test.data)) + + data_frame = {} + for row in range(0, len(rowArrays)): + valuesDict = {} + for i in range(1, len(colHeadersValues)): + valuesDict[colHeadersValues[i]+"-01-01T00:00:00.000Z"] = rowArrays[row][i] + + #valuesDict = OrderedDict(sorted(valuesDict.items(), key=lambda t: t[0])) + + spatial = -1 + + for individual in individuals_list: + if rowArrays[row][0] == individual.title: + spatial = individual.id + + data_frame[str(spatial)] = valuesDict + + + + data_dict = {"resolution":"year", "unit":25, "data_frame": str(data_frame)} + + #pandas_data = pandas.DataFrame(data_dict) + + dictString = str(data_dict).replace("{'", '{"') + dictString = str(dictString).replace("':", '":') + dictString = str(dictString).replace("['", '["') + dictString = str(dictString).replace("]'", ']"') + dictString = str(dictString).replace("']", '"]') + dictString = str(dictString).replace("',", '",') + dictString = str(dictString).replace(", '", ', "') + dictString = str(dictString).replace("'", '"') + + print("data_dict ", dictString) + + + new_dataset = Dataset(title=array['label'], description=array['label'], keywords=array['label'], time_resolution="year", + time_start=colHeadersValues[1], time_end=colHeadersValues[len(colHeadersValues)-1], + spatials=spatials, language_id=38, resource_issued=datetime.datetime.today().strftime('%Y-%m-%d'), + resource_url="http://appsso.eurostat.ec.europa.eu/nui/show.do?dataset="+ str(code) +"&lang=en", + resource_id=71, unit_id=25, indicator_id=new_indicator.id, class_id=1, policy_domains=[1], + license_id=96, + data=dictString) + + new_dataset.save() + + + From e1981563427ed337136449037986c61dbf5d807e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=BCneyt=20Hizli?= Date: Tue, 8 Nov 2016 17:06:51 +0100 Subject: [PATCH 2/5] several updates --- .../management/commands/eurostat.py | 69 +++++++++---------- 1 file changed, 33 insertions(+), 36 deletions(-) diff --git a/apps/datasetmanager/management/commands/eurostat.py b/apps/datasetmanager/management/commands/eurostat.py index f244701..d837c13 100644 --- a/apps/datasetmanager/management/commands/eurostat.py +++ b/apps/datasetmanager/management/commands/eurostat.py @@ -3,6 +3,7 @@ from django.db import models from django.apps import apps as django_apps from collections import OrderedDict +from django.conf import settings import json import requests import datetime @@ -102,53 +103,49 @@ def handle(self, *args, **options): resultArray = [] - dataset_test = Dataset.objects.get(pk=19) - - print("dataset ", type(dataset_test.data)) - - data_frame = {} for row in range(0, len(rowArrays)): - valuesDict = {} - for i in range(1, len(colHeadersValues)): - valuesDict[colHeadersValues[i]+"-01-01T00:00:00.000Z"] = rowArrays[row][i] - - #valuesDict = OrderedDict(sorted(valuesDict.items(), key=lambda t: t[0])) - - spatial = -1 - + values = {} + spatial = "" for individual in individuals_list: if rowArrays[row][0] == individual.title: - spatial = individual.id - - data_frame[str(spatial)] = valuesDict - + spatial = individual.title + for i in range(1, len(colHeadersValues)): + if type(rowArrays[row][i]) is str: + values[colHeadersValues[i]] = None + else: + values[colHeadersValues[i]] = rowArrays[row][i] - data_dict = {"resolution":"year", "unit":25, "data_frame": str(data_frame)} - #pandas_data = pandas.DataFrame(data_dict) + resultArray.append({"row":row+1, "individual":spatial, "values":values}) - dictString = str(data_dict).replace("{'", '{"') - dictString = str(dictString).replace("':", '":') - dictString = str(dictString).replace("['", '["') - dictString = str(dictString).replace("]'", ']"') - dictString = str(dictString).replace("']", '"]') - dictString = str(dictString).replace("',", '",') - dictString = str(dictString).replace(", '", ', "') - dictString = str(dictString).replace("'", '"') - print("data_dict ", dictString) + headers = {'content-type': 'application/json', 'x-user-path':'https://adhocracy-prod.policycompass.eu/api/principals/users/0000072/', + 'x-user-token':'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzUxMiJ9.eyJpYXQiOjE0Nzg1Mjg2NDIsInN1YiI6Ii9wcmluY2lwYWxzL3VzZXJzLzAwMDAwNzIiLCJleHAiOjE0ODExMjA2NDJ9.HhV3dnlZp_B_A0q6Ijq7MMwC8tbcDMz72ZqP2XW9WCumLZVVoMoGSaxk7CyZkC-F4jSHrx4htn9h5ZqOx35l3Q'} + payload = {"time": + {"resolution":"year","start":colHeadersValues[1],"end":colHeadersValues[len(colHeadersValues)-1]}, + "resource":{"issued":datetime.datetime.today().strftime('%Y-%m-%d'), + "external_resource":71, + "url":"http://appsso.eurostat.ec.europa.eu/nui/show.do?dataset="+ str(code) +"&lang=en" + }, + "policy_domains":[1], + "title":array['label'], + "keywords":array['label'], + "license_id":96, + "description":array['label'], + "spatials":spatials, + "language_id":38, + "indicator_id":new_indicator.id, + "class_id":1, + "user_id":1, + "unit_id":25, + "data":{"table":resultArray}, + "is_draft":True} - new_dataset = Dataset(title=array['label'], description=array['label'], keywords=array['label'], time_resolution="year", - time_start=colHeadersValues[1], time_end=colHeadersValues[len(colHeadersValues)-1], - spatials=spatials, language_id=38, resource_issued=datetime.datetime.today().strftime('%Y-%m-%d'), - resource_url="http://appsso.eurostat.ec.europa.eu/nui/show.do?dataset="+ str(code) +"&lang=en", - resource_id=71, unit_id=25, indicator_id=new_indicator.id, class_id=1, policy_domains=[1], - license_id=96, - data=dictString) - new_dataset.save() + print("payload " , json.dumps({"table":resultArray})) + result = requests.post(settings.PC_SERVICES['references']['base_url'] + '/api/v1/datasetmanager/datasets', data=json.dumps(payload), headers=headers) From 5733ce3e8974ca6286c968ee2e991c8d18145b05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=BCneyt=20Hizli?= Date: Wed, 9 Nov 2016 13:08:16 +0100 Subject: [PATCH 3/5] automatic selection of filters is working --- .../management/commands/eurostat.py | 218 ++++++++++-------- 1 file changed, 119 insertions(+), 99 deletions(-) diff --git a/apps/datasetmanager/management/commands/eurostat.py b/apps/datasetmanager/management/commands/eurostat.py index d837c13..e9571b9 100644 --- a/apps/datasetmanager/management/commands/eurostat.py +++ b/apps/datasetmanager/management/commands/eurostat.py @@ -1,151 +1,171 @@ -from django.core.management.base import BaseCommand, CommandError -from apps.datasetmanager.models import Dataset -from django.db import models +from django.core.management.base import BaseCommand from django.apps import apps as django_apps from collections import OrderedDict from django.conf import settings +from pandasdmx import Request import json import requests import datetime -import pandas + class Command(BaseCommand): help = 'Eurostat' def handle(self, *args, **options): + if len(args) == 1 and args[0] == 'ALL': + estat = Request("ESTAT") + dataflow = estat.get(resource_type='dataflow') + dataflows = dataflow.msg.dataflows + args = dataflows.keys() + + filters = [] + query = " " + filters_description = "" + for code in args: data = requests.get("http://ec.europa.eu/eurostat/wdds/rest/data/v2.1/json/en/" + str(code) + "?precision=1") - if data.status_code == 400: - errorDict = {"result": 400} - errorString = str(errorDict).replace("'", '"') - errorJson = json.loads(errorString) - return Response(errorJson) - elif data.status_code == 416: - errorDict = {"result": 416} - errorString = str(errorDict).replace("'", '"') - errorJson = json.loads(errorString) - return Response(errorJson) + if data.status_code == 416: + estat = Request("ESTAT") - array = data.json(object_pairs_hook=OrderedDict) + dataflow = estat.get(resource_type='dataflow') - rowHeaders = array['dimension']['geo']['category']['label'] + dataflows = dataflow.msg.dataflows - rowHeadersValues = list(rowHeaders.values()) + dsd_id = dataflows[code].structure.id - rowLen = len(rowHeadersValues) + dsd_resp = estat.get(resource_type='datastructure', resource_id=dsd_id) - colHeaders = array['dimension']['time']['category']['label'] + dsd = dsd_resp.msg.datastructures[dsd_id] - colHeadersVals = list(colHeaders.values()) + dimensionsList = list(dsd.dimensions) - colLen = len(colHeadersVals) + 1 + for f in range(0, len(dimensionsList)): + if(dimensionsList[f] != "TIME_PERIOD" and dimensionsList[f] != "GEO" and dimensionsList[f] != "FREQ"): + dimensionsValuesList = list(dsd.dimensions[dimensionsList[f]].local_repr.enum.values()) + filters.append([dimensionsList[f], [dimensionsValuesList[0].id]]) + filters_description += dimensionsList[f] + ": " + dimensionsValuesList[0].name.en + ", " - colHeadersValues = [] + if(filters): + for key in range(0, len(filters)): + for value in range(0, len(filters[key][1])): + query += '&' + filters[key][0].lower() + '=' + filters[key][1][value] - colHeadersValues.append("") + data = requests.get("http://ec.europa.eu/eurostat/wdds/rest/data/v2.1/json/en/" + str(code) + "?precision=1" + query.strip()) - for q in range(0, colLen - 1): - colHeadersValues.append(colHeadersVals[q]) + array = data.json(object_pairs_hook=OrderedDict) - val = array['value'] + try: + rowHeaders = array['dimension']['geo']['category']['label'] - values = list(val.values()) + rowHeadersValues = list(rowHeaders.values()) - rowArrays = [] + rowLen = len(rowHeadersValues) - index = 0 - for row in range(0, rowLen): - colArray = [] - position = row * (colLen - 1) - colArray.append(rowHeadersValues[row]) - for col in range(0, colLen - 1): - if val.get(str(position + col)) is None: - colArray.append("") - else: - colArray.append(values[index]) - index += 1 + colHeaders = array['dimension']['time']['category']['label'] - rowArrays.append(colArray) + colHeadersVals = list(colHeaders.values()) - Individual = django_apps.get_model("referencepool", "Individual") - individuals_list = Individual.objects.all() + colLen = len(colHeadersVals) + 1 - DataClass = django_apps.get_model("referencepool", "DataClass") - data_class = DataClass.objects.get(title="Country") + colHeadersValues = [] - Indicator = django_apps.get_model("indicatorservice", "Indicator") + colHeadersValues.append("") - try: - new_indicator = Indicator.objects.get(name=array['label']) - except: - new_indicator = Indicator(name=array['label'], description=array['label']) - new_indicator.save() + for q in range(0, colLen - 1): + colHeadersValues.append(colHeadersVals[q]) + + val = array['value'] + values = list(val.values()) - spatials = [] + rowArrays = [] + index = 0 + for row in range(0, rowLen): + colArray = [] + position = row * (colLen - 1) + colArray.append(rowHeadersValues[row]) + for col in range(0, colLen - 1): + if val.get(str(position + col)) is None: + colArray.append("") + else: + colArray.append(values[index]) + index += 1 - for i in range(0, len(rowHeaders)): - spatial_key = list(rowHeaders.keys())[i] - spatial_value = list(rowHeaders.values())[i] - found = False - for individual in individuals_list: - if spatial_value == individual.title: - spatials.append(individual.id) - found = True + rowArrays.append(colArray) - if found is False: - new_individual = Individual(title=spatial_value, code=spatial_key, data_class=data_class) - new_individual.save() - spatials.append(new_individual.id) + Individual = django_apps.get_model("referencepool", "Individual") + individuals_list = Individual.objects.all() + DataClass = django_apps.get_model("referencepool", "DataClass") + data_class = DataClass.objects.get(title="Country") - resultArray = [] + Indicator = django_apps.get_model("indicatorservice", "Indicator") - for row in range(0, len(rowArrays)): - values = {} - spatial = "" - for individual in individuals_list: - if rowArrays[row][0] == individual.title: - spatial = individual.title + try: + new_indicator = Indicator.objects.get(name=array['label']) + except: + new_indicator = Indicator(name=array['label'], description=array['label']) + new_indicator.save() - for i in range(1, len(colHeadersValues)): - if type(rowArrays[row][i]) is str: - values[colHeadersValues[i]] = None - else: - values[colHeadersValues[i]] = rowArrays[row][i] + spatials = [] + for i in range(0, len(rowHeaders)): + spatial_key = list(rowHeaders.keys())[i] + spatial_value = list(rowHeaders.values())[i] + found = False + for individual in individuals_list: + if spatial_value == individual.title: + spatials.append(individual.id) + found = True - resultArray.append({"row":row+1, "individual":spatial, "values":values}) + if found is False: + new_individual = Individual(title=spatial_value, code=spatial_key, data_class=data_class) + new_individual.save() + spatials.append(new_individual.id) + resultArray = [] - headers = {'content-type': 'application/json', 'x-user-path':'https://adhocracy-prod.policycompass.eu/api/principals/users/0000072/', - 'x-user-token':'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzUxMiJ9.eyJpYXQiOjE0Nzg1Mjg2NDIsInN1YiI6Ii9wcmluY2lwYWxzL3VzZXJzLzAwMDAwNzIiLCJleHAiOjE0ODExMjA2NDJ9.HhV3dnlZp_B_A0q6Ijq7MMwC8tbcDMz72ZqP2XW9WCumLZVVoMoGSaxk7CyZkC-F4jSHrx4htn9h5ZqOx35l3Q'} + for row in range(0, len(rowArrays)): + values = {} + spatial = "" + for individual in individuals_list: + if rowArrays[row][0] == individual.title: + spatial = individual.title - payload = {"time": - {"resolution":"year","start":colHeadersValues[1],"end":colHeadersValues[len(colHeadersValues)-1]}, - "resource":{"issued":datetime.datetime.today().strftime('%Y-%m-%d'), - "external_resource":71, - "url":"http://appsso.eurostat.ec.europa.eu/nui/show.do?dataset="+ str(code) +"&lang=en" - }, - "policy_domains":[1], - "title":array['label'], - "keywords":array['label'], - "license_id":96, - "description":array['label'], - "spatials":spatials, - "language_id":38, - "indicator_id":new_indicator.id, - "class_id":1, - "user_id":1, - "unit_id":25, - "data":{"table":resultArray}, - "is_draft":True} + for i in range(1, len(colHeadersValues)): + if type(rowArrays[row][i]) is str: + values[colHeadersValues[i]] = None + else: + values[colHeadersValues[i]] = rowArrays[row][i] + resultArray.append({"row": row + 1, "individual": spatial, "values": values}) + headers = {'content-type': 'application/json', 'x-user-path': 'https://adhocracy-prod.policycompass.eu/api/principals/users/0000072/', 'x-user-token': 'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzUxMiJ9.eyJpYXQiOjE0Nzg1Mjg2NDIsInN1YiI6Ii9wcmluY2lwYWxzL3VzZXJzLzAwMDAwNzIiLCJleHAiOjE0ODExMjA2NDJ9.HhV3dnlZp_B_A0q6Ijq7MMwC8tbcDMz72ZqP2XW9WCumLZVVoMoGSaxk7CyZkC-F4jSHrx4htn9h5ZqOx35l3Q'} - print("payload " , json.dumps({"table":resultArray})) + payload = {} + payload['time'] = {"resolution": "year", "start": colHeadersValues[1], "end": colHeadersValues[len(colHeadersValues) - 1]} + payload["resource"] = {"issued": datetime.datetime.today().strftime('%Y-%m-%d'), "external_resource": 71, "url": "http://appsso.eurostat.ec.europa.eu/nui/show.do?dataset=" + str(code) + "&lang=en"} + payload["policy_domains"] = [1] + payload["title"] = array['label'] + payload["keywords"] = array['label'] + payload["license_id"] = 96 + payload["description"] = array['label'] + payload["spatials"] = spatials + payload["language_id"] = 38 + payload["indicator_id"] = new_indicator.id + payload["class_id"] = 1 + payload["user_id"] = 1 + payload["unit_id"] = 25 + payload["data"] = {"table": resultArray} + payload["is_draft"] = False - result = requests.post(settings.PC_SERVICES['references']['base_url'] + '/api/v1/datasetmanager/datasets', data=json.dumps(payload), headers=headers) + if len(filters) > 0: + payload['description'] = payload['description'] + " (selected filters: " + filters_description[:-2] + ")" + requests.post(settings.PC_SERVICES['references']['base_url'] + '/api/v1/datasetmanager/datasets', data=json.dumps(payload), headers=headers) + print("Download of '", code, "' successful") + + except: + print("ERROR: Download of '", code, "' unsuccessful") From 4e5ce6e66421aba9f1b5bdca7731bfca989eabfb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=BCneyt=20Hizli?= Date: Wed, 9 Nov 2016 13:31:00 +0100 Subject: [PATCH 4/5] added comments --- .../management/commands/eurostat.py | 31 +++++++++---------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/apps/datasetmanager/management/commands/eurostat.py b/apps/datasetmanager/management/commands/eurostat.py index e9571b9..839ff23 100644 --- a/apps/datasetmanager/management/commands/eurostat.py +++ b/apps/datasetmanager/management/commands/eurostat.py @@ -12,7 +12,9 @@ class Command(BaseCommand): help = 'Eurostat' def handle(self, *args, **options): + # if all datasets should be downloaded if len(args) == 1 and args[0] == 'ALL': + # usage of pandasdmx estat = Request("ESTAT") dataflow = estat.get(resource_type='dataflow') dataflows = dataflow.msg.dataflows @@ -22,24 +24,22 @@ def handle(self, *args, **options): query = " " filters_description = "" + # iterate over every Eurostat code in arguments for code in args: data = requests.get("http://ec.europa.eu/eurostat/wdds/rest/data/v2.1/json/en/" + str(code) + "?precision=1") + # selection of filters if data.status_code == 416: + # usage of pandasdmx estat = Request("ESTAT") - dataflow = estat.get(resource_type='dataflow') - dataflows = dataflow.msg.dataflows - dsd_id = dataflows[code].structure.id - dsd_resp = estat.get(resource_type='datastructure', resource_id=dsd_id) - dsd = dsd_resp.msg.datastructures[dsd_id] - dimensionsList = list(dsd.dimensions) + # choose first option of any filter, except TIME_PERIOD, GEO and FREQ for f in range(0, len(dimensionsList)): if(dimensionsList[f] != "TIME_PERIOD" and dimensionsList[f] != "GEO" and dimensionsList[f] != "FREQ"): dimensionsValuesList = list(dsd.dimensions[dimensionsList[f]].local_repr.enum.values()) @@ -51,37 +51,31 @@ def handle(self, *args, **options): for value in range(0, len(filters[key][1])): query += '&' + filters[key][0].lower() + '=' + filters[key][1][value] + # new request with selected filters data = requests.get("http://ec.europa.eu/eurostat/wdds/rest/data/v2.1/json/en/" + str(code) + "?precision=1" + query.strip()) array = data.json(object_pairs_hook=OrderedDict) try: + # get countries rowHeaders = array['dimension']['geo']['category']['label'] - rowHeadersValues = list(rowHeaders.values()) - rowLen = len(rowHeadersValues) - + # get years colHeaders = array['dimension']['time']['category']['label'] - colHeadersVals = list(colHeaders.values()) - colLen = len(colHeadersVals) + 1 - colHeadersValues = [] - colHeadersValues.append("") for q in range(0, colLen - 1): colHeadersValues.append(colHeadersVals[q]) val = array['value'] - values = list(val.values()) - rowArrays = [] - index = 0 + for row in range(0, rowLen): colArray = [] position = row * (colLen - 1) @@ -111,6 +105,7 @@ def handle(self, *args, **options): spatials = [] + # choose already existing individuals from referencepool or create missing individuals for i in range(0, len(rowHeaders)): spatial_key = list(rowHeaders.keys())[i] spatial_value = list(rowHeaders.values())[i] @@ -127,6 +122,7 @@ def handle(self, *args, **options): resultArray = [] + # create result array in correct format for row in range(0, len(rowArrays)): values = {} spatial = "" @@ -142,8 +138,10 @@ def handle(self, *args, **options): resultArray.append({"row": row + 1, "individual": spatial, "values": values}) + # headers for POST request headers = {'content-type': 'application/json', 'x-user-path': 'https://adhocracy-prod.policycompass.eu/api/principals/users/0000072/', 'x-user-token': 'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzUxMiJ9.eyJpYXQiOjE0Nzg1Mjg2NDIsInN1YiI6Ii9wcmluY2lwYWxzL3VzZXJzLzAwMDAwNzIiLCJleHAiOjE0ODExMjA2NDJ9.HhV3dnlZp_B_A0q6Ijq7MMwC8tbcDMz72ZqP2XW9WCumLZVVoMoGSaxk7CyZkC-F4jSHrx4htn9h5ZqOx35l3Q'} + # prepare new dataset payload = {} payload['time'] = {"resolution": "year", "start": colHeadersValues[1], "end": colHeadersValues[len(colHeadersValues) - 1]} payload["resource"] = {"issued": datetime.datetime.today().strftime('%Y-%m-%d'), "external_resource": 71, "url": "http://appsso.eurostat.ec.europa.eu/nui/show.do?dataset=" + str(code) + "&lang=en"} @@ -161,6 +159,7 @@ def handle(self, *args, **options): payload["data"] = {"table": resultArray} payload["is_draft"] = False + # add selected filters to description if len(filters) > 0: payload['description'] = payload['description'] + " (selected filters: " + filters_description[:-2] + ")" From f4a6c71f9d87cb8a7b3896ff8c6db85b031aca66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=BCneyt=20Hizli?= Date: Wed, 9 Nov 2016 14:09:02 +0100 Subject: [PATCH 5/5] corrected filter error --- apps/datasetmanager/management/commands/eurostat.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/apps/datasetmanager/management/commands/eurostat.py b/apps/datasetmanager/management/commands/eurostat.py index 839ff23..a92ed74 100644 --- a/apps/datasetmanager/management/commands/eurostat.py +++ b/apps/datasetmanager/management/commands/eurostat.py @@ -20,12 +20,12 @@ def handle(self, *args, **options): dataflows = dataflow.msg.dataflows args = dataflows.keys() - filters = [] - query = " " - filters_description = "" - # iterate over every Eurostat code in arguments for code in args: + filters = [] + query = " " + filters_description = "" + data = requests.get("http://ec.europa.eu/eurostat/wdds/rest/data/v2.1/json/en/" + str(code) + "?precision=1") # selection of filters @@ -54,9 +54,8 @@ def handle(self, *args, **options): # new request with selected filters data = requests.get("http://ec.europa.eu/eurostat/wdds/rest/data/v2.1/json/en/" + str(code) + "?precision=1" + query.strip()) - array = data.json(object_pairs_hook=OrderedDict) - try: + array = data.json(object_pairs_hook=OrderedDict) # get countries rowHeaders = array['dimension']['geo']['category']['label'] rowHeadersValues = list(rowHeaders.values())