From 74fd31ed55e544d2ff7207b5ebe09b7740ae0b24 Mon Sep 17 00:00:00 2001 From: missinglink Date: Thu, 1 Aug 2019 15:22:24 +0200 Subject: [PATCH] feat(statcan): experiment importing statcan data --- bin/cmd/import.js | 2 +- import/source/statcan.js | 8 ++ import/source/statcan/download.txt | 1 + import/source/statcan/map/geometries.js | 17 +++ import/source/statcan/map/geometries.test.js | 112 +++++++++++++++++++ import/source/statcan/map/names.js | 9 ++ import/source/statcan/map/names.test.js | 35 ++++++ import/source/statcan/map/place.js | 42 +++++++ import/source/statcan/map/place.test.js | 48 ++++++++ import/source/statcan/map/properties.js | 9 ++ import/source/statcan/map/properties.test.js | 40 +++++++ 11 files changed, 322 insertions(+), 1 deletion(-) create mode 100644 import/source/statcan.js create mode 100644 import/source/statcan/download.txt create mode 100644 import/source/statcan/map/geometries.js create mode 100644 import/source/statcan/map/geometries.test.js create mode 100644 import/source/statcan/map/names.js create mode 100644 import/source/statcan/map/names.test.js create mode 100644 import/source/statcan/map/place.js create mode 100644 import/source/statcan/map/place.test.js create mode 100644 import/source/statcan/map/properties.js create mode 100644 import/source/statcan/map/properties.test.js diff --git a/bin/cmd/import.js b/bin/cmd/import.js index 9f65cd4..125440c 100644 --- a/bin/cmd/import.js +++ b/bin/cmd/import.js @@ -9,7 +9,7 @@ module.exports = { yargs.positional('source', { type: 'string', describe: 'name of data source', - choices: ['whosonfirst', 'osmium', 'zcta'], + choices: ['whosonfirst', 'osmium', 'zcta', 'statcan'], demand: 'source is required' }) diff --git a/import/source/statcan.js b/import/source/statcan.js new file mode 100644 index 0000000..10ef03e --- /dev/null +++ b/import/source/statcan.js @@ -0,0 +1,8 @@ +const file = require('../../import/file') + +module.exports = { + ingress: file, + record_separator: /\r?\n/, + format: 'json', + mapper: require('./statcan/map/place') +} diff --git a/import/source/statcan/download.txt b/import/source/statcan/download.txt new file mode 100644 index 0000000..785c391 --- /dev/null +++ b/import/source/statcan/download.txt @@ -0,0 +1 @@ +https://www12.statcan.gc.ca/census-recensement/2011/geo/bound-limit/bound-limit-eng.cfm \ No newline at end of file diff --git a/import/source/statcan/map/geometries.js b/import/source/statcan/map/geometries.js new file mode 100644 index 0000000..0106adf --- /dev/null +++ b/import/source/statcan/map/geometries.js @@ -0,0 +1,17 @@ +const _ = require('lodash') +const format = require('../../../format') +const Geometry = require('../../../../model/Geometry') + +function mapper (place, doc) { + const geometry = _.get(doc, 'geometry') + const isPolygon = _.get(geometry, 'type', '').trim().toUpperCase().endsWith('POLYGON') + + if (geometry) { + place.addGeometry(new Geometry( + format.from('geometry', 'geojson', geometry), + isPolygon ? 'boundary' : 'centroid' + )) + } +} + +module.exports = mapper diff --git a/import/source/statcan/map/geometries.test.js b/import/source/statcan/map/geometries.test.js new file mode 100644 index 0000000..0a8ecfc --- /dev/null +++ b/import/source/statcan/map/geometries.test.js @@ -0,0 +1,112 @@ +const Place = require('../../../../model/Place') +const Geometry = require('../../../../model/Geometry') +const map = require('./geometries') + +module.exports.tests = {} + +module.exports.tests.mapper = (test) => { + test('mapper: geometry empty', (t) => { + let p = new Place() + map(p, {}) + + t.equals(p.geometry.length, 0) + t.end() + }) + test('mapper: maps polygon', (t) => { + let p = new Place() + map(p, { + geometry: { + 'type': 'Polygon', + 'coordinates': [ + [ + [ + 39.0234375, + 48.922499263758255 + ], + [ + 47.8125, + 39.639537564366684 + ], + [ + 61.17187499999999, + 49.83798245308484 + ], + [ + 39.0234375, + 48.922499263758255 + ] + ] + ] + } + }) + t.equals(p.geometry.length, 1) + t.true(p.geometry[0] instanceof Geometry) + t.equal(p.geometry[0].geometry.constructor.name.toUpperCase(), 'POLYGON') + t.equal(p.geometry[0].role, 'boundary') + t.end() + }) + test('mapper: maps point', (t) => { + let p = new Place() + map(p, { + geometry: { + 'type': 'Point', + 'coordinates': [ + 59.0625, + 47.989921667414194 + ] + } + }) + t.equals(p.geometry.length, 1) + t.true(p.geometry[0] instanceof Geometry) + t.equal(p.geometry[0].geometry.constructor.name.toUpperCase(), 'POINT') + t.equal(p.geometry[0].role, 'centroid') + t.end() + }) + test('mapper: internal point', (t) => { + let p = new Place() + map(p, { + geometry: { + 'type': 'Polygon', + 'coordinates': [ + [ + [ + 39.0234375, + 48.922499263758255 + ], + [ + 47.8125, + 39.639537564366684 + ], + [ + 61.17187499999999, + 49.83798245308484 + ], + [ + 39.0234375, + 48.922499263758255 + ] + ] + ] + }, + properties: { + 'INTPTLON10': 1.1, + 'INTPTLAT10': 2.2 + } + }) + t.equals(p.geometry.length, 2) + t.true(p.geometry[1] instanceof Geometry) + t.equal(p.geometry[1].geometry.constructor.name.toUpperCase(), 'POINT') + t.equal(p.geometry[1].role, 'centroid') + t.end() + }) +} + +module.exports.all = (tape) => { + function test (name, testFunction) { + return tape(`geometries: ${name}`, testFunction) + } + + for (var testCase in module.exports.tests) { + module.exports.tests[testCase](test) + } +} diff --git a/import/source/statcan/map/names.js b/import/source/statcan/map/names.js new file mode 100644 index 0000000..cc3b3bb --- /dev/null +++ b/import/source/statcan/map/names.js @@ -0,0 +1,9 @@ +const _ = require('lodash') +const Name = require('../../../../model/Name') + +function mapper (place, properties) { + // generic name properties + place.addName(new Name('eng', 'default', false, _.get(properties, 'CDNAME', '').trim())) +} + +module.exports = mapper diff --git a/import/source/statcan/map/names.test.js b/import/source/statcan/map/names.test.js new file mode 100644 index 0000000..4eacb2c --- /dev/null +++ b/import/source/statcan/map/names.test.js @@ -0,0 +1,35 @@ +const Place = require('../../../../model/Place') +const map = require('./names') + +module.exports.tests = {} + +module.exports.tests.mapper = (test) => { + test('mapper: properties empty', (t) => { + let p = new Place() + map(p, {}) + + t.equals(p.name.length, 0) + t.end() + }) + test('mapper: name', (t) => { + let p = new Place() + map(p, { 'ZCTA5CE10': ' example1 ' }) + + t.equals(p.name.length, 1) + t.equals(p.name[0].lang, 'und') + t.equals(p.name[0].tag, 'default') + t.equals(p.name[0].abbr, false) + t.equals(p.name[0].name, 'example1') + t.end() + }) +} + +module.exports.all = (tape) => { + function test (name, testFunction) { + return tape(`names: ${name}`, testFunction) + } + + for (var testCase in module.exports.tests) { + module.exports.tests[testCase](test) + } +} diff --git a/import/source/statcan/map/place.js b/import/source/statcan/map/place.js new file mode 100644 index 0000000..00149d2 --- /dev/null +++ b/import/source/statcan/map/place.js @@ -0,0 +1,42 @@ +const _ = require('lodash') +const Identity = require('../../../../model/Identity') +const Ontology = require('../../../../model/Ontology') +const Place = require('../../../../model/Place') + +const map = { + properties: require('./properties'), + names: require('./names'), + geometries: require('./geometries') +} + +// { +// "CSDUID": "5933045", +// "CSDNAME": "Sun Peaks Mountain", +// "CSDTYPE": "VL", +// "PRUID": "59", +// "PRNAME": "British Columbia / Colombie-Britannique", +// "CDUID": "5933", +// "CDNAME": "Thompson-Nicola", +// "CDTYPE": "RD" +// } + +function mapper (doc) { + // get document properties + const properties = _.get(doc, 'properties') + if (!_.isPlainObject(properties)) { return null } + + // instantiate a new place + const place = new Place( + new Identity('statcan', _.get(properties, 'CDUID', '').trim()), + new Ontology('admin', _.get(properties, 'CDTYPE', '').trim()) + ) + + // run mappers + map.properties(place, properties) + map.names(place, properties) + map.geometries(place, doc) + + return place +} + +module.exports = mapper diff --git a/import/source/statcan/map/place.test.js b/import/source/statcan/map/place.test.js new file mode 100644 index 0000000..089be58 --- /dev/null +++ b/import/source/statcan/map/place.test.js @@ -0,0 +1,48 @@ +const Place = require('../../../../model/Place') +const map = require('./place') + +module.exports.tests = {} + +module.exports.tests.mapper = (test) => { + test('mapper: properties empty', (t) => { + let place = map({}) + t.equal(place, null) + t.end() + }) + test('mapper: maps identity & ontology', (t) => { + let place = map({ + properties: { + 'ZCTA5CE10': '90210' + } + }) + t.true(place instanceof Place) + t.equal(place.identity.source, 'uscensus') + t.equal(place.identity.id, 'zcta:90210') + t.equal(place.ontology.class, 'admin') + t.equal(place.ontology.type, 'postalcode') + t.end() + }) + test('mapper: maps geometry', (t) => { + let place = map({ + properties: { + 'ZCTA5CE10': '90210' + }, + geometry: require('../../../../test/fixture/geojson.triangle') + }) + t.true(place instanceof Place) + t.equal(place.geometry.length, 1) + t.equal(place.geometry[0].geometry.constructor.name.toUpperCase(), 'POLYGON') + t.equal(place.geometry[0].role, 'boundary') + t.end() + }) +} + +module.exports.all = (tape) => { + function test (name, testFunction) { + return tape(`place: ${name}`, testFunction) + } + + for (var testCase in module.exports.tests) { + module.exports.tests[testCase](test) + } +} diff --git a/import/source/statcan/map/properties.js b/import/source/statcan/map/properties.js new file mode 100644 index 0000000..f27bee0 --- /dev/null +++ b/import/source/statcan/map/properties.js @@ -0,0 +1,9 @@ +const Property = require('../../../../model/Property') + +function mapper (place, properties) { + for (let key in properties) { + place.addProperty(new Property(`statcan:${key}`, properties[key])) + } +} + +module.exports = mapper diff --git a/import/source/statcan/map/properties.test.js b/import/source/statcan/map/properties.test.js new file mode 100644 index 0000000..1ef563f --- /dev/null +++ b/import/source/statcan/map/properties.test.js @@ -0,0 +1,40 @@ +const Place = require('../../../../model/Place') +const map = require('./properties') + +module.exports.tests = {} + +module.exports.tests.mapper = (test) => { + test('mapper: properties empty', (t) => { + let p = new Place() + map(p, {}) + + t.equals(p.property.length, 0) + t.end() + }) + test('mapper: uscensus-specific properties', (t) => { + let p = new Place() + map(p, { + 'CLASSFP10': 'B5', + 'MTFCC10': 'G6350', + 'INTPTLON10': 1.1, + 'INTPTLAT10': 2.2 + }) + + t.equals(p.property.length, 2) + t.equals(p.property[0].key, 'uscensus:CLASSFP10', 'CLASSFP10') + t.equals(p.property[0].value, 'B5', 'CLASSFP10') + t.equals(p.property[1].key, 'uscensus:MTFCC10', 'MTFCC10') + t.equals(p.property[1].value, 'G6350', 'MTFCC10') + t.end() + }) +} + +module.exports.all = (tape) => { + function test (name, testFunction) { + return tape(`properties: ${name}`, testFunction) + } + + for (var testCase in module.exports.tests) { + module.exports.tests[testCase](test) + } +}