diff --git a/cmd/import.sh b/cmd/import.sh index c7adc73c..044b6f60 100644 --- a/cmd/import.sh +++ b/cmd/import.sh @@ -3,7 +3,7 @@ set -e; # per-source imports function import_wof(){ compose_run 'whosonfirst' './bin/start'; } -function import_oa(){ compose_run 'openaddresses' "./bin/parallel ${OPENADDRESSES_PARALLELISM:-1}"; } +function import_oa(){ compose_run -T 'openaddresses' "./bin/parallel ${OPENADDRESSES_PARALLELISM:-1}"; } function import_osm(){ compose_run 'openstreetmap' './bin/start'; } function import_polylines(){ compose_run 'polylines' './bin/start'; } function import_geonames(){ compose_run 'geonames' './bin/start'; } diff --git a/cmd/prepare.sh b/cmd/prepare.sh index ab4ea1eb..26dc4118 100644 --- a/cmd/prepare.sh +++ b/cmd/prepare.sh @@ -9,9 +9,34 @@ function prepare_placeholder(){ compose_run -T 'placeholder' ./cmd/build.sh; } +function prepare_spatial(){ + compose_run -T --entrypoint='bash' 'spatial' << 'EOF' + #!/bin/bash + set -euo pipefail + + TARGET_DB='/data/spatial/docker.spatial.db' + WOF_SQLITE_DIR='/data/whosonfirst/sqlite' + SQL_EXTRACT_QUERY="SELECT json_extract(body, '$') FROM geojson" + + mkdir -p $(dirname "${TARGET_DB}") + rm -rf "${TARGET_DB}" + + export_all_wof_databases_as_geojson() { + for db in $(find "${WOF_SQLITE_DIR}" -name '*.db' -type f -maxdepth 1); do + 1>&2 echo "[reading] $(basename ${db})" + /opt/spatial/bin/sqlite3 "${db}" "${SQL_EXTRACT_QUERY}" + done + } + + export_all_wof_databases_as_geojson \ + | node bin/spatial.js import whosonfirst --db="${TARGET_DB}" +EOF +} + register 'prepare' 'polylines' 'export road network from openstreetmap into polylines format' prepare_polylines register 'prepare' 'interpolation' 'build interpolation sqlite databases' prepare_interpolation register 'prepare' 'placeholder' 'build placeholder sqlite databases' prepare_placeholder +register 'prepare' 'spatial' 'build spatial database' prepare_spatial # prepare all the data to be used by imports function prepare_all(){ diff --git a/projects/portland-metro/Dockerfile.oa.plus.spatial b/projects/portland-metro/Dockerfile.oa.plus.spatial new file mode 100644 index 00000000..7c6c30db --- /dev/null +++ b/projects/portland-metro/Dockerfile.oa.plus.spatial @@ -0,0 +1,8 @@ +FROM pelias/spatial:pip_beta AS spatial +FROM pelias/openaddresses:master + +# copy spatial runtime +COPY --from=spatial --chown=pelias /opt/spatial /opt/spatial + +# copy spatial code +COPY --from=spatial --chown=pelias /code /code/spatial diff --git a/projects/portland-metro/admin-lookup/SpatialServiceConfig.js b/projects/portland-metro/admin-lookup/SpatialServiceConfig.js new file mode 100644 index 00000000..ed499e06 --- /dev/null +++ b/projects/portland-metro/admin-lookup/SpatialServiceConfig.js @@ -0,0 +1,23 @@ +const pmw = require('pelias-microservice-wrapper') + +class SpatialServiceConfig extends pmw.ServiceConfiguration { + constructor() { + super('spatial', { + url: 'http://spatial:4770/' + }) + } + getUrl() { + return `${this.baseUrl}query/pip/beta`; + } + getParameters(params) { + return params; + } + isEnabled() { + return true + } + createClient() { + return pmw.service(this) + } +}; + +module.exports = SpatialServiceConfig diff --git a/projects/portland-metro/admin-lookup/codec.js b/projects/portland-metro/admin-lookup/codec.js new file mode 100644 index 00000000..5b3e8965 --- /dev/null +++ b/projects/portland-metro/admin-lookup/codec.js @@ -0,0 +1,23 @@ +const _ = require('lodash') +const Document = require('pelias-model').Document + +module.exports.marshal = (doc) => { + return doc +} + +module.exports.unmarshal = (data) => { + + // assign document prototype + // let doc = new Document(data.source, data.layer, data.source_id) + // doc = _.merge(doc, data) + + let doc = Object.setPrototypeOf(data, Document.prototype) + + // create a non-enumerable property for metadata + Object.defineProperty(doc, '_meta', { writable: true, value: {} }); + + // create a non-enumerable property for post-processing scripts + Object.defineProperty(doc, '_post', { writable: true, value: [] }); + + return doc +} diff --git a/projects/portland-metro/admin-lookup/localClient.js b/projects/portland-metro/admin-lookup/localClient.js new file mode 100644 index 00000000..268ddbdc --- /dev/null +++ b/projects/portland-metro/admin-lookup/localClient.js @@ -0,0 +1,45 @@ +const _ = require('lodash') +const codec = require('./codec') +const cpuCount = require('os').cpus().length +const Pool = require('worker-thread-pool'); + +// // const threads = require('worker_threads') +// const worker = new threads.Worker('./node_modules/pelias-wof-admin-lookup/queryServiceWorkerThread.js', { +// workerData: { +// readonly: true, +// filename: '/data/spatial/docker.spatial.db' +// } +// }); + + +const pool = new Pool({ + size: cpuCount, + path: './node_modules/pelias-wof-admin-lookup/queryServiceWorkerThread.js' +}) + +function lookup(doc, cb){ + // console.error('lookup', query) + // message.doc = codec.marshal(message.doc) + + // const crypto = require('crypto') + // const token = crypto.randomBytes(64).toString('hex').substr(0, 8); + + pool.run({ centroid: doc.getCentroid() }) + .then((message) => { + // doc = codec.unmarshal(doc) + // console.error('worker message', message) + + message.addParent.forEach(ap => { + // console.error('addParent', ap) + doc.addParent(...ap) + }) + + cb() + }) + .catch(({ err }) => { + // console.error(message) + cb({ err }) + }) +} + +module.exports = lookup diff --git a/projects/portland-metro/admin-lookup/main.js b/projects/portland-metro/admin-lookup/main.js new file mode 100644 index 00000000..dabaca5e --- /dev/null +++ b/projects/portland-metro/admin-lookup/main.js @@ -0,0 +1,40 @@ +const _ = require('lodash') +const cpuCount = require('os').cpus().length +// const through = require('through2'); +const transform = require('parallel-transform'); +const parallelism = cpuCount +const logger = require('pelias-logger').get('admin-lookup'); +const streamOptions = { highwaterMark: 2000, ordered: false } + +// network client +// const SpatialServiceConfig = require('./SpatialServiceConfig'); +// const client = new SpatialServiceConfig().createClient() + +// local client +const client = require('./localClient') + +function create(){ + // return through.obj(streamOptions, (doc, enc, next) => { + return transform(parallelism, streamOptions, (doc, next) => { + client(doc, ({ err }) => { + if (err) { + console.error('PIP error', err) + + // if there's an error, just log it and move on + logger.error(`PIP server failed: ${(err.message || JSON.stringify(err))}`, { + id: doc.getGid(), + lat: centroid.lat, + lon: centroid.lon + }); + + // don't pass the unmodified doc along + return next() + } + + // console.error(query) + next(null, doc); + }); + }) +} + +module.exports = { create } diff --git a/projects/portland-metro/admin-lookup/node_modules/aggregate-error/index.d.ts b/projects/portland-metro/admin-lookup/node_modules/aggregate-error/index.d.ts new file mode 100644 index 00000000..0a7926b3 --- /dev/null +++ b/projects/portland-metro/admin-lookup/node_modules/aggregate-error/index.d.ts @@ -0,0 +1,14 @@ +/** + * Create an error from multiple errors. + */ +export default class AggregateError extends Error implements Iterable { + readonly name: 'AggregateError'; + + /** + * @param errors - If a string, a new `Error` is created with the string as the error message. If a non-Error object, a new `Error` is created with all properties from the object copied over. + * @returns An Error that is also an [`Iterable`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Iterators_and_Generators#Iterables) for the individual errors. + */ + constructor(errors: ReadonlyArray); + + [Symbol.iterator](): IterableIterator; +} diff --git a/projects/portland-metro/admin-lookup/node_modules/aggregate-error/index.js b/projects/portland-metro/admin-lookup/node_modules/aggregate-error/index.js new file mode 100644 index 00000000..54b842b3 --- /dev/null +++ b/projects/portland-metro/admin-lookup/node_modules/aggregate-error/index.js @@ -0,0 +1,48 @@ +'use strict'; +const indentString = require('indent-string'); +const cleanStack = require('clean-stack'); + +const cleanInternalStack = stack => stack.replace(/\s+at .*aggregate-error\/index.js:\d+:\d+\)?/g, ''); + +class AggregateError extends Error { + constructor(errors) { + if (!Array.isArray(errors)) { + throw new TypeError(`Expected input to be an Array, got ${typeof errors}`); + } + + errors = [...errors].map(error => { + if (error instanceof Error) { + return error; + } + + if (error !== null && typeof error === 'object') { + // Handle plain error objects with message property and/or possibly other metadata + return Object.assign(new Error(error.message), error); + } + + return new Error(error); + }); + + let message = errors + .map(error => { + // The `stack` property is not standardized, so we can't assume it exists + return typeof error.stack === 'string' ? cleanInternalStack(cleanStack(error.stack)) : String(error); + }) + .join('\n'); + message = '\n' + indentString(message, 4); + super(message); + + this.name = 'AggregateError'; + + Object.defineProperty(this, '_errors', {value: errors}); + } + + * [Symbol.iterator]() { + for (const error of this._errors) { + yield error; + } + } +} + +module.exports = AggregateError; +module.exports.default = AggregateError; diff --git a/projects/portland-metro/admin-lookup/node_modules/aggregate-error/license b/projects/portland-metro/admin-lookup/node_modules/aggregate-error/license new file mode 100644 index 00000000..e7af2f77 --- /dev/null +++ b/projects/portland-metro/admin-lookup/node_modules/aggregate-error/license @@ -0,0 +1,9 @@ +MIT License + +Copyright (c) Sindre Sorhus (sindresorhus.com) + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/projects/portland-metro/admin-lookup/node_modules/aggregate-error/package.json b/projects/portland-metro/admin-lookup/node_modules/aggregate-error/package.json new file mode 100644 index 00000000..78a4e7d7 --- /dev/null +++ b/projects/portland-metro/admin-lookup/node_modules/aggregate-error/package.json @@ -0,0 +1,74 @@ +{ + "_from": "aggregate-error@^2.0.0", + "_id": "aggregate-error@2.2.0", + "_inBundle": false, + "_integrity": "sha512-E5n+IZkhh22/pFdUvHUU/o9z752lc+7tgHt+FXS/g6BjlbE9249dGmuS/SxIWMPhTljZJkFN+7OXE0+O5+WT8w==", + "_location": "/aggregate-error", + "_phantomChildren": {}, + "_requested": { + "type": "range", + "registry": true, + "raw": "aggregate-error@^2.0.0", + "name": "aggregate-error", + "escapedName": "aggregate-error", + "rawSpec": "^2.0.0", + "saveSpec": null, + "fetchSpec": "^2.0.0" + }, + "_requiredBy": [ + "/worker-thread-pool" + ], + "_resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-2.2.0.tgz", + "_shasum": "f54b464db18cc77c907ae084451f39134707134a", + "_spec": "aggregate-error@^2.0.0", + "_where": "/Users/peter/code/pelias/docker/projects/portland-metro/admin-lookup/node_modules/worker-thread-pool", + "author": { + "name": "Sindre Sorhus", + "email": "sindresorhus@gmail.com", + "url": "sindresorhus.com" + }, + "bugs": { + "url": "https://github.com/sindresorhus/aggregate-error/issues" + }, + "bundleDependencies": false, + "dependencies": { + "clean-stack": "^2.0.0", + "indent-string": "^3.0.0" + }, + "deprecated": false, + "description": "Create an error from multiple errors", + "devDependencies": { + "ava": "^1.2.1", + "tsd-check": "^0.3.0", + "xo": "^0.24.0" + }, + "engines": { + "node": ">=6" + }, + "files": [ + "index.js", + "index.d.ts" + ], + "homepage": "https://github.com/sindresorhus/aggregate-error#readme", + "keywords": [ + "aggregate", + "error", + "err", + "combine", + "multiple", + "many", + "collection", + "iterable", + "iterator" + ], + "license": "MIT", + "name": "aggregate-error", + "repository": { + "type": "git", + "url": "git+https://github.com/sindresorhus/aggregate-error.git" + }, + "scripts": { + "test": "xo && ava && tsd-check" + }, + "version": "2.2.0" +} diff --git a/projects/portland-metro/admin-lookup/node_modules/aggregate-error/readme.md b/projects/portland-metro/admin-lookup/node_modules/aggregate-error/readme.md new file mode 100644 index 00000000..a03ecaca --- /dev/null +++ b/projects/portland-metro/admin-lookup/node_modules/aggregate-error/readme.md @@ -0,0 +1,66 @@ +# aggregate-error [![Build Status](https://travis-ci.org/sindresorhus/aggregate-error.svg?branch=master)](https://travis-ci.org/sindresorhus/aggregate-error) + +> Create an error from multiple errors + + +## Install + +``` +$ npm install aggregate-error +``` + + +## Usage + +```js +const AggregateError = require('aggregate-error'); + +const error = new AggregateError([new Error('foo'), 'bar', {message: 'baz'}]); + +throw error; +/* +AggregateError: + Error: foo + at Object. (/Users/sindresorhus/dev/aggregate-error/example.js:3:33) + Error: bar + at Object. (/Users/sindresorhus/dev/aggregate-error/example.js:3:13) + Error: baz + at Object. (/Users/sindresorhus/dev/aggregate-error/example.js:3:13) + at AggregateError (/Users/sindresorhus/dev/aggregate-error/index.js:19:3) + at Object. (/Users/sindresorhus/dev/aggregate-error/example.js:3:13) + at Module._compile (module.js:556:32) + at Object.Module._extensions..js (module.js:565:10) + at Module.load (module.js:473:32) + at tryModuleLoad (module.js:432:12) + at Function.Module._load (module.js:424:3) + at Module.runMain (module.js:590:10) + at run (bootstrap_node.js:394:7) + at startup (bootstrap_node.js:149:9) +*/ + +for (const individualError of error) { + console.log(individualError); +} +//=> [Error: foo] +//=> [Error: bar] +//=> [Error: baz] +``` + + +## API + +### AggregateError(errors) + +Returns an `Error` that is also an [`Iterable`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Iterators_and_Generators#Iterables) for the individual errors. + +#### errors + +Type: `Array` + +If a string, a new `Error` is created with the string as the error message.
+If a non-Error object, a new `Error` is created with all properties from the object copied over. + + +## License + +MIT © [Sindre Sorhus](https://sindresorhus.com) diff --git a/projects/portland-metro/admin-lookup/node_modules/clean-stack/index.d.ts b/projects/portland-metro/admin-lookup/node_modules/clean-stack/index.d.ts new file mode 100644 index 00000000..ed589950 --- /dev/null +++ b/projects/portland-metro/admin-lookup/node_modules/clean-stack/index.d.ts @@ -0,0 +1,47 @@ +declare namespace cleanStack { + interface Options { + /** + Prettify the file paths in the stack: + + `/Users/sindresorhus/dev/clean-stack/unicorn.js:2:15` → `~/dev/clean-stack/unicorn.js:2:15` + + @default false + */ + readonly pretty?: boolean; + } +} + +/** +Clean up error stack traces. Removes the mostly unhelpful internal Node.js entries. + +@param stack - The `stack` property of an `Error`. + +@example +``` +import cleanStack = require('clean-stack'); + +const error = new Error('Missing unicorn'); + +console.log(error.stack); + +// Error: Missing unicorn +// at Object. (/Users/sindresorhus/dev/clean-stack/unicorn.js:2:15) +// at Module._compile (module.js:409:26) +// at Object.Module._extensions..js (module.js:416:10) +// at Module.load (module.js:343:32) +// at Function.Module._load (module.js:300:12) +// at Function.Module.runMain (module.js:441:10) +// at startup (node.js:139:18) + +console.log(cleanStack(error.stack)); + +// Error: Missing unicorn +// at Object. (/Users/sindresorhus/dev/clean-stack/unicorn.js:2:15) +``` +*/ +declare function cleanStack( + stack: string, + options?: cleanStack.Options +): string; + +export = cleanStack; diff --git a/projects/portland-metro/admin-lookup/node_modules/clean-stack/index.js b/projects/portland-metro/admin-lookup/node_modules/clean-stack/index.js new file mode 100644 index 00000000..8c1dcc4c --- /dev/null +++ b/projects/portland-metro/admin-lookup/node_modules/clean-stack/index.js @@ -0,0 +1,40 @@ +'use strict'; +const os = require('os'); + +const extractPathRegex = /\s+at.*(?:\(|\s)(.*)\)?/; +const pathRegex = /^(?:(?:(?:node|(?:internal\/[\w/]*|.*node_modules\/(?:babel-polyfill|pirates)\/.*)?\w+)\.js:\d+:\d+)|native)/; +const homeDir = typeof os.homedir === 'undefined' ? '' : os.homedir(); + +module.exports = (stack, options) => { + options = Object.assign({pretty: false}, options); + + return stack.replace(/\\/g, '/') + .split('\n') + .filter(line => { + const pathMatches = line.match(extractPathRegex); + if (pathMatches === null || !pathMatches[1]) { + return true; + } + + const match = pathMatches[1]; + + // Electron + if ( + match.includes('.app/Contents/Resources/electron.asar') || + match.includes('.app/Contents/Resources/default_app.asar') + ) { + return false; + } + + return !pathRegex.test(match); + }) + .filter(line => line.trim() !== '') + .map(line => { + if (options.pretty) { + return line.replace(extractPathRegex, (m, p1) => m.replace(p1, p1.replace(homeDir, '~'))); + } + + return line; + }) + .join('\n'); +}; diff --git a/projects/portland-metro/admin-lookup/node_modules/clean-stack/license b/projects/portland-metro/admin-lookup/node_modules/clean-stack/license new file mode 100644 index 00000000..e7af2f77 --- /dev/null +++ b/projects/portland-metro/admin-lookup/node_modules/clean-stack/license @@ -0,0 +1,9 @@ +MIT License + +Copyright (c) Sindre Sorhus (sindresorhus.com) + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/projects/portland-metro/admin-lookup/node_modules/clean-stack/package.json b/projects/portland-metro/admin-lookup/node_modules/clean-stack/package.json new file mode 100644 index 00000000..1fd43320 --- /dev/null +++ b/projects/portland-metro/admin-lookup/node_modules/clean-stack/package.json @@ -0,0 +1,71 @@ +{ + "_from": "clean-stack@^2.0.0", + "_id": "clean-stack@2.2.0", + "_inBundle": false, + "_integrity": "sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A==", + "_location": "/clean-stack", + "_phantomChildren": {}, + "_requested": { + "type": "range", + "registry": true, + "raw": "clean-stack@^2.0.0", + "name": "clean-stack", + "escapedName": "clean-stack", + "rawSpec": "^2.0.0", + "saveSpec": null, + "fetchSpec": "^2.0.0" + }, + "_requiredBy": [ + "/aggregate-error" + ], + "_resolved": "https://registry.npmjs.org/clean-stack/-/clean-stack-2.2.0.tgz", + "_shasum": "ee8472dbb129e727b31e8a10a427dee9dfe4008b", + "_spec": "clean-stack@^2.0.0", + "_where": "/Users/peter/code/pelias/docker/projects/portland-metro/admin-lookup/node_modules/aggregate-error", + "author": { + "name": "Sindre Sorhus", + "email": "sindresorhus@gmail.com", + "url": "sindresorhus.com" + }, + "browser": { + "os": false + }, + "bugs": { + "url": "https://github.com/sindresorhus/clean-stack/issues" + }, + "bundleDependencies": false, + "deprecated": false, + "description": "Clean up error stack traces", + "devDependencies": { + "ava": "^1.4.1", + "tsd": "^0.7.2", + "xo": "^0.24.0" + }, + "engines": { + "node": ">=6" + }, + "files": [ + "index.js", + "index.d.ts" + ], + "homepage": "https://github.com/sindresorhus/clean-stack#readme", + "keywords": [ + "clean", + "stack", + "trace", + "traces", + "error", + "err", + "electron" + ], + "license": "MIT", + "name": "clean-stack", + "repository": { + "type": "git", + "url": "git+https://github.com/sindresorhus/clean-stack.git" + }, + "scripts": { + "test": "xo && ava && tsd" + }, + "version": "2.2.0" +} diff --git a/projects/portland-metro/admin-lookup/node_modules/clean-stack/readme.md b/projects/portland-metro/admin-lookup/node_modules/clean-stack/readme.md new file mode 100644 index 00000000..8d441604 --- /dev/null +++ b/projects/portland-metro/admin-lookup/node_modules/clean-stack/readme.md @@ -0,0 +1,76 @@ +# clean-stack [![Build Status](https://travis-ci.org/sindresorhus/clean-stack.svg?branch=master)](https://travis-ci.org/sindresorhus/clean-stack) + +> Clean up error stack traces + +Removes the mostly unhelpful internal Node.js entries. + +Also works in Electron. + + +## Install + +``` +$ npm install clean-stack +``` + + +## Usage + +```js +const cleanStack = require('clean-stack'); + +const error = new Error('Missing unicorn'); + +console.log(error.stack); +/* +Error: Missing unicorn + at Object. (/Users/sindresorhus/dev/clean-stack/unicorn.js:2:15) + at Module._compile (module.js:409:26) + at Object.Module._extensions..js (module.js:416:10) + at Module.load (module.js:343:32) + at Function.Module._load (module.js:300:12) + at Function.Module.runMain (module.js:441:10) + at startup (node.js:139:18) +*/ + +console.log(cleanStack(error.stack)); +/* +Error: Missing unicorn + at Object. (/Users/sindresorhus/dev/clean-stack/unicorn.js:2:15) +*/ +``` + + +## API + +### cleanStack(stack, [options]) + +#### stack + +Type: `string` + +The `stack` property of an `Error`. + +#### options + +Type: `Object` + +##### pretty + +Type: `boolean`
+Default: `false` + +Prettify the file paths in the stack: + +`/Users/sindresorhus/dev/clean-stack/unicorn.js:2:15` → `~/dev/clean-stack/unicorn.js:2:15` + + +## Related + +- [extrack-stack](https://github.com/sindresorhus/extract-stack) - Extract the actual stack of an error +- [stack-utils](https://github.com/tapjs/stack-utils) - Captures and cleans stack traces + + +## License + +MIT © [Sindre Sorhus](https://sindresorhus.com) diff --git a/projects/portland-metro/admin-lookup/node_modules/indent-string/index.js b/projects/portland-metro/admin-lookup/node_modules/indent-string/index.js new file mode 100644 index 00000000..a48199ca --- /dev/null +++ b/projects/portland-metro/admin-lookup/node_modules/indent-string/index.js @@ -0,0 +1,27 @@ +'use strict'; +module.exports = (str, count, opts) => { + // Support older versions: use the third parameter as options.indent + // TODO: Remove the workaround in the next major version + const options = typeof opts === 'object' ? Object.assign({indent: ' '}, opts) : {indent: opts || ' '}; + count = count === undefined ? 1 : count; + + if (typeof str !== 'string') { + throw new TypeError(`Expected \`input\` to be a \`string\`, got \`${typeof str}\``); + } + + if (typeof count !== 'number') { + throw new TypeError(`Expected \`count\` to be a \`number\`, got \`${typeof count}\``); + } + + if (typeof options.indent !== 'string') { + throw new TypeError(`Expected \`options.indent\` to be a \`string\`, got \`${typeof options.indent}\``); + } + + if (count === 0) { + return str; + } + + const regex = options.includeEmptyLines ? /^/mg : /^(?!\s*$)/mg; + return str.replace(regex, options.indent.repeat(count)); +} +; diff --git a/projects/portland-metro/admin-lookup/node_modules/indent-string/license b/projects/portland-metro/admin-lookup/node_modules/indent-string/license new file mode 100644 index 00000000..e7af2f77 --- /dev/null +++ b/projects/portland-metro/admin-lookup/node_modules/indent-string/license @@ -0,0 +1,9 @@ +MIT License + +Copyright (c) Sindre Sorhus (sindresorhus.com) + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/projects/portland-metro/admin-lookup/node_modules/indent-string/package.json b/projects/portland-metro/admin-lookup/node_modules/indent-string/package.json new file mode 100644 index 00000000..f275e59e --- /dev/null +++ b/projects/portland-metro/admin-lookup/node_modules/indent-string/package.json @@ -0,0 +1,68 @@ +{ + "_from": "indent-string@^3.0.0", + "_id": "indent-string@3.2.0", + "_inBundle": false, + "_integrity": "sha1-Sl/W0nzDMvN+VBmlBNu4NxBckok=", + "_location": "/indent-string", + "_phantomChildren": {}, + "_requested": { + "type": "range", + "registry": true, + "raw": "indent-string@^3.0.0", + "name": "indent-string", + "escapedName": "indent-string", + "rawSpec": "^3.0.0", + "saveSpec": null, + "fetchSpec": "^3.0.0" + }, + "_requiredBy": [ + "/aggregate-error" + ], + "_resolved": "https://registry.npmjs.org/indent-string/-/indent-string-3.2.0.tgz", + "_shasum": "4a5fd6d27cc332f37e5419a504dbb837105c9289", + "_spec": "indent-string@^3.0.0", + "_where": "/Users/peter/code/pelias/docker/projects/portland-metro/admin-lookup/node_modules/aggregate-error", + "author": { + "name": "Sindre Sorhus", + "email": "sindresorhus@gmail.com", + "url": "sindresorhus.com" + }, + "bugs": { + "url": "https://github.com/sindresorhus/indent-string/issues" + }, + "bundleDependencies": false, + "deprecated": false, + "description": "Indent each line in a string", + "devDependencies": { + "ava": "*", + "xo": "*" + }, + "engines": { + "node": ">=4" + }, + "files": [ + "index.js" + ], + "homepage": "https://github.com/sindresorhus/indent-string#readme", + "keywords": [ + "indent", + "string", + "str", + "pad", + "align", + "line", + "text", + "each", + "every" + ], + "license": "MIT", + "name": "indent-string", + "repository": { + "type": "git", + "url": "git+https://github.com/sindresorhus/indent-string.git" + }, + "scripts": { + "test": "xo && ava" + }, + "version": "3.2.0" +} diff --git a/projects/portland-metro/admin-lookup/node_modules/indent-string/readme.md b/projects/portland-metro/admin-lookup/node_modules/indent-string/readme.md new file mode 100644 index 00000000..d2592d73 --- /dev/null +++ b/projects/portland-metro/admin-lookup/node_modules/indent-string/readme.md @@ -0,0 +1,72 @@ +# indent-string [![Build Status](https://travis-ci.org/sindresorhus/indent-string.svg?branch=master)](https://travis-ci.org/sindresorhus/indent-string) + +> Indent each line in a string + + +## Install + +``` +$ npm install indent-string +``` + + +## Usage + +```js +const indentString = require('indent-string'); + +indentString('Unicorns\nRainbows', 4); +//=> ' Unicorns' +//=> ' Rainbows' + +indentString('Unicorns\nRainbows', 4, {indent: '♥'}); +//=> '♥♥♥♥Unicorns' +//=> '♥♥♥♥Rainbows' +``` + + +## API + +### indentString(input, [count], [options]) + +#### input + +Type: `string` + +String you want to indent. + +#### count + +Type: `number`
+Default: `1` + +How many times you want `indent` repeated. + +#### options + +Type: `Object`
+ +##### indent + +Type: `string`
+Default: `' '` + +String to use for the indent. + +##### includeEmptyLines + +Type: `boolean`
+Default: `false` + +Also indent empty lines. + + +## Related + +- [indent-string-cli](https://github.com/sindresorhus/indent-string-cli) - CLI for this module +- [strip-indent](https://github.com/sindresorhus/strip-indent) - Strip leading whitespace from every line in a string + + +## License + +MIT © [Sindre Sorhus](https://sindresorhus.com) diff --git a/projects/portland-metro/admin-lookup/node_modules/worker-thread-pool/.travis.yml b/projects/portland-metro/admin-lookup/node_modules/worker-thread-pool/.travis.yml new file mode 100644 index 00000000..56e925c2 --- /dev/null +++ b/projects/portland-metro/admin-lookup/node_modules/worker-thread-pool/.travis.yml @@ -0,0 +1,13 @@ +language: node_js + +node_js: + - "11" + - "10" + +script: +- npm run coveralls + +notifications: + email: + on_success: never + on_failure: always \ No newline at end of file diff --git a/projects/portland-metro/admin-lookup/node_modules/worker-thread-pool/LICENSE b/projects/portland-metro/admin-lookup/node_modules/worker-thread-pool/LICENSE new file mode 100644 index 00000000..cde62e43 --- /dev/null +++ b/projects/portland-metro/admin-lookup/node_modules/worker-thread-pool/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2018-2019 Denis Fäcke + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/projects/portland-metro/admin-lookup/node_modules/worker-thread-pool/README.md b/projects/portland-metro/admin-lookup/node_modules/worker-thread-pool/README.md new file mode 100644 index 00000000..794c7750 --- /dev/null +++ b/projects/portland-metro/admin-lookup/node_modules/worker-thread-pool/README.md @@ -0,0 +1,72 @@ +# worker-thread-pool + +[![Build Status](https://travis-ci.org/SerayaEryn/worker-thread-pool.svg?branch=master)](https://travis-ci.org/SerayaEryn/worker-thread-pool) +[![Coverage Status](https://coveralls.io/repos/github/SerayaEryn/worker-thread-pool/badge.svg?branch=master)](https://coveralls.io/github/SerayaEryn/worker-thread-pool?branch=master) +[![JavaScript Style Guide](https://img.shields.io/badge/code_style-standard-brightgreen.svg)](https://standardjs.com) +[![NPM version](https://img.shields.io/npm/v/worker-thread-pool.svg?style=flat)](https://www.npmjs.com/package/worker-thread-pool) [![Greenkeeper badge](https://badges.greenkeeper.io/SerayaEryn/worker-thread-pool.svg)](https://greenkeeper.io/) + +An easy way to create a pool of worker threads. + +## Usage + +If being used with node v10.5.0 to v11.6.0 the `worker-thread-pool` module requires node to be started with the `--experimental-worker` flag. + +```js +//main.js +const Pool = require('worker-thread-pool'); + +const pool = new Pool({ + path: __dirname + '/worker.js' +}); +pool.run({name: 'world'}) + .then((result) => { + //... + }) +``` + +```js +//worker.js +const { parentPort } = require('worker_threads'); + +parentPort.on('message', (message) => { + message.port.postMessage('hello ' + message.name); + message.port.close(); +}); +``` + +## API + +### Pool(options) + +Creates a new pool with workers for the specified javascript file. + +#### options + +##### path + +The path to the javascript file containing the source code to be executed in the thread pool. + +##### size (optional) + +The size of the thread pool. Defaults to `4`. + +### Pool#run(workerData) + +Passes the `workerData` to the worker and waits until the worker sends back an answer. Resolves the answer of the worker in a Promise. + +### Poll#queueLength() + +Returns the current length of the queue. + +### Poll#poolLength() + +Returns the current size of the pool. + +### Pool#close() + +Removes all workers from the pool, calls `terminate` on them and then emits a `close` event. +If an error occurs during an `error` event will be emitted. + +## License + +[MIT](./LICENSE) \ No newline at end of file diff --git a/projects/portland-metro/admin-lookup/node_modules/worker-thread-pool/lib/Worker.js b/projects/portland-metro/admin-lookup/node_modules/worker-thread-pool/lib/Worker.js new file mode 100644 index 00000000..7558f789 --- /dev/null +++ b/projects/portland-metro/admin-lookup/node_modules/worker-thread-pool/lib/Worker.js @@ -0,0 +1,122 @@ +'use strict' + +const { Worker, MessageChannel } = require('worker_threads') +const EventEmitter = require('events') +const AggregateError = require('aggregate-error') + +const addNewWorkerToPool = Symbol('addNewWorkerToPool') +const onMessage = Symbol('onMessage') +const onRelease = Symbol('onRelease') +const pool = Symbol('pool') +const queue = Symbol('queue') +const run = Symbol('run') +const size = Symbol('size') +const closing = Symbol('closing') +const path = Symbol('path') + +module.exports = class ThreadPool extends EventEmitter { + constructor (options) { + super() + const opts = options + this[path] = opts.path + this[size] = opts.size || 4 + this[pool] = [] + this[queue] = [] + this[closing] = false + for (let i = 0; i < this[size]; i++) { + this[addNewWorkerToPool]() + } + this[onRelease] = this[onRelease].bind(this) + this.on('release', this[onRelease]) + } + + run (workerData) { + return new Promise((resolve, reject) => { + if (this[pool].length > 0) { + const worker = this[pool].shift() + this[run](worker, workerData, resolve, reject) + } else { + this[queue].push((worker) => { + this[run](worker, workerData, resolve, reject) + }) + } + }) + } + + close () { + this[closing] = true + const promises = [] + const size = this.poolLength() + for (let index = 0; index < size; index++) { + const worker = this[pool].shift() + promises.push(new Promise((resolve) => { + worker.terminate(resolve) + })) + } + Promise.all(promises) + .then(handleErrors) + .then(() => this.emit('close')) + .catch((error) => { + this.emit('error', error) + this.emit('close') + }) + } + + queueLength () { + return this[queue].length + } + + poolLength () { + return this[pool].length + } + + [onRelease] (worker) { + if (this[queue].length > 0) { + const cb = this[queue].shift() + cb(worker) + } else { + this[pool].push(worker) + } + } + + [run] (worker, workerData, resolve, reject) { + let messageChannel = new MessageChannel() + workerData.port = messageChannel.port1 + messageChannel.port2.once('message', (result) => { + worker.removeAllListeners('error') + this[onMessage](resolve, worker, result) + }) + worker.once('error', (err) => { + reject(err) + this[addNewWorkerToPool]() + }) + worker.postMessage(workerData, [messageChannel.port1]) + } + + [onMessage] (resolve, worker, result) { + this.emit('release', worker) + resolve(result) + } + + [addNewWorkerToPool] () { + const worker = new Worker(this[path]) + worker.once('exit', (exitCode) => { + if (exitCode !== 0 && !this[closing]) { + this[addNewWorkerToPool]() + } + }) + this[pool].push(worker) + } +} + +function handleErrors (maybeErrors) { + const errors = maybeErrors.filter(notNull) + if (errors.length === 0) { + return + } + throw new AggregateError(errors) +} + +function notNull (error) { + return error != null +} diff --git a/projects/portland-metro/admin-lookup/node_modules/worker-thread-pool/package.json b/projects/portland-metro/admin-lookup/node_modules/worker-thread-pool/package.json new file mode 100644 index 00000000..2cf209b9 --- /dev/null +++ b/projects/portland-metro/admin-lookup/node_modules/worker-thread-pool/package.json @@ -0,0 +1,71 @@ +{ + "_from": "worker-thread-pool", + "_id": "worker-thread-pool@1.0.0", + "_inBundle": false, + "_integrity": "sha512-YSi61A+UZ2MSx8YMM/VngKVfoB5VvwkvFkgI5OFeBJyMyjrhdDX6yBlZBCaNdw5A7jW36I47JxxGmygOIuT7IQ==", + "_location": "/worker-thread-pool", + "_phantomChildren": {}, + "_requested": { + "type": "tag", + "registry": true, + "raw": "worker-thread-pool", + "name": "worker-thread-pool", + "escapedName": "worker-thread-pool", + "rawSpec": "", + "saveSpec": null, + "fetchSpec": "latest" + }, + "_requiredBy": [ + "#USER", + "/" + ], + "_resolved": "https://registry.npmjs.org/worker-thread-pool/-/worker-thread-pool-1.0.0.tgz", + "_shasum": "2f9a73f0994a53c332e8385ce89ff6b3bf87f5c3", + "_spec": "worker-thread-pool", + "_where": "/Users/peter/code/pelias/docker/projects/portland-metro/admin-lookup", + "author": { + "name": "Denis Fäcke" + }, + "bugs": { + "url": "https://github.com/SerayaEryn/worker-thread-pool/issues" + }, + "bundleDependencies": false, + "dependencies": { + "aggregate-error": "^2.0.0" + }, + "deprecated": false, + "description": "A easy way to create a pool of worker threads.", + "devDependencies": { + "coveralls": "^3.0.2", + "standard": "^12.0.1", + "tap": "^12.0.1" + }, + "engines": { + "node": ">= 10.5.0" + }, + "homepage": "https://github.com/SerayaEryn/worker-thread-pool#readme", + "keywords": [ + "pool", + "worker", + "thread-pool", + "workers", + "thread", + "worker_threads", + "threads" + ], + "license": "MIT", + "main": "lib/Worker.js", + "name": "worker-thread-pool", + "repository": { + "type": "git", + "url": "git+https://github.com/SerayaEryn/worker-thread-pool.git" + }, + "scripts": { + "coverage-report": "npm run coveralls && tap --coverage-report=lcov", + "coveralls": "npm run unit -- --cov", + "lint": "standard lib/* test/*", + "test": "npm run lint && npm run unit", + "unit": "tap test/*.test.js --node-arg=--experimental-worker" + }, + "version": "1.0.0" +} diff --git a/projects/portland-metro/admin-lookup/node_modules/worker-thread-pool/test/boom.js b/projects/portland-metro/admin-lookup/node_modules/worker-thread-pool/test/boom.js new file mode 100644 index 00000000..ba002cfe --- /dev/null +++ b/projects/portland-metro/admin-lookup/node_modules/worker-thread-pool/test/boom.js @@ -0,0 +1,9 @@ +'use strict' + +const { parentPort } = require('worker_threads') + +parentPort.on('message', handleMessage) + +function handleMessage (message) { + throw new Error('boooom') +} diff --git a/projects/portland-metro/admin-lookup/node_modules/worker-thread-pool/test/pool.test.js b/projects/portland-metro/admin-lookup/node_modules/worker-thread-pool/test/pool.test.js new file mode 100644 index 00000000..23dd3007 --- /dev/null +++ b/projects/portland-metro/admin-lookup/node_modules/worker-thread-pool/test/pool.test.js @@ -0,0 +1,81 @@ +'use strict' + +const t = require('tap') +const test = t.test +const Pool = require('../lib/Worker') +const path = require('path') + +test('should create pool', (t) => { + t.plan(2) + const pool = new Pool({ + path: path.join(__dirname, '/worker.js') + }) + + return pool.run({ test: 1 }) + .then((result) => { + t.strictEquals(result, 'hello world') + pool.close() + }) + .then(() => t.pass()) +}) + +test('should close pool', (t) => { + t.plan(2) + const pool = new Pool({ + path: path.join(__dirname, '/worker.js'), + size: 1 + }) + + return Promise.all([ + pool.run({ test: 1 }), + pool.run({ test: 2 }), + pool.run({ test: 3 }), + pool.run({ test: 4 }) + ]) + .then((result) => { + t.deepEquals(result, ['hello world', 'hello world', 'hello world', 'hello world']) + pool.close() + }) + .then(() => { + t.equals(pool.poolLength(), 0) + }) +}) + +test('should create new worker if error in worker', (t) => { + t.plan(3) + const pool = new Pool({ + path: path.join(__dirname, '/boom.js'), + size: 1 + }) + + return pool.run({ test: 1 }) + .catch((err) => { + t.equals(err.message, 'boooom') + t.equals(pool.poolLength(), 1) + pool.close() + t.equals(pool.poolLength(), 0) + }) +}) + +test('should emit close', (t) => { + t.plan(1) + const pool = new Pool({ + path: path.join(__dirname, '/boom.js'), + size: 1 + }) + + pool.on('close', () => t.pass('close emitted')) + pool.close() +}) + +test('should return queue length', (t) => { + t.plan(2) + const pool = new Pool({ + path: path.join(__dirname, '/boom.js'), + size: 1 + }) + + t.strictEquals(pool.queueLength(), 0) + pool.on('close', () => t.pass('close emitted')) + pool.close() +}) diff --git a/projects/portland-metro/admin-lookup/node_modules/worker-thread-pool/test/worker.js b/projects/portland-metro/admin-lookup/node_modules/worker-thread-pool/test/worker.js new file mode 100644 index 00000000..043edd95 --- /dev/null +++ b/projects/portland-metro/admin-lookup/node_modules/worker-thread-pool/test/worker.js @@ -0,0 +1,12 @@ +'use strict' + +const { parentPort } = require('worker_threads') + +parentPort.on('message', handleMessage) + +function handleMessage (message) { + setTimeout(() => { + message.port.postMessage('hello world') + message.port.close() + }, 50) +} diff --git a/projects/portland-metro/admin-lookup/package-lock.json b/projects/portland-metro/admin-lookup/package-lock.json new file mode 100644 index 00000000..ae311971 --- /dev/null +++ b/projects/portland-metro/admin-lookup/package-lock.json @@ -0,0 +1,33 @@ +{ + "requires": true, + "lockfileVersion": 1, + "dependencies": { + "aggregate-error": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-2.2.0.tgz", + "integrity": "sha512-E5n+IZkhh22/pFdUvHUU/o9z752lc+7tgHt+FXS/g6BjlbE9249dGmuS/SxIWMPhTljZJkFN+7OXE0+O5+WT8w==", + "requires": { + "clean-stack": "^2.0.0", + "indent-string": "^3.0.0" + } + }, + "clean-stack": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/clean-stack/-/clean-stack-2.2.0.tgz", + "integrity": "sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A==" + }, + "indent-string": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-3.2.0.tgz", + "integrity": "sha1-Sl/W0nzDMvN+VBmlBNu4NxBckok=" + }, + "worker-thread-pool": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/worker-thread-pool/-/worker-thread-pool-1.0.0.tgz", + "integrity": "sha512-YSi61A+UZ2MSx8YMM/VngKVfoB5VvwkvFkgI5OFeBJyMyjrhdDX6yBlZBCaNdw5A7jW36I47JxxGmygOIuT7IQ==", + "requires": { + "aggregate-error": "^2.0.0" + } + } + } +} diff --git a/projects/portland-metro/admin-lookup/package.json b/projects/portland-metro/admin-lookup/package.json new file mode 100644 index 00000000..a927655e --- /dev/null +++ b/projects/portland-metro/admin-lookup/package.json @@ -0,0 +1,6 @@ +{ + "main": "./main.js", + "dependencies": { + "worker-thread-pool": "^1.0.0" + } +} diff --git a/projects/portland-metro/admin-lookup/queryServiceWorkerThread.js b/projects/portland-metro/admin-lookup/queryServiceWorkerThread.js new file mode 100644 index 00000000..04de37ab --- /dev/null +++ b/projects/portland-metro/admin-lookup/queryServiceWorkerThread.js @@ -0,0 +1,134 @@ +const _ = require('lodash') +// const codec = require('./codec') +const threads = require('worker_threads') + +const QueryService = require('/code/spatial/service/QueryService') +const middleware = require('/code/spatial/server/routes/pip_beta') +const service = new QueryService({ + readonly: true, + filename: '/data/spatial/docker.spatial.db' +}) + +const adminLayers = [ + 'neighbourhood', 'borough', + 'locality', 'localadmin', + 'county', 'macrocounty', + 'region', 'macroregion', + 'dependency', 'country', + 'empire', 'continent' +] + +threads.parentPort.on('message', ({ centroid, port }) => { + // console.error('worker message', doc) + // doc = codec.unmarshal(doc) + + // generate query + const query = { + lon: centroid.lon, + lat: centroid.lat, + lang: 'und', + tag: 'default' + } + + lookup(query, (err, res) => { + if (err) { + console.error('worker error', err) + port.postMessage({ err }); + port.close(); + return + } + + const addParent = [] + mapPIPResultsToDocument(addParent, res) + port.postMessage({ addParent }) + port.close() + + // console.error('mapped', doc) + + // port.postMessage({ doc }); + // port.close(); + }) +}); + +console.error('worker start') + +function lookup(query, cb) { + + const req = {} + _.set(req, 'app.locals.service', service) + _.set(req, 'query', query) + + const res = {} + res.status = () => res + res.json = (data) => cb(null, data) + + middleware(req, res) +} + +function mapPIPResultsToDocument(foo, res) { + const hits = _.get(res, 'hits', []) + const place = _.get(res, 'place', {}) + const name = _.get(res, 'name', {}) + let hierarchy = _.get(res, 'hierarchy', {}) + + // hard-core the WOF hierarchy branch to use + const branch = 'wof:0' + hierarchy = _.get(hierarchy, branch, {}) + + // iterate over layers from most granular to least granular + for (const layer of adminLayers) { + + let parentIdentity = _.find(hits, (identity) => { + const p = _.get(place, identity) + return _.get(p, 'type') === layer && _.get(p, 'class') === 'admin' + }) + + // no match found at this layer + if (!parentIdentity) { continue } + + // walk hierarchy to add parents + const seen = {} + while (parentIdentity) { + + // prevent infinite recursion + if (_.has(seen, parentIdentity)) { break } + _.set(seen, parentIdentity); + + // load parent data + const parent = { + place: _.get(place, parentIdentity), + name: _.get(_.get(name, parentIdentity), 'und|default') + } + + // parent not found (not loaded into the spatial DB?) + if (!_.has(parent, 'place.id')) { + // logger.warn(`parent not found: ${parentIdentity}`); + break + } + + // call postmessage with parents to add + try { + foo.push([ + _.get(parent, 'place.type'), + _.get(parent, 'name.name[0]'), + _.get(parent, 'place.id'), + _.get(parent, 'name.abbr[0]') + ]) + } + catch (err) { + logger.warn('invalid value', { + result: { + type: layer, + values: parent + } + }); + } + + // find next parent + parentIdentity = _.get(hierarchy, parentIdentity) + } + + // do not check other layers + break; + } +} diff --git a/projects/portland-metro/docker-compose.yml b/projects/portland-metro/docker-compose.yml index 729f5026..14160805 100644 --- a/projects/portland-metro/docker-compose.yml +++ b/projects/portland-metro/docker-compose.yml @@ -53,12 +53,16 @@ services: - "./blacklist/:/data/blacklist" openaddresses: image: pelias/openaddresses:master + build: + context: . + dockerfile: Dockerfile.oa.plus.spatial container_name: pelias_openaddresses user: "${DOCKER_USER}" volumes: - "./pelias.json:/code/pelias.json" - "${DATA_DIR}:/data" - "./blacklist/:/data/blacklist" + - "./admin-lookup:/code/pelias/openaddresses/node_modules/pelias-wof-admin-lookup:cached" transit: image: pelias/transit:master container_name: pelias_transit @@ -91,16 +95,16 @@ services: volumes: - "./pelias.json:/code/pelias.json" - "${DATA_DIR}:/data" - pip: - image: pelias/pip-service:master - container_name: pelias_pip-service + spatial: + image: pelias/spatial:pip_beta + container_name: pelias_spatial user: "${DOCKER_USER}" restart: always - environment: [ "PORT=4200" ] - ports: [ "4200:4200" ] + environment: [ "PORT=4770" ] + ports: [ "4770:4770" ] + command: [ "server", "--db=/data/spatial/docker.spatial.db" ] volumes: - - "./pelias.json:/code/pelias.json" - - "${DATA_DIR}:/data" + - "${DATA_DIR}:/data:cached" elasticsearch: image: pelias/elasticsearch:7.5.1 container_name: pelias_elasticsearch