From e9f11f6707dd77301886255724aa7ee696da10b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franti=C5=A1ek=20Kolovsk=C3=BD?= Date: Fri, 2 Nov 2018 11:06:06 +0100 Subject: [PATCH] data loading classes --- doc/multiple_loading.md | 34 ++++++++++++++++++++++++++ src/experimental/ClassicDataLoader.js | 13 ++++++++++ src/experimental/MultipleDataLoader.js | 22 +++++++++++++++++ 3 files changed, 69 insertions(+) create mode 100644 doc/multiple_loading.md create mode 100644 src/experimental/ClassicDataLoader.js create mode 100644 src/experimental/MultipleDataLoader.js diff --git a/doc/multiple_loading.md b/doc/multiple_loading.md new file mode 100644 index 000000000..eb99cf107 --- /dev/null +++ b/doc/multiple_loading.md @@ -0,0 +1,34 @@ +Multiple data loader +==================== +Metadata file +------------- +```json +{ + "files": ["file1", "file2"] +} +``` +Name convention +--------------- +Every file contain N records. The records are continuously numbered form 0. The name of file is first_record_ID/N. + +For example: +``` +filename ID range +0.csv <0, N - 1> +1.csv +2.csv <2N, 3N - 1> +... +``` +```csv +ID,x,y,accident_index,timestamp,accident_severity,road_type,speed_limit +0,127.59738453,85.12878333,201243R249122,1356994800,2,6,30 +1,128.43117867,85.63623004,2012460222555,1356995160,3,6,60 +2,127.8970816,85.90564471,2012471300002,1356995340,2,3,30 +3,127.9295808,85.04589612,201201TB01286,1356995520,2,6,30 +4,126.33486436,82.66046783,201206N068460,1356996600,3,6,30 +5,128.03807716,85.07476281,201201KF69105,1356996600,3,6,30 +6,127.78275271,85.23497257,201201VK30371,1356996780,3,3,30 +7,126.86014507,82.51395481,2012130000038,1356996900,3,6,30 +8,128.06074311,85.06769119,201201JI40714,1356997500,3,6,30 +9,126.94752924,83.52010252,201230D002704,1356998160,2,6,30 +``` \ No newline at end of file diff --git a/src/experimental/ClassicDataLoader.js b/src/experimental/ClassicDataLoader.js new file mode 100644 index 000000000..ebc063f3f --- /dev/null +++ b/src/experimental/ClassicDataLoader.js @@ -0,0 +1,13 @@ +/** + * Classic data loader + * @param {string} datafile csv file with data + * @param callback call back function + * @constructor + */ +WGL.experimental.ClassicDataLoader = function (datafile, callback) { + this.load = function(){ + d3.csv(datafile, function (err, data) { + callback(data); + }); + }; +}; \ No newline at end of file diff --git a/src/experimental/MultipleDataLoader.js b/src/experimental/MultipleDataLoader.js new file mode 100644 index 000000000..26cd791f2 --- /dev/null +++ b/src/experimental/MultipleDataLoader.js @@ -0,0 +1,22 @@ +/** + * Data loader for multiple files + * @param {string} metafile metadata file + * @param callback call back function, one parameter for data + * @constructor + */ +WGL.experimental.MultipleDataLoader = function (metafile, callback) { + this.load = function(){ + let patha = metafile.split("/"); + let path = metafile.replace(patha[patha.length - 1], ""); + + d3.json(metafile, function (err, config) { + let q = queue(); + config.files.forEach(function (data, i) { + q.defer(d3.csv, path+config.files[i]); + }); + q.awaitAll(function (err, datas) { + callback(datas.flat()); + }); + }); + }; +}; \ No newline at end of file