diff --git a/Dockerfile b/Dockerfile index b589105..499b819 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM gradle:4.7.0-jdk8-alpine AS build +FROM gradle:7.1-jdk11 AS build COPY --chown=gradle:gradle *.gradle /home/gradle/src/ COPY --chown=gradle:gradle ./external/ /home/gradle/src/external/ COPY --chown=gradle:gradle ./src/ /home/gradle/src/src/ @@ -8,7 +8,7 @@ RUN gradle allJars --no-daemon RUN ls -l /home/gradle/src/* RUN ls -l /home/gradle/src/build/libs/* -FROM openjdk:8-jre-slim +FROM openjdk:11-jre-slim RUN mkdir /app diff --git a/build-docker.sh b/build-docker.sh new file mode 100755 index 0000000..5e22e2c --- /dev/null +++ b/build-docker.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -e +set -x + +docker build --tag shst-builder . diff --git a/build-local.sh b/build-local.sh new file mode 100755 index 0000000..fbdf8c4 --- /dev/null +++ b/build-local.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +set -e +set -x + +/opt/homebrew/opt/gradle@7/bin/gradle allJars --stacktrace diff --git a/build.gradle b/build.gradle index 67e422c..b2c6e34 100644 --- a/build.gradle +++ b/build.gradle @@ -3,10 +3,13 @@ version '0.3.2' buildscript { - repositories { jcenter() - mavenCentral()} + repositories { + maven { url "https://plugins.gradle.org/m2/" } + mavenCentral() + } dependencies { - classpath 'com.github.jengelman.gradle.plugins:shadow:2.0.1' + //classpath 'com.github.jengelman.gradle.plugins:shadow:2.0.1' + classpath 'gradle.plugin.com.github.johnrengelman:shadow:7.1.1' //classpath 'com.google.protobuf:protobuf-gradle-plugin:0.8.3' } // Note PF : this plugin does not support resource transformation (needed for akka configuration files } @@ -23,34 +26,34 @@ import com.github.jengelman.gradle.plugins.shadow.tasks.* sourceCompatibility = 1.8 repositories { - maven { url "http://repository.apache.org" } - jcenter() + maven { url "https://repository.apache.org" } + mavenCentral() mavenLocal() } dependencies { - testCompile group: 'junit', name: 'junit', version: '4.11' + testImplementation group: 'junit', name: 'junit', version: '4.11' } -def flinkversion = "1.3.2" +def flinkversion = "1.3.3" def protobufversion = "3.4.0" dependencies { - compile 'org.slf4j:slf4j-api:2.0.3' - compile 'org.slf4j:slf4j-jdk14:2.0.3' - compile files('external/osmosis-osm-binary.jar') - compile 'com.esri.geometry:esri-geometry-api:1.2.1' - compile "org.apache.flink:flink-java:${flinkversion}" - compile "org.apache.flink:flink-clients_2.10:${flinkversion}" - compile "com.google.protobuf:protobuf-java:${protobufversion}" - compile "com.jsoniter:jsoniter:0.9.15" - compile 'net.sf.geographiclib:GeographicLib-Java:1.46' - compile 'commons-cli:commons-cli:1.4' + implementation 'org.slf4j:slf4j-api:2.0.3' + implementation 'org.slf4j:slf4j-jdk14:2.0.3' + implementation files('external/osmosis-osm-binary.jar') + implementation 'com.esri.geometry:esri-geometry-api:1.2.1' + implementation "org.apache.flink:flink-java:${flinkversion}" + implementation "org.apache.flink:flink-clients_2.10:${flinkversion}" + implementation "com.google.protobuf:protobuf-java:${protobufversion}" + implementation "com.jsoniter:jsoniter:0.9.15" + implementation 'net.sf.geographiclib:GeographicLib-Java:1.46' + implementation 'commons-cli:commons-cli:1.4' - testCompile "junit:junit:4.11" + testImplementation "junit:junit:4.11" } @@ -115,7 +118,7 @@ shadowJar { // and permit to use the soft from a command line task allJars(type: ShadowJar) { from sourceSets.main.output - configurations = [project.configurations.compile] + configurations = [project.configurations.runtimeClasspath] transform(com.github.jengelman.gradle.plugins.shadow.transformers.AppendingTransformer) { resource = 'reference.conf' } manifest { diff --git a/generate-tiles-docker.sh b/generate-tiles-docker.sh new file mode 100755 index 0000000..97ccaf1 --- /dev/null +++ b/generate-tiles-docker.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -e +set -x + +docker run --rm \ + -v $(pwd)/data:/data \ + -v $(pwd)/out:/out \ + shst-builder \ + 2G greater-london-latest.osm.pbf diff --git a/generate-tiles-local.sh b/generate-tiles-local.sh new file mode 100755 index 0000000..445b838 --- /dev/null +++ b/generate-tiles-local.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +set -e +set -x + +# Usage: ./generate-tiles-local.sh [osmPbf] [xmx] [javabin] [jarfile] +osmPbf=${1:-greater-london-latest.osm.pbf} +xmx=${2:-2G} +javabin=${3:-/opt/homebrew/opt/openjdk@11/bin/java} +jarfile=${4:-build/libs/sharedstreets-builder-0.3.2.jar} + + +pbfName=$(basename $osmPbf .osm.pbf) + +# Download OSM PBF +# wget https://download.geofabrik.de/europe/united-kingdom/england/greater-london-latest.osm.pbf -O data/greater-london-latest.osm.pbf + +${javabin} -Xmx${xmx} -jar ${jarfile} --input data/${osmPbf} --output out/shst/tiles/osm/latest/${pbfName} diff --git a/src/main/java/io/sharedstreets/tools/builder/ProcessPBF.java b/src/main/java/io/sharedstreets/tools/builder/ProcessPBF.java index 8a47a8d..7b5ba48 100644 --- a/src/main/java/io/sharedstreets/tools/builder/ProcessPBF.java +++ b/src/main/java/io/sharedstreets/tools/builder/ProcessPBF.java @@ -125,7 +125,7 @@ public static void main(String[] args) throws Exception { conf.setFloat(ConfigConstants.TASK_MANAGER_MEMORY_FRACTION_KEY, 0.5f); final ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment(conf); ExecutionConfig execConfig = env.getConfig(); - execConfig.setParallelism(54); + //execConfig.setParallelism(54); // load osm data from PBF input OSMDataStream dataStream = new OSMDataStream(inputFile, env);