From 58882f0b8e7af6640fa390effd8d412223a10580 Mon Sep 17 00:00:00 2001 From: Mikko Haavisto Date: Fri, 8 Dec 2023 11:22:03 +0200 Subject: [PATCH 1/4] gradle7 build + helper scripts --- build-local.sh | 3 +++ build.gradle | 41 ++++++++++++++++++++++------------------- generate-tiles-local.sh | 13 +++++++++++++ 3 files changed, 38 insertions(+), 19 deletions(-) create mode 100755 build-local.sh create mode 100755 generate-tiles-local.sh diff --git a/build-local.sh b/build-local.sh new file mode 100755 index 0000000..e53edcf --- /dev/null +++ b/build-local.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +/opt/homebrew/opt/gradle@7/bin/gradle allJars --stacktrace diff --git a/build.gradle b/build.gradle index 67e422c..b2c6e34 100644 --- a/build.gradle +++ b/build.gradle @@ -3,10 +3,13 @@ version '0.3.2' buildscript { - repositories { jcenter() - mavenCentral()} + repositories { + maven { url "https://plugins.gradle.org/m2/" } + mavenCentral() + } dependencies { - classpath 'com.github.jengelman.gradle.plugins:shadow:2.0.1' + //classpath 'com.github.jengelman.gradle.plugins:shadow:2.0.1' + classpath 'gradle.plugin.com.github.johnrengelman:shadow:7.1.1' //classpath 'com.google.protobuf:protobuf-gradle-plugin:0.8.3' } // Note PF : this plugin does not support resource transformation (needed for akka configuration files } @@ -23,34 +26,34 @@ import com.github.jengelman.gradle.plugins.shadow.tasks.* sourceCompatibility = 1.8 repositories { - maven { url "http://repository.apache.org" } - jcenter() + maven { url "https://repository.apache.org" } + mavenCentral() mavenLocal() } dependencies { - testCompile group: 'junit', name: 'junit', version: '4.11' + testImplementation group: 'junit', name: 'junit', version: '4.11' } -def flinkversion = "1.3.2" +def flinkversion = "1.3.3" def protobufversion = "3.4.0" dependencies { - compile 'org.slf4j:slf4j-api:2.0.3' - compile 'org.slf4j:slf4j-jdk14:2.0.3' - compile files('external/osmosis-osm-binary.jar') - compile 'com.esri.geometry:esri-geometry-api:1.2.1' - compile "org.apache.flink:flink-java:${flinkversion}" - compile "org.apache.flink:flink-clients_2.10:${flinkversion}" - compile "com.google.protobuf:protobuf-java:${protobufversion}" - compile "com.jsoniter:jsoniter:0.9.15" - compile 'net.sf.geographiclib:GeographicLib-Java:1.46' - compile 'commons-cli:commons-cli:1.4' + implementation 'org.slf4j:slf4j-api:2.0.3' + implementation 'org.slf4j:slf4j-jdk14:2.0.3' + implementation files('external/osmosis-osm-binary.jar') + implementation 'com.esri.geometry:esri-geometry-api:1.2.1' + implementation "org.apache.flink:flink-java:${flinkversion}" + implementation "org.apache.flink:flink-clients_2.10:${flinkversion}" + implementation "com.google.protobuf:protobuf-java:${protobufversion}" + implementation "com.jsoniter:jsoniter:0.9.15" + implementation 'net.sf.geographiclib:GeographicLib-Java:1.46' + implementation 'commons-cli:commons-cli:1.4' - testCompile "junit:junit:4.11" + testImplementation "junit:junit:4.11" } @@ -115,7 +118,7 @@ shadowJar { // and permit to use the soft from a command line task allJars(type: ShadowJar) { from sourceSets.main.output - configurations = [project.configurations.compile] + configurations = [project.configurations.runtimeClasspath] transform(com.github.jengelman.gradle.plugins.shadow.transformers.AppendingTransformer) { resource = 'reference.conf' } manifest { diff --git a/generate-tiles-local.sh b/generate-tiles-local.sh new file mode 100755 index 0000000..2bab79e --- /dev/null +++ b/generate-tiles-local.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +# Usage: ./generate-tiles-local.sh [tiles] [memory] [javabin] [jarfile] +tiles=${1:-greater-london-latest} +memory=${2:-2G} +javabin=${3:-/opt/homebrew/opt/openjdk@11/bin/java} +jarfile=${4:-build/libs/sharedstreets-builder-0.3.2.jar} + +# Download OSM PBF +# wget https://download.geofabrik.de/europe/united-kingdom/england/greater-london-latest.osm.pbf -O data/greater-london-latest.osm.pbf + +${javabin} -Xmx${memory} -jar ${jarfile} --input data/${tiles}.osm.pbf --output ${tiles} + From 21b6226b06a51505876033beefb4a6149b791891 Mon Sep 17 00:00:00 2001 From: Mikko Haavisto Date: Fri, 8 Dec 2023 11:29:12 +0200 Subject: [PATCH 2/4] comment out explicit parallelism --- src/main/java/io/sharedstreets/tools/builder/ProcessPBF.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/io/sharedstreets/tools/builder/ProcessPBF.java b/src/main/java/io/sharedstreets/tools/builder/ProcessPBF.java index 8a47a8d..7b5ba48 100644 --- a/src/main/java/io/sharedstreets/tools/builder/ProcessPBF.java +++ b/src/main/java/io/sharedstreets/tools/builder/ProcessPBF.java @@ -125,7 +125,7 @@ public static void main(String[] args) throws Exception { conf.setFloat(ConfigConstants.TASK_MANAGER_MEMORY_FRACTION_KEY, 0.5f); final ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment(conf); ExecutionConfig execConfig = env.getConfig(); - execConfig.setParallelism(54); + //execConfig.setParallelism(54); // load osm data from PBF input OSMDataStream dataStream = new OSMDataStream(inputFile, env); From 17a15a65504b740ffe0534b2fae490681739fa0b Mon Sep 17 00:00:00 2001 From: Mikko Haavisto Date: Fri, 8 Dec 2023 11:56:54 +0200 Subject: [PATCH 3/4] docker build for gradle7 --- Dockerfile | 4 ++-- build-docker.sh | 5 +++++ generate-tiles-docker.sh | 9 +++++++++ 3 files changed, 16 insertions(+), 2 deletions(-) create mode 100755 build-docker.sh create mode 100755 generate-tiles-docker.sh diff --git a/Dockerfile b/Dockerfile index b589105..499b819 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM gradle:4.7.0-jdk8-alpine AS build +FROM gradle:7.1-jdk11 AS build COPY --chown=gradle:gradle *.gradle /home/gradle/src/ COPY --chown=gradle:gradle ./external/ /home/gradle/src/external/ COPY --chown=gradle:gradle ./src/ /home/gradle/src/src/ @@ -8,7 +8,7 @@ RUN gradle allJars --no-daemon RUN ls -l /home/gradle/src/* RUN ls -l /home/gradle/src/build/libs/* -FROM openjdk:8-jre-slim +FROM openjdk:11-jre-slim RUN mkdir /app diff --git a/build-docker.sh b/build-docker.sh new file mode 100755 index 0000000..5e22e2c --- /dev/null +++ b/build-docker.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -e +set -x + +docker build --tag shst-builder . diff --git a/generate-tiles-docker.sh b/generate-tiles-docker.sh new file mode 100755 index 0000000..97ccaf1 --- /dev/null +++ b/generate-tiles-docker.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -e +set -x + +docker run --rm \ + -v $(pwd)/data:/data \ + -v $(pwd)/out:/out \ + shst-builder \ + 2G greater-london-latest.osm.pbf From ce6d6c8b101b550134a7dcba1b1501fc042b25d3 Mon Sep 17 00:00:00 2001 From: Mikko Haavisto Date: Fri, 8 Dec 2023 11:57:24 +0200 Subject: [PATCH 4/4] unify docker and local scripts --- build-local.sh | 3 +++ generate-tiles-local.sh | 15 ++++++++++----- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/build-local.sh b/build-local.sh index e53edcf..fbdf8c4 100755 --- a/build-local.sh +++ b/build-local.sh @@ -1,3 +1,6 @@ #!/usr/bin/env bash +set -e +set -x + /opt/homebrew/opt/gradle@7/bin/gradle allJars --stacktrace diff --git a/generate-tiles-local.sh b/generate-tiles-local.sh index 2bab79e..445b838 100755 --- a/generate-tiles-local.sh +++ b/generate-tiles-local.sh @@ -1,13 +1,18 @@ #!/usr/bin/env bash -# Usage: ./generate-tiles-local.sh [tiles] [memory] [javabin] [jarfile] -tiles=${1:-greater-london-latest} -memory=${2:-2G} +set -e +set -x + +# Usage: ./generate-tiles-local.sh [osmPbf] [xmx] [javabin] [jarfile] +osmPbf=${1:-greater-london-latest.osm.pbf} +xmx=${2:-2G} javabin=${3:-/opt/homebrew/opt/openjdk@11/bin/java} jarfile=${4:-build/libs/sharedstreets-builder-0.3.2.jar} + +pbfName=$(basename $osmPbf .osm.pbf) + # Download OSM PBF # wget https://download.geofabrik.de/europe/united-kingdom/england/greater-london-latest.osm.pbf -O data/greater-london-latest.osm.pbf -${javabin} -Xmx${memory} -jar ${jarfile} --input data/${tiles}.osm.pbf --output ${tiles} - +${javabin} -Xmx${xmx} -jar ${jarfile} --input data/${osmPbf} --output out/shst/tiles/osm/latest/${pbfName}