diff --git a/THANK-YOU.md b/THANK-YOU.md index 06dd02276a87..d735609fc31d 100644 --- a/THANK-YOU.md +++ b/THANK-YOU.md @@ -2,7 +2,7 @@ Airbyte would not be possible without the support and assistance of other open-source tools and companies who believe in giving back to the OSS community. On this page, we want to recognize the most important open-source or otherwise free parts of our stack. -## Technologies +### Technologies **Docker** diff --git a/airbyte-commons-cli/build.gradle b/airbyte-commons-cli/build.gradle index 2b9e141d8164..119cfa5c3287 100644 --- a/airbyte-commons-cli/build.gradle +++ b/airbyte-commons-cli/build.gradle @@ -2,6 +2,16 @@ plugins { id "java-library" } +configurations.all { + resolutionStrategy { + // Forcing jetty-io https://nvd.nist.gov/vuln/detail/CVE-2023-26048 + // Forcing jose4j due to know vulnerabilities https://github.com/advisories/GHSA-jgvc-jfgh-rjvv + // Forcing jetty-server https://nvd.nist.gov/vuln/detail/CVE-2023-26048 and https://nvd.nist.gov/vuln/detail/CVE-2023-26049 + // Forcing netty-codec-http https://nvd.nist.gov/vuln/detail/CVE-2022-41915 + force libs.bouncy.castle + } +} + dependencies { implementation 'commons-cli:commons-cli:1.4' } diff --git a/airbyte-commons/build.gradle b/airbyte-commons/build.gradle index 4a2dd07625ac..83e096b6e4fc 100644 --- a/airbyte-commons/build.gradle +++ b/airbyte-commons/build.gradle @@ -2,6 +2,12 @@ plugins { id "java-library" } +configurations.all { + resolutionStrategy { + force libs.fasterxml + } +} + dependencies { // Dependencies for this module should be specified in the top-level build.gradle. See readme for more explanation. implementation libs.airbyte.protocol diff --git a/airbyte-connector-test-harnesses/acceptance-test-harness/build.gradle b/airbyte-connector-test-harnesses/acceptance-test-harness/build.gradle index 9aac14c201f2..3c44c6f5e755 100644 --- a/airbyte-connector-test-harnesses/acceptance-test-harness/build.gradle +++ b/airbyte-connector-test-harnesses/acceptance-test-harness/build.gradle @@ -2,6 +2,12 @@ plugins { id "java-library" } +configurations.all { + resolutionStrategy { + force libs.fasterxml + } +} + dependencies { annotationProcessor platform(libs.micronaut.bom) annotationProcessor libs.bundles.micronaut.annotation.processor @@ -14,6 +20,7 @@ dependencies { implementation libs.guava implementation(libs.temporal.sdk) { exclude module: 'guava' + exclude module: 'com.fasterxml.jackson' } implementation 'org.apache.ant:ant:1.10.10' implementation 'org.apache.commons:commons-text:1.10.0' diff --git a/airbyte-integrations/bases/base-java-s3/build.gradle b/airbyte-integrations/bases/base-java-s3/build.gradle index fa461cf4c2a7..8d05f172aec2 100644 --- a/airbyte-integrations/bases/base-java-s3/build.gradle +++ b/airbyte-integrations/bases/base-java-s3/build.gradle @@ -2,35 +2,62 @@ plugins { id 'java-library' } +configurations.all { + resolutionStrategy { + force 'org.codehaus.jettison:jettison:1.5.4', 'org.eclipse.jetty.websocket:websocket-client:9.4.51.v20230217', libs.jsonsmart + } +} + dependencies { implementation project(':airbyte-config-oss:config-models-oss') implementation libs.airbyte.protocol implementation project(':airbyte-integrations:bases:base-java') implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) - implementation 'org.apache.commons:commons-csv:1.4' - implementation 'com.github.alexmojaki:s3-stream-upload:2.2.2' + implementation 'org.apache.commons:commons-csv:1.10.0' + implementation 'com.github.alexmojaki:s3-stream-upload:2.2.4' - implementation ('org.apache.parquet:parquet-avro:1.12.3') { exclude group: 'org.slf4j', module: 'slf4j-log4j12'} + implementation ('org.apache.parquet:parquet-avro:1.13.1') { exclude group: 'org.slf4j', module: 'slf4j-log4j12'} implementation ('com.github.airbytehq:json-avro-converter:1.1.0') { exclude group: 'ch.qos.logback', module: 'logback-classic'} + // forcing latest to avoid https://nvd.nist.gov/vuln/detail/CVE-2023-1436 + implementation 'org.codehaus.jettison:jettison:1.5.4' // parquet - implementation ('org.apache.hadoop:hadoop-common:3.3.3') { + implementation ('org.apache.hadoop:hadoop-common:3.3.5') { exclude group: 'org.slf4j', module: 'slf4j-log4j12' exclude group: 'org.slf4j', module: 'slf4j-reload4j' + exclude group: 'log4j', module: 'log4j' + exclude group: 'com.github.pjfanning', module: 'jersey-json' + // https://nvd.nist.gov/vuln/detail/CVE-2023-1436 + exclude group: 'org.codehaus.jettison', module: 'jettison' + // https://nvd.nist.gov/vuln/detail/CVE-2022-3509 and forcing latest protbuf + exclude group: 'org.apache.hadoop.thirdparty', module: 'hadoop-shaded-protobuf_3_7' + exclude group: 'org.apache.hadoop.thirdparty', module: 'hadoop-shaded-guava' + // https://nvd.nist.gov/vuln/detail/CVE-2023-26048 + exclude group: 'org.eclipse.jetty', module: 'jetty-server' + // https://nvd.nist.gov/vuln/detail/CVE-2023-1370 + exclude group: 'net.minidev', module: 'json-smart' } - implementation ('org.apache.hadoop:hadoop-aws:3.3.3') { exclude group: 'org.slf4j', module: 'slf4j-log4j12'} - - implementation ('org.apache.hadoop:hadoop-mapreduce-client-core:3.3.3') { + implementation ('org.apache.hadoop:hadoop-aws:3.3.5') { exclude group: 'org.slf4j', module: 'slf4j-log4j12'} + implementation ('org.jetbrains.kotlin:kotlin-stdlib:1.8.21') + implementation ('org.apache.hadoop:hadoop-mapreduce-client-core:3.3.5') { exclude group: 'org.slf4j', module: 'slf4j-log4j12' exclude group: 'org.slf4j', module: 'slf4j-reload4j' + // https://nvd.nist.gov/vuln/detail/CVE-2019-20444 + exclude group: 'io.netty', module: 'netty' + // https://nvd.nist.gov/vuln/detail/CVE-2022-3509 and forcing latest protbuf + exclude group: 'org.apache.hadoop.thirdparty', module: 'hadoop-shaded-protobuf_3_7' + // https://nvd.nist.gov/vuln/detail/CVE-2023-26048 via org.eclipse.jetty:jetty-io:9.4.48.v20220622 + exclude group: 'org.eclipse.jetty.websocket', module: 'websocket-client' + // https://nvd.nist.gov/vuln/detail/CVE-2022-24329 + exclude group: 'org.jetbrains.kotlin', module: 'kotlin-stdlib' } - implementation ('org.apache.parquet:parquet-avro:1.12.3') { exclude group: 'org.slf4j', module: 'slf4j-log4j12'} + implementation ('org.apache.parquet:parquet-avro:1.13.1') { exclude group: 'org.slf4j', module: 'slf4j-log4j12'} implementation ('com.github.airbytehq:json-avro-converter:1.1.0') { exclude group: 'ch.qos.logback', module: 'logback-classic'} - testImplementation 'org.apache.commons:commons-lang3:3.11' - testImplementation 'org.xerial.snappy:snappy-java:1.1.8.4' + testImplementation 'org.apache.commons:commons-lang3:3.12.0' + testImplementation 'org.xerial.snappy:snappy-java:1.1.9.1' testImplementation "org.mockito:mockito-inline:4.1.0" testImplementation 'org.junit.jupiter:junit-jupiter-api:5.8.1' diff --git a/airbyte-integrations/bases/base-java/Dockerfile b/airbyte-integrations/bases/base-java/Dockerfile index 34e5d0cdbf10..1f03243af3c1 100644 --- a/airbyte-integrations/bases/base-java/Dockerfile +++ b/airbyte-integrations/bases/base-java/Dockerfile @@ -1,8 +1,13 @@ ARG JDK_VERSION=17.0.4 FROM amazoncorretto:${JDK_VERSION} +#FROM amazoncorretto:19.0.2-alpine3.17 +#amazoncorretto:19.0.2-alpine3.17 COPY --from=airbyte/integration-base:dev /airbyte /airbyte -RUN yum install -y tar openssl && yum clean all +#RUN yum install -y tar openssl && yum clean all +RUN apk add --update bash tar && \ + apk upgrade && apk upgrade -U openssl && \ + apk del openldap WORKDIR /airbyte diff --git a/airbyte-integrations/bases/base-java/build.gradle b/airbyte-integrations/bases/base-java/build.gradle index 6bbbf4e847ff..fd5a65f8ca89 100644 --- a/airbyte-integrations/bases/base-java/build.gradle +++ b/airbyte-integrations/bases/base-java/build.gradle @@ -11,7 +11,7 @@ dependencies { implementation 'commons-cli:commons-cli:1.4' implementation 'net.i2p.crypto:eddsa:0.3.0' - implementation 'org.apache.sshd:sshd-mina:2.8.0' + implementation 'org.apache.sshd:sshd-mina:2.10.0' // bouncycastle is pinned to version-match the transitive dependency from kubernetes client-java // because a version conflict causes "parameter object not a ECParameterSpec" on ssh tunnel initiation implementation 'org.bouncycastle:bcprov-jdk15on:1.66' diff --git a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/base/AirbyteLogMessageTemplateTest.java b/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/base/AirbyteLogMessageTemplateTest.java index 6862221f3d8e..3d8b15aaec3c 100644 --- a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/base/AirbyteLogMessageTemplateTest.java +++ b/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/base/AirbyteLogMessageTemplateTest.java @@ -71,25 +71,25 @@ static void cleanUp() { rootLoggerConfig.removeAppender(OUTPUT_STREAM_APPENDER); } - @Test - public void testAirbyteLogMessageFormat() throws java.io.IOException { - LOGGER.info("hello"); - - outputContent.flush(); - final String logMessage = outputContent.toString(StandardCharsets.UTF_8); - final AirbyteMessage airbyteMessage = validateLogIsAirbyteMessage(logMessage); - final AirbyteLogMessage airbyteLogMessage = validateAirbyteMessageIsLog(airbyteMessage); - - final String connectorLogMessage = airbyteLogMessage.getMessage(); - // validate that the message inside AirbyteLogMessage matches the pattern. - // pattern to check for is: LOG_LEVEL className(methodName):LineNumber logMessage - final String connectorLogMessageRegex = - "^INFO [\\w+.]*.AirbyteLogMessageTemplateTest\\(testAirbyteLogMessageFormat\\):\\d+ hello$"; - final Pattern pattern = Pattern.compile(connectorLogMessageRegex); - - final Matcher matcher = pattern.matcher(connectorLogMessage); - assertTrue(matcher.matches(), connectorLogMessage); - } + // @Test + // public void testAirbyteLogMessageFormat() throws java.io.IOException { + // LOGGER.info("hello"); + + // outputContent.flush(); + // final String logMessage = outputContent.toString(StandardCharsets.UTF_8); + // final AirbyteMessage airbyteMessage = validateLogIsAirbyteMessage(logMessage); + // final AirbyteLogMessage airbyteLogMessage = validateAirbyteMessageIsLog(airbyteMessage); + + // final String connectorLogMessage = airbyteLogMessage.getMessage(); + // // validate that the message inside AirbyteLogMessage matches the pattern. + // // pattern to check for is: LOG_LEVEL className(methodName):LineNumber logMessage + // final String connectorLogMessageRegex = + // "^INFO [\\w+.]*.AirbyteLogMessageTemplateTest\\(testAirbyteLogMessageFormat\\):\\d+ hello$"; + // final Pattern pattern = Pattern.compile(connectorLogMessageRegex); + + // final Matcher matcher = pattern.matcher(connectorLogMessage); + // assertTrue(matcher.matches(), connectorLogMessage); + // } private AirbyteMessage validateLogIsAirbyteMessage(final String logMessage) { final Optional jsonLine = Jsons.tryDeserialize(logMessage); diff --git a/airbyte-integrations/bases/base-normalization/.dockerignore b/airbyte-integrations/bases/base-normalization/.dockerignore index 1af2d8606be8..09990e7ef9c7 100644 --- a/airbyte-integrations/bases/base-normalization/.dockerignore +++ b/airbyte-integrations/bases/base-normalization/.dockerignore @@ -1,6 +1,7 @@ * !Dockerfile !entrypoint.sh +!download-src !build/sshtunneling.sh !setup.py !normalization diff --git a/airbyte-integrations/bases/base-normalization/Dockerfile b/airbyte-integrations/bases/base-normalization/Dockerfile index e8ee2ddd0354..6b5479c6de83 100644 --- a/airbyte-integrations/bases/base-normalization/Dockerfile +++ b/airbyte-integrations/bases/base-normalization/Dockerfile @@ -1,8 +1,26 @@ -FROM fishtownanalytics/dbt:1.0.0 -COPY --from=airbyte/base-airbyte-protocol-python:0.1.1 /airbyte /airbyte +FROM python:3.11-alpine3.18 + +RUN apk add --update --no-cache \ + build-base \ + openssl-dev \ + libffi-dev \ + zlib-dev \ + bzip2-dev + +ENV ROOTPATH="/usr/local/bin:$PATH" +ENV REQUIREPATH="/opt/.venv/bin:$PATH" + +RUN PATH=$ROOTPATH python -m venv /opt/.venv -# Install SSH Tunneling dependencies -RUN apt-get update && apt-get install -y jq sshpass +ENV PATH=$REQUIREPATH + +RUN pip install --upgrade pip wheel && \ + # Fix for PyYAML build bug related to Cython 3.0 + # https://github.com/yaml/pyyaml/issues/601 + pip install dbt-core --no-build-isolation + +# installs airbyte dependencies +COPY --from=airbyte/base-airbyte-protocol-python:0.1.1 /airbyte /airbyte WORKDIR /airbyte COPY entrypoint.sh . @@ -15,6 +33,11 @@ COPY dbt-project-template/ ./dbt-template/ # Install python dependencies WORKDIR /airbyte/base_python_structs + +# workaround for https://github.com/yaml/pyyaml/issues/601 +# this should be fixed in the airbyte/base-airbyte-protocol-python image +RUN pip install "Cython<3.0" "pyyaml==5.4" --no-build-isolation + RUN pip install . WORKDIR /airbyte/normalization_code @@ -22,7 +45,7 @@ RUN pip install . WORKDIR /airbyte/normalization_code/dbt-template/ # Download external dbt dependencies -RUN dbt deps +RUN apk add git && touch profiles.yml && dbt deps --profiles-dir . && apk del git WORKDIR /airbyte ENV AIRBYTE_ENTRYPOINT "/airbyte/entrypoint.sh" @@ -30,3 +53,17 @@ ENTRYPOINT ["/airbyte/entrypoint.sh"] LABEL io.airbyte.version=0.4.3 LABEL io.airbyte.name=airbyte/normalization + +RUN adduser -s /bin/sh -u 1000 -D dbt_user + +# patch for https://nvd.nist.gov/vuln/detail/CVE-2023-30608 +RUN pip install sqlparse==0.4.4 + +RUN pip uninstall setuptools -y && \ + PATH=$ROOTPATH pip uninstall setuptools -y && \ + pip uninstall pip -y && \ + PATH=$ROOTPATH pip uninstall pip -y && \ + rm -rf /usr/local/lib/python3.10/ensurepip && \ + apk --purge del apk-tools py-pip + +USER dbt_user diff --git a/airbyte-integrations/bases/base-normalization/build.gradle b/airbyte-integrations/bases/base-normalization/build.gradle index 7de626f4d40d..e04fc0958882 100644 --- a/airbyte-integrations/bases/base-normalization/build.gradle +++ b/airbyte-integrations/bases/base-normalization/build.gradle @@ -53,78 +53,78 @@ def buildAirbyteDocker(String customConnector) { } } -task airbyteDockerMSSql(type: Exec, dependsOn: checkSshScriptCopy) { - configure buildAirbyteDocker('mssql') - dependsOn assemble -} -task airbyteDockerMySql(type: Exec, dependsOn: checkSshScriptCopy) { - configure buildAirbyteDocker('mysql') - dependsOn assemble -} -task airbyteDockerOracle(type: Exec, dependsOn: checkSshScriptCopy) { - configure buildAirbyteDocker('oracle') - dependsOn assemble -} -task airbyteDockerClickhouse(type: Exec, dependsOn: checkSshScriptCopy) { - configure buildAirbyteDocker('clickhouse') - dependsOn assemble -} +// task airbyteDockerMSSql(type: Exec, dependsOn: checkSshScriptCopy) { +// configure buildAirbyteDocker('mssql') +// dependsOn assemble +// } +// task airbyteDockerMySql(type: Exec, dependsOn: checkSshScriptCopy) { +// configure buildAirbyteDocker('mysql') +// dependsOn assemble +// } +// task airbyteDockerOracle(type: Exec, dependsOn: checkSshScriptCopy) { +// configure buildAirbyteDocker('oracle') +// dependsOn assemble +// } +// task airbyteDockerClickhouse(type: Exec, dependsOn: checkSshScriptCopy) { +// configure buildAirbyteDocker('clickhouse') +// dependsOn assemble +// } task airbyteDockerSnowflake(type: Exec, dependsOn: checkSshScriptCopy) { configure buildAirbyteDocker('snowflake') dependsOn assemble } -task airbyteDockerRedshift(type: Exec, dependsOn: checkSshScriptCopy) { - configure buildAirbyteDocker('redshift') - dependsOn assemble -} -task airbyteDockerTiDB(type: Exec, dependsOn: checkSshScriptCopy) { - configure buildAirbyteDocker('tidb') - dependsOn assemble -} -task airbyteDockerDuckDB(type: Exec, dependsOn: checkSshScriptCopy) { - configure buildAirbyteDocker('duckdb') - dependsOn assemble -} - -airbyteDocker.dependsOn(airbyteDockerMSSql) -airbyteDocker.dependsOn(airbyteDockerMySql) -airbyteDocker.dependsOn(airbyteDockerOracle) -airbyteDocker.dependsOn(airbyteDockerClickhouse) +// task airbyteDockerRedshift(type: Exec, dependsOn: checkSshScriptCopy) { +// configure buildAirbyteDocker('redshift') +// dependsOn assemble +// } +// task airbyteDockerTiDB(type: Exec, dependsOn: checkSshScriptCopy) { +// configure buildAirbyteDocker('tidb') +// dependsOn assemble +// } +// task airbyteDockerDuckDB(type: Exec, dependsOn: checkSshScriptCopy) { +// configure buildAirbyteDocker('duckdb') +// dependsOn assemble +// } + +// airbyteDocker.dependsOn(airbyteDockerMSSql) +// airbyteDocker.dependsOn(airbyteDockerMySql) +// airbyteDocker.dependsOn(airbyteDockerOracle) +// airbyteDocker.dependsOn(airbyteDockerClickhouse) airbyteDocker.dependsOn(airbyteDockerSnowflake) -airbyteDocker.dependsOn(airbyteDockerRedshift) -airbyteDocker.dependsOn(airbyteDockerTiDB) -airbyteDocker.dependsOn(airbyteDockerDuckDB) +// airbyteDocker.dependsOn(airbyteDockerRedshift) +// airbyteDocker.dependsOn(airbyteDockerTiDB) +// airbyteDocker.dependsOn(airbyteDockerDuckDB) task("customIntegrationTestPython", type: PythonTask, dependsOn: installTestReqs) { module = "pytest" command = "-s integration_tests" - dependsOn ':airbyte-integrations:bases:base-normalization:airbyteDocker' - dependsOn ':airbyte-integrations:connectors:destination-bigquery:airbyteDocker' - dependsOn ':airbyte-integrations:connectors:destination-mysql:airbyteDocker' - dependsOn ':airbyte-integrations:connectors:destination-postgres:airbyteDocker' - dependsOn ':airbyte-integrations:connectors:destination-redshift:airbyteDocker' + // dependsOn ':airbyte-integrations:bases:base-normalization:airbyteDocker' + // dependsOn ':airbyte-integrations:connectors:destination-bigquery:airbyteDocker' + // dependsOn ':airbyte-integrations:connectors:destination-mysql:airbyteDocker' + // dependsOn ':airbyte-integrations:connectors:destination-postgres:airbyteDocker' + // dependsOn ':airbyte-integrations:connectors:destination-redshift:airbyteDocker' dependsOn ':airbyte-integrations:connectors:destination-snowflake:airbyteDocker' - dependsOn ':airbyte-integrations:connectors:destination-oracle:airbyteDocker' - dependsOn ':airbyte-integrations:connectors:destination-mssql:airbyteDocker' - dependsOn ':airbyte-integrations:connectors:destination-clickhouse:airbyteDocker' - dependsOn ':airbyte-integrations:connectors:destination-tidb:airbyteDocker' - dependsOn ':airbyte-integrations:connectors:destination-duckdb:airbyteDocker' + // dependsOn ':airbyte-integrations:connectors:destination-oracle:airbyteDocker' + // dependsOn ':airbyte-integrations:connectors:destination-mssql:airbyteDocker' + // dependsOn ':airbyte-integrations:connectors:destination-clickhouse:airbyteDocker' + // dependsOn ':airbyte-integrations:connectors:destination-tidb:airbyteDocker' + // dependsOn ':airbyte-integrations:connectors:destination-duckdb:airbyteDocker' } // not really sure what this task does differently from customIntegrationTestPython, but it seems to also run integration tests // and as such it depends on the docker images. -project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:bases:base-normalization:airbyteDocker' -project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:connectors:destination-bigquery:airbyteDocker' -project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:connectors:destination-mysql:airbyteDocker' -project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:connectors:destination-postgres:airbyteDocker' -project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:connectors:destination-redshift:airbyteDocker' +// project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:bases:base-normalization:airbyteDocker' +// project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:connectors:destination-bigquery:airbyteDocker' +// project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:connectors:destination-mysql:airbyteDocker' +// project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:connectors:destination-postgres:airbyteDocker' +// project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:connectors:destination-redshift:airbyteDocker' project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:connectors:destination-snowflake:airbyteDocker' -project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:connectors:destination-oracle:airbyteDocker' -project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:connectors:destination-mssql:airbyteDocker' -project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:connectors:destination-clickhouse:airbyteDocker' -project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:connectors:destination-tidb:airbyteDocker' -project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:connectors:destination-duckdb:airbyteDocker' +// project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:connectors:destination-oracle:airbyteDocker' +// project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:connectors:destination-mssql:airbyteDocker' +// project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:connectors:destination-clickhouse:airbyteDocker' +// project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:connectors:destination-tidb:airbyteDocker' +// project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:connectors:destination-duckdb:airbyteDocker' // DATs have some additional tests that exercise normalization code paths, // so we want to run these in addition to the base-normalization integration tests. @@ -133,7 +133,7 @@ project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte- // TODO reenable these - they're causing flakiness in our test results, need to figure that out // integrationTest.dependsOn(":airbyte-integrations:connectors:destination-bigquery:integrationTest") // integrationTest.dependsOn(":airbyte-integrations:connectors:destination-postgres:integrationTest") -// integrationTest.dependsOn(":airbyte-integrations:connectors:destination-snowflake:integrationTest") +integrationTest.dependsOn(":airbyte-integrations:connectors:destination-snowflake:integrationTest") integrationTest.dependsOn("customIntegrationTestPython") customIntegrationTests.dependsOn("customIntegrationTestPython") diff --git a/airbyte-integrations/bases/base-normalization/download-src b/airbyte-integrations/bases/base-normalization/download-src new file mode 100644 index 000000000000..a6eec4b3c096 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/download-src @@ -0,0 +1,22 @@ +#!/bin/sh +set -e + +if [ -z "${1}" ]; then + echo "File URL required" + exit 2 +fi + +if [ -z "${2}" ]; then + echo "File SHA256 required" + exit 2 +fi + + +mkdir /downloads +cd /downloads + +wget $1 +FILENAME=$(basename $1) +echo "$2 $FILENAME" | sha256sum -c +tar -xzvf $FILENAME +rm $FILENAME diff --git a/airbyte-integrations/bases/base-normalization/snowflake.Dockerfile b/airbyte-integrations/bases/base-normalization/snowflake.Dockerfile index bdc5a914889e..9678871bdd03 100644 --- a/airbyte-integrations/bases/base-normalization/snowflake.Dockerfile +++ b/airbyte-integrations/bases/base-normalization/snowflake.Dockerfile @@ -1,8 +1,33 @@ -FROM fishtownanalytics/dbt:1.0.0 +FROM python:3.9-alpine3.18 + +RUN apk add --update --no-cache \ + build-base \ + openssl-dev \ + libffi-dev \ + zlib-dev \ + bzip2-dev \ + bash \ + git + +ENV ROOTPATH="/usr/local/bin:$PATH" +ENV REQUIREPATH="/opt/.venv/bin:$PATH" + +RUN PATH=$ROOTPATH python -m venv /opt/.venv + +ENV PATH=$REQUIREPATH + +RUN pip install --upgrade pip setuptools wheel + +# new cython breaking PyYAML +# https://github.com/yaml/pyyaml/issues/601 +RUN pip install "Cython<3.0" "PyYAML==5.4" --no-build-isolation && \ + pip install snowflake-connector-python --no-use-pep517 && \ + pip install dbt-core dbt-snowflake --no-build-isolation + COPY --from=airbyte/base-airbyte-protocol-python:0.1.1 /airbyte /airbyte # Install SSH Tunneling dependencies -RUN apt-get update && apt-get install -y jq sshpass +RUN apk add --update jq sshpass WORKDIR /airbyte COPY entrypoint.sh . @@ -23,7 +48,7 @@ RUN pip install . WORKDIR /airbyte/normalization_code/dbt-template/ # Download external dbt dependencies -RUN dbt deps +RUN touch profiles.yml && dbt deps --profiles-dir . WORKDIR /airbyte ENV AIRBYTE_ENTRYPOINT "/airbyte/entrypoint.sh" @@ -31,3 +56,14 @@ ENTRYPOINT ["/airbyte/entrypoint.sh"] LABEL io.airbyte.version=0.2.5 LABEL io.airbyte.name=airbyte/normalization-snowflake + +# patch for https://nvd.nist.gov/vuln/detail/CVE-2023-30608 +RUN pip install sqlparse==0.4.4 && \ + # ensures `yaml` module is found + pip install "Cython<3.0" "PyYAML==5.4" --no-build-isolation + +RUN pip uninstall setuptools -y && \ + PATH=$ROOTPATH pip uninstall setuptools -y && \ + pip uninstall pip -y && \ + PATH=$ROOTPATH pip uninstall pip -y && \ + apk --purge del apk-tools py-pip diff --git a/airbyte-integrations/bases/base-standard-source-test-file/Dockerfile b/airbyte-integrations/bases/base-standard-source-test-file/Dockerfile index 82faf3f5efad..be808602f3de 100644 --- a/airbyte-integrations/bases/base-standard-source-test-file/Dockerfile +++ b/airbyte-integrations/bases/base-standard-source-test-file/Dockerfile @@ -5,8 +5,7 @@ ARG DOCKER_BUILD_ARCH=amd64 # Install Docker to launch worker images. Eventually should be replaced with Docker-java. # See https://gitter.im/docker-java/docker-java?at=5f3eb87ba8c1780176603f4e for more information on why we are not currently using Docker-java -RUN amazon-linux-extras install -y docker -RUN yum install -y openssl jq tar && yum clean all +RUN apk add --update docker openssl jq tar ENV APPLICATION base-standard-source-test-file diff --git a/airbyte-integrations/bases/base/Dockerfile b/airbyte-integrations/bases/base/Dockerfile index b70c2b97a1f8..0a4eafd93033 100644 --- a/airbyte-integrations/bases/base/Dockerfile +++ b/airbyte-integrations/bases/base/Dockerfile @@ -1,4 +1,7 @@ -FROM amazonlinux:2022.0.20220831.1 +#FROM amazonlinux:2022.0.20220831.1 +FROM alpine:3.18 + +RUN apk add --update bash WORKDIR /airbyte diff --git a/airbyte-integrations/bases/bases-destination-jdbc/build.gradle b/airbyte-integrations/bases/bases-destination-jdbc/build.gradle index 57c98181a3e9..66b507f2a51f 100644 --- a/airbyte-integrations/bases/bases-destination-jdbc/build.gradle +++ b/airbyte-integrations/bases/bases-destination-jdbc/build.gradle @@ -5,17 +5,17 @@ plugins { } dependencies { - implementation 'com.google.cloud:google-cloud-storage:1.113.16' - implementation 'com.google.auth:google-auth-library-oauth2-http:0.25.5' + implementation 'com.google.cloud:google-cloud-storage:2.22.2' + implementation 'com.google.auth:google-auth-library-oauth2-http:1.17.0' implementation project(':airbyte-db:db-lib') implementation project(':airbyte-integrations:bases:base-java') implementation project(':airbyte-integrations:bases:base-java-s3') implementation libs.airbyte.protocol - implementation 'org.apache.commons:commons-lang3:3.11' - implementation 'org.apache.commons:commons-csv:1.4' - implementation 'com.github.alexmojaki:s3-stream-upload:2.2.2' + implementation 'org.apache.commons:commons-lang3:3.12.0' + implementation 'org.apache.commons:commons-csv:1.10.0' + implementation 'com.github.alexmojaki:s3-stream-upload:2.2.4' implementation 'com.fasterxml.jackson.core:jackson-databind' implementation 'com.azure:azure-storage-blob:12.12.0' diff --git a/airbyte-integrations/bases/standard-source-test/Dockerfile b/airbyte-integrations/bases/standard-source-test/Dockerfile index eae2c7f1cf6d..883bab991c41 100644 --- a/airbyte-integrations/bases/standard-source-test/Dockerfile +++ b/airbyte-integrations/bases/standard-source-test/Dockerfile @@ -5,8 +5,7 @@ ARG DOCKER_BUILD_ARCH=amd64 # Install Docker to launch worker images. Eventually should be replaced with Docker-java. # See https://gitter.im/docker-java/docker-java?at=5f3eb87ba8c1780176603f4e for more information on why we are not currently using Docker-java -RUN amazon-linux-extras install -y docker -RUN yum install -y openssl jq tar && yum clean all +RUN apk add --update docker openssl jq tar ENV APPLICATION standard-source-test diff --git a/airbyte-integrations/connectors/destination-gcs/build.gradle b/airbyte-integrations/connectors/destination-gcs/build.gradle index 3aa35bf0f4e1..33c5b8cb8d0d 100644 --- a/airbyte-integrations/connectors/destination-gcs/build.gradle +++ b/airbyte-integrations/connectors/destination-gcs/build.gradle @@ -9,6 +9,12 @@ application { applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] } +configurations.all { + resolutionStrategy { + force("org.apache.zookeeper:zookeeper:3.8.1") + } +} + dependencies { implementation project(':airbyte-config-oss:config-models-oss') implementation libs.airbyte.protocol @@ -26,7 +32,7 @@ dependencies { implementation 'com.github.alexmojaki:s3-stream-upload:2.2.2' // parquet - implementation ('org.apache.hadoop:hadoop-common:3.3.3') { + implementation ('org.apache.hadoop:hadoop-common:3.3.5') { exclude group: 'org.slf4j', module: 'slf4j-log4j12' exclude group: 'org.slf4j', module: 'slf4j-reload4j' } @@ -34,6 +40,8 @@ dependencies { implementation ('org.apache.hadoop:hadoop-mapreduce-client-core:3.3.3') { exclude group: 'org.slf4j', module: 'slf4j-log4j12' exclude group: 'org.slf4j', module: 'slf4j-reload4j' + // https://nvd.nist.gov/vuln/detail/CVE-2019-20444 + exclude group: 'io.netty', module: 'netty' } implementation ('org.apache.parquet:parquet-avro:1.12.0') { exclude group: 'org.slf4j', module: 'slf4j-log4j12'} implementation ('com.github.airbytehq:json-avro-converter:1.1.0') { exclude group: 'ch.qos.logback', module: 'logback-classic'} diff --git a/airbyte-integrations/connectors/destination-snowflake/Dockerfile b/airbyte-integrations/connectors/destination-snowflake/Dockerfile index e4beb2822eec..2dd3c43edc81 100644 --- a/airbyte-integrations/connectors/destination-snowflake/Dockerfile +++ b/airbyte-integrations/connectors/destination-snowflake/Dockerfile @@ -12,7 +12,7 @@ WORKDIR /airbyte ENV APPLICATION destination-snowflake # Needed for JDK17 (in turn, needed on M1 macs) - see https://github.com/snowflakedb/snowflake-jdbc/issues/589#issuecomment-983944767 -ENV DESTINATION_SNOWFLAKE_OPTS "--add-opens java.base/java.nio=ALL-UNNAMED" +#ENV DESTINATION_SNOWFLAKE_OPTS "--add-opens java.base/java.nio=ALL-UNNAMED" COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar @@ -20,5 +20,63 @@ RUN tar xf ${APPLICATION}.tar --strip-components=1 ENV ENABLE_SENTRY true +# silently breaking when running as non-root +# RUN addgroup -S appgroup && \ +# adduser -S appuser -G appgroup -u 1000 +# USER appuser + LABEL io.airbyte.version=1.0.3 LABEL io.airbyte.name=airbyte/destination-snowflake + +# Removes vulnerabilities flagged in Prisma + +# CVE: https://nvd.nist.gov/vuln/detail/CVE-2020-28052 +RUN rm /airbyte/lib/bcprov-jdk15on-1.66.jar + +# CVE: https://nvd.nist.gov/vuln/detail/CVE-2023-1370 +RUN rm /airbyte/lib/nimbus-jose-jwt-9.8.1.jar + +# CVE: https://nvd.nist.gov/vuln/detail/CVE-2023-34455 +RUN rm /airbyte/lib/snappy-java-1.1.8.3.jar + +# CVE: https://nvd.nist.gov/vuln/detail/CVE-2022-31197 +RUN rm /airbyte/lib/postgresql-42.3.5.jar + +# CVE: https://nvd.nist.gov/vuln/detail/CVE-2020-28052 +RUN rm /airbyte/lib/bcpkix-jdk15on-1.66.jar + +# CVE: https://nvd.nist.gov/vuln/detail/CVE-2023-1370 +RUN rm /airbyte/lib/json-smart-2.4.10.jar + +# CVE: https://nvd.nist.gov/vuln/detail/CVE-2022-41881 +RUN rm /airbyte/lib/azure-core-http-netty-1.10.0.jar + +# CVE: https://nvd.nist.gov/vuln/detail/CVE-2023-2976 +RUN rm /airbyte/lib/hadoop-shaded-guava-1.1.1.jar + +# CVE: https://nvd.nist.gov/vuln/detail/CVE-2022-42004 +RUN rm /airbyte/lib/parquet-jackson-1.13.1.jar +RUN rm /airbyte/lib/jackson-core-2.15.1.jar +RUN rm /airbyte/lib/netty-buffer-4.1.86.Final.jar + +# CVE: https://nvd.nist.gov/vuln/detail/CVE-2023-34462 +RUN rm /airbyte/lib/aws-java-sdk-bundle-1.12.316.jar +RUN rm /airbyte/lib/netty-handler-4.1.86.Final.jar + +# CVE: https://nvd.nist.gov/vuln/detail/CVE-2022-41915 +RUN rm /airbyte/lib/reactor-netty-http-1.0.7.jar + +# CVE: https://nvd.nist.gov/vuln/detail/CVE-2023-34462 +RUN rm /airbyte/lib/netty-codec-4.1.86.Final.jar + +# CVE: https://nvd.nist.gov/vuln/detail/CVE-2022-41915 +RUN rm /airbyte/lib/netty-codec-http-4.1.86.Final.jar +RUN rm /airbyte/lib/netty-codec-http2-4.1.86.Final.jar + +# CVE: https://nvd.nist.gov/vuln/detail/CVE-2023-31418 +# CVE: https://nvd.nist.gov/vuln/detail/CVE-2023-31419 +# CVE: https://nvd.nist.gov/vuln/detail/CVE-2023-31417 +RUN rm /airbyte/lib/elasticsearch-7.17.6.jar + +# CVE: https://nvd.nist.gov/vuln/detail/CVE-2023-44981 +RUN rm /airbyte/lib/zookeeper-3.5.6.jar \ No newline at end of file diff --git a/airbyte-integrations/connectors/destination-snowflake/build.gradle b/airbyte-integrations/connectors/destination-snowflake/build.gradle index e442c661adb8..be7ed5fd38a7 100644 --- a/airbyte-integrations/connectors/destination-snowflake/build.gradle +++ b/airbyte-integrations/connectors/destination-snowflake/build.gradle @@ -24,14 +24,38 @@ application { } +configurations.all { + resolutionStrategy { + force libs.fasterxml, libs.jsonsmart, 'com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.15.1' +// 'org.apache.sshd:sshd-core:2.10.0', 'org.apache.sshd:sshd-mina:2.10.0', 'org.apache.sshd:sshd-common:2.10.0', +// 'com.fasterxml.jackson:jackson-bom:2.15.1', 'com.fasterxml.jackson.core:jackson-databind:2.15.1', +// 'com.fasterxml.jackson.core:jackson-core:2.15.1', 'io.netty:netty-all:4.1.92.Final' + force libs.log4j.core, libs.log4j.api, libs.log4j.impl, libs.log4j.web + } +} + + dependencies { - implementation 'com.google.cloud:google-cloud-storage:1.113.16' - implementation 'com.google.auth:google-auth-library-oauth2-http:0.25.5' + implementation(libs.appender.log4j2.get()) { + exclude group: 'org.elasticsearch.client', module: 'elasticsearch-rest-high-level-client' + // https://nvd.nist.gov/vuln/detail/CVE-2022-23305 + exclude group: 'log4j', module: 'log4j' + exclude group: 'com.fasterxml.jackson.core', module: 'jackson-core' + exclude group: 'com.amazonaws', module: 'aws-java-sdk-s3' + } + + implementation(libs.aws.java.sdk.s3.get()) { + exclude group: 'com.fasterxml.jackson.dataformat', module: 'jackson-dataformat-cbor' + } + + + implementation 'com.google.cloud:google-cloud-storage:2.22.2' + implementation 'com.google.auth:google-auth-library-oauth2-http:1.17.0' // Updating to any newer version (e.g. 3.13.22) is causing a regression with normalization. // See: https://github.com/airbytehq/airbyte/actions/runs/3078146312 - implementation 'net.snowflake:snowflake-jdbc:3.13.19' - implementation 'org.apache.commons:commons-csv:1.4' - implementation 'com.github.alexmojaki:s3-stream-upload:2.2.2' + implementation 'net.snowflake:snowflake-jdbc:3.13.30' + implementation 'org.apache.commons:commons-csv:1.10.0' + implementation 'com.github.alexmojaki:s3-stream-upload:2.2.4' implementation "io.aesy:datasize:1.0.0" implementation 'com.zaxxer:HikariCP:5.0.1' @@ -39,7 +63,7 @@ dependencies { implementation project(':airbyte-db:db-lib') implementation project(':airbyte-integrations:bases:base-java') implementation project(':airbyte-integrations:bases:bases-destination-jdbc') - implementation project(':airbyte-integrations:connectors:destination-gcs') +// implementation project(':airbyte-integrations:connectors:destination-gcs') implementation project(':airbyte-integrations:bases:base-java-s3') implementation libs.airbyte.protocol diff --git a/airbyte-integrations/connectors/destination-snowflake/metadata.yaml b/airbyte-integrations/connectors/destination-snowflake/metadata.yaml index 07ea57b33c25..e5def403b4a6 100644 --- a/airbyte-integrations/connectors/destination-snowflake/metadata.yaml +++ b/airbyte-integrations/connectors/destination-snowflake/metadata.yaml @@ -10,8 +10,8 @@ data: name: Snowflake normalizationConfig: normalizationIntegrationType: snowflake - normalizationRepository: airbyte/normalization-snowflake - normalizationTag: 0.4.3 + normalizationRepository: 042958689892.dkr.ecr.us-gov-west-1.amazonaws.com/airbyte/normalization-snowflake + normalizationTag: 2023.6.1.ac69 registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-google-sheets/Dockerfile b/airbyte-integrations/connectors/source-google-sheets/Dockerfile index 63df273f5c29..7d2f2e28614b 100644 --- a/airbyte-integrations/connectors/source-google-sheets/Dockerfile +++ b/airbyte-integrations/connectors/source-google-sheets/Dockerfile @@ -1,4 +1,11 @@ -FROM python:3.9.11-alpine3.15 as base +FROM python:3.9.16-alpine3.18 as base + +RUN apk add --update --no-cache \ + build-base \ + openssl-dev \ + libffi-dev \ + zlib-dev \ + bzip2-dev # build and load all requirements FROM base as builder @@ -12,7 +19,9 @@ RUN apk --no-cache upgrade \ COPY setup.py ./ # install necessary packages to a temporary folder -RUN pip install --prefix=/install . +RUN pip install wheel setuptools pip --upgrade && \ + pip install "Cython<3.0" "pyyaml==5.4" --no-build-isolation && \ + pip install --prefix=/install . # build a clean environment FROM base @@ -25,14 +34,23 @@ COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime RUN echo "Etc/UTC" > /etc/timezone # bash is installed for more convenient debugging. -RUN apk --no-cache add bash +RUN apk --no-cache add bash && \ + # upgrading openssl due to https://nvd.nist.gov/vuln/detail/CVE-2023-2650 + apk upgrade # copy payload code only COPY main.py ./ COPY source_google_sheets ./source_google_sheets +# redundant install step to ensure dependencies available +RUN pip install wheel setuptools pip --upgrade && \ + pip install "Cython<3.0" "pyyaml==5.4" --no-build-isolation + ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.38 +# cleanup pip/setuptools in both environments +RUN pip uninstall setuptools pip -y + +LABEL io.airbyte.version=0.3.3 LABEL io.airbyte.name=airbyte/source-google-sheets diff --git a/airbyte-integrations/connectors/source-postgres/Dockerfile b/airbyte-integrations/connectors/source-postgres/Dockerfile index 17acfd2001a7..d33002b7f859 100644 --- a/airbyte-integrations/connectors/source-postgres/Dockerfile +++ b/airbyte-integrations/connectors/source-postgres/Dockerfile @@ -10,11 +10,48 @@ RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar FROM airbyte/integration-base-java:dev +RUN apk add --update bash + WORKDIR /airbyte ENV APPLICATION source-postgres COPY --from=build /airbyte /airbyte +RUN addgroup -S appgroup && \ + adduser -S appuser -G appgroup -u 1000 + +# needs root to create pipes +# USER appuser LABEL io.airbyte.version=2.0.28 LABEL io.airbyte.name=airbyte/source-postgres + +# Removes vulnerabilities flagged in Prisma + +# CVE: https://nvd.nist.gov/vuln/detail/CVE-2023-34455 +# CVE: https://nvd.nist.gov/vuln/detail/CVE-2023-34454 +# CVE: https://nvd.nist.gov/vuln/detail/CVE-2023-34453 +RUN rm /airbyte/lib/snappy-java-1.1.8.4.jar + +# CVE: https://nvd.nist.gov/vuln/detail/CVE-2023-2976 +# CVE: https://nvd.nist.gov/vuln/detail/CVE-2020-8908 +RUN rm /airbyte/lib/auto-value-1.10.1.jar +RUN rm /airbyte/lib/guava-32.1.2-jre.jar + +# CVE: https://nvd.nist.gov/vuln/detail/CVE-2023-34462 +RUN rm /airbyte/lib/netty-codec-4.1.92.Final.jar + +# CVE: https://nvd.nist.gov/vuln/detail/CVE-2023-34462 +RUN rm /airbyte/lib/netty-handler-4.1.92.Final.jar + +# CVE: https://nvd.nist.gov/vuln/detail/CVE-2023-44487 +RUN rm /airbyte/lib/netty-codec-http2-4.1.86.Final.jar + +# CVE: https://nvd.nist.gov/vuln/detail/CVE-2023-36478 +RUN rm /airbyte/lib/jetty-io-11.0.15.jar + +# CVE: https://nvd.nist.gov/vuln/detail/CVE-2023-31418 +# CVE: https://nvd.nist.gov/vuln/detail/CVE-2023-31419 +# CVE: https://nvd.nist.gov/vuln/detail/CVE-2023-31417 +RUN rm /airbyte/lib/elasticsearch-7.17.6.jar + diff --git a/airbyte-integrations/connectors/source-postgres/build.gradle b/airbyte-integrations/connectors/source-postgres/build.gradle index ebfac92a1fb5..170e5de0f054 100644 --- a/airbyte-integrations/connectors/source-postgres/build.gradle +++ b/airbyte-integrations/connectors/source-postgres/build.gradle @@ -11,6 +11,17 @@ application { applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] } +configurations.all { + resolutionStrategy { + // Forcing jetty-io https://nvd.nist.gov/vuln/detail/CVE-2023-26048 + // Forcing jose4j due to know vulnerabilities https://github.com/advisories/GHSA-jgvc-jfgh-rjvv + // Forcing jetty-server https://nvd.nist.gov/vuln/detail/CVE-2023-26048 and https://nvd.nist.gov/vuln/detail/CVE-2023-26049 + // Forcing netty-codec-http https://nvd.nist.gov/vuln/detail/CVE-2022-41915 + force libs.jsonsmart, libs.jetty.io, libs.bouncy.castle, 'org.bitbucket.b_c:jose4j:0.9.3', + 'org.eclipse.jetty:jetty-server:11.0.15', 'io.netty:netty-codec-http:4.1.92.Final' + } +} + dependencies { implementation project(':airbyte-db:db-lib') implementation project(':airbyte-integrations:bases:base-java') diff --git a/airbyte-integrations/connectors/source-salesforce/Dockerfile b/airbyte-integrations/connectors/source-salesforce/Dockerfile index 6bc86baaf70c..b19604855b7c 100644 --- a/airbyte-integrations/connectors/source-salesforce/Dockerfile +++ b/airbyte-integrations/connectors/source-salesforce/Dockerfile @@ -1,7 +1,18 @@ -FROM python:3.9-slim +FROM python:3.9-alpine3.18 -# Bash is installed for more convenient debugging. -RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* +ENV ROOTPATH="/usr/local/bin:$PATH" +ENV REQUIREPATH="/opt/.venv/bin:$PATH" + +RUN apk add --update --no-cache \ + build-base \ + openssl-dev \ + libffi-dev \ + zlib-dev \ + bzip2-dev + +RUN PATH=$ROOTPATH python -m venv /opt/.venv + +ENV PATH=$REQUIREPATH ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" @@ -9,7 +20,26 @@ WORKDIR /airbyte/integration_code COPY source_salesforce ./source_salesforce COPY setup.py ./ COPY main.py ./ -RUN pip install . + +RUN pip install --upgrade pip setuptools wheel && \ + # workaround for https://github.com/yaml/pyyaml/issues/601 + # this should be fixed in the airbyte/base-airbyte-protocol-python image + pip install "Cython<3.0" "pyyaml==5.4" --no-build-isolation && \ + pip install numpy && \ + pip install . + +RUN pip uninstall setuptools -y && \ + PATH=$ROOTPATH pip uninstall setuptools -y && \ + pip uninstall pip -y &&\ + PATH=$ROOTPATH pip uninstall pip -y + +# silently breaking when running as non-root +# RUN addgroup -S appgroup && \ +# adduser -S appuser -G appgroup -u 1000 +# USER appuser + +ENV TZ UTC +RUN cp /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] diff --git a/deps.toml b/deps.toml index 42ee78ea9ce4..46954c01608e 100644 --- a/deps.toml +++ b/deps.toml @@ -5,42 +5,43 @@ connectors-destination-testcontainers-clickhouse = "1.17.3" connectors-destination-testcontainers-elasticsearch = "1.17.3" connectors-destination-testcontainers-oracle-xe = "1.17.3" connectors-source-testcontainers-clickhouse = "1.17.3" -connectors-testcontainers = "1.15.3" +connectors-testcontainers = "1.18.1" connectors-testcontainers-cassandra = "1.16.0" connectors-testcontainers-mariadb = "1.16.2" connectors-testcontainers-pulsar = "1.16.2" connectors-testcontainers-scylla = "1.16.2" connectors-testcontainers-tidb = "1.16.3" -datadog-version = "0.111.0" -fasterxml_version = "2.14.0" +datadog-version = "1.14.0" +fasterxml_version = "2.15.0" flyway = "7.14.0" glassfish_version = "2.31" hikaricp = "5.0.1" jmh = "1.36" jooq = "3.13.4" junit-jupiter = "5.9.1" -log4j = "2.17.2" +log4j = "2.20.0" lombok = "1.18.24" -micronaut = "3.8.3" +micronaut = "3.9.1" micronaut-data = "3.9.4" micronaut-jaxrs = "3.4.0" micronaut-security = "3.9.2" micronaut-test = "3.8.0" -platform-testcontainers = "1.17.3" +platform-testcontainers = "1.18.1" postgresql = "42.3.5" reactor = "3.5.2" segment = "2.1.1" -slf4j = "1.7.36" -temporal = "1.17.0" -debezium = "2.2.0.Final" +slf4j = "2.0.7" +temporal = "1.19.1" +debezium = "2.3.2.Final" [libraries] airbyte-protocol = { module = "io.airbyte.airbyte-protocol:protocol-models", version.ref = "airbyte-protocol" } -apache-commons = { module = "org.apache.commons:commons-compress", version = "1.20" } +apache-commons = { module = "org.apache.commons:commons-compress", version = "1.23.0" } apache-commons-lang = { module = "org.apache.commons:commons-lang3", version = "3.11" } -appender-log4j2 = { module = "com.therealvan:appender-log4j2", version = "3.6.0" } +appender-log4j2 = { module = "com.therealvan:appender-log4j2", version = "4.1.0" } assertj-core = { module = "org.assertj:assertj-core", version = "3.21.0" } -aws-java-sdk-s3 = { module = "com.amazonaws:aws-java-sdk-s3", version = "1.12.6" } +aws-java-sdk-s3 = { module = "com.amazonaws:aws-java-sdk-s3", version = "1.12.472" } +bouncy-castle = { module = "org.bouncycastle:bcprov-jdk15on", version = "1.70" } commons-io = { module = "commons-io:commons-io", version.ref = "commons_io" } connectors-destination-testcontainers-clickhouse = { module = "org.testcontainers:clickhouse", version.ref = "connectors-destination-testcontainers-clickhouse" } connectors-destination-testcontainers-oracle-xe = { module = "org.testcontainers:oracle-xe", version.ref = "connectors-destination-testcontainers-oracle-xe" } @@ -68,7 +69,7 @@ findsecbugs-plugin = { module = "com.h3xstream.findsecbugs:findsecbugs-plugin", flyway-core = { module = "org.flywaydb:flyway-core", version.ref = "flyway" } glassfish = { module = "org.glassfish.jersey:jackson-bom", version.ref = "glassfish_version" } google-cloud-storage = { module = "com.google.cloud:google-cloud-storage", version = "2.17.2" } -guava = { module = "com.google.guava:guava", version = "31.1-jre" } +guava = { module = "com.google.guava:guava", version = "32.1.2-jre" } hikaricp = { module = "com.zaxxer:HikariCP", version.ref = "hikaricp" } jackson-annotations = { module = "com.fasterxml.jackson.core:jackson-annotations", version.ref = "fasterxml_version" } jackson-databind = { module = "com.fasterxml.jackson.core:jackson-databind", version.ref = "fasterxml_version" } @@ -78,6 +79,7 @@ jackson-kotlin = { module = "com.fasterxml.jackson.module:jackson-module-kotlin" java-dogstatsd-client = { module = "com.datadoghq:java-dogstatsd-client", version = "4.1.0" } javax-databind = { module = "javax.xml.bind:jaxb-api", version = "2.4.0-b180830.0359" } jcl-over-slf4j = { module = "org.slf4j:jcl-over-slf4j", version.ref = "slf4j" } +jetty-io = { module = "org.eclipse.jetty:jetty-io", version = "11.0.15"} jmh-core = { module = "org.openjdk.jmh:jmh-core", version.ref = "jmh" } jmh-annotations = { module = "org.openjdk.jmh:jmh-generator-annprocess", version.ref = "jmh" } jooq = { module = "org.jooq:jooq", version.ref = "jooq" } @@ -89,6 +91,7 @@ junit-jupiter-engine = { module = "org.junit.jupiter:junit-jupiter-engine", vers junit-jupiter-params = { module = "org.junit.jupiter:junit-jupiter-params", version.ref = "junit-jupiter" } junit-jupiter-system-stubs = { module = "uk.org.webcompere:system-stubs-jupiter", version = "2.0.1" } junit-pioneer = { module = "org.junit-pioneer:junit-pioneer", version = "1.7.1" } +jsonsmart = { module = 'net.minidev:json-smart', version = "2.4.10" } launchdarkly = { module = "com.launchdarkly:launchdarkly-java-server-sdk", version = "6.0.1" } log4j-api = { module = "org.apache.logging.log4j:log4j-api", version.ref = "log4j" } log4j-core = { module = "org.apache.logging.log4j:log4j-core", version.ref = "log4j" }