.gitlab-ci.yml

include:
  - template: Code-Quality.gitlab-ci.yml
  - template: Jobs/SAST.gitlab-ci.yml
  - template: Jobs/Secret-Detection.gitlab-ci.yml
  - project: 'dev/ext-project-packaging/ci-includes'
    file: 'gitbox-deb-ci.yml'

variables:
  LANG: C.UTF-8
  IMAGE_NAME: linotp-build-test-environment_buster
  BUILD_CODE_IMAGE: $CI_REGISTRY/$IMAGE_NAME:$CI_COMMIT_REF_NAME
  LATEST_CODE_IMAGE: $CI_REGISTRY/$IMAGE_NAME:$CI_PIPELINE_ID
  DOCKERFY_URL: https://debs-devel.corp.linotp.de/misc/dockerfy/dockerfy
  DEBIAN_MIRROR: ftp.de.debian.org
  DEPENDENCY_DEB_REPO: http://debs-devel.corp.linotp.de/linotp3 buster linotp
  # DEPENDENCY_GPG_KEYID: 7CC736921C5A21B7
  DEPENDENCY_GPG_KEYID: ""
  DEPENDENCY_GPG_KEY_URL: http://debs-devel.corp.linotp.de/misc/gpg/linotp-dev-archive-keyring.gpg
  DOCKER_TAGS: $CI_PIPELINE_ID
  IMAGE_PREFIX_CI: $CI_REGISTRY/$CI_PROJECT_PATH/ci
  NOSE_COVER_PACKAGE: linotp
  DOCKER_COMPOSE_VERSION: "1.22.0"
  # Docker network per build - needed for DNS resolution between services (service linotp needs db)
  FF_NETWORK_PER_BUILD: 1
  BLACKDOG_LDAP_IMAGE: $CI_REGISTRY/docker-images/blackdog-ldap/blackdog-ldap:latest
  BLACKDOG_MYSQL_IMAGE: $CI_REGISTRY/docker-images/blackdog-mysql/blackdog-mysql:latest
  PYPI_IMAGE: $IMAGE_PREFIX_CI/linotp-testenv-pypi
  DOCKER_IMAGE_TAG: $CI_REGISTRY_IMAGE/b/$CI_COMMIT_REF_SLUG:$CI_PIPELINE_ID
  DOCKER_TEST_IMG_IMAGE_TAG: $IMAGE_PREFIX_CI/$CI_COMMIT_REF_SLUG-testimg:$CI_PIPELINE_ID
  DOCKER_TEST_ENV_IMAGE_TAG: $IMAGE_PREFIX_CI/$CI_COMMIT_REF_SLUG-testenv:$CI_PIPELINE_ID
  DOCKER_RELEASE_IMAGE: $CI_REGISTRY_IMAGE/linotp
  # Variables for security scan evaluation
  SECRET_DETECTION_REPORT_FILE: gl-secret-detection-report.json
  SAST_DETECTION_REPORT_FILE: gl-sast-report.json
  # Variables for upload-debs
  REPOSITORY: linotp3
  COMPONENT: linotp
  # uncomment `CI_DEBUG_SERVICES` to capture logs of job-services (e.g. the linotp container in e2e tests)
  # CI_DEBUG_SERVICES: "true"


workflow:
  rules:
    - if: $CI_PIPELINE_SOURCE == 'schedule'              # Execute nightly pipeline
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' # Execute pipeline in merge request context
    - if: '$CI_PIPELINE_SOURCE == "web"'                 # Execute pipeline manually triggered from the web
    - if: $CI_COMMIT_TAG                                 # Execute pipeline on release tag
    - if: '$CI_COMMIT_BRANCH =~ /^(branch-v|master)/'    # Execute pipeline when a new commit is pushed to a stable or master branch
    - if: $NIGHTLY_PIPELINE                              # Execute pipeline if run with this variable set

stages:
  - pypi-image
  - pypi-tests
  - check-and-build
  - test
  - lint
  - coverage-report
  - apidocs
  - upload-image
  - upload-debian-packages
  - iso

build-and-upload-pypi-image-testenv:
  stage: pypi-image
  # image: docker:latest
  # TODO: use docker:latest again once the pipeline-issue
  # https://forum.gitlab.com/t/the-ci-cd-pipeline-suddenly-fails-without-any-changes-done-to-the-repository/81117
  # has been resolved.
  # For now we use the proposed solution with the previous (and working) image `docker:20.10.22-dind` referenced by hash
  # docker@sha256:c8bb6fa5388b56304dd770c4bc0478de81ce18540173b1a589178c0d31bfce90
  image: docker@sha256:c8bb6fa5388b56304dd770c4bc0478de81ce18540173b1a589178c0d31bfce90
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule"
      variables:
        PYPI_IMAGE_TAG: $PYPI_IMAGE:$CI_COMMIT_SHA
    - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH =~ /^(branch-v|master)/
      variables:
        PYPI_IMAGE_TAG: $PYPI_IMAGE:$CI_COMMIT_SHA
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
      when: manual
      allow_failure: true
      variables:
        PYPI_IMAGE_TAG: $PYPI_IMAGE:$CI_MERGE_REQUEST_TARGET_BRANCH_SHA
  variables:
    PYPI_IMAGE_TAG: $PYPI_IMAGE:$CI_COMMIT_REF_NAME
  before_script:
    - !reference [.docker-registry]
  script:
    - docker build -t "$PYPI_IMAGE_TAG" -f docker/Dockerfile.testenv-pypi .
    - docker push "$PYPI_IMAGE_TAG"

build-and-upload-docker:
  stage: check-and-build
  needs: []
  # image: docker:latest
  # TODO: use docker:latest again once the pipeline-issue
  # https://forum.gitlab.com/t/the-ci-cd-pipeline-suddenly-fails-without-any-changes-done-to-the-repository/81117
  # has been resolved.
  # For now we use the proposed solution with the previous (and working) image `docker:20.10.22-dind` referenced by hash
  # docker@sha256:c8bb6fa5388b56304dd770c4bc0478de81ce18540173b1a589178c0d31bfce90
  image: docker@sha256:c8bb6fa5388b56304dd770c4bc0478de81ce18540173b1a589178c0d31bfce90
  before_script:
    - !reference [.docker-registry]
  script:
    - docker build -t "$DOCKER_IMAGE_TAG" -f docker/Dockerfile.linotp .
    - docker push "$DOCKER_IMAGE_TAG"
    - docker build -t "$DOCKER_TEST_ENV_IMAGE_TAG" --build-arg BASE_IMAGE="$DOCKER_IMAGE_TAG" -f docker/Dockerfile.linotp-testenv .
    - docker push "$DOCKER_TEST_ENV_IMAGE_TAG"
    - docker build -t "$DOCKER_TEST_IMG_IMAGE_TAG" --build-arg BASE_IMAGE="$DOCKER_IMAGE_TAG" -f docker/Dockerfile.linotp-testimg .
    - docker push "$DOCKER_TEST_IMG_IMAGE_TAG"

container_scanning:
  image:
    name: docker.io/aquasec/trivy:latest
    entrypoint: [""]
  needs: ["build-and-upload-docker"]
  variables:
    SEVERITIES: CRITICAL
    GIT_STRATEGY: none
    TRIVY_USERNAME: "$CI_REGISTRY_USER"
    TRIVY_PASSWORD: "$CI_REGISTRY_PASSWORD"
    TRIVY_AUTH_URL: "$CI_REGISTRY"
    FULL_IMAGE_NAME: $DOCKER_IMAGE_TAG
  script:
    - trivy --version
    # cache cleanup is needed when scanning images with the same tags, it does not remove the database
    - time trivy clean --all
    # update vulnerabilities db
    - time trivy image --download-db-only --no-progress --cache-dir .trivycache/
    # Builds report and puts it in the default workdir $CI_PROJECT_DIR, so `artifacts:` can take it from there
    - time trivy image --scanners vuln --exit-code 0 --cache-dir .trivycache/ --no-progress --format template --template "@/contrib/gitlab.tpl"
        --output "$CI_PROJECT_DIR/gl-container-scanning-report.json" "$FULL_IMAGE_NAME"
    # Prints full report
    - time trivy image --scanners vuln --exit-code 0 --cache-dir .trivycache/ --no-progress "$FULL_IMAGE_NAME"
    # Fail on critical vulnerabilities that could be fixed
    - time trivy image --scanners vuln --exit-code 1 --cache-dir .trivycache/ --severity "$SEVERITIES" --ignore-unfixed --no-progress "$FULL_IMAGE_NAME"
  cache:
    paths:
      - .trivycache/
  # Enables https://docs.gitlab.com/ee/user/application_security/container_scanning/ (Container Scanning report is available on GitLab EE Ultimate or GitLab.com Gold)
  artifacts:
    when: always
    expose_as: container_scanning
    paths:
      - $CI_PROJECT_DIR/gl-container-scanning-report.json
    expire_in: 2 weeks
    reports:
      container_scanning: gl-container-scanning-report.json

secret_detection:
  needs: []
  rules:
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
  artifacts:
    paths:
      - $SECRET_DETECTION_REPORT_FILE
    expire_in: 1 hour

.report_evaluation:
  rules:
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
  variables:
      SEVERITIES: "Critical,High"
      GIT_STRATEGY: none
  cache: {}
  before_script:
    - apt-get install -y jq
  script:
    - |
      if [ -f "$REPORT_FILE" ]; then
        VULNERABILITIES_COUNT=$(jq --arg severities "$SEVERITIES" '.vulnerabilities | map(select(.severity as $s | $severities | split(",") | any(. == $s))) | length' "$REPORT_FILE")
        if [ "$VULNERABILITIES_COUNT" -gt 0 ]; then
          echo "Vulnerabilities detected. Please analyze the artifact $REPORT_FILE used by the '${CI_JOB_NAME}' job."
          jq --arg severities "$SEVERITIES" '.vulnerabilities | map(select(.severity as $s | $severities | split(",") | any(. == $s)))' "$REPORT_FILE"
          exit 80
        else
          echo "No vulnerabilities detected."
        fi
      else
        echo "Artifact $REPORT_FILE does not exist. The job creating it for '${CI_JOB_NAME}' likely didn't create one. Hence, no evaluation can be performed."
      fi

secret_detection_evaluation:
  extends: .report_evaluation
  needs: ["secret_detection"]
  variables:
    REPORT_FILE: $SECRET_DETECTION_REPORT_FILE

semgrep-sast:
  needs: []
  rules:
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
  variables:
    SAST_SEMGREP_METRICS: "false"
  artifacts:
    paths:
      - $SAST_DETECTION_REPORT_FILE
    expire_in: 1 hour

semgrep-sast_evaluation:
  extends: .report_evaluation
  needs: ["semgrep-sast"]
  variables:
    REPORT_FILE: $SAST_DETECTION_REPORT_FILE
  before_script:
    - apt-get install -y jq
    # remove cmd call from report by hand (because "# nosec" does not work in semgrep job)
    # Update this command in case the job fails after altering "mysql_cmd.py"
    - jq '(.vulnerabilities[] | select(.location.file == "linotp/cli/mysql_cmd.py" and .location.start_line == 140)) |= null' $REPORT_FILE > tmp.json && mv tmp.json $REPORT_FILE

pre-commit-check:
  stage: check-and-build
  needs: []
  image: python:3.7-buster
  interruptible: true
  cache:
    paths:
      - $CI_PROJECT_DIR/.pre-commit-cache
  rules:
    - exists:
      - .pre-commit-config.yaml
  before_script:
    - pip install pre-commit
  script:
    - PRE_COMMIT_HOME=$CI_PROJECT_DIR/.pre-commit-cache pre-commit run --all-files

.docker-build-args: &docker-build-args |
  DOCKER_BUILD_ARGS="--build-arg BUILDKIT_INLINE_CACHE=1"
  # Use cached docker images in non-nightly pipelines
  if [ -z "${NIGHTLY_PIPELINE}" ]; then
    DOCKER_BUILD_ARGS="$DOCKER_BUILD_ARGS --cache-from=$BUILDER_CACHE_IMAGE --cache-from=$LINOTP_CACHE_IMAGE"       
  fi

# Build docker images and debs
# This builds the .debs and docker images (builder, linotp, testenv).
# Everything else depends on these, so we build everything in one go
# to make the build of all the images as fast as possible. We use
# Docker caching to speed things up.
.build-debs-and-images:
  stage: check-and-build
  image: docker:latest
  interruptible: true
  retry:
    max: 2
  needs: []
  before_script:
    - !reference [.docker-registry]
  script:
    - apk add --no-cache make
    - TESTENV_DEB_IMAGE=$IMAGE_PREFIX_CI/linotp-testenv-deb-$DEBIAN_RELEASE_NAME
    - SOFTHSM_DEB_IMAGE=$IMAGE_PREFIX_CI/linotp-softhsm-deb-$DEBIAN_RELEASE_NAME
    - BUILDER_CACHE_KEY=$(sha256sum docker/Dockerfile.builder-deb | cut -d ' ' -f1)
    - BUILDER_CACHE_IMAGE=$IMAGE_PREFIX_CI/linotp-builder-deb-$DEBIAN_RELEASE_NAME
    - LINOTP_CACHE_KEY=$(sha256sum docker/Dockerfile.linotp-deb | cut -d ' ' -f1)
    - LINOTP_CACHE_IMAGE=$IMAGE_PREFIX_CI/linotp-deb-$DEBIAN_RELEASE_NAME
    # Pull the old images to speedup the build process, the hope is
    # that the modified time will always be the same unless the file
    # has actually been changed
    - export DOCKER_BUILDKIT=1
    - docker pull debian:$DEBIAN_RELEASE_NAME
    - docker pull $BUILDER_CACHE_IMAGE:$BUILDER_CACHE_KEY || true
    - docker pull $LINOTP_CACHE_IMAGE:$LINOTP_CACHE_KEY || true
    # Build the images
    - *docker-build-args
    - >
        make
        docker-build-linotp-builder
        docker-build-linotp
        docker-build-linotp-test-image
        docker-build-linotp-softhsm
        BUILDDIR="${CI_PROJECT_DIR}"
        DOCKER_BUILD_ARGS="${DOCKER_BUILD_ARGS}"
    # Tag and push images to repository in the form:
    #  registry/ci/release/linotp-IMAGENAME:pipelineID
    #  registry/ci/release/linotp-IMAGENAME:sha256
    # First for linotp-builder
    - docker tag linotp-builder:latest "$BUILDER_CACHE_IMAGE:$BUILDER_CACHE_KEY"
    - docker tag linotp-builder:latest "$BUILDER_CACHE_IMAGE:$CI_PIPELINE_ID"
    - docker push "$BUILDER_CACHE_IMAGE:$BUILDER_CACHE_KEY"
    - docker push "$BUILDER_CACHE_IMAGE:$CI_PIPELINE_ID"
    # Now do the same for linotp
    - docker tag linotp:latest "$LINOTP_CACHE_IMAGE:$LINOTP_CACHE_KEY"
    - docker tag linotp:latest "$LINOTP_CACHE_IMAGE:$CI_PIPELINE_ID"
    - docker push "$LINOTP_CACHE_IMAGE:$LINOTP_CACHE_KEY"
    - docker push "$LINOTP_CACHE_IMAGE:$CI_PIPELINE_ID"
    # Now we push the testenv and softhsm images. We do not bother to cache
    # this because they change every time we do a new linotp build.
    - docker tag linotp-testenv:latest "$TESTENV_DEB_IMAGE:$CI_PIPELINE_ID"
    - docker push "$TESTENV_DEB_IMAGE:$CI_PIPELINE_ID"
    - docker tag linotp-softhsm:latest "$SOFTHSM_DEB_IMAGE:$CI_PIPELINE_ID"
    - docker push "$SOFTHSM_DEB_IMAGE:$CI_PIPELINE_ID"
    # Log all the images to stdout
    - docker images
    # Save build job id to artifacts for later uploads
    - echo "$CI_JOB_ID" > apt/"$DEBIAN_RELEASE_NAME"_build_job_id.txt
    - echo "DEB_ARTIFACTS_URL=https://${GITLAB_DOMAIN_NAME}/api/v4/projects/${CI_PROJECT_ID}/jobs/${CI_JOB_ID}/artifacts" >> build.env

build:debian-buster:
  extends: .build-debs-and-images
  variables:
    DEBIAN_RELEASE_NAME: buster
  artifacts:
    expose_as: apt-buster
    paths:
      - apt/
    name: "linotp-debs${CI_COMMIT_REF_SLUG}"
    expire_in: 2 weeks
    reports:
      dotenv: build.env


lint:
  stage: lint
  when: manual
  image: $IMAGE_PREFIX_CI/linotp-testenv-deb-$DEBIAN_RELEASE_NAME:$CI_PIPELINE_ID
  needs: ["build:debian-buster"]
  interruptible: true
  variables:
    LINTED_MODULE: ""
    DEBIAN_RELEASE_NAME: buster
    EXIT_ON_FATAL: "1"
    EXIT_ON_ERROR: "0"
    EXIT_ON_WARNING: "0"
  script:
    - pylint3 linotp/${LINTED_MODULE} && STATUS=0 || STATUS=$?
    - echo STATUS:$STATUS
    - "test $(( $STATUS & 32 )) -eq 0 || (echo 'Fail: usage error'; false)"
    - "test $(( $EXIT_ON_FATAL && $STATUS & 1 )) -eq 0 || (echo 'Fail: Fatal errors'; false)"
    - "test $(( $EXIT_ON_ERROR && $STATUS & 2 )) -eq 0 || (echo 'Fail: Errors'; false)"
    - "test $(( $EXIT_ON_WARNING && $STATUS & 4 )) -eq 0 || (echo 'Fail: Warnings'; false)"
  parallel:
    matrix:
      - LINTED_MODULE: [controllers, lib, model, provider, tokens, useridresolver]

# Provide pytest arguments based on base rules
# The tests need a number of arguments based on the type
# of job they represent. (database type, parallel run, nightly, etc.).
# We also need do decide whether to include parallel arguments depending on
# the job variables provided. We implement this as a shell command so that
# we can make use of the shell to put all the pieces together in once place.
#
# Pytest arguments can be specified in these variables:
#  NIGHTLY_ARGS     - Only used if this is a nightly (extended) pipeline or merge-train
#  NON_NIGHTLY_ARGS - Only used for normal pipelines (MRs, merges, tags)
#  ALL_JOB_ARGS     - Both nightly and non-nightly jobs
#
# Behaviour configuration:
#  NIGHTLY_PIPELINE - If set, the pipeline runs in nightly mode and enables an
#                     extended set of tests
#
# Other variables used:
#  CI_JOB_NAME      - the test report file is derived from the job name
#  CI_NODE_TOTAL    - parallel run configuration
#  CI_NODE_INDEX
.pytest-args: &pytest-args |
  # Junit XML filename is based on the job name
  PYTESTARGS="--junitxml=`echo ${CI_JOB_NAME} | tr ' /' '_'`.xml"
  if [ -n "${CI_NODE_INDEX}" ]; then
    PYTESTARGS="$PYTESTARGS --test-group-count=${CI_NODE_TOTAL} --test-group=${CI_NODE_INDEX}"
  fi
  # Add arguments from job definition
  PYTESTARGS="$PYTESTARGS ${ALL_JOB_ARGS}"
  # Job definition of nightly / non-nightly arguments
  if [ -n "${NIGHTLY_PIPELINE}" ]; then
    PYTESTARGS="$PYTESTARGS ${NIGHTLY_ARGS}"
  else
    PYTESTARGS="$PYTESTARGS ${NON_NIGHTLY_ARGS}"
  fi
  #
  if [ -n "${PYPI_ARGS}" ]; then
    PYTESTARGS="$PYTESTARGS ${PYPI_ARGS}"
  fi
  # Coverage report
  PYTESTARGS="$PYTESTARGS --cov=linotp --cov-report=xml:${CI_PROJECT_DIR}/coverage.xml"

# Nightly tests are scheduled to run once a day and contain
# more tests than the regular set
# Merge_trains and tag-pipelines too run against more to prevent a broken master
.nightly-test-set-rules:
  rules:
    - if: $NIGHTLY_PIPELINE
      when: always
    - if: $CI_MERGE_REQUEST_EVENT_TYPE == 'merge_train' || $CI_COMMIT_TAG 
      variables:
        NIGHTLY_PIPELINE: 1
      when: always

# Tests should run on Debian buster
.buster-tests:
  variables:
    DEBIAN_RELEASE_NAME: buster
  needs: ["build:debian-buster"]

# The next sections provide different database configurations
.mysql-vars: &mysql-vars
  MYSQL_DATABASE: linotp_db
  MYSQL_ROOT_PASSWORD: rootpass
  MYSQL_USER: linotp
  MYSQL_PASSWORD: linotppass
  LINOTP_DB_HOST: "db"
  LINOTP_DB_PORT: "3306"
  LINOTP_DB_WAITTIME: 2m
  LINOTP_DATABASE_URI: "mysql+mysqldb://linotp:linotppass@db/linotp_db?charset=utf8"
  LINOTP_PYTEST_DATABASE_URI: "mysql+mysqldb://linotp:linotppass@db/linotp_db?charset=utf8"

.mariadb-database:
  rules:
    - !reference [.nightly-test-set-rules, rules]
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'

  variables:
    <<: *mysql-vars
    DB: mariadb:10.1
  services:
    - name: ${DB}
      alias: db

.mysql-database:
  extends:
    - .nightly-test-set-rules
  variables:
    <<: *mysql-vars
    DB: mysql:5.5
  services:
    - name: ${DB}
      alias: db
      command: ["mysqld", "--character-set-server=utf8", "--collation-server=utf8_general_ci"]

.mysql-lower-case-table-names-database:
  extends:
    - .nightly-test-set-rules
  variables:
    <<: *mysql-vars
    DB: mysql:5.5
  services:
    - name: ${DB}
      alias: db
      command: ["mysqld", "--character-set-server=utf8", "--collation-server=utf8_general_ci", "--lower-case-table-names=1"]
  script:
      # check that config table is stored lower case, i.e. the tests executed verify that lower case table names work.
    - |
      TABLES=$(python3 -c "from sqlalchemy import create_engine; print(*create_engine('$LINOTP_DATABASE_URI').table_names(), sep='\n')")
      echo $TABLES
      if [[ "$TABLES" =~ "Config" || ! "$TABLES" =~ "config" ]]; then
        echo "ERROR: Expected to find table 'config' and not 'Config' in lower case table name tests."
        exit 1
      fi

.postgres-database:
  extends:
    - .nightly-test-set-rules
  variables:
    DB: postgres:12
    POSTGRES_DB: linotp_db
    POSTGRES_USER: linotp
    POSTGRES_PASSWORD: "linotppass"
    POSTGRES_HOST_AUTH_METHOD: trust
    LINOTP_DB_HOST: "db"
    LINOTP_DB_PORT: "3306"
    LINOTP_DB_WAITTIME: 2m
    LINOTP_DATABASE_URI: postgres://linotp:linotppass@db/linotp_db
    LINOTP_PYTEST_DATABASE_URI: postgres://linotp:linotppass@db/linotp_db
  services:
    - name: ${DB}
      alias: db

## Test stages
.test-base:
  interruptible: true
  parallel: 4
  artifacts:
    reports:
      coverage_report:
        coverage_format: cobertura
        path: ${CI_PROJECT_DIR}/coverage.xml

.test-base-pypi:
  extends: .test-base
  stage: pypi-tests
  image: $PYPI_IMAGE_TAG
  rules:
    - if: $CI_MERGE_REQUEST_EVENT_TYPE == 'merged_result'
  variables:
    PYPI_IMAGE_TAG: $PYPI_IMAGE:$CI_MERGE_REQUEST_TARGET_BRANCH_SHA
    PYPI_ARGS: "-m \"not fails_in_pypi_tests\" -n auto"
  before_script:
    - *pytest-args
  
.test-base-debs:
  extends:
    - .test-base
    - .nightly-test-set-rules
  stage: test
  needs: build:debian-buster
  image: $IMAGE_PREFIX_CI/linotp-testenv-deb-$DEBIAN_RELEASE_NAME:$CI_PIPELINE_ID

unit-test-pypi:
  extends: .test-base-pypi
  script:
      - pip3 install -e .
      - make unittests PYTESTARGS="$PYTESTARGS"

.functional-test-base-pypi:
  extends:
    - .test-base-pypi
  artifacts:
    reports:
      junit: linotp/tests/functional/functional-pypi*.xml
  script:
    - make functionaltests PYTESTARGS="$PYTESTARGS"

functional-test-pypi:
  extends: .functional-test-base-pypi

# Unit tests
unit-test:
  extends:
    - .test-base-debs
    - .buster-tests
  artifacts:
    reports:
      junit: linotp/tests/unit/unit-*.xml
  script:
    - *pytest-args
    - make unittests PYTESTARGS="$PYTESTARGS"

# Functional tests
.functional-test-base:
  extends:
    - .test-base-debs
    - .buster-tests
  artifacts:
    reports:
      junit: linotp/tests/functional/functional-*.xml
  script:
    - *pytest-args
    - make functionaltests PYTESTARGS="$PYTESTARGS"

.functional-test-base-dbs:
  extends: .functional-test-base

functional-test:
  extends: .functional-test-base

functional-test-mariadb:
  extends:
    - .functional-test-base-dbs
    - .mariadb-database
    - .nightly-test-set-rules

functional-test-postgres:
  extends:
    - .functional-test-base-dbs
    - .postgres-database

functional-test-mysql:
  extends:
    - .functional-test-base-dbs
    - .mysql-database

functional-test-mysql-lower-case-table-names:
  extends:
    - .functional-test-base-dbs
    - .mysql-lower-case-table-names-database
  script:
    - !reference [".functional-test-base-dbs", "script"]
    - !reference [".mysql-lower-case-table-names-database", "script"]

.e2e-test:
  extends:
    - .test-base-debs
    - .buster-tests
  variables:
    LINOTP_IMAGE: "linotp-deb"
    LINOTP_HOST: "test-linotp"
    LINOTP_PORT: "443"
    LINOTP_PROTOCOL: "https"
    LINOTP_USERNAME: "admin"
    LINOTP_PASSWORD: "admin"
    HEALTHCHECK_PORT: "80"
    SELENIUM_DRIVER: "chrome"
    SELENIUM_PROTOCOL: "http"
    SELENIUM_HOST: "test-chrome"
    SELENIUM_PORT: "4444"
    SSL_TRUSTED_CERT: "
      Issuer: O = LinOTP-DE, CN = LinOTP-DE Root CA
        Validity
            Not Before: Feb  6 12:15:37 2022 GMT
            Not After : Feb  4 12:15:37 2032 GMT
        Subject: O = LinOTP-DE, CN = LinOTP-DE Root CA
      -----BEGIN CERTIFICATE-----
      MIIBojCCAUmgAwIBAgIQPlwfzlZDQsPhD1rWE9Ux3TAKBggqhkjOPQQDAjAwMRIw
      EAYDVQQKEwlMaW5PVFAtREUxGjAYBgNVBAMTEUxpbk9UUC1ERSBSb290IENBMB4X
      DTIyMDIwNjEyMTUzN1oXDTMyMDIwNDEyMTUzN1owMDESMBAGA1UEChMJTGluT1RQ
      LURFMRowGAYDVQQDExFMaW5PVFAtREUgUm9vdCBDQTBZMBMGByqGSM49AgEGCCqG
      SM49AwEHA0IABGZPiX58lcLz+oX/CNZFdJI3bWE9KdxRlmypYWwbZUqkhLn1ARWc
      lblmltOU/L6/XlbYuLWdTE3Hk1VF7UA+zdejRTBDMA4GA1UdDwEB/wQEAwIBBjAS
      BgNVHRMBAf8ECDAGAQH/AgEBMB0GA1UdDgQWBBRY28kvC11GjduDBsUaQvIr87ke
      MDAKBggqhkjOPQQDAgNHADBEAiB9pumbZbFk5ChludETNKxFzSVRCx7Cbzm1zNCw
      TGfofQIgFClkyscaKq+ALGjKzDAf+oF4A1BgOzqdFxafFePRH54=
      -----END CERTIFICATE-----
      "
  artifacts:
    name: browser-screenshots
    paths:
      - Screenshots/
    when: on_failure
    reports:
      junit: linotp/tests/integration/e2e-*.xml
  script:
    - cd linotp/tests/integration
    - >
        /usr/local/bin/dockerfy
        --template docker_cfg.ini.tmpl:/tmp/server_cfg.ini
        --wait tcp://$SELENIUM_HOST:$SELENIUM_PORT
        --timeout 60m
        --wait tcp://$LINOTP_HOST:$LINOTP_PORT
        --timeout 60m
    - *pytest-args
    - make integrationtests TCFILE=/tmp/server_cfg.ini PYTESTARGS="$PYTESTARGS"
  after_script:
    - mv linotp/tests/integration/Screenshots .

e2e-test-mariadb:
  extends:
    - .e2e-test
    - .mariadb-database
  services:
    - name: "${IMAGE_PREFIX_CI}/${LINOTP_IMAGE}-${DEBIAN_RELEASE_NAME}:${CI_PIPELINE_ID}"
      alias: test-linotp
    - name: $BLACKDOG_LDAP_IMAGE
      alias: blackdog-ldap
    - name: $BLACKDOG_MYSQL_IMAGE
      alias: blackdog-mysql
    - name: ${DB}
      alias: db
    - name: selenium/standalone-chrome-debug:latest
      alias: test-chrome
  allow_failure: false

.e2e-test-docker:
  image:
    name: $DOCKER_TEST_ENV_IMAGE_TAG
    entrypoint: [""]
  extends:
    - .e2e-test
  needs:
    - build-and-upload-docker
  variables:
    LINOTP_PORT: "5000"
    LINOTP_PROTOCOL: "http"
    SKIP_ANGULAR_SELF_SERVICE_TEST: 1
  services:
    - name: $BLACKDOG_LDAP_IMAGE
      alias: blackdog-ldap
    - name: $BLACKDOG_MYSQL_IMAGE
      alias: blackdog-mysql
    - name: selenium/standalone-chrome-debug:latest
      alias: test-chrome
    # implicitly start linotp latest, as it relies on the DB
    - name: $DOCKER_TEST_IMG_IMAGE_TAG
      alias: test-linotp
      command:
        - "--with-bootstrap"
      variables:
        LINOTP_DB_WAITTIME: 10s
        LINOTP_ADMIN_PASSWORD: admin
        LINOTP_LOG_LEVEL: INFO
        LINOTP_SESSION_COOKIE_SECURE: "false"
        # reenable selfservice for some e2e tests
        LINOTP_DISABLE_CONTROLLERS: "gettoken"
        LINOTP_SITE_ROOT_REDIRECT: ""

e2e-test-docker-mariadb:
  extends:
    - .e2e-test-docker
    - .mariadb-database
  services: 
    - !reference [".e2e-test-docker", "services"]
    - name: ${DB}
      alias: db
      command: ["mysqld", "--character-set-server=utf8", "--collation-server=utf8_unicode_ci"]

e2e-test-mysql-lower-case-table-names:
  extends:
    - .e2e-test-docker
    - .mysql-lower-case-table-names-database
  services: 
    - !reference [".e2e-test-docker", "services"]
    - !reference [".mysql-lower-case-table-names-database", "services"]
  script:
    - !reference [".e2e-test-docker", "script"]
    - !reference [".mysql-lower-case-table-names-database", "script"]

# End to end test using Soft HSM image
e2e-test-softhsm:
  extends:
    - e2e-test-mariadb
    - .nightly-test-set-rules
  variables:
    ALL_JOB_ARGS: "-m smoketest -v"
    LINOTP_IMAGE: "linotp-softhsm-deb"
  allow_failure: true
  parallel:
    # too few tests -> dont split
    matrix: 
    - CI_NODE_INDEX: [1]
      CI_NODE_TOTAL: [1]

e2e-test-docker-softhsm:
  extends:
    - e2e-test-docker-mariadb
    - .nightly-test-set-rules
  variables:
    ALL_JOB_ARGS: "-m smoketest -v"
  allow_failure: true
  parallel:
    # too few tests -> dont split
    matrix: 
    - CI_NODE_INDEX: [1]
      CI_NODE_TOTAL: [1]

# Auto-generate API docs using Sphinx.
apidocs:
  stage: apidocs
  image: $IMAGE_PREFIX_CI/linotp-deb-buster:${CI_PIPELINE_ID}
  interruptible: true
  needs: ["build:debian-buster"]
  script:
    - python3 --version
    - pip3 install -e ".[apidocs]"
    - cd ${CI_PROJECT_DIR}/api-doc
    - make apidocs
    - make html
    - mkdir ${CI_PROJECT_DIR}/api-docs
    - cp -a build/html/. ${CI_PROJECT_DIR}/api-docs
  artifacts:
    name: "linotp-apidocs"
    paths:
      - api-docs
    expire_in: 2 weeks

# Upload deb packages
.debian-releases:
  parallel:
    matrix:
      - DEBIAN_RELEASE_NAME:
        - buster


upload-debs:
  stage: upload-debian-packages
  extends: .upload-debs

# Upload container image

upload-container-image:
  stage: upload-image
  extends: .upload-container-image

# Build and upload are being handled by dev/linotp/sva2/build-isos
.build-and-upload-iso:
  stage: iso
  trigger:
    project: dev/linotp/sva2/build-isos
    strategy: depend

build-and-upload-iso:
  extends: .build-and-upload-iso
  variables:
    UPLOAD: "yes"
  rules:
    - if: '$CI_COMMIT_TAG =~ /^dev\//'                  # Release candidate for QA
      variables:
        BUILD: golden
    - if: '$CI_COMMIT_TAG =~ /^release/'                # Release version
      variables:
        BUILD: release

build-iso:
  extends: .build-and-upload-iso
  when: manual
  allow_failure: true
  variables:
    DEB_ARTIFACTS_URL: $DEB_ARTIFACTS_URL
    BUILD: commit
    UPLOAD: "no"
  rules:
    - if: $CI_PIPELINE_SOURCE == 'schedule'            # Nightly/Scheduled pipeline
    - if: $CI_MERGE_REQUEST_REF_PATH                    # Commit on branch