diff --git a/.drone.yml b/.drone.yml index fe200b8ff540..f4d09a737d28 100644 --- a/.drone.yml +++ b/.drone.yml @@ -1,16 +1,180 @@ -pipeline: - run_tests: - image: riot/riotbuild:latest - environment: - - CLICOLOR_FORCE=1 - - NPROC_MAX=4 - commands: - - pwd - - printenv - - git status - - git remote -v - - ./dist/tools/ci/build_and_test.sh -matrix: - BUILDTEST_MCU_GROUP: - - static-tests - - host +kind: pipeline +type: docker +name: build + +node: + type: build-manager + +trigger: + event: + - pull_request + +environment: + DRONE_CLONE_REPO: riot-ci/RIOT + CLICOLOR_FORCE: 1 + NPROC_MAX: 4 + GIT_CACHE_DIR: /cache/git + +clone: + disable: true + +services: + # this service launches a container running an autossh session that + # connects to murdock-slave@ci.riot-os.org, creating port forwards to the + # disque and redis instances there. It will be automatically killed at the end + # of this pipeline. + - name: services + image: kaspar030/autossh + settings: + ssh_key: + from_secret: ssh_key_murdock + user: murdock-slave + host: ci.riot-os.org + ssh_args: "-L0.0.0.0:7711:localhost:7711 -L0.0.0.0:6379:localhost:6379" + ports: [ 6379, 7711 ] + +steps: + # As we're building distributed, we need to make sure that every worker is + # using the exact same commit. + # We also want to test the PR branch *merged into current master*. + # This step creates that merge commit and pushes it to riot-ci/RIOT. + # The resulting commit hash will be stored in "merge_commit" in the working + # directory (which is shared with the subsequent steps by drone via a docker + # volume). + - name: clone + image: kaspar030/drone-clone + settings: + SSH_KEY: + from_secret: ssh_key_clone + mode: clone + base_repo: ${DRONE_REPO_LINK} + base_branch: ${DRONE_REPO_BRANCH} + build_repo: ${DRONE_GIT_HTTP_URL} + build_branch: ${DRONE_SOURCE_BRANCH} + build_commit: ${CI_COMMIT_SHA} + merge_commit_repo: riot-ci/RIOT + + # This build step launches the distributed build. + - name: build + image: kaspar030/drone-dwq + commands: + # build a global build id (unique over all repositories) + # this is also used as queue for dwq + - export DRONE_BUILD_ID=drone-${DRONE_REPO}-${DRONE_BUILD_NUMBER} + + - "# setting disque parameters" + - export DWQ_DISQUE_URL="services:7711" + - export DWQ_QUEUE=$${DRONE_BUILD_ID} + - export DWQ_COMMIT="$$(cat merge_commit)" + - export DWQ_REPO="https://github.com/$${DRONE_CLONE_REPO}" + + - "# ensuring redis is operational" + - redis-cli -h services ping + + - "# ensuring disque is operational" + - dwqc "true" + + - "# launch the distributed build" + - > + dwqc -E BOARDS=native './.murdock get_jobs' + | dwqc --quiet -P --outfile result.json + + # This step processes the build results + - name: process-results + image: kaspar030/drone-postbuild + commands: + # build a global build id (unique over all repositories) + # this is also used as queue for dwq + - export DRONE_BUILD_ID=drone-${DRONE_REPO}-${DRONE_BUILD_NUMBER} + - "# running post build scripts" + - > + for script in /scripts/post-build.d/*; do + echo " running $script" + $script + done + - "# parsing build results" + - /scripts/parse_result.py result.json + + when: + status: + - success + # also run on failure, we want to see why it failed :) + - failure + + + # This step sends a shutdown signal to all build workers. + # It needs to be executed by all workers. + - name: shutdown-workers + image: kaspar030/drone-dwq + commands: + # setting disque parameters + - export DWQ_DISQUE_URL="services:7711" + + # send shutdown command to workers. NUMBER (4) MUST MATCH parallelism: parameter + # in next pipeline! + - export DRONE_BUILD_ID=drone-${DRONE_REPO}-${DRONE_BUILD_NUMBER} + - dwqm control --shutdown $$(for i in $$(seq 4); do echo $${DRONE_BUILD_ID}; done) + + when: + status: + - success + - failure + +--- +kind: pipeline +type: docker +name: build-worker +parallelism: 4 + +#node: +# type: build-worker + +trigger: + event: + - pull_request + +clone: + disable: true + +services: + # this service launches a container running an autossh session that + # connects to murdock-slave@ci.riot-os.org, creating port forwards to the + # disque and redis instances there. It will be automatically killed at the end + # of this pipeline. + - name: services + image: kaspar030/autossh + settings: + ssh_key: + from_secret: ssh_key_murdock + user: murdock-slave + host: ci.riot-os.org + ssh_args: "-L0.0.0.0:7711:localhost:7711 -L0.0.0.0:6379:localhost:6379" + ports: [ 6379, 7711 ] + +steps: + # This step runs the dwq worker, which will execute build jobs sent by the + # "build" step from above + - name: handle-build-jobs + image: kaspar030/riotdocker:latest + environment: + CCACHE: ccache + CCACHE_DIR: /cache/ccache + NPROC_MAX: 4 + GIT_CACHE_DIR: /cache/git + CLICOLOR_FORCE: 1 + + commands: + - "#" + - "# Running on runner ${DRONE_STAGE_MACHINE}" + - "#" + # setting disque parameters + - export DWQ_DISQUE_URL="services:7711" + # ensure ccache folder exists + - mkdir -p $${CCACHE_DIR} + - "# ensure git cache is initialized" + - git-cache init + # get latest dwq version (TODO: rely on container or pin) + - pip3 install --upgrade dwq + # launch dwq worker + - export DRONE_BUILD_ID=drone-${DRONE_REPO}-${DRONE_BUILD_NUMBER} + - dwqw --name $${DRONE_BUILD_ID} --jobs 4 --queues $${DRONE_BUILD_ID}