From 4313561af5befa48c53eff8cdb77288cc213cb44 Mon Sep 17 00:00:00 2001 From: Kaspar Schleiser Date: Wed, 18 Nov 2020 14:04:58 +0100 Subject: [PATCH 01/43] .drone.yml: build on murdock --- .drone.yml | 71 ++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 55 insertions(+), 16 deletions(-) diff --git a/.drone.yml b/.drone.yml index fe200b8ff540..21935f831107 100644 --- a/.drone.yml +++ b/.drone.yml @@ -1,16 +1,55 @@ -pipeline: - run_tests: - image: riot/riotbuild:latest - environment: - - CLICOLOR_FORCE=1 - - NPROC_MAX=4 - commands: - - pwd - - printenv - - git status - - git remote -v - - ./dist/tools/ci/build_and_test.sh -matrix: - BUILDTEST_MCU_GROUP: - - static-tests - - host +kind: pipeline +type: docker +name: static_tests +trigger: + event: + - pull_request + +environment: + DRONE_CLONE_REPO: riot-ci/RIOT + CLICOLOR_FORCE: 1 + NPROC_MAX: 4 + GIT_CACHE_DIR: /cache/git + +clone: + disable: true + +services: + - name: disque + image: kaspar030/autossh + settings: + ssh_key: + from_secret: ssh_key_murdock + user: murdock-slave + host: ci.riot-os.org + ssh_args: "-L0.0.0.0:7711:localhost:7711 -L0.0.0.0:6379:localhost:6379" + ports: [ 6379, 7711 ] + +steps: + - name: clone + image: kaspar030/drone-clone + settings: + SSH_KEY: + from_secret: ssh_key_clone + mode: clone + base_repo: ${DRONE_REPO_LINK} + base_branch: ${DRONE_REPO_BRANCH} + build_repo: ${DRONE_GIT_HTTP_URL} + build_branch: ${DRONE_SOURCE_BRANCH} + build_commit: ${CI_COMMIT_SHA} + merge_commit_repo: riot-ci/RIOT + + - name: build + image: kaspar030/drone-dwq + commands: + + # setting disque parameters + - export DWQ_DISQUE_URL="disque:7711" + - export DWQ_COMMIT="$$(cat merge_commit)" + - export DWQ_REPO="https://github.com/$${DRONE_CLONE_REPO}" + + # running trivial "true" on default queue + - dwqc "true" + + # ensuring redis is operational + - redis-cli -h disque ping From 88e5ff4b1adfacaf4aa900cd0585782980581ced Mon Sep 17 00:00:00 2001 From: Kaspar Schleiser Date: Wed, 16 Dec 2020 11:53:12 +0100 Subject: [PATCH 02/43] fixup! .drone.yml: build on murdock --- .drone.yml | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 21935f831107..58073ac4745f 100644 --- a/.drone.yml +++ b/.drone.yml @@ -1,6 +1,6 @@ kind: pipeline type: docker -name: static_tests +name: build trigger: event: - pull_request @@ -15,6 +15,10 @@ clone: disable: true services: + # this service launches a container running an autossh session that + # connects to murdock-slave@ci.riot-os.org, creating port forwards to the + # disque and redis instances there. It will be automatically killed at the end + # of this pipeline. - name: disque image: kaspar030/autossh settings: @@ -26,6 +30,12 @@ services: ports: [ 6379, 7711 ] steps: + # As we're building distributed, we need to make sure that every worker is using the exact same commit. + # We also want to test the PR branch *merged into current master*. + # This step creates that merge commit and pushes it to riot-ci/RIOT. + # The resulting commit hash will be stored in "merge_commit" in the working + # directory (which is shared with the subsequent steps by drone via a docker + # volume). - name: clone image: kaspar030/drone-clone settings: @@ -39,6 +49,7 @@ steps: build_commit: ${CI_COMMIT_SHA} merge_commit_repo: riot-ci/RIOT + # This build step launches the distributed build. - name: build image: kaspar030/drone-dwq commands: @@ -53,3 +64,8 @@ steps: # ensuring redis is operational - redis-cli -h disque ping + + # launch the actual build + - dwqc -E APPS=examples/hello-world -E BOARDS=native \ + './.murdock get_jobs' \ + | dwqc --quiet --progress --outfile result.json From 64c62edb20aba4741804da83b3fdb2750ac85867 Mon Sep 17 00:00:00 2001 From: Kaspar Schleiser Date: Wed, 16 Dec 2020 11:54:57 +0100 Subject: [PATCH 03/43] fixup! fixup! .drone.yml: build on murdock --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 58073ac4745f..44ad379a0199 100644 --- a/.drone.yml +++ b/.drone.yml @@ -68,4 +68,4 @@ steps: # launch the actual build - dwqc -E APPS=examples/hello-world -E BOARDS=native \ './.murdock get_jobs' \ - | dwqc --quiet --progress --outfile result.json + | dwqc --quiet -P --outfile result.json From e1c53a94cc115cb66f8c8aa01cba727537d310a9 Mon Sep 17 00:00:00 2001 From: Kaspar Schleiser Date: Wed, 16 Dec 2020 11:57:34 +0100 Subject: [PATCH 04/43] fixup! fixup! fixup! .drone.yml: build on murdock --- .drone.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.drone.yml b/.drone.yml index 44ad379a0199..16fe1cef4249 100644 --- a/.drone.yml +++ b/.drone.yml @@ -66,6 +66,6 @@ steps: - redis-cli -h disque ping # launch the actual build - - dwqc -E APPS=examples/hello-world -E BOARDS=native \ - './.murdock get_jobs' \ - | dwqc --quiet -P --outfile result.json + - > + dwqc -E APPS=examples/hello-world -E BOARDS=native './.murdock get_jobs' + | dwqc --quiet -P --outfile result.json From 91e8f1e2bfa97f841d88e2fc1d6909775f2120ce Mon Sep 17 00:00:00 2001 From: Kaspar Schleiser Date: Wed, 16 Dec 2020 13:46:51 +0100 Subject: [PATCH 05/43] fixup! fixup! fixup! fixup! .drone.yml: build on murdock --- .drone.yml | 55 +++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 52 insertions(+), 3 deletions(-) diff --git a/.drone.yml b/.drone.yml index 16fe1cef4249..c5cf9cfb6335 100644 --- a/.drone.yml +++ b/.drone.yml @@ -66,6 +66,55 @@ steps: - redis-cli -h disque ping # launch the actual build - - > - dwqc -E APPS=examples/hello-world -E BOARDS=native './.murdock get_jobs' - | dwqc --quiet -P --outfile result.json + #- > + # dwqc -E APPS=examples/hello-world -E BOARDS=native './.murdock get_jobs' + # | dwqc --quiet -P --outfile result.json + + - dwqc -q drone-breeze hostname + - dwqm control --shutdown drone-breeze + +kind: pipeline +type: docker +name: dwq-worker-breeze +trigger: + event: + - pull_request + +environment: + DRONE_CLONE_REPO: riot-ci/RIOT + CLICOLOR_FORCE: 1 + NPROC_MAX: 4 + GIT_CACHE_DIR: /cache/git + +clone: + disable: true + +services: + # this service launches a container running an autossh session that + # connects to murdock-slave@ci.riot-os.org, creating port forwards to the + # disque and redis instances there. It will be automatically killed at the end + # of this pipeline. + - name: disque + image: kaspar030/autossh + settings: + ssh_key: + from_secret: ssh_key_murdock + user: murdock-slave + host: ci.riot-os.org + ssh_args: "-L0.0.0.0:7711:localhost:7711 -L0.0.0.0:6379:localhost:6379" + ports: [ 6379, 7711 ] + + # This build step launches the distributed build. + - name: build + image: kaspar030/riotdocker:latest + environment: + - CCACHE=ccache + - CCACHE_DIR=/cache/ccache + commands: + # setting disque parameters + - export DWQ_DISQUE_URL="disque:7711" + - mkdir -p $${CCACHE_DIR} + - git-cache init + - sleep 10 + - dwqw --name drone-breeze --jobs 16 --queues drone-breeze + From 76e929b9363a2c5ba5c7d931a4bddb907958e06b Mon Sep 17 00:00:00 2001 From: Kaspar Schleiser Date: Wed, 16 Dec 2020 13:47:37 +0100 Subject: [PATCH 06/43] fixup! fixup! fixup! fixup! fixup! .drone.yml: build on murdock --- .drone.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.drone.yml b/.drone.yml index c5cf9cfb6335..cecaff90ae69 100644 --- a/.drone.yml +++ b/.drone.yml @@ -108,8 +108,8 @@ services: - name: build image: kaspar030/riotdocker:latest environment: - - CCACHE=ccache - - CCACHE_DIR=/cache/ccache + CCACHE: ccache + CCACHE_DIR: /cache/ccache commands: # setting disque parameters - export DWQ_DISQUE_URL="disque:7711" From 6d842f06311dbc0550729c975cab2043d9bffe17 Mon Sep 17 00:00:00 2001 From: Kaspar Schleiser Date: Wed, 16 Dec 2020 13:48:30 +0100 Subject: [PATCH 07/43] fixup! fixup! fixup! fixup! fixup! fixup! .drone.yml: build on murdock --- .drone.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.drone.yml b/.drone.yml index cecaff90ae69..f4a4b56a5a1c 100644 --- a/.drone.yml +++ b/.drone.yml @@ -73,6 +73,7 @@ steps: - dwqc -q drone-breeze hostname - dwqm control --shutdown drone-breeze +--- kind: pipeline type: docker name: dwq-worker-breeze From 0005250c5c15f59d441ebe5f6667337d8244b150 Mon Sep 17 00:00:00 2001 From: Kaspar Schleiser Date: Wed, 16 Dec 2020 13:49:17 +0100 Subject: [PATCH 08/43] fixup! fixup! fixup! fixup! fixup! fixup! fixup! .drone.yml: build on murdock --- .drone.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.drone.yml b/.drone.yml index f4a4b56a5a1c..cbfd3a49640e 100644 --- a/.drone.yml +++ b/.drone.yml @@ -105,6 +105,7 @@ services: ssh_args: "-L0.0.0.0:7711:localhost:7711 -L0.0.0.0:6379:localhost:6379" ports: [ 6379, 7711 ] +steps: # This build step launches the distributed build. - name: build image: kaspar030/riotdocker:latest From 31853326d9b76542f1220944becab07cc3a236a6 Mon Sep 17 00:00:00 2001 From: Kaspar Schleiser Date: Wed, 16 Dec 2020 14:37:08 +0100 Subject: [PATCH 09/43] fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! .drone.yml: build on murdock --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index cbfd3a49640e..b3faeeebad1f 100644 --- a/.drone.yml +++ b/.drone.yml @@ -117,6 +117,6 @@ steps: - export DWQ_DISQUE_URL="disque:7711" - mkdir -p $${CCACHE_DIR} - git-cache init - - sleep 10 + - pip3 install --upgrade dwq - dwqw --name drone-breeze --jobs 16 --queues drone-breeze From e7ddd0c6800c4a568201532fc7353b139e6ffcaf Mon Sep 17 00:00:00 2001 From: Kaspar Schleiser Date: Wed, 16 Dec 2020 14:54:30 +0100 Subject: [PATCH 10/43] fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! .drone.yml: build on murdock --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index b3faeeebad1f..0899538c6015 100644 --- a/.drone.yml +++ b/.drone.yml @@ -60,7 +60,7 @@ steps: - export DWQ_REPO="https://github.com/$${DRONE_CLONE_REPO}" # running trivial "true" on default queue - - dwqc "true" + - dwqc -q drone-breeze "true" # ensuring redis is operational - redis-cli -h disque ping From af36d60322a9483ca6b74ae1f4dcc50a6817802f Mon Sep 17 00:00:00 2001 From: Kaspar Schleiser Date: Wed, 16 Dec 2020 15:47:47 +0100 Subject: [PATCH 11/43] fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! .drone.yml: build on murdock --- .drone.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.drone.yml b/.drone.yml index 0899538c6015..373b1a01d39d 100644 --- a/.drone.yml +++ b/.drone.yml @@ -120,3 +120,5 @@ steps: - pip3 install --upgrade dwq - dwqw --name drone-breeze --jobs 16 --queues drone-breeze +matrix: + many: [ 1, 2 ] From afbe3a0cff11edbd26c52c3c9f62683ba6c77ff5 Mon Sep 17 00:00:00 2001 From: Kaspar Schleiser Date: Wed, 16 Dec 2020 20:27:44 +0100 Subject: [PATCH 12/43] Revert "fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! .drone.yml: build on murdock" This reverts commit af36d60322a9483ca6b74ae1f4dcc50a6817802f. --- .drone.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.drone.yml b/.drone.yml index 373b1a01d39d..0899538c6015 100644 --- a/.drone.yml +++ b/.drone.yml @@ -120,5 +120,3 @@ steps: - pip3 install --upgrade dwq - dwqw --name drone-breeze --jobs 16 --queues drone-breeze -matrix: - many: [ 1, 2 ] From 3cdfbb0b62180a21fc26c183fb3860bebbe0076f Mon Sep 17 00:00:00 2001 From: Kaspar Schleiser Date: Wed, 16 Dec 2020 23:16:33 +0100 Subject: [PATCH 13/43] fixup! Revert "fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! .drone.yml: build on murdock" --- .drone.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.drone.yml b/.drone.yml index 0899538c6015..12649c4e3c91 100644 --- a/.drone.yml +++ b/.drone.yml @@ -53,6 +53,7 @@ steps: - name: build image: kaspar030/drone-dwq commands: + - false # setting disque parameters - export DWQ_DISQUE_URL="disque:7711" @@ -113,6 +114,7 @@ steps: CCACHE: ccache CCACHE_DIR: /cache/ccache commands: + - false # setting disque parameters - export DWQ_DISQUE_URL="disque:7711" - mkdir -p $${CCACHE_DIR} From c3144fdfb783f58b4dbe1de0c31786310698e264 Mon Sep 17 00:00:00 2001 From: Kaspar Schleiser Date: Thu, 17 Dec 2020 12:29:47 +0100 Subject: [PATCH 14/43] fixup! fixup! Revert "fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! .drone.yml: build on murdock" --- .drone.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.drone.yml b/.drone.yml index 12649c4e3c91..a47c9501e1e8 100644 --- a/.drone.yml +++ b/.drone.yml @@ -78,6 +78,8 @@ steps: kind: pipeline type: docker name: dwq-worker-breeze +parallelism: 2 + trigger: event: - pull_request From 40d5f374b6e561b86ebce2f599a17cc30ae6ce97 Mon Sep 17 00:00:00 2001 From: Kaspar Schleiser Date: Thu, 17 Dec 2020 12:58:28 +0100 Subject: [PATCH 15/43] fixup! fixup! fixup! Revert "fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! .drone.yml: build on murdock" --- .drone.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index a47c9501e1e8..c65fcd1e8bbd 100644 --- a/.drone.yml +++ b/.drone.yml @@ -123,4 +123,3 @@ steps: - git-cache init - pip3 install --upgrade dwq - dwqw --name drone-breeze --jobs 16 --queues drone-breeze - From 8dbe40521b953a0d3ee62e1b35f89b48d104f277 Mon Sep 17 00:00:00 2001 From: Kaspar Schleiser Date: Thu, 17 Dec 2020 13:15:44 +0100 Subject: [PATCH 16/43] fixup! fixup! fixup! fixup! Revert "fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! .drone.yml: build on murdock" --- .drone.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.drone.yml b/.drone.yml index c65fcd1e8bbd..81c2f30ba2fc 100644 --- a/.drone.yml +++ b/.drone.yml @@ -53,8 +53,6 @@ steps: - name: build image: kaspar030/drone-dwq commands: - - false - # setting disque parameters - export DWQ_DISQUE_URL="disque:7711" - export DWQ_COMMIT="$$(cat merge_commit)" @@ -72,7 +70,8 @@ steps: # | dwqc --quiet -P --outfile result.json - dwqc -q drone-breeze hostname - - dwqm control --shutdown drone-breeze + - dwqc -q drone-breeze hostname + - dwqm control --shutdown drone-breeze drone-breeze --- kind: pipeline @@ -116,7 +115,6 @@ steps: CCACHE: ccache CCACHE_DIR: /cache/ccache commands: - - false # setting disque parameters - export DWQ_DISQUE_URL="disque:7711" - mkdir -p $${CCACHE_DIR} From c985bcec3a92117f91970dae3bdcdd3625511b6c Mon Sep 17 00:00:00 2001 From: Kaspar Schleiser Date: Thu, 17 Dec 2020 13:32:16 +0100 Subject: [PATCH 17/43] fixup! fixup! fixup! fixup! fixup! Revert "fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! .drone.yml: build on murdock" --- .drone.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.drone.yml b/.drone.yml index 81c2f30ba2fc..3b78ccfc5783 100644 --- a/.drone.yml +++ b/.drone.yml @@ -115,6 +115,7 @@ steps: CCACHE: ccache CCACHE_DIR: /cache/ccache commands: + - printenv | sort # setting disque parameters - export DWQ_DISQUE_URL="disque:7711" - mkdir -p $${CCACHE_DIR} From 51dd03c178fad2ff819799677bf80ff37d1d337e Mon Sep 17 00:00:00 2001 From: Kaspar Schleiser Date: Thu, 17 Dec 2020 14:53:00 +0100 Subject: [PATCH 18/43] fixup! fixup! fixup! fixup! fixup! fixup! Revert "fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! .drone.yml: build on murdock" --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 3b78ccfc5783..d55d8c8bb03d 100644 --- a/.drone.yml +++ b/.drone.yml @@ -77,7 +77,7 @@ steps: kind: pipeline type: docker name: dwq-worker-breeze -parallelism: 2 +parallelism: 8 trigger: event: From 232ca70e3056707e0bddef7f3412f893a0673ca4 Mon Sep 17 00:00:00 2001 From: Kaspar Schleiser Date: Thu, 17 Dec 2020 15:31:46 +0100 Subject: [PATCH 19/43] fixup! fixup! fixup! fixup! fixup! fixup! fixup! Revert "fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! .drone.yml: build on murdock" --- .drone.yml | 58 ++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 39 insertions(+), 19 deletions(-) diff --git a/.drone.yml b/.drone.yml index d55d8c8bb03d..5991b2141c6f 100644 --- a/.drone.yml +++ b/.drone.yml @@ -19,7 +19,7 @@ services: # connects to murdock-slave@ci.riot-os.org, creating port forwards to the # disque and redis instances there. It will be automatically killed at the end # of this pipeline. - - name: disque + - name: services image: kaspar030/autossh settings: ssh_key: @@ -30,7 +30,8 @@ services: ports: [ 6379, 7711 ] steps: - # As we're building distributed, we need to make sure that every worker is using the exact same commit. + # As we're building distributed, we need to make sure that every worker is + # using the exact same commit. # We also want to test the PR branch *merged into current master*. # This step creates that merge commit and pushes it to riot-ci/RIOT. # The resulting commit hash will be stored in "merge_commit" in the working @@ -54,7 +55,7 @@ steps: image: kaspar030/drone-dwq commands: # setting disque parameters - - export DWQ_DISQUE_URL="disque:7711" + - export DWQ_DISQUE_URL="services:7711" - export DWQ_COMMIT="$$(cat merge_commit)" - export DWQ_REPO="https://github.com/$${DRONE_CLONE_REPO}" @@ -64,31 +65,42 @@ steps: # ensuring redis is operational - redis-cli -h disque ping + # build a global build id (unique over all repositories) + - export DRONE_BUILD_ID=drone-${DRONE_REPO}-{DRONE_BUILD_NUMBER} + # launch the actual build #- > # dwqc -E APPS=examples/hello-world -E BOARDS=native './.murdock get_jobs' # | dwqc --quiet -P --outfile result.json - - dwqc -q drone-breeze hostname - - dwqc -q drone-breeze hostname - - dwqm control --shutdown drone-breeze drone-breeze + - dwqc -q $${DRONE_BUILD_ID} hostname + - dwqc -q $${DRONE_BUILD_ID} hostname + # This step sends a shutdown signal to all build workers. + # It needs to be executed by all workers. + - name: shutdown-workers + image: kaspar030/drone-dwq + commands: + # setting disque parameters + - export DWQ_DISQUE_URL="disque:7711" + + # send shutdown command to workers. NUMBER (8) MUST MATCH parallelism: parameter + # in next pipeline! + - dwqm control --shutdown $$(for i in $$(seq 8); do echo $${DRONE_BUILD_ID); done) + when: + status: + - success + - failure --- kind: pipeline type: docker -name: dwq-worker-breeze +name: build-worker parallelism: 8 trigger: event: - pull_request -environment: - DRONE_CLONE_REPO: riot-ci/RIOT - CLICOLOR_FORCE: 1 - NPROC_MAX: 4 - GIT_CACHE_DIR: /cache/git - clone: disable: true @@ -97,7 +109,7 @@ services: # connects to murdock-slave@ci.riot-os.org, creating port forwards to the # disque and redis instances there. It will be automatically killed at the end # of this pipeline. - - name: disque + - name: services image: kaspar030/autossh settings: ssh_key: @@ -108,17 +120,25 @@ services: ports: [ 6379, 7711 ] steps: - # This build step launches the distributed build. - - name: build + # This step + - name: handle-build-jobs image: kaspar030/riotdocker:latest environment: CCACHE: ccache CCACHE_DIR: /cache/ccache + NPROC_MAX: 4 + GIT_CACHE_DIR: /cache/git + CLICOLOR_FORCE: 1 + commands: - - printenv | sort # setting disque parameters - - export DWQ_DISQUE_URL="disque:7711" + - export DWQ_DISQUE_URL="services:7711" + # ensure ccache folder exists - mkdir -p $${CCACHE_DIR} + # ensure git cache is initialized - git-cache init + # get latest dwq version (TODO: rely on container or pin) - pip3 install --upgrade dwq - - dwqw --name drone-breeze --jobs 16 --queues drone-breeze + # launch dwq worker + - export DRONE_BUILD_ID=drone-${DRONE_REPO}-{DRONE_BUILD_NUMBER} + - dwqw --name $${DRONE_BUILD_ID} --jobs 4 --queues $${DRONE_BUILD_ID} From 16646cacdda3123641ed2f6be54a9b712a30e01b Mon Sep 17 00:00:00 2001 From: Kaspar Schleiser Date: Thu, 17 Dec 2020 15:37:05 +0100 Subject: [PATCH 20/43] fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! Revert "fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! .drone.yml: build on murdock" --- .drone.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 5991b2141c6f..43c1fd71048f 100644 --- a/.drone.yml +++ b/.drone.yml @@ -1,6 +1,7 @@ kind: pipeline type: docker name: build + trigger: event: - pull_request @@ -86,7 +87,8 @@ steps: # send shutdown command to workers. NUMBER (8) MUST MATCH parallelism: parameter # in next pipeline! - - dwqm control --shutdown $$(for i in $$(seq 8); do echo $${DRONE_BUILD_ID); done) + - dwqm control --shutdown $$(for i in $$(seq 8); do echo $${DRONE_BUILD_ID}; done) + when: status: - success From ba98fa41d240e75838a0509f955b5ddbc4140a03 Mon Sep 17 00:00:00 2001 From: Kaspar Schleiser Date: Thu, 17 Dec 2020 15:44:43 +0100 Subject: [PATCH 21/43] fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! Revert "fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! .drone.yml: build on murdock" --- .drone.yml | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/.drone.yml b/.drone.yml index 43c1fd71048f..724a8bb926b0 100644 --- a/.drone.yml +++ b/.drone.yml @@ -60,14 +60,15 @@ steps: - export DWQ_COMMIT="$$(cat merge_commit)" - export DWQ_REPO="https://github.com/$${DRONE_CLONE_REPO}" + # build a global build id (unique over all repositories) + # this is also used as queue for dwq + - export DRONE_BUILD_ID=drone-${DRONE_REPO}-${DRONE_BUILD_NUMBER} + # running trivial "true" on default queue - - dwqc -q drone-breeze "true" + - dwqc -q $${DRONE_BUILD_ID} "true" # ensuring redis is operational - - redis-cli -h disque ping - - # build a global build id (unique over all repositories) - - export DRONE_BUILD_ID=drone-${DRONE_REPO}-{DRONE_BUILD_NUMBER} + - redis-cli -h services ping # launch the actual build #- > @@ -83,17 +84,19 @@ steps: image: kaspar030/drone-dwq commands: # setting disque parameters - - export DWQ_DISQUE_URL="disque:7711" + - export DWQ_DISQUE_URL="services:7711" # send shutdown command to workers. NUMBER (8) MUST MATCH parallelism: parameter # in next pipeline! - - dwqm control --shutdown $$(for i in $$(seq 8); do echo $${DRONE_BUILD_ID}; done) + #- dwqm control --shutdown $$(for i in $$(seq 8); do echo $${DRONE_BUILD_ID}; done) when: status: - success - failure + --- + kind: pipeline type: docker name: build-worker @@ -142,5 +145,5 @@ steps: # get latest dwq version (TODO: rely on container or pin) - pip3 install --upgrade dwq # launch dwq worker - - export DRONE_BUILD_ID=drone-${DRONE_REPO}-{DRONE_BUILD_NUMBER} + - export DRONE_BUILD_ID=drone-${DRONE_REPO}-${DRONE_BUILD_NUMBER} - dwqw --name $${DRONE_BUILD_ID} --jobs 4 --queues $${DRONE_BUILD_ID} From 36e2fc7ae6dbb51b205c9a6cec492cc46793456c Mon Sep 17 00:00:00 2001 From: Kaspar Schleiser Date: Thu, 17 Dec 2020 15:46:12 +0100 Subject: [PATCH 22/43] fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! Revert "fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! .drone.yml: build on murdock" --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 724a8bb926b0..b99e007c9d7e 100644 --- a/.drone.yml +++ b/.drone.yml @@ -88,7 +88,7 @@ steps: # send shutdown command to workers. NUMBER (8) MUST MATCH parallelism: parameter # in next pipeline! - #- dwqm control --shutdown $$(for i in $$(seq 8); do echo $${DRONE_BUILD_ID}; done) + - dwqm control --shutdown $$(for i in $$(seq 8); do echo $${DRONE_BUILD_ID}; done) when: status: From 01ee774f710f5d31b8c65dc423acacf4aee93926 Mon Sep 17 00:00:00 2001 From: Kaspar Schleiser Date: Thu, 17 Dec 2020 16:00:33 +0100 Subject: [PATCH 23/43] fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! Revert "fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! .drone.yml: build on murdock" --- .drone.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.drone.yml b/.drone.yml index b99e007c9d7e..a0c8c4803221 100644 --- a/.drone.yml +++ b/.drone.yml @@ -88,6 +88,7 @@ steps: # send shutdown command to workers. NUMBER (8) MUST MATCH parallelism: parameter # in next pipeline! + - echo dwqm control --shutdown $$(for i in $$(seq 8); do echo $${DRONE_BUILD_ID}; done) - dwqm control --shutdown $$(for i in $$(seq 8); do echo $${DRONE_BUILD_ID}; done) when: From b811d3d01d8726f1ccd7c8c67a9b342447527df7 Mon Sep 17 00:00:00 2001 From: Kaspar Schleiser Date: Thu, 17 Dec 2020 16:28:28 +0100 Subject: [PATCH 24/43] fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! Revert "fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! .drone.yml: build on murdock" --- .drone.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.drone.yml b/.drone.yml index a0c8c4803221..09e3dedf22f6 100644 --- a/.drone.yml +++ b/.drone.yml @@ -103,6 +103,9 @@ type: docker name: build-worker parallelism: 8 +node: + type: build-worker + trigger: event: - pull_request From fe3fdaa87f2506b7be4229daf501b185fa510c40 Mon Sep 17 00:00:00 2001 From: Kaspar Schleiser Date: Thu, 17 Dec 2020 16:31:24 +0100 Subject: [PATCH 25/43] fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! Revert "fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! .drone.yml: build on murdock" --- .drone.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.drone.yml b/.drone.yml index 09e3dedf22f6..d7d23aa8c617 100644 --- a/.drone.yml +++ b/.drone.yml @@ -88,8 +88,10 @@ steps: # send shutdown command to workers. NUMBER (8) MUST MATCH parallelism: parameter # in next pipeline! - - echo dwqm control --shutdown $$(for i in $$(seq 8); do echo $${DRONE_BUILD_ID}; done) - - dwqm control --shutdown $$(for i in $$(seq 8); do echo $${DRONE_BUILD_ID}; done) + - > + for i in $$(seq 8); do + dwqm control --shutdown $${DRONE_BUILD_ID} + done when: status: From 6212182005d6d78fe6afb8e42fc994b717ba38cf Mon Sep 17 00:00:00 2001 From: Kaspar Schleiser Date: Thu, 17 Dec 2020 16:33:39 +0100 Subject: [PATCH 26/43] fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! Revert "fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! .drone.yml: build on murdock" --- .drone.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.drone.yml b/.drone.yml index d7d23aa8c617..da18040c0b11 100644 --- a/.drone.yml +++ b/.drone.yml @@ -88,10 +88,8 @@ steps: # send shutdown command to workers. NUMBER (8) MUST MATCH parallelism: parameter # in next pipeline! - - > - for i in $$(seq 8); do - dwqm control --shutdown $${DRONE_BUILD_ID} - done + - export DRONE_BUILD_ID=drone-${DRONE_REPO}-${DRONE_BUILD_NUMBER} + - dwqm control --shutdown $$(for i in $$(seq 8); do echo $${DRONE_BUILD_ID}; done) when: status: From 0a898ad5f89a0c9c8ac9df0da287f09cc7b723cb Mon Sep 17 00:00:00 2001 From: Kaspar Schleiser Date: Thu, 17 Dec 2020 16:44:03 +0100 Subject: [PATCH 27/43] fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! Revert "fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! .drone.yml: build on murdock" --- .drone.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.drone.yml b/.drone.yml index da18040c0b11..fda069a999ad 100644 --- a/.drone.yml +++ b/.drone.yml @@ -2,6 +2,9 @@ kind: pipeline type: docker name: build +node: + type: build-manager + trigger: event: - pull_request From d767e8b3311a1fd005c69c0e5dae7d17d24e12b9 Mon Sep 17 00:00:00 2001 From: Kaspar Schleiser Date: Thu, 17 Dec 2020 16:51:15 +0100 Subject: [PATCH 28/43] fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! Revert "fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! .drone.yml: build on murdock" --- .drone.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.drone.yml b/.drone.yml index fda069a999ad..ed78472127ee 100644 --- a/.drone.yml +++ b/.drone.yml @@ -89,10 +89,10 @@ steps: # setting disque parameters - export DWQ_DISQUE_URL="services:7711" - # send shutdown command to workers. NUMBER (8) MUST MATCH parallelism: parameter + # send shutdown command to workers. NUMBER (4) MUST MATCH parallelism: parameter # in next pipeline! - export DRONE_BUILD_ID=drone-${DRONE_REPO}-${DRONE_BUILD_NUMBER} - - dwqm control --shutdown $$(for i in $$(seq 8); do echo $${DRONE_BUILD_ID}; done) + - dwqm control --shutdown $$(for i in $$(seq 4); do echo $${DRONE_BUILD_ID}; done) when: status: @@ -104,7 +104,7 @@ steps: kind: pipeline type: docker name: build-worker -parallelism: 8 +parallelism: 4 node: type: build-worker From cdf5edfdffe7b8e0cb7eec8937d1704f4dfaf5ed Mon Sep 17 00:00:00 2001 From: Kaspar Schleiser Date: Thu, 17 Dec 2020 17:31:46 +0100 Subject: [PATCH 29/43] fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! Revert "fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! .drone.yml: build on murdock" --- .drone.yml | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/.drone.yml b/.drone.yml index ed78472127ee..4485bae6717c 100644 --- a/.drone.yml +++ b/.drone.yml @@ -58,28 +58,27 @@ steps: - name: build image: kaspar030/drone-dwq commands: + # build a global build id (unique over all repositories) + # this is also used as queue for dwq + - export DRONE_BUILD_ID=drone-${DRONE_REPO}-${DRONE_BUILD_NUMBER} + # setting disque parameters - export DWQ_DISQUE_URL="services:7711" + - export DWQ_QUEUE=$${DRONE_BUILD_ID} - export DWQ_COMMIT="$$(cat merge_commit)" - export DWQ_REPO="https://github.com/$${DRONE_CLONE_REPO}" - # build a global build id (unique over all repositories) - # this is also used as queue for dwq - - export DRONE_BUILD_ID=drone-${DRONE_REPO}-${DRONE_BUILD_NUMBER} # running trivial "true" on default queue - - dwqc -q $${DRONE_BUILD_ID} "true" + - dwqc "true" # ensuring redis is operational - redis-cli -h services ping # launch the actual build - #- > - # dwqc -E APPS=examples/hello-world -E BOARDS=native './.murdock get_jobs' - # | dwqc --quiet -P --outfile result.json - - - dwqc -q $${DRONE_BUILD_ID} hostname - - dwqc -q $${DRONE_BUILD_ID} hostname + - > + dwqc -E APPS=examples/hello-world -E BOARDS=native './.murdock get_jobs' + | dwqc --quiet -P --outfile result.json # This step sends a shutdown signal to all build workers. # It needs to be executed by all workers. From 9011e0d1978824f088979093a38594c0514c77c9 Mon Sep 17 00:00:00 2001 From: Kaspar Schleiser Date: Thu, 17 Dec 2020 17:37:37 +0100 Subject: [PATCH 30/43] fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! Revert "fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! .drone.yml: build on murdock" --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 4485bae6717c..ea67f4e54783 100644 --- a/.drone.yml +++ b/.drone.yml @@ -77,7 +77,7 @@ steps: # launch the actual build - > - dwqc -E APPS=examples/hello-world -E BOARDS=native './.murdock get_jobs' + dwqc -E BOARDS=native './.murdock get_jobs' | dwqc --quiet -P --outfile result.json # This step sends a shutdown signal to all build workers. From 0d6fe344405b294098a63e0687b4c47b6f664eb4 Mon Sep 17 00:00:00 2001 From: Kaspar Schleiser Date: Fri, 18 Dec 2020 13:50:16 +0100 Subject: [PATCH 31/43] fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! Revert "fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! .drone.yml: build on murdock" --- .drone.yml | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/.drone.yml b/.drone.yml index ea67f4e54783..085f5a80dc4d 100644 --- a/.drone.yml +++ b/.drone.yml @@ -68,18 +68,38 @@ steps: - export DWQ_COMMIT="$$(cat merge_commit)" - export DWQ_REPO="https://github.com/$${DRONE_CLONE_REPO}" + # ensuring redis is operational + - redis-cli -h services ping # running trivial "true" on default queue - dwqc "true" - # ensuring redis is operational - - redis-cli -h services ping - # launch the actual build - > dwqc -E BOARDS=native './.murdock get_jobs' | dwqc --quiet -P --outfile result.json + # This build step processes the build results + - name: build + image: kaspar030/drone-postbuild + commands: + # build a global build id (unique over all repositories) + # this is also used as queue for dwq + - export DRONE_BUILD_ID=drone-${DRONE_REPO}-${DRONE_BUILD_NUMBER} + - > + for script in /scripts/postbuild.d; do + $script + done + + - /scripts/parse_results.py + + when: + status: + - success + # also run on failure, we want to see why it failed :) + - failure + + # This step sends a shutdown signal to all build workers. # It needs to be executed by all workers. - name: shutdown-workers From 4fc588d11444a952face5eee9fcb066933f914e7 Mon Sep 17 00:00:00 2001 From: Kaspar Schleiser Date: Fri, 18 Dec 2020 13:51:17 +0100 Subject: [PATCH 32/43] fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! Revert "fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! .drone.yml: build on murdock" --- .drone.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.drone.yml b/.drone.yml index 085f5a80dc4d..b5dd9a13623b 100644 --- a/.drone.yml +++ b/.drone.yml @@ -79,8 +79,8 @@ steps: dwqc -E BOARDS=native './.murdock get_jobs' | dwqc --quiet -P --outfile result.json - # This build step processes the build results - - name: build + # This step processes the build results + - name: process-results image: kaspar030/drone-postbuild commands: # build a global build id (unique over all repositories) From 46bc1654c17ac36e4fe942f8b1158b10cf885973 Mon Sep 17 00:00:00 2001 From: Kaspar Schleiser Date: Fri, 18 Dec 2020 14:07:19 +0100 Subject: [PATCH 33/43] fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! Revert "fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! .drone.yml: build on murdock" --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index b5dd9a13623b..be8fae14c41e 100644 --- a/.drone.yml +++ b/.drone.yml @@ -87,7 +87,7 @@ steps: # this is also used as queue for dwq - export DRONE_BUILD_ID=drone-${DRONE_REPO}-${DRONE_BUILD_NUMBER} - > - for script in /scripts/postbuild.d; do + for script in /scripts/post-build.d; do $script done From e8be310548580e6963d68e1b4332082b687e7448 Mon Sep 17 00:00:00 2001 From: Kaspar Schleiser Date: Fri, 18 Dec 2020 14:17:59 +0100 Subject: [PATCH 34/43] fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! Revert "fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! .drone.yml: build on murdock" --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index be8fae14c41e..35f39590dbfa 100644 --- a/.drone.yml +++ b/.drone.yml @@ -87,7 +87,7 @@ steps: # this is also used as queue for dwq - export DRONE_BUILD_ID=drone-${DRONE_REPO}-${DRONE_BUILD_NUMBER} - > - for script in /scripts/post-build.d; do + for script in /scripts/post-build.d/*; do $script done From f4d87c433be027967fa95f73fcc413140d431d49 Mon Sep 17 00:00:00 2001 From: Kaspar Schleiser Date: Fri, 18 Dec 2020 14:23:32 +0100 Subject: [PATCH 35/43] fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! Revert "fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! .drone.yml: build on murdock" --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 35f39590dbfa..76fe2942b936 100644 --- a/.drone.yml +++ b/.drone.yml @@ -91,7 +91,7 @@ steps: $script done - - /scripts/parse_results.py + - /scripts/parse_result.py when: status: From 7508388567efedd5cfa1bc1fb358cfb3a9280328 Mon Sep 17 00:00:00 2001 From: Kaspar Schleiser Date: Fri, 18 Dec 2020 14:57:52 +0100 Subject: [PATCH 36/43] fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! Revert "fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! .drone.yml: build on murdock" --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 76fe2942b936..b6c4d53d3d90 100644 --- a/.drone.yml +++ b/.drone.yml @@ -91,7 +91,7 @@ steps: $script done - - /scripts/parse_result.py + - /scripts/parse_result.py result.json when: status: From f3f5614dcd5e72d596dfc1d0d95f9309c008d22f Mon Sep 17 00:00:00 2001 From: Kaspar Schleiser Date: Fri, 18 Dec 2020 15:03:48 +0100 Subject: [PATCH 37/43] fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! Revert "fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! .drone.yml: build on murdock" --- .drone.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.drone.yml b/.drone.yml index b6c4d53d3d90..99b88897e1b9 100644 --- a/.drone.yml +++ b/.drone.yml @@ -162,6 +162,7 @@ steps: CLICOLOR_FORCE: 1 commands: + - \# Running on runner ${DRONE_STAGE_MACHINE} # setting disque parameters - export DWQ_DISQUE_URL="services:7711" # ensure ccache folder exists From ba27a09338eb20387231eb3310e7a575c475e8f8 Mon Sep 17 00:00:00 2001 From: Kaspar Schleiser Date: Fri, 18 Dec 2020 15:45:28 +0100 Subject: [PATCH 38/43] fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! Revert "fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! .drone.yml: build on murdock" --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 99b88897e1b9..0846e9619788 100644 --- a/.drone.yml +++ b/.drone.yml @@ -162,7 +162,7 @@ steps: CLICOLOR_FORCE: 1 commands: - - \# Running on runner ${DRONE_STAGE_MACHINE} + - "# Running on runner ${DRONE_STAGE_MACHINE}" # setting disque parameters - export DWQ_DISQUE_URL="services:7711" # ensure ccache folder exists From 71fda9138ccd8d645b84c115b92caa2924081967 Mon Sep 17 00:00:00 2001 From: Kaspar Schleiser Date: Fri, 18 Dec 2020 15:48:26 +0100 Subject: [PATCH 39/43] fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! Revert "fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! .drone.yml: build on murdock" --- .drone.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.drone.yml b/.drone.yml index 0846e9619788..c7d820508299 100644 --- a/.drone.yml +++ b/.drone.yml @@ -62,19 +62,19 @@ steps: # this is also used as queue for dwq - export DRONE_BUILD_ID=drone-${DRONE_REPO}-${DRONE_BUILD_NUMBER} - # setting disque parameters + - "# setting disque parameters" - export DWQ_DISQUE_URL="services:7711" - export DWQ_QUEUE=$${DRONE_BUILD_ID} - export DWQ_COMMIT="$$(cat merge_commit)" - export DWQ_REPO="https://github.com/$${DRONE_CLONE_REPO}" - # ensuring redis is operational + - "# ensuring redis is operational" - redis-cli -h services ping - # running trivial "true" on default queue + - "# running trivial "true" on default queue" - dwqc "true" - # launch the actual build + - "# launch the distributed build" - > dwqc -E BOARDS=native './.murdock get_jobs' | dwqc --quiet -P --outfile result.json @@ -86,11 +86,13 @@ steps: # build a global build id (unique over all repositories) # this is also used as queue for dwq - export DRONE_BUILD_ID=drone-${DRONE_REPO}-${DRONE_BUILD_NUMBER} + - "# running post build scripts" - > for script in /scripts/post-build.d/*; do + echo " running $script" $script done - + - "# parsing build results" - /scripts/parse_result.py result.json when: @@ -162,7 +164,9 @@ steps: CLICOLOR_FORCE: 1 commands: + - "#" - "# Running on runner ${DRONE_STAGE_MACHINE}" + - "#" # setting disque parameters - export DWQ_DISQUE_URL="services:7711" # ensure ccache folder exists From 22e0fc24869d672d4053e4ee9ae20afc0be67cbb Mon Sep 17 00:00:00 2001 From: Kaspar Schleiser Date: Fri, 18 Dec 2020 15:52:18 +0100 Subject: [PATCH 40/43] fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! Revert "fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! .drone.yml: build on murdock" --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index c7d820508299..acb51629da60 100644 --- a/.drone.yml +++ b/.drone.yml @@ -71,7 +71,7 @@ steps: - "# ensuring redis is operational" - redis-cli -h services ping - - "# running trivial "true" on default queue" + - "# ensuring disque is operational" - dwqc "true" - "# launch the distributed build" From 8f557e694b93de66a7cb3b8c37db71c9b49cb399 Mon Sep 17 00:00:00 2001 From: Kaspar Schleiser Date: Fri, 18 Dec 2020 22:52:29 +0100 Subject: [PATCH 41/43] fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! Revert "fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! .drone.yml: build on murdock" --- .drone.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.drone.yml b/.drone.yml index acb51629da60..4cab7e3ac04b 100644 --- a/.drone.yml +++ b/.drone.yml @@ -127,8 +127,8 @@ type: docker name: build-worker parallelism: 4 -node: - type: build-worker +#node: +# type: build-worker trigger: event: From edc79f48084f9e83942c6c9b0d99652c32ca00bd Mon Sep 17 00:00:00 2001 From: Kaspar Schleiser Date: Sat, 19 Dec 2020 00:21:09 +0100 Subject: [PATCH 42/43] fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! Revert "fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! .drone.yml: build on murdock" --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 4cab7e3ac04b..ddc9dae0317d 100644 --- a/.drone.yml +++ b/.drone.yml @@ -171,7 +171,7 @@ steps: - export DWQ_DISQUE_URL="services:7711" # ensure ccache folder exists - mkdir -p $${CCACHE_DIR} - # ensure git cache is initialized + - "# ensure git cache is initialized" - git-cache init # get latest dwq version (TODO: rely on container or pin) - pip3 install --upgrade dwq From ecb63a834a9a7fd1ad14d08c65e254adf4885d6c Mon Sep 17 00:00:00 2001 From: Kaspar Schleiser Date: Sat, 19 Dec 2020 00:26:46 +0100 Subject: [PATCH 43/43] fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! Revert "fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! .drone.yml: build on murdock" --- .drone.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.drone.yml b/.drone.yml index ddc9dae0317d..f4d09a737d28 100644 --- a/.drone.yml +++ b/.drone.yml @@ -121,7 +121,6 @@ steps: - failure --- - kind: pipeline type: docker name: build-worker @@ -153,7 +152,8 @@ services: ports: [ 6379, 7711 ] steps: - # This step + # This step runs the dwq worker, which will execute build jobs sent by the + # "build" step from above - name: handle-build-jobs image: kaspar030/riotdocker:latest environment: