From c898b0d1c1fd644d84af54800349dcc4aba02eb8 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Tue, 20 Sep 2016 14:37:02 +0200 Subject: [PATCH 01/25] Upgrades to latest Git release --- git/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/git/Dockerfile b/git/Dockerfile index 276f51f..08a6143 100644 --- a/git/Dockerfile +++ b/git/Dockerfile @@ -1,9 +1,9 @@ FROM httpd:2.4.23 -ENV GIT_VERSION 2.9.3 +ENV GIT_VERSION 2.10.0 ENV GIT_VERSION_TGZ_URL https://www.kernel.org/pub/software/scm/git/git-$GIT_VERSION.tar.gz -ENV GIT_VERSION_TGZ_SHA1 ae90c4e5008ae10c8a67a51ff3dbea8364d97168 +ENV GIT_VERSION_TGZ_SHA1 2d588afe7adb11ea11e0787c4a2f01329a0f2f55 RUN depsRuntime=' \ libcurl3 \ From 076b26cb2122f764182e73554064856ea0c1ee69 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Tue, 20 Sep 2016 15:29:10 +0200 Subject: [PATCH 02/25] Break up into separate images for git installation and git httpd configuration --- git/Dockerfile | 45 +------------------------------------------- gitclient/Dockerfile | 44 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 44 deletions(-) create mode 100644 gitclient/Dockerfile diff --git a/git/Dockerfile b/git/Dockerfile index 08a6143..7676108 100644 --- a/git/Dockerfile +++ b/git/Dockerfile @@ -1,47 +1,4 @@ - -FROM httpd:2.4.23 - -ENV GIT_VERSION 2.10.0 -ENV GIT_VERSION_TGZ_URL https://www.kernel.org/pub/software/scm/git/git-$GIT_VERSION.tar.gz -ENV GIT_VERSION_TGZ_SHA1 2d588afe7adb11ea11e0787c4a2f01329a0f2f55 - -RUN depsRuntime=' \ - libcurl3 \ - libexpat1 \ - gettext \ - libssl1.0.0 \ - ' \ - && depsBuild=' \ - curl ca-certificates \ - gcc \ - make \ - autoconf \ - libcurl4-gnutls-dev \ - libexpat1-dev \ - gettext \ - libz-dev \ - libssl-dev \ - ' \ - set -x \ - && apt-get update \ - && apt-get install -y --no-install-recommends $depsRuntime \ - && apt-get install -y --no-install-recommends $depsBuild \ - && rm -r /var/lib/apt/lists/* \ - && curl -SL "$GIT_VERSION_TGZ_URL" -o git-$GIT_VERSION.tar.gz \ - && echo "$GIT_VERSION_TGZ_SHA1 git-$GIT_VERSION.tar.gz" | sha1sum -c - \ - && mkdir -p src/git \ - && tar -xvf git-$GIT_VERSION.tar.gz -C src/git --strip-components=1 \ - && rm git-$GIT_VERSION.tar.gz* \ - && cd src/git \ - && make configure \ - && ./configure --prefix=/usr \ - && make all \ - && make install \ - && cd ../../ \ - && rm -r src/git \ - && apt-get purge -y --auto-remove $depsBuild - -EXPOSE 80 +FROM solsson/httpd-gitclient RUN sed -i 's|#LoadModule cgid_module|LoadModule cgid_module|' conf/httpd.conf \ && sed -i 's|#LoadModule rewrite_module|LoadModule rewrite_module|' conf/httpd.conf \ diff --git a/gitclient/Dockerfile b/gitclient/Dockerfile new file mode 100644 index 0000000..2194d64 --- /dev/null +++ b/gitclient/Dockerfile @@ -0,0 +1,44 @@ + +FROM httpd:2.4.23 + +ENV GIT_VERSION 2.10.0 +ENV GIT_VERSION_TGZ_URL https://www.kernel.org/pub/software/scm/git/git-$GIT_VERSION.tar.gz +ENV GIT_VERSION_TGZ_SHA1 2d588afe7adb11ea11e0787c4a2f01329a0f2f55 + +RUN depsRuntime=' \ + libcurl3 \ + libexpat1 \ + gettext \ + libssl1.0.0 \ + ' \ + && depsBuild=' \ + curl ca-certificates \ + gcc \ + make \ + autoconf \ + libcurl4-gnutls-dev \ + libexpat1-dev \ + gettext \ + libz-dev \ + libssl-dev \ + ' \ + set -x \ + && apt-get update \ + && apt-get install -y --no-install-recommends $depsRuntime \ + && apt-get install -y --no-install-recommends $depsBuild \ + && rm -r /var/lib/apt/lists/* \ + && curl -SL "$GIT_VERSION_TGZ_URL" -o git-$GIT_VERSION.tar.gz \ + && echo "$GIT_VERSION_TGZ_SHA1 git-$GIT_VERSION.tar.gz" | sha1sum -c - \ + && mkdir -p src/git \ + && tar -xvf git-$GIT_VERSION.tar.gz -C src/git --strip-components=1 \ + && rm git-$GIT_VERSION.tar.gz* \ + && cd src/git \ + && make configure \ + && ./configure --prefix=/usr \ + && make all \ + && make install \ + && cd ../../ \ + && rm -r src/git \ + && apt-get purge -y --auto-remove $depsBuild + +EXPOSE 80 From 8900286bd9492940a9c8dd227bb753e825290ba5 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Tue, 20 Sep 2016 15:30:33 +0200 Subject: [PATCH 03/25] Omit the organization prefix so we can use FROM with docker-compose build+image --- git/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/Dockerfile b/git/Dockerfile index 7676108..55c75a6 100644 --- a/git/Dockerfile +++ b/git/Dockerfile @@ -1,4 +1,4 @@ -FROM solsson/httpd-gitclient +FROM httpd-gitclient RUN sed -i 's|#LoadModule cgid_module|LoadModule cgid_module|' conf/httpd.conf \ && sed -i 's|#LoadModule rewrite_module|LoadModule rewrite_module|' conf/httpd.conf \ From 583cb982d5cb472b2e859b43a94c5c0ff7a98c6c Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Tue, 20 Sep 2016 15:31:44 +0200 Subject: [PATCH 04/25] Use one container to initialize git repositories with content for a second container, to emulate Kubernetes gitVolume --- build-contracts/docker-compose.yml | 21 +++++++++++++++++++++ httpd-gitconf/Dockerfile | 4 ++++ 2 files changed, 25 insertions(+) create mode 100644 build-contracts/docker-compose.yml create mode 100644 httpd-gitconf/Dockerfile diff --git a/build-contracts/docker-compose.yml b/build-contracts/docker-compose.yml new file mode 100644 index 0000000..56a88e8 --- /dev/null +++ b/build-contracts/docker-compose.yml @@ -0,0 +1,21 @@ +version: '2' +services: + gitclient.build: + build: ../gitclient + image: httpd-gitclient + entrypoint: ["echo", "This service was just a build job. Exiting."] + git.init: + build: ../git + image: httpd-gitclient + depends_on: + - gitclient.build + expose: + - "80" + volumes: + - ./data-git/conf:/usr/local/apache2/conf + - ./data-git/cert:/usr/local/apache2/cert + httpd: + build: ../httpd-gitconf + image: httpd-gitconf + volumes_from: + - git.init diff --git a/httpd-gitconf/Dockerfile b/httpd-gitconf/Dockerfile new file mode 100644 index 0000000..2f07d22 --- /dev/null +++ b/httpd-gitconf/Dockerfile @@ -0,0 +1,4 @@ + +# TODO but we need openidc here too + +FROM httpd-gitclient From 0286ced13a198b9908aacb501b28099e5311a94c Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Tue, 20 Sep 2016 15:35:26 +0200 Subject: [PATCH 05/25] Removes misleading image: from downstrea, but keeps depends_on because it seems required for build dependency --- build-contracts/docker-compose.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/build-contracts/docker-compose.yml b/build-contracts/docker-compose.yml index 56a88e8..4d9ef14 100644 --- a/build-contracts/docker-compose.yml +++ b/build-contracts/docker-compose.yml @@ -6,7 +6,6 @@ services: entrypoint: ["echo", "This service was just a build job. Exiting."] git.init: build: ../git - image: httpd-gitclient depends_on: - gitclient.build expose: From bc6ce0acc1c94ac70efaa197d5862b2a777e4a67 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Tue, 20 Sep 2016 16:43:48 +0200 Subject: [PATCH 06/25] libcurl-gnutls required for local git http clone, and with libcurl present we might just have curl CLI there too --- gitclient/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gitclient/Dockerfile b/gitclient/Dockerfile index 2194d64..0619af2 100644 --- a/gitclient/Dockerfile +++ b/gitclient/Dockerfile @@ -7,6 +7,8 @@ ENV GIT_VERSION_TGZ_SHA1 2d588afe7adb11ea11e0787c4a2f01329a0f2f55 RUN depsRuntime=' \ libcurl3 \ + libcurl3-gnutls \ + curl \ libexpat1 \ gettext \ libssl1.0.0 \ From ecb23ecbcb9a6a1bf1626359e786a2dba0ac423c Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Tue, 20 Sep 2016 19:42:55 +0200 Subject: [PATCH 07/25] Looks like there's a severe regression in solsson/httpd:git for 2.9+, with 404 on git clone http://$(docker-compose port git2 80)/git/Org/repo.git when the repo exists at /opt/git/Org/repo.git --- build-contracts/docker-compose.yml | 20 ++++++++++++-------- build-contracts/githost/Dockerfile | 17 +++++++++++++++++ build-contracts/githost/gitconfig | 3 +++ 3 files changed, 32 insertions(+), 8 deletions(-) create mode 100644 build-contracts/githost/Dockerfile create mode 100644 build-contracts/githost/gitconfig diff --git a/build-contracts/docker-compose.yml b/build-contracts/docker-compose.yml index 4d9ef14..32503ca 100644 --- a/build-contracts/docker-compose.yml +++ b/build-contracts/docker-compose.yml @@ -4,17 +4,21 @@ services: build: ../gitclient image: httpd-gitclient entrypoint: ["echo", "This service was just a build job. Exiting."] - git.init: + git.build: build: ../git + image: httpd-git + entrypoint: ["echo", "This service was just a build job. Exiting."] + githost: + build: ./githost depends_on: - - gitclient.build - expose: + - git.build + ports: + - "80" + git2: + image: solsson/httpd@sha256:5bd6cc8c9c3a3dc5d9cd816a602dd7eeb576520343dc9521a594ea13f7ca9d33 + ports: - "80" - volumes: - - ./data-git/conf:/usr/local/apache2/conf - - ./data-git/cert:/usr/local/apache2/cert httpd: build: ../httpd-gitconf image: httpd-gitconf - volumes_from: - - git.init + #entrypoint: git-clones-then-httpd-foreground diff --git a/build-contracts/githost/Dockerfile b/build-contracts/githost/Dockerfile new file mode 100644 index 0000000..59fc9d4 --- /dev/null +++ b/build-contracts/githost/Dockerfile @@ -0,0 +1,17 @@ +FROM httpd-git + +#RUN rm conf/git/git-readonly.conf + +RUN mkdir -p /opt/git/Gitconf \ + && git init --bare /opt/git/Gitconf/conf.git \ + && git init --bare /opt/git/Gitconf/cert.git + +RUN git config --global user.email "you@example.com" \ + && git config --global user.name "Your Name" \ + && git clone /opt/git/Gitconf/conf.git /tmp/conf \ + && cd /tmp/conf/ \ + && cp /usr/local/apache2/conf/httpd.conf /tmp/conf/ \ + && sed -i 's/^Include/#Include/' /tmp/conf/httpd.conf \ + && git add httpd.conf \ + && git commit -m "Gets httpd up and running" \ + && git push origin master diff --git a/build-contracts/githost/gitconfig b/build-contracts/githost/gitconfig new file mode 100644 index 0000000..29ace58 --- /dev/null +++ b/build-contracts/githost/gitconfig @@ -0,0 +1,3 @@ +[user] + email = you@example.com + name = Your Name From fcd8aca659c234bf75c7370456e43c580fca302a Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Tue, 20 Sep 2016 19:44:42 +0200 Subject: [PATCH 08/25] Verified that config reload works with docker httpd, noted as pseudo code --- httpd-gitconf/conf-reload | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 httpd-gitconf/conf-reload diff --git a/httpd-gitconf/conf-reload b/httpd-gitconf/conf-reload new file mode 100644 index 0000000..cfc5f45 --- /dev/null +++ b/httpd-gitconf/conf-reload @@ -0,0 +1,14 @@ + +# timestamp +RUNLABEL=() + +# keep current config state for undo +for (d in conf cert) + git checkout -b previous-$RUNLABEL + +function rollback + +# test config +apachectl configtest || rollback + +apachectl graceful || rollback From 98c80c84f8818d20e19f29ff32930e96e81cad52 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Thu, 22 Sep 2016 08:54:13 +0200 Subject: [PATCH 09/25] Test setup, now let's do the scripting --- build-contracts/docker-compose.yml | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/build-contracts/docker-compose.yml b/build-contracts/docker-compose.yml index 32503ca..d23efaf 100644 --- a/build-contracts/docker-compose.yml +++ b/build-contracts/docker-compose.yml @@ -6,6 +6,8 @@ services: entrypoint: ["echo", "This service was just a build job. Exiting."] git.build: build: ../git + depends_on: + - gitclient.build image: httpd-git entrypoint: ["echo", "This service was just a build job. Exiting."] githost: @@ -14,11 +16,26 @@ services: - git.build ports: - "80" - git2: - image: solsson/httpd@sha256:5bd6cc8c9c3a3dc5d9cd816a602dd7eeb576520343dc9521a594ea13f7ca9d33 - ports: - - "80" httpd: build: ../httpd-gitconf image: httpd-gitconf - #entrypoint: git-clones-then-httpd-foreground + volumes: + - ./gitconf-test:/gitconf-test + entrypoint: /gitconf-test/httpd-entrypoint.sh + conftest: + image: httpd-gitclient + depends_on: + - gitclient.build + labels: + com.yolean.build-contract: "*" + volumes: + - ./gitconf-test:/gitconf-test + entrypoint: /gitconf-test/reconf-test.sh + # if this container stops the test should fail, so it can't terminate like a com.yolean.build-contract + uptest: + image: httpd-gitclient + depends_on: + - gitclient.build + volumes: + - ./gitconf-test:/gitconf-test + entrypoint: /gitconf-test/up-test.sh From 8aede88110e6bdd881dec5933d6c82c46926b46b Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Thu, 22 Sep 2016 14:19:24 +0200 Subject: [PATCH 10/25] Refactors git httpd config to allow push, if auth is configured. Yolean/build-contract tests. --- build-contracts/docker-compose.yml | 29 ++++++++++- build-contracts/githost/Dockerfile | 21 +++++--- build-contracts/githost/auth-anon.conf | 11 ++++ build-contracts/githost/gitconfig | 3 -- build-contracts/perlspec/Dockerfile | 14 ++++++ build-contracts/test/git-http-spec.pl | 70 ++++++++++++++++++++++++++ git/Dockerfile | 6 ++- git/conf/git-access.conf | 18 +++++++ git/conf/git-readonly.conf | 6 --- git/conf/git.conf | 10 ++-- 10 files changed, 162 insertions(+), 26 deletions(-) create mode 100644 build-contracts/githost/auth-anon.conf delete mode 100644 build-contracts/githost/gitconfig create mode 100644 build-contracts/perlspec/Dockerfile create mode 100755 build-contracts/test/git-http-spec.pl create mode 100644 git/conf/git-access.conf delete mode 100644 git/conf/git-readonly.conf diff --git a/build-contracts/docker-compose.yml b/build-contracts/docker-compose.yml index d23efaf..deceace 100644 --- a/build-contracts/docker-compose.yml +++ b/build-contracts/docker-compose.yml @@ -10,24 +10,49 @@ services: - gitclient.build image: httpd-git entrypoint: ["echo", "This service was just a build job. Exiting."] + # httpd-git acceptance testing githost: build: ./githost depends_on: - git.build ports: - "80" + readonly: + build: ./githost + depends_on: + - git.build + ports: + - "80" + environment: + - GIT_READONLY=1 + test.git: + build: ./perlspec + labels: + com.yolean.build-contract: "*" + links: + - githost + - readonly + volumes: + - ./test:/test + command: /test/git-http-spec.pl + # httpd-gitconf acceptance testing httpd: build: ../httpd-gitconf image: httpd-gitconf + links: + - githost volumes: - ./gitconf-test:/gitconf-test - entrypoint: /gitconf-test/httpd-entrypoint.sh + entrypoint: /gitconf-test/httpd-entrypoint-gitconf conftest: image: httpd-gitclient depends_on: - gitclient.build labels: com.yolean.build-contract: "*" + links: + - githost + - httpd volumes: - ./gitconf-test:/gitconf-test entrypoint: /gitconf-test/reconf-test.sh @@ -36,6 +61,8 @@ services: image: httpd-gitclient depends_on: - gitclient.build + links: + - httpd volumes: - ./gitconf-test:/gitconf-test entrypoint: /gitconf-test/up-test.sh diff --git a/build-contracts/githost/Dockerfile b/build-contracts/githost/Dockerfile index 59fc9d4..cb48cdd 100644 --- a/build-contracts/githost/Dockerfile +++ b/build-contracts/githost/Dockerfile @@ -1,17 +1,22 @@ FROM httpd-git -#RUN rm conf/git/git-readonly.conf +RUN sed -i 's|^#LoadModule authn_anon_module|LoadModule authn_anon_module|' conf/httpd.conf -RUN mkdir -p /opt/git/Gitconf \ - && git init --bare /opt/git/Gitconf/conf.git \ - && git init --bare /opt/git/Gitconf/cert.git +COPY auth-anon.conf conf/git/ + +RUN mkdir -p /opt/git/Test \ + && git init --bare /opt/git/Test/test.git \ + && git init --bare /opt/git/Test/conf.git \ + && git init --bare /opt/git/Test/cert.git \ + && chown -R daemon /opt/git RUN git config --global user.email "you@example.com" \ && git config --global user.name "Your Name" \ - && git clone /opt/git/Gitconf/conf.git /tmp/conf \ + && git clone /opt/git/Test/conf.git /tmp/conf \ && cd /tmp/conf/ \ - && cp /usr/local/apache2/conf/httpd.conf /tmp/conf/ \ - && sed -i 's/^Include/#Include/' /tmp/conf/httpd.conf \ - && git add httpd.conf \ + && cp /usr/local/apache2/conf/httpd.conf . \ + && cp /usr/local/apache2/conf/mime.types . \ + && sed -i 's/^Include/#Include/' httpd.conf \ + && git add * \ && git commit -m "Gets httpd up and running" \ && git push origin master diff --git a/build-contracts/githost/auth-anon.conf b/build-contracts/githost/auth-anon.conf new file mode 100644 index 0000000..2f7de7a --- /dev/null +++ b/build-contracts/githost/auth-anon.conf @@ -0,0 +1,11 @@ + + AuthName "If visitors get this auth prompt you are at risk" + AuthType Basic + AuthBasicProvider anon + + Anonymous_NoUserID off + Anonymous_MustGiveEmail off + Anonymous_VerifyEmail off + Anonymous_LogEmail off + Anonymous "*" + diff --git a/build-contracts/githost/gitconfig b/build-contracts/githost/gitconfig deleted file mode 100644 index 29ace58..0000000 --- a/build-contracts/githost/gitconfig +++ /dev/null @@ -1,3 +0,0 @@ -[user] - email = you@example.com - name = Your Name diff --git a/build-contracts/perlspec/Dockerfile b/build-contracts/perlspec/Dockerfile new file mode 100644 index 0000000..b948122 --- /dev/null +++ b/build-contracts/perlspec/Dockerfile @@ -0,0 +1,14 @@ +FROM perl:5.24 + +# http://stackoverflow.com/questions/3462058/how-do-i-automate-cpan-configuration +RUN (echo y;echo o conf prerequisites_policy follow;echo o conf commit) | cpan + +RUN cpan install Test::Spec + +RUN git --version && cpan install Git::Wrapper + +RUN git config --global user.email "perlspec-testing@example.com" \ + && git config --global user.name "Perlspec Testint" + +# gotcha, /usr/bin/perl fails to include CPAN modules +ENTRYPOINT ["/usr/local/bin/perl", "-w"] diff --git a/build-contracts/test/git-http-spec.pl b/build-contracts/test/git-http-spec.pl new file mode 100755 index 0000000..ba18c41 --- /dev/null +++ b/build-contracts/test/git-http-spec.pl @@ -0,0 +1,70 @@ +#!/usr/bin/perl -w +use strict; + +use Test::Spec; + +# Didn't like this much, let's see if we need it +#use Git::Wrapper; +#my $git = Git::Wrapper->new('/tmp/test'); + +my $testkey = time(); +mkdir "/tmp/testrun-$testkey"; +chdir "/tmp/testrun-$testkey"; +print "# testrun /tmp/testrun-$testkey\n"; + +describe "Clone at /git/[org]/[repo]" => sub { + + it "Allowed" => sub { + `git clone http://githost/git/Test/test.git ./test`; + is($?, 0); + }; + + it "Produces a local repo" => sub { + ok(-e 'test/.git' and -d 'test/.git'); + }; + +}; + +describe "Readonly" => sub { + + it "Same clone behavior as regular host" => sub { + `git clone http://readonly/git/Test/test.git ./readonly`; + is($?, 0); + }; + + it "Same fetch" => sub { + `cd test/ && git remote add readonly http://readonly/git/Test/test.git && git fetch readonly`; + is($?, 0); + }; + + it "Denies push" => sub { + `cd test/ && echo test > test1.txt && git add test1.txt && git commit -m "Test 1"`; + is($?, 0); + `cd test/ && git push readonly master`; + isnt($?, 0); + }; + + # TODO test for status code 403 at GET /git/Test/test.git/info/refs?service=git-receive-pack + # as the test above passes for status 500 (i.e. auth not configured) too + # Or we can possibly just check for git auth attempt "fatal: could not read Username for 'http://readonly': No such device or address" + +}; + +describe "Push" => sub { + + it "Requires authentication (with default config, custom auth conf needed)" => sub { + `cd test/ && git push origin master`; + isnt($?, 0); + }; + + it "Test container runs mod_auth_anon so any username will do here" => sub { + `cd test/ && git remote add auth 'http://testuser:\@githost/git/Test/test.git' && git remote -v`; + ## more presistent auth + #`echo 'http://testuser:@githost' >> ~/.git-credentials` + #`cd test/ && git config credential.helper store && git push origin master`; + is($?, 0); + }; + +}; + +runtests unless caller; diff --git a/git/Dockerfile b/git/Dockerfile index 55c75a6..f4878a1 100644 --- a/git/Dockerfile +++ b/git/Dockerfile @@ -4,5 +4,9 @@ RUN sed -i 's|#LoadModule cgid_module|LoadModule cgid_module|' conf/httpd.conf \ && sed -i 's|#LoadModule rewrite_module|LoadModule rewrite_module|' conf/httpd.conf \ && echo "Include conf/git/*.conf" >> conf/httpd.conf +ENV GIT_PROJECT_ROOT="/opt/git" +ENV GIT_HTTP_EXPORT_ALL="1" +ENV GIT_READONLY="" + ADD conf/git.conf /usr/local/apache2/conf/git/ -ADD conf/git-readonly.conf /usr/local/apache2/conf/git/ +ADD conf/git-access.conf /usr/local/apache2/conf/git/ diff --git a/git/conf/git-access.conf b/git/conf/git-access.conf new file mode 100644 index 0000000..502a81d --- /dev/null +++ b/git/conf/git-access.conf @@ -0,0 +1,18 @@ + +RewriteEngine On + +RewriteCond %{QUERY_STRING} service=git-receive-pack [OR] +RewriteCond %{REQUEST_URI} /git-receive-pack$ +RewriteCond %{GIT_READONLY} !^$ +RewriteRule ^/git/ - [F] + +RewriteCond %{QUERY_STRING} service=git-receive-pack [OR] +RewriteCond %{REQUEST_URI} /git-receive-pack$ +RewriteRule ^/git/ - [E=AUTHREQUIRED] + + + Require valid-user + Order deny,allow + Deny from env=AUTHREQUIRED + Satisfy any + diff --git a/git/conf/git-readonly.conf b/git/conf/git-readonly.conf deleted file mode 100644 index d4926c2..0000000 --- a/git/conf/git-readonly.conf +++ /dev/null @@ -1,6 +0,0 @@ - - - Order deny,allow - Deny from env=GITWRITE - Satisfy any - diff --git a/git/conf/git.conf b/git/conf/git.conf index 9c402a3..3f829c8 100644 --- a/git/conf/git.conf +++ b/git/conf/git.conf @@ -1,13 +1,9 @@ -SetEnv GIT_PROJECT_ROOT /opt/git -SetEnv GIT_HTTP_EXPORT_ALL 1 +PassEnv GIT_PROJECT_ROOT +PassEnv GIT_HTTP_EXPORT_ALL + ScriptAlias /git/ /usr/libexec/git-core/git-http-backend/ Options +ExecCGI Require all granted - -RewriteEngine On -RewriteCond %{QUERY_STRING} service=git-receive-pack [OR] -RewriteCond %{REQUEST_URI} /git-receive-pack$ -RewriteRule ^/git/ - [E=AUTHREQUIRED] From 72a8d4b10d85955699d6efc54e1b8a776c7d84a1 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Thu, 22 Sep 2016 14:21:32 +0200 Subject: [PATCH 11/25] wip httpd with conf from git clone --- .../gitconf-test/httpd-entrypoint-gitconf | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100755 build-contracts/gitconf-test/httpd-entrypoint-gitconf diff --git a/build-contracts/gitconf-test/httpd-entrypoint-gitconf b/build-contracts/gitconf-test/httpd-entrypoint-gitconf new file mode 100755 index 0000000..71b2150 --- /dev/null +++ b/build-contracts/gitconf-test/httpd-entrypoint-gitconf @@ -0,0 +1,12 @@ +#!/usr/bin/perl -w +use strict; + +`git clone http://githost/git/Test/conf.git /tmp/conf`; $? == 0 or die; +`git clone http://githost/git/Test/cert.git /tmp/cert`; $? == 0 or die; +`rm /usr/local/apache2/conf -Rf`; $? == 0 or die; +`mv /tmp/conf /usr/local/apache2/conf`; $? == 0 or die; +`mv /tmp/cert /usr/local/apache2/cert`; $? == 0 or die; +`apachectl configtest`; $? == 0 or die; + +# now the real entrypoint +exec 'httpd-foreground'; From 62c2c10204b064106faa0f01bc98a2e89df58515 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 23 Sep 2016 13:01:06 +0200 Subject: [PATCH 12/25] Starts HTTP level specs for the httpd-reconf concept --- build-contracts/docker-compose.yml | 18 +++--------------- build-contracts/perlspec/Dockerfile | 2 ++ .../{conf-reload => Httpd-Conf-Reload.pm} | 5 +++++ 3 files changed, 10 insertions(+), 15 deletions(-) rename httpd-gitconf/{conf-reload => Httpd-Conf-Reload.pm} (79%) diff --git a/build-contracts/docker-compose.yml b/build-contracts/docker-compose.yml index deceace..40a13eb 100644 --- a/build-contracts/docker-compose.yml +++ b/build-contracts/docker-compose.yml @@ -44,10 +44,8 @@ services: volumes: - ./gitconf-test:/gitconf-test entrypoint: /gitconf-test/httpd-entrypoint-gitconf - conftest: - image: httpd-gitclient - depends_on: - - gitclient.build + test.reconf: + build: ./perlspec labels: com.yolean.build-contract: "*" links: @@ -55,14 +53,4 @@ services: - httpd volumes: - ./gitconf-test:/gitconf-test - entrypoint: /gitconf-test/reconf-test.sh - # if this container stops the test should fail, so it can't terminate like a com.yolean.build-contract - uptest: - image: httpd-gitclient - depends_on: - - gitclient.build - links: - - httpd - volumes: - - ./gitconf-test:/gitconf-test - entrypoint: /gitconf-test/up-test.sh + command: /gitconf-test/reconf-spec.pl diff --git a/build-contracts/perlspec/Dockerfile b/build-contracts/perlspec/Dockerfile index b948122..9008eeb 100644 --- a/build-contracts/perlspec/Dockerfile +++ b/build-contracts/perlspec/Dockerfile @@ -5,6 +5,8 @@ RUN (echo y;echo o conf prerequisites_policy follow;echo o conf commit) | cpan RUN cpan install Test::Spec +RUN cpan install HTTP::Tiny + RUN git --version && cpan install Git::Wrapper RUN git config --global user.email "perlspec-testing@example.com" \ diff --git a/httpd-gitconf/conf-reload b/httpd-gitconf/Httpd-Conf-Reload.pm similarity index 79% rename from httpd-gitconf/conf-reload rename to httpd-gitconf/Httpd-Conf-Reload.pm index cfc5f45..6a9c1db 100644 --- a/httpd-gitconf/conf-reload +++ b/httpd-gitconf/Httpd-Conf-Reload.pm @@ -1,3 +1,8 @@ +#!/usr/bin/perl -w +use strict; + +use Httpd-Conf-Reload; + # timestamp RUNLABEL=() From 047933a658c8417e1e4c3f695fabf960640b85fb Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 23 Sep 2016 14:34:30 +0200 Subject: [PATCH 13/25] Automatic test runs would be great for development --- build-contracts/perlspec/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/build-contracts/perlspec/Dockerfile b/build-contracts/perlspec/Dockerfile index 9008eeb..fa37528 100644 --- a/build-contracts/perlspec/Dockerfile +++ b/build-contracts/perlspec/Dockerfile @@ -7,6 +7,8 @@ RUN cpan install Test::Spec RUN cpan install HTTP::Tiny +RUN cpan install App::prowess + RUN git --version && cpan install Git::Wrapper RUN git config --global user.email "perlspec-testing@example.com" \ From b70bf92ba5de2d82e166ad1ddfeb3732462e044c Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 23 Sep 2016 14:39:34 +0200 Subject: [PATCH 14/25] Test::Continuous worked with Docker for Mac, App::prowess didn't --- build-contracts/perlspec/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-contracts/perlspec/Dockerfile b/build-contracts/perlspec/Dockerfile index fa37528..e331c80 100644 --- a/build-contracts/perlspec/Dockerfile +++ b/build-contracts/perlspec/Dockerfile @@ -7,7 +7,7 @@ RUN cpan install Test::Spec RUN cpan install HTTP::Tiny -RUN cpan install App::prowess +RUN cpan install Test::Continuous RUN git --version && cpan install Git::Wrapper From 5b7b3a15c630d690057ebf73fad98cb9800ba832 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 23 Sep 2016 15:43:29 +0200 Subject: [PATCH 15/25] Just switch entrypoint to get continous testing. Validated with both libs. --- build-contracts/docker-compose.yml | 6 ++-- build-contracts/gitconf-test/reconf-spec.t | 41 ++++++++++++++++++++++ build-contracts/perlspec/Dockerfile | 13 +++++-- 3 files changed, 55 insertions(+), 5 deletions(-) create mode 100755 build-contracts/gitconf-test/reconf-spec.t diff --git a/build-contracts/docker-compose.yml b/build-contracts/docker-compose.yml index 40a13eb..ea67220 100644 --- a/build-contracts/docker-compose.yml +++ b/build-contracts/docker-compose.yml @@ -52,5 +52,7 @@ services: - githost - httpd volumes: - - ./gitconf-test:/gitconf-test - command: /gitconf-test/reconf-spec.pl + - ./gitconf-test:/project/t + # watch + #entrypoint: autoprove + #entrypoint: prowess diff --git a/build-contracts/gitconf-test/reconf-spec.t b/build-contracts/gitconf-test/reconf-spec.t new file mode 100755 index 0000000..bca9a3f --- /dev/null +++ b/build-contracts/gitconf-test/reconf-spec.t @@ -0,0 +1,41 @@ +#!/usr/bin/perl -w +use strict; + +use Test::Spec; + +my $testkey = time(); +mkdir "/tmp/testrun-$testkey"; +chdir "/tmp/testrun-$testkey"; +print "# testrun /tmp/testrun-$testkey\n"; + +use HTTP::Tiny; + +my $r; + +describe "Httpd state at container startup" => sub { + + it "Should be running" => sub { + $r = HTTP::Tiny->new->head('http://httpd/'); + is($r->{status}, 200); + }; + + it "Should have a typical 404 error page" => sub { + $r = HTTP::Tiny->new->head('http://httpd/testing/notfound'); + is($r->{status}, 404); + isnt($r->{content}, 'Custom.'); + }; + +}; + +describe "A shared git remote" => sub { + + it "Is alive" => sub { + $r = HTTP::Tiny->new->head('http://githost/'); + is($r->{status}, 200); + }; + +}; + + + +runtests unless caller; diff --git a/build-contracts/perlspec/Dockerfile b/build-contracts/perlspec/Dockerfile index e331c80..8f9303e 100644 --- a/build-contracts/perlspec/Dockerfile +++ b/build-contracts/perlspec/Dockerfile @@ -7,12 +7,19 @@ RUN cpan install Test::Spec RUN cpan install HTTP::Tiny -RUN cpan install Test::Continuous +# We need one of these and I don't know which one is more stable or useful yet +# To evaluate, toggle entrypoint between prove/autoprove/prowess +RUN cpan install Test::Continuous App::prowess RUN git --version && cpan install Git::Wrapper RUN git config --global user.email "perlspec-testing@example.com" \ && git config --global user.name "Perlspec Testint" -# gotcha, /usr/bin/perl fails to include CPAN modules -ENTRYPOINT ["/usr/local/bin/perl", "-w"] +# Official perl image gotcha, /usr/bin/perl fails to include CPAN modules +RUN mv /usr/bin/perl /usr/bin/perl.org \ + && ln -s /usr/local/bin/perl /usr/bin/perl + +WORKDIR /project + +ENTRYPOINT ["prove"] From 52b22b2818cc0c2f6137b6fd6b473f73d7ca5cdb Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 23 Sep 2016 15:50:54 +0200 Subject: [PATCH 16/25] Adapt to new perlspec entrypoint --- build-contracts/docker-compose.yml | 3 +-- build-contracts/test/{git-http-spec.pl => git-http-spec.t} | 0 2 files changed, 1 insertion(+), 2 deletions(-) rename build-contracts/test/{git-http-spec.pl => git-http-spec.t} (100%) diff --git a/build-contracts/docker-compose.yml b/build-contracts/docker-compose.yml index ea67220..3d2c7af 100644 --- a/build-contracts/docker-compose.yml +++ b/build-contracts/docker-compose.yml @@ -33,8 +33,7 @@ services: - githost - readonly volumes: - - ./test:/test - command: /test/git-http-spec.pl + - ./test:/project/t # httpd-gitconf acceptance testing httpd: build: ../httpd-gitconf diff --git a/build-contracts/test/git-http-spec.pl b/build-contracts/test/git-http-spec.t similarity index 100% rename from build-contracts/test/git-http-spec.pl rename to build-contracts/test/git-http-spec.t From 4e825c1c7e790491dde3c50e9be3a85a0719fc21 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 23 Sep 2016 17:19:27 +0200 Subject: [PATCH 17/25] Starts a script concept for the reconf operation --- build-contracts/docker-compose.yml | 3 +- build-contracts/gitconf-test/reconf-spec.t | 44 +++++++++++++++++++ build-contracts/perlspec/Dockerfile | 2 +- httpd-gitconf/HttpdReconf.pm | 29 ++++++++++++ .../{Httpd-Conf-Reload.pm => httpd-reconf} | 12 ++++- 5 files changed, 87 insertions(+), 3 deletions(-) create mode 100644 httpd-gitconf/HttpdReconf.pm rename httpd-gitconf/{Httpd-Conf-Reload.pm => httpd-reconf} (58%) mode change 100644 => 100755 diff --git a/build-contracts/docker-compose.yml b/build-contracts/docker-compose.yml index 3d2c7af..a83025d 100644 --- a/build-contracts/docker-compose.yml +++ b/build-contracts/docker-compose.yml @@ -42,6 +42,7 @@ services: - githost volumes: - ./gitconf-test:/gitconf-test + - ../httpd-gitconf:/perldev entrypoint: /gitconf-test/httpd-entrypoint-gitconf test.reconf: build: ./perlspec @@ -52,6 +53,6 @@ services: - httpd volumes: - ./gitconf-test:/project/t - # watch + # Of these test watch tools prowess' terminal usage seems to work better with docker-compose run, but neither understands Ctrl+C so you need docker kill #entrypoint: autoprove #entrypoint: prowess diff --git a/build-contracts/gitconf-test/reconf-spec.t b/build-contracts/gitconf-test/reconf-spec.t index bca9a3f..1f71268 100755 --- a/build-contracts/gitconf-test/reconf-spec.t +++ b/build-contracts/gitconf-test/reconf-spec.t @@ -9,6 +9,7 @@ chdir "/tmp/testrun-$testkey"; print "# testrun /tmp/testrun-$testkey\n"; use HTTP::Tiny; +use JSON; my $r; @@ -34,6 +35,49 @@ describe "A shared git remote" => sub { is($r->{status}, 200); }; + it "Has a conf repo to clone" => sub { + `git clone http://githost/git/Test/conf.git`; + is($?, 0); + ok(-e 'conf/.git'); + }; + + it "Has a cert repo to clone" => sub { + `git clone http://githost/git/Test/cert.git`; + is($?, 0); + ok(-e 'conf/.git'); + }; + +}; + +describe "Extenal conf modification over git" => sub { + + it "Add a simple one liner that can be detected over HTTP" => sub { + `echo 'ErrorDocument 404 "Custom."' >> conf/httpd.conf`; + is($?, 0); + `cd conf/; git add httpd.conf; git commit -m "Change 404 page"`; + is($?, 0); + }; + + it "Trigger httpd reconf using REST endpoint" => sub { + my $http = HTTP::Tiny->new(); + my $r = $http->post( + 'http://httpd/admin/reconf' => { + content => to_json( + {} + ), + headers => { + 'Accept' => 'application/json', + }, + }, + ); + is($r->{status}, 200); + }; + + it "Shuld now have affected httpd's runtime conf" => sub { + $r = HTTP::Tiny->new->head('http://httpd/testing/notfound'); + is($r->{content}, 'Custom.'); + }; + }; diff --git a/build-contracts/perlspec/Dockerfile b/build-contracts/perlspec/Dockerfile index 8f9303e..1fd1df0 100644 --- a/build-contracts/perlspec/Dockerfile +++ b/build-contracts/perlspec/Dockerfile @@ -5,7 +5,7 @@ RUN (echo y;echo o conf prerequisites_policy follow;echo o conf commit) | cpan RUN cpan install Test::Spec -RUN cpan install HTTP::Tiny +RUN cpan install HTTP::Tiny JSON # We need one of these and I don't know which one is more stable or useful yet # To evaluate, toggle entrypoint between prove/autoprove/prowess diff --git a/httpd-gitconf/HttpdReconf.pm b/httpd-gitconf/HttpdReconf.pm new file mode 100644 index 0000000..7eefb48 --- /dev/null +++ b/httpd-gitconf/HttpdReconf.pm @@ -0,0 +1,29 @@ +package HttpdReconf; +use strict; +use warnings; + +use Exporter qw(import); + +our @EXPORT_OK = qw(refresh configtest reload); + +my $httpdhome = '/usr/local/apache2'; + +sub refresh { + `cd $httpdhome/conf; git pull --rebase`; + $? == 0 or return 0; + #if (-e "$httpdhome/cert" and -d "$httpdhome/cert") { + # `cd $httpdhome/cert; git pull --rebase`; + # $? == 0 or return 0; + #} + return 1; +}; + +sub configtest { + print "Configtest!\n"; +}; + +sub reload { + print "Reload!\n" +}; + +1; diff --git a/httpd-gitconf/Httpd-Conf-Reload.pm b/httpd-gitconf/httpd-reconf old mode 100644 new mode 100755 similarity index 58% rename from httpd-gitconf/Httpd-Conf-Reload.pm rename to httpd-gitconf/httpd-reconf index 6a9c1db..9b51ad8 --- a/httpd-gitconf/Httpd-Conf-Reload.pm +++ b/httpd-gitconf/httpd-reconf @@ -1,8 +1,16 @@ #!/usr/bin/perl -w + use strict; -use Httpd-Conf-Reload; +use HttpdReconf qw(refresh configtest reload); + +refresh() or die('Refresh failed'); + +configtest() or die('Configtest failed'); +reload() or die('Configtest failed'); + +=pod # timestamp RUNLABEL=() @@ -17,3 +25,5 @@ function rollback apachectl configtest || rollback apachectl graceful || rollback + +=cut From 06abc1348e129b099d8c7971846e33539144318b Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sat, 24 Sep 2016 06:22:11 +0200 Subject: [PATCH 18/25] The build+FROM gotcha --- build-contracts/docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/build-contracts/docker-compose.yml b/build-contracts/docker-compose.yml index a83025d..97b2d81 100644 --- a/build-contracts/docker-compose.yml +++ b/build-contracts/docker-compose.yml @@ -38,6 +38,8 @@ services: httpd: build: ../httpd-gitconf image: httpd-gitconf + depends_on: + - gitclient.build links: - githost volumes: From 5ed59995c28c0671c13439f23553af9a965a9f5b Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sat, 24 Sep 2016 06:23:15 +0200 Subject: [PATCH 19/25] No runtimeDeps may exist in buildDeps. Regarding ca-certificates they'are of little value in the image because they get outdated. --- gitclient/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitclient/Dockerfile b/gitclient/Dockerfile index 0619af2..8b5be52 100644 --- a/gitclient/Dockerfile +++ b/gitclient/Dockerfile @@ -14,7 +14,7 @@ RUN depsRuntime=' \ libssl1.0.0 \ ' \ && depsBuild=' \ - curl ca-certificates \ + ca-certificates \ gcc \ make \ autoconf \ From ab42aeb0e9beeb18109b3e9ab42c7c30c4144d65 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sat, 24 Sep 2016 20:16:06 +0200 Subject: [PATCH 20/25] Make som kind of lib for the various tasks invovled, experimenting with logging and system exec --- httpd-gitconf/Dockerfile | 27 ++++++++++++++++++++++++++- httpd-gitconf/HttpdControl.pm | 27 +++++++++++++++++++++++++++ httpd-gitconf/HttpdReconf.pm | 29 ----------------------------- httpd-gitconf/httpd-reconf | 27 +++++++++++++++++++++++++-- 4 files changed, 78 insertions(+), 32 deletions(-) create mode 100644 httpd-gitconf/HttpdControl.pm delete mode 100644 httpd-gitconf/HttpdReconf.pm diff --git a/httpd-gitconf/Dockerfile b/httpd-gitconf/Dockerfile index 2f07d22..ee8e5ce 100644 --- a/httpd-gitconf/Dockerfile +++ b/httpd-gitconf/Dockerfile @@ -1,4 +1,29 @@ # TODO but we need openidc here too - FROM httpd-gitclient + +# Needed during development at least +RUN mkdir -p /usr/local/lib/site_perl/Data \ + && curl http://api.metacpan.org/source/SMUELLER/Data-Dumper-2.161/Dumper.pm -o /usr/local/lib/site_perl/Data/Dumper.pm \ + && echo "c3d1692479123e7c21d3e47a08550fde6fcbcbdf /usr/local/lib/site_perl/Data/Dumper.pm" | sha1sum -c - + +# Evaluated this logging lib +#RUN mkdir -p /usr/local/lib/site_perl/CGI \ +# && curl http://api.metacpan.org/source/JMOORE/CGI-Log-1.00/Log.pm -o /usr/local/lib/site_perl/CGI/Log.pm \ +# && echo "d02ad2ad622ee1953f51e0c74b9af52a94bb6d0b /usr/local/lib/site_perl/CGI/Log.pm" | sha1sum -c - + +# Found a logging lib that is a simple abstraction on print and also replaces die, without the dependencies of Carp +RUN mkdir -p /usr/local/lib/site_perl/Scalar \ + && curl http://api.metacpan.org/source/PEVANS/Scalar-List-Utils-1.45/lib/Scalar/Util.pm -o /usr/local/lib/site_perl/Scalar/Util.pm \ + && echo "a85497bb2f8979b6eb76cfdc7bd7dc4bcc70b64e /usr/local/lib/site_perl/Scalar/Util.pm" | sha1sum -c - \ + && mkdir -p /usr/local/lib/site_perl/Term \ + && curl http://api.metacpan.org/source/RRA/Term-ANSIColor-4.05/lib/Term/ANSIColor.pm -o /usr/local/lib/site_perl/Term/ANSIColor.pm \ + && echo "07499818b26ab025d726e56b2c798cee14ad61a6 /usr/local/lib/site_perl/Term/ANSIColor.pm" | sha1sum -c - \ + && mkdir -p /usr/local/lib/site_perl/Log \ + && curl http://api.metacpan.org/source/KAZEBURO/Log-Minimal-0.19/lib/Log/Minimal.pm -o /usr/local/lib/site_perl/Log/Minimal.pm \ + && echo "917f7f526e286d7ae684d6a2e7468729d500f7a3 /usr/local/lib/site_perl/Log/Minimal.pm" | sha1sum -c - + +COPY HttpdControl.pm /usr/local/lib/site_perl/ +COPY ReconfDirGit.pm /usr/local/lib/site_perl/ + +COPY httpd-reconf /usr/local/bin/ diff --git a/httpd-gitconf/HttpdControl.pm b/httpd-gitconf/HttpdControl.pm new file mode 100644 index 0000000..5b4ecf9 --- /dev/null +++ b/httpd-gitconf/HttpdControl.pm @@ -0,0 +1,27 @@ +package HttpdControl; + +use strict; +use warnings; + +sub new { + my ($class, %args) = @_; + return bless { %args }, $class; +} + +sub configtest { + my ($self) = @_; + my $out = `apachectl configtest`; + my $result = $?; + print $out; + return $result == 0; +} + +sub reload { + my ($self) = @_; + my $out = `apachectl graceful`; + my $result = $?; + print $out; + return $result == 0; +} + +1; diff --git a/httpd-gitconf/HttpdReconf.pm b/httpd-gitconf/HttpdReconf.pm deleted file mode 100644 index 7eefb48..0000000 --- a/httpd-gitconf/HttpdReconf.pm +++ /dev/null @@ -1,29 +0,0 @@ -package HttpdReconf; -use strict; -use warnings; - -use Exporter qw(import); - -our @EXPORT_OK = qw(refresh configtest reload); - -my $httpdhome = '/usr/local/apache2'; - -sub refresh { - `cd $httpdhome/conf; git pull --rebase`; - $? == 0 or return 0; - #if (-e "$httpdhome/cert" and -d "$httpdhome/cert") { - # `cd $httpdhome/cert; git pull --rebase`; - # $? == 0 or return 0; - #} - return 1; -}; - -sub configtest { - print "Configtest!\n"; -}; - -sub reload { - print "Reload!\n" -}; - -1; diff --git a/httpd-gitconf/httpd-reconf b/httpd-gitconf/httpd-reconf index 9b51ad8..bfff92a 100755 --- a/httpd-gitconf/httpd-reconf +++ b/httpd-gitconf/httpd-reconf @@ -1,12 +1,35 @@ #!/usr/bin/perl -w - use strict; +use warnings; + +use ReconfDirGit; +use HttpdControl; + +my $conf = ReconfDirGit->new( dir => '/usr/local/apache2/conf' ); +print $conf->dir(); +print ": current rev = "; +print $conf->rev(); +print "\n"; + +my $control = HttpdControl->new(); + +if ($control->configtest()) { + print " - Config is OK.\n"; +} else { + die("TODO revert to last known good config"); +} + +$conf->mark_good(); + +$conf->pull_rebase(); + +print "Continued?"; use HttpdReconf qw(refresh configtest reload); refresh() or die('Refresh failed'); -configtest() or die('Configtest failed'); + reload() or die('Configtest failed'); From d3284dea528c154b2221c54c6808189432d0fd75 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sat, 24 Sep 2016 20:46:42 +0200 Subject: [PATCH 21/25] lotsa gotchas --- httpd-gitconf/ReconfDirGit.pm | 54 +++++++++++++++++++++++++++++++++++ httpd-gitconf/httpd-reconf | 13 +++------ 2 files changed, 58 insertions(+), 9 deletions(-) create mode 100644 httpd-gitconf/ReconfDirGit.pm diff --git a/httpd-gitconf/ReconfDirGit.pm b/httpd-gitconf/ReconfDirGit.pm new file mode 100644 index 0000000..1275ee2 --- /dev/null +++ b/httpd-gitconf/ReconfDirGit.pm @@ -0,0 +1,54 @@ +package ReconfDirGit; + +use strict; +use warnings; + +use Log::Minimal env_debug => 'DEBUG'; + +sub new { + my ($class, %args) = @_; + return bless { %args }, $class; +} + +sub dir { + my ($self) = @_; + return $self->{dir}; +} + +sub rev { + my ($self) = @_; + my $rev = `cd $self->{dir} && git rev-parse --verify HEAD`; + ($? == 0) or die('TODO throw exception instead'); + chomp($rev); + return $rev; +} + +sub branch { + my ($self) = @_; + my $branch = `cd $self->{dir} && git rev-parse --abbrev-ref HEAD`; + $? == 0 or die('Failed to read current branch'); + chomp($branch); + return $branch; +} + +sub mark_good { + my ($self) = @_; + my $current = $self->branch(); + debugf(`cd $self->{dir} && git checkout -B reconf-last-known-good-configuration && git checkout $current`); + ($? == 0) or die('TODO throw exception instead'); +} + +sub pull_rebase { + my ($self) = @_; + my $rev = $self->rev(); + my $current = $self->branch(); + debugf("cd $self->{dir} && git rev-parse --verify $self->{remote}/$current"); + my $remoterev = `cd $self->{dir} && git rev-parse --verify $self->{remote}/$current`; + chomp($remoterev); + ($rev eq $remoterev) or croakf("Local rev $rev is out of sync with $self->{remote}/$current $remoterev"); + debugf("$current == $self->{remote}/$current == $rev"); + `cd $self->{dir} && git fetch $self->{remote} && git rebase $self->{remote}/$current`; + ($? == 0) or croakf('TODO throw exception instead'); +} + +1; diff --git a/httpd-gitconf/httpd-reconf b/httpd-gitconf/httpd-reconf index bfff92a..85a53cf 100755 --- a/httpd-gitconf/httpd-reconf +++ b/httpd-gitconf/httpd-reconf @@ -5,7 +5,10 @@ use warnings; use ReconfDirGit; use HttpdControl; -my $conf = ReconfDirGit->new( dir => '/usr/local/apache2/conf' ); +my $conf = ReconfDirGit->new( + dir => '/usr/local/apache2/conf', + remote => 'origin' +); print $conf->dir(); print ": current rev = "; print $conf->rev(); @@ -25,14 +28,6 @@ $conf->pull_rebase(); print "Continued?"; -use HttpdReconf qw(refresh configtest reload); - -refresh() or die('Refresh failed'); - - - -reload() or die('Configtest failed'); - =pod # timestamp From 136ec8ccd49893b0688e49fbd42d180754dd9261 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sun, 25 Sep 2016 05:53:48 +0200 Subject: [PATCH 22/25] Allows logging level to be increased using the same env as with node.js --- build-contracts/docker-compose.yml | 2 ++ httpd-gitconf/HttpdControl.pm | 2 ++ httpd-gitconf/httpd-reconf | 2 ++ 3 files changed, 6 insertions(+) diff --git a/build-contracts/docker-compose.yml b/build-contracts/docker-compose.yml index 97b2d81..532426f 100644 --- a/build-contracts/docker-compose.yml +++ b/build-contracts/docker-compose.yml @@ -46,6 +46,8 @@ services: - ./gitconf-test:/gitconf-test - ../httpd-gitconf:/perldev entrypoint: /gitconf-test/httpd-entrypoint-gitconf + environment: + - DEBUG=* test.reconf: build: ./perlspec labels: diff --git a/httpd-gitconf/HttpdControl.pm b/httpd-gitconf/HttpdControl.pm index 5b4ecf9..88dbec2 100644 --- a/httpd-gitconf/HttpdControl.pm +++ b/httpd-gitconf/HttpdControl.pm @@ -3,6 +3,8 @@ package HttpdControl; use strict; use warnings; +use Log::Minimal env_debug => 'DEBUG'; + sub new { my ($class, %args) = @_; return bless { %args }, $class; diff --git a/httpd-gitconf/httpd-reconf b/httpd-gitconf/httpd-reconf index 85a53cf..a03dde0 100755 --- a/httpd-gitconf/httpd-reconf +++ b/httpd-gitconf/httpd-reconf @@ -2,6 +2,8 @@ use strict; use warnings; +use Log::Minimal env_debug => 'DEBUG'; + use ReconfDirGit; use HttpdControl; From 00c575d67590bcbcb39f4fb8cde913203f9351e1 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sun, 25 Sep 2016 05:54:15 +0200 Subject: [PATCH 23/25] Something of a complete, but untested, workflow --- httpd-gitconf/HttpdControl.pm | 3 ++- httpd-gitconf/ReconfDirGit.pm | 20 ++++++++++++++------ httpd-gitconf/httpd-reconf | 28 +++++++++++++++++++++++----- 3 files changed, 39 insertions(+), 12 deletions(-) diff --git a/httpd-gitconf/HttpdControl.pm b/httpd-gitconf/HttpdControl.pm index 88dbec2..4e0f415 100644 --- a/httpd-gitconf/HttpdControl.pm +++ b/httpd-gitconf/HttpdControl.pm @@ -14,7 +14,8 @@ sub configtest { my ($self) = @_; my $out = `apachectl configtest`; my $result = $?; - print $out; + chomp($out); + debugf($out); return $result == 0; } diff --git a/httpd-gitconf/ReconfDirGit.pm b/httpd-gitconf/ReconfDirGit.pm index 1275ee2..93b2e81 100644 --- a/httpd-gitconf/ReconfDirGit.pm +++ b/httpd-gitconf/ReconfDirGit.pm @@ -18,7 +18,7 @@ sub dir { sub rev { my ($self) = @_; my $rev = `cd $self->{dir} && git rev-parse --verify HEAD`; - ($? == 0) or die('TODO throw exception instead'); + ($? == 0) or croakf("Failed to read current rev: $rev"); chomp($rev); return $rev; } @@ -26,8 +26,9 @@ sub rev { sub branch { my ($self) = @_; my $branch = `cd $self->{dir} && git rev-parse --abbrev-ref HEAD`; - $? == 0 or die('Failed to read current branch'); + $? == 0 or croakf("Failed to read current branch: $branch"); chomp($branch); + $branch or croakf("Falsey branch name at ".$self->rev()); return $branch; } @@ -35,10 +36,11 @@ sub mark_good { my ($self) = @_; my $current = $self->branch(); debugf(`cd $self->{dir} && git checkout -B reconf-last-known-good-configuration && git checkout $current`); - ($? == 0) or die('TODO throw exception instead'); + ($? == 0) or croakf('Branch last good conf failed at $current'); + debugf("reconf-last-known-good-configuration saved at ".$self->rev()); } -sub pull_rebase { +sub fetch_rebase { my ($self) = @_; my $rev = $self->rev(); my $current = $self->branch(); @@ -47,8 +49,14 @@ sub pull_rebase { chomp($remoterev); ($rev eq $remoterev) or croakf("Local rev $rev is out of sync with $self->{remote}/$current $remoterev"); debugf("$current == $self->{remote}/$current == $rev"); - `cd $self->{dir} && git fetch $self->{remote} && git rebase $self->{remote}/$current`; - ($? == 0) or croakf('TODO throw exception instead'); + debugf(`cd $self->{dir} && git fetch $self->{remote} && git rebase $self->{remote}/$current`); + ($? == 0) or croakf("Fetch + rebase failed for $self->{remote}/$current"); +} + +sub revert_to_good { + my ($self) = @_; + debugf(`git checkout reconf-last-known-good-configuration`); + ($? == 0) or croakf("Revert failed. Now at ".$self->rev()); } 1; diff --git a/httpd-gitconf/httpd-reconf b/httpd-gitconf/httpd-reconf index a03dde0..afbeffa 100755 --- a/httpd-gitconf/httpd-reconf +++ b/httpd-gitconf/httpd-reconf @@ -13,22 +13,40 @@ my $conf = ReconfDirGit->new( ); print $conf->dir(); print ": current rev = "; -print $conf->rev(); -print "\n"; +my $start = $conf->rev(); +print "$start\n"; my $control = HttpdControl->new(); if ($control->configtest()) { - print " - Config is OK.\n"; + #print " - Config is OK.\n"; } else { die("TODO revert to last known good config"); } $conf->mark_good(); -$conf->pull_rebase(); +if (!$conf->fetch_rebase()) { -print "Continued?"; + critf("Failed to refresh configuration from remote"); + +} else { + + my $after = $conf->rev(); + debugf("Rebase done. At $after."); + + if ($control->configtest()) { + $control->reload(); + infof("Config reloaded successfuly at $after"); + # Keep last-known-good in case monitoring detects a regression + } else { + warnf("Invalid configuration at $after, reverting"); + $conf->revert_to_good(); + } + +} + +infof "Done."; =pod From 8dc6fd50244a707bb687145f211d41a20290785b Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sun, 25 Sep 2016 10:51:56 +0200 Subject: [PATCH 24/25] Expose port for testing --- build-contracts/docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/build-contracts/docker-compose.yml b/build-contracts/docker-compose.yml index 532426f..36ac8b7 100644 --- a/build-contracts/docker-compose.yml +++ b/build-contracts/docker-compose.yml @@ -46,6 +46,8 @@ services: - ./gitconf-test:/gitconf-test - ../httpd-gitconf:/perldev entrypoint: /gitconf-test/httpd-entrypoint-gitconf + ports: + - "80" environment: - DEBUG=* test.reconf: From 2374b7bcb666e38d2438f21ae14993b3ece69224 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sun, 25 Sep 2016 10:53:00 +0200 Subject: [PATCH 25/25] Support fixing invalid conf by pushing another git commit from somewhere --- httpd-gitconf/ReconfDirGit.pm | 9 ++++++--- httpd-gitconf/httpd-reconf | 10 ++-------- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/httpd-gitconf/ReconfDirGit.pm b/httpd-gitconf/ReconfDirGit.pm index 93b2e81..292865e 100644 --- a/httpd-gitconf/ReconfDirGit.pm +++ b/httpd-gitconf/ReconfDirGit.pm @@ -35,9 +35,10 @@ sub branch { sub mark_good { my ($self) = @_; my $current = $self->branch(); - debugf(`cd $self->{dir} && git checkout -B reconf-last-known-good-configuration && git checkout $current`); + ($current =~ /^reconf_last-known-good_/) and die("Invalid state. Current branch is the checkpoint $current"); + debugf(`cd $self->{dir} && git checkout -B _reconf_last-known-good_$current && git checkout $current`); ($? == 0) or croakf('Branch last good conf failed at $current'); - debugf("reconf-last-known-good-configuration saved at ".$self->rev()); + debugf("_reconf_last-known-good_$current saved at ".$self->rev()); } sub fetch_rebase { @@ -55,7 +56,9 @@ sub fetch_rebase { sub revert_to_good { my ($self) = @_; - debugf(`git checkout reconf-last-known-good-configuration`); + my $current = $self->branch(); + ($current =~ /^reconf_last-known-good_/) and die("Current branch is already at the checkpoint $current"); + debugf(`cd $self->{dir} && git checkout _reconf_last-known-good_$current && git checkout -B $current`); ($? == 0) or croakf("Revert failed. Now at ".$self->rev()); } diff --git a/httpd-gitconf/httpd-reconf b/httpd-gitconf/httpd-reconf index afbeffa..af66880 100755 --- a/httpd-gitconf/httpd-reconf +++ b/httpd-gitconf/httpd-reconf @@ -19,19 +19,14 @@ print "$start\n"; my $control = HttpdControl->new(); if ($control->configtest()) { - #print " - Config is OK.\n"; + $conf->mark_good(); } else { - die("TODO revert to last known good config"); + warnf("Starting from invalid config. There's hopefully a _reconf_last-known-good_ branch already."); } -$conf->mark_good(); - if (!$conf->fetch_rebase()) { - critf("Failed to refresh configuration from remote"); - } else { - my $after = $conf->rev(); debugf("Rebase done. At $after."); @@ -43,7 +38,6 @@ if (!$conf->fetch_rebase()) { warnf("Invalid configuration at $after, reverting"); $conf->revert_to_good(); } - } infof "Done.";