diff --git a/build-contracts/docker-compose.yml b/build-contracts/docker-compose.yml new file mode 100644 index 0000000..36ac8b7 --- /dev/null +++ b/build-contracts/docker-compose.yml @@ -0,0 +1,64 @@ +version: '2' +services: + gitclient.build: + build: ../gitclient + image: httpd-gitclient + entrypoint: ["echo", "This service was just a build job. Exiting."] + git.build: + build: ../git + depends_on: + - gitclient.build + image: httpd-git + entrypoint: ["echo", "This service was just a build job. Exiting."] + # httpd-git acceptance testing + githost: + build: ./githost + depends_on: + - git.build + ports: + - "80" + readonly: + build: ./githost + depends_on: + - git.build + ports: + - "80" + environment: + - GIT_READONLY=1 + test.git: + build: ./perlspec + labels: + com.yolean.build-contract: "*" + links: + - githost + - readonly + volumes: + - ./test:/project/t + # httpd-gitconf acceptance testing + httpd: + build: ../httpd-gitconf + image: httpd-gitconf + depends_on: + - gitclient.build + links: + - githost + volumes: + - ./gitconf-test:/gitconf-test + - ../httpd-gitconf:/perldev + entrypoint: /gitconf-test/httpd-entrypoint-gitconf + ports: + - "80" + environment: + - DEBUG=* + test.reconf: + build: ./perlspec + labels: + com.yolean.build-contract: "*" + links: + - githost + - httpd + volumes: + - ./gitconf-test:/project/t + # Of these test watch tools prowess' terminal usage seems to work better with docker-compose run, but neither understands Ctrl+C so you need docker kill + #entrypoint: autoprove + #entrypoint: prowess diff --git a/build-contracts/gitconf-test/httpd-entrypoint-gitconf b/build-contracts/gitconf-test/httpd-entrypoint-gitconf new file mode 100755 index 0000000..71b2150 --- /dev/null +++ b/build-contracts/gitconf-test/httpd-entrypoint-gitconf @@ -0,0 +1,12 @@ +#!/usr/bin/perl -w +use strict; + +`git clone http://githost/git/Test/conf.git /tmp/conf`; $? == 0 or die; +`git clone http://githost/git/Test/cert.git /tmp/cert`; $? == 0 or die; +`rm /usr/local/apache2/conf -Rf`; $? == 0 or die; +`mv /tmp/conf /usr/local/apache2/conf`; $? == 0 or die; +`mv /tmp/cert /usr/local/apache2/cert`; $? == 0 or die; +`apachectl configtest`; $? == 0 or die; + +# now the real entrypoint +exec 'httpd-foreground'; diff --git a/build-contracts/gitconf-test/reconf-spec.t b/build-contracts/gitconf-test/reconf-spec.t new file mode 100755 index 0000000..1f71268 --- /dev/null +++ b/build-contracts/gitconf-test/reconf-spec.t @@ -0,0 +1,85 @@ +#!/usr/bin/perl -w +use strict; + +use Test::Spec; + +my $testkey = time(); +mkdir "/tmp/testrun-$testkey"; +chdir "/tmp/testrun-$testkey"; +print "# testrun /tmp/testrun-$testkey\n"; + +use HTTP::Tiny; +use JSON; + +my $r; + +describe "Httpd state at container startup" => sub { + + it "Should be running" => sub { + $r = HTTP::Tiny->new->head('http://httpd/'); + is($r->{status}, 200); + }; + + it "Should have a typical 404 error page" => sub { + $r = HTTP::Tiny->new->head('http://httpd/testing/notfound'); + is($r->{status}, 404); + isnt($r->{content}, 'Custom.'); + }; + +}; + +describe "A shared git remote" => sub { + + it "Is alive" => sub { + $r = HTTP::Tiny->new->head('http://githost/'); + is($r->{status}, 200); + }; + + it "Has a conf repo to clone" => sub { + `git clone http://githost/git/Test/conf.git`; + is($?, 0); + ok(-e 'conf/.git'); + }; + + it "Has a cert repo to clone" => sub { + `git clone http://githost/git/Test/cert.git`; + is($?, 0); + ok(-e 'conf/.git'); + }; + +}; + +describe "Extenal conf modification over git" => sub { + + it "Add a simple one liner that can be detected over HTTP" => sub { + `echo 'ErrorDocument 404 "Custom."' >> conf/httpd.conf`; + is($?, 0); + `cd conf/; git add httpd.conf; git commit -m "Change 404 page"`; + is($?, 0); + }; + + it "Trigger httpd reconf using REST endpoint" => sub { + my $http = HTTP::Tiny->new(); + my $r = $http->post( + 'http://httpd/admin/reconf' => { + content => to_json( + {} + ), + headers => { + 'Accept' => 'application/json', + }, + }, + ); + is($r->{status}, 200); + }; + + it "Shuld now have affected httpd's runtime conf" => sub { + $r = HTTP::Tiny->new->head('http://httpd/testing/notfound'); + is($r->{content}, 'Custom.'); + }; + +}; + + + +runtests unless caller; diff --git a/build-contracts/githost/Dockerfile b/build-contracts/githost/Dockerfile new file mode 100644 index 0000000..cb48cdd --- /dev/null +++ b/build-contracts/githost/Dockerfile @@ -0,0 +1,22 @@ +FROM httpd-git + +RUN sed -i 's|^#LoadModule authn_anon_module|LoadModule authn_anon_module|' conf/httpd.conf + +COPY auth-anon.conf conf/git/ + +RUN mkdir -p /opt/git/Test \ + && git init --bare /opt/git/Test/test.git \ + && git init --bare /opt/git/Test/conf.git \ + && git init --bare /opt/git/Test/cert.git \ + && chown -R daemon /opt/git + +RUN git config --global user.email "you@example.com" \ + && git config --global user.name "Your Name" \ + && git clone /opt/git/Test/conf.git /tmp/conf \ + && cd /tmp/conf/ \ + && cp /usr/local/apache2/conf/httpd.conf . \ + && cp /usr/local/apache2/conf/mime.types . \ + && sed -i 's/^Include/#Include/' httpd.conf \ + && git add * \ + && git commit -m "Gets httpd up and running" \ + && git push origin master diff --git a/build-contracts/githost/auth-anon.conf b/build-contracts/githost/auth-anon.conf new file mode 100644 index 0000000..2f7de7a --- /dev/null +++ b/build-contracts/githost/auth-anon.conf @@ -0,0 +1,11 @@ + + AuthName "If visitors get this auth prompt you are at risk" + AuthType Basic + AuthBasicProvider anon + + Anonymous_NoUserID off + Anonymous_MustGiveEmail off + Anonymous_VerifyEmail off + Anonymous_LogEmail off + Anonymous "*" + diff --git a/build-contracts/perlspec/Dockerfile b/build-contracts/perlspec/Dockerfile new file mode 100644 index 0000000..1fd1df0 --- /dev/null +++ b/build-contracts/perlspec/Dockerfile @@ -0,0 +1,25 @@ +FROM perl:5.24 + +# http://stackoverflow.com/questions/3462058/how-do-i-automate-cpan-configuration +RUN (echo y;echo o conf prerequisites_policy follow;echo o conf commit) | cpan + +RUN cpan install Test::Spec + +RUN cpan install HTTP::Tiny JSON + +# We need one of these and I don't know which one is more stable or useful yet +# To evaluate, toggle entrypoint between prove/autoprove/prowess +RUN cpan install Test::Continuous App::prowess + +RUN git --version && cpan install Git::Wrapper + +RUN git config --global user.email "perlspec-testing@example.com" \ + && git config --global user.name "Perlspec Testint" + +# Official perl image gotcha, /usr/bin/perl fails to include CPAN modules +RUN mv /usr/bin/perl /usr/bin/perl.org \ + && ln -s /usr/local/bin/perl /usr/bin/perl + +WORKDIR /project + +ENTRYPOINT ["prove"] diff --git a/build-contracts/test/git-http-spec.t b/build-contracts/test/git-http-spec.t new file mode 100755 index 0000000..ba18c41 --- /dev/null +++ b/build-contracts/test/git-http-spec.t @@ -0,0 +1,70 @@ +#!/usr/bin/perl -w +use strict; + +use Test::Spec; + +# Didn't like this much, let's see if we need it +#use Git::Wrapper; +#my $git = Git::Wrapper->new('/tmp/test'); + +my $testkey = time(); +mkdir "/tmp/testrun-$testkey"; +chdir "/tmp/testrun-$testkey"; +print "# testrun /tmp/testrun-$testkey\n"; + +describe "Clone at /git/[org]/[repo]" => sub { + + it "Allowed" => sub { + `git clone http://githost/git/Test/test.git ./test`; + is($?, 0); + }; + + it "Produces a local repo" => sub { + ok(-e 'test/.git' and -d 'test/.git'); + }; + +}; + +describe "Readonly" => sub { + + it "Same clone behavior as regular host" => sub { + `git clone http://readonly/git/Test/test.git ./readonly`; + is($?, 0); + }; + + it "Same fetch" => sub { + `cd test/ && git remote add readonly http://readonly/git/Test/test.git && git fetch readonly`; + is($?, 0); + }; + + it "Denies push" => sub { + `cd test/ && echo test > test1.txt && git add test1.txt && git commit -m "Test 1"`; + is($?, 0); + `cd test/ && git push readonly master`; + isnt($?, 0); + }; + + # TODO test for status code 403 at GET /git/Test/test.git/info/refs?service=git-receive-pack + # as the test above passes for status 500 (i.e. auth not configured) too + # Or we can possibly just check for git auth attempt "fatal: could not read Username for 'http://readonly': No such device or address" + +}; + +describe "Push" => sub { + + it "Requires authentication (with default config, custom auth conf needed)" => sub { + `cd test/ && git push origin master`; + isnt($?, 0); + }; + + it "Test container runs mod_auth_anon so any username will do here" => sub { + `cd test/ && git remote add auth 'http://testuser:\@githost/git/Test/test.git' && git remote -v`; + ## more presistent auth + #`echo 'http://testuser:@githost' >> ~/.git-credentials` + #`cd test/ && git config credential.helper store && git push origin master`; + is($?, 0); + }; + +}; + +runtests unless caller; diff --git a/git/Dockerfile b/git/Dockerfile index 276f51f..f4878a1 100644 --- a/git/Dockerfile +++ b/git/Dockerfile @@ -1,51 +1,12 @@ - -FROM httpd:2.4.23 - -ENV GIT_VERSION 2.9.3 -ENV GIT_VERSION_TGZ_URL https://www.kernel.org/pub/software/scm/git/git-$GIT_VERSION.tar.gz -ENV GIT_VERSION_TGZ_SHA1 ae90c4e5008ae10c8a67a51ff3dbea8364d97168 - -RUN depsRuntime=' \ - libcurl3 \ - libexpat1 \ - gettext \ - libssl1.0.0 \ - ' \ - && depsBuild=' \ - curl ca-certificates \ - gcc \ - make \ - autoconf \ - libcurl4-gnutls-dev \ - libexpat1-dev \ - gettext \ - libz-dev \ - libssl-dev \ - ' \ - set -x \ - && apt-get update \ - && apt-get install -y --no-install-recommends $depsRuntime \ - && apt-get install -y --no-install-recommends $depsBuild \ - && rm -r /var/lib/apt/lists/* \ - && curl -SL "$GIT_VERSION_TGZ_URL" -o git-$GIT_VERSION.tar.gz \ - && echo "$GIT_VERSION_TGZ_SHA1 git-$GIT_VERSION.tar.gz" | sha1sum -c - \ - && mkdir -p src/git \ - && tar -xvf git-$GIT_VERSION.tar.gz -C src/git --strip-components=1 \ - && rm git-$GIT_VERSION.tar.gz* \ - && cd src/git \ - && make configure \ - && ./configure --prefix=/usr \ - && make all \ - && make install \ - && cd ../../ \ - && rm -r src/git \ - && apt-get purge -y --auto-remove $depsBuild - -EXPOSE 80 +FROM httpd-gitclient RUN sed -i 's|#LoadModule cgid_module|LoadModule cgid_module|' conf/httpd.conf \ && sed -i 's|#LoadModule rewrite_module|LoadModule rewrite_module|' conf/httpd.conf \ && echo "Include conf/git/*.conf" >> conf/httpd.conf +ENV GIT_PROJECT_ROOT="/opt/git" +ENV GIT_HTTP_EXPORT_ALL="1" +ENV GIT_READONLY="" + ADD conf/git.conf /usr/local/apache2/conf/git/ -ADD conf/git-readonly.conf /usr/local/apache2/conf/git/ +ADD conf/git-access.conf /usr/local/apache2/conf/git/ diff --git a/git/conf/git-access.conf b/git/conf/git-access.conf new file mode 100644 index 0000000..502a81d --- /dev/null +++ b/git/conf/git-access.conf @@ -0,0 +1,18 @@ + +RewriteEngine On + +RewriteCond %{QUERY_STRING} service=git-receive-pack [OR] +RewriteCond %{REQUEST_URI} /git-receive-pack$ +RewriteCond %{GIT_READONLY} !^$ +RewriteRule ^/git/ - [F] + +RewriteCond %{QUERY_STRING} service=git-receive-pack [OR] +RewriteCond %{REQUEST_URI} /git-receive-pack$ +RewriteRule ^/git/ - [E=AUTHREQUIRED] + + + Require valid-user + Order deny,allow + Deny from env=AUTHREQUIRED + Satisfy any + diff --git a/git/conf/git-readonly.conf b/git/conf/git-readonly.conf deleted file mode 100644 index d4926c2..0000000 --- a/git/conf/git-readonly.conf +++ /dev/null @@ -1,6 +0,0 @@ - - - Order deny,allow - Deny from env=GITWRITE - Satisfy any - diff --git a/git/conf/git.conf b/git/conf/git.conf index 9c402a3..3f829c8 100644 --- a/git/conf/git.conf +++ b/git/conf/git.conf @@ -1,13 +1,9 @@ -SetEnv GIT_PROJECT_ROOT /opt/git -SetEnv GIT_HTTP_EXPORT_ALL 1 +PassEnv GIT_PROJECT_ROOT +PassEnv GIT_HTTP_EXPORT_ALL + ScriptAlias /git/ /usr/libexec/git-core/git-http-backend/ Options +ExecCGI Require all granted - -RewriteEngine On -RewriteCond %{QUERY_STRING} service=git-receive-pack [OR] -RewriteCond %{REQUEST_URI} /git-receive-pack$ -RewriteRule ^/git/ - [E=AUTHREQUIRED] diff --git a/gitclient/Dockerfile b/gitclient/Dockerfile new file mode 100644 index 0000000..8b5be52 --- /dev/null +++ b/gitclient/Dockerfile @@ -0,0 +1,46 @@ + +FROM httpd:2.4.23 + +ENV GIT_VERSION 2.10.0 +ENV GIT_VERSION_TGZ_URL https://www.kernel.org/pub/software/scm/git/git-$GIT_VERSION.tar.gz +ENV GIT_VERSION_TGZ_SHA1 2d588afe7adb11ea11e0787c4a2f01329a0f2f55 + +RUN depsRuntime=' \ + libcurl3 \ + libcurl3-gnutls \ + curl \ + libexpat1 \ + gettext \ + libssl1.0.0 \ + ' \ + && depsBuild=' \ + ca-certificates \ + gcc \ + make \ + autoconf \ + libcurl4-gnutls-dev \ + libexpat1-dev \ + gettext \ + libz-dev \ + libssl-dev \ + ' \ + set -x \ + && apt-get update \ + && apt-get install -y --no-install-recommends $depsRuntime \ + && apt-get install -y --no-install-recommends $depsBuild \ + && rm -r /var/lib/apt/lists/* \ + && curl -SL "$GIT_VERSION_TGZ_URL" -o git-$GIT_VERSION.tar.gz \ + && echo "$GIT_VERSION_TGZ_SHA1 git-$GIT_VERSION.tar.gz" | sha1sum -c - \ + && mkdir -p src/git \ + && tar -xvf git-$GIT_VERSION.tar.gz -C src/git --strip-components=1 \ + && rm git-$GIT_VERSION.tar.gz* \ + && cd src/git \ + && make configure \ + && ./configure --prefix=/usr \ + && make all \ + && make install \ + && cd ../../ \ + && rm -r src/git \ + && apt-get purge -y --auto-remove $depsBuild + +EXPOSE 80 diff --git a/httpd-gitconf/Dockerfile b/httpd-gitconf/Dockerfile new file mode 100644 index 0000000..ee8e5ce --- /dev/null +++ b/httpd-gitconf/Dockerfile @@ -0,0 +1,29 @@ + +# TODO but we need openidc here too +FROM httpd-gitclient + +# Needed during development at least +RUN mkdir -p /usr/local/lib/site_perl/Data \ + && curl http://api.metacpan.org/source/SMUELLER/Data-Dumper-2.161/Dumper.pm -o /usr/local/lib/site_perl/Data/Dumper.pm \ + && echo "c3d1692479123e7c21d3e47a08550fde6fcbcbdf /usr/local/lib/site_perl/Data/Dumper.pm" | sha1sum -c - + +# Evaluated this logging lib +#RUN mkdir -p /usr/local/lib/site_perl/CGI \ +# && curl http://api.metacpan.org/source/JMOORE/CGI-Log-1.00/Log.pm -o /usr/local/lib/site_perl/CGI/Log.pm \ +# && echo "d02ad2ad622ee1953f51e0c74b9af52a94bb6d0b /usr/local/lib/site_perl/CGI/Log.pm" | sha1sum -c - + +# Found a logging lib that is a simple abstraction on print and also replaces die, without the dependencies of Carp +RUN mkdir -p /usr/local/lib/site_perl/Scalar \ + && curl http://api.metacpan.org/source/PEVANS/Scalar-List-Utils-1.45/lib/Scalar/Util.pm -o /usr/local/lib/site_perl/Scalar/Util.pm \ + && echo "a85497bb2f8979b6eb76cfdc7bd7dc4bcc70b64e /usr/local/lib/site_perl/Scalar/Util.pm" | sha1sum -c - \ + && mkdir -p /usr/local/lib/site_perl/Term \ + && curl http://api.metacpan.org/source/RRA/Term-ANSIColor-4.05/lib/Term/ANSIColor.pm -o /usr/local/lib/site_perl/Term/ANSIColor.pm \ + && echo "07499818b26ab025d726e56b2c798cee14ad61a6 /usr/local/lib/site_perl/Term/ANSIColor.pm" | sha1sum -c - \ + && mkdir -p /usr/local/lib/site_perl/Log \ + && curl http://api.metacpan.org/source/KAZEBURO/Log-Minimal-0.19/lib/Log/Minimal.pm -o /usr/local/lib/site_perl/Log/Minimal.pm \ + && echo "917f7f526e286d7ae684d6a2e7468729d500f7a3 /usr/local/lib/site_perl/Log/Minimal.pm" | sha1sum -c - + +COPY HttpdControl.pm /usr/local/lib/site_perl/ +COPY ReconfDirGit.pm /usr/local/lib/site_perl/ + +COPY httpd-reconf /usr/local/bin/ diff --git a/httpd-gitconf/HttpdControl.pm b/httpd-gitconf/HttpdControl.pm new file mode 100644 index 0000000..4e0f415 --- /dev/null +++ b/httpd-gitconf/HttpdControl.pm @@ -0,0 +1,30 @@ +package HttpdControl; + +use strict; +use warnings; + +use Log::Minimal env_debug => 'DEBUG'; + +sub new { + my ($class, %args) = @_; + return bless { %args }, $class; +} + +sub configtest { + my ($self) = @_; + my $out = `apachectl configtest`; + my $result = $?; + chomp($out); + debugf($out); + return $result == 0; +} + +sub reload { + my ($self) = @_; + my $out = `apachectl graceful`; + my $result = $?; + print $out; + return $result == 0; +} + +1; diff --git a/httpd-gitconf/ReconfDirGit.pm b/httpd-gitconf/ReconfDirGit.pm new file mode 100644 index 0000000..292865e --- /dev/null +++ b/httpd-gitconf/ReconfDirGit.pm @@ -0,0 +1,65 @@ +package ReconfDirGit; + +use strict; +use warnings; + +use Log::Minimal env_debug => 'DEBUG'; + +sub new { + my ($class, %args) = @_; + return bless { %args }, $class; +} + +sub dir { + my ($self) = @_; + return $self->{dir}; +} + +sub rev { + my ($self) = @_; + my $rev = `cd $self->{dir} && git rev-parse --verify HEAD`; + ($? == 0) or croakf("Failed to read current rev: $rev"); + chomp($rev); + return $rev; +} + +sub branch { + my ($self) = @_; + my $branch = `cd $self->{dir} && git rev-parse --abbrev-ref HEAD`; + $? == 0 or croakf("Failed to read current branch: $branch"); + chomp($branch); + $branch or croakf("Falsey branch name at ".$self->rev()); + return $branch; +} + +sub mark_good { + my ($self) = @_; + my $current = $self->branch(); + ($current =~ /^reconf_last-known-good_/) and die("Invalid state. Current branch is the checkpoint $current"); + debugf(`cd $self->{dir} && git checkout -B _reconf_last-known-good_$current && git checkout $current`); + ($? == 0) or croakf('Branch last good conf failed at $current'); + debugf("_reconf_last-known-good_$current saved at ".$self->rev()); +} + +sub fetch_rebase { + my ($self) = @_; + my $rev = $self->rev(); + my $current = $self->branch(); + debugf("cd $self->{dir} && git rev-parse --verify $self->{remote}/$current"); + my $remoterev = `cd $self->{dir} && git rev-parse --verify $self->{remote}/$current`; + chomp($remoterev); + ($rev eq $remoterev) or croakf("Local rev $rev is out of sync with $self->{remote}/$current $remoterev"); + debugf("$current == $self->{remote}/$current == $rev"); + debugf(`cd $self->{dir} && git fetch $self->{remote} && git rebase $self->{remote}/$current`); + ($? == 0) or croakf("Fetch + rebase failed for $self->{remote}/$current"); +} + +sub revert_to_good { + my ($self) = @_; + my $current = $self->branch(); + ($current =~ /^reconf_last-known-good_/) and die("Current branch is already at the checkpoint $current"); + debugf(`cd $self->{dir} && git checkout _reconf_last-known-good_$current && git checkout -B $current`); + ($? == 0) or croakf("Revert failed. Now at ".$self->rev()); +} + +1; diff --git a/httpd-gitconf/httpd-reconf b/httpd-gitconf/httpd-reconf new file mode 100755 index 0000000..af66880 --- /dev/null +++ b/httpd-gitconf/httpd-reconf @@ -0,0 +1,61 @@ +#!/usr/bin/perl -w +use strict; +use warnings; + +use Log::Minimal env_debug => 'DEBUG'; + +use ReconfDirGit; +use HttpdControl; + +my $conf = ReconfDirGit->new( + dir => '/usr/local/apache2/conf', + remote => 'origin' +); +print $conf->dir(); +print ": current rev = "; +my $start = $conf->rev(); +print "$start\n"; + +my $control = HttpdControl->new(); + +if ($control->configtest()) { + $conf->mark_good(); +} else { + warnf("Starting from invalid config. There's hopefully a _reconf_last-known-good_ branch already."); +} + +if (!$conf->fetch_rebase()) { + critf("Failed to refresh configuration from remote"); +} else { + my $after = $conf->rev(); + debugf("Rebase done. At $after."); + + if ($control->configtest()) { + $control->reload(); + infof("Config reloaded successfuly at $after"); + # Keep last-known-good in case monitoring detects a regression + } else { + warnf("Invalid configuration at $after, reverting"); + $conf->revert_to_good(); + } +} + +infof "Done."; + +=pod + +# timestamp +RUNLABEL=() + +# keep current config state for undo +for (d in conf cert) + git checkout -b previous-$RUNLABEL + +function rollback + +# test config +apachectl configtest || rollback + +apachectl graceful || rollback + +=cut