diff --git a/.buildpacks b/.buildpacks deleted file mode 100644 index 4d7e2859be..0000000000 --- a/.buildpacks +++ /dev/null @@ -1,2 +0,0 @@ -https://github.com/heroku/heroku-buildpack-ruby.git -https://github.com/ryandotsmith/nginx-buildpack.git diff --git a/.gitignore b/.gitignore index 7ea2cdf138..7164b0d14f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,13 @@ config/travis.yml +config/travis/* config/database.yml config/nginx.conf config/skylight.yml tmp/ + logs/ +log/ .yardoc .coverage diff --git a/.ruby-version b/.ruby-version index 585940699b..2bf1c1ccf3 100644 --- a/.ruby-version +++ b/.ruby-version @@ -1 +1 @@ -2.2.3 +2.3.1 diff --git a/.travis.yml b/.travis.yml index 767aa5d722..505e664d12 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,7 @@ language: ruby sudo: false -rvm: 2.2.3 +rvm: 2.3.1 env: global: diff --git a/Gemfile b/Gemfile index bee58fa549..10e1d278d6 100644 --- a/Gemfile +++ b/Gemfile @@ -1,33 +1,37 @@ source 'https://rubygems.org' gemspec -gem 's3', github: 'travis-ci/s3' +ruby '2.3.1' -gem 'travis-support', github: 'travis-ci/travis-support' -gem 'travis-amqp', github: 'travis-ci/travis-amqp' +gem 's3', git: 'https://github.com/travis-ci/s3' + +gem 'mime-types' + +gem 'travis-support', git: 'https://github.com/travis-ci/travis-support' +gem 'travis-amqp', git: 'https://github.com/travis-ci/travis-amqp' gem 'travis-config', '~> 0.1.0' -gem 'travis-settings', github: 'travis-ci/travis-settings' -gem 'travis-sidekiqs', github: 'travis-ci/travis-sidekiqs' +gem 'travis-settings', git: 'https://github.com/travis-ci/travis-settings' +gem 'travis-sidekiqs', git: 'https://github.com/travis-ci/travis-sidekiqs' +gem 'travis-lock', git: 'https://github.com/travis-ci/travis-lock' -gem 'travis-yaml', github: 'travis-ci/travis-yaml' -gem 'mustermann', github: 'rkh/mustermann' +gem 'travis-yaml', git: 'https://github.com/travis-ci/travis-yaml' +gem 'mustermann' gem 'sinatra' -gem 'sinatra-contrib', require: nil #github: 'sinatra/sinatra-contrib', require: nil +gem 'sinatra-contrib', require: nil #git: 'https://github.com/sinatra/sinatra-contrib', require: nil gem 'active_model_serializers' gem 'unicorn' gem 'sentry-raven' -gem 'yard-sinatra', github: 'rkh/yard-sinatra' +gem 'yard-sinatra', git: 'https://github.com/rkh/yard-sinatra' gem 'rack-contrib' -gem 'rack-cache', github: 'rtomayko/rack-cache' +gem 'rack-cache', git: 'https://github.com/rtomayko/rack-cache' gem 'rack-attack', '5.0.0.beta1' gem 'gh' gem 'bunny', '~> 0.8.0' gem 'dalli' gem 'pry' gem 'metriks', '0.9.9.6' -gem 'metriks-librato_metrics', github: 'eric/metriks-librato_metrics' -gem 'micro_migrations' +gem 'metriks-librato_metrics', git: 'https://github.com/eric/metriks-librato_metrics' gem 'simplecov' gem 'skylight', '~> 0.6.0.beta.1' gem 'stackprof' @@ -36,8 +40,11 @@ gem 'netaddr' gem 'jemalloc' gem 'customerio' +gem "redlock" +gem 'rake', '~> 0.9.2' + group :development, :test do - gem 'travis-migrations', github: 'travis-ci/travis-migrations' + gem 'travis-migrations', git: 'https://github.com/travis-ci/travis-migrations' end group :test do @@ -47,6 +54,7 @@ group :test do gem 'mocha', '~> 0.12' gem 'database_cleaner', '~> 0.8.0' gem 'timecop', '~> 0.8.0' + gem 'webmock' end group :development do @@ -54,7 +62,3 @@ group :development do gem 'rerun' gem 'rb-fsevent', '~> 0.9.1' end - -group :development, :test do - gem 'rake', '~> 0.9.2' -end diff --git a/Gemfile.lock b/Gemfile.lock index daead028fc..9daf9249a5 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,52 +1,51 @@ GIT - remote: git://github.com/eric/metriks-librato_metrics.git - revision: 2c124f024fd2e34378260cd24b0e8687ff9bd196 + remote: https://github.com/eric/metriks-librato_metrics + revision: 3acb88b36c014a8b93bdd91a867c1b3db781ad4e specs: - metriks-librato_metrics (1.0.5) + metriks-librato_metrics (1.0.6) metriks (>= 0.9.9.6) GIT - remote: git://github.com/rkh/mustermann.git - revision: 9611951c5c789ad8227a740ae142c157a8bf7401 - specs: - mustermann (0.4.0) - tool (~> 0.2) - -GIT - remote: git://github.com/rkh/yard-sinatra.git + remote: https://github.com/rkh/yard-sinatra revision: 00774d355123617ff0faa7e0ebd54c4cdcfcdf93 specs: yard-sinatra (1.0.0) yard (~> 0.7) GIT - remote: git://github.com/rtomayko/rack-cache.git + remote: https://github.com/rtomayko/rack-cache revision: 2d6618172c39c53dceab2e2946d87496154f3e52 specs: rack-cache (1.6.1) rack (>= 0.4) GIT - remote: git://github.com/travis-ci/s3.git + remote: https://github.com/travis-ci/s3 revision: 386361c1b0ede19cde0ddaf86e41a16308575f5d specs: s3 (0.3.21) proxies (~> 0.2.0) GIT - remote: git://github.com/travis-ci/travis-amqp.git + remote: https://github.com/travis-ci/travis-amqp revision: 3966b3651adfd1c544f53d49d5986ac16cd9c4bc specs: travis-amqp (0.0.1) GIT - remote: git://github.com/travis-ci/travis-migrations.git - revision: d50bc1b1e38f5eab1550acbe753bff75602ae778 + remote: https://github.com/travis-ci/travis-lock + revision: b418401c79e082aa53a62364f7b01a9be6f0d768 + specs: + travis-lock (0.1.1) + +GIT + remote: https://github.com/travis-ci/travis-migrations + revision: 9df23c494c90ed64f5151d4d5009c430b2f8c010 specs: travis-migrations (0.0.3) GIT - remote: git://github.com/travis-ci/travis-settings.git + remote: https://github.com/travis-ci/travis-settings revision: d510e63b6c6f059cccae141c265e7a0c7236d1fd specs: travis-settings (0.0.1) @@ -54,7 +53,7 @@ GIT virtus GIT - remote: git://github.com/travis-ci/travis-sidekiqs.git + remote: https://github.com/travis-ci/travis-sidekiqs revision: c5d4a4abc6c3737f9c43d3333efb94daa18b9fbb specs: travis-sidekiqs (0.0.1) @@ -62,13 +61,13 @@ GIT sidekiq GIT - remote: git://github.com/travis-ci/travis-support.git + remote: https://github.com/travis-ci/travis-support revision: 2cd02d2a06fdd1e2fc2f129148c168b56f7c190f specs: travis-support (0.0.1) GIT - remote: git://github.com/travis-ci/travis-yaml.git + remote: https://github.com/travis-ci/travis-yaml revision: 032caed23af8ed1ed55e9204bb91316f3ada2f74 specs: travis-yaml (0.2.0) @@ -80,17 +79,18 @@ PATH activerecord (~> 3.2.19) coder (~> 0.4.0) composite_primary_keys (~> 5.0) + fog-aws (~> 0.12.0) + fog-google (~> 0.4.2) google-api-client (~> 0.9.4) hashr memcachier multi_json - mustermann (~> 0.4) + mustermann (~> 1.0.0.beta2) pg pusher (~> 0.14.0) rack-contrib (~> 1.1) rack-ssl (~> 1.3, >= 1.3.3) - railties (~> 3.2.19) - redcarpet (~> 2.1) + redcarpet (>= 3.2.3) redis (~> 3.0) rollout (~> 1.1.0) simple_states (~> 1.0.0) @@ -104,30 +104,21 @@ PATH GEM remote: https://rubygems.org/ specs: - actionpack (3.2.22.2) - activemodel (= 3.2.22.2) - activesupport (= 3.2.22.2) - builder (~> 3.0.0) - erubis (~> 2.7.0) - journey (~> 1.0.4) - rack (~> 1.4.5) - rack-cache (~> 1.2) - rack-test (~> 0.6.1) - sprockets (~> 2.2.1) active_model_serializers (0.9.5) activemodel (>= 3.2) - activemodel (3.2.22.2) - activesupport (= 3.2.22.2) + activemodel (3.2.22.5) + activesupport (= 3.2.22.5) builder (~> 3.0.0) - activerecord (3.2.22.2) - activemodel (= 3.2.22.2) - activesupport (= 3.2.22.2) + activerecord (3.2.22.5) + activemodel (= 3.2.22.5) + activesupport (= 3.2.22.5) arel (~> 3.0.2) tzinfo (~> 0.3.29) - activesupport (3.2.22.2) + activesupport (3.2.22.5) i18n (~> 0.6, >= 0.6.4) multi_json (~> 1.0) - addressable (2.4.0) + addressable (2.5.0) + public_suffix (~> 2.0, >= 2.0.2) arel (3.0.3) atomic (1.1.99) avl_tree (1.1.3) @@ -145,7 +136,9 @@ GEM composite_primary_keys (5.0.14) activerecord (~> 3.2.0, >= 3.2.9) concurrent-ruby (1.0.2) - connection_pool (2.2.0) + connection_pool (2.2.1) + crack (0.4.3) + safe_yaml (~> 1.0.0) customerio (1.0.0) multi_json (~> 1.0) dalli (2.7.6) @@ -155,14 +148,34 @@ GEM diff-lcs (1.2.5) docile (1.1.5) equalizer (0.0.11) - erubis (2.7.0) + excon (0.54.0) factory_girl (2.4.2) activesupport faraday (0.9.2) multipart-post (>= 1.2, < 3) - ffi (1.9.10) + ffi (1.9.14) + fog-aws (0.12.0) + fog-core (~> 1.38) + fog-json (~> 1.0) + fog-xml (~> 0.1) + ipaddress (~> 0.8) + fog-core (1.43.0) + builder + excon (~> 0.49) + formatador (~> 0.2) + fog-google (0.4.2) + fog-core + fog-json + fog-xml + fog-json (1.0.2) + fog-core (~> 1.0) + multi_json (~> 1.10) + fog-xml (0.1.2) + fog-core + nokogiri (~> 1.5, >= 1.5.11) foreman (0.82.0) thor (~> 0.19.1) + formatador (0.2.5) gh (0.14.0) addressable backports @@ -171,7 +184,7 @@ GEM net-http-persistent (>= 2.7) net-http-pipeline git-version-bump (0.15.1) - google-api-client (0.9.8) + google-api-client (0.9.20) addressable (~> 2.3) googleauth (~> 0.5) httpclient (~> 2.7) @@ -180,7 +193,6 @@ GEM mime-types (>= 1.6) representable (~> 2.3.0) retriable (~> 2.0) - thor (~> 0.19) googleauth (0.5.1) faraday (~> 0.9) jwt (~> 1.4) @@ -189,17 +201,17 @@ GEM multi_json (~> 1.11) os (~> 0.9) signet (~> 0.7) + hashdiff (0.3.1) hashr (0.0.22) - hike (1.2.3) hitimes (1.2.4) - httpclient (2.8.0) + httpclient (2.8.2.4) hurley (0.2) i18n (0.7.0) ice_nine (0.11.2) + ipaddress (0.8.3) jemalloc (1.0.1) - journey (1.0.4) - json (1.8.3) - jwt (1.5.4) + json (2.0.2) + jwt (1.5.6) kgio (2.10.0) listen (3.1.5) rb-fsevent (~> 0.9, >= 0.9.4) @@ -210,37 +222,42 @@ GEM little-plugger (~> 1.1) multi_json (~> 1.10) memcachier (0.0.2) - memoist (0.14.0) + memoist (0.15.0) metaclass (0.0.4) method_source (0.8.2) metriks (0.9.9.6) atomic (~> 1.0) avl_tree (~> 1.1.2) hitimes (~> 1.1) - micro_migrations (0.0.2) - activerecord - railties - mime-types (1.25.1) + mime-types (3.1) + mime-types-data (~> 3.2015) + mime-types-data (3.2016.0521) + mini_portile2 (2.1.0) mocha (0.14.0) metaclass (~> 0.0.1) multi_json (1.12.1) multipart-post (2.0.0) - net-http-persistent (2.9.4) + mustermann (1.0.0.beta2) + net-http-persistent (3.0.0) + connection_pool (~> 2.2) net-http-pipeline (1.0.1) netaddr (1.5.1) + nokogiri (1.6.8.1) + mini_portile2 (~> 2.1.0) os (0.9.6) - pg (0.18.4) + pg (0.19.0) proxies (0.2.1) - pry (0.10.3) + pry (0.10.4) coderay (~> 1.1.0) method_source (~> 0.8.1) slop (~> 3.4) + public_suffix (2.0.4) pusher (0.14.6) httpclient (~> 2.5) multi_json (~> 1.0) pusher-signature (~> 0.1.8) pusher-signature (0.1.8) - rack (1.4.7) + rack (1.6.5) rack-attack (5.0.0.beta1) rack rack-contrib (1.4.0) @@ -248,28 +265,21 @@ GEM rack (~> 1.4) rack-protection (1.5.3) rack - rack-ssl (1.3.4) + rack-ssl (1.4.1) rack rack-test (0.6.3) rack (>= 1.0) - railties (3.2.22.2) - actionpack (= 3.2.22.2) - activesupport (= 3.2.22.2) - rack-ssl (~> 1.3.2) - rake (>= 0.8.7) - rdoc (~> 3.4) - thor (>= 0.14.6, < 2.0) - raindrops (0.16.0) + raindrops (0.17.0) rake (0.9.6) - rb-fsevent (0.9.7) + rb-fsevent (0.9.8) rb-inotify (0.9.7) ffi (>= 0.5.0) - rdoc (3.12.2) - json (~> 1.4) - redcarpet (2.3.0) - redis (3.3.0) + redcarpet (3.3.4) + redis (3.3.1) redis-namespace (1.5.2) redis (~> 3.0, >= 3.0.4) + redlock (0.1.8) + redis (~> 3, >= 3.0.0) representable (2.3.0) uber (~> 0.0.7) rerun (0.11.0) @@ -287,14 +297,16 @@ GEM rspec-core (>= 2.99.0.beta1) rspec-expectations (>= 2.99.0.beta1) rspec-mocks (2.99.4) - ruby_dep (1.3.1) - sentry-raven (1.0.0) - faraday (>= 0.7.6) - sidekiq (4.1.2) + ruby_dep (1.5.0) + safe_yaml (1.0.4) + sentry-raven (2.1.2) + faraday (>= 0.7.6, < 0.10.x) + sidekiq (4.2.6) concurrent-ruby (~> 1.0) connection_pool (~> 2.2, >= 2.2.0) + rack-protection (>= 1.5.0) redis (~> 3.2, >= 3.2.1) - signet (0.7.2) + signet (0.7.3) addressable (~> 2.3) faraday (~> 0.9) jwt (~> 1.5) @@ -302,13 +314,13 @@ GEM simple_states (1.0.1) activesupport hashr (~> 0.0.10) - simplecov (0.11.2) + simplecov (0.12.0) docile (~> 1.1.0) - json (~> 1.8) + json (>= 1.8, < 3) simplecov-html (~> 0.10.0) simplecov-html (0.10.0) - sinatra (1.4.6) - rack (~> 1.4) + sinatra (1.4.7) + rack (~> 1.5) rack-protection (~> 1.4) tilt (>= 1.3, < 3) sinatra-contrib (1.4.7) @@ -321,31 +333,30 @@ GEM skylight (0.6.2.beta.2) activesupport (>= 3.0.0) slop (3.6.0) - sprockets (2.2.3) - hike (~> 1.2) - multi_json (~> 1.0) - rack (~> 1.0) - tilt (~> 1.1, != 1.3.0) - stackprof (0.2.9) + stackprof (0.2.10) thor (0.19.1) thread_safe (0.3.5) - tilt (1.4.1) + tilt (2.0.5) timecop (0.8.1) tool (0.2.3) travis-config (0.1.4) hashr (~> 0.0) - tzinfo (0.3.49) + tzinfo (0.3.52) uber (0.0.15) - unicorn (5.1.0) + unicorn (5.2.0) kgio (~> 2.6) raindrops (~> 0.7) - useragent (0.16.7) + useragent (0.16.8) virtus (1.0.5) axiom-types (~> 0.1) coercible (~> 1.0) descendants_tracker (~> 0.0, >= 0.0.3) equalizer (~> 0.0, >= 0.0.9) - yard (0.8.7.6) + webmock (2.1.0) + addressable (>= 2.3.6) + crack (>= 0.3.2) + hashdiff + yard (0.9.5) PLATFORMS ruby @@ -362,9 +373,9 @@ DEPENDENCIES jemalloc metriks (= 0.9.9.6) metriks-librato_metrics! - micro_migrations + mime-types mocha (~> 0.12) - mustermann! + mustermann netaddr pry rack-attack (= 5.0.0.beta1) @@ -372,6 +383,7 @@ DEPENDENCIES rack-contrib rake (~> 0.9.2) rb-fsevent (~> 0.9.1) + redlock rerun rspec (~> 2.13) rspec-its @@ -386,13 +398,18 @@ DEPENDENCIES travis-amqp! travis-api! travis-config (~> 0.1.0) + travis-lock! travis-migrations! travis-settings! travis-sidekiqs! travis-support! travis-yaml! unicorn + webmock yard-sinatra! +RUBY VERSION + ruby 2.3.1p112 + BUNDLED WITH - 1.12.5 + 1.13.6 diff --git a/Procfile b/Procfile index b6c965fa41..4fa50ead4a 100644 --- a/Procfile +++ b/Procfile @@ -1,4 +1,4 @@ web: ./script/server console: bundle exec je ./script/console sidekiq: bundle exec je sidekiq -c 4 -r ./lib/travis/sidekiq.rb -q build_cancellations, -q build_restarts, -q job_cancellations, -q job_restarts -start_crons: ./script/start_crons +cron: bin/cron diff --git a/README.md b/README.md index 73ee8aae45..fd70d18e27 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ -# The public Travis API +# Travis API -This is the app running on https://api.travis-ci.org/ +[![Build Status](https://travis-ci.org/travis-ci/travis-api.svg?branch=master)](https://travis-ci.org/travis-ci/travis-api) + +https://api.travis-ci.org ## Requirements @@ -38,7 +40,7 @@ NB detail for how `rake` sets up the database can be found in the `Rakefile`. In $ RAILS_ENV=development bundle exec rake db:create $ RAILS_ENV=test bundle exec rake db:create ``` -#### Optional +#### Travis Logs DB setup Clone `travis-logs` and copy the `logs` database (assume the PostgreSQL user is `postgres`): ```sh-session $ cd .. @@ -78,3 +80,13 @@ If you have problems with Nginx because the websocket is already in use, try res ### API documentation v3 documentation can be found at https://developer.travis-ci.org which is a repository that can be found at https://github.com/travis-pro/developer + +### Adding V3 Endpoints Developer Documentation +Start with the find/get spec (for example: spec/v3/services/caches/find_spec.rb) for your new endpoint. If you don't have a find route start with whatever route you want to add first. Run the test and add the files you need to clear the errors. They should be: + - A service (lib/travis/api/v3/services/caches/find.rb) + - A query (lib/travis/api/v3/queries/caches.rb) + - Register the service in v3/services.rb (alphabetical order please) + - Add a route (v3/routes.rb) + Re-run the test at this point. Depending on what objects you are returning you may also need to add: + - Add a model (either pulls from the DB or a wrapper for the class of the objects returned from another source (s3 for example), or that structures the result you will be passing back to the client) + - Add a renderer (if needed to display your new model/object/collection) diff --git a/Rakefile b/Rakefile index 6e2e257407..0e45e7ae13 100644 --- a/Rakefile +++ b/Rakefile @@ -1,10 +1,13 @@ namespace :db do - env = ENV["RAILS_ENV"] + env = ENV["RAILS_ENV"] || 'development' if env != 'production' desc "Create and migrate the #{env} database" task :create do sh "createdb travis_#{env}" rescue nil - sh "psql -q travis_#{env} < #{Gem.loaded_specs['travis-migrations'].full_gem_path}/db/structure.sql" + sh "psql -q travis_#{env} < #{Gem.loaded_specs['travis-migrations'].full_gem_path}/db/main/structure.sql" + + sh "createdb travis_logs_#{env}" rescue nil + sh "psql -q travis_logs_#{env} < #{Gem.loaded_specs['travis-migrations'].full_gem_path}/db/logs/structure.sql" end end end diff --git a/bin/cron b/bin/cron new file mode 100644 index 0000000000..a783985d8f --- /dev/null +++ b/bin/cron @@ -0,0 +1,13 @@ +#!/usr/bin/env ruby + +$stdout.sync = true + +$LOAD_PATH << 'lib' + +require 'bundler/setup' +require 'travis/api/app' +require 'travis/api/app/schedulers/schedule_cron_jobs' + +Travis::Api::App.new +Travis.logger.info "Starting cron jobs scheduler now" +Travis::Api::App::Schedulers::ScheduleCronJobs.run diff --git a/config/database.yml b/config/database.yml deleted file mode 100644 index 97ba5e01cf..0000000000 --- a/config/database.yml +++ /dev/null @@ -1,22 +0,0 @@ -# This is only used for standalone_migrations. In other contexts -# Travis.config.database will be used. - -defaults: &defaults - adapter: postgresql - encoding: unicode - pool: 5 - min_messages: warning - username: <%= ENV['USER'] %> - host: localhost - -production: - <<: *defaults - database: travis_development - -development: - <<: *defaults - database: travis_development - -test: - <<: *defaults - database: travis_test diff --git a/config/nginx.conf.erb b/config/nginx.conf.erb index 0d752e1b72..492b6c283d 100644 --- a/config/nginx.conf.erb +++ b/config/nginx.conf.erb @@ -36,6 +36,20 @@ http { keepalive_timeout 5; location / { + if ($request_method = 'OPTIONS') { + # mirroring https://github.com/travis-ci/travis-api/blob/master/lib/travis/api/app/cors.rb + # performing in nginx to short-circuit those requests and respond more quickly + + add_header 'Access-Control-Allow-Origin' '*'; + add_header 'Access-Control-Allow-Credentials' 'true'; + add_header 'Access-Control-Expose-Headers' 'Content-Type, Cache-Control, Expires, Etag, Last-Modified'; + add_header 'Access-Control-Allow-Methods' 'HEAD, GET, POST, PATCH, PUT, DELETE'; + add_header 'Access-Control-Allow-Headers' 'Content-Type, Authorization, Accept, If-None-Match, If-Modified-Since, X-User-Agent, Travis-API-Version'; + add_header 'Access-Control-Max-Age' 86400; + + return 204; + } + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Host $http_host; proxy_redirect off; diff --git a/heroku-buildpack-sigsci.tgz b/heroku-buildpack-sigsci.tgz deleted file mode 100644 index 1b0b6c83ac..0000000000 Binary files a/heroku-buildpack-sigsci.tgz and /dev/null differ diff --git a/lib/tasks/populate_dont_run_if_recent_build_exists.rake b/lib/tasks/populate_dont_run_if_recent_build_exists.rake new file mode 100644 index 0000000000..b0c8e80395 --- /dev/null +++ b/lib/tasks/populate_dont_run_if_recent_build_exists.rake @@ -0,0 +1,14 @@ +# Populate :dont_run_if_recent_build_exists in Crons table +# after the column has been added +desc "Populate dont_run_if_recent_build_exists for all cron jobs" +task :populate_dont_run_if_recent_build_exists do + require "travis" + require "travis/api/v3" + Travis::Database.connect + + Travis::API::V3::Models::Cron.all.each do |cron| + cron.update_attribute(:dont_run_if_recent_build_exists, cron.disable_by_build) + end +end + +task default: :populate_dont_run_if_recent_build_exists diff --git a/lib/tasks/populate_next_and_last_runs_for_crons.rake b/lib/tasks/populate_next_and_last_runs_for_crons.rake new file mode 100644 index 0000000000..f42906f052 --- /dev/null +++ b/lib/tasks/populate_next_and_last_runs_for_crons.rake @@ -0,0 +1,25 @@ +# Populate :next_run and :last_run in Crons table +# after these columns have been added +desc "Populate next and last runs for all cron jobs" +task :populate_next_last_cron_runs do + require "travis" + require "travis/api/v3" + Travis::Database.connect + + Travis::API::V3::Models::Cron.all.each do |cron| + cron.update_attribute(:last_run, last_cron_build_date(cron)) + cron.schedule_next_build(from: cron.last_run || cron.created_at) + end +end + +task default: :populate_next_last_cron_runs + +def last_cron_build_date(cron) + last_cron_build = Travis::API::V3::Models::Build.where( + repository_id: cron.branch.repository.id, + branch: cron.branch.name, + event_type: "cron" + ).order("id DESC").first + return last_cron_build.created_at unless last_cron_build.nil? + nil +end diff --git a/lib/travis/api/app/cors.rb b/lib/travis/api/app/cors.rb index 1b93c751db..43cfd8318e 100644 --- a/lib/travis/api/app/cors.rb +++ b/lib/travis/api/app/cors.rb @@ -13,6 +13,7 @@ class Cors < Base end options // do + # make sure to update nginx.conf.erb when you update this headers['Access-Control-Allow-Methods'] = "HEAD, GET, POST, PATCH, PUT, DELETE" headers['Access-Control-Allow-Headers'] = "Content-Type, Authorization, Accept, If-None-Match, If-Modified-Since, X-User-Agent, Travis-API-Version" diff --git a/lib/travis/api/app/endpoint/authorization.rb b/lib/travis/api/app/endpoint/authorization.rb index a8676fa48c..0597077332 100644 --- a/lib/travis/api/app/endpoint/authorization.rb +++ b/lib/travis/api/app/endpoint/authorization.rb @@ -383,8 +383,6 @@ def target_ok?(target_origin) uri.scheme == 'https' elsif uri.host =~ /\A(.+\.)?travis-ci\.(com|org)\Z/ uri.scheme == 'https' - elsif uri.host =~ /\A(.+\.)?travis-lite\.com\Z/ - uri.scheme == 'https' elsif uri.host == 'localhost' or uri.host == '127.0.0.1' uri.port > 1023 end diff --git a/lib/travis/api/app/endpoint/builds.rb b/lib/travis/api/app/endpoint/builds.rb index b23b82580e..36d7a8b704 100644 --- a/lib/travis/api/app/endpoint/builds.rb +++ b/lib/travis/api/app/endpoint/builds.rb @@ -1,6 +1,4 @@ require 'travis/api/app' -require 'travis/api/workers/build_cancellation' -require 'travis/api/workers/build_restart' require 'travis/api/enqueue/services/restart_model' require 'travis/api/enqueue/services/cancel_model' diff --git a/lib/travis/api/app/endpoint/endpoints.rb b/lib/travis/api/app/endpoint/endpoints.rb index ffe2219c57..ac2e3d5503 100644 --- a/lib/travis/api/app/endpoint/endpoints.rb +++ b/lib/travis/api/app/endpoint/endpoints.rb @@ -10,7 +10,9 @@ class Endpoints < Endpoint set :setup do endpoint_files = Dir.glob(File.expand_path("../*.rb", __FILE__)) - YARD::Registry.load(endpoint_files, true) + + # Only force reparse in development, as yardoc is generated the first run + YARD::Registry.load(endpoint_files, Travis.env == 'development') YARD::Sinatra.routes.each do |route| namespace = route.namespace diff --git a/lib/travis/api/app/endpoint/jobs.rb b/lib/travis/api/app/endpoint/jobs.rb index 4a217625ab..5c92849ffe 100644 --- a/lib/travis/api/app/endpoint/jobs.rb +++ b/lib/travis/api/app/endpoint/jobs.rb @@ -1,6 +1,4 @@ require 'travis/api/app' -require 'travis/api/workers/job_cancellation' -require 'travis/api/workers/job_restart' require 'travis/api/enqueue/services/restart_model' require 'travis/api/enqueue/services/cancel_model' diff --git a/lib/travis/api/app/endpoint/repos.rb b/lib/travis/api/app/endpoint/repos.rb index b4f9843813..cc9d5d50d0 100644 --- a/lib/travis/api/app/endpoint/repos.rb +++ b/lib/travis/api/app/endpoint/repos.rb @@ -122,7 +122,7 @@ class Repos < Endpoint # json(:repository) get '/:owner_name/:name' do prefer_follower do - respond_with service(:find_repo, params) + respond_with service(:find_repo, params), type_hint: Repository end end diff --git a/lib/travis/api/app/helpers/respond_with.rb b/lib/travis/api/app/helpers/respond_with.rb index 08182992d2..0c7992a546 100644 --- a/lib/travis/api/app/helpers/respond_with.rb +++ b/lib/travis/api/app/helpers/respond_with.rb @@ -16,10 +16,12 @@ module RespondWith def respond_with(resource, options = {}) result = respond(resource, options) + if result && response.content_type =~ /application\/json/ status STATUS[result[:result]] if result.is_a?(Hash) && result[:result].is_a?(Symbol) result = prettify_result? ? JSON.pretty_generate(result) : result.to_json end + halt result || 404 end diff --git a/lib/travis/api/app/responders/image.rb b/lib/travis/api/app/responders/image.rb index 5ce738146c..c90a4b399c 100644 --- a/lib/travis/api/app/responders/image.rb +++ b/lib/travis/api/app/responders/image.rb @@ -17,7 +17,9 @@ def apply end def apply? - super && resource.is_a?(Repository) + # :type_hint is returned by repos endpoint + # so that we can return a 'unknown.png' image + (super && resource.is_a?(Repository)) || (acceptable_format? && resource.nil? && options[:type_hint] == Repository) end private @@ -31,7 +33,11 @@ def filename end def result - Repository::StatusImage.new(resource, params[:branch]).result + if resource + Repository::StatusImage.new(resource, params[:branch]).result + else + 'unknown' + end end def root diff --git a/lib/travis/api/app/schedulers/schedule_cron_jobs.rb b/lib/travis/api/app/schedulers/schedule_cron_jobs.rb new file mode 100644 index 0000000000..a90d1ff0ac --- /dev/null +++ b/lib/travis/api/app/schedulers/schedule_cron_jobs.rb @@ -0,0 +1,51 @@ +require "travis/lock" +require "redlock" +require "metriks" + +class Travis::Api::App + module Schedulers + class ScheduleCronJobs < Travis::Services::Base + register :schedule_cron_jobs + + def self.run + loop do + begin + Travis::Lock.exclusive("enqueue_cron_jobs", options) do + Metriks.timer("api.cron_scheduler.enqueue").time { enqueue } + end + rescue Travis::Lock::Redis::LockError => e + Travis.logger.error e.message + end + sleep(Travis::API::V3::Models::Cron::SCHEDULER_INTERVAL) + end + end + + def self.options + @options ||= { + strategy: :redis, + url: Travis.config.redis.url, + retries: 0 + } + end + + def self.enqueue + scheduled = Travis::API::V3::Models::Cron.scheduled + count = scheduled.count + + Travis.logger.info "Found #{count} cron jobs to enqueue" + + Metriks.gauge("api.cron_scheduler.upcoming_jobs").set(count) + + scheduled.each do |cron| + begin + cron.needs_new_build? ? cron.enqueue : cron.skip_and_schedule_next_build + rescue => e + Metriks.meter("api.cron_scheduler.enqueue.error").mark + Raven.capture_exception(e, tags: { 'cron_id' => cron.try(:id) }) + next + end + end + end + end + end +end \ No newline at end of file diff --git a/lib/travis/api/v3.rb b/lib/travis/api/v3.rb index f05aec5e06..e76e821707 100644 --- a/lib/travis/api/v3.rb +++ b/lib/travis/api/v3.rb @@ -10,6 +10,7 @@ def load_dir(dir, recursive: true) def response(payload, headers = {}, content_type: 'application/json'.freeze, status: 200) payload = JSON.pretty_generate(payload) unless payload.is_a? String + content_type = 'text/plain'.freeze if payload.is_a? String headers = { 'Content-Type'.freeze => content_type, 'Content-Length'.freeze => payload.bytesize.to_s }.merge!(headers) [status, headers, [payload]] end @@ -36,6 +37,8 @@ def location(env) InsufficientAccess = ClientError .create(status: 403) JobAlreadyRunning = ClientError .create('job already running, cannot restart', status: 409) JobNotCancelable = ClientError .create('job is not running, cannot cancel', status: 409) + JobUnfinished = ClientError .create('job still running, cannot remove log yet', status: 409) + LogAlreadyRemoved = ClientError .create('log has already been removed', status: 409) LoginRequired = ClientError .create('login required', status: 403) MethodNotAllowed = ClientError .create('method not allowed', status: 405) NotImplemented = ServerError .create('request not (yet) implemented', status: 501) diff --git a/lib/travis/api/v3/access_control/generic.rb b/lib/travis/api/v3/access_control/generic.rb index 705e5071a6..232aa5dfec 100644 --- a/lib/travis/api/v3/access_control/generic.rb +++ b/lib/travis/api/v3/access_control/generic.rb @@ -23,6 +23,10 @@ def writable?(object) full_access? or dispatch(object, :writable?) end + def adminable?(object) + dispatch(object, :adminable?) + end + def admin_for(repository) raise AdminAccessRequired, repository: repository end @@ -99,6 +103,10 @@ def organization_visible?(organization) full_access? or public_api? end + def ssl_key_writable?(ssl_key) + writable? ssl_key.repository + end + def user_visible?(user) unrestricted_api? end @@ -107,6 +115,10 @@ def user_writable?(user) self.user == user end + def repository_adminable?(repository) + false + end + def repository_visible?(repository) return true if unrestricted_api? and not repository.private? private_repository_visible?(repository) diff --git a/lib/travis/api/v3/access_control/user.rb b/lib/travis/api/v3/access_control/user.rb index c2ac4ae9d0..75b58fe739 100644 --- a/lib/travis/api/v3/access_control/user.rb +++ b/lib/travis/api/v3/access_control/user.rb @@ -41,6 +41,10 @@ def job_restartable?(job) permission?(:pull, job.repository) end + def repository_adminable?(repository) + permission?(:admin, repository) + end + def organization_visible?(organization) super or organization_writable?(organization) end diff --git a/lib/travis/api/v3/github.rb b/lib/travis/api/v3/github.rb index cb9c389c97..aa68498569 100644 --- a/lib/travis/api/v3/github.rb +++ b/lib/travis/api/v3/github.rb @@ -14,6 +14,10 @@ class GitHub } private_constant :DEFAULT_OPTIONS + EVENTS = %i(push pull_request issue_comment public member create delete + membership repository) + private_constant :EVENTS + def self.client_config { api_url: DEFAULT_OPTIONS[:api_url], @@ -38,7 +42,7 @@ def set_hook(repository, flag) hooks_url = "repos/#{repository.slug}/hooks" payload = { name: 'travis'.freeze, - events: [:push, :pull_request, :issue_comment, :public, :member], + events: EVENTS, active: flag, config: { domain: Travis.config.service_hook_url || '' } } diff --git a/lib/travis/api/v3/models/admin_settings.rb b/lib/travis/api/v3/models/admin_settings.rb index 4cf5cc9343..f5511bc473 100644 --- a/lib/travis/api/v3/models/admin_settings.rb +++ b/lib/travis/api/v3/models/admin_settings.rb @@ -2,7 +2,7 @@ module Travis::API::V3 class Models::AdminSettings < Models::JsonSlice - pair Models::AdminSetting + child Models::AdminSetting attribute :api_builds_rate_limit, Integer end end diff --git a/lib/travis/api/v3/models/branch.rb b/lib/travis/api/v3/models/branch.rb index 147a8bfe44..88cefb5a5b 100644 --- a/lib/travis/api/v3/models/branch.rb +++ b/lib/travis/api/v3/models/branch.rb @@ -4,7 +4,7 @@ class Models::Branch < Model belongs_to :last_build, class_name: 'Travis::API::V3::Models::Build'.freeze has_many :builds, foreign_key: [:repository_id, :branch], primary_key: [:repository_id, :name], order: 'builds.id DESC'.freeze, conditions: { event_type: 'push' } has_many :commits, foreign_key: [:repository_id, :branch], primary_key: [:repository_id, :name], order: 'commits.id DESC'.freeze - has_one :cron, dependent: :delete + has_one :cron, dependent: :destroy def default_branch name == repository.default_branch_name diff --git a/lib/travis/api/v3/models/cache.rb b/lib/travis/api/v3/models/cache.rb new file mode 100644 index 0000000000..b35c9e5bbb --- /dev/null +++ b/lib/travis/api/v3/models/cache.rb @@ -0,0 +1,20 @@ +module Travis::API::V3 + class Models::Cache + attr_accessor :repo, :repository_id, :size, :slug, :branch, :last_modified + + def self.factory(caches, repo) + caches.map do |c| + new(c, repo) + end + end + + def initialize(cache, repo) + @repo = repo + @repository_id = repo.id + @size = Integer(cache.content_length) + @slug = cache.key.to_s.split('/').last + @branch = cache.key[%r{^\d+/(.*)/[^/]+$}, 1] + @last_modified = cache.last_modified + end + end +end diff --git a/lib/travis/api/v3/models/cron.rb b/lib/travis/api/v3/models/cron.rb index e86527a993..68a270dafe 100644 --- a/lib/travis/api/v3/models/cron.rb +++ b/lib/travis/api/v3/models/cron.rb @@ -1,75 +1,88 @@ module Travis::API::V3 class Models::Cron < Model + SCHEDULER_INTERVAL = 1.minute belongs_to :branch + after_create :schedule_first_build - LastBuild = -1 - ThisBuild = 0 - NextBuild = 1 - - def next_enqueuing - if disable_by_build && last_non_cron_build_date > planned_time(LastBuild) - planned_time(NextBuild) - elsif last_cron_build_date >= planned_time(LastBuild) - planned_time(ThisBuild) - else - Time.now - 5.minutes - end - end + scope :scheduled, -> { where("next_run <= '#{DateTime.now.utc}'") } - def planned_time(in_builds = ThisBuild) - case interval - when 'daily' - planned_time_daily(in_builds) - when 'weekly' - planned_time_weekly(in_builds) - when 'monthly' - planned_time_monthly(in_builds) + TIME_INTERVALS = { + "daily" => :day, + "weekly" => :week, + "monthly" => :month + } + + def schedule_next_build(from: nil) + # Make sure the next build will always be in the future + if (from && (from <= (DateTime.now.utc - 1.send(TIME_INTERVALS[interval])))) + from = DateTime.now.utc end + + update_attribute(:next_run, (from || last_run || DateTime.now.utc) + 1.send(TIME_INTERVALS[interval])) end - def planned_time_daily(in_builds) - now = DateTime.now - build_today = DateTime.new(now.year, now.month, now.day, created_at.hour) - return build_today + 1 + in_builds if (now > build_today) - build_today + in_builds + def schedule_first_build + update_attribute(:next_run, DateTime.now.utc + SCHEDULER_INTERVAL) end - def planned_time_weekly(in_builds) - now = DateTime.now - build_today = DateTime.new(now.year, now.month, now.day, created_at.hour) - next_time = build_today + ((created_at.wday - now.wday) % 7) - return build_today + 7 * (1 + in_builds) if (now > next_time) - next_time + 7 * in_builds + def needs_new_build? + always_run? || !last_non_cron_build_time || last_non_cron_build_time < 24.hour.ago end - def planned_time_monthly(in_builds) - now = DateTime.now - created = DateTime.new(created_at.year, created_at.month, created_at.day, created_at.hour) - month_since_creation = (now.year * 12 + now.month) - (created_at.year * 12 + created_at.month) - this_month = created >> month_since_creation - return created >> (month_since_creation + 1 + in_builds) if (now > this_month) - created >> (month_since_creation + in_builds) + def skip_and_schedule_next_build + # Update last_run also, because this build wasn't enqueued through Queries::Crons + last_build_time = last_non_cron_build_time + update_attribute(:last_run, last_build_time) + + schedule_next_build(from: DateTime.now) end - def last_cron_build_date - last_cron_build = Models::Build.where( - :repository_id => branch.repository.id, - :branch => branch.name, - :event_type => 'cron' - ).order("id DESC").first - return last_cron_build.created_at unless last_cron_build.nil? - Time.at(0) + def enqueue + if !branch.repository.github_id + raise StandardError, "Repository does not have a github_id" + end + + if !branch.exists_on_github + self.destroy + return false + end + + user_id = branch.repository.users.detect { |u| u.github_oauth_token }.try(:id) + user_id ||= branch.repository.owner.id + + payload = { + repository: { + id: branch.repository.github_id, + owner_name: branch.repository.owner_name, + name: branch.repository.name }, + branch: branch.name, + user: { id: user_id } + } + + class_name, queue = Query.sidekiq_queue(:build_request) + + ::Sidekiq::Client.push( + 'queue'.freeze => queue, + 'class'.freeze => class_name, + 'args'.freeze => [{ + type: 'cron'.freeze, + payload: JSON.dump(payload), + credentials: {} + }]) + + update_attribute(:last_run, DateTime.now.utc) + schedule_next_build end - def last_non_cron_build_date - last_build = Models::Build.where( - :repository_id => branch.repository.id, - :branch => branch.name - ).where(['event_type NOT IN (?)', ['cron']]).order("id DESC").first - return last_build.created_at unless last_build.nil? - Time.at(0) + private + def always_run? + !dont_run_if_recent_build_exists end + def last_non_cron_build_time + last_build = Build.find_by_id(branch.last_build_id) + last_build.finished_at.to_datetime.utc if last_build + end end end diff --git a/lib/travis/api/v3/models/env_var.rb b/lib/travis/api/v3/models/env_var.rb index 27f2606f81..ae937d1010 100644 --- a/lib/travis/api/v3/models/env_var.rb +++ b/lib/travis/api/v3/models/env_var.rb @@ -6,13 +6,13 @@ class Models::EnvVar < Travis::Settings::Model attribute :public, Boolean attribute :repository_id, Integer - def repository - @repository ||= Models::Repository.find(repository_id) - end - validates_each :id, :name do |record, attr, value| others = record.repository.env_vars.select { |ev| ev.id != record.id } record.errors.add(:base, :duplicate_resource) if others.find { |ev| ev.send(attr) == record.send(attr) } end + + def repository + @repository ||= Models::Repository.find(repository_id) + end end end diff --git a/lib/travis/api/v3/models/env_vars.rb b/lib/travis/api/v3/models/env_vars.rb index 1e4b7d0860..28a6186ee3 100644 --- a/lib/travis/api/v3/models/env_vars.rb +++ b/lib/travis/api/v3/models/env_vars.rb @@ -1,11 +1,30 @@ +require_relative './json_sync' + module Travis::API::V3 class Models::EnvVars < Travis::Settings::Collection + include Models::JsonSync model Models::EnvVar + # See Models::JsonSync + def to_h + { 'env_vars' => map(&:to_h).map(&:stringify_keys) } + end + + def create(attributes) + super(attributes).tap { sync! } + end + + def add(env_var) + destroy(env_var.id) if find(env_var.id) + create(env_var.attributes) + end + + def destroy(id) + super(id).tap { sync! } + end + def repository @repository ||= Models::Repository.find(additional_attributes[:repository_id]) end - - undef :to_hash end end diff --git a/lib/travis/api/v3/models/json_slice.rb b/lib/travis/api/v3/models/json_slice.rb index 5feb063a76..6fda05f808 100644 --- a/lib/travis/api/v3/models/json_slice.rb +++ b/lib/travis/api/v3/models/json_slice.rb @@ -1,46 +1,36 @@ +require_relative './json_sync' + module Travis::API::V3 class Models::JsonSlice - include Enumerable - include Virtus.model - - attr_reader :parent, :attr + include Virtus.model, Enumerable, Models::JsonSync - def self.pair(klass) - @@pair = klass + def self.child(klass) + @@child_klass = klass end - def pair - self.class.class_variable_get(:@@pair) + def child_klass + @@child_klass end def each(&block) return enum_for(:each) unless block_given? - attributes.keys.each { |name| yield read(name) } + attributes.keys.each { |id| yield read(id) } self end def read(name) value = send(name) - pair.new(name, value, parent) unless value.nil? + child_klass.new(name, value, parent) unless value.nil? end def update(name, value) send(:"#{name}=", value) - @sync.call if @sync + sync! read(name) end def to_h Hash[map { |x| [x.name, x.value] }] end - - def parent_attr(parent, attr) - @parent, @attr = parent, attr - @sync = -> do - previous = @parent.send(:"#{@attr}") - @parent.send(:"#{@attr}=", previous.merge(to_h).to_json) - @parent.save! - end - end end end diff --git a/lib/travis/api/v3/models/json_sync.rb b/lib/travis/api/v3/models/json_sync.rb new file mode 100644 index 0000000000..6c6e61567d --- /dev/null +++ b/lib/travis/api/v3/models/json_sync.rb @@ -0,0 +1,22 @@ +module Travis::API::V3 + module Models::JsonSync + attr_reader :parent, :attr + + def sync(parent, attr) + @parent, @attr = parent, attr + @sync = -> do + previous = @parent.send(:"#{@attr}") + @parent.send(:"#{@attr}=", previous.merge(to_h).to_json) + @parent.save! + end + end + + def sync! + @sync.call if @sync + end + + def to_h + raise NotImplementedError + end + end +end diff --git a/lib/travis/api/v3/models/log.rb b/lib/travis/api/v3/models/log.rb index d1c3219bf3..88923f4919 100644 --- a/lib/travis/api/v3/models/log.rb +++ b/lib/travis/api/v3/models/log.rb @@ -1,7 +1,23 @@ module Travis::API::V3 class Models::Log < Model + establish_connection(Travis.config.logs_database) + belongs_to :job belongs_to :removed_by, class_name: 'User', foreign_key: :removed_by - has_many :log_parts, dependent: :destroy + has_many :log_parts, dependent: :destroy, order: 'number ASC' + + def clear!(user) + removed_at = Time.now.utc + message ="Log removed by #{user.name} at #{removed_at}" + update_attributes!( + :content => nil, + :aggregated_at => nil, + :archived_at => nil, + :removed_at => removed_at, + :removed_by => user + ) + log_parts.destroy_all + log_parts.create(content: message, number: 1, final: true) + end end end diff --git a/lib/travis/api/v3/models/log_part.rb b/lib/travis/api/v3/models/log_part.rb index 43fc737030..054e4b8c9e 100644 --- a/lib/travis/api/v3/models/log_part.rb +++ b/lib/travis/api/v3/models/log_part.rb @@ -1,5 +1,6 @@ module Travis::API::V3 class Models::LogPart < Model + establish_connection(Travis.config.logs_database) belongs_to :log end end diff --git a/lib/travis/api/v3/models/repository.rb b/lib/travis/api/v3/models/repository.rb index 219b6081e3..bff0808f32 100644 --- a/lib/travis/api/v3/models/repository.rb +++ b/lib/travis/api/v3/models/repository.rb @@ -12,7 +12,7 @@ class Models::Repository < Model belongs_to :last_build, class_name: 'Travis::API::V3::Models::Build'.freeze belongs_to :current_build, class_name: 'Travis::API::V3::Models::Build'.freeze - has_one :key, class_name: 'Travis::API::V3::Models::SSLKey'.freeze + has_one :key, class_name: 'Travis::API::V3::Models::SslKey'.freeze has_one :default_branch, foreign_key: [:repository_id, :name], primary_key: [:id, :default_branch], @@ -70,16 +70,17 @@ def settings end def user_settings - Models::UserSettings.new(settings).tap { |us| us.parent_attr(self, :settings) } + Models::UserSettings.new(settings).tap { |us| us.sync(self, :settings) } end def admin_settings - Models::AdminSettings.new(settings).tap { |as| as.parent_attr(self, :settings) } + Models::AdminSettings.new(settings).tap { |as| as.sync(self, :settings) } end def env_vars - Models::EnvVars.new.tap do |vars| - vars.load(settings.fetch('env_vars', []), repository_id: self.id) + Models::EnvVars.new.tap do |ev| + ev.load(settings.fetch('env_vars', []), repository_id: id) + ev.sync(self, :settings) end end end diff --git a/lib/travis/api/v3/models/ssl_key.rb b/lib/travis/api/v3/models/ssl_key.rb index bd9685c762..34b96ed61c 100644 --- a/lib/travis/api/v3/models/ssl_key.rb +++ b/lib/travis/api/v3/models/ssl_key.rb @@ -1,9 +1,31 @@ +require 'openssl' + module Travis::API::V3 - class Models::SSLKey < Model + class Models::SslKey < Model belongs_to :repository serialize :private_key, Travis::Settings::EncryptedColumn.new + def generate_keys! + self.public_key = self.private_key = nil + generate_keys + end + + def generate_keys + unless public_key && private_key + keys = OpenSSL::PKey::RSA.generate(Travis.config.repository.ssl_key.size) + self.public_key = keys.public_key.to_s + self.private_key = keys.to_pem + end + end + + def fingerprint + return unless public_key + rsa_key = OpenSSL::PKey::RSA.new(public_key) + public_ssh_rsa = "\x00\x00\x00\x07ssh-rsa" + rsa_key.e.to_s(0) + rsa_key.n.to_s(0) + OpenSSL::Digest::MD5.new(public_ssh_rsa).hexdigest.scan(/../).join(':') + end + def encoded_public_key key = build_key.public_key ['ssh-rsa ', "\0\0\0\assh-rsa#{sized_bytes(key.e)}#{sized_bytes(key.n)}"].pack('a*m').gsub("\n", '') diff --git a/lib/travis/api/v3/models/user_settings.rb b/lib/travis/api/v3/models/user_settings.rb index a3c31c76ec..c8abd60d70 100644 --- a/lib/travis/api/v3/models/user_settings.rb +++ b/lib/travis/api/v3/models/user_settings.rb @@ -1,11 +1,13 @@ module Travis::API::V3 class Models::UserSettings < Models::JsonSlice - pair Models::UserSetting + child Models::UserSetting attribute :builds_only_with_travis_yml, Boolean, default: false attribute :build_pushes, Boolean, default: true attribute :build_pull_requests, Boolean, default: true attribute :maximum_number_of_builds, Integer, default: 0 + attribute :auto_cancel_pushes, Boolean, default: false + attribute :auto_cancel_pull_requests, Boolean, default: false def repository_id parent && parent.id diff --git a/lib/travis/api/v3/permissions/generic.rb b/lib/travis/api/v3/permissions/generic.rb index 2ff7025fe2..2ed1f5e7e5 100644 --- a/lib/travis/api/v3/permissions/generic.rb +++ b/lib/travis/api/v3/permissions/generic.rb @@ -64,5 +64,6 @@ def cancelable? def restartable? access_control.restartable? object end + end end diff --git a/lib/travis/api/v3/permissions/job.rb b/lib/travis/api/v3/permissions/job.rb index 835ecd4302..3cadbec53f 100644 --- a/lib/travis/api/v3/permissions/job.rb +++ b/lib/travis/api/v3/permissions/job.rb @@ -13,5 +13,9 @@ def restart? def debug? write? end + + def delete_log? + write? + end end end diff --git a/lib/travis/api/v3/permissions/repository.rb b/lib/travis/api/v3/permissions/repository.rb index 1935387e26..820e4ecfa8 100644 --- a/lib/travis/api/v3/permissions/repository.rb +++ b/lib/travis/api/v3/permissions/repository.rb @@ -29,5 +29,17 @@ def create_cron? def change_settings? write? end + + def change_env_vars? + write? + end + + def change_key? + write? + end + + def admin? + access_control.adminable? object + end end end diff --git a/lib/travis/api/v3/queries/caches.rb b/lib/travis/api/v3/queries/caches.rb new file mode 100644 index 0000000000..dfd675c846 --- /dev/null +++ b/lib/travis/api/v3/queries/caches.rb @@ -0,0 +1,36 @@ +module Travis::API::V3 + class Queries::Caches < RemoteQuery + params :match, :branch + + def find(repo) + @repo = repo + caches = fetch + filter Models::Cache.factory(caches, repo) + end + + def delete(repo) + @repo = repo + destroyed_caches = remove + filter Models::Cache.factory(destroyed_caches, repo) + end + + def filter(list) + return list unless match + list.select{|c| c.slug.include? match} + end + + private + + def prefix + "#{@repo.github_id.to_s}/#{branch}" + end + + def s3_config + config.cache_options.try(:s3) || {} + end + + def gcs_config + config.cache_options.try(:gcs) || {} + end + end +end diff --git a/lib/travis/api/v3/queries/cron.rb b/lib/travis/api/v3/queries/cron.rb index 43d3cb5f5a..4b65ef66c2 100644 --- a/lib/travis/api/v3/queries/cron.rb +++ b/lib/travis/api/v3/queries/cron.rb @@ -13,9 +13,9 @@ def find_for_branch(branch) branch.cron end - def create(branch, interval, disable_by_build) + def create(branch, interval, dont_run_if_recent_build_exists) branch.cron.destroy unless branch.cron.nil? - Models::Cron.create(branch: branch, interval: interval, disable_by_build: disable_by_build) + Models::Cron.create(branch: branch, interval: interval, dont_run_if_recent_build_exists: dont_run_if_recent_build_exists) end end end diff --git a/lib/travis/api/v3/queries/crons.rb b/lib/travis/api/v3/queries/crons.rb index 83a00245ff..fceb2e45de 100644 --- a/lib/travis/api/v3/queries/crons.rb +++ b/lib/travis/api/v3/queries/crons.rb @@ -1,43 +1,7 @@ module Travis::API::V3 class Queries::Crons < Query - def find(repository) - Models::Cron.where(:branch_id => repository.branches) - end - - def start_all() - Models::Cron.all.select do |cron| - begin - @cron = cron - start(cron) if cron.next_enqueuing <= Time.now - rescue => e - Raven.capture_exception(e, tags: { 'cron_id' => @cron.try(:id) }) - sleep(10) # This ensures the dyno does not spin down before the http request to send the error to sentry completes - next - end - end - end - - def start(cron) - branch = cron.branch - raise ServerError, 'repository does not have a github_id'.freeze unless branch.repository.github_id - unless branch.exists_on_github - cron.destroy - return false - end - - user_id = branch.repository.users.detect { |u| u.github_oauth_token }.try(:id) - user_id ||= branch.repository.owner.id - - payload = { - repository: { id: branch.repository.github_id, owner_name: branch.repository.owner_name, name: branch.repository.name }, - branch: branch.name, - user: { id: user_id } - } - - class_name, queue = Query.sidekiq_queue(:build_request) - ::Sidekiq::Client.push('queue'.freeze => queue, 'class'.freeze => class_name, 'args'.freeze => [{type: 'cron'.freeze, payload: JSON.dump(payload), credentials: {}}]) - true + Models::Cron.where(branch_id: repository.branches) end end end diff --git a/lib/travis/api/v3/queries/env_var.rb b/lib/travis/api/v3/queries/env_var.rb index 4ffaeab955..d2f48570bd 100644 --- a/lib/travis/api/v3/queries/env_var.rb +++ b/lib/travis/api/v3/queries/env_var.rb @@ -9,7 +9,7 @@ def find(repository) def update(repository) if env_var = find(repository) env_var.update(env_var_params) - repository.save! + repository.env_vars.add(env_var) env_var end end diff --git a/lib/travis/api/v3/queries/log.rb b/lib/travis/api/v3/queries/log.rb new file mode 100644 index 0000000000..eced10a887 --- /dev/null +++ b/lib/travis/api/v3/queries/log.rb @@ -0,0 +1,56 @@ +module Travis::API::V3 + class Queries::Log < RemoteQuery + + def find(job) + @job = job + #check for the log in the Logs DB + log = Models::Log.find_by_job_id(@job.id) + raise EntityMissing, 'log not found'.freeze if log.nil? + #if the log has been archived, go to s3 + if log.archived_at + content = fetch.get(prefix).try(:body) + create_log_parts(log, content) + #if log has been aggregated, look at log.content + elsif log.aggregated_at + create_log_parts(log, log.content) + end + log + end + + def create_log_parts(log, content) + log.log_parts << Models::LogPart.new(log_id: log.id, content: content, number: 0, created_at: log.created_at) + end + + def delete(user, job) + @job = job + log = Models::Log.find_by_job_id(@job.id) + raise EntityMissing, 'log not found'.freeze if log.nil? + raise LogAlreadyRemoved if log.removed_at || log.removed_by + raise JobUnfinished unless @job.finished_at? + + remove if log.archived_at + + log.clear!(user) + log + end + + private + + def prefix + "jobs/#{@job.id}/log.txt" + end + + def bucket_name + hostname('archive') + end + + def hostname(name) + "#{name}#{'-staging' if Travis.env == 'staging'}.#{Travis.config.host.split('.')[-2, 2].join('.')}" + end + + def s3_config + conf = config.logs_options.try(:s3) || {} + conf.merge!(bucket_name: bucket_name) + end + end +end diff --git a/lib/travis/api/v3/queries/ssh_key.rb b/lib/travis/api/v3/queries/ssh_key.rb new file mode 100644 index 0000000000..5140154302 --- /dev/null +++ b/lib/travis/api/v3/queries/ssh_key.rb @@ -0,0 +1,15 @@ +module Travis::API::V3 + class Queries::SshKey < Query + def find(repository) + repository.key + end + + def regenerate(repository) + key = repository.key || repository.create_key + key.tap do |key| + key.generate_keys! + key.save! + end + end + end +end diff --git a/lib/travis/api/v3/queries/user_settings.rb b/lib/travis/api/v3/queries/user_settings.rb index 359d7394b1..a918356184 100644 --- a/lib/travis/api/v3/queries/user_settings.rb +++ b/lib/travis/api/v3/queries/user_settings.rb @@ -1,7 +1,26 @@ module Travis::API::V3 class Queries::UserSettings < Query - def find(repository) - repository.user_settings + FEATURE_FLAGGED = %i( + auto_cancel_pushes + auto_cancel_pull_requests + ) + + def find(repo) + filter(repo, repo.user_settings) end + + private + + def filter(repo, settings) + settings.select { |setting| !flagged?(setting) || active?(repo) } + end + + def flagged?(setting) + FEATURE_FLAGGED.include?(setting.name) + end + + def active?(repo) + Travis::Features.owner_active?(:auto_cancel, repo.owner) + end end end diff --git a/lib/travis/api/v3/remote_query.rb b/lib/travis/api/v3/remote_query.rb new file mode 100644 index 0000000000..714c99522d --- /dev/null +++ b/lib/travis/api/v3/remote_query.rb @@ -0,0 +1,66 @@ +require 'fog/aws' +require 'fog/google' + +module Travis::API::V3 + class RemoteQuery < Query + def set + # This is for the future when we use the API to create a file on a remote + raise NotImplemented + fetch.create( + key: 'file key', + body: File.open("path to file"), + public: false + ) + end + + def fetch + storage_files + end + + def remove + caches = fetch + caches.each do |cache| + cache.destroy + end + end + + private + + def storage_files + storage_bucket.files + end + + def storage_bucket + aws_bucket = s3_bucket + return aws_bucket if aws_bucket + gcs_bucket + end + + def prefix + warn 'prefix in RemoteQuery called. If you wanted a prefix filter please impliment it in the subclass.' + '' + end + + def s3_bucket + s3 = Fog::Storage.new(aws_access_key_id: s3_config[:access_key_id], aws_secret_access_key: s3_config[:secret_access_key], provider: 'AWS') + s3.directories.get(s3_config[:bucket_name], prefix: prefix) + end + + def gcs_bucket + gcs = Fog::Storage::Google.new(google_json_key_string: gcs_config[:json_key], google_project: gcs_config[:google_project]) + gcs.directories.get(gcs_config[:bucket_name], prefix: prefix) + end + + def config + Travis.config + end + + def s3_config + raise NotImplemented + end + + def gcs_config + raise NotImplemented + end + end +end diff --git a/lib/travis/api/v3/renderer/cache.rb b/lib/travis/api/v3/renderer/cache.rb new file mode 100644 index 0000000000..67b6254bdf --- /dev/null +++ b/lib/travis/api/v3/renderer/cache.rb @@ -0,0 +1,6 @@ +module Travis::API::V3 + class Renderer::Cache < Renderer::ModelRenderer + representation(:minimal, :repository_id, :size, :slug, :branch, :last_modified) + representation(:standard, *representations[:minimal], :repo) + end +end diff --git a/lib/travis/api/v3/renderer/caches.rb b/lib/travis/api/v3/renderer/caches.rb new file mode 100644 index 0000000000..990ea92342 --- /dev/null +++ b/lib/travis/api/v3/renderer/caches.rb @@ -0,0 +1,6 @@ +module Travis::API::V3 + class Renderer::Caches < Renderer::CollectionRenderer + type :caches + collection_key :caches + end +end diff --git a/lib/travis/api/v3/renderer/cron.rb b/lib/travis/api/v3/renderer/cron.rb index 95419867df..4a18e9e5e8 100644 --- a/lib/travis/api/v3/renderer/cron.rb +++ b/lib/travis/api/v3/renderer/cron.rb @@ -3,7 +3,7 @@ module Travis::API::V3 class Renderer::Cron < Renderer::ModelRenderer representation(:minimal, :id) - representation(:standard, :id, :repository, :branch, :interval, :disable_by_build, :next_enqueuing, :created_at) + representation(:standard, :id, :repository, :branch, :interval, :dont_run_if_recent_build_exists, :last_run, :next_run, :created_at) def repository model.branch.repository diff --git a/lib/travis/api/v3/renderer/env_vars.rb b/lib/travis/api/v3/renderer/env_vars.rb index 9a47a84cb8..e1f1e817e4 100644 --- a/lib/travis/api/v3/renderer/env_vars.rb +++ b/lib/travis/api/v3/renderer/env_vars.rb @@ -1,6 +1,6 @@ module Travis::API::V3 class Renderer::EnvVars < Renderer::CollectionRenderer type :env_vars - collection_key :env_vars + collection_key :env_vars end end diff --git a/lib/travis/api/v3/renderer/log.rb b/lib/travis/api/v3/renderer/log.rb new file mode 100644 index 0000000000..95b8600006 --- /dev/null +++ b/lib/travis/api/v3/renderer/log.rb @@ -0,0 +1,28 @@ +module Travis::API::V3 + module Renderer::Log + extend self + + def render(log, **options) + text?(options) ? text(log) : json(log, **options) + end + + private + + def text?(options) + options[:accept] == 'text/plain'.freeze + end + + def text(log) + log.log_parts.map(&:content).join("\n") + end + + def json(log, **options) + Json.new(log, **options).render(:standard) + end + + class Json < Renderer::ModelRenderer + type :log + representation :standard, :id, :content, :log_parts + end + end +end diff --git a/lib/travis/api/v3/renderer/log_part.rb b/lib/travis/api/v3/renderer/log_part.rb new file mode 100644 index 0000000000..54043a5234 --- /dev/null +++ b/lib/travis/api/v3/renderer/log_part.rb @@ -0,0 +1,6 @@ +module Travis::API::V3 + class Renderer::LogPart < Renderer::ModelRenderer + representation(:minimal, :content, :number) + representation(:standard, *representations[:minimal], :log) + end +end diff --git a/lib/travis/api/v3/renderer/log_parts.rb b/lib/travis/api/v3/renderer/log_parts.rb new file mode 100644 index 0000000000..6d945a3238 --- /dev/null +++ b/lib/travis/api/v3/renderer/log_parts.rb @@ -0,0 +1,6 @@ +module Travis::API::V3 + class Renderer::LogParts < Renderer::CollectionRenderer + type :log_parts + collection_key :log_parts + end +end diff --git a/lib/travis/api/v3/renderer/ssh_key.rb b/lib/travis/api/v3/renderer/ssh_key.rb new file mode 100644 index 0000000000..b85b889ce3 --- /dev/null +++ b/lib/travis/api/v3/renderer/ssh_key.rb @@ -0,0 +1,5 @@ +module Travis::API::V3 + class Renderer::SshKey < Renderer::ModelRenderer + representation :standard, :id, :public_key, :fingerprint + end +end diff --git a/lib/travis/api/v3/result.rb b/lib/travis/api/v3/result.rb index 42a6c7271c..323aeb88c8 100644 --- a/lib/travis/api/v3/result.rb +++ b/lib/travis/api/v3/result.rb @@ -23,13 +23,14 @@ def ignored_param(param, reason: nil, **info) def render(params, env) href = self.href href = V3.location(env) if href.nil? and env['REQUEST_METHOD'.freeze] == 'GET'.freeze - include = params['include'.freeze].to_s.split(?,.freeze) + include = params.to_h['include'.freeze].to_s.split(?,.freeze) add_info Renderer[type].render(resource, href: href, script_name: env['SCRIPT_NAME'.freeze], include: include, access_control: access_control, - meta_data: meta_data) + meta_data: meta_data, + accept: env.fetch('HTTP_ACCEPT'.freeze, 'application/json'.freeze)) end def add_info(payload) diff --git a/lib/travis/api/v3/result/head.rb b/lib/travis/api/v3/result/head.rb new file mode 100644 index 0000000000..d15f0bc575 --- /dev/null +++ b/lib/travis/api/v3/result/head.rb @@ -0,0 +1,7 @@ +module Travis::API::V3 + class Result::Head < Result + def render(*) + ''.freeze + end + end +end diff --git a/lib/travis/api/v3/routes.rb b/lib/travis/api/v3/routes.rb index 5a7c206d2c..9517d63fe6 100644 --- a/lib/travis/api/v3/routes.rb +++ b/lib/travis/api/v3/routes.rb @@ -43,6 +43,13 @@ module Routes post :cancel, '/cancel' post :restart, '/restart' post :debug, '/debug' + + resource :log do + route '/log' + get :find + delete :delete + end + end resource :lint do @@ -107,6 +114,12 @@ module Routes get :find end + resource :caches do + route '/caches' + get :find + delete :delete + end + resource :crons do route '/crons' get :for_repository @@ -141,6 +154,12 @@ module Routes patch :update delete :delete end + + resource :ssh_key do + route '/ssh_key' + get :find + post :create + end end resource :user do diff --git a/lib/travis/api/v3/service.rb b/lib/travis/api/v3/service.rb index 7a87f44c83..0d4f69d587 100644 --- a/lib/travis/api/v3/service.rb +++ b/lib/travis/api/v3/service.rb @@ -89,6 +89,14 @@ def result(*args) Result.new(access_control, *args) end + def head(*args) + Result::Head.new(access_control, *args) + end + + def deleted + head result_type, nil, status: 204 + end + def run not_found unless result = run! result = result(result_type, result) unless result.is_a? Result diff --git a/lib/travis/api/v3/services.rb b/lib/travis/api/v3/services.rb index dd70757b3e..4222fc1028 100644 --- a/lib/travis/api/v3/services.rb +++ b/lib/travis/api/v3/services.rb @@ -9,6 +9,7 @@ module Services Broadcasts = Module.new { extend Services } Build = Module.new { extend Services } Builds = Module.new { extend Services } + Caches = Module.new { extend Services } Cron = Module.new { extend Services } Crons = Module.new { extend Services } EnvVar = Module.new { extend Services } @@ -16,12 +17,14 @@ module Services Job = Module.new { extend Services } Jobs = Module.new { extend Services } Lint = Module.new { extend Services } + Log = Module.new { extend Services } Organization = Module.new { extend Services } Organizations = Module.new { extend Services } Owner = Module.new { extend Services } Repositories = Module.new { extend Services } Repository = Module.new { extend Services } Requests = Module.new { extend Services } + SshKey = Module.new { extend Services } User = Module.new { extend Services } UserSetting = Module.new { extend Services } UserSettings = Module.new { extend Services } diff --git a/lib/travis/api/v3/services/caches/delete.rb b/lib/travis/api/v3/services/caches/delete.rb new file mode 100644 index 0000000000..7502c0305f --- /dev/null +++ b/lib/travis/api/v3/services/caches/delete.rb @@ -0,0 +1,9 @@ +module Travis::API::V3 + class Services::Caches::Delete < Service + params :match, :branch + + def run! + query.delete(find(:repository)) + end + end +end diff --git a/lib/travis/api/v3/services/caches/find.rb b/lib/travis/api/v3/services/caches/find.rb new file mode 100644 index 0000000000..db5d5c840d --- /dev/null +++ b/lib/travis/api/v3/services/caches/find.rb @@ -0,0 +1,9 @@ +module Travis::API::V3 + class Services::Caches::Find < Service + params :match, :branch + + def run! + query.find(find(:repository)) + end + end +end diff --git a/lib/travis/api/v3/services/cron/create.rb b/lib/travis/api/v3/services/cron/create.rb index 2fb366e109..7c0dd0e07b 100644 --- a/lib/travis/api/v3/services/cron/create.rb +++ b/lib/travis/api/v3/services/cron/create.rb @@ -1,7 +1,7 @@ module Travis::API::V3 class Services::Cron::Create < Service result_type :cron - params :interval, :disable_by_build + params :interval, :dont_run_if_recent_build_exists def run! repository = check_login_and_find(:repository) @@ -10,7 +10,7 @@ def run! raise Error.new('Invalid value for interval. Interval must be "daily", "weekly" or "monthly"!', status: 422) unless ["daily", "weekly", "monthly"].include?(params["interval"]) access_control.permissions(repository).create_cron! access_control.permissions(branch.cron).delete! if branch.cron - query.create(branch, params["interval"], params["disable_by_build"] ? params["disable_by_build"] : false) + query.create(branch, params["interval"], params["dont_run_if_recent_build_exists"] ? params["dont_run_if_recent_build_exists"] : false) end end diff --git a/lib/travis/api/v3/services/cron/delete.rb b/lib/travis/api/v3/services/cron/delete.rb index 59aa6d10b9..3162630cdb 100644 --- a/lib/travis/api/v3/services/cron/delete.rb +++ b/lib/travis/api/v3/services/cron/delete.rb @@ -1,11 +1,10 @@ module Travis::API::V3 class Services::Cron::Delete < Service - #params :id - def run! cron = check_login_and_find access_control.permissions(cron).delete! cron.destroy + deleted end end end diff --git a/lib/travis/api/v3/services/crons/start.rb b/lib/travis/api/v3/services/crons/start.rb deleted file mode 100644 index 104d43f5a3..0000000000 --- a/lib/travis/api/v3/services/crons/start.rb +++ /dev/null @@ -1,9 +0,0 @@ -module Travis::API::V3 - class Services::Crons::Start < Service - - def run! - query.start_all() - end - - end -end diff --git a/lib/travis/api/v3/services/env_var/delete.rb b/lib/travis/api/v3/services/env_var/delete.rb index 3ea473ab1f..ecd3534bc1 100644 --- a/lib/travis/api/v3/services/env_var/delete.rb +++ b/lib/travis/api/v3/services/env_var/delete.rb @@ -5,7 +5,8 @@ class Services::EnvVar::Delete < Service def run! repository = check_login_and_find(:repository) - query.delete(repository) + access_control.permissions(repository).change_env_vars! + query.delete(repository) and deleted end end end diff --git a/lib/travis/api/v3/services/log/delete.rb b/lib/travis/api/v3/services/log/delete.rb new file mode 100644 index 0000000000..ee3fd8e7b1 --- /dev/null +++ b/lib/travis/api/v3/services/log/delete.rb @@ -0,0 +1,9 @@ +module Travis::API::V3 + class Services::Log::Delete < Service + def run! + job = check_login_and_find(:job) + access_control.permissions(job).delete_log! + query.delete(access_control.user, job) + end + end +end diff --git a/lib/travis/api/v3/services/log/find.rb b/lib/travis/api/v3/services/log/find.rb new file mode 100644 index 0000000000..2ccf8bc566 --- /dev/null +++ b/lib/travis/api/v3/services/log/find.rb @@ -0,0 +1,8 @@ +module Travis::API::V3 + class Services::Log::Find < Service + def run! + job = check_login_and_find(:job) + query.find(job) + end + end +end diff --git a/lib/travis/api/v3/services/ssh_key/create.rb b/lib/travis/api/v3/services/ssh_key/create.rb new file mode 100644 index 0000000000..b245935fe2 --- /dev/null +++ b/lib/travis/api/v3/services/ssh_key/create.rb @@ -0,0 +1,10 @@ +module Travis::API::V3 + class Services::SshKey::Create < Service + def run! + repository = check_login_and_find(:repository) + access_control.permissions(repository).change_key! + ssh_key = query.regenerate(repository) + result(:ssh_key, ssh_key, status: 201) + end + end +end diff --git a/lib/travis/api/v3/services/ssh_key/find.rb b/lib/travis/api/v3/services/ssh_key/find.rb new file mode 100644 index 0000000000..dba3ed4cbc --- /dev/null +++ b/lib/travis/api/v3/services/ssh_key/find.rb @@ -0,0 +1,8 @@ +module Travis::API::V3 + class Services::SshKey::Find < Service + def run! + repository = check_login_and_find(:repository) + query.find(repository) + end + end +end diff --git a/lib/travis/api/workers/build_cancellation.rb b/lib/travis/api/workers/build_cancellation.rb deleted file mode 100644 index bf1eca7642..0000000000 --- a/lib/travis/api/workers/build_cancellation.rb +++ /dev/null @@ -1,19 +0,0 @@ -require 'sidekiq/worker' -require 'multi_json' - -module Travis - module Sidekiq - class BuildCancellation - class ProcessingError < StandardError; end - - include ::Sidekiq::Worker - sidekiq_options queue: :build_cancellations - - def perform(data) - user = User.find(data['user_id']) - Travis.service(:cancel_build, user, { id: data['id'], source: data['source'] }).run - end - - end - end -end diff --git a/lib/travis/api/workers/build_restart.rb b/lib/travis/api/workers/build_restart.rb deleted file mode 100644 index 427bc24be6..0000000000 --- a/lib/travis/api/workers/build_restart.rb +++ /dev/null @@ -1,19 +0,0 @@ -require 'sidekiq/worker' -require 'multi_json' - -module Travis - module Sidekiq - class BuildRestart - class ProcessingError < StandardError; end - - include ::Sidekiq::Worker - sidekiq_options queue: :build_restarts - - def perform(data) - user = User.find(data['user_id']) - Travis.service(:reset_model, user, build_id: data['id']).run - end - - end - end -end diff --git a/lib/travis/api/workers/job_cancellation.rb b/lib/travis/api/workers/job_cancellation.rb deleted file mode 100644 index 3210a556a4..0000000000 --- a/lib/travis/api/workers/job_cancellation.rb +++ /dev/null @@ -1,18 +0,0 @@ -require 'sidekiq/worker' -require 'multi_json' - -module Travis - module Sidekiq - class JobCancellation - class ProcessingError < StandardError; end - - include ::Sidekiq::Worker - sidekiq_options queue: :job_cancellations - - def perform(data) - user = User.find(data['user_id']) - Travis.service(:cancel_job, user, { id: data['id'], source: data['source'] }).run - end - end - end -end diff --git a/lib/travis/api/workers/job_restart.rb b/lib/travis/api/workers/job_restart.rb deleted file mode 100644 index 88352a5875..0000000000 --- a/lib/travis/api/workers/job_restart.rb +++ /dev/null @@ -1,18 +0,0 @@ -require 'sidekiq/worker' -require 'multi_json' - -module Travis - module Sidekiq - class JobRestart - class ProcessingError < StandardError; end - - include ::Sidekiq::Worker - sidekiq_options queue: :job_restarts - - def perform(data) - user = User.find(data['user_id']) - Travis.service(:reset_model, user, job_id: data['id']).run - end - end - end -end diff --git a/lib/travis/config/defaults.rb b/lib/travis/config/defaults.rb index 397a1be679..9c9cb75877 100644 --- a/lib/travis/config/defaults.rb +++ b/lib/travis/config/defaults.rb @@ -18,7 +18,9 @@ class Config < Hashr assets: { host: HOSTS[Travis.env.to_sym] }, amqp: { username: 'guest', password: 'guest', host: 'localhost', prefetch: 1 }, database: { adapter: 'postgresql', database: "travis_#{Travis.env}", encoding: 'unicode', min_messages: 'warning', variables: { statement_timeout: 10_000 } }, - s3: { access_key_id: '', secret_access_key: '' }, + logs_database: { adapter: 'postgresql', database: "travis_logs_#{Travis.env}", encoding: 'unicode', min_messages: 'warning', variables: { statement_timeout: 10_000 } }, + logs_options: { s3: { access_key_id: '', secret_access_key: ''}}, + s3: { access_key_id: '', secret_access_key: ''}, pusher: { app_id: 'app-id', key: 'key', secret: 'secret' }, sidekiq: { namespace: 'sidekiq', pool_size: 1 }, smtp: {}, diff --git a/lib/travis/github/services/set_hook.rb b/lib/travis/github/services/set_hook.rb index 338e01fcd4..036f70d6bf 100644 --- a/lib/travis/github/services/set_hook.rb +++ b/lib/travis/github/services/set_hook.rb @@ -5,7 +5,8 @@ module Travis module Github module Services class SetHook < Travis::Services::Base - EVENTS = [:push, :pull_request, :issue_comment, :public, :member] + EVENTS = %i(push pull_request issue_comment public member create delete + membership repository) register :github_set_hook diff --git a/lib/travis/sidekiq.rb b/lib/travis/sidekiq.rb index 077e6055ab..69369b5538 100644 --- a/lib/travis/sidekiq.rb +++ b/lib/travis/sidekiq.rb @@ -1,10 +1,6 @@ #$: << './lib' require 'sidekiq' require 'travis' -require 'travis/api/workers/build_cancellation' -require 'travis/api/workers/build_restart' -require 'travis/api/workers/job_cancellation' -require 'travis/api/workers/job_restart' require 'travis/support/amqp' pool_size = ENV['SIDEKIQ_DB_POOL_SIZE'] || 5 diff --git a/lib/travis/testing/factories.rb b/lib/travis/testing/factories.rb index 5490e84b76..02350551f1 100644 --- a/lib/travis/testing/factories.rb +++ b/lib/travis/testing/factories.rb @@ -131,5 +131,27 @@ api_username "travis-ci" api_key "0123456789abcdef" end + + factory :branch, class: Travis::API::V3::Models::Branch do + name Random.rand(1..1000) + repository_id { Factory(:repository).id } + end + + factory :v3_build, class: Travis::API::V3::Models::Build do + owner { User.first || Factory(:user) } + repository { Repository.first || Factory(:repository) } + association :request + association :commit + started_at { Time.now.utc } + finished_at { Time.now.utc } + number 1 + state :passed + end + + factory :cron, class: Travis::API::V3::Models::Cron do + branch { Factory(:branch) } + interval "daily" + dont_run_if_recent_build_exists false + end end diff --git a/script/start_crons b/script/start_crons deleted file mode 100755 index 63a0273efc..0000000000 --- a/script/start_crons +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env ruby -# encoding: UTF-8 - -require 'bundler/setup' -require 'travis/api/app' - -Travis::Api::App.setup - -Travis::API::V3::Services::Crons::Start.new(nil,nil,nil).run! diff --git a/sigsci-buildpack-nginx.tgz b/sigsci-buildpack-nginx.tgz deleted file mode 100644 index 47dcb4b609..0000000000 Binary files a/sigsci-buildpack-nginx.tgz and /dev/null differ diff --git a/spec/integration/error_handling_spec.rb b/spec/integration/error_handling_spec.rb index 4faedbc274..4246c4362d 100644 --- a/spec/integration/error_handling_spec.rb +++ b/spec/integration/error_handling_spec.rb @@ -25,7 +25,7 @@ class TestError < StandardError error = TestError.new('Konstantin broke all the thingz!') Travis::Api::App::Endpoint::Repos.any_instance.stubs(:service).raises(error) Raven.expects(:send_event).with do |event| - event.message == "#{error.class}: #{error.message}" + event['logentry']['message'] == "#{error.class}: #{error.message}" end expect { get "/repos" }.to raise_error(TestError) sleep 0.1 diff --git a/spec/integration/v2/repositories_spec.rb b/spec/integration/v2/repositories_spec.rb index c4be1d74c0..7a366bed1d 100644 --- a/spec/integration/v2/repositories_spec.rb +++ b/spec/integration/v2/repositories_spec.rb @@ -144,9 +144,12 @@ response.should deliver_cc_xml_for(Repository.timeline) end - it 'responds with 404 when repo can\'t be found and format is png' do + it 'responds with 200 and image when repo can\'t be found and format is png' do result = get('/repos/foo/bar.png', {}, 'HTTP_ACCEPT' => 'image/png; version=2') - result.status.should == 404 + result.status.should == 200 + result.headers['Content-Type'].should == 'image/png' + result.body.should_not == '' + result.should deliver_result_image_for('unknown') end it 'responds with 404 when repo can\'t be found and format is other than png' do diff --git a/spec/lib/schedulers/schedule_cron_jobs_spec.rb b/spec/lib/schedulers/schedule_cron_jobs_spec.rb new file mode 100644 index 0000000000..6179f396d3 --- /dev/null +++ b/spec/lib/schedulers/schedule_cron_jobs_spec.rb @@ -0,0 +1,82 @@ +require "sentry-raven" +require "travis/api/app/schedulers/schedule_cron_jobs" + +describe "ScheduleCronJobs" do + let(:subject) { Travis::Api::App::Schedulers::ScheduleCronJobs.enqueue } + + before do + Travis::Api::App::Schedulers::ScheduleCronJobs.stubs(:options).returns({ + strategy: :redis, + url: 'redis://localhost:6379', + retries: 2 + }) + end + + let(:error) { StandardError.new("Konstantin broke all the thingz!") } + let!(:scheduler_interval) { Travis::API::V3::Models::Cron::SCHEDULER_INTERVAL + 1.minute } + + describe "enqueue" do + it 'continues running crons if one breaks' do + cron1 = Factory(:cron) + cron2 = Factory(:cron) + Timecop.travel(scheduler_interval.from_now) + + Travis::API::V3::Models::Cron.any_instance.expects(:branch).raises(error) + Raven.expects(:capture_exception).with(error, tags: {'cron_id' => cron1.id }) + + + Travis::API::V3::Models::Cron.any_instance.expects(:branch).raises(error) + Raven.expects(:capture_exception).with(error, tags: {'cron_id' => cron2.id }) + + subject + + Timecop.return + cron1.destroy + cron2.destroy + end + + it "raises exception when enqueue method errors" do + cron1 = Factory(:cron) + Timecop.travel(scheduler_interval.from_now) + + Travis::API::V3::Models::Cron.any_instance.stubs(:enqueue).raises(error) + + Raven.expects(:capture_exception).with(error, tags: {'cron_id' => cron1.id }) + + subject + + Timecop.return + cron1.destroy + end + + context "dont_run_if_recent_build_exists is true" do + let!(:cron) { Factory(:cron, dont_run_if_recent_build_exists: true) } + + before { Timecop.freeze(DateTime.now) } + + context "no new build in the last 24h" do + before do + last_build = Factory.create(:build, + repository_id: cron.branch.repository.id, + finished_at: DateTime.now - 1.hour) + cron.branch.update_attribute(:last_build_id, last_build.id) + Timecop.travel(scheduler_interval.from_now) + end + + after { Timecop.return } + + it "skips enqueuing a cron job" do + Sidekiq::Client.any_instance.expects(:push).never + subject + end + + it "schedules the next cron job" do + subject + + cron.reload + expect(cron.next_run.to_i).to eql (Time.now.utc + 1.day).to_i + end + end + end + end +end diff --git a/spec/lib/services/find_caches_spec.rb b/spec/lib/services/find_caches_spec.rb index 17bced135e..a98d6e87b7 100644 --- a/spec/lib/services/find_caches_spec.rb +++ b/spec/lib/services/find_caches_spec.rb @@ -62,11 +62,11 @@ describe 'with multiple buckets' do let(:cache_options) { { - s3: { bucket_name: '', access_key_id: '', secret_access_key: '' }, s3: { bucket_name: '', access_key_id: '', secret_access_key: '' } + s3: { bucket_name: '', access_key_id: '', secret_access_key: '' } } } its(:size) do skip "this isn't valid anymore we don't use multiple buckets" - should be == 4 + should be == 4 end end end diff --git a/spec/lib/travis/config_spec.rb b/spec/lib/travis/config_spec.rb index d3f31d9c80..12f843ae39 100644 --- a/spec/lib/travis/config_spec.rb +++ b/spec/lib/travis/config_spec.rb @@ -73,6 +73,16 @@ :variables => { :statement_timeout => 10000 } } end + + it 'logs database' do + config.logs_database.should == { + :adapter => 'postgresql', + :database => 'travis_logs_test', + :encoding => 'unicode', + :min_messages => 'warning', + :variables => { :statement_timeout => 10000 } + } + end end describe 'resource urls' do @@ -86,6 +96,8 @@ it { config.database.port.should == 1234 } it { config.database.database.should == 'database' } it { config.database.encoding.should == 'unicode' } + it { config.database.variables.application_name.should_not be_empty } + it { config.database.variables.statement_timeout.should eq 10000 } end describe 'with a DATABASE_URL set' do @@ -98,6 +110,8 @@ it { config.database.port.should == 1234 } it { config.database.database.should == 'database' } it { config.database.encoding.should == 'unicode' } + it { config.database.variables.application_name.should_not be_empty } + it { config.database.variables.statement_timeout.should eq 10000 } end describe 'with a TRAVIS_LOGS_DATABASE_URL set' do @@ -110,6 +124,8 @@ it { config.logs_database.port.should == 1234 } it { config.logs_database.database.should == 'database' } it { config.logs_database.encoding.should == 'unicode' } + it { config.logs_database.variables.application_name.should_not be_empty } + it { config.logs_database.variables.statement_timeout.should eq 10000 } end describe 'with a LOGS_DATABASE_URL set' do @@ -122,6 +138,8 @@ it { config.logs_database.port.should == 1234 } it { config.logs_database.database.should == 'database' } it { config.logs_database.encoding.should == 'unicode' } + it { config.logs_database.variables.application_name.should_not be_empty } + it { config.logs_database.variables.statement_timeout.should eq 10000 } end describe 'with a TRAVIS_RABBITMQ_URL set' do diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 74fcf50e6f..a7db1c1778 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -10,6 +10,7 @@ require 'multi_json' require 'pry' require 'stackprof' +require 'webmock/rspec' require 'travis/api/app' require 'travis/testing' @@ -24,6 +25,7 @@ require 'support/s3' require 'support/test_helpers' require 'support/shared_examples' +require 'support/active_record' module TestHelpers include Sinatra::TestHelpers diff --git a/spec/support/active_record.rb b/spec/support/active_record.rb index d0e1874c6d..0481146bdf 100644 --- a/spec/support/active_record.rb +++ b/spec/support/active_record.rb @@ -8,11 +8,16 @@ # TODO why not make this use Travis::Database.connect ? config = Travis.config.database.to_h -config.merge!('adapter' => 'jdbcpostgresql', 'username' => ENV['USER']) if RUBY_PLATFORM == 'java' +logs_config = Travis.config.logs_database.to_h ActiveRecord::Base.default_timezone = :utc ActiveRecord::Base.logger = Logger.new('log/test.db.log') -ActiveRecord::Base.configurations = { 'test' => config } +ActiveRecord::Base.configurations = { + 'test' => config, + 'logs_test' => logs_config, +} + +Travis::LogsModel.establish_connection('logs_test') ActiveRecord::Base.establish_connection('test') DatabaseCleaner.clean_with :truncation diff --git a/spec/support/gcs.rb b/spec/support/gcs.rb index 11ff98d662..4ad419adc6 100644 --- a/spec/support/gcs.rb +++ b/spec/support/gcs.rb @@ -39,4 +39,4 @@ class FakeAuthorization let(:gcs_storage) { FakeService.new } end end -end \ No newline at end of file +end diff --git a/spec/v3/models/branch_spec.rb b/spec/v3/models/branch_spec.rb new file mode 100644 index 0000000000..87f9223d06 --- /dev/null +++ b/spec/v3/models/branch_spec.rb @@ -0,0 +1,9 @@ +describe Travis::API::V3::Models::Branch do + let!(:subject) { Factory(:branch) } + + it "cron should be deleted when the related branch is deleted" do + cron = Factory(:cron, branch: subject) + subject.destroy + Travis::API::V3::Models::Cron.find_by_id(cron.id).should be nil + end +end diff --git a/spec/v3/models/cron_spec.rb b/spec/v3/models/cron_spec.rb index 7b9dd93ae9..46ace8da2b 100644 --- a/spec/v3/models/cron_spec.rb +++ b/spec/v3/models/cron_spec.rb @@ -1,47 +1,22 @@ describe Travis::API::V3::Models::Cron do - let(:repo) { Travis::API::V3::Models::Repository.where(owner_name: 'svenfuchs', name: 'minimal').first } - let(:branch) { Travis::API::V3::Models::Branch.create(repository: repo, name: 'cron test') } + let(:subject) { Factory(:cron, branch_id: Factory(:branch).id) } - describe "next build time is calculated correctly on year changes" do + let!(:scheduler_interval) { Travis::API::V3::Models::Cron::SCHEDULER_INTERVAL + 1.minute } + describe "scheduled scope" do + it "collects all upcoming cron jobs" do + cron1 = Factory(:cron) + cron2 = Factory(:cron) - before do + cron2.update_attribute(:next_run, 2.hours.from_now) + Timecop.travel(scheduler_interval.from_now) + Travis::API::V3::Models::Cron.scheduled.count.should eql 1 Timecop.return - Timecop.travel(DateTime.new(2015, 12, 31, 16)) - end - - after do - Timecop.return - Timecop.freeze(Time.now.utc) - end - - it "for daily builds" do - cron = Travis::API::V3::Models::Cron.create(branch_id: branch.id, interval: 'daily', disable_by_build: false) - build = Travis::API::V3::Models::Build.create(:repository_id => repo.id, :branch_name => branch.name, :event_type => 'cron') - expect(cron.next_enqueuing).to be == DateTime.new(2016, 1, 1, 16) - build.destroy - cron.destroy - end - - it "for weekly builds" do - cron = Travis::API::V3::Models::Cron.create(branch_id: branch.id, interval: 'weekly', disable_by_build: false) - build = Travis::API::V3::Models::Build.create(:repository_id => repo.id, :branch_name => branch.name, :event_type => 'cron') - expect(cron.next_enqueuing).to be == DateTime.new(2016, 1, 7, 16) - build.destroy - cron.destroy - end - - it "for monthly builds" do - cron = Travis::API::V3::Models::Cron.create(branch_id: branch.id, interval: 'monthly', disable_by_build: false) - build = Travis::API::V3::Models::Build.create(:repository_id => repo.id, :branch_name => branch.name, :event_type => 'cron') - expect(cron.next_enqueuing).to be == DateTime.new(2016, 1, 31, 16) - build.destroy - cron.destroy + cron1.destroy + cron2.destroy end - end - describe "push build is ignored if disable by build is false" do - + describe "next build time is calculated correctly on year changes" do before do Timecop.return Timecop.travel(DateTime.new(2015, 12, 31, 16)) @@ -53,184 +28,111 @@ end it "for daily builds" do - cron = Travis::API::V3::Models::Cron.create(branch_id: branch.id, interval: 'daily', disable_by_build: false) - cron_build = Travis::API::V3::Models::Build.create(:repository_id => repo.id, :branch_name => branch.name, :event_type => 'cron') - push_build = Travis::API::V3::Models::Build.create(:repository_id => repo.id, :branch_name => branch.name, :event_type => 'push') - expect(cron.next_enqueuing).to be == DateTime.new(2016, 1, 1, 16) - cron_build.destroy - push_build.destroy - cron.destroy + subject.schedule_next_build(from: DateTime.now) + subject.next_run.should be == DateTime.new(2016, 1, 1, 16) end it "for weekly builds" do - cron = Travis::API::V3::Models::Cron.create(branch_id: branch.id, interval: 'weekly', disable_by_build: false) - cron_build = Travis::API::V3::Models::Build.create(:repository_id => repo.id, :branch_name => branch.name, :event_type => 'cron') - push_build = Travis::API::V3::Models::Build.create(:repository_id => repo.id, :branch_name => branch.name, :event_type => 'push') - expect(cron.next_enqueuing).to be == DateTime.new(2016, 1, 7, 16) - cron_build.destroy - push_build.destroy - cron.destroy + subject.interval = "weekly" + subject.schedule_next_build(from: DateTime.now) + subject.next_run.should be == DateTime.new(2016, 1, 7, 16) end it "for monthly builds" do - cron = Travis::API::V3::Models::Cron.create(branch_id: branch.id, interval: 'monthly', disable_by_build: false) - cron_build = Travis::API::V3::Models::Build.create(:repository_id => repo.id, :branch_name => branch.name, :event_type => 'cron') - push_build = Travis::API::V3::Models::Build.create(:repository_id => repo.id, :branch_name => branch.name, :event_type => 'push') - expect(cron.next_enqueuing).to be == DateTime.new(2016, 1, 31, 16) - cron_build.destroy - push_build.destroy - cron.destroy + subject.interval = "monthly" + subject.schedule_next_build(from: DateTime.now) + subject.next_run.should be == DateTime.new(2016, 1, 31, 16) end - end - describe "disable by build works with build" do + context "for daily runs, when last_run is set" do + before { Timecop.freeze(Time.now) } + after { Timecop.return } - before do - Timecop.return - Timecop.travel(DateTime.new(2015, 12, 31, 16)) + it "sets the next_run correctly" do + subject.last_run = 1.day.ago.utc + 5.minutes + subject.schedule_next_build + subject.next_run.to_i.should eql 5.minutes.from_now.utc.to_i end + end - after do - Timecop.return - Timecop.freeze(Time.now.utc) - end + context "when last_run is not set" do + before { Timecop.freeze(Time.now) } + after { Timecop.return } - it "for daily builds" do - cron = Travis::API::V3::Models::Cron.create(branch_id: branch.id, interval: 'daily', disable_by_build: true) - build = Travis::API::V3::Models::Build.create(:repository_id => repo.id, :branch_name => branch.name, :event_type => 'push') - expect(cron.next_enqueuing).to be == DateTime.new(2016, 1, 2, 16) - build.destroy - cron.destroy + context "and from: is not passed" do + it "sets the next_run from now" do + subject.schedule_next_build + subject.next_run.should be == DateTime.now + 1.day + end end - - it "for weekly builds" do - cron = Travis::API::V3::Models::Cron.create(branch_id: branch.id, interval: 'weekly', disable_by_build: true) - build = Travis::API::V3::Models::Build.create(:repository_id => repo.id, :branch_name => branch.name, :event_type => 'push') - expect(cron.next_enqueuing).to be == DateTime.new(2016, 1, 14, 16) - build.destroy - cron.destroy + context "and from: is passed" do + it "sets the next_run from from:" do + subject.schedule_next_build(from: DateTime.now + 3.day) + subject.next_run.should be == DateTime.now + 4.day + end end - it "for monthly builds" do - cron = Travis::API::V3::Models::Cron.create(branch_id: branch.id, interval: 'monthly', disable_by_build: true) - build = Travis::API::V3::Models::Build.create(:repository_id => repo.id, :branch_name => branch.name, :event_type => 'push') - expect(cron.next_enqueuing).to be == DateTime.new(2016, 2, 29, 16) # it's a leap year :-D - build.destroy - cron.destroy + context "and from: is more than one interval in the past" do + it "ensures that the next_run is in the future" do + subject.schedule_next_build(from: DateTime.now - 2.day) + subject.next_run.should be >= DateTime.now + end end - end - describe "disable by build works without build" do + describe "enqueue" do + before { Timecop.freeze(Time.now) } + after { Timecop.return } - before do - Timecop.return - Timecop.travel(DateTime.new(2015, 12, 31, 16)) + it "enqueues the cron" do + Sidekiq::Client.any_instance.expects(:push).once + subject.enqueue end - after do - Timecop.return - Timecop.freeze(Time.now.utc) + it "set the last_run time to now" do + subject.enqueue + subject.last_run.should be == DateTime.now.utc end - it "for daily builds" do - cron = Travis::API::V3::Models::Cron.create(branch_id: branch.id, interval: 'daily', disable_by_build: true) - build = Travis::API::V3::Models::Build.create(:repository_id => repo.id, :branch_name => branch.name, :event_type => 'cron') - expect(cron.next_enqueuing).to be == DateTime.new(2016, 1, 1, 16) - build.destroy - cron.destroy + it "schedules the next run" do + subject.enqueue + subject.next_run.should be == DateTime.now.utc + 1.day end - it "for weekly builds" do - cron = Travis::API::V3::Models::Cron.create(branch_id: branch.id, interval: 'weekly', disable_by_build: true) - build = Travis::API::V3::Models::Build.create(:repository_id => repo.id, :branch_name => branch.name, :event_type => 'cron') - expect(cron.next_enqueuing).to be == DateTime.new(2016, 1, 7, 16) - build.destroy - cron.destroy + it "raises error when repo doesn't have a github id" do + subject.branch.repository.github_id = nil + expect { subject.enqueue }.to raise_error(StandardError) end - it "for monthly builds" do - cron = Travis::API::V3::Models::Cron.create(branch_id: branch.id, interval: 'monthly', disable_by_build: true) - build = Travis::API::V3::Models::Build.create(:repository_id => repo.id, :branch_name => branch.name, :event_type => 'cron') - expect(cron.next_enqueuing).to be == DateTime.new(2016, 1, 31, 16) - build.destroy - cron.destroy + it "destroys cron if branch does not exist on github" do + subject.branch.exists_on_github = false + expect{ subject.enqueue }.to change { Travis::API::V3::Models::Cron.count}.by(-1) end - end - describe "build starts now if next build time is in the past" do - - before do - Timecop.return - end - - after do - Timecop.return - Timecop.freeze(Time.now.utc) - end - - it "for daily builds with disable_by_build true" do - Timecop.travel(DateTime.new(2015, 12, 31, 16)) - cron = Travis::API::V3::Models::Cron.create(branch_id: branch.id, interval: 'daily', disable_by_build: true) - build = Travis::API::V3::Models::Build.create(:repository_id => repo.id, :branch_name => branch.name, :event_type => 'cron') - Timecop.freeze(DateTime.new(2016, 1, 1, 19)) - expect(cron.next_enqueuing).to be == DateTime.now - 5.minutes - build.destroy - cron.destroy + context "when always_run? is false" do + context "when no build has existed before running a cron build" do + let(:cron) { Factory(:cron, branch_id: Factory(:branch).id, dont_run_if_recent_build_exists: true) } + it "needs_new_build? returns true" do + cron.needs_new_build?.should be_truthy + end end - it "for daily builds with disable_by_build false" do - Timecop.travel(DateTime.new(2015, 12, 31, 16)) - cron = Travis::API::V3::Models::Cron.create(branch_id: branch.id, interval: 'daily', disable_by_build: false) - build = Travis::API::V3::Models::Build.create(:repository_id => repo.id, :branch_name => branch.name, :event_type => 'cron') - Timecop.freeze(DateTime.new(2016, 1, 1, 19)) - expect(cron.next_enqueuing).to be == DateTime.now - 5.minutes - build.destroy - cron.destroy - end + context "when there was a build in the last 24h" do + let(:cron) { Factory(:cron, branch_id: Factory(:branch, last_build: Factory(:v3_build)).id, dont_run_if_recent_build_exists: true) } - it "for weekly builds with disable_by_build true" do - Timecop.travel(DateTime.new(2015, 12, 31, 16)) - cron = Travis::API::V3::Models::Cron.create(branch_id: branch.id, interval: 'weekly', disable_by_build: true) - build = Travis::API::V3::Models::Build.create(:repository_id => repo.id, :branch_name => branch.name, :event_type => 'cron') - Timecop.freeze(DateTime.new(2016, 1, 7, 19)) - expect(cron.next_enqueuing).to be == DateTime.now - 5.minutes - build.destroy - cron.destroy - end - - it "for weekly builds with disable_by_build false" do - Timecop.travel(DateTime.new(2015, 12, 31, 16)) - cron = Travis::API::V3::Models::Cron.create(branch_id: branch.id, interval: 'weekly', disable_by_build: false) - build = Travis::API::V3::Models::Build.create(:repository_id => repo.id, :branch_name => branch.name, :event_type => 'cron') - Timecop.freeze(DateTime.new(2016, 1, 7, 19)) - expect(cron.next_enqueuing).to be == DateTime.now - 5.minutes - build.destroy - cron.destroy - end - - it "for monthly builds with disable_by_build true" do - Timecop.travel(DateTime.new(2015, 12, 31, 16)) - cron = Travis::API::V3::Models::Cron.create(branch_id: branch.id, interval: 'monthly', disable_by_build: true) - build = Travis::API::V3::Models::Build.create(:repository_id => repo.id, :branch_name => branch.name, :event_type => 'cron') - Timecop.freeze(DateTime.new(2016, 1, 31, 19)) - expect(cron.next_enqueuing).to be == DateTime.now - 5.minutes - build.destroy - cron.destroy + it "needs_new_build? returns false" do + cron.needs_new_build?.should be_falsey + end end + end - it "for monthly builds with disable_by_build false" do - Timecop.travel(DateTime.new(2015, 12, 31, 16)) - cron = Travis::API::V3::Models::Cron.create(branch_id: branch.id, interval: 'monthly', disable_by_build: false) - build = Travis::API::V3::Models::Build.create(:repository_id => repo.id, :branch_name => branch.name, :event_type => 'cron') - Timecop.freeze(DateTime.new(2016, 1, 31, 19)) - expect(cron.next_enqueuing).to be == DateTime.now - 5.minutes - build.destroy - cron.destroy + context "when repo ownership is transferred" do + it "enqueues a cron for the repo with the new owner" do + subject.branch.repository.update_attribute(:owner, Factory(:user, name: "Yoda", login: "yoda", email: "yoda@yoda.com")) + Sidekiq::Client.any_instance.expects(:push).once + subject.enqueue end - end - end diff --git a/spec/v3/queries/cron_spec.rb b/spec/v3/queries/cron_spec.rb deleted file mode 100644 index 1418a6e0c1..0000000000 --- a/spec/v3/queries/cron_spec.rb +++ /dev/null @@ -1,48 +0,0 @@ -require 'sentry-raven' - -describe Travis::API::V3::Queries::Crons do - let(:user) { Travis::API::V3::Models::User.find_by_login('svenfuchs') } - let(:repo) { Travis::API::V3::Models::Repository.where(owner_name: 'svenfuchs', name: 'minimal').first } - let(:existing_branch) { Travis::API::V3::Models::Branch.create(repository: repo, name: 'cron-test-existing', exists_on_github: true) } - let(:existing_branch2) { Travis::API::V3::Models::Branch.create(repository: repo, name: 'cron-test-existing2', exists_on_github: true) } - let(:non_existing_branch) { Travis::API::V3::Models::Branch.create(repository: repo, name: 'cron-test-non-existing', exists_on_github: false) } - let(:query) { Travis::API::V3::Queries::Crons.new({}, 'Overview') -} - - describe "start all" do - before { Travis::API::V3::Models::Permission.create(repository: repo, user: repo.owner, push: true) } - - it "starts crons on existing branches" do - cron = Travis::API::V3::Models::Cron.create(branch_id: existing_branch.id, interval: 'daily', disable_by_build: false) - expect(query.start_all).to include(cron) - end - - it "delete crons on branches not existing on GitHub" do - cron = Travis::API::V3::Models::Cron.create(branch_id: non_existing_branch.id, interval: 'daily', disable_by_build: false) - expect(query.start_all).to_not include(cron) - expect(Travis::API::V3::Models::Cron.where(id: cron.id).length).to equal(0) - end - - it 'enques error into a thread' do - cron = Travis::API::V3::Models::Cron.create(branch_id: existing_branch.id, interval: 'daily', disable_by_build: false) - error = StandardError.new('Konstantin broke all the thingz!') - Travis::API::V3::Queries::Crons.any_instance.expects(:sleep).with(10) - Travis::API::V3::Models::Cron.any_instance.stubs(:branch).raises(error) - Raven.expects(:capture_exception).with(error, tags: {'cron_id' => cron.id }) - query.start_all - end - - it 'continues running crons if one breaks' do - cron = Travis::API::V3::Models::Cron.create(branch_id: existing_branch.id, interval: 'daily', disable_by_build: false) - cron2 = Travis::API::V3::Models::Cron.create(branch_id: existing_branch2.id, interval: 'daily', disable_by_build: false) - - error = StandardError.new('Konstantin broke all the thingz!') - Travis::API::V3::Models::Cron.any_instance.stubs(:branch).raises(error) - - Travis::API::V3::Queries::Crons.any_instance.expects(:sleep).twice.with(10) - Raven.expects(:capture_exception).with(error, tags: {'cron_id' => cron.id }) - Raven.expects(:capture_exception).with(error, tags: {'cron_id' => cron2.id }) - query.start_all - end - end -end diff --git a/spec/v3/services/branch/find_spec.rb b/spec/v3/services/branch/find_spec.rb index 199716b922..63c72a4b1f 100644 --- a/spec/v3/services/branch/find_spec.rb +++ b/spec/v3/services/branch/find_spec.rb @@ -1,4 +1,4 @@ -describe Travis::API::V3::Services::Repository::Find, set_app: true do +describe Travis::API::V3::Services::Branch::Find, set_app: true do let(:repo) { Travis::API::V3::Models::Repository.where(owner_name: 'svenfuchs', name: 'minimal').first } let(:build) { repo.builds.first } let(:jobs) { Travis::API::V3::Models::Build.find(build.id).jobs } diff --git a/spec/v3/services/caches/delete_spec.rb b/spec/v3/services/caches/delete_spec.rb new file mode 100644 index 0000000000..7cdc37fe9a --- /dev/null +++ b/spec/v3/services/caches/delete_spec.rb @@ -0,0 +1,208 @@ +describe Travis::API::V3::Services::Caches::Delete, set_app: true do + let(:repo) { Travis::API::V3::Models::Repository.where(owner_name: 'svenfuchs', name: 'minimal').first } + let(:build) { repo.builds.first } + let(:jobs) { Travis::API::V3::Models::Build.find(build.id).jobs } + let(:s3_bucket_name) { "travis-cache-staging-org" } + let(:storage) { Fog::Storage.new({ + aws_access_key_id: "key", + aws_secret_access_key: "secret", + provider: "AWS" + }) + } + let(:bucket){ storage.directories.create(key: s3_bucket_name) } + let(:file1){ bucket.files.create( + key: "#{repo.github_id}/ha-bug-rm_rf/cache-linux-precise-lkjdhfsod8fu4tc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855--rvm-2.2.5--gemfile-Gemfile.tgz", + body: "cached file for one branch.") + } + let(:file2){ bucket.files.create( + key: "#{repo.github_id}/master/cache-linux-precise-e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855--rvm-default--gemfile-Gemfile.tgz", + body: "Cached file for the awesome master branch.") + } + let(:result) { [ + { + "@type"=>"cache", + "@representation"=>"standard", + "repository_id"=>1, + "size"=>file1.content_length, + "branch"=>"ha-bug-rm_rf", + "slug"=>"cache-linux-precise-lkjdhfsod8fu4tc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855--rvm-2.2.5--gemfile-Gemfile.tgz", + "repo"=>{ + "@type"=>"repository", + "@href"=>"/v3/repo/#{repo.id}", + "@representation"=>"minimal", + "id"=>repo.id, + "name"=>repo.name, + "slug"=>repo.slug + } + }, + { + "@type"=>"cache", + "@representation"=>"standard", + "repository_id"=>1, + "size"=>file2.content_length, + "branch"=>"master", + "slug"=>"cache-linux-precise-e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855--rvm-default--gemfile-Gemfile.tgz", + "repo"=>{ + "@type"=>"repository", + "@href"=>"/v3/repo/#{repo.id}", + "@representation"=>"minimal", + "id"=>repo.id, + "name"=>repo.name, + "slug"=>repo.slug + } + } + ] + } + + before { repo.default_branch.save! } + before do + Fog.mock! + Travis.config.cache_options.s3 = { access_key_id: 'key', secret_access_key: 'secret', bucket_name: s3_bucket_name } + Timecop.return + Timecop.freeze(Time.now.utc) + result[0]["last_modified"] = Time.now.utc.strftime("%FT%TZ") + result[1]["last_modified"] = Time.now.utc.strftime("%FT%TZ") + end + after do + Fog::Mock.reset + Timecop.return + end + + describe "delete all on s3" do + before { delete("/v3/repo/#{repo.id}/caches") } + example { expect(last_response).to be_ok } + example do + expect(JSON.load(body)).to be == { + "@type"=>"caches", + "@representation"=>"standard", + "caches"=> result + } + end + end + + describe "delete for branch" do + example do + delete("/v3/repo/#{repo.id}/caches", { branch: result[0]["branch"] } ) + expect(JSON.load(body)).to be == { + "@type"=>"caches", + "@representation"=>"standard", + "caches"=> [result[0]] + } + end + end + + describe "delete for match" do + example do + delete("/v3/repo/#{repo.id}/caches", { branch: result[0]["branch"], match: 'dhfsod8fu4' } ) + expect(JSON.load(body)).to be == { + "@type"=>"caches", + "@representation"=>"standard", + "caches"=> [result[0]] + } + end + end + + describe "delete all on gcs" do + let(:xml_content) { + " + + #{s3_bucket_name} + + + 1000 + false + + #{repo.github_id}/ha-bug-rm_rf/cache-linux-precise-lkjdhfsod8fu4tc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855--rvm-2.2.5--gemfile-Gemfile.tgz + 2009-10-12T17:50:30.000Z + "hgb9dede5f27731c9771645a39863328" + 20308738 + STANDARD + + 75aa57f09aa0c8caeab4f8c24e99d10f8e7faeebf76c078efc7c6caea54ba06a + mtd@amazon.com + + + + #{repo.github_id}/master/cache-linux-precise-e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855--rvm-default--gemfile-Gemfile.tgz + 2009-10-12T17:50:30.000Z + "1b2cf535f27731c974343645a3985328" + 64994 + STANDARD_IA + + 75aa57f09aa0c8caeab4f8c24e99d10f8e7faeebf76c078efc7c6caea54ba06a + mtd@amazon.com + + + " + } + + let(:xml_content_single_repo) { + " + + #{s3_bucket_name} + + + 1000 + false + + + 2009-10-12T17:50:30.000Z + "hgb9dede5f27731c9771645a39863328" + 20308738 + STANDARD + + 75aa57f09aa0c8caeab4f8c24e99d10f8e7faeebf76c078efc7c6caea54ba06a + mtd@amazon.com + + + " + } + + let(:empty_xml_content) { + " + + #{s3_bucket_name} + + + 1000 + false + " + } + + before do + stub_request(:get, "https://#{s3_bucket_name}.s3.amazonaws.com/?prefix=#{repo.id}/"). + to_return(:status => 200, :body => empty_xml_content, :headers => {}) + stub_request(:post, "https://accounts.google.com/o/oauth2/token"). + to_return(:status => 200, :body => {authorization: 'skdjfhdkfh'}.to_json, :headers => {content_type: 'application/json'}) + + stub_request(:get, "https://www.googleapis.com/storage/v1/b/travis-cache-staging-org-gce/?prefix=#{repo.id}/"). + to_return(:status => 200, :body => xml_content, :headers => {}) + + ## FIXME: figure out mock for gcs + # Fog.mock! + # Travis.config.cache_options.gcs = { json_key: 'key', google_project: 'pj', bucket_name: 'bucket' } + # storage = Fog::Storage::Google.new({ + # google_json_key_string: 'key', + # google_project: 'pj' + # }) + # + # ## FIXME: + # bucket = storage.directories.create(key: "#{repo.id}/branch") + # file = bucket.files.create( + # key: "some file", + # body: "something to test" + # ) + end + # after { Fog::Mock.reset } + + before { delete("/v3/repo/#{repo.id}/caches") } + skip do + expect(JSON.load(body)).to be == { + "@type"=>"caches", + "@href"=>"/v3/repo/#{repo.id}/caches", + "@representation"=>"standard", + "caches"=> result + } + end + end +end diff --git a/spec/v3/services/caches/find_spec.rb b/spec/v3/services/caches/find_spec.rb new file mode 100644 index 0000000000..dec0ca856c --- /dev/null +++ b/spec/v3/services/caches/find_spec.rb @@ -0,0 +1,186 @@ +describe Travis::API::V3::Services::Caches::Find, set_app: true do + let(:repo) { Travis::API::V3::Models::Repository.where(owner_name: 'svenfuchs', name: 'minimal').first } + let(:build) { repo.builds.first } + let(:jobs) { Travis::API::V3::Models::Build.find(build.id).jobs } + let(:s3_bucket_name) { "travis-cache-staging-org" } + let(:result) { [ + { + "@type"=>"cache", + "@representation"=>"standard", + "repository_id"=>1, + "size"=>20308738, + "branch"=>"ha-bug-rm_rf", + "last_modified"=>"2009-10-12T17:50:30Z", + "slug"=>"cache-linux-precise-lkjdhfsod8fu4tc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855--rvm-2.2.5--gemfile-Gemfile.tgz", + "repo"=>{ + "@type"=>"repository", + "@href"=>"/v3/repo/#{repo.id}", + "@representation"=>"minimal", + "id"=>repo.id, + "name"=>repo.name, + "slug"=>repo.slug + } + }, + { + "@type"=>"cache", + "@representation"=>"standard", + "repository_id"=>1, + "size"=>64994, + "branch"=>"master", + "last_modified"=>"2009-10-12T17:50:30Z", + "slug"=>"cache-linux-precise-e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855--rvm-default--gemfile-Gemfile.tgz", + "repo"=>{ + "@type"=>"repository", + "@href"=>"/v3/repo/#{repo.id}", + "@representation"=>"minimal", + "id"=>repo.id, + "name"=>repo.name, + "slug"=>repo.slug + } + } + ] + } + let(:xml_content) { + " + + #{s3_bucket_name} + + + 1000 + false + + #{repo.github_id}/ha-bug-rm_rf/cache-linux-precise-lkjdhfsod8fu4tc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855--rvm-2.2.5--gemfile-Gemfile.tgz + 2009-10-12T17:50:30.000Z + "hgb9dede5f27731c9771645a39863328" + 20308738 + STANDARD + + 75aa57f09aa0c8caeab4f8c24e99d10f8e7faeebf76c078efc7c6caea54ba06a + mtd@amazon.com + + + + #{repo.github_id}/master/cache-linux-precise-e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855--rvm-default--gemfile-Gemfile.tgz + 2009-10-12T17:50:30.000Z + "1b2cf535f27731c974343645a3985328" + 64994 + STANDARD_IA + + 75aa57f09aa0c8caeab4f8c24e99d10f8e7faeebf76c078efc7c6caea54ba06a + mtd@amazon.com + + + " + } + + let(:xml_content_single_repo) { + " + + #{s3_bucket_name} + + + 1000 + false + + #{repo.github_id}/ha-bug-rm_rf/cache-linux-precise-lkjdhfsod8fu4tc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855--rvm-2.2.5--gemfile-Gemfile.tgz + 2009-10-12T17:50:30.000Z + "hgb9dede5f27731c9771645a39863328" + 20308738 + STANDARD + + 75aa57f09aa0c8caeab4f8c24e99d10f8e7faeebf76c078efc7c6caea54ba06a + mtd@amazon.com + + + " + } + + let(:empty_xml_content) { + " + + #{s3_bucket_name} + + + 1000 + false + " + } + before do + repo.default_branch.save! + Fog.unmock! + Travis.config.cache_options.s3 = { access_key_id: 'key', secret_access_key: 'secret', bucket_name: s3_bucket_name } + end + + describe "existing cache on s3" do + before do + stub_request(:get, "https://#{s3_bucket_name}.s3.amazonaws.com/?prefix=#{repo.id}/"). + to_return(:status => 200, :body => xml_content, :headers => {}) + end + before { get("/v3/repo/#{repo.id}/caches") } + example { expect(last_response).to be_ok } + example do + expect(JSON.load(body)).to be == { + "@type"=>"caches", + "@href"=>"/v3/repo/#{repo.id}/caches", + "@representation"=>"standard", + "caches"=> result + } + end + end + + describe "filter by branch" do + before do + stub_request(:get, "https://#{s3_bucket_name}.s3.amazonaws.com/?prefix=#{repo.id}/#{result[0]["branch"]}"). + to_return(:status => 200, :body => xml_content_single_repo, :headers => {}) + end + + example do + get("/v3/repo/#{repo.id}/caches", { branch: result[0]["branch"] } ) + expect(JSON.load(body)).to be == { + "@type"=>"caches", + "@href"=>"/v3/repo/1/caches?branch=#{result[0]["branch"]}", + "@representation"=>"standard", + "caches"=> [result[0]] + } + end + end + + describe "filter by match" do + before do + stub_request(:get, "https://#{s3_bucket_name}.s3.amazonaws.com/?prefix=#{repo.id}/#{result[0]["branch"]}"). + to_return(:status => 200, :body => xml_content, :headers => {}) + end + + example do + get("/v3/repo/#{repo.id}/caches", { branch: result[0]["branch"], match: 'dhfsod8fu4' } ) + expect(JSON.load(body)).to be == { + "@type"=>"caches", + "@href"=>"/v3/repo/1/caches?branch=#{result[0]["branch"]}&match=dhfsod8fu4", + "@representation"=>"standard", + "caches"=> [result[0]] + } + end + end + + describe "existing cache on gcs" do + before do + stub_request(:get, "https://#{s3_bucket_name}.s3.amazonaws.com/?prefix=#{repo.id}/"). + to_return(:status => 200, :body => empty_xml_content, :headers => {}) + stub_request(:post, "https://accounts.google.com/o/oauth2/token"). + to_return(:status => 200, :body => {authorization: 'skdjfhdkfh'}.to_json, :headers => {content_type: 'application/json'}) + + stub_request(:get, "https://www.googleapis.com/storage/v1/b/travis-cache-staging-org-gce/?prefix=#{repo.id}/"). + to_return(:status => 200, :body => xml_content, :headers => {}) + + end + before { get("/v3/repo/#{repo.id}/caches") } + skip do + expect(JSON.load(body)).to be == { + "@type"=>"caches", + "@href"=>"/v3/repo/#{repo.id}/caches", + "@representation"=>"standard", + "caches"=> result + } + end + end +end diff --git a/spec/v3/services/cron/create_spec.rb b/spec/v3/services/cron/create_spec.rb index bce74790e5..0eab1853a0 100644 --- a/spec/v3/services/cron/create_spec.rb +++ b/spec/v3/services/cron/create_spec.rb @@ -6,8 +6,8 @@ let(:current_cron) {Travis::API::V3::Models::Cron.where(branch_id: branch.id).last} let(:token) { Travis::Api::App::AccessToken.create(user: repo.owner, app_id: 1) } let(:headers) {{ 'HTTP_AUTHORIZATION' => "token #{token}", "Content-Type" => "application/json" }} - let(:options) {{ "interval" => "monthly", "disable_by_build" => false }} - let(:wrong_options) {{ "interval" => "notExisting", "disable_by_build" => false }} + let(:options) {{ "interval" => "monthly", "dont_run_if_recent_build_exists" => false }} + let(:wrong_options) {{ "interval" => "notExisting", "dont_run_if_recent_build_exists" => false }} let(:parsed_body) { JSON.load(body) } before do @@ -61,10 +61,12 @@ "@representation" => "minimal", "name" => "#{branch.name}" }, "interval" => "monthly", - "disable_by_build" => false, - "next_enqueuing" => current_cron.next_enqueuing.strftime('%Y-%m-%dT%H:%M:%SZ'), + "dont_run_if_recent_build_exists" => false, + "last_run" => current_cron.last_run, + "next_run" => current_cron.next_run.strftime('%Y-%m-%dT%H:%M:%SZ'), "created_at" => current_cron.created_at.strftime('%Y-%m-%dT%H:%M:%SZ') }} + example { expect(current_cron.next_run).to_not be nil } end describe "creating multiple cron jobs for one branch" do diff --git a/spec/v3/services/cron/delete_spec.rb b/spec/v3/services/cron/delete_spec.rb index ec7d17c177..f415f3e9be 100644 --- a/spec/v3/services/cron/delete_spec.rb +++ b/spec/v3/services/cron/delete_spec.rb @@ -25,34 +25,9 @@ describe "deleting a cron job by id" do before { Travis::API::V3::Models::Permission.create(repository: repo, user: repo.owner, push: true) } before { delete("/v3/cron/#{cron.id}", {}, headers) } - example { expect(last_response).to be_ok } + example { expect(last_response.status).to eq 204 } example { expect(Travis::API::V3::Models::Cron.where(id: cron.id)).to be_empty } - example { expect(parsed_body).to be == { - "@type" => "cron", - "@href" => "/v3/cron/#{cron.id}", - "@representation" => "standard", - "@permissions" => { - "read" => true, - "delete" => true, - "start" => true }, - "id" => cron.id, - "repository" => { - "@type" => "repository", - "@href" => "/v3/repo/#{repo.id}", - "@representation" => "minimal", - "id" => repo.id, - "name" => "minimal", - "slug" => "svenfuchs/minimal" }, - "branch" => { - "@type" => "branch", - "@href" => "/v3/repo/#{repo.id}/branch/#{branch.name}", - "@representation" => "minimal", - "name" => branch.name }, - "interval" => "daily", - "disable_by_build" => true, - "next_enqueuing" => cron.next_enqueuing.strftime('%Y-%m-%dT%H:%M:%SZ'), - "created_at" => cron.created_at.strftime('%Y-%m-%dT%H:%M:%SZ') - }} + example { expect(parsed_body).to be_nil } end describe "try deleting a cron job without login" do diff --git a/spec/v3/services/cron/find_spec.rb b/spec/v3/services/cron/find_spec.rb index 4c6d33a052..681723d029 100644 --- a/spec/v3/services/cron/find_spec.rb +++ b/spec/v3/services/cron/find_spec.rb @@ -20,7 +20,7 @@ end describe "fetching a cron job by id" do - before { get("/v3/cron/#{cron.id}") } + before { get("/v3/cron/#{cron.id}") } example { expect(last_response).to be_ok } example { expect(parsed_body).to be == { "@type" => "cron", @@ -44,8 +44,9 @@ "@representation" => "minimal", "name" => branch.name }, "interval" => "daily", - "disable_by_build" => true, - "next_enqueuing" => cron.next_enqueuing.strftime('%Y-%m-%dT%H:%M:%SZ'), + "dont_run_if_recent_build_exists" => false, + "last_run" => cron.last_run, + "next_run" => cron.next_run.strftime('%Y-%m-%dT%H:%M:%SZ'), "created_at" => cron.created_at.strftime('%Y-%m-%dT%H:%M:%SZ') }} end @@ -104,8 +105,9 @@ "@representation" => "minimal", "name" => branch.name }, "interval" => "daily", - "disable_by_build" => true, - "next_enqueuing" => cron.next_enqueuing.strftime('%Y-%m-%dT%H:%M:%SZ'), + "dont_run_if_recent_build_exists" => false, + "last_run" => cron.last_run, + "next_run" => cron.next_run.strftime('%Y-%m-%dT%H:%M:%SZ'), "created_at" => cron.created_at.strftime('%Y-%m-%dT%H:%M:%SZ') }} end diff --git a/spec/v3/services/cron/for_branch_spec.rb b/spec/v3/services/cron/for_branch_spec.rb index 5ce98d1a8d..7fb3a82b20 100644 --- a/spec/v3/services/cron/for_branch_spec.rb +++ b/spec/v3/services/cron/for_branch_spec.rb @@ -45,8 +45,9 @@ "@representation" => "minimal", "name" => branch.name }, "interval" => "daily", - "disable_by_build" => true, - "next_enqueuing" => cron.next_enqueuing.strftime('%Y-%m-%dT%H:%M:%SZ'), + "dont_run_if_recent_build_exists" => false, + "last_run" => cron.last_run, + "next_run" => cron.next_run.strftime('%Y-%m-%dT%H:%M:%SZ'), "created_at" => cron.created_at.strftime('%Y-%m-%dT%H:%M:%SZ') }} end diff --git a/spec/v3/services/crons/for_repository_spec.rb b/spec/v3/services/crons/for_repository_spec.rb index ca59886df0..3a07f0b905 100644 --- a/spec/v3/services/crons/for_repository_spec.rb +++ b/spec/v3/services/crons/for_repository_spec.rb @@ -65,8 +65,9 @@ "@representation" => "minimal", "name" => "#{branch.name}" }, "interval" => "daily", - "disable_by_build" => true, - "next_enqueuing" => cron.next_enqueuing.strftime('%Y-%m-%dT%H:%M:%SZ'), + "dont_run_if_recent_build_exists" => false, + "last_run" => cron.last_run, + "next_run" => cron.next_run.strftime('%Y-%m-%dT%H:%M:%SZ'), "created_at" => cron.created_at.strftime('%Y-%m-%dT%H:%M:%SZ') } ] diff --git a/spec/v3/services/env_var/delete_spec.rb b/spec/v3/services/env_var/delete_spec.rb index 305b01dad1..a04ec6f592 100644 --- a/spec/v3/services/env_var/delete_spec.rb +++ b/spec/v3/services/env_var/delete_spec.rb @@ -11,23 +11,58 @@ include_examples 'not authenticated' end - describe 'authenticated, missing repo' do - before { delete("/v3/repo/999999999/env_var/foo", {}, auth_headers) } - include_examples 'missing repo' + describe 'authenticated, wrong permissions' do + before do + Travis::API::V3::Models::Permission.create(repository: repo, user: repo.owner, pull: true) + delete("/v3/repo/#{repo.id}/env_var/foo", {}, auth_headers) + end + example { expect(last_response.status).to eq 403 } + example do + expect(JSON.load(last_response.body)).to eq( + '@type' => 'error', + 'error_type' => 'insufficient_access', + 'error_message' => 'operation requires change_env_vars access to repository', + 'resource_type' => 'repository', + 'permission' => 'change_env_vars', + 'repository' => { + '@type' => 'repository', + '@href' => "/v3/repo/#{repo.id}", + '@representation' => 'minimal', + 'id' => repo.id, + 'name' => repo.name, + 'slug' => repo.slug + } + ) + end end - describe 'authenticated, missing repo, missing env var' do - before { delete("/v3/repo/#{repo.id}/env_var/#{env_var[:id]}", {}, auth_headers) } - include_examples 'missing env_var' - end + context 'authenticated, right permissions' do + before { Travis::API::V3::Models::Permission.create(repository: repo, user: repo.owner, push: true) } - describe 'authenticated, missing repo, existing env var' do - before do - repo.update_attributes(settings: JSON.generate(env_vars: [env_var])) - delete("/v3/repo/#{repo.id}/env_var/#{env_var[:id]}", {}, auth_headers) + describe 'missing repo' do + before { delete("/v3/repo/999999999/env_var/foo", {}, auth_headers) } + include_examples 'missing repo' end - example { expect(last_response.status).to eq 200 } - example { expect(JSON.parse(last_response.body)["id"]).to eq(env_var[:id]) } + describe 'existing repo, missing env var' do + before { delete("/v3/repo/#{repo.id}/env_var/#{env_var[:id]}", {}, auth_headers) } + include_examples 'missing env_var' + end + + describe 'existing repo, existing env var' do + before do + repo.update_attributes(settings: JSON.generate(env_vars: [env_var], foo: 'bar')) + delete("/v3/repo/#{repo.id}/env_var/#{env_var[:id]}", {}, auth_headers) + end + + example { expect(last_response.status).to eq 204 } + example { expect(last_response.body).to be_empty } + example 'persists changes' do + expect(repo.reload.env_vars.find(env_var[:id])).to be_nil + end + example 'does not clobber other settings' do + expect(repo.reload.settings['foo']).to eq 'bar' + end + end end end diff --git a/spec/v3/services/env_var/update_spec.rb b/spec/v3/services/env_var/update_spec.rb index c4cab5798b..d11cdfced6 100644 --- a/spec/v3/services/env_var/update_spec.rb +++ b/spec/v3/services/env_var/update_spec.rb @@ -30,7 +30,7 @@ end before do - repo.update_attributes(settings: JSON.generate(env_vars: [env_var])) + repo.update_attributes(settings: JSON.generate(env_vars: [env_var], foo: 'bar')) patch("/v3/repo/#{repo.id}/env_var/#{env_var[:id]}", JSON.generate(params), auth_headers.merge(json_headers)) end @@ -46,5 +46,11 @@ 'public' => env_var[:public] ) end + example 'persists changes' do + expect(repo.reload.settings['env_vars'].first['name']).to eq 'QUX' + end + example 'does not clobber other settings' do + expect(repo.reload.settings['foo']).to eq 'bar' + end end end diff --git a/spec/v3/services/env_vars/create_spec.rb b/spec/v3/services/env_vars/create_spec.rb index 48e82f21f3..b6e4433a9e 100644 --- a/spec/v3/services/env_vars/create_spec.rb +++ b/spec/v3/services/env_vars/create_spec.rb @@ -63,6 +63,9 @@ ) expect(response).to include('@href', 'id') end + example 'persists changes' do + expect(repo.reload.settings['env_vars'].first['name']).to eq 'FOO' + end end describe 'public' do diff --git a/spec/v3/services/job/find_spec.rb b/spec/v3/services/job/find_spec.rb index 1030565719..3ec930e4ba 100644 --- a/spec/v3/services/job/find_spec.rb +++ b/spec/v3/services/job/find_spec.rb @@ -23,7 +23,8 @@ "read" => true, "cancel" => false, "restart" => false, - "debug" => false }, + "debug" => false, + "delete_log" => false }, "id" => job.id, "number" => job.number, "state" => job.state, @@ -83,6 +84,7 @@ let(:headers) {{ 'HTTP_AUTHORIZATION' => "token #{token}" }} before { Travis::API::V3::Models::Permission.create(repository: repo, user: repo.owner, pull: true) } before { repo.update_attribute(:private, true) } + before { Travis::API::V3::Permissions::Job.any_instance.stubs(:delete_log?).returns(true) } before { get("/v3/job/#{job.id}", {}, headers) } after { repo.update_attribute(:private, false) } example { expect(last_response).to be_ok } @@ -94,7 +96,8 @@ "read" => true, "cancel" => true, "restart" => true, - "debug" => false }, + "debug" => false, + "delete_log" => true }, "id" => job.id, "number" => job.number, "state" => job.state, diff --git a/spec/v3/services/jobs/find_spec.rb b/spec/v3/services/jobs/find_spec.rb index 760fcbfa88..6d4d82b32f 100644 --- a/spec/v3/services/jobs/find_spec.rb +++ b/spec/v3/services/jobs/find_spec.rb @@ -20,7 +20,8 @@ "read" => true, "cancel" => false, "restart" => false, - "debug" => false }, + "debug" => false, + "delete_log" => false }, "id" => jobs[0].id, "number" => "#{jobs[0].number}", "state" => "configured", @@ -68,7 +69,8 @@ "read" => true, "cancel" => false, "restart" => false, - "debug" => false }, + "debug" => false, + "delete_log" => false}, "id" => jobs[1].id, "number" => "#{jobs[1].number}", "state" => "configured", @@ -116,7 +118,8 @@ "read" => true, "cancel" => false, "restart" => false, - "debug" => false }, + "debug" => false, + "delete_log" => false}, "id" => jobs[2].id, "number" => "#{jobs[2].number}", "state" => "configured", @@ -164,7 +167,8 @@ "read" => true, "cancel" => false, "restart" => false, - "debug" => false }, + "debug" => false, + "delete_log" => false}, "id" => jobs[3].id, "number" => "#{jobs[3].number}", "state" => "configured", @@ -210,7 +214,7 @@ } end - describe "jobs private repository, private API, authenticated as user with access" do + describe "jobs private repository, private API, authenticated as user with pull access" do let(:token) { Travis::Api::App::AccessToken.create(user: repo.owner, app_id: 1) } let(:headers) {{ 'HTTP_AUTHORIZATION' => "token #{token}" }} before { Travis::API::V3::Models::Permission.create(repository: repo, user: repo.owner, pull: true) } @@ -230,7 +234,8 @@ "read" => true, "cancel" => true, "restart" => true, - "debug" => false }, + "debug" => false, + "delete_log" => false }, "id" => jobs[0].id, "number" => "#{jobs[0].number}", "state" => "configured", @@ -278,7 +283,8 @@ "read" => true, "cancel" => true, "restart" => true, - "debug" => false }, + "debug" => false, + "delete_log" => false }, "id" => jobs[1].id, "number" => "#{jobs[1].number}", "state" => "configured", @@ -326,7 +332,8 @@ "read" => true, "cancel" => true, "restart" => true, - "debug" => false }, + "debug" => false, + "delete_log" => false }, "id" => jobs[2].id, "number" => "#{jobs[2].number}", "state" => "configured", @@ -374,7 +381,224 @@ "read" => true, "cancel" => true, "restart" => true, - "debug" => false }, + "debug" => false, + "delete_log" => false }, + "id" => jobs[3].id, + "number" => "#{jobs[3].number}", + "state" => "configured", + "started_at" => "2010-11-12T13:00:00Z", + "finished_at" => nil, + "build" => { + "@type" => "build", + "@href" => "/v3/build/#{build.id}", + "@representation"=> "minimal", + "id" => build.id, + "number" => build.number, + "state" => "configured", + "duration" => nil, + "event_type" => "push", + "previous_state" => "passed", + "started_at" => "2010-11-12T13:00:00Z", + "finished_at" => nil}, + "queue" => "builds.linux", + "repository" =>{ + "@type" => "repository", + "@href" => "/v3/repo/1", + "@representation"=>"minimal", + "id" => repo.id, + "name" => "minimal", + "slug" => "svenfuchs/minimal"}, + "commit" =>{ + "@type" => "commit", + "@representation"=> "minimal", + "id" => commit.id, + "sha" => commit.commit, + "ref" => commit.ref, + "message" => commit.message, + "compare_url" => commit.compare_url, + "committed_at" =>"2010-11-12T12:55:00Z"}, + "owner" =>{ + "@type" => "user", + "@href" => "/v3/user/1", + "@representation"=> "minimal", + "id" => 1, + "login" => "svenfuchs"}} + ] + } + } + end + +describe "jobs private repository, private API, authenticated as user with push access" do + let(:token) { Travis::Api::App::AccessToken.create(user: repo.owner, app_id: 1) } + let(:headers) {{ 'HTTP_AUTHORIZATION' => "token #{token}" }} + before { Travis::API::V3::Models::Permission.create(repository: repo, user: repo.owner, pull: true, push: true) } + before { Travis::API::V3::Permissions::Job.any_instance.stubs(:delete_log?).returns(true) } + before { Travis::API::V3::Permissions::Job.any_instance.stubs(:debug?).returns(true) } + before { repo.update_attribute(:private, true) } + before { get("/v3/build/#{build.id}/jobs", {}, headers) } + after { repo.update_attribute(:private, false) } + example { expect(last_response).to be_ok } + example { expect(parsed_body).to be == { + "@type" => "jobs", + "@href" => "/v3/build/#{build.id}/jobs", + "@representation" => "standard", + "jobs" => [{ + "@type" => "job", + "@href" => "/v3/job/#{jobs[0].id}", + "@representation" => "standard", + "@permissions" => { + "read" => true, + "cancel" => true, + "restart" => true, + "debug" => true, + "delete_log" => true }, + "id" => jobs[0].id, + "number" => "#{jobs[0].number}", + "state" => "configured", + "started_at" => "2010-11-12T13:00:00Z", + "finished_at" => nil, + "build" => { + "@type" => "build", + "@href" => "/v3/build/#{build.id}", + "@representation"=> "minimal", + "id" => build.id, + "number" => build.number, + "state" => "configured", + "duration" => nil, + "event_type" => "push", + "previous_state" => "passed", + "started_at" => "2010-11-12T13:00:00Z", + "finished_at" => nil}, + "queue" => "builds.linux", + "repository" =>{ + "@type" => "repository", + "@href" => "/v3/repo/1", + "@representation"=>"minimal", + "id" => repo.id, + "name" => "minimal", + "slug" => "svenfuchs/minimal"}, + "commit" =>{ + "@type" => "commit", + "@representation"=> "minimal", + "id" => commit.id, + "sha" => commit.commit, + "ref" => commit.ref, + "message" => commit.message, + "compare_url" => commit.compare_url, + "committed_at" =>"2010-11-12T12:55:00Z"}, + "owner" =>{ + "@type" => "user", + "@href" => "/v3/user/1", + "@representation"=> "minimal", + "id" => 1, + "login" => "svenfuchs"}}, + {"@type" => "job", + "@href" => "/v3/job/#{jobs[1].id}", + "@representation" => "standard", + "@permissions" => { + "read" => true, + "cancel" => true, + "restart" => true, + "debug" => true, + "delete_log" => true }, + "id" => jobs[1].id, + "number" => "#{jobs[1].number}", + "state" => "configured", + "started_at" => "2010-11-12T13:00:00Z", + "finished_at" => nil, + "build" => { + "@type" => "build", + "@href" => "/v3/build/#{build.id}", + "@representation"=> "minimal", + "id" => build.id, + "number" => build.number, + "state" => "configured", + "duration" => nil, + "event_type" => "push", + "previous_state" => "passed", + "started_at" => "2010-11-12T13:00:00Z", + "finished_at" => nil}, + "queue" => "builds.linux", + "repository" =>{ + "@type" => "repository", + "@href" => "/v3/repo/1", + "@representation"=>"minimal", + "id" => repo.id, + "name" => "minimal", + "slug" => "svenfuchs/minimal"}, + "commit" =>{ + "@type" => "commit", + "@representation"=> "minimal", + "id" => commit.id, + "sha" => commit.commit, + "ref" => commit.ref, + "message" => commit.message, + "compare_url" => commit.compare_url, + "committed_at" =>"2010-11-12T12:55:00Z"}, + "owner" =>{ + "@type" => "user", + "@href" => "/v3/user/1", + "@representation"=> "minimal", + "id" => 1, + "login" => "svenfuchs"}}, + {"@type" => "job", + "@href" => "/v3/job/#{jobs[2].id}", + "@representation" => "standard", + "@permissions" => { + "read" => true, + "cancel" => true, + "restart" => true, + "debug" => true, + "delete_log" => true }, + "id" => jobs[2].id, + "number" => "#{jobs[2].number}", + "state" => "configured", + "started_at" => "2010-11-12T13:00:00Z", + "finished_at" => nil, + "build" => { + "@type" => "build", + "@href" => "/v3/build/#{build.id}", + "@representation"=> "minimal", + "id" => build.id, + "number" => build.number, + "state" => "configured", + "duration" => nil, + "event_type" => "push", + "previous_state" => "passed", + "started_at" => "2010-11-12T13:00:00Z", + "finished_at" => nil}, + "queue" => "builds.linux", + "repository" =>{ + "@type" => "repository", + "@href" => "/v3/repo/1", + "@representation"=>"minimal", + "id" => repo.id, + "name" => "minimal", + "slug" => "svenfuchs/minimal"}, + "commit" =>{ + "@type" => "commit", + "@representation"=> "minimal", + "id" => commit.id, + "sha" => commit.commit, + "ref" => commit.ref, + "message" => commit.message, + "compare_url" => commit.compare_url, + "committed_at" =>"2010-11-12T12:55:00Z"}, + "owner" =>{ + "@type" => "user", + "@href" => "/v3/user/1", + "@representation"=> "minimal", + "id" => 1, + "login" => "svenfuchs"}}, + {"@type" => "job", + "@href" => "/v3/job/#{jobs[3].id}", + "@representation" => "standard", + "@permissions" => { + "read" => true, + "cancel" => true, + "restart" => true, + "debug" => true, + "delete_log" => true }, "id" => jobs[3].id, "number" => "#{jobs[3].number}", "state" => "configured", diff --git a/spec/v3/services/log/delete_spec.rb b/spec/v3/services/log/delete_spec.rb new file mode 100644 index 0000000000..7e936727e6 --- /dev/null +++ b/spec/v3/services/log/delete_spec.rb @@ -0,0 +1,175 @@ +require 'spec_helper' + +describe Travis::API::V3::Services::Log::Delete, set_app: true do + let(:user) { Factory.create(:user) } + let(:repo) { Factory.create(:repository, owner_name: user.login, name: 'minimal', owner: user)} + let(:repo2) { Factory.create(:repository, owner_name: user.login, name: 'minimal2', owner: user)} + let(:build) { Factory.create(:build, repository: repo) } + let(:build2) { Factory.create(:build, repository: repo2) } + let(:job) { Travis::API::V3::Models::Job.create(build: build) } + let(:job2) { Travis::API::V3::Models::Job.create(build: build2) } + let(:job3) { Travis::API::V3::Models::Job.create(build: build2) } + let(:s3job) { Travis::API::V3::Models::Job.create(build: build) } + let(:token) { Travis::Api::App::AccessToken.create(user: user, app_id: 1) } + let(:headers) { { 'HTTP_AUTHORIZATION' => "token #{token}" } } + let(:parsed_body) { JSON.load(body) } + let(:log) { Travis::API::V3::Models::Log.create(job: job) } + let(:log2) { Travis::API::V3::Models::Log.create(job: job2) } + let(:s3log) { Travis::API::V3::Models::Log.create(job: s3job, content: 'minimal log 1') } + let(:xml_content) { + " + + bucket + + + 1000 + false + + jobs/#{s3job.id}/log.txt + 2009-10-12T17:50:30.000Z + "hgb9dede5f27731c9771645a39863328" + 20308738 + STANDARD + + 75aa57f09aa0c8caeab4f8c24e99d10f8e7faeebf76c078efc7c6caea54ba06a + mtd@amazon.com + + $ git clean -fdx\nRemoving Gemfile.lock\n$ git fetch + + + " + } + + before do + Travis::API::V3::AccessControl::LegacyToken.any_instance.stubs(:visible?).returns(true) + Travis::API::V3::Permissions::Job.any_instance.stubs(:delete_log?).returns(true) + stub_request(:get, "https://bucket.s3.amazonaws.com/?max-keys=1000"). + to_return(:status => 200, :body => xml_content, :headers => {}) + stub_request(:get, "https://s3.amazonaws.com/archive.travis-ci.org/?prefix=jobs/#{log.job.id}/log.txt"). + to_return(:status => 200, :body => xml_content, :headers => {}) + end + + describe "not authenticated" do + before { delete("/v3/job/#{log.job.id}/log") } + example { expect(last_response.status).to be == 403 } + example { expect(JSON.load(body)).to be == { + "@type" => "error", + "error_type" => "login_required", + "error_message" => "login required" + }} + end + + describe "missing log, authenticated" do + before { job3.update_attributes(finished_at: Time.now, state: "passed")} + + example do + delete("/v3/job/#{job3.id}/log", {}, headers) + expect(last_response.status).to be == 404 + expect(JSON.load(body)).to be == { + "@type" => "error", + "error_type" => "not_found", + "error_message" => "log not found" + } + end + end + + describe 'existing db log, authenticated' do + before do + Timecop.return + Timecop.freeze(Time.now.utc) + job.update_attributes(finished_at: Time.now) + end + after { Timecop.return } + example do + delete("/v3/job/#{log.job.id}/log", {}, headers) + expect(last_response.status).to be == 200 + expect(JSON.load(body)).to be == {"@type"=>"log", + "@href"=>"/v3/job/#{log.job.id}/log", + "@representation"=>"standard", + "id"=>log.id, + "content"=>nil, + "log_parts"=>[{ + "@type"=>"log_part", + "@representation"=>"minimal", + "content"=>"Log removed by Sven Fuchs at #{Time.now.utc}", + "number"=>1}]} + end + end + + context 's3 log, authenticated' do + before do + s3job.update_attributes(finished_at: Time.now) + Fog.mock! + Travis.config.logs_options.s3 = { access_key_id: 'key', secret_access_key: 'secret' } + storage = Fog::Storage.new({ + :aws_access_key_id => "key", + :aws_secret_access_key => "secret", + :provider => "AWS" + }) + bucket = storage.directories.create(:key => 'archive.travis-ci.org') + file = bucket.files.create( + :key => "jobs/#{s3job.id}/log.txt", + :body => "$ git clean -fdx\nRemoving Gemfile.lock\n$ git fetch" + ) + end + after { Fog::Mock.reset } + + describe 'updates log, inserts new log part' do + before do + Timecop.return + Timecop.freeze(Time.now.utc) + end + after { Timecop.return } + example do + s3log.update_attributes(archived_at: Time.now) + delete("/v3/job/#{s3log.job.id}/log", {}, headers) + expect(last_response.status).to be == 200 + expect(JSON.load(body)).to be == {"@type"=>"log", + "@href"=>"/v3/job/#{s3log.job.id}/log", + "@representation"=>"standard", + "id"=>s3log.id, + "content"=>nil, + "log_parts"=>[{ + "@type"=>"log_part", + "@representation"=>"minimal", + "content"=>"Log removed by Sven Fuchs at #{Time.now.utc}", + "number"=>1}]} + end + end + end + + context 'when job for log is still running, authenticated' do + example do + delete("/v3/job/#{log2.job.id}/log", {}, headers) + expect(last_response.status).to be == 409 + expect(parsed_body).to eq({ + "@type"=>"error", + "error_type"=>"job_unfinished", + "error_message"=>"job still running, cannot remove log yet"}) + end + end + + context 'when log already removed_at, authenticated' do + before { log2.update_attributes(removed_at: Time.now) } + example do + delete("/v3/job/#{log2.job.id}/log", {}, headers) + expect(last_response.status).to be == 409 + expect(parsed_body).to eq({ + "@type"=>"error", + "error_type"=>"log_already_removed", + "error_message"=>"log has already been removed"}) + end + end + + context 'when log already removed_by, authenticated' do + before { log2.update_attributes(removed_by: user) } + example do + delete("/v3/job/#{log2.job.id}/log", {}, headers) + expect(last_response.status).to be == 409 + expect(parsed_body).to eq({ + "@type"=>"error", + "error_type"=>"log_already_removed", + "error_message"=>"log has already been removed"}) + end + end +end diff --git a/spec/v3/services/log/find_spec.rb b/spec/v3/services/log/find_spec.rb new file mode 100644 index 0000000000..f114d11811 --- /dev/null +++ b/spec/v3/services/log/find_spec.rb @@ -0,0 +1,158 @@ +require 'spec_helper' + +describe Travis::API::V3::Services::Log::Find, set_app: true do + let(:user) { Factory.create(:user) } + let(:repo) { Factory.create(:repository, owner_name: user.login, name: 'minimal', owner: user)} + let(:build) { Factory.create(:build, repository: repo) } + let(:job) { Travis::API::V3::Models::Job.create(build: build) } + let(:job2) { Travis::API::V3::Models::Job.create(build: build)} + let(:job3) { Travis::API::V3::Models::Job.create(build: build)} + let(:s3job) { Travis::API::V3::Models::Job.create(build: build) } + let(:token) { Travis::Api::App::AccessToken.create(user: user, app_id: 1) } + let(:headers) { { 'HTTP_AUTHORIZATION' => "token #{token}" } } + let(:parsed_body) { JSON.load(body) } + let(:log) { Travis::API::V3::Models::Log.create(job: job) } + let(:log2) { Travis::API::V3::Models::Log.create(job: job2) } + let(:log3) { Travis::API::V3::Models::Log.create(job: job3) } + let(:s3log) { Travis::API::V3::Models::Log.create(job: s3job, content: 'minimal log 1') } + let(:find_log) { "string" } + let(:xml_content) { + " + + bucket + + + 1000 + false + + jobs/#{s3job.id}/log.txt + 2009-10-12T17:50:30.000Z + "hgb9dede5f27731c9771645a39863328" + 20308738 + STANDARD + + 75aa57f09aa0c8caeab4f8c24e99d10f8e7faeebf76c078efc7c6caea54ba06a + mtd@amazon.com + + $ git clean -fdx\nRemoving Gemfile.lock\n$ git fetch + + + " + } + + + before do + log3.delete + Travis::API::V3::AccessControl::LegacyToken.any_instance.stubs(:visible?).returns(true) + stub_request(:get, "https://bucket.s3.amazonaws.com/?max-keys=1000"). + to_return(:status => 200, :body => xml_content, :headers => {}) + stub_request(:get, "https://s3.amazonaws.com/archive.travis-ci.com/?prefix=jobs/#{s3job.id}/log.txt"). + to_return(status: 200, body: xml_content, headers: {}) + Fog.mock! + Travis.config.logs_options.s3 = { access_key_id: 'key', secret_access_key: 'secret' } + storage = Fog::Storage.new({ + :aws_access_key_id => "key", + :aws_secret_access_key => "secret", + :provider => "AWS" + }) + bucket = storage.directories.create(:key => 'archive.travis-ci.org') + file = bucket.files.create( + :key => "jobs/#{s3job.id}/log.txt", + :body => "$ git clean -fdx\nRemoving Gemfile.lock\n$ git fetch" + ) + end + after { Fog::Mock.reset } + + context 'when log stored in db' do + describe 'returns log with an array of Log Parts' do + example do + log_part = log.log_parts.create(content: "logging it", number: 0) + get("/v3/job/#{log.job.id}/log", {}, headers) + expect(parsed_body).to eq( + '@href' => "/v3/job/#{log.job.id}/log", + '@representation' => 'standard', + '@type' => 'log', + 'content' => nil, + 'id' => log.id, + 'log_parts' => [{ + "@type" => "log_part", + "@representation" => "minimal", + "content" => log_part.content, + "number" => log_part.number }]) + end + end + + describe 'returns aggregated log with an array of Log Parts' do + before { log2.update_attributes(aggregated_at: Time.now, content: "aggregating!")} + example do + get("/v3/job/#{log2.job.id}/log", {}, headers) + expect(parsed_body).to eq( + '@type' => 'log', + '@href' => "/v3/job/#{log2.job.id}/log", + '@representation' => 'standard', + 'content' => "aggregating!", + 'id' => log2.id, + 'log_parts' => [{ + "@type" => "log_part", + "@representation" => "minimal", + "content" => "aggregating!", + "number" => 0 }]) + end + end + + describe 'returns log as plain text' do + example do + log_part = log.log_parts.create(content: "logging it", number: 1) + log_part2 = log.log_parts.create(content: "logging more", number: 2) + log_part3 = log.log_parts.create(content: "logging forever", number: 3) + + get("/v3/job/#{log.job.id}/log", {}, headers.merge('HTTP_ACCEPT' => 'text/plain')) + expect(body).to eq( + "logging it\nlogging more\nlogging forever") + end + end + end + + context 'when log not found in db but stored on S3' do + describe 'returns log with an array of Log Parts' do + example do + s3log.update_attributes(archived_at: Time.now) + get("/v3/job/#{s3log.job.id}/log", {}, headers) + + expect(parsed_body).to eq( + '@type' => 'log', + '@href' => "/v3/job/#{s3job.id}/log", + '@representation' => 'standard', + 'id' => s3log.id, + 'content' => 'minimal log 1', + 'log_parts' => [{ + "@type"=>"log_part", + "@representation"=>"minimal", + "content"=>"$ git clean -fdx\nRemoving Gemfile.lock\n$ git fetch", + "number"=>0}]) + end + end + describe 'returns log as plain text' do + example do + s3log.update_attributes(archived_at: Time.now) + get("/v3/job/#{s3log.job.id}/log", {}, headers.merge('HTTP_ACCEPT' => 'text/plain')) + expect(last_response.headers).to include("Content-Type" => "text/plain") + expect(body).to eq( + "$ git clean -fdx\nRemoving Gemfile.lock\n$ git fetch") + end + end + end + + context 'when log not found anywhere' do + describe 'does not return log - returns error' do + example do + log3.delete + get("/v3/job/#{job3.id}/log", {}, headers) + expect(parsed_body).to eq({ + "@type"=>"error", + "error_type"=>"not_found", + "error_message"=>"log not found"}) + end + end + end +end diff --git a/spec/v3/services/owner/find_spec.rb b/spec/v3/services/owner/find_spec.rb index 3b7df32715..7a11c4481f 100644 --- a/spec/v3/services/owner/find_spec.rb +++ b/spec/v3/services/owner/find_spec.rb @@ -66,7 +66,10 @@ "unstar" => false, "create_request" => false, "create_cron" => false, - "change_settings" => false + "change_settings" => false, + "change_env_vars" => false, + "change_key" => false, + "admin" => false }, "id" => repo.id, "name" => "example-repo", @@ -116,7 +119,10 @@ "unstar" => false, "create_request"=> false, "create_cron" => false, - "change_settings" => false + "change_settings" => false, + "change_env_vars" => false, + "change_key" => false, + "admin" => false }, "id" => repo.id, "name" => "example-repo", diff --git a/spec/v3/services/repositories/for_current_user_spec.rb b/spec/v3/services/repositories/for_current_user_spec.rb index ea2682c4e2..d7a34a0f1b 100644 --- a/spec/v3/services/repositories/for_current_user_spec.rb +++ b/spec/v3/services/repositories/for_current_user_spec.rb @@ -5,7 +5,7 @@ let(:token) { Travis::Api::App::AccessToken.create(user: repo.owner, app_id: 1) } let(:headers) {{ 'HTTP_AUTHORIZATION' => "token #{token}" }} - before { Travis::API::V3::Models::Permission.create(repository: repo, user: repo.owner, pull: true, push: true) } + before { Travis::API::V3::Models::Permission.create(repository: repo, user: repo.owner, pull: true, push: true, admin: true) } before { repo.update_attribute(:private, true) } after { repo.update_attribute(:private, false) } @@ -44,7 +44,10 @@ "unstar" => true, "create_request" => true, "create_cron" => false, - "change_settings" => true + "change_settings" => true, + "change_env_vars" => true, + "change_key" => true, + "admin" => true }, "id" => repo.id, "name" => "minimal", diff --git a/spec/v3/services/repositories/for_owner_spec.rb b/spec/v3/services/repositories/for_owner_spec.rb index a9adb542a2..15d197989b 100644 --- a/spec/v3/services/repositories/for_owner_spec.rb +++ b/spec/v3/services/repositories/for_owner_spec.rb @@ -45,7 +45,10 @@ "unstar" => false, "create_request" => false, "create_cron" => false, - "change_settings" => false + "change_settings" => false, + "change_env_vars" => false, + "change_key" => false, + "admin" => false }, "id" => repo.id, "name" => "minimal", @@ -124,7 +127,10 @@ "unstar" => false, "create_request"=> false, "create_cron" => false, - "change_settings" => false + "change_settings" => false, + "change_env_vars" => false, + "change_key" => false, + "admin" => false }, "id" => 1, "name" => "minimal", @@ -155,7 +161,10 @@ "unstar" => false, "create_request"=> false, "create_cron" => false, - "change_settings" => false + "change_settings" => false, + "change_env_vars" => false, + "change_key" => false, + "admin" => false }, "id" => repo2.id, "name" => "maximal", diff --git a/spec/v3/services/repository/find_spec.rb b/spec/v3/services/repository/find_spec.rb index efd18456b8..6fca7eaea7 100644 --- a/spec/v3/services/repository/find_spec.rb +++ b/spec/v3/services/repository/find_spec.rb @@ -1,4 +1,5 @@ describe Travis::API::V3::Services::Repository::Find, set_app: true do + let(:user) { User.where(login: 'svenfuchs').first } let(:repo) { Travis::API::V3::Models::Repository.where(owner_name: 'svenfuchs', name: 'minimal').first } let(:build) { repo.builds.first } let(:jobs) { Travis::API::V3::Models::Build.find(build.id).jobs } @@ -36,7 +37,10 @@ "unstar" => false, "create_request" => false, "create_cron" => false, - "change_settings" => false + "change_settings" => false, + "change_env_vars" => false, + "change_key" => false, + "admin" => false }, "id" => repo.id, "name" => "minimal", @@ -99,7 +103,7 @@ describe "private repository, private API, authenticated as user with access" do let(:token) { Travis::Api::App::AccessToken.create(user: repo.owner, app_id: 1) } let(:headers) {{ 'HTTP_AUTHORIZATION' => "token #{token}" }} - before { Travis::API::V3::Models::Permission.create(repository: repo, user: repo.owner, pull: true) } + before { Travis::API::V3::Models::Permission.create(repository: repo, user: repo.owner, pull: true, admin: false) } before { repo.update_attribute(:private, true) } before { get("/v3/repo/#{repo.id}", {}, headers) } after { repo.update_attribute(:private, false) } @@ -116,7 +120,10 @@ "unstar" => false, "create_request" => false, "create_cron" => false, - "change_settings" => false + "change_settings" => false, + "change_env_vars" => false, + "change_key" => false, + "admin" => false }, "id" => repo.id, "name" => "minimal", @@ -181,7 +188,10 @@ "unstar" => true, "create_request" => true, "create_cron" => false, - "change_settings" => true + "change_settings" => true, + "change_env_vars" => true, + "change_key" => true, + "admin" => false }, "id" => repo.id, "name" => "minimal", @@ -225,19 +235,13 @@ }} end - describe "private repository without cron feature, authenticated as internal application with full access, scoped to the right org" do - let(:app_name) { 'travis-example' } - let(:app_secret) { '12345678' } - let(:sign_opts) { "a=#{app_name}:s=#{repo.owner_name}" } - let(:signature) { OpenSSL::HMAC.hexdigest('sha256', app_secret, sign_opts) } - let(:headers) {{ 'HTTP_AUTHORIZATION' => "signature #{sign_opts}:#{signature}" }} - before { Travis.config.applications = { app_name => { full_access: true, secret: app_secret }}} - + describe "private repository without cron feature, authenticated as user with admin access" do + let(:token) { Travis::Api::App::AccessToken.create(user: repo.owner, app_id: 1) } + let(:headers) {{ 'HTTP_AUTHORIZATION' => "token #{token}" }} + before { Travis::API::V3::Models::Permission.create(repository: repo, user: repo.owner, pull: true, push: true, admin: true) } before { repo.update_attribute(:private, true) } before { get("/v3/repo/#{repo.id}", {}, headers) } - before { repo.update_attribute(:private, false) } - example { expect(last_response).to be_ok } example { expect(parsed_body).to be == { @@ -252,7 +256,10 @@ "unstar" => true, "create_request" => true, "create_cron" => false, - "change_settings" => true + "change_settings" => true, + "change_env_vars" => true, + "change_key" => true, + "admin" => true }, "id" => repo.id, "name" => "minimal", diff --git a/spec/v3/services/ssh_key/create_spec.rb b/spec/v3/services/ssh_key/create_spec.rb new file mode 100644 index 0000000000..3c6ea08b5c --- /dev/null +++ b/spec/v3/services/ssh_key/create_spec.rb @@ -0,0 +1,80 @@ +require 'spec_helper' + +describe Travis::API::V3::Services::SshKey::Create, set_app: true do + let(:repo) do + Travis::API::V3::Models::Repository.where(owner_name: 'svenfuchs', name: 'minimal').first_or_create.tap do |repo| + repo.create_key.tap { |key| key.generate_keys!; key.save! } + end + end + let(:token) { Travis::Api::App::AccessToken.create(user: repo.owner, app_id: 1) } + let(:other_user) { FactoryGirl.create(:user) } + let(:other_token) { Travis::Api::App::AccessToken.create(user: other_user, app_id: 2) } + let(:auth_headers) { { 'HTTP_AUTHORIZATION' => "token #{token}" } } + + describe 'not authenticated' do + before { post("/v3/repo/#{repo.id}/ssh_key") } + include_examples 'not authenticated' + end + + context 'authenticated as wrong user' do + describe 'not allowed' do + before { post("/v3/repo/#{repo.id}/ssh_key", {}, 'HTTP_AUTHORIZATION' => "token #{other_token}") } + + example { expect(last_response.status).to eq(403) } + example do + expect(JSON.load(body)).to eq( + '@type' => 'error', + 'error_type' => 'insufficient_access', + 'error_message' => 'operation requires change_key access to repository', + 'permission' => 'change_key', + 'resource_type' => 'repository', + 'repository' => { + '@type' => 'repository', + '@href' => "/v3/repo/#{repo.id}", + '@representation' => 'minimal', + 'id' => repo.id, + 'name' => 'minimal', + 'slug' => 'svenfuchs/minimal' + } + ) + end + end + end + + context 'authenticated' do + describe 'missing repo' do + before { post("/v3/repo/999999999/ssh_key", {}, auth_headers) } + include_examples 'missing repo' + end + + describe 'existing repo, creates key when none exists' do + before do + Travis::API::V3::Models::Permission.create(repository: repo, user: repo.owner, push: true) + repo.key.destroy + post("/v3/repo/#{repo.id}/ssh_key", {}, auth_headers) + end + + example { expect(last_response.status).to eq 201 } + example do + expect(JSON.parse(last_response.body)).to include *%w{@type @href @representation id public_key fingerprint} + end + end + + describe 'existing repo, regenerates key when one exists' do + let!(:key) { repo.key } + + before do + Travis::API::V3::Models::Permission.create(repository: repo, user: repo.owner, push: true) + post("/v3/repo/#{repo.id}/ssh_key", {}, auth_headers) + end + + example { expect(last_response.status).to eq 201 } + example do + result = JSON.parse(last_response.body) + expect(result).to include *%w{@type @href @representation id public_key fingerprint} + expect(result['id']).to eq key.reload.id + expect(result['fingerprint']).to eq(key.reload.fingerprint) + end + end + end +end diff --git a/spec/v3/services/ssh_key/find_spec.rb b/spec/v3/services/ssh_key/find_spec.rb new file mode 100644 index 0000000000..2546686e33 --- /dev/null +++ b/spec/v3/services/ssh_key/find_spec.rb @@ -0,0 +1,55 @@ +require 'spec_helper' + +describe Travis::API::V3::Services::SshKey::Find, set_app: true do + let(:repo) do + Travis::API::V3::Models::Repository.where(owner_name: 'svenfuchs', name: 'minimal').first_or_create.tap do |repo| + repo.create_key.tap { |key| key.generate_keys!; key.save! } + end + end + let(:token) { Travis::Api::App::AccessToken.create(user: repo.owner, app_id: 1) } + let(:auth_headers) { { 'HTTP_AUTHORIZATION' => "token #{token}" } } + + describe 'not authenticated' do + before { get("/v3/repo/#{repo.id}/ssh_key") } + include_examples 'not authenticated' + end + + context 'authenticated' do + describe 'missing repo' do + before { get("/v3/repo/999999999/ssh_key", {}, auth_headers) } + include_examples 'missing repo' + end + + describe 'existing repo, no key' do + before do + repo.key.destroy + get("/v3/repo/#{repo.id}/ssh_key", {}, auth_headers) + end + + example { expect(last_response.status).to eq 404 } + example do + expect(JSON.parse(last_response.body)).to eq( + '@type' => 'error', + 'error_message' => 'ssh_key not found (or insufficient access)', + 'error_type' => 'not_found', + 'resource_type' => 'ssh_key' + ) + end + end + + describe 'existing repo, existing key' do + before { get("/v3/repo/#{repo.id}/ssh_key", {}, auth_headers) } + example { expect(last_response.status).to eq 200 } + example do + expect(JSON.parse(last_response.body)).to eq( + '@type' => 'ssh_key', + '@href' => "/v3/repo/#{repo.id}/ssh_key", + '@representation' => 'standard', + 'id' => repo.key.id, + 'public_key' => repo.key.public_key, + 'fingerprint' => repo.key.fingerprint + ) + end + end + end +end diff --git a/spec/v3/services/user_settings/find_spec.rb b/spec/v3/services/user_settings/find_spec.rb index d351f64d90..294f4ca736 100644 --- a/spec/v3/services/user_settings/find_spec.rb +++ b/spec/v3/services/user_settings/find_spec.rb @@ -25,6 +25,9 @@ end end + before { Travis::Features.activate_owner(:auto_cancel, repo.owner) } + after { Travis::Features.deactivate_owner(:auto_cancel, repo.owner) } + describe 'authenticated, existing repo, repo has no settings, return defaults' do before { get("/v3/repo/#{repo.id}/settings", {}, auth_headers) } @@ -38,7 +41,9 @@ { '@type' => 'user_setting','@href' => "/v3/repo/#{repo.id}/setting/builds_only_with_travis_yml", '@representation' => 'standard', 'name' => 'builds_only_with_travis_yml', 'value' => false }, { '@type' => 'user_setting','@href' => "/v3/repo/#{repo.id}/setting/build_pushes", '@representation' => 'standard', 'name' => 'build_pushes', 'value' => true }, { '@type' => 'user_setting','@href' => "/v3/repo/#{repo.id}/setting/build_pull_requests", '@representation' => 'standard', 'name' => 'build_pull_requests', 'value' => true }, - { '@type' => 'user_setting','@href' => "/v3/repo/#{repo.id}/setting/maximum_number_of_builds", '@representation' => 'standard', 'name' => 'maximum_number_of_builds', 'value' => 0 } + { '@type' => 'user_setting','@href' => "/v3/repo/#{repo.id}/setting/maximum_number_of_builds", '@representation' => 'standard', 'name' => 'maximum_number_of_builds', 'value' => 0 }, + { '@type' => 'user_setting','@href' => "/v3/repo/#{repo.id}/setting/auto_cancel_pushes", '@representation' => 'standard', 'name' => 'auto_cancel_pushes', 'value' => false }, + { '@type' => 'user_setting','@href' => "/v3/repo/#{repo.id}/setting/auto_cancel_pull_requests", '@representation' => 'standard', 'name' => 'auto_cancel_pull_requests', 'value' => false }, ] ) end @@ -60,7 +65,9 @@ { '@type' => 'user_setting','@href' => "/v3/repo/#{repo.id}/setting/builds_only_with_travis_yml", '@representation' => 'standard', 'name' => 'builds_only_with_travis_yml', 'value' => false }, { '@type' => 'user_setting','@href' => "/v3/repo/#{repo.id}/setting/build_pushes", '@representation' => 'standard', 'name' => 'build_pushes', 'value' => false }, { '@type' => 'user_setting','@href' => "/v3/repo/#{repo.id}/setting/build_pull_requests", '@representation' => 'standard', 'name' => 'build_pull_requests', 'value' => true }, - { '@type' => 'user_setting','@href' => "/v3/repo/#{repo.id}/setting/maximum_number_of_builds", '@representation' => 'standard', 'name' => 'maximum_number_of_builds', 'value' => 0 } + { '@type' => 'user_setting','@href' => "/v3/repo/#{repo.id}/setting/maximum_number_of_builds", '@representation' => 'standard', 'name' => 'maximum_number_of_builds', 'value' => 0 }, + { '@type' => 'user_setting','@href' => "/v3/repo/#{repo.id}/setting/auto_cancel_pushes", '@representation' => 'standard', 'name' => 'auto_cancel_pushes', 'value' => false }, + { '@type' => 'user_setting','@href' => "/v3/repo/#{repo.id}/setting/auto_cancel_pull_requests", '@representation' => 'standard', 'name' => 'auto_cancel_pull_requests', 'value' => false }, ] ) end diff --git a/travis-api.gemspec b/travis-api.gemspec index 2563014e83..36d0677225 100644 --- a/travis-api.gemspec +++ b/travis-api.gemspec @@ -15,18 +15,20 @@ Gem::Specification.new do |s| s.add_dependency 'composite_primary_keys', '~> 5.0' s.add_dependency 'sinatra', '~> 1.3' s.add_dependency 'sinatra-contrib', '~> 1.3' - s.add_dependency 'mustermann', '~> 0.4' - s.add_dependency 'redcarpet', '~> 2.1' + s.add_dependency 'mustermann', '~> 1.0.0.beta2' + s.add_dependency 'redcarpet', '>= 3.2.3' s.add_dependency 'rack-ssl', '~> 1.3', '>= 1.3.3' s.add_dependency 'rack-contrib', '~> 1.1' s.add_dependency 'memcachier' s.add_dependency 'useragent' s.add_dependency 'tool' + s.add_dependency 'google-api-client', '~> 0.9.4' + s.add_dependency 'fog-aws', '~> 0.12.0' + s.add_dependency 'fog-google', '~> 0.4.2' # from travis-core gemspec s.add_dependency 'activerecord', '~> 3.2.19' - s.add_dependency 'railties', '~> 3.2.19' s.add_dependency 'rollout', '~> 1.1.0' s.add_dependency 'coder', '~> 0.4.0' s.add_dependency 'virtus', '~> 1.0.0' @@ -35,5 +37,4 @@ Gem::Specification.new do |s| s.add_dependency 'simple_states', '~> 1.0.0' s.add_dependency 'pusher', '~> 0.14.0' s.add_dependency 'multi_json' - s.add_dependency 'google-api-client', '~> 0.9.4' end