diff --git a/Gemfile b/Gemfile index 7bea20ca..05cd9ed1 100644 --- a/Gemfile +++ b/Gemfile @@ -6,7 +6,10 @@ git_source(:github) do |repo_name| "https://github.com/#{repo_name}.git" end -gem 'rails', '~> 7.1.6' +gem "mutex_m", "~> 0.3.0" +gem "csv", "~> 3.3" + +gem 'rails', '~> 8.0.2' gem 'zeitwerk', '~> 2.6.18' # keep zeitwerk 2.6 until Ruby is 3.2 or higher gem 'pg', '~> 1.6.2' diff --git a/Gemfile.lock b/Gemfile.lock index 4e6d2065..8ceb6814 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -9,85 +9,77 @@ GEM remote: https://rubygems.org/ specs: 3scale-api (1.4.0) - actioncable (7.1.6) - actionpack (= 7.1.6) - activesupport (= 7.1.6) + actioncable (8.0.2.1) + actionpack (= 8.0.2.1) + activesupport (= 8.0.2.1) nio4r (~> 2.0) websocket-driver (>= 0.6.1) zeitwerk (~> 2.6) - actionmailbox (7.1.6) - actionpack (= 7.1.6) - activejob (= 7.1.6) - activerecord (= 7.1.6) - activestorage (= 7.1.6) - activesupport (= 7.1.6) - mail (>= 2.7.1) - net-imap - net-pop - net-smtp - actionmailer (7.1.6) - actionpack (= 7.1.6) - actionview (= 7.1.6) - activejob (= 7.1.6) - activesupport (= 7.1.6) - mail (~> 2.5, >= 2.5.4) - net-imap - net-pop - net-smtp + actionmailbox (8.0.2.1) + actionpack (= 8.0.2.1) + activejob (= 8.0.2.1) + activerecord (= 8.0.2.1) + activestorage (= 8.0.2.1) + activesupport (= 8.0.2.1) + mail (>= 2.8.0) + actionmailer (8.0.2.1) + actionpack (= 8.0.2.1) + actionview (= 8.0.2.1) + activejob (= 8.0.2.1) + activesupport (= 8.0.2.1) + mail (>= 2.8.0) rails-dom-testing (~> 2.2) - actionpack (7.1.6) - actionview (= 7.1.6) - activesupport (= 7.1.6) - cgi + actionpack (8.0.2.1) + actionview (= 8.0.2.1) + activesupport (= 8.0.2.1) nokogiri (>= 1.8.5) - racc rack (>= 2.2.4) rack-session (>= 1.0.1) rack-test (>= 0.6.3) rails-dom-testing (~> 2.2) rails-html-sanitizer (~> 1.6) - actiontext (7.1.6) - actionpack (= 7.1.6) - activerecord (= 7.1.6) - activestorage (= 7.1.6) - activesupport (= 7.1.6) + useragent (~> 0.16) + actiontext (8.0.2.1) + actionpack (= 8.0.2.1) + activerecord (= 8.0.2.1) + activestorage (= 8.0.2.1) + activesupport (= 8.0.2.1) globalid (>= 0.6.0) nokogiri (>= 1.8.5) - actionview (7.1.6) - activesupport (= 7.1.6) + actionview (8.0.2.1) + activesupport (= 8.0.2.1) builder (~> 3.1) - cgi erubi (~> 1.11) rails-dom-testing (~> 2.2) rails-html-sanitizer (~> 1.6) - activejob (7.1.6) - activesupport (= 7.1.6) + activejob (8.0.2.1) + activesupport (= 8.0.2.1) globalid (>= 0.3.6) - activemodel (7.1.6) - activesupport (= 7.1.6) - activerecord (7.1.6) - activemodel (= 7.1.6) - activesupport (= 7.1.6) + activemodel (8.0.2.1) + activesupport (= 8.0.2.1) + activerecord (8.0.2.1) + activemodel (= 8.0.2.1) + activesupport (= 8.0.2.1) timeout (>= 0.4.0) - activestorage (7.1.6) - actionpack (= 7.1.6) - activejob (= 7.1.6) - activerecord (= 7.1.6) - activesupport (= 7.1.6) + activestorage (8.0.2.1) + actionpack (= 8.0.2.1) + activejob (= 8.0.2.1) + activerecord (= 8.0.2.1) + activesupport (= 8.0.2.1) marcel (~> 1.0) - activesupport (7.1.6) + activesupport (8.0.2.1) base64 benchmark (>= 0.3) bigdecimal - concurrent-ruby (~> 1.0, >= 1.0.2) + concurrent-ruby (~> 1.0, >= 1.3.1) connection_pool (>= 2.2.5) drb i18n (>= 1.6, < 2) logger (>= 1.4.2) minitest (>= 5.1) - mutex_m securerandom (>= 0.3) - tzinfo (~> 2.0) + tzinfo (~> 2.0, >= 2.0.5) + uri (>= 0.13.1) addressable (2.8.0) public_suffix (>= 2.0.2, < 5.0) ansi (1.5.0) @@ -103,16 +95,16 @@ GEM concurrent-ruby (~> 1.0) builder (3.3.0) byebug (12.0.0) - cgi (0.5.0) codecov (0.4.3) simplecov (>= 0.15, < 0.22) coderay (1.1.3) concurrent-ruby (1.3.5) - connection_pool (2.5.4) + connection_pool (3.0.2) crack (0.4.5) rexml crass (1.0.6) - date (3.5.0) + csv (3.3.5) + date (3.5.1) debug_inspector (1.2.0) docile (1.3.5) drb (2.2.3) @@ -142,8 +134,7 @@ GEM dry-inflector (~> 1.0) dry-logic (~> 1.4) zeitwerk (~> 2.6) - erb (4.0.4) - cgi (>= 0.3.3) + erb (6.0.0) erubi (1.13.1) excon (0.112.0) faraday (1.3.0) @@ -266,7 +257,7 @@ GEM pry-stack_explorer (0.6.1) binding_of_caller (~> 1.0) pry (~> 0.13) - psych (5.2.6) + psych (5.3.0) date stringio public_suffix (4.0.6) @@ -287,20 +278,20 @@ GEM rackup (1.0.1) rack (< 3) webrick - rails (7.1.6) - actioncable (= 7.1.6) - actionmailbox (= 7.1.6) - actionmailer (= 7.1.6) - actionpack (= 7.1.6) - actiontext (= 7.1.6) - actionview (= 7.1.6) - activejob (= 7.1.6) - activemodel (= 7.1.6) - activerecord (= 7.1.6) - activestorage (= 7.1.6) - activesupport (= 7.1.6) + rails (8.0.2.1) + actioncable (= 8.0.2.1) + actionmailbox (= 8.0.2.1) + actionmailer (= 8.0.2.1) + actionpack (= 8.0.2.1) + actiontext (= 8.0.2.1) + actionview (= 8.0.2.1) + activejob (= 8.0.2.1) + activemodel (= 8.0.2.1) + activerecord (= 8.0.2.1) + activestorage (= 8.0.2.1) + activesupport (= 8.0.2.1) bundler (>= 1.15.0) - railties (= 7.1.6) + railties (= 8.0.2.1) rails-dom-testing (2.3.0) activesupport (>= 5.0.0) minitest @@ -308,19 +299,17 @@ GEM rails-html-sanitizer (1.6.2) loofah (~> 2.21) nokogiri (>= 1.15.7, != 1.16.7, != 1.16.6, != 1.16.5, != 1.16.4, != 1.16.3, != 1.16.2, != 1.16.1, != 1.16.0.rc1, != 1.16.0) - railties (7.1.6) - actionpack (= 7.1.6) - activesupport (= 7.1.6) - cgi - irb + railties (8.0.2.1) + actionpack (= 8.0.2.1) + activesupport (= 8.0.2.1) + irb (~> 1.13) rackup (>= 1.0.0) rake (>= 12.2) thor (~> 1.0, >= 1.2.2) - tsort (>= 0.2) zeitwerk (~> 2.6) rainbow (3.1.1) rake (13.3.1) - rdoc (6.15.1) + rdoc (6.17.0) erb psych (>= 4.0.0) tsort @@ -374,10 +363,10 @@ GEM rack (~> 2.2) rack-protection (= 2.2.3) tilt (~> 2.0) - stringio (3.1.8) + stringio (3.1.9) thor (1.4.0) tilt (2.0.11) - timeout (0.4.4) + timeout (0.5.0) tomlrb (2.0.3) tsort (0.2.0) tzinfo (2.0.6) @@ -385,6 +374,8 @@ GEM unicode-display_width (3.2.0) unicode-emoji (~> 4.1) unicode-emoji (4.1.0) + uri (1.1.1) + useragent (0.16.11) validate_url (1.0.15) activemodel (>= 3.0.0) public_suffix @@ -392,7 +383,7 @@ GEM addressable (>= 2.3.6) crack (>= 0.3.2) hashdiff (>= 0.4.0, < 2.0.0) - webrick (1.8.2) + webrick (1.9.2) websocket-driver (0.8.0) base64 websocket-extensions (>= 0.1.0) @@ -426,6 +417,7 @@ DEPENDENCIES bootsnap (>= 1.4.4) bugsnag codecov + csv (~> 3.3) httpclient! k8s-ruby license_finder (~> 7.0.1) @@ -433,6 +425,7 @@ DEPENDENCIES message_bus minitest-reporters minitest-stub-const + mutex_m (~> 0.3.0) oauth2 pg (~> 1.6.2) prometheus-client (~> 2.1.0) @@ -443,7 +436,7 @@ DEPENDENCIES puma (~> 5.2) que (~> 2.4.1) que-web - rails (~> 7.1.6) + rails (~> 8.0.2) responders (~> 3.0.1) rubocop rubocop-performance diff --git a/app/models/integration.rb b/app/models/integration.rb index c95a87bc..8082533b 100644 --- a/app/models/integration.rb +++ b/app/models/integration.rb @@ -4,7 +4,7 @@ class Integration < ApplicationRecord belongs_to :model has_many :integration_states, dependent: :destroy - enum state: %i[active disabled].index_with{ |status| status.to_s } + enum :state, %i[active disabled].index_with{ |status| status.to_s } def self.tenant_or_model(tenant, model) by_tenant = where(tenant: tenant, model_id: nil) diff --git a/bin/dev b/bin/dev new file mode 100755 index 00000000..5f91c205 --- /dev/null +++ b/bin/dev @@ -0,0 +1,2 @@ +#!/usr/bin/env ruby +exec "./bin/rails", "server", *ARGV diff --git a/bin/rubocop b/bin/rubocop new file mode 100755 index 00000000..40330c0f --- /dev/null +++ b/bin/rubocop @@ -0,0 +1,8 @@ +#!/usr/bin/env ruby +require "rubygems" +require "bundler/setup" + +# explicit rubocop config increases performance slightly while avoiding config confusion. +ARGV.unshift("--config", File.expand_path("../.rubocop.yml", __dir__)) + +load Gem.bin_path("rubocop", "rubocop") diff --git a/bin/setup b/bin/setup index 3cd5a9d7..be3db3c0 100755 --- a/bin/setup +++ b/bin/setup @@ -1,7 +1,6 @@ #!/usr/bin/env ruby require "fileutils" -# path to your application root. APP_ROOT = File.expand_path("..", __dir__) def system!(*args) @@ -14,7 +13,6 @@ FileUtils.chdir APP_ROOT do # Add necessary setup steps to this file. puts "== Installing dependencies ==" - system! "gem install bundler --conservative" system("bundle check") || system!("bundle install") # puts "\n== Copying sample files ==" @@ -28,6 +26,9 @@ FileUtils.chdir APP_ROOT do puts "\n== Removing old logs and tempfiles ==" system! "bin/rails log:clear tmp:clear" - puts "\n== Restarting application server ==" - system! "bin/rails restart" + unless ARGV.include?("--skip-server") + puts "\n== Starting development server ==" + STDOUT.flush # flush the output before exec(2) so that it displays + exec "bin/dev" + end end diff --git a/config/application.rb b/config/application.rb index 47d46216..ce383241 100644 --- a/config/application.rb +++ b/config/application.rb @@ -26,7 +26,7 @@ class Application < Rails::Application # Please, add to the `ignore` list any other `lib` subdirectories that do # not contain `.rb` files, or that should not be reloaded or eager loaded. # Common ones are `templates`, `generators`, or `middleware`, for example. - # config.autoload_lib(ignore: %w(tasks puma generators prometheus que)) + # config.autoload_lib(ignore: %w(assets tasks puma generators prometheus que)) # Que needs :sql because of advanced PostgreSQL features config.active_record.schema_format = :sql diff --git a/config/environments/development.rb b/config/environments/development.rb index 98b3b542..c22359d2 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -7,9 +7,7 @@ ActionDispatch::DebugLocks # Settings specified here will take precedence over those in config/application.rb. - # In the development environment your application's code is reloaded any time - # it changes. This slows down response time but is perfect for development - # since you don't have to restart the web server when you make code changes. + # Make code changes take effect immediately without server restart. config.enable_reloading = true # Do not eager load code on boot. @@ -18,11 +16,11 @@ # Show full error reports. config.consider_all_requests_local = true - # Enable server timing + # Enable server timing. config.server_timing = true - # Enable/disable caching. By default caching is disabled. - # Run rails dev:cache to toggle caching. + # Enable/disable Action Controller caching. By default Action Controller caching is disabled. + # Run rails dev:cache to toggle Action Controller caching. if Rails.root.join("tmp/caching-dev.txt").exist? config.cache_store = :memory_store config.public_file_server.headers = { @@ -49,16 +47,21 @@ # Highlight code that triggered database queries in logs. config.active_record.verbose_query_logs = true + # Append comments with runtime information tags to SQL queries in logs. + config.active_record.query_log_tags_enabled = true + # Highlight code that enqueued background job in logs. config.active_job.verbose_enqueue_logs = true - # Raises error for missing translations. # config.i18n.raise_on_missing_translations = true # Annotate rendered view with file names. # config.action_view.annotate_rendered_view_with_filenames = true - # Raise error when a before_action's only/except options reference missing actions + # Raise error when a before_action's only/except options reference missing actions. config.action_controller.raise_on_missing_callback_actions = true + + # Apply autocorrection by RuboCop to files generated by `bin/rails generate`. + # config.generators.apply_rubocop_autocorrect_after_generate! end diff --git a/config/environments/production.rb b/config/environments/production.rb index 0f66f30a..2d5bfafb 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -6,18 +6,14 @@ # Code is not reloaded between requests. config.enable_reloading = false - # Eager load code on boot. This eager loads most of Rails and - # your application in memory, allowing both threaded web servers - # and those relying on copy on write to perform better. - # Rake tasks automatically ignore this option for performance. + # Eager load code on boot for better performance and memory savings (ignored by Rake tasks). config.eager_load = true - # Full error reports are disabled and caching is turned on. + # Full error reports are disabled. config.consider_all_requests_local = false - # Ensures that a master key has been made available in ENV["RAILS_MASTER_KEY"], config/master.key, or an environment - # key such as config/credentials/production.key. This key is used to decrypt credentials (and other encrypted files). - # config.require_master_key = true + # Cache assets for far-future expiry since they are all digest stamped. + # config.public_file_server.headers = { "cache-control" => "public, max-age=#{1.year.to_i}" } # Disable serving static files from `public/`, relying on NGINX/Apache to do so instead. config.public_file_server.enabled = ENV["RAILS_SERVE_STATIC_FILES"].present? @@ -25,10 +21,6 @@ # Enable serving of images, stylesheets, and JavaScripts from an asset server. # config.asset_host = "http://assets.example.com" - # Specifies the header that your server uses for sending files. - # config.action_dispatch.x_sendfile_header = "X-Sendfile" # for Apache - # config.action_dispatch.x_sendfile_header = "X-Accel-Redirect" # for NGINX - # Assume all access to the app is happening through a SSL-terminating reverse proxy. # Can be used together with config.force_ssl for Strict-Transport-Security and secure cookies. # config.assume_ssl = true @@ -36,54 +28,52 @@ # Force all access to the app over SSL, use Strict-Transport-Security, and use secure cookies. # config.force_ssl = true - # Log to STDOUT by default - # config.logger = ActiveSupport::Logger.new(STDOUT) - # .tap { |logger| logger.formatter = ::Logger::Formatter.new } - # .then { |logger| ActiveSupport::TaggedLogging.new(logger) } + # Skip http-to-https redirect for the default health check endpoint. + # config.ssl_options = { redirect: { exclude: ->(request) { request.path == "/up" } } } - # Prepend all log lines with the following tags. - config.log_tags = [ ] + # Log to STDOUT with the current request id as a default log tag. + config.log_tags = [] - # "info" includes generic and useful information about system operation, but avoids logging too much - # information to avoid inadvertent exposure of personally identifiable information (PII). If you - # want to log everything, set the level to "debug". + # Change to "debug" to log everything (including potentially personally-identifiable information!) config.log_level = ENV.fetch("RAILS_LOG_LEVEL", "info") # Use default logging formatter so that PID and timestamp are not suppressed. config.log_formatter = ::Logger::Formatter.new - # Use a different logger for distributed setups. - # require "syslog/logger" - # config.logger = ActiveSupport::TaggedLogging.new(Syslog::Logger.new "app-name") - if ENV["RAILS_LOG_TO_STDOUT"].present? logger = ActiveSupport::Logger.new(STDOUT) logger.formatter = config.log_formatter config.logger = ActiveSupport::TaggedLogging.new(logger) end - # Use a different cache store in production. + # Prevent health checks from clogging up the logs. + config.silence_healthcheck_path = "/up" + + # Don't log any deprecations. + config.active_support.report_deprecations = false + + # Replace the default in-process memory cache store with a durable alternative. # config.cache_store = :mem_cache_store - # Use a real queuing backend for Active Job (and separate queues per environment). + # Replace the default in-process and non-durable queuing backend for Active Job. # config.active_job.queue_adapter = :resque - # config.active_job.queue_name_prefix = "zync_production" # Enable locale fallbacks for I18n (makes lookups for any locale fall back to # the I18n.default_locale when a translation cannot be found). config.i18n.fallbacks = true - # Don't log any deprecations. - config.active_support.report_deprecations = false - # Do not dump schema after migrations. config.active_record.dump_schema_after_migration = false + # Only use :id for inspections in production. + config.active_record.attributes_for_inspect = [ :id ] + # Enable DNS rebinding protection and other `Host` header attacks. # config.hosts = [ # "example.com", # Allow requests from example.com # /.*\.example\.com/ # Allow requests from subdomains like `www.example.com` # ] + # # Skip DNS rebinding protection for the default health check endpoint. # config.host_authorization = { exclude: ->(request) { request.path == "/up" } } end diff --git a/config/environments/test.rb b/config/environments/test.rb index d349c355..99fc3af3 100644 --- a/config/environments/test.rb +++ b/config/environments/test.rb @@ -1,5 +1,3 @@ -require "active_support/core_ext/integer/time" - # The test environment is used exclusively to run your application's # test suite. You never need to work with it otherwise. Remember that # your test database is "scratch space" for the test suite and is wiped @@ -49,6 +47,8 @@ # Annotate rendered view with file names. # config.action_view.annotate_rendered_view_with_filenames = true - # Raise error when a before_action's only/except options reference missing actions + # Raise error when a before_action's only/except options reference missing actions. config.action_controller.raise_on_missing_callback_actions = true + + config.active_job.queue_adapter = :test end diff --git a/config/initializers/filter_parameter_logging.rb b/config/initializers/filter_parameter_logging.rb index c72afd9d..cb1afaa6 100644 --- a/config/initializers/filter_parameter_logging.rb +++ b/config/initializers/filter_parameter_logging.rb @@ -4,5 +4,5 @@ # Use this to limit dissemination of sensitive information. # See the ActiveSupport::ParameterFilter documentation for supported notations and behaviors. Rails.application.config.filter_parameters += [ - :password, :access_token + :password, :access_token, :passw, :email, :secret, :token, :_key, :crypt, :salt, :certificate, :otp, :ssn, :cvv, :cvc ] diff --git a/config/initializers/new_framework_defaults_7_2.rb b/config/initializers/new_framework_defaults_7_2.rb new file mode 100644 index 00000000..9b322a33 --- /dev/null +++ b/config/initializers/new_framework_defaults_7_2.rb @@ -0,0 +1,70 @@ +# Be sure to restart your server when you modify this file. +# +# This file eases your Rails 7.2 framework defaults upgrade. +# +# Uncomment each configuration one by one to switch to the new default. +# Once your application is ready to run with all new defaults, you can remove +# this file and set the `config.load_defaults` to `7.2`. +# +# Read the Guide for Upgrading Ruby on Rails for more info on each option. +# https://guides.rubyonrails.org/upgrading_ruby_on_rails.html + +### +# Controls whether Active Job's `#perform_later` and similar methods automatically defer +# the job queuing to after the current Active Record transaction is committed. +# +# Example: +# Topic.transaction do +# topic = Topic.create(...) +# NewTopicNotificationJob.perform_later(topic) +# end +# +# In this example, if the configuration is set to `:never`, the job will +# be enqueued immediately, even though the `Topic` hasn't been committed yet. +# Because of this, if the job is picked up almost immediately, or if the +# transaction doesn't succeed for some reason, the job will fail to find this +# topic in the database. +# +# If `enqueue_after_transaction_commit` is set to `:default`, the queue adapter +# will define the behaviour. +# +# Note: Active Job backends can disable this feature. This is generally done by +# backends that use the same database as Active Record as a queue, hence they +# don't need this feature. +#++ +# Rails.application.config.active_job.enqueue_after_transaction_commit = :default + +### +# Adds image/webp to the list of content types Active Storage considers as an image +# Prevents automatic conversion to a fallback PNG, and assumes clients support WebP, as they support gif, jpeg, and png. +# This is possible due to broad browser support for WebP, but older browsers and email clients may still not support +# WebP. Requires imagemagick/libvips built with WebP support. +#++ +# Rails.application.config.active_storage.web_image_content_types = %w[image/png image/jpeg image/gif image/webp] + +### +# Enable validation of migration timestamps. When set, an ActiveRecord::InvalidMigrationTimestampError +# will be raised if the timestamp prefix for a migration is more than a day ahead of the timestamp +# associated with the current time. This is done to prevent forward-dating of migration files, which can +# impact migration generation and other migration commands. +# +# Applications with existing timestamped migrations that do not adhere to the +# expected format can disable validation by setting this config to `false`. +#++ +Rails.application.config.active_record.validate_migration_timestamps = true + +### +# Controls whether the PostgresqlAdapter should decode dates automatically with manual queries. +# +# Example: +# ActiveRecord::ConnectionAdapters::PostgreSQLAdapter.select_value("select '2024-01-01'::date") #=> Date +# +# This query used to return a `String`. +#++ +Rails.application.config.active_record.postgresql_adapter_decode_dates = true + +### +# Enables YJIT as of Ruby 3.3, to bring sizeable performance improvements. If you are +# deploying to a memory constrained environment you may want to set this to `false`. +#++ +Rails.application.config.yjit = true diff --git a/config/initializers/new_framework_defaults_8_0.rb b/config/initializers/new_framework_defaults_8_0.rb new file mode 100644 index 00000000..04ab90f8 --- /dev/null +++ b/config/initializers/new_framework_defaults_8_0.rb @@ -0,0 +1,30 @@ +# Be sure to restart your server when you modify this file. +# +# This file eases your Rails 8.0 framework defaults upgrade. +# +# Uncomment each configuration one by one to switch to the new default. +# Once your application is ready to run with all new defaults, you can remove +# this file and set the `config.load_defaults` to `8.0`. +# +# Read the Guide for Upgrading Ruby on Rails for more info on each option. +# https://guides.rubyonrails.org/upgrading_ruby_on_rails.html + +### +# Specifies whether `to_time` methods preserve the UTC offset of their receivers or preserves the timezone. +# If set to `:zone`, `to_time` methods will use the timezone of their receivers. +# If set to `:offset`, `to_time` methods will use the UTC offset. +# If `false`, `to_time` methods will convert to the local system UTC offset instead. +#++ +Rails.application.config.active_support.to_time_preserves_timezone = :zone + +### +# When both `If-Modified-Since` and `If-None-Match` are provided by the client +# only consider `If-None-Match` as specified by RFC 7232 Section 6. +# If set to `false` both conditions need to be satisfied. +#++ +Rails.application.config.action_dispatch.strict_freshness = true + +### +# Set `Regexp.timeout` to `1`s by default to improve security over Regexp Denial-of-Service attacks. +#++ +Regexp.timeout = 1 diff --git a/config/puma.rb b/config/puma.rb index 0c0ba2d2..b4ce3bff 100644 --- a/config/puma.rb +++ b/config/puma.rb @@ -1,9 +1,29 @@ -# Puma can serve each request in a thread from an internal thread pool. -# The `threads` method setting takes two numbers: a minimum and maximum. -# Any libraries that use thread pools should be configured to match -# the maximum value specified for Puma. Default is set to 5 threads for minimum -# and maximum; this matches the default thread size of Active Record. +# This configuration file will be evaluated by Puma. The top-level methods that +# are invoked here are part of Puma's configuration DSL. For more information +# about methods provided by the DSL, see https://puma.io/puma/Puma/DSL.html. # +# Puma starts a configurable number of processes (workers) and each process +# serves each request in a thread from an internal thread pool. +# +# You can control the number of workers using ENV["WEB_CONCURRENCY"]. You +# should only set this value when you want to run 2 or more workers. The +# default is already 1. +# +# The ideal number of threads per worker depends both on how much time the +# application spends waiting for IO operations and on how much you wish to +# prioritize throughput over latency. +# +# As a rule of thumb, increasing the number of threads will increase how much +# traffic a given process can handle (throughput), but due to CRuby's +# Global VM Lock (GVL) it has diminishing returns and will degrade the +# response time (latency) of the application. +# +# The default is set to 3 threads as it's deemed a decent compromise between +# throughput and latency for the average Rails application. +# +# Any libraries that use a connection pool or another resource pool should +# be configured to provide at least as many connections as the number of +# threads. This includes Active Record's `pool` parameter in `database.yml`. max_threads_count = ENV.fetch("RAILS_MAX_THREADS") { 5 } min_threads_count = ENV.fetch("RAILS_MIN_THREADS") { max_threads_count } threads min_threads_count, max_threads_count @@ -14,30 +34,16 @@ worker_timeout 3600 if ENV.fetch("RAILS_ENV", "development") == "development" # Specifies the `port` that Puma will listen on to receive requests; default is 3000. -# -port ENV.fetch("PORT") { 3000 } +port ENV.fetch("PORT", 3000) # Specifies the `environment` that Puma will run in. # environment ENV.fetch("RAILS_ENV") { "development" } -# Specifies the `pidfile` that Puma will use. +# Specify the PID file. Defaults to tmp/pids/server.pid in development. +# In other environments, only set the PID file if requested. pidfile ENV.fetch("PIDFILE") { "tmp/pids/server.pid" } -# Specifies the number of `workers` to boot in clustered mode. -# Workers are forked web server processes. If using threads and workers together -# the concurrency of the application would be max `threads` * `workers`. -# Workers do not work on JRuby or Windows (both of which do not support -# processes). -# -# workers ENV.fetch("WEB_CONCURRENCY") { 2 } - -# Use the `preload_app!` method when specifying a `workers` number. -# This directive tells Puma to first boot the application and load code -# before forking the application. This takes advantage of Copy On Write -# process behavior so workers use less memory. -# -# preload_app! queue_requests false # let the higher layer figure that out diff --git a/lib/prometheus/que_stats.rb b/lib/prometheus/que_stats.rb index b61f91cf..ce707fdd 100644 --- a/lib/prometheus/que_stats.rb +++ b/lib/prometheus/que_stats.rb @@ -69,7 +69,7 @@ def call(*filters) with(common_tables) execute do |connection| - connection.select_all(relation.to_sql) + connection.select_all(relation) end end diff --git a/test/lib/prometheus/que_stats_test.rb b/test/lib/prometheus/que_stats_test.rb index 0bdef172..90bb24a2 100644 --- a/test/lib/prometheus/que_stats_test.rb +++ b/test/lib/prometheus/que_stats_test.rb @@ -12,73 +12,89 @@ class Prometheus::QueStatsTest < ActiveSupport::TestCase Prometheus::QueStats.read_only_transaction = @_readonly_transaction end + def with_que_adapter(&block) + ApplicationJob.stub(:queue_adapter, ActiveJob::QueueAdapters::QueAdapter.new, &block) + end + test 'worker stats' do assert Prometheus::QueStats::WorkerStats.new.call end test 'job stats' do - Que.stop! - ApplicationJob.perform_later - assert_equal 1, stats_count - assert_equal 1, stats_count(where: ['1 > 0']) - assert_equal 1, stats_count(where: ['1 > 0', '2 > 1']) - assert_equal 0, stats_count(where: ['1 > 0', '2 < 1']) + with_que_adapter do + Que.stop! + ApplicationJob.perform_later + assert_equal 1, stats_count + assert_equal 1, stats_count(where: ['1 > 0']) + assert_equal 1, stats_count(where: ['1 > 0', '2 > 1']) + assert_equal 0, stats_count(where: ['1 > 0', '2 < 1']) + end end test 'ready jobs stats' do - Que.stop! - assert_equal 0, stats_count(type: :ready) - jobs = Array.new(3) { ApplicationJob.perform_later } - jobs << ApplicationJob.set(wait_until: 1.day.from_now).perform_later - assert_equal 3, stats_count(type: :ready) - update_job(jobs[0], error_count: 1) - assert_equal 2, stats_count(type: :ready) - update_job(jobs[1], expired_at: 1.minute.ago) - assert_equal 1, stats_count(type: :ready) - update_job(jobs[2], finished_at: 1.minute.ago) - assert_equal 0, stats_count(type: :ready) + with_que_adapter do + Que.stop! + assert_equal 0, stats_count(type: :ready) + jobs = Array.new(3) { ApplicationJob.perform_later } + jobs << ApplicationJob.set(wait_until: 1.day.from_now).perform_later + assert_equal 3, stats_count(type: :ready) + update_job(jobs[0], error_count: 1) + assert_equal 2, stats_count(type: :ready) + update_job(jobs[1], expired_at: 1.minute.ago) + assert_equal 1, stats_count(type: :ready) + update_job(jobs[2], finished_at: 1.minute.ago) + assert_equal 0, stats_count(type: :ready) + end end test 'scheduled jobs stats' do - Que.stop! - assert_equal 0, stats_count(type: :scheduled) - jobs = [ApplicationJob, ApplicationJob.set(wait_until: 1.day.from_now), ApplicationJob.set(wait_until: 2.days.from_now)].map(&:perform_later) - assert_equal 2, stats_count(type: :scheduled) - update_job(jobs[1], error_count: 16, expired_at: 1.minute.ago) - assert_equal 1, stats_count(type: :scheduled) - update_job(jobs.last, run_at: 1.minute.ago) - assert_equal 0, stats_count(type: :scheduled) + with_que_adapter do + Que.stop! + assert_equal 0, stats_count(type: :scheduled) + jobs = [ApplicationJob, ApplicationJob.set(wait_until: 1.day.from_now), ApplicationJob.set(wait_until: 2.days.from_now)].map(&:perform_later) + assert_equal 2, stats_count(type: :scheduled) + update_job(jobs[1], error_count: 16, expired_at: 1.minute.ago) + assert_equal 1, stats_count(type: :scheduled) + update_job(jobs.last, run_at: 1.minute.ago) + assert_equal 0, stats_count(type: :scheduled) + end end test 'finished jobs stats' do - Que.stop! - assert_equal 0, stats_count(type: :finished) - jobs = Array.new(2) { ApplicationJob.perform_later } - assert_equal 0, stats_count(type: :finished) - update_job(jobs.first, finished_at: Time.now) - assert_equal 1, stats_count(type: :finished) + with_que_adapter do + Que.stop! + assert_equal 0, stats_count(type: :finished) + jobs = Array.new(2) { ApplicationJob.perform_later } + assert_equal 0, stats_count(type: :finished) + update_job(jobs.first, finished_at: Time.now) + assert_equal 1, stats_count(type: :finished) + end end test 'failed jobs stats' do - Que.stop! - assert_equal 0, stats_count(type: :failed) - jobs = Array.new(2) { ApplicationJob.perform_later } - assert_equal 0, stats_count(type: :failed) - update_job(jobs.first, error_count: 1) - assert_equal 1, stats_count(type: :failed) - update_job(jobs.first, error_count: 15) - assert_equal 1, stats_count(type: :failed) - update_job(jobs.first, error_count: 16, expired_at: Time.now.utc) - assert_equal 0, stats_count(type: :failed) + with_que_adapter do + Que.stop! + assert_equal 0, stats_count(type: :failed) + jobs = Array.new(2) { ApplicationJob.perform_later } + assert_equal 0, stats_count(type: :failed) + update_job(jobs.first, error_count: 1) + assert_equal 1, stats_count(type: :failed) + update_job(jobs.first, error_count: 15) + assert_equal 1, stats_count(type: :failed) + update_job(jobs.first, error_count: 16, expired_at: Time.now.utc) + assert_equal 0, stats_count(type: :failed) + end end test 'expired jobs stats' do - Que.stop! - assert_equal 0, stats_count(type: :expired) - jobs = Array.new(2) { ApplicationJob.perform_later } - assert_equal 0, stats_count(type: :expired) - update_job(jobs.first, error_count: 16, expired_at: Time.now.utc) - assert_equal 1, stats_count(type: :expired) + with_que_adapter do + Que.stop! + assert_equal 0, stats_count(type: :expired) + jobs = Array.new(2) { ApplicationJob.perform_later } + assert_equal 0, stats_count(type: :expired) + update_job(jobs.first, error_count: 16, expired_at: Time.now.utc) + assert_equal 1, stats_count(type: :expired) + end end class WithTransaction < ActiveSupport::TestCase @@ -91,20 +107,22 @@ def test_readonly_transaction end test 'serialize metrics' do - Que.stop! + with_que_adapter do + Que.stop! - job = ApplicationJob.new - job.enqueue + job = ApplicationJob.new + job.enqueue - job.scheduled_at = 1.day.ago - job.enqueue + job.scheduled_at = 1.day.ago + job.enqueue - job.executions = 1 - job.enqueue + job.executions = 1 + job.enqueue - Yabeda.collectors.each(&:call) + Yabeda.collectors.each(&:call) - assert Prometheus::Client::Formats::Text.marshal(Yabeda::Prometheus.registry) + assert Prometheus::Client::Formats::Text.marshal(Yabeda::Prometheus.registry) + end end protected diff --git a/test/models/model_test.rb b/test/models/model_test.rb index f1625bc0..2429aa62 100644 --- a/test/models/model_test.rb +++ b/test/models/model_test.rb @@ -2,26 +2,40 @@ require 'test_helper' class ModelTest < ActiveSupport::TestCase + self.use_transactional_tests = false + def test_weak_lock - locking_connection = Model.connection_pool.checkout + model = Model.first! + + + # The events in the two threads must happen in a particular order: + # 1. The new thread must lock the record and wait. This way the lock + # and the transaction are kept open. + # 2. The main thread must wait until the lock is taken, only then it can + # try to take the lock + # We use queues to signal threads + locked_signal = Queue.new + release_signal = Queue.new - fiber = Model.stub(:connection, locking_connection) do - Fiber.new do - locking_connection.transaction do - Fiber.yield Model.first!.weak_lock - end + # Take the lock and keep it taken + thread = Thread.new do + Model.transaction do + model.weak_lock + locked_signal.push(true) + release_signal.pop end end - locked_model = fiber.resume - refute_equal Model.connection, locking_connection + # Wait till the lock is taken + locked_signal.pop - connection = Model.connection_pool.checkout - - Model.stub(:connection, connection) do - assert_raises Model::LockTimeoutError do - Model.find(locked_model.id).weak_lock - end + # Try to take the lock, we expect it to fail + assert_raises Model::LockTimeoutError do + model.weak_lock end + + # Release the thread + release_signal.push(true) + thread.join end end