From 2fa7947f765bea0b31b50eb166aeadf76f245519 Mon Sep 17 00:00:00 2001 From: Bernard Lambeau Date: Mon, 29 Sep 2025 18:29:31 +0200 Subject: [PATCH] Add an inherits argument to db:flush, and skip seed files that are unchanged. --- CHANGELOG.md | 7 ++++- Gemfile.lock | 47 ++++++++++++++++++++----------- README.md | 34 +++++++++++----------- dbagent.gemspec | 2 +- lib/db_agent/seeder/actions.rb | 2 +- lib/db_agent/seeder/relational.rb | 37 +++++++++++++++--------- tasks/db.rake | 5 ++-- 7 files changed, 82 insertions(+), 52 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a3a8638..a0d559e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,9 @@ -## 3.9.0 +## 3.9.1 + +* `rake db:flush` now supports a second argument, which is the seed to inherit + from. Tables whose data is unchanged are no longer flushed on disk. + +## 3.9.0 - 2025-09-29 * Add support for multiple databases, via a DBAGENT_DATABASES environment variables diff --git a/Gemfile.lock b/Gemfile.lock index e5fee84..4cfa8f3 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -3,7 +3,7 @@ PATH specs: dbagent (3.9.0) base64 (~> 0.3) - bmg (~> 0.21) + bmg (~> 0.23.5) net-ping (~> 2) path (~> 2) pg (~> 1) @@ -17,43 +17,50 @@ GEM remote: http://rubygems.org/ specs: base64 (0.3.0) - bigdecimal (3.1.6) - bmg (0.21.3) + bigdecimal (3.2.3) + bmg (0.23.5) path (>= 2.0) predicate (>= 2.7.1, < 3.0) daemons (1.4.1) - diff-lcs (1.5.1) + diff-lcs (1.6.2) eventmachine (1.2.7) - minitest (5.22.2) + minitest (5.25.5) mustermann (2.0.2) ruby2_keywords (~> 0.0.1) net-ping (2.0.8) path (2.1.0) - pg (1.5.5) - predicate (2.8.0) + pg (1.6.2) + pg (1.6.2-aarch64-linux) + pg (1.6.2-aarch64-linux-musl) + pg (1.6.2-aarch64-mingw-ucrt) + pg (1.6.2-arm64-darwin) + pg (1.6.2-x86_64-darwin) + pg (1.6.2-x86_64-linux) + pg (1.6.2-x86_64-linux-musl) + predicate (2.9.0) minitest (>= 5.0) sexpr (~> 1.1) - rack (2.2.8.1) + rack (2.2.18) rack-protection (2.2.4) rack rack-test (1.1.0) rack (>= 1.0, < 3) - rake (13.1.0) - rspec (3.13.0) + rake (13.3.0) + rspec (3.13.1) rspec-core (~> 3.13.0) rspec-expectations (~> 3.13.0) rspec-mocks (~> 3.13.0) - rspec-core (3.13.0) + rspec-core (3.13.5) rspec-support (~> 3.13.0) - rspec-expectations (3.13.0) + rspec-expectations (3.13.5) diff-lcs (>= 1.2.0, < 2.0) rspec-support (~> 3.13.0) - rspec-mocks (3.13.0) + rspec-mocks (3.13.5) diff-lcs (>= 1.2.0, < 2.0) rspec-support (~> 3.13.0) - rspec-support (3.13.0) + rspec-support (3.13.6) ruby2_keywords (0.0.5) - sequel (5.77.0) + sequel (5.96.0) bigdecimal sexpr (1.1.0) sinatra (2.2.4) @@ -65,11 +72,17 @@ GEM daemons (~> 1.0, >= 1.0.9) eventmachine (~> 1.0, >= 1.0.4) rack (>= 1, < 3) - tilt (2.3.0) + tilt (2.6.1) PLATFORMS + aarch64-linux + aarch64-linux-musl + aarch64-mingw-ucrt + arm64-darwin ruby + x86_64-darwin x86_64-linux + x86_64-linux-musl DEPENDENCIES bundler (~> 2) @@ -78,4 +91,4 @@ DEPENDENCIES rspec (~> 3.6) BUNDLED WITH - 2.4.6 + 2.6.2 diff --git a/README.md b/README.md index 31fd0ff..56be367 100644 --- a/README.md +++ b/README.md @@ -42,23 +42,23 @@ See the examples folder for details. The following rake tasks helps you managing the database. They must typically be executed on the docker container. ``` -rake db:check-seeds # Checks that all seeds can be installed correctly -rake db:create # Creates an fresh new user & database (USE WITH CARE) -rake db:drop # Drops the user & database (USE WITH CARE) -rake db:flush[to] # Flushes the database as a particular data set -rake db:migrate # Runs migrations on the current database -rake db:ping # Pings the database, making sure everything's ready for migration -rake db:rebuild # Rebuilds the database from scratch (USE WITH CARE) -rake db:repl # Opens a database REPL -rake db:seed[from] # Seeds the database with a particular data set -rake db:spy # Dumps the schema documentation into database/schema -rake db:backup # Makes a database backup to the backups folder -rake db:restore[match] # Restore the last matching database backup file from backups folder -rake db:revive # Shortcut for both db:restore and db:migrate -rake db:wait_server # Waits until the postgresql host seems available -rake db:wait # Waits until the postgresql database seems available -rake db:tables # List tables with those with fewer dependencies first -rake db:dependencies[of] # List tables that depend of a given one +rake db:check-seeds # Checks that all seeds can be installed correctly +rake db:create # Creates an fresh new user & database (USE WITH CARE) +rake db:drop # Drops the user & database (USE WITH CARE) +rake db:flush[to,inherits?] # Flushes the database as a particular data set +rake db:migrate # Runs migrations on the current database +rake db:ping # Pings the database, making sure everything's ready for migration +rake db:rebuild # Rebuilds the database from scratch (USE WITH CARE) +rake db:repl # Opens a database REPL +rake db:seed[from] # Seeds the database with a particular data set +rake db:spy # Dumps the schema documentation into database/schema +rake db:backup # Makes a database backup to the backups folder +rake db:restore[match] # Restore the last matching database backup file from backups folder +rake db:revive # Shortcut for both db:restore and db:migrate +rake db:wait_server # Waits until the postgresql host seems available +rake db:wait # Waits until the postgresql database seems available +rake db:tables # List tables with those with fewer dependencies first +rake db:dependencies[of] # List tables that depend of a given one ``` ## Available webservices diff --git a/dbagent.gemspec b/dbagent.gemspec index dd38f55..b0460cf 100644 --- a/dbagent.gemspec +++ b/dbagent.gemspec @@ -19,7 +19,7 @@ Gem::Specification.new do |s| s.add_dependency 'rake', '~> 13' s.add_dependency 'path', '~> 2' s.add_dependency 'sinatra', '~> 2' - s.add_dependency 'bmg', '~> 0.21' + s.add_dependency 'bmg', '~> 0.23.5' s.add_dependency 'net-ping', "~> 2" s.add_dependency "predicate", "~> 2", ">= 2.7.1" s.add_dependency 'thin', '~> 1' diff --git a/lib/db_agent/seeder/actions.rb b/lib/db_agent/seeder/actions.rb index 5dbd20a..c1adae1 100644 --- a/lib/db_agent/seeder/actions.rb +++ b/lib/db_agent/seeder/actions.rb @@ -11,7 +11,7 @@ def insert_script(from) def flush_empty(to = "empty") end - def flush(to) + def flush(to, inherits = "empty") end def check_seeds diff --git a/lib/db_agent/seeder/relational.rb b/lib/db_agent/seeder/relational.rb index f58a7cf..a7bf989 100644 --- a/lib/db_agent/seeder/relational.rb +++ b/lib/db_agent/seeder/relational.rb @@ -59,19 +59,20 @@ def flush_empty(to = "empty") end end - def flush(to) - target = data_folder.seed_folder(to).path.rm_rf.mkdir_p - source = data_folder.seed_folder('empty') - seed_files = source.seed_files_per_table - - (target/"metadata.json").write <<-JSON.strip - { "inherits": "empty" } + def flush(to, inherits = "empty") + target_path = data_folder.seed_folder(to).path.rm_rf.mkdir_p + inherit_seed = data_folder.seed_folder(inherits) + inherit_path = inherit_seed.path + seed_files = inherit_seed.seed_files_per_table + + (target_path/"metadata.json").write <<-JSON.strip + { "inherits": "#{inherits}" } JSON - seed_files.each_pair do |table_name, source_file| - target_file = target/source_file.basename.to_s + seed_files.each_pair do |table_name, inherit_file| + target_file = target_path/inherit_file.basename.to_s table = file2table(target_file) - flush_table(table, target_file, true) + flush_table(table, target_file, inherit_file) end end @@ -90,11 +91,13 @@ def check_seeds private - def flush_table(table, target_file, skip_empty) + def flush_table(table, target_file, inherit_file) data = viewpoint.send(table.gsub(/\./, '__').to_sym).to_a table_name = qualify_table(table) - if data.empty? && skip_empty - LOGGER.info("Skipping table `#{table_name}` since empty") + if data.empty? + LOGGER.info("Skipping table `#{table_name}`: empty") + elsif same_data?(inherit_file, data) + LOGGER.info("Skipping table `#{table_name}`: same as inherited one") else LOGGER.info("Flushing table `#{table_name}`") json = JSON.pretty_generate(data) @@ -102,6 +105,14 @@ def flush_table(table, target_file, skip_empty) end end + def same_data?(inherit_file, data) + return false unless inherit_file.file? + + r = Bmg.json(inherit_file) + s = Bmg.in_memory(data) + r.to_set == s.to_set + end + def before_seeding!(seed_folder) seed_folder.before_seeding_files.each do |file| LOGGER.info("Executing `#{file}`") diff --git a/tasks/db.rake b/tasks/db.rake index 517aa9e..16d7ec9 100644 --- a/tasks/db.rake +++ b/tasks/db.rake @@ -88,8 +88,9 @@ namespace :db do task :insert_script => :require desc "Flushes the database as a particular data set" - task :flush, :to do |t,args| - db_handler.seeder.flush(args[:to] || Time.now.strftime("%Y%M%d%H%M%S").to_s) + task :flush, [:to, :inherits] do |t,args| + args.with_defaults(:to => Time.now.strftime("%Y%M%d%H%M%S").to_s, :inherits => "empty") + db_handler.seeder.flush(args[:to], args[:inherits]) end task :flush => :require