Add an inherits argument to db:flush, and skip seed files that are un…

CHANGELOG.md

@@ -1,4 +1,9 @@
-## 3.9.0
+## 3.9.1
+
+* `rake db:flush` now supports a second argument, which is the seed to inherit
+  from. Tables whose data is unchanged are no longer flushed on disk.
+
+## 3.9.0 - 2025-09-29
 
 * Add support for multiple databases, via a DBAGENT_DATABASES environment
   variables

Gemfile.lock

@@ -3,7 +3,7 @@ PATH
   specs:
     dbagent (3.9.0)
       base64 (~> 0.3)
-      bmg (~> 0.21)
+      bmg (~> 0.23.5)
       net-ping (~> 2)
       path (~> 2)
       pg (~> 1)
@@ -17,43 +17,50 @@ GEM
   remote: http://rubygems.org/
   specs:
     base64 (0.3.0)
-    bigdecimal (3.1.6)
-    bmg (0.21.3)
+    bigdecimal (3.2.3)
+    bmg (0.23.5)
       path (>= 2.0)
       predicate (>= 2.7.1, < 3.0)
     daemons (1.4.1)
-    diff-lcs (1.5.1)
+    diff-lcs (1.6.2)
     eventmachine (1.2.7)
-    minitest (5.22.2)
+    minitest (5.25.5)
     mustermann (2.0.2)
       ruby2_keywords (~> 0.0.1)
     net-ping (2.0.8)
     path (2.1.0)
-    pg (1.5.5)
-    predicate (2.8.0)
+    pg (1.6.2)
+    pg (1.6.2-aarch64-linux)
+    pg (1.6.2-aarch64-linux-musl)
+    pg (1.6.2-aarch64-mingw-ucrt)
+    pg (1.6.2-arm64-darwin)
+    pg (1.6.2-x86_64-darwin)
+    pg (1.6.2-x86_64-linux)
+    pg (1.6.2-x86_64-linux-musl)
+    predicate (2.9.0)
       minitest (>= 5.0)
       sexpr (~> 1.1)
-    rack (2.2.8.1)
+    rack (2.2.18)
     rack-protection (2.2.4)
       rack
     rack-test (1.1.0)
       rack (>= 1.0, < 3)
-    rake (13.1.0)
-    rspec (3.13.0)
+    rake (13.3.0)
+    rspec (3.13.1)
       rspec-core (~> 3.13.0)
       rspec-expectations (~> 3.13.0)
       rspec-mocks (~> 3.13.0)
-    rspec-core (3.13.0)
+    rspec-core (3.13.5)
       rspec-support (~> 3.13.0)
-    rspec-expectations (3.13.0)
+    rspec-expectations (3.13.5)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
-    rspec-mocks (3.13.0)
+    rspec-mocks (3.13.5)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
-    rspec-support (3.13.0)
+    rspec-support (3.13.6)
     ruby2_keywords (0.0.5)
-    sequel (5.77.0)
+    sequel (5.96.0)
       bigdecimal
     sexpr (1.1.0)
     sinatra (2.2.4)
@@ -65,11 +72,17 @@ GEM
       daemons (~> 1.0, >= 1.0.9)
       eventmachine (~> 1.0, >= 1.0.4)
       rack (>= 1, < 3)
-    tilt (2.3.0)
+    tilt (2.6.1)
 
 PLATFORMS
+  aarch64-linux
+  aarch64-linux-musl
+  aarch64-mingw-ucrt
+  arm64-darwin
   ruby
+  x86_64-darwin
   x86_64-linux
+  x86_64-linux-musl
 
 DEPENDENCIES
   bundler (~> 2)
@@ -78,4 +91,4 @@ DEPENDENCIES
   rspec (~> 3.6)
 
 BUNDLED WITH
-   2.4.6
+   2.6.2

README.md

@@ -42,23 +42,23 @@ See the examples folder for details.
 The following rake tasks helps you managing the database. They must typically be executed on the docker container.
 
 ```
-rake db:check-seeds      # Checks that all seeds can be installed correctly
-rake db:create           # Creates an fresh new user & database (USE WITH CARE)
-rake db:drop             # Drops the user & database (USE WITH CARE)
-rake db:flush[to]        # Flushes the database as a particular data set
-rake db:migrate          # Runs migrations on the current database
-rake db:ping             # Pings the database, making sure everything's ready for migration
-rake db:rebuild          # Rebuilds the database from scratch (USE WITH CARE)
-rake db:repl             # Opens a database REPL
-rake db:seed[from]       # Seeds the database with a particular data set
-rake db:spy              # Dumps the schema documentation into database/schema
-rake db:backup           # Makes a database backup to the backups folder
-rake db:restore[match]   # Restore the last matching database backup file from backups folder
-rake db:revive           # Shortcut for both db:restore and db:migrate
-rake db:wait_server      # Waits until the postgresql host seems available
-rake db:wait             # Waits until the postgresql database seems available
-rake db:tables           # List tables with those with fewer dependencies first
-rake db:dependencies[of] # List tables that depend of a given one
+rake db:check-seeds          # Checks that all seeds can be installed correctly
+rake db:create               # Creates an fresh new user & database (USE WITH CARE)
+rake db:drop                 # Drops the user & database (USE WITH CARE)
+rake db:flush[to,inherits?]  # Flushes the database as a particular data set
+rake db:migrate              # Runs migrations on the current database
+rake db:ping                 # Pings the database, making sure everything's ready for migration
+rake db:rebuild              # Rebuilds the database from scratch (USE WITH CARE)
+rake db:repl                 # Opens a database REPL
+rake db:seed[from]           # Seeds the database with a particular data set
+rake db:spy                  # Dumps the schema documentation into database/schema
+rake db:backup               # Makes a database backup to the backups folder
+rake db:restore[match]       # Restore the last matching database backup file from backups folder
+rake db:revive               # Shortcut for both db:restore and db:migrate
+rake db:wait_server          # Waits until the postgresql host seems available
+rake db:wait                 # Waits until the postgresql database seems available
+rake db:tables               # List tables with those with fewer dependencies first
+rake db:dependencies[of]     # List tables that depend of a given one
 ```
 
 ## Available webservices

dbagent.gemspec

@@ -19,7 +19,7 @@ Gem::Specification.new do |s|
   s.add_dependency 'rake', '~> 13'
   s.add_dependency 'path', '~> 2'
   s.add_dependency 'sinatra', '~> 2'
-  s.add_dependency 'bmg', '~> 0.21'
+  s.add_dependency 'bmg', '~> 0.23.5'
   s.add_dependency 'net-ping', "~> 2"
   s.add_dependency "predicate", "~> 2", ">= 2.7.1"
   s.add_dependency 'thin', '~> 1'

lib/db_agent/seeder/actions.rb

@@ -11,7 +11,7 @@ def insert_script(from)
       def flush_empty(to = "empty")
       end
 
-      def flush(to)
+      def flush(to, inherits = "empty")
       end
 
       def check_seeds

lib/db_agent/seeder/relational.rb

@@ -59,19 +59,20 @@ def flush_empty(to = "empty")
         end
       end
 
-      def flush(to)
-        target = data_folder.seed_folder(to).path.rm_rf.mkdir_p
-        source = data_folder.seed_folder('empty')
-        seed_files = source.seed_files_per_table
-
-        (target/"metadata.json").write <<-JSON.strip
-          { "inherits": "empty" }
+      def flush(to, inherits = "empty")
+        target_path = data_folder.seed_folder(to).path.rm_rf.mkdir_p
+        inherit_seed = data_folder.seed_folder(inherits)
+        inherit_path = inherit_seed.path
+        seed_files = inherit_seed.seed_files_per_table
+
+        (target_path/"metadata.json").write <<-JSON.strip
+          { "inherits": "#{inherits}" }
         JSON
 
-        seed_files.each_pair do |table_name, source_file|
-          target_file = target/source_file.basename.to_s
+        seed_files.each_pair do |table_name, inherit_file|
+          target_file = target_path/inherit_file.basename.to_s
           table = file2table(target_file)
-          flush_table(table, target_file, true)
+          flush_table(table, target_file, inherit_file)
         end
       end
 
@@ -90,18 +91,28 @@ def check_seeds
 
     private
 
-      def flush_table(table, target_file, skip_empty)
+      def flush_table(table, target_file, inherit_file)
         data = viewpoint.send(table.gsub(/\./, '__').to_sym).to_a
         table_name = qualify_table(table)
-        if data.empty? && skip_empty
-          LOGGER.info("Skipping table `#{table_name}` since empty")
+        if data.empty?
+          LOGGER.info("Skipping table `#{table_name}`: empty")
+        elsif same_data?(inherit_file, data)
+          LOGGER.info("Skipping table `#{table_name}`: same as inherited one")
         else
           LOGGER.info("Flushing table `#{table_name}`")
           json = JSON.pretty_generate(data)
           target_file.write(json)
         end
       end
 
+      def same_data?(inherit_file, data)
+        return false unless inherit_file.file?
+
+        r = Bmg.json(inherit_file)
+        s = Bmg.in_memory(data)
+        r.to_set == s.to_set
+      end
+
       def before_seeding!(seed_folder)
         seed_folder.before_seeding_files.each do |file|
           LOGGER.info("Executing `#{file}`")

tasks/db.rake

@@ -88,8 +88,9 @@ namespace :db do
   task :insert_script => :require
 
   desc "Flushes the database as a particular data set"
-  task :flush, :to do |t,args|
-    db_handler.seeder.flush(args[:to] || Time.now.strftime("%Y%M%d%H%M%S").to_s)
+  task :flush, [:to, :inherits] do |t,args|
+    args.with_defaults(:to => Time.now.strftime("%Y%M%d%H%M%S").to_s, :inherits => "empty")
+    db_handler.seeder.flush(args[:to], args[:inherits])
   end
   task :flush => :require