From 8c7e5aa9da8202c4861c0c55d7a32d3dcc719462 Mon Sep 17 00:00:00 2001 From: Bernard Lambeau Date: Sat, 27 Sep 2025 15:09:58 +0200 Subject: [PATCH 1/4] Use .volumes everywhere ; add make clean --- .dockerignore | 2 +- .gitignore | 4 ++-- Makefile | 3 +++ examples/multi-db/docker-compose.yml | 2 +- examples/suppliers-and-parts/docker-compose.yml | 2 +- examples/todo/docker-compose.yml | 2 +- 6 files changed, 9 insertions(+), 6 deletions(-) diff --git a/.dockerignore b/.dockerignore index 814c8cb..2daab34 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,4 +1,4 @@ .bundle **/vendor/bundle -**/volumes +**/.volumes /examples diff --git a/.gitignore b/.gitignore index 6513cef..61fb674 100644 --- a/.gitignore +++ b/.gitignore @@ -1,11 +1,11 @@ **/.bundle **/vendor/bundle /bin -**/volumes **/schema/spy examples/suppliers-and-parts/data/tmp/ examples/suppliers-and-parts/data/incity/ examples/suppliers-and-parts/data/new_empty/ examples/suppliers-and-parts/backups/*.sql pkg/* -/.volumes +.volumes +**/.volumes diff --git a/Makefile b/Makefile index 1053e63..0c0b41b 100644 --- a/Makefile +++ b/Makefile @@ -55,3 +55,6 @@ package: gem.push: ls pkg/dbagent*.gem | xargs gem push + +clean: + find . -name .volumes | xargs rm -rf diff --git a/examples/multi-db/docker-compose.yml b/examples/multi-db/docker-compose.yml index 0e4869d..21d7698 100644 --- a/examples/multi-db/docker-compose.yml +++ b/examples/multi-db/docker-compose.yml @@ -18,7 +18,7 @@ services: - 5432:5432 volumes: - ./initdb:/docker-entrypoint-initdb.d - - ./volumes/pgdata:/var/lib/postgresql/data + - ./.volumes/pgdata:/var/lib/postgresql/data # The `database` agent is the logical DbAgent tool. The agent is NOT intended to # be started in production. diff --git a/examples/suppliers-and-parts/docker-compose.yml b/examples/suppliers-and-parts/docker-compose.yml index 638e3f9..17ca2a6 100644 --- a/examples/suppliers-and-parts/docker-compose.yml +++ b/examples/suppliers-and-parts/docker-compose.yml @@ -14,7 +14,7 @@ services: POSTGRES_USER: dbagent POSTGRES_DB: suppliers-and-parts volumes: - - ./volumes/pgdata:/var/lib/postgresql/data + - ./.volumes/pgdata:/var/lib/postgresql/data # The `database` agent is the logical DbAgent tool. The agent is NOT intended to # be started in production. diff --git a/examples/todo/docker-compose.yml b/examples/todo/docker-compose.yml index 702b225..b56f751 100644 --- a/examples/todo/docker-compose.yml +++ b/examples/todo/docker-compose.yml @@ -14,7 +14,7 @@ services: POSTGRES_USER: todo POSTGRES_DB: todo volumes: - - ./volumes/pgdata:/var/lib/postgresql/data + - ./.volumes/pgdata:/var/lib/postgresql/data # The `database` agent is the logical DbAgent tool. The agent is NOT intended to # be started in production. From cb72d9a55780ee02e7a4541e202db873d7a442e7 Mon Sep 17 00:00:00 2001 From: Bernard Lambeau Date: Sat, 27 Sep 2025 15:14:24 +0200 Subject: [PATCH 2/4] Use same user/pass on all examples. --- examples/multi-db/docker-compose.yml | 8 ++++---- examples/suppliers-and-parts/docker-compose.yml | 2 ++ examples/todo/docker-compose.yml | 6 ++++-- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/examples/multi-db/docker-compose.yml b/examples/multi-db/docker-compose.yml index 21d7698..95f21ea 100644 --- a/examples/multi-db/docker-compose.yml +++ b/examples/multi-db/docker-compose.yml @@ -11,9 +11,9 @@ services: postgres: image: postgres:15 environment: - POSTGRES_USER: postgres + POSTGRES_USER: dbagent + POSTGRES_PASSWORD: dbagent POSTGRES_DB: postgres - POSTGRES_PASSWORD: password ports: - 5432:5432 volumes: @@ -36,6 +36,6 @@ services: - ./schema:/home/app/schema environment: DBAGENT_HOST: postgres - DBAGENT_USER: postgres + DBAGENT_USER: dbagent + DBAGENT_PASSWORD: dbagent DBAGENT_DB: postgres - DBAGENT_PASSWORD: password diff --git a/examples/suppliers-and-parts/docker-compose.yml b/examples/suppliers-and-parts/docker-compose.yml index 17ca2a6..1498820 100644 --- a/examples/suppliers-and-parts/docker-compose.yml +++ b/examples/suppliers-and-parts/docker-compose.yml @@ -12,6 +12,7 @@ services: image: postgres:15 environment: POSTGRES_USER: dbagent + POSTGRES_PASSWORD: dbagent POSTGRES_DB: suppliers-and-parts volumes: - ./.volumes/pgdata:/var/lib/postgresql/data @@ -33,4 +34,5 @@ services: environment: DBAGENT_HOST: postgres DBAGENT_USER: dbagent + DBAGENT_PASSWORD: dbagent DBAGENT_DB: suppliers-and-parts diff --git a/examples/todo/docker-compose.yml b/examples/todo/docker-compose.yml index b56f751..9302f43 100644 --- a/examples/todo/docker-compose.yml +++ b/examples/todo/docker-compose.yml @@ -11,7 +11,8 @@ services: postgres: image: postgres:9.5 environment: - POSTGRES_USER: todo + POSTGRES_USER: dbagent + POSTGRES_PASSWORD: dbagent POSTGRES_DB: todo volumes: - ./.volumes/pgdata:/var/lib/postgresql/data @@ -31,5 +32,6 @@ services: - ./data:/home/app/data - ./schema:/home/app/schema environment: - DBAGENT_USER: todo + DBAGENT_USER: dbagent + DBAGENT_PASSWORD: dbagent DBAGENT_DB: todo From 618ae76926f054b2d4cf3aa2e256d5ec8d242227 Mon Sep 17 00:00:00 2001 From: Bernard Lambeau Date: Sat, 27 Sep 2025 15:50:43 +0200 Subject: [PATCH 3/4] Refactor with a Composite pattern on handler & seeder. --- lib/db_agent/db_handler.rb | 94 +----------------- lib/db_agent/db_handler/actions.rb | 40 ++++++++ lib/db_agent/db_handler/composite.rb | 29 ++++++ lib/db_agent/db_handler/mssql.rb | 1 + lib/db_agent/db_handler/mysql.rb | 2 + lib/db_agent/db_handler/postgresql.rb | 1 + lib/db_agent/db_handler/relational.rb | 102 ++++++++++++++++++++ lib/db_agent/seeder.rb | 126 +----------------------- lib/db_agent/seeder/actions.rb | 22 +++++ lib/db_agent/seeder/composite.rb | 21 ++++ lib/db_agent/seeder/relational.rb | 133 ++++++++++++++++++++++++++ lib/db_agent/webapp.rb | 4 +- tasks/db.rake | 10 +- 13 files changed, 365 insertions(+), 220 deletions(-) create mode 100644 lib/db_agent/db_handler/actions.rb create mode 100644 lib/db_agent/db_handler/composite.rb create mode 100644 lib/db_agent/db_handler/relational.rb create mode 100644 lib/db_agent/seeder/actions.rb create mode 100644 lib/db_agent/seeder/composite.rb create mode 100644 lib/db_agent/seeder/relational.rb diff --git a/lib/db_agent/db_handler.rb b/lib/db_agent/db_handler.rb index 436ab7f..f1b1be7 100644 --- a/lib/db_agent/db_handler.rb +++ b/lib/db_agent/db_handler.rb @@ -20,86 +20,6 @@ def initialize(options) attr_reader :data_folder, :viewpoints_folder attr_reader :migrations_table, :superuser_migrations_table - def ping - puts "Using #{config}" - sequel_db.test_connection - puts "Everything seems fine!" - end - - def create - raise NotImplementedError - end - - def drop - raise NotImplementedError - end - - def backup - raise NotImplementedError - end - - def repl - raise NotImplementedError - end - - def wait_server - require 'net/ping' - raise "No host found" unless config[:host] - check = Net::Ping::External.new(config[:host]) - print "Trying to ping `#{config[:host]}`\n" - wait_timeout_in_seconds.downto(0) do |i| - print "." - if check.ping? - print "\nServer found.\n" - break - elsif i == 0 - print "\n" - raise "Server not found, I give up." - else - sleep(1) - end - end - end - - def wait - print "Using #{config}\n" - wait_timeout_in_seconds.downto(0) do |i| - print "." - begin - sequel_db.test_connection - print "\nDatabase is there. Great.\n" - break - rescue Sequel::Error - if i==0 - print "\n" - raise - end - sleep(1) - end - end - end - - def restore(t, args) - raise NotImplementedError - end - - def migrate(version = nil) - Sequel.extension :migration - sf = migrations_folder/'superuser' - if sf.exists? && !sf.glob('*.rb').empty? - Sequel::Migrator.run(sequel_superdb, migrations_folder/'superuser', table: superuser_migrations_table, target: version) - end - Sequel::Migrator.run(sequel_db, migrations_folder, table: migrations_table, target: version) - end - - def repl - raise NotImplementedError - end - - def spy - raise NotImplementedError - end - def self.factor(options) case options[:config][:adapter] when 'postgres' @@ -113,15 +33,6 @@ def self.factor(options) end end - def sequel_db - @sequel_db ||= ::Sequel.connect(config) - end - - def sequel_superdb - raise "No superconfig set" if superconfig.nil? - @sequel_superdb ||= ::Sequel.connect(superconfig) - end - def system(cmd, *args) puts cmd ::Kernel.system(cmd, *args) @@ -135,7 +46,7 @@ def require_viewpoints! # Forking def fork(options = {}) - DbHandler.new(@original_options.merge(options)) + DbHandler.factor(@original_options.merge(options)) end def fork_config(partial_config = {}) @@ -154,6 +65,9 @@ def print(*args) end # class DbHandler end # module DbAgent +require_relative 'db_handler/actions' +require_relative 'db_handler/composite' +require_relative 'db_handler/relational' require_relative 'db_handler/postgresql' require_relative 'db_handler/mssql' require_relative 'db_handler/mysql' diff --git a/lib/db_agent/db_handler/actions.rb b/lib/db_agent/db_handler/actions.rb new file mode 100644 index 0000000..085be98 --- /dev/null +++ b/lib/db_agent/db_handler/actions.rb @@ -0,0 +1,40 @@ +module DbAgent + class DbHandler + module Actions + + def ping + end + + def create + end + + def drop + end + + def backup + end + + def repl + end + + def wait_server + end + + def wait + end + + def restore(t, args) + end + + def migrate(version = nil) + end + + def repl + end + + def spy + end + + end # module Actions + end # class DbHandler +end # module DbAgent diff --git a/lib/db_agent/db_handler/composite.rb b/lib/db_agent/db_handler/composite.rb new file mode 100644 index 0000000..ff16b76 --- /dev/null +++ b/lib/db_agent/db_handler/composite.rb @@ -0,0 +1,29 @@ +module DbAgent + class DbHandler + class Composite < DbHandler + + def each_db_handler + database_names.each do |name| + yield(self.fork_config(database: name)) + end + end + + def database_names + [ config[:database] ] + end + + Actions.public_instance_methods.each do |meth| + define_method(meth) do |*args, &bl| + each_db_handler do |db| + db.send(meth, *args, &bl) + end + end + end + + def seeder + Seeder::Composite.new(self) + end + + end # class Composite + end # class DbHandler +end # module DbAgent diff --git a/lib/db_agent/db_handler/mssql.rb b/lib/db_agent/db_handler/mssql.rb index faf52a0..641ea62 100644 --- a/lib/db_agent/db_handler/mssql.rb +++ b/lib/db_agent/db_handler/mssql.rb @@ -1,6 +1,7 @@ module DbAgent class DbHandler class MSSQL < DbHandler + include Relational def create raise diff --git a/lib/db_agent/db_handler/mysql.rb b/lib/db_agent/db_handler/mysql.rb index 828ad90..62dd551 100644 --- a/lib/db_agent/db_handler/mysql.rb +++ b/lib/db_agent/db_handler/mysql.rb @@ -3,6 +3,8 @@ module DbAgent class DbHandler class MySQL < DbHandler + include Relational + def create shell mysql('-e', "'CREATE DATABASE #{config[:database]}'") end diff --git a/lib/db_agent/db_handler/postgresql.rb b/lib/db_agent/db_handler/postgresql.rb index 12fb7cc..62dfa40 100644 --- a/lib/db_agent/db_handler/postgresql.rb +++ b/lib/db_agent/db_handler/postgresql.rb @@ -1,6 +1,7 @@ module DbAgent class DbHandler class PostgreSQL < DbHandler + include Relational def create shell pg_cmd("createuser","--no-createdb","--no-createrole","--no-superuser","--no-password",config[:user]), diff --git a/lib/db_agent/db_handler/relational.rb b/lib/db_agent/db_handler/relational.rb new file mode 100644 index 0000000..ded9812 --- /dev/null +++ b/lib/db_agent/db_handler/relational.rb @@ -0,0 +1,102 @@ +module DbAgent + class DbHandler + module Relational + + def ping + puts "Using #{config}" + sequel_db.test_connection + puts "Everything seems fine!" + end + + def create + raise NotImplementedError + end + + def drop + raise NotImplementedError + end + + def backup + raise NotImplementedError + end + + def repl + raise NotImplementedError + end + + def wait_server + require 'net/ping' + raise "No host found" unless config[:host] + check = Net::Ping::External.new(config[:host]) + print "Trying to ping `#{config[:host]}`\n" + wait_timeout_in_seconds.downto(0) do |i| + print "." + if check.ping? + print "\nServer found.\n" + break + elsif i == 0 + print "\n" + raise "Server not found, I give up." + else + sleep(1) + end + end + end + + def wait + print "Using #{config}\n" + wait_timeout_in_seconds.downto(0) do |i| + print "." + begin + sequel_db.test_connection + print "\nDatabase is there. Great.\n" + break + rescue Sequel::Error + if i==0 + print "\n" + raise + end + sleep(1) + end + end + end + + def restore(t, args) + raise NotImplementedError + end + + def migrate(version = nil) + Sequel.extension :migration + sf = migrations_folder/'superuser' + if sf.exists? && !sf.glob('*.rb').empty? + Sequel::Migrator.run(sequel_superdb, migrations_folder/'superuser', table: superuser_migrations_table, target: version) + end + Sequel::Migrator.run(sequel_db, migrations_folder, table: migrations_table, target: version) + end + + def repl + raise NotImplementedError + end + + def spy + raise NotImplementedError + end + + public + + def seeder + Seeder::Relational.new(self) + end + + def sequel_db + @sequel_db ||= ::Sequel.connect(config) + end + + def sequel_superdb + raise "No superconfig set" if superconfig.nil? + @sequel_superdb ||= ::Sequel.connect(superconfig) + end + + end # module Relational + end # class DbHandler +end # module DbAgent diff --git a/lib/db_agent/seeder.rb b/lib/db_agent/seeder.rb index bd2fa9d..3003de9 100644 --- a/lib/db_agent/seeder.rb +++ b/lib/db_agent/seeder.rb @@ -8,128 +8,8 @@ def initialize(handler) end attr_reader :handler, :data_folder - def install(from) - seed_folder = data_folder.seed_folder(from) - seed_files = seed_folder.seed_files_per_table - - handler.sequel_db.transaction do - before_seeding!(seed_folder) - - # Truncate tables - seed_files.keys.reverse.each do |table| - LOGGER.info("Emptying table `#{table}`") - handler.sequel_db[table].delete - end - - # Fill them - seed_files.keys.each do |table| - LOGGER.info("Filling table `#{table}`") - file = seed_files[table] - data = file.load - raise "Empty file: #{file}" if data.nil? - - handler.sequel_db[table].multi_insert(data) - end - - after_seeding!(seed_folder) - end - end - - def insert_script(from) - seed_folder = data_folder.seed_folder(from) - seed_files = seed_folder.seed_files_per_table - - # Fill them - seed_files.keys.each do |table| - file = seed_files[table] - data = file.load - next if data.empty? - - keys = data.first.keys - values = data.map{|t| - keys.map{|k| t[k] } - } - puts handler.sequel_db[table].multi_insert_sql(keys, values) - end - end - - def flush_empty(to = "empty") - target = (handler.data_folder/to).rm_rf.mkdir_p - (target/"metadata.json").write <<-JSON.strip - {} - JSON - TableOrderer.new(handler).tsort.each_with_index do |table_name, index| - (target/"#{(index*10).to_s.rjust(5,"0")}-#{table_name}.json").write("[]") - end - end - - def flush(to) - target = (handler.data_folder/to).rm_rf.mkdir_p - source = (handler.data_folder/"empty") - (target/"metadata.json").write <<-JSON.strip - { "inherits": "empty" } - JSON - seed_files(source).each do |f| - flush_seed_file(f, to) - end - end - - def flush_seed_file(f, to) - target = (handler.data_folder/to) - table = file2table(f) - flush_table(table, target, f.basename, true) - end - - def flush_table(table, target_folder, file_name, skip_empty) - data = viewpoint.send(table.gsub(/\./, '__').to_sym).to_a - table_name = qualify_table(table) - if data.empty? && skip_empty - LOGGER.info("Skipping table `#{table_name}` since empty") - else - LOGGER.info("Flushing table `#{table_name}`") - json = JSON.pretty_generate(data) - (target_folder/file_name).write(json) - end - end - - def each_seed(install = true) - handler.data_folder.glob('**/*') do |file| - next unless file.directory? - next unless (file/"metadata.json").exists? - - base = file.relative_to(handler.data_folder) - begin - Seeder.new(handler).install(base) - puts "#{base} OK" - yield(self, file) if block_given? - rescue => ex - puts "KO on #{file}" - puts ex.message - end if install - end - end - - private - - def before_seeding!(seed_folder) - seed_folder.before_seeding_files.each do |file| - handler.sequel_db.execute(file.read) - end - end - - def after_seeding!(seed_folder) - seed_folder.after_seeding_files.each do |file| - handler.sequel_db.execute(file.read) - end - end - - def viewpoint - @viewpoint ||= if vp = ENV['DBAGENT_VIEWPOINT'] - Kernel.const_get(vp).new(handler.sequel_db) - else - Viewpoint::Base.new(handler.sequel_db) - end - end - end # class Seeder end # module DbAgent +require_relative 'seeder/actions' +require_relative 'seeder/composite' +require_relative 'seeder/relational' diff --git a/lib/db_agent/seeder/actions.rb b/lib/db_agent/seeder/actions.rb new file mode 100644 index 0000000..516c941 --- /dev/null +++ b/lib/db_agent/seeder/actions.rb @@ -0,0 +1,22 @@ +module DbAgent + class Seeder + module Actions + + def install(from) + end + + def insert_script(from) + end + + def flush_empty(to = "empty") + end + + def flush(to) + end + + def check_seeds(install = true) + end + + end # module Actions + end # class Seeder +end # module DbAgent diff --git a/lib/db_agent/seeder/composite.rb b/lib/db_agent/seeder/composite.rb new file mode 100644 index 0000000..f8fbe43 --- /dev/null +++ b/lib/db_agent/seeder/composite.rb @@ -0,0 +1,21 @@ +module DbAgent + class Seeder + class Composite < Seeder + + def each_db_seeder + handler.each_db_handler do |db_handler| + yield(db_handler.seeder) + end + end + + Actions.public_instance_methods.each do |meth| + define_method(meth) do |*args, &bl| + each_db_seeder do |seeder| + seeder.send(meth, *args, &bl) + end + end + end + + end # class Composite + end # class Seeder +end # module DbAgent diff --git a/lib/db_agent/seeder/relational.rb b/lib/db_agent/seeder/relational.rb new file mode 100644 index 0000000..0247b11 --- /dev/null +++ b/lib/db_agent/seeder/relational.rb @@ -0,0 +1,133 @@ +module DbAgent + class Seeder + class Relational < Seeder + + def install(from) + seed_folder = data_folder.seed_folder(from) + seed_files = seed_folder.seed_files_per_table + + sequel_db.transaction do + before_seeding!(seed_folder) + + # Truncate tables + seed_files.keys.reverse.each do |table| + LOGGER.info("Emptying table `#{table}`") + sequel_db[table].delete + end + + # Fill them + seed_files.keys.each do |table| + LOGGER.info("Filling table `#{table}`") + file = seed_files[table] + data = file.load + raise "Empty file: #{file}" if data.nil? + + sequel_db[table].multi_insert(data) + end + + after_seeding!(seed_folder) + end + end + + def insert_script(from) + seed_folder = data_folder.seed_folder(from) + seed_files = seed_folder.seed_files_per_table + + # Fill them + seed_files.keys.each do |table| + file = seed_files[table] + data = file.load + next if data.empty? + + keys = data.first.keys + values = data.map{|t| + keys.map{|k| t[k] } + } + puts sequel_db[table].multi_insert_sql(keys, values) + end + end + + def flush_empty(to = "empty") + target = (handler.data_folder/to).rm_rf.mkdir_p + (target/"metadata.json").write <<-JSON.strip + {} + JSON + TableOrderer.new(handler).tsort.each_with_index do |table_name, index| + (target/"#{(index*10).to_s.rjust(5,"0")}-#{table_name}.json").write("[]") + end + end + + def flush(to) + target = (handler.data_folder/to).rm_rf.mkdir_p + source = (handler.data_folder/"empty") + (target/"metadata.json").write <<-JSON.strip + { "inherits": "empty" } + JSON + seed_files(source).each do |f| + flush_seed_file(f, to) + end + end + + def check_seeds(install = true) + handler.data_folder.glob('**/*') do |file| + next unless file.directory? + next unless (file/"metadata.json").exists? + + base = file.relative_to(handler.data_folder) + begin + Seeder.new(handler).install(base) + puts "#{base} OK" + rescue => ex + puts "KO on #{file}" + puts ex.message + end if install + end + end + + private + + def flush_seed_file(f, to) + target = (handler.data_folder/to) + table = file2table(f) + flush_table(table, target, f.basename, true) + end + + def flush_table(table, target_folder, file_name, skip_empty) + data = viewpoint.send(table.gsub(/\./, '__').to_sym).to_a + table_name = qualify_table(table) + if data.empty? && skip_empty + LOGGER.info("Skipping table `#{table_name}` since empty") + else + LOGGER.info("Flushing table `#{table_name}`") + json = JSON.pretty_generate(data) + (target_folder/file_name).write(json) + end + end + + def before_seeding!(seed_folder) + seed_folder.before_seeding_files.each do |file| + sequel_db.execute(file.read) + end + end + + def after_seeding!(seed_folder) + seed_folder.after_seeding_files.each do |file| + sequel_db.execute(file.read) + end + end + + def viewpoint + @viewpoint ||= if vp = ENV['DBAGENT_VIEWPOINT'] + Kernel.const_get(vp).new(sequel_db) + else + Viewpoint::Base.new(sequel_db) + end + end + + def sequel_db + handler.send(:sequel_db) + end + + end # class Relational + end # class Seeder +end # module DbAgent diff --git a/lib/db_agent/webapp.rb b/lib/db_agent/webapp.rb index db2e6b5..4514eb2 100644 --- a/lib/db_agent/webapp.rb +++ b/lib/db_agent/webapp.rb @@ -28,13 +28,13 @@ class Webapp < Sinatra::Base end post '/seeds/install' do - Seeder.new(settings.db_handler).install(request["id"]) + settings.db_handler.seeder.install(request["id"]) "ok" end post '/seeds/flush' do seed_name = request["id"] - Seeder.new(settings.db_handler).flush(request["id"]) + settings.db_handler.seeder.flush(request["id"]) "ok" end diff --git a/tasks/db.rake b/tasks/db.rake index 32f68ce..be013af 100644 --- a/tasks/db.rake +++ b/tasks/db.rake @@ -71,31 +71,31 @@ namespace :db do desc "Checks that all seeds can be installed correctly" task :"check-seeds" do - Seeder.new(db_handler).each_seed(true) + db_handler.seeder.check_seeds(true) end task :"check-seeds" => :require desc "Seeds the database with a particular data set" task :seed, :from do |t,args| - Seeder.new(db_handler).install(args[:from] || 'empty') + db_handler.seeder.install(args[:from] || 'empty') end task :seed => :require desc "Prints an INSERT script for a particular data set" task :insert_script, :from do |t,args| - Seeder.new(db_handler).insert_script(args[:from] || 'empty') + db_handler.seeder.insert_script(args[:from] || 'empty') end task :insert_script => :require desc "Flushes the database as a particular data set" task :flush, :to do |t,args| - Seeder.new(db_handler).flush(args[:to] || Time.now.strftime("%Y%M%d%H%M%S").to_s) + db_handler.seeder.flush(args[:to] || Time.now.strftime("%Y%M%d%H%M%S").to_s) end task :flush => :require desc "Flushes the initial empty files as a data set" task :flush_empty, :to do |t,args| - Seeder.new(db_handler).flush_empty(args[:to] || Time.now.strftime("%Y%M%d%H%M%S").to_s) + db_handler.seeder.flush_empty(args[:to] || Time.now.strftime("%Y%M%d%H%M%S").to_s) end task :flush_empty => :require From f8c9da68895ddb8919148cb34590dedac356b7ec Mon Sep 17 00:00:00 2001 From: Bernard Lambeau Date: Sat, 27 Sep 2025 16:13:33 +0200 Subject: [PATCH 4/4] Complete support for multiple databases. --- .gitignore | 2 +- examples/multi-db/data/base/db1/01-todo.json | 4 +- examples/multi-db/data/base/db2/01-users.json | 6 ++ examples/multi-db/data/base/db2/metadata.json | 1 + lib/db_agent.rb | 1 + lib/db_agent/data_folder.rb | 19 +++++- lib/db_agent/db_handler.rb | 26 ++++---- lib/db_agent/db_handler/composite.rb | 20 ++++-- lib/db_agent/db_handler/relational.rb | 6 +- lib/db_agent/seed_folder.rb | 31 +++++---- lib/db_agent/seed_utils.rb | 6 -- lib/db_agent/seeder.rb | 7 +- lib/db_agent/seeder/composite.rb | 6 +- lib/db_agent/seeder/relational.rb | 33 +++++----- spec/db_handler/test_composite.rb | 65 +++++++++++++++++++ spec/spec_helper.rb | 4 ++ 16 files changed, 172 insertions(+), 65 deletions(-) create mode 100644 examples/multi-db/data/base/db2/01-users.json create mode 100644 examples/multi-db/data/base/db2/metadata.json create mode 100644 spec/db_handler/test_composite.rb diff --git a/.gitignore b/.gitignore index 61fb674..03e568c 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,7 @@ **/vendor/bundle /bin **/schema/spy -examples/suppliers-and-parts/data/tmp/ +examples/*/data/tmp/ examples/suppliers-and-parts/data/incity/ examples/suppliers-and-parts/data/new_empty/ examples/suppliers-and-parts/backups/*.sql diff --git a/examples/multi-db/data/base/db1/01-todo.json b/examples/multi-db/data/base/db1/01-todo.json index ab67d4e..3d3fb8b 100644 --- a/examples/multi-db/data/base/db1/01-todo.json +++ b/examples/multi-db/data/base/db1/01-todo.json @@ -1,8 +1,6 @@ [ { "id": 1, - "title": "Write more documenation", - "description": "Get started guide should probably be longer", - "done": false + "title": "Write more documenation" } ] diff --git a/examples/multi-db/data/base/db2/01-users.json b/examples/multi-db/data/base/db2/01-users.json new file mode 100644 index 0000000..e651e5e --- /dev/null +++ b/examples/multi-db/data/base/db2/01-users.json @@ -0,0 +1,6 @@ +[ + { + "id": 1, + "name": "Bernard Lambeau" + } +] diff --git a/examples/multi-db/data/base/db2/metadata.json b/examples/multi-db/data/base/db2/metadata.json new file mode 100644 index 0000000..bde3048 --- /dev/null +++ b/examples/multi-db/data/base/db2/metadata.json @@ -0,0 +1 @@ +{ "inherits": "empty" } diff --git a/lib/db_agent.rb b/lib/db_agent.rb index a91a515..2518a58 100644 --- a/lib/db_agent.rb +++ b/lib/db_agent.rb @@ -64,6 +64,7 @@ def self.default_superconfig(cfg = default_config) def self.default_handler cfg = default_config DbHandler.factor({ + databases: ENV['DBAGENT_DATABASES'], config: cfg, superconfig: default_superconfig(cfg), root: ROOT_FOLDER, diff --git a/lib/db_agent/data_folder.rb b/lib/db_agent/data_folder.rb index ad81eea..d0c2495 100644 --- a/lib/db_agent/data_folder.rb +++ b/lib/db_agent/data_folder.rb @@ -1,13 +1,26 @@ module DbAgent class DataFolder - def initialize(db_handler) + def initialize(db_handler, database_suffix = nil) @db_handler = db_handler + @database_suffix = database_suffix end attr_reader :db_handler - def seed_folder(seed, database = nil) - SeedFolder.new(self, seed, database) + def seed_folder(seed, database_suffix = @database_suffix) + SeedFolder.new(self, seed, database_suffix) + end + + def path + db_handler.data_folder + end + + def /(part) + path/part + end + + def glob(*args, &bl) + path.glob(*args, &bl) end end # class DataFolder diff --git a/lib/db_agent/db_handler.rb b/lib/db_agent/db_handler.rb index f1b1be7..d7422de 100644 --- a/lib/db_agent/db_handler.rb +++ b/lib/db_agent/db_handler.rb @@ -2,7 +2,7 @@ module DbAgent class DbHandler def initialize(options) - @original_options = options + @options = options @config = options[:config] @superconfig = options[:superconfig] @root_folder = options[:root] @@ -15,21 +15,25 @@ def initialize(options) @superuser_migrations_table = options[:superuser_migrations_table] || 'superuser_migrations' require_viewpoints! end - attr_reader :config, :superconfig + attr_reader :options, :config, :superconfig attr_reader :backup_folder, :schema_folder, :migrations_folder attr_reader :data_folder, :viewpoints_folder attr_reader :migrations_table, :superuser_migrations_table def self.factor(options) - case options[:config][:adapter] - when 'postgres' - PostgreSQL.new(options) - when 'mssql' - MSSQL.new(options) - when /mysql/ - MySQL.new(options) + if options[:databases] + Composite.new(options) else - PostgreSQL.new(options) + case options[:config][:adapter] + when 'postgres' + PostgreSQL.new(options) + when 'mssql' + MSSQL.new(options) + when /mysql/ + MySQL.new(options) + else + PostgreSQL.new(options) + end end end @@ -46,7 +50,7 @@ def require_viewpoints! # Forking def fork(options = {}) - DbHandler.factor(@original_options.merge(options)) + DbHandler.factor(@options.merge(options)) end def fork_config(partial_config = {}) diff --git a/lib/db_agent/db_handler/composite.rb b/lib/db_agent/db_handler/composite.rb index ff16b76..cd553bf 100644 --- a/lib/db_agent/db_handler/composite.rb +++ b/lib/db_agent/db_handler/composite.rb @@ -4,23 +4,35 @@ class Composite < DbHandler def each_db_handler database_names.each do |name| - yield(self.fork_config(database: name)) + yield(name, self.fork_config(database: name).fork({ + databases: nil, + backup: @backup_folder/name, + schema: @schema_folder/name, + migrations: @migrations_folder/name, + })) end end def database_names - [ config[:database] ] + case dbs = options[:databases] + when '/from-empty-seeds' + (data_folder/'empty').glob('*').filter{|f| f.directory? }.map{|f| f.basename.to_s } + when NilClass + [config[:database]] + else + dbs.split(/\s*,\s*/) + end end Actions.public_instance_methods.each do |meth| define_method(meth) do |*args, &bl| - each_db_handler do |db| + each_db_handler do |dbname, db| db.send(meth, *args, &bl) end end end - def seeder + def seeder(database_suffix = nil) Seeder::Composite.new(self) end diff --git a/lib/db_agent/db_handler/relational.rb b/lib/db_agent/db_handler/relational.rb index ded9812..ab75443 100644 --- a/lib/db_agent/db_handler/relational.rb +++ b/lib/db_agent/db_handler/relational.rb @@ -69,7 +69,7 @@ def migrate(version = nil) Sequel.extension :migration sf = migrations_folder/'superuser' if sf.exists? && !sf.glob('*.rb').empty? - Sequel::Migrator.run(sequel_superdb, migrations_folder/'superuser', table: superuser_migrations_table, target: version) + Sequel::Migrator.run(sequel_superdb, sf, table: superuser_migrations_table, target: version) end Sequel::Migrator.run(sequel_db, migrations_folder, table: migrations_table, target: version) end @@ -84,8 +84,8 @@ def spy public - def seeder - Seeder::Relational.new(self) + def seeder(database_suffix = nil) + Seeder::Relational.new(self, database_suffix) end def sequel_db diff --git a/lib/db_agent/seed_folder.rb b/lib/db_agent/seed_folder.rb index 4188c3e..e00eb71 100644 --- a/lib/db_agent/seed_folder.rb +++ b/lib/db_agent/seed_folder.rb @@ -2,9 +2,9 @@ module DbAgent class SeedFolder include SeedUtils - def initialize(data_folder, seed = 'empty', database = nil) + def initialize(data_folder, seed = 'empty', database_suffix = nil) @data_folder = data_folder - @database = database + @database_suffix = database_suffix @seed = seed end attr_reader :data_folder, :seed @@ -14,33 +14,33 @@ def db_handler end def metadata - @metadata ||= (folder(seed)/"metadata.json").load + @metadata ||= (path(seed)/"metadata.json").load end - def folder(seed = self.seed) - if @database - db_handler.data_folder/seed/@database + def path(seed = self.seed) + if @database_suffix + data_folder/seed/@database_suffix else - db_handler.data_folder/seed + data_folder/seed end end def parent @parent ||= if inherits = metadata["inherits"] - SeedFolder.new(data_folder, inherits, @database) + SeedFolder.new(data_folder, inherits, @database_suffix) else NullObject.new(data_folder) end end def before_seeding_files - f = (folder/'before_seeding.sql') + f = (path/'before_seeding.sql') fs = f.file? ? [f] : [] parent.before_seeding_files + fs end def after_seeding_files - f = (folder/'after_seeding.sql') + f = (path/'after_seeding.sql') fs = f.file? ? [f] : [] parent.after_seeding_files + fs end @@ -58,8 +58,10 @@ def seed_files_per_table end end + protected + def _seed_files_per_table - folder = self.folder(seed) + folder = self.path(seed) map = parent._seed_files_per_table seed_files(folder).each do |f| @@ -68,7 +70,12 @@ def _seed_files_per_table map end - protected :_seed_files_per_table + + def seed_files(folder) + folder + .glob("*.json") + .reject{|f| f.basename.to_s =~ /^metadata/ } + end class NullObject < SeedFolder def _seed_files_per_table diff --git a/lib/db_agent/seed_utils.rb b/lib/db_agent/seed_utils.rb index 410c1da..c09d811 100644 --- a/lib/db_agent/seed_utils.rb +++ b/lib/db_agent/seed_utils.rb @@ -1,12 +1,6 @@ module DbAgent module SeedUtils - def seed_files(folder) - folder - .glob("*.json") - .reject{|f| f.basename.to_s =~ /^metadata/ } - end - def file2table(file) file.basename.rm_ext.to_s[/^\d+-(.*)/, 1] end diff --git a/lib/db_agent/seeder.rb b/lib/db_agent/seeder.rb index 3003de9..513929a 100644 --- a/lib/db_agent/seeder.rb +++ b/lib/db_agent/seeder.rb @@ -2,11 +2,12 @@ module DbAgent class Seeder include SeedUtils - def initialize(handler) + def initialize(handler, database_suffix = nil) @handler = handler - @data_folder = DataFolder.new(handler) + @database_suffix = database_suffix + @data_folder = DataFolder.new(handler, database_suffix) end - attr_reader :handler, :data_folder + attr_reader :handler, :data_folder, :database_suffix end # class Seeder end # module DbAgent diff --git a/lib/db_agent/seeder/composite.rb b/lib/db_agent/seeder/composite.rb index f8fbe43..5958c79 100644 --- a/lib/db_agent/seeder/composite.rb +++ b/lib/db_agent/seeder/composite.rb @@ -3,14 +3,14 @@ class Seeder class Composite < Seeder def each_db_seeder - handler.each_db_handler do |db_handler| - yield(db_handler.seeder) + handler.each_db_handler do |dbname, db_handler| + yield(dbname, db_handler.seeder(dbname)) end end Actions.public_instance_methods.each do |meth| define_method(meth) do |*args, &bl| - each_db_seeder do |seeder| + each_db_seeder do |dbname, seeder| seeder.send(meth, *args, &bl) end end diff --git a/lib/db_agent/seeder/relational.rb b/lib/db_agent/seeder/relational.rb index 0247b11..21d81fc 100644 --- a/lib/db_agent/seeder/relational.rb +++ b/lib/db_agent/seeder/relational.rb @@ -48,34 +48,41 @@ def insert_script(from) end def flush_empty(to = "empty") - target = (handler.data_folder/to).rm_rf.mkdir_p + target = data_folder.seed_folder(to).path.rm_rf.mkdir_p + (target/"metadata.json").write <<-JSON.strip {} JSON + TableOrderer.new(handler).tsort.each_with_index do |table_name, index| (target/"#{(index*10).to_s.rjust(5,"0")}-#{table_name}.json").write("[]") end end def flush(to) - target = (handler.data_folder/to).rm_rf.mkdir_p - source = (handler.data_folder/"empty") + target = data_folder.seed_folder(to).path.rm_rf.mkdir_p + source = data_folder.seed_folder('empty') + seed_files = source.seed_files_per_table + (target/"metadata.json").write <<-JSON.strip { "inherits": "empty" } JSON - seed_files(source).each do |f| - flush_seed_file(f, to) + + seed_files.each_pair do |table_name, source_file| + target_file = target/source_file.basename.to_s + table = file2table(target_file) + flush_table(table, target_file, true) end end def check_seeds(install = true) - handler.data_folder.glob('**/*') do |file| + data_folder.glob('**/*') do |file| next unless file.directory? next unless (file/"metadata.json").exists? - base = file.relative_to(handler.data_folder) + base = file.relative_to(data_folder.path) begin - Seeder.new(handler).install(base) + handler.seeder.install(base) puts "#{base} OK" rescue => ex puts "KO on #{file}" @@ -86,13 +93,7 @@ def check_seeds(install = true) private - def flush_seed_file(f, to) - target = (handler.data_folder/to) - table = file2table(f) - flush_table(table, target, f.basename, true) - end - - def flush_table(table, target_folder, file_name, skip_empty) + def flush_table(table, target_file, skip_empty) data = viewpoint.send(table.gsub(/\./, '__').to_sym).to_a table_name = qualify_table(table) if data.empty? && skip_empty @@ -100,7 +101,7 @@ def flush_table(table, target_folder, file_name, skip_empty) else LOGGER.info("Flushing table `#{table_name}`") json = JSON.pretty_generate(data) - (target_folder/file_name).write(json) + target_file.write(json) end end diff --git a/spec/db_handler/test_composite.rb b/spec/db_handler/test_composite.rb new file mode 100644 index 0000000..1112966 --- /dev/null +++ b/spec/db_handler/test_composite.rb @@ -0,0 +1,65 @@ + +require 'spec_helper' + +module DbAgent + describe DbHandler::Composite do + + let(:config) { + { + user: 'postgres', + database: 'sap', + } + } + + let(:handler) { + DbHandler::Composite.new({ + databases: databases, + config: config, + root: examples_folder/subfolder + }) + } + + context 'on a single database with no instruction' do + let(:subfolder) { + 'suppliers-and-parts' + } + + let(:databases) { + nil + } + + it 'uses the main database' do + expect(handler.database_names).to eql(['sap']) + end + end + + context 'on multiple database with an explicit list' do + let(:subfolder) { + 'multi-db' + } + + let(:databases) { + 'db1,db2' + } + + it 'uses the main database' do + expect(handler.database_names).to eql(['db1', 'db2']) + end + end + + context 'on multiple database from empty seeds' do + let(:subfolder) { + 'multi-db' + } + + let(:databases) { + '/from-empty-seeds' + } + + it 'uses the main database' do + expect(handler.database_names).to eql(['db1', 'db2']) + end + end + + end +end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index ce07bcc..2398716 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -13,6 +13,10 @@ def database @db ||= DbAgent.default_handler.sequel_db end + def examples_folder + Path.backfind('.[Gemfile]')/'examples' + end + end RSpec.configure do |c|