From 6ca9127870e4a1316e523d5a84904ffe5295115c Mon Sep 17 00:00:00 2001 From: Bernard Lambeau Date: Sat, 27 Sep 2025 11:18:29 +0200 Subject: [PATCH 1/3] fixup! Fix test suite. --- Makefile | 7 +++---- docker-compose.yml | 2 +- examples/suppliers-and-parts/data/base/metadata.json | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index d18bb8d..1053e63 100644 --- a/Makefile +++ b/Makefile @@ -36,19 +36,18 @@ exec_test: bundle exec rake db:seed[base]; \ bundle exec rake db:insert_script[base]; \ bundle exec rake db:flush[tmp]; \ + rm -rf examples/suppliers-and-parts/data/tmp bundle exec rake db:flush_empty[new_empty]; \ + rm -rf examples/suppliers-and-parts/data/new_empty; \ bundle exec rake db:spy; \ bundle exec rake db:backup; \ ' -clean: - rm -rf examples/suppliers-and-parts/data/new_empty examples/suppliers-and-parts/data/tmp - down: docker stop db dbagent || true docker network rm agent-network || true -test: down prepare exec_test clean down +test: down prepare exec_test down package: bundle install diff --git a/docker-compose.yml b/docker-compose.yml index 9b20617..5476374 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,4 +11,4 @@ services: POSTGRES_PASSWORD: dbagent POSTGRES_DB: suppliers-and-parts volumes: - - ./volumes/pgdata:/var/lib/postgresql/data + - .volumes/pgdata:/var/lib/postgresql/data diff --git a/examples/suppliers-and-parts/data/base/metadata.json b/examples/suppliers-and-parts/data/base/metadata.json index 0967ef4..bde3048 100644 --- a/examples/suppliers-and-parts/data/base/metadata.json +++ b/examples/suppliers-and-parts/data/base/metadata.json @@ -1 +1 @@ -{} +{ "inherits": "empty" } From 5db729b553d1956466623e9e026cd847d33c3572 Mon Sep 17 00:00:00 2001 From: Bernard Lambeau Date: Sat, 27 Sep 2025 11:48:53 +0200 Subject: [PATCH 2/3] Add SeedFolder#before_seeding_files and SeedFolder#after_seeding_files --- .../data/hooks/after_seeding.sql | 0 .../data/hooks/before_seeding.sql | 0 .../data/hooks/child/after_seeding.sql | 0 .../data/hooks/child/before_seeding.sql | 0 .../data/hooks/child/metadata.json | 1 + .../data/hooks/metadata.json | 1 + lib/db_agent/seed_folder.rb | 54 ++++++++++++++----- lib/db_agent/seeder.rb | 25 +++++---- spec/test_data_folder.rb | 52 +++++++++++++++++- 9 files changed, 105 insertions(+), 28 deletions(-) create mode 100644 examples/suppliers-and-parts/data/hooks/after_seeding.sql create mode 100644 examples/suppliers-and-parts/data/hooks/before_seeding.sql create mode 100644 examples/suppliers-and-parts/data/hooks/child/after_seeding.sql create mode 100644 examples/suppliers-and-parts/data/hooks/child/before_seeding.sql create mode 100644 examples/suppliers-and-parts/data/hooks/child/metadata.json create mode 100644 examples/suppliers-and-parts/data/hooks/metadata.json diff --git a/examples/suppliers-and-parts/data/hooks/after_seeding.sql b/examples/suppliers-and-parts/data/hooks/after_seeding.sql new file mode 100644 index 0000000..e69de29 diff --git a/examples/suppliers-and-parts/data/hooks/before_seeding.sql b/examples/suppliers-and-parts/data/hooks/before_seeding.sql new file mode 100644 index 0000000..e69de29 diff --git a/examples/suppliers-and-parts/data/hooks/child/after_seeding.sql b/examples/suppliers-and-parts/data/hooks/child/after_seeding.sql new file mode 100644 index 0000000..e69de29 diff --git a/examples/suppliers-and-parts/data/hooks/child/before_seeding.sql b/examples/suppliers-and-parts/data/hooks/child/before_seeding.sql new file mode 100644 index 0000000..e69de29 diff --git a/examples/suppliers-and-parts/data/hooks/child/metadata.json b/examples/suppliers-and-parts/data/hooks/child/metadata.json new file mode 100644 index 0000000..84d17ba --- /dev/null +++ b/examples/suppliers-and-parts/data/hooks/child/metadata.json @@ -0,0 +1 @@ +{ "inherits": "hooks" } diff --git a/examples/suppliers-and-parts/data/hooks/metadata.json b/examples/suppliers-and-parts/data/hooks/metadata.json new file mode 100644 index 0000000..6ccd29b --- /dev/null +++ b/examples/suppliers-and-parts/data/hooks/metadata.json @@ -0,0 +1 @@ +{ "inherits": "base" } diff --git a/lib/db_agent/seed_folder.rb b/lib/db_agent/seed_folder.rb index 1707f49..6197873 100644 --- a/lib/db_agent/seed_folder.rb +++ b/lib/db_agent/seed_folder.rb @@ -5,8 +5,9 @@ class SeedFolder def initialize(data_folder, seed = 'empty') @data_folder = data_folder @seed = seed + @metadata = (folder(seed)/"metadata.json").load end - attr_reader :data_folder, :seed + attr_reader :data_folder, :seed, :metadata def db_handler data_folder.db_handler @@ -16,9 +17,29 @@ def folder(seed = self.seed) db_handler.data_folder/seed end + def parent + @parent ||= if inherits = metadata["inherits"] + SeedFolder.new(data_folder, inherits) + else + NullObject.new(data_folder) + end + end + + def before_seeding_files + f = (folder/'before_seeding.sql') + fs = f.file? ? [f] : [] + parent.before_seeding_files + fs + end + + def after_seeding_files + f = (folder/'after_seeding.sql') + fs = f.file? ? [f] : [] + parent.after_seeding_files + fs + end + # Returns a Hash[Sequel.qualify(table_name) => Path] def seed_files_per_table - pairs = _seed_files_per_table(seed) + pairs = _seed_files_per_table pairs .keys .sort{|p1,p2| @@ -29,23 +50,30 @@ def seed_files_per_table end end - def _seed_files_per_table(seed) + def _seed_files_per_table folder = self.folder(seed) - data = {} - - # load metadata and install parent dataset if any - metadata = (folder/"metadata.json").load - if parent = metadata["inherits"] - data = _seed_files_per_table(parent) - end + map = parent._seed_files_per_table seed_files(folder).each do |f| - data[file2table(f)] = f + map[file2table(f)] = f end - data + map end - private :_seed_files_per_table + protected :_seed_files_per_table + + class NullObject < SeedFolder + def _seed_files_per_table + {} + end + def before_seeding_files + [] + end + + def after_seeding_files + [] + end + end end # class SeedFolder end # module DbAgent diff --git a/lib/db_agent/seeder.rb b/lib/db_agent/seeder.rb index 25637d1..bd2fa9d 100644 --- a/lib/db_agent/seeder.rb +++ b/lib/db_agent/seeder.rb @@ -9,11 +9,11 @@ def initialize(handler) attr_reader :handler, :data_folder def install(from) - handler.sequel_db.transaction do - before_seeding! + seed_folder = data_folder.seed_folder(from) + seed_files = seed_folder.seed_files_per_table - seed_folder = data_folder.seed_folder(from) - seed_files = seed_folder.seed_files_per_table + handler.sequel_db.transaction do + before_seeding!(seed_folder) # Truncate tables seed_files.keys.reverse.each do |table| @@ -111,17 +111,16 @@ def each_seed(install = true) private - def before_seeding! - file = handler.data_folder/"before_seeding.sql" - return unless file.exists? - - handler.sequel_db.execute(file.read) + def before_seeding!(seed_folder) + seed_folder.before_seeding_files.each do |file| + handler.sequel_db.execute(file.read) + end end - def after_seeding!(seed_folder, folder = seed_folder.folder) - file = folder/"after_seeding.sql" - handler.sequel_db.execute(file.read) if file.exists? - after_seeding!(seed_folder, folder.parent) unless folder == handler.data_folder + def after_seeding!(seed_folder) + seed_folder.after_seeding_files.each do |file| + handler.sequel_db.execute(file.read) + end end def viewpoint diff --git a/spec/test_data_folder.rb b/spec/test_data_folder.rb index 1097597..719ebe5 100644 --- a/spec/test_data_folder.rb +++ b/spec/test_data_folder.rb @@ -14,7 +14,7 @@ module DbAgent }) } - context 'on a singledb' do + context 'on a singledb and base seed' do let(:root) { Path.backfind('.[Gemfile]')/'examples/suppliers-and-parts' } @@ -24,12 +24,60 @@ module DbAgent end it 'helps getting merged_data' do - expect(subject.seed_folder('base').seed_files_per_table).to eql({ + seed_folder = subject.seed_folder('base') + expect(seed_folder.seed_files_per_table).to eql({ :suppliers => root/'data/base/100-suppliers.json', :parts => root/'data/base/200-parts.json', Sequel.qualify(:public, :supplies) => root/'data/base/300-public.supplies.json', }) end + + it 'helps getting before and after seeding files' do + seed_folder = subject.seed_folder('base') + expect(seed_folder.before_seeding_files).to eql([]) + expect(seed_folder.after_seeding_files).to eql([]) + end + end + + context 'on a singledb and hooks seed' do + let(:root) { + Path.backfind('.[Gemfile]')/'examples/suppliers-and-parts' + } + + it 'works' do + expect(subject).to be_a(DataFolder) + end + + it 'helps getting merged_data' do + seed_folder = subject.seed_folder('hooks') + expect(seed_folder.seed_files_per_table).to eql({ + :suppliers => root/'data/base/100-suppliers.json', + :parts => root/'data/base/200-parts.json', + Sequel.qualify(:public, :supplies) => root/'data/base/300-public.supplies.json', + }) + end + + it 'helps getting before and after seeding files' do + seed_folder = subject.seed_folder('hooks') + expect(seed_folder.before_seeding_files).to eql([ + root/'data/hooks/before_seeding.sql', + ]) + expect(seed_folder.after_seeding_files).to eql([ + root/'data/hooks/after_seeding.sql', + ]) + end + + it 'helps getting before and after seeding files recursively' do + seed_folder = subject.seed_folder('hooks/child') + expect(seed_folder.before_seeding_files).to eql([ + root/'data/hooks/before_seeding.sql', + root/'data/hooks/child/before_seeding.sql', + ]) + expect(seed_folder.after_seeding_files).to eql([ + root/'data/hooks/after_seeding.sql', + root/'data/hooks/child/after_seeding.sql', + ]) + end end end end From 046232d42942d86c3f92fa90ec8f94e91cb3e33c Mon Sep 17 00:00:00 2001 From: Bernard Lambeau Date: Sat, 27 Sep 2025 12:10:03 +0200 Subject: [PATCH 3/3] Start support for multiple databases. --- examples/multi-db/Dockerfile | 5 +++ examples/multi-db/README.md | 33 +++++++++++++++ examples/multi-db/data/base/db1/01-todo.json | 8 ++++ .../multi-db/data/base/db1/after_seeding.sql | 0 .../multi-db/data/base/db1/before_seeding.sql | 0 examples/multi-db/data/base/db1/metadata.json | 1 + examples/multi-db/data/empty/db1/01-todo.json | 1 + .../multi-db/data/empty/db1/after_seeding.sql | 0 .../data/empty/db1/before_seeding.sql | 0 .../multi-db/data/empty/db1/metadata.json | 1 + .../multi-db/data/empty/db2/01-users.json | 1 + .../multi-db/data/empty/db2/metadata.json | 1 + examples/multi-db/docker-compose.yml | 41 +++++++++++++++++++ .../multi-db/initdb/multiple-databases.sql | 2 + .../migrations/db1/20250927120800_todo.rb | 11 +++++ .../migrations/db2/20250927120800_users.rb | 11 +++++ lib/db_agent/data_folder.rb | 4 +- lib/db_agent/seed_folder.rb | 18 +++++--- spec/test_data_folder.rb | 30 +++++++++++++- 19 files changed, 159 insertions(+), 9 deletions(-) create mode 100644 examples/multi-db/Dockerfile create mode 100644 examples/multi-db/README.md create mode 100644 examples/multi-db/data/base/db1/01-todo.json create mode 100644 examples/multi-db/data/base/db1/after_seeding.sql create mode 100644 examples/multi-db/data/base/db1/before_seeding.sql create mode 100644 examples/multi-db/data/base/db1/metadata.json create mode 100644 examples/multi-db/data/empty/db1/01-todo.json create mode 100644 examples/multi-db/data/empty/db1/after_seeding.sql create mode 100644 examples/multi-db/data/empty/db1/before_seeding.sql create mode 100644 examples/multi-db/data/empty/db1/metadata.json create mode 100644 examples/multi-db/data/empty/db2/01-users.json create mode 100644 examples/multi-db/data/empty/db2/metadata.json create mode 100644 examples/multi-db/docker-compose.yml create mode 100644 examples/multi-db/initdb/multiple-databases.sql create mode 100644 examples/multi-db/migrations/db1/20250927120800_todo.rb create mode 100644 examples/multi-db/migrations/db2/20250927120800_users.rb diff --git a/examples/multi-db/Dockerfile b/examples/multi-db/Dockerfile new file mode 100644 index 0000000..eebd4d7 --- /dev/null +++ b/examples/multi-db/Dockerfile @@ -0,0 +1,5 @@ +FROM enspirit/dbagent + +COPY initdb /home/app/initdb +COPY migrations /home/app/migrations +COPY data /home/app/data diff --git a/examples/multi-db/README.md b/examples/multi-db/README.md new file mode 100644 index 0000000..de9f5d2 --- /dev/null +++ b/examples/multi-db/README.md @@ -0,0 +1,33 @@ +# Multiple databases managed through DbAgent + +This example provides a basic configuration to maintaing multiple databases +on the same postgresql server. + +## Get started + +``` +docker-compose up +``` + +In another terminal, let migrate & spy the database: + +``` +docker-compose exec dbagent bash +rake db:ping +rake db:migrate +rake db:spy +``` + +You can browse the database schema in a web browser: + +``` +http://127.0.0.1:8080/schema/ +``` + +Let now install some data, then look at it: + +``` +echo "SELECT * FROM todo" | rake db:repl +curl -X POST http://127.0.0.1/seeds/install -d "id=test" +echo "SELECT * FROM todo" | rake db:repl +``` diff --git a/examples/multi-db/data/base/db1/01-todo.json b/examples/multi-db/data/base/db1/01-todo.json new file mode 100644 index 0000000..ab67d4e --- /dev/null +++ b/examples/multi-db/data/base/db1/01-todo.json @@ -0,0 +1,8 @@ +[ + { + "id": 1, + "title": "Write more documenation", + "description": "Get started guide should probably be longer", + "done": false + } +] diff --git a/examples/multi-db/data/base/db1/after_seeding.sql b/examples/multi-db/data/base/db1/after_seeding.sql new file mode 100644 index 0000000..e69de29 diff --git a/examples/multi-db/data/base/db1/before_seeding.sql b/examples/multi-db/data/base/db1/before_seeding.sql new file mode 100644 index 0000000..e69de29 diff --git a/examples/multi-db/data/base/db1/metadata.json b/examples/multi-db/data/base/db1/metadata.json new file mode 100644 index 0000000..bde3048 --- /dev/null +++ b/examples/multi-db/data/base/db1/metadata.json @@ -0,0 +1 @@ +{ "inherits": "empty" } diff --git a/examples/multi-db/data/empty/db1/01-todo.json b/examples/multi-db/data/empty/db1/01-todo.json new file mode 100644 index 0000000..0637a08 --- /dev/null +++ b/examples/multi-db/data/empty/db1/01-todo.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/examples/multi-db/data/empty/db1/after_seeding.sql b/examples/multi-db/data/empty/db1/after_seeding.sql new file mode 100644 index 0000000..e69de29 diff --git a/examples/multi-db/data/empty/db1/before_seeding.sql b/examples/multi-db/data/empty/db1/before_seeding.sql new file mode 100644 index 0000000..e69de29 diff --git a/examples/multi-db/data/empty/db1/metadata.json b/examples/multi-db/data/empty/db1/metadata.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/examples/multi-db/data/empty/db1/metadata.json @@ -0,0 +1 @@ +{} diff --git a/examples/multi-db/data/empty/db2/01-users.json b/examples/multi-db/data/empty/db2/01-users.json new file mode 100644 index 0000000..0637a08 --- /dev/null +++ b/examples/multi-db/data/empty/db2/01-users.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/examples/multi-db/data/empty/db2/metadata.json b/examples/multi-db/data/empty/db2/metadata.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/examples/multi-db/data/empty/db2/metadata.json @@ -0,0 +1 @@ +{} diff --git a/examples/multi-db/docker-compose.yml b/examples/multi-db/docker-compose.yml new file mode 100644 index 0000000..0e4869d --- /dev/null +++ b/examples/multi-db/docker-compose.yml @@ -0,0 +1,41 @@ +version: '2' + +services: + + # The `postgres` agent is simply a PostgreSQL database. This agent may be + # replaced by a real database service, such as AWS RDS, provided the + # configuration of the other agents is adapted below. + # + # We keep the files in a mounted volume to keep the database state accross + # executions. + postgres: + image: postgres:15 + environment: + POSTGRES_USER: postgres + POSTGRES_DB: postgres + POSTGRES_PASSWORD: password + ports: + - 5432:5432 + volumes: + - ./initdb:/docker-entrypoint-initdb.d + - ./volumes/pgdata:/var/lib/postgresql/data + + # The `database` agent is the logical DbAgent tool. The agent is NOT intended to + # be started in production. + # + # Mounted volume is just a handy tool to hack on source code in development + # mode. + dbagent: + build: . + ports: + - 8080:80 + volumes: + - ./backups:/home/app/backups + - ./migrations:/home/app/migrations + - ./data:/home/app/data + - ./schema:/home/app/schema + environment: + DBAGENT_HOST: postgres + DBAGENT_USER: postgres + DBAGENT_DB: postgres + DBAGENT_PASSWORD: password diff --git a/examples/multi-db/initdb/multiple-databases.sql b/examples/multi-db/initdb/multiple-databases.sql new file mode 100644 index 0000000..1a18e9c --- /dev/null +++ b/examples/multi-db/initdb/multiple-databases.sql @@ -0,0 +1,2 @@ +CREATE DATABASE db1; +CREATE DATABASE db2; diff --git a/examples/multi-db/migrations/db1/20250927120800_todo.rb b/examples/multi-db/migrations/db1/20250927120800_todo.rb new file mode 100644 index 0000000..907434c --- /dev/null +++ b/examples/multi-db/migrations/db1/20250927120800_todo.rb @@ -0,0 +1,11 @@ +Sequel.migration do + up do + run <<-SQL + CREATE TABLE todo ( + id SERIAL NOT NULL, + title VARCHAR(255) NOT NULL, + PRIMARY KEY (id) + ); + SQL + end +end diff --git a/examples/multi-db/migrations/db2/20250927120800_users.rb b/examples/multi-db/migrations/db2/20250927120800_users.rb new file mode 100644 index 0000000..ff3176b --- /dev/null +++ b/examples/multi-db/migrations/db2/20250927120800_users.rb @@ -0,0 +1,11 @@ +Sequel.migration do + up do + run <<-SQL + CREATE TABLE users ( + id SERIAL NOT NULL, + name VARCHAR(255) NOT NULL, + PRIMARY KEY (id) + ); + SQL + end +end diff --git a/lib/db_agent/data_folder.rb b/lib/db_agent/data_folder.rb index 29edcd9..ad81eea 100644 --- a/lib/db_agent/data_folder.rb +++ b/lib/db_agent/data_folder.rb @@ -6,8 +6,8 @@ def initialize(db_handler) end attr_reader :db_handler - def seed_folder(seed) - SeedFolder.new(self, seed) + def seed_folder(seed, database = nil) + SeedFolder.new(self, seed, database) end end # class DataFolder diff --git a/lib/db_agent/seed_folder.rb b/lib/db_agent/seed_folder.rb index 6197873..4188c3e 100644 --- a/lib/db_agent/seed_folder.rb +++ b/lib/db_agent/seed_folder.rb @@ -2,24 +2,32 @@ module DbAgent class SeedFolder include SeedUtils - def initialize(data_folder, seed = 'empty') + def initialize(data_folder, seed = 'empty', database = nil) @data_folder = data_folder + @database = database @seed = seed - @metadata = (folder(seed)/"metadata.json").load end - attr_reader :data_folder, :seed, :metadata + attr_reader :data_folder, :seed def db_handler data_folder.db_handler end + def metadata + @metadata ||= (folder(seed)/"metadata.json").load + end + def folder(seed = self.seed) - db_handler.data_folder/seed + if @database + db_handler.data_folder/seed/@database + else + db_handler.data_folder/seed + end end def parent @parent ||= if inherits = metadata["inherits"] - SeedFolder.new(data_folder, inherits) + SeedFolder.new(data_folder, inherits, @database) else NullObject.new(data_folder) end diff --git a/spec/test_data_folder.rb b/spec/test_data_folder.rb index 719ebe5..7fbb554 100644 --- a/spec/test_data_folder.rb +++ b/spec/test_data_folder.rb @@ -23,7 +23,7 @@ module DbAgent expect(subject).to be_a(DataFolder) end - it 'helps getting merged_data' do + it 'helps getting seed files per table' do seed_folder = subject.seed_folder('base') expect(seed_folder.seed_files_per_table).to eql({ :suppliers => root/'data/base/100-suppliers.json', @@ -39,6 +39,32 @@ module DbAgent end end + context 'on a multi-db and base seed' do + let(:root) { + Path.backfind('.[Gemfile]')/'examples/multi-db' + } + + it 'helps getting seed files per table' do + seed_folder = subject.seed_folder('base', 'db1') + expect(seed_folder.seed_files_per_table).to eql({ + :todo => root/'data/base/db1/01-todo.json', + }) + end + + + it 'helps getting before and after seeding files' do + seed_folder = subject.seed_folder('base', 'db1') + expect(seed_folder.before_seeding_files).to eql([ + root/'data/empty/db1/before_seeding.sql', + root/'data/base/db1/before_seeding.sql', + ]) + expect(seed_folder.after_seeding_files).to eql([ + root/'data/empty/db1/after_seeding.sql', + root/'data/base/db1/after_seeding.sql', + ]) + end + end + context 'on a singledb and hooks seed' do let(:root) { Path.backfind('.[Gemfile]')/'examples/suppliers-and-parts' @@ -48,7 +74,7 @@ module DbAgent expect(subject).to be_a(DataFolder) end - it 'helps getting merged_data' do + it 'helps getting seed files per table' do seed_folder = subject.seed_folder('hooks') expect(seed_folder.seed_files_per_table).to eql({ :suppliers => root/'data/base/100-suppliers.json',