From 626165b68bee020a81770aef44a2364e85209e8a Mon Sep 17 00:00:00 2001 From: Bernard Lambeau Date: Sat, 27 Sep 2025 10:26:21 +0200 Subject: [PATCH 1/4] Extract file discovery logic to DataFolder and SeedFolder. --- Makefile | 1 + lib/db_agent.rb | 3 ++ lib/db_agent/data_folder.rb | 14 +++++++++ lib/db_agent/seed_folder.rb | 51 ++++++++++++++++++++++++++++++ lib/db_agent/seed_utils.rb | 20 ++++++++++++ lib/db_agent/seeder.rb | 63 ++++++------------------------------- spec/test_data_folder.rb | 35 +++++++++++++++++++++ 7 files changed, 134 insertions(+), 53 deletions(-) create mode 100644 lib/db_agent/data_folder.rb create mode 100644 lib/db_agent/seed_folder.rb create mode 100644 lib/db_agent/seed_utils.rb create mode 100644 spec/test_data_folder.rb diff --git a/Makefile b/Makefile index 4a31123..44a0a7f 100644 --- a/Makefile +++ b/Makefile @@ -29,6 +29,7 @@ exec_test: docker exec -t dbagent bundle exec rake test docker exec -t dbagent bundle exec rake db:wait db:ping docker exec -t dbagent bundle exec rake db:migrate + docker exec -t dbagent bundle exec rake db:check-seeds docker exec -t dbagent bundle exec rake db:seed[base] docker exec -t dbagent bundle exec rake db:insert_script[base] docker exec -t dbagent bundle exec rake db:flush[tmp] diff --git a/lib/db_agent.rb b/lib/db_agent.rb index 9d99b19..a91a515 100644 --- a/lib/db_agent.rb +++ b/lib/db_agent.rb @@ -74,6 +74,9 @@ def self.default_handler end # module DbAgent require 'db_agent/viewpoint' +require 'db_agent/seed_utils' +require 'db_agent/data_folder' +require 'db_agent/seed_folder' require 'db_agent/seeder' require 'db_agent/table_orderer' require 'db_agent/db_handler' diff --git a/lib/db_agent/data_folder.rb b/lib/db_agent/data_folder.rb new file mode 100644 index 0000000..29edcd9 --- /dev/null +++ b/lib/db_agent/data_folder.rb @@ -0,0 +1,14 @@ +module DbAgent + class DataFolder + + def initialize(db_handler) + @db_handler = db_handler + end + attr_reader :db_handler + + def seed_folder(seed) + SeedFolder.new(self, seed) + end + + end # class DataFolder +end # module DbAgent diff --git a/lib/db_agent/seed_folder.rb b/lib/db_agent/seed_folder.rb new file mode 100644 index 0000000..1707f49 --- /dev/null +++ b/lib/db_agent/seed_folder.rb @@ -0,0 +1,51 @@ +module DbAgent + class SeedFolder + include SeedUtils + + def initialize(data_folder, seed = 'empty') + @data_folder = data_folder + @seed = seed + end + attr_reader :data_folder, :seed + + def db_handler + data_folder.db_handler + end + + def folder(seed = self.seed) + db_handler.data_folder/seed + end + + # Returns a Hash[Sequel.qualify(table_name) => Path] + def seed_files_per_table + pairs = _seed_files_per_table(seed) + pairs + .keys + .sort{|p1,p2| + pairs[p1].basename <=> pairs[p2].basename + } + .each_with_object({}) do |name,index| + index[qualify_table(name)] = pairs[name] + end + end + + def _seed_files_per_table(seed) + folder = self.folder(seed) + data = {} + + # load metadata and install parent dataset if any + metadata = (folder/"metadata.json").load + if parent = metadata["inherits"] + data = _seed_files_per_table(parent) + end + + seed_files(folder).each do |f| + data[file2table(f)] = f + end + + data + end + private :_seed_files_per_table + + end # class SeedFolder +end # module DbAgent diff --git a/lib/db_agent/seed_utils.rb b/lib/db_agent/seed_utils.rb new file mode 100644 index 0000000..410c1da --- /dev/null +++ b/lib/db_agent/seed_utils.rb @@ -0,0 +1,20 @@ +module DbAgent + module SeedUtils + + def seed_files(folder) + folder + .glob("*.json") + .reject{|f| f.basename.to_s =~ /^metadata/ } + end + + def file2table(file) + file.basename.rm_ext.to_s[/^\d+-(.*)/, 1] + end + + def qualify_table(table_name) + parts = table_name.to_s.split('.') + parts.size === 2 ? Sequel.qualify(*parts.map(&:to_sym)) : table_name.to_sym + end + + end # module SeedUtils +end # module DbAgent diff --git a/lib/db_agent/seeder.rb b/lib/db_agent/seeder.rb index 0af107f..d2c7264 100644 --- a/lib/db_agent/seeder.rb +++ b/lib/db_agent/seeder.rb @@ -1,19 +1,21 @@ module DbAgent class Seeder + include SeedUtils def initialize(handler) @handler = handler + @data_folder = DataFolder.new(handler) end - attr_reader :handler + attr_reader :handler, :data_folder def install(from) handler.sequel_db.transaction do before_seeding! - folder = handler.data_folder/from + seed_folder = data_folder.seed_folder(from) # load files in order - pairs = merged_data(from) + pairs = seed_folder.seed_files_per_table # Truncate tables pairs.keys.reverse.each do |table| @@ -31,15 +33,15 @@ def install(from) handler.sequel_db[table].multi_insert(data) end - after_seeding!(folder) + after_seeding!(seed_folder) end end def insert_script(from) - folder = handler.data_folder/from + seed_folder = data_folder.seed_folder(from) # load files in order - pairs = merged_data(from) + pairs = seed_folder.seed_files_per_table # Fill them pairs.keys.each do |table| @@ -120,55 +122,10 @@ def before_seeding! handler.sequel_db.execute(file.read) end - def after_seeding!(folder) + def after_seeding!(seed_folder, folder = seed_folder.folder) file = folder/"after_seeding.sql" handler.sequel_db.execute(file.read) if file.exists? - after_seeding!(folder.parent) unless folder == handler.data_folder - end - - # Returns a Hash[Sequel.qualify(table_name) => Path] - def merged_data(from) - pairs = _merged_data(from) - pairs - .keys - .sort{|p1,p2| - pairs[p1].basename <=> pairs[p2].basename - } - .each_with_object({}) do |name,index| - index[qualify_table(name)] = pairs[name] - end - end - - def _merged_data(from) - folder = handler.data_folder/from - data = {} - - # load metadata and install parent dataset if any - metadata = (folder/"metadata.json").load - if parent = metadata["inherits"] - data = _merged_data(parent) - end - - seed_files(folder).each do |f| - data[file2table(f)] = f - end - - data - end - - def seed_files(folder) - folder - .glob("*.json") - .reject{|f| f.basename.to_s =~ /^metadata/ } - end - - def file2table(f) - f.basename.rm_ext.to_s[/^\d+-(.*)/, 1] - end - - def qualify_table(table_name) - parts = table_name.to_s.split('.') - parts.size === 2 ? Sequel.qualify(*parts.map(&:to_sym)) : table_name.to_sym + after_seeding!(seed_folder, folder.parent) unless folder == handler.data_folder end def viewpoint diff --git a/spec/test_data_folder.rb b/spec/test_data_folder.rb new file mode 100644 index 0000000..1097597 --- /dev/null +++ b/spec/test_data_folder.rb @@ -0,0 +1,35 @@ +require 'spec_helper' + +module DbAgent + describe DataFolder do + + subject { + DataFolder.new(db_handler) + } + + let(:db_handler) { + DbHandler.new({ + config: {}, + root: root, + }) + } + + context 'on a singledb' do + let(:root) { + Path.backfind('.[Gemfile]')/'examples/suppliers-and-parts' + } + + it 'works' do + expect(subject).to be_a(DataFolder) + end + + it 'helps getting merged_data' do + expect(subject.seed_folder('base').seed_files_per_table).to eql({ + :suppliers => root/'data/base/100-suppliers.json', + :parts => root/'data/base/200-parts.json', + Sequel.qualify(:public, :supplies) => root/'data/base/300-public.supplies.json', + }) + end + end + end +end From 1172bca8f6b6800da18b06e3205ecaa494214ea9 Mon Sep 17 00:00:00 2001 From: Bernard Lambeau Date: Sat, 27 Sep 2025 10:28:43 +0200 Subject: [PATCH 2/4] fixup! Extract file discovery logic to DataFolder and SeedFolder. --- lib/db_agent/seeder.rb | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/lib/db_agent/seeder.rb b/lib/db_agent/seeder.rb index d2c7264..25637d1 100644 --- a/lib/db_agent/seeder.rb +++ b/lib/db_agent/seeder.rb @@ -13,20 +13,18 @@ def install(from) before_seeding! seed_folder = data_folder.seed_folder(from) - - # load files in order - pairs = seed_folder.seed_files_per_table + seed_files = seed_folder.seed_files_per_table # Truncate tables - pairs.keys.reverse.each do |table| + seed_files.keys.reverse.each do |table| LOGGER.info("Emptying table `#{table}`") handler.sequel_db[table].delete end # Fill them - pairs.keys.each do |table| + seed_files.keys.each do |table| LOGGER.info("Filling table `#{table}`") - file = pairs[table] + file = seed_files[table] data = file.load raise "Empty file: #{file}" if data.nil? @@ -39,13 +37,11 @@ def install(from) def insert_script(from) seed_folder = data_folder.seed_folder(from) - - # load files in order - pairs = seed_folder.seed_files_per_table + seed_files = seed_folder.seed_files_per_table # Fill them - pairs.keys.each do |table| - file = pairs[table] + seed_files.keys.each do |table| + file = seed_files[table] data = file.load next if data.empty? From 2cbc9cec2b17a054fa7734c5ebff0f7fe4d36d40 Mon Sep 17 00:00:00 2001 From: Bernard Lambeau Date: Sat, 27 Sep 2025 11:05:19 +0200 Subject: [PATCH 3/4] Fix test suite. --- .gitignore | 3 ++- Gemfile.lock | 2 +- Makefile | 34 ++++++++++++++++++++-------------- dbagent.gemspec | 2 +- tasks/test.rake | 9 +++++---- 5 files changed, 29 insertions(+), 21 deletions(-) diff --git a/.gitignore b/.gitignore index c143629..6513cef 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,5 @@ examples/suppliers-and-parts/data/tmp/ examples/suppliers-and-parts/data/incity/ examples/suppliers-and-parts/data/new_empty/ examples/suppliers-and-parts/backups/*.sql -pkg/* \ No newline at end of file +pkg/* +/.volumes diff --git a/Gemfile.lock b/Gemfile.lock index 6e22d21..81b263e 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -75,7 +75,7 @@ DEPENDENCIES bundler (~> 2) dbagent! rack-test (~> 1) - rspec (~> 3) + rspec (~> 3.6) BUNDLED WITH 2.4.6 diff --git a/Makefile b/Makefile index 44a0a7f..283cdfe 100644 --- a/Makefile +++ b/Makefile @@ -19,30 +19,36 @@ image.push: image docker push enspirit/dbagent:${DOCKER_TAG} prepare: image + rm -rf .volumes docker network create agent-network || true - docker run -d --rm --name db -v db-data:/var/lib/postgresql/data --env POSTGRES_USER=dbagent --env POSTGRES_DB=suppliers-and-parts --env POSTGRES_PASSWORD=dbagent --network=agent-network --user $(id -u):$(id -g) postgres:15 - docker run -d --rm --name dbagent --env DBAGENT_HOST=db -v$(PWD)/lib:/home/data/lib -v $(DATA_PATH)/data:/home/app/data -v $(DATA_PATH)/migrations:/home/app/migrations -v $(DATA_PATH)/backups:/home/app/backups -v $(DATA_PATH)/viewpoints:/home/app/viewpoints -v $(PWD)/tasks:/home/app/tasks -v $(PWD)/lib:/home/app/lib --network=agent-network --user $(id -u):$(id -g) enspirit/dbagent + docker run -d --rm --name db -v ./.volumes/pgdata:/var/lib/postgresql/data --env POSTGRES_USER=dbagent --env POSTGRES_DB=suppliers-and-parts --env POSTGRES_PASSWORD=dbagent --network=agent-network postgres:15 + docker run -d --rm --name dbagent --env DBAGENT_HOST=db --env DBAGENT_ROOT_FOLDER=/home/app/examples/suppliers-and-parts -v $(PWD)/examples:/home/app/examples -v $(PWD)/lib:/home/app/lib -v $(PWD)/spec:/home/app/spec -v $(PWD)/tasks:/home/app/tasks --network=agent-network enspirit/dbagent docker exec -t dbagent bundle install docker ps exec_test: - docker exec -t dbagent bundle exec rake test - docker exec -t dbagent bundle exec rake db:wait db:ping - docker exec -t dbagent bundle exec rake db:migrate - docker exec -t dbagent bundle exec rake db:check-seeds - docker exec -t dbagent bundle exec rake db:seed[base] - docker exec -t dbagent bundle exec rake db:insert_script[base] - docker exec -t dbagent bundle exec rake db:flush[tmp] - docker exec -e DBAGENT_VIEWPOINT=DbAgent::Viewpoint::InCity -t dbagent bundle exec rake db:flush[incity] - docker exec -t dbagent bundle exec rake db:flush_empty[new_empty] - docker exec -t dbagent bundle exec rake db:spy - docker exec -t dbagent bundle exec rake db:backup + docker exec -t dbagent sh -c '\ + set -e; \ + bundle exec rake db:wait db:ping; \ + bundle exec rake db:migrate; \ + bundle exec rake test; \ + bundle exec rake db:check-seeds; \ + bundle exec rake db:seed[base]; \ + bundle exec rake db:insert_script[base]; \ + bundle exec rake db:flush[tmp]; \ + bundle exec rake db:flush_empty[new_empty]; \ + bundle exec rake db:spy; \ + bundle exec rake db:backup; \ + ' + +clean: + rm -rf examples/suppliers-and-parts/data/new_empty examples/suppliers-and-parts/data/tmp down: docker stop db dbagent || true docker network rm agent-network || true -test: prepare exec_test down +test: down prepare exec_test clean down package: bundle install diff --git a/dbagent.gemspec b/dbagent.gemspec index a62af0c..dd38f55 100644 --- a/dbagent.gemspec +++ b/dbagent.gemspec @@ -26,6 +26,6 @@ Gem::Specification.new do |s| s.add_dependency 'base64', '~> 0.3' s.add_development_dependency 'bundler', '~> 2' - s.add_development_dependency 'rspec', '~> 3' + s.add_development_dependency 'rspec', '~> 3.6' s.add_development_dependency 'rack-test', '~> 1' end diff --git a/tasks/test.rake b/tasks/test.rake index 1c53d0e..0aa2793 100644 --- a/tasks/test.rake +++ b/tasks/test.rake @@ -1,9 +1,10 @@ +require "rspec/core/rake_task" namespace :test do - desc %q{Run all RSpec tests} - task :unit do - require 'rspec' - RSpec::Core::Runner.run(%w[-I. -Ilib -Ispec --pattern=spec/**/test_*.rb --color .]) + desc "Runs unit tests" + RSpec::Core::RakeTask.new(:unit) do |t| + t.pattern = "spec/**/test_*.rb" + t.rspec_opts = ["-Ilib", "-Ispec", "--color", "--backtrace", "--format=progress"] end task :all => :"unit" From 118b638ce8a95d04d1744464867cc120fbbc6e47 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 27 Sep 2025 09:06:17 +0000 Subject: [PATCH 4/4] Bump rack from 2.2.8.1 to 2.2.18 Bumps [rack](https://github.com/rack/rack) from 2.2.8.1 to 2.2.18. - [Release notes](https://github.com/rack/rack/releases) - [Changelog](https://github.com/rack/rack/blob/main/CHANGELOG.md) - [Commits](https://github.com/rack/rack/compare/v2.2.8.1...v2.2.18) --- updated-dependencies: - dependency-name: rack dependency-version: 2.2.18 dependency-type: indirect ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 81b263e..1e362d8 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -33,7 +33,7 @@ GEM predicate (2.8.0) minitest (>= 5.0) sexpr (~> 1.1) - rack (2.2.8.1) + rack (2.2.18) rack-protection (2.2.4) rack rack-test (1.1.0)