Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 3 additions & 4 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -36,19 +36,18 @@ exec_test:
bundle exec rake db:seed[base]; \
bundle exec rake db:insert_script[base]; \
bundle exec rake db:flush[tmp]; \
rm -rf examples/suppliers-and-parts/data/tmp
bundle exec rake db:flush_empty[new_empty]; \
rm -rf examples/suppliers-and-parts/data/new_empty; \
bundle exec rake db:spy; \
bundle exec rake db:backup; \
'

clean:
rm -rf examples/suppliers-and-parts/data/new_empty examples/suppliers-and-parts/data/tmp

down:
docker stop db dbagent || true
docker network rm agent-network || true

test: down prepare exec_test clean down
test: down prepare exec_test down

package:
bundle install
Expand Down
2 changes: 1 addition & 1 deletion docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,4 +11,4 @@ services:
POSTGRES_PASSWORD: dbagent
POSTGRES_DB: suppliers-and-parts
volumes:
- ./volumes/pgdata:/var/lib/postgresql/data
- .volumes/pgdata:/var/lib/postgresql/data
5 changes: 5 additions & 0 deletions examples/multi-db/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
FROM enspirit/dbagent

COPY initdb /home/app/initdb
COPY migrations /home/app/migrations
COPY data /home/app/data
33 changes: 33 additions & 0 deletions examples/multi-db/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
# Multiple databases managed through DbAgent

This example provides a basic configuration to maintaing multiple databases
on the same postgresql server.

## Get started

```
docker-compose up
```

In another terminal, let migrate & spy the database:

```
docker-compose exec dbagent bash
rake db:ping
rake db:migrate
rake db:spy
```

You can browse the database schema in a web browser:

```
http://127.0.0.1:8080/schema/
```

Let now install some data, then look at it:

```
echo "SELECT * FROM todo" | rake db:repl
curl -X POST http://127.0.0.1/seeds/install -d "id=test"
echo "SELECT * FROM todo" | rake db:repl
```
8 changes: 8 additions & 0 deletions examples/multi-db/data/base/db1/01-todo.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
[
{
"id": 1,
"title": "Write more documenation",
"description": "Get started guide should probably be longer",
"done": false
}
]
Empty file.
Empty file.
1 change: 1 addition & 0 deletions examples/multi-db/data/base/db1/metadata.json
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "inherits": "empty" }
1 change: 1 addition & 0 deletions examples/multi-db/data/empty/db1/01-todo.json
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
[]
Empty file.
Empty file.
1 change: 1 addition & 0 deletions examples/multi-db/data/empty/db1/metadata.json
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{}
1 change: 1 addition & 0 deletions examples/multi-db/data/empty/db2/01-users.json
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
[]
1 change: 1 addition & 0 deletions examples/multi-db/data/empty/db2/metadata.json
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{}
41 changes: 41 additions & 0 deletions examples/multi-db/docker-compose.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
version: '2'

services:

# The `postgres` agent is simply a PostgreSQL database. This agent may be
# replaced by a real database service, such as AWS RDS, provided the
# configuration of the other agents is adapted below.
#
# We keep the files in a mounted volume to keep the database state accross
# executions.
postgres:
image: postgres:15
environment:
POSTGRES_USER: postgres
POSTGRES_DB: postgres
POSTGRES_PASSWORD: password
ports:
- 5432:5432
volumes:
- ./initdb:/docker-entrypoint-initdb.d
- ./volumes/pgdata:/var/lib/postgresql/data

# The `database` agent is the logical DbAgent tool. The agent is NOT intended to
# be started in production.
#
# Mounted volume is just a handy tool to hack on source code in development
# mode.
dbagent:
build: .
ports:
- 8080:80
volumes:
- ./backups:/home/app/backups
- ./migrations:/home/app/migrations
- ./data:/home/app/data
- ./schema:/home/app/schema
environment:
DBAGENT_HOST: postgres
DBAGENT_USER: postgres
DBAGENT_DB: postgres
DBAGENT_PASSWORD: password
2 changes: 2 additions & 0 deletions examples/multi-db/initdb/multiple-databases.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
CREATE DATABASE db1;
CREATE DATABASE db2;
11 changes: 11 additions & 0 deletions examples/multi-db/migrations/db1/20250927120800_todo.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
Sequel.migration do
up do
run <<-SQL
CREATE TABLE todo (
id SERIAL NOT NULL,
title VARCHAR(255) NOT NULL,
PRIMARY KEY (id)
);
SQL
end
end
11 changes: 11 additions & 0 deletions examples/multi-db/migrations/db2/20250927120800_users.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
Sequel.migration do
up do
run <<-SQL
CREATE TABLE users (
id SERIAL NOT NULL,
name VARCHAR(255) NOT NULL,
PRIMARY KEY (id)
);
SQL
end
end
2 changes: 1 addition & 1 deletion examples/suppliers-and-parts/data/base/metadata.json
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{}
{ "inherits": "empty" }
Empty file.
Empty file.
Empty file.
Empty file.
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "inherits": "hooks" }
1 change: 1 addition & 0 deletions examples/suppliers-and-parts/data/hooks/metadata.json
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "inherits": "base" }
4 changes: 2 additions & 2 deletions lib/db_agent/data_folder.rb
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,8 @@ def initialize(db_handler)
end
attr_reader :db_handler

def seed_folder(seed)
SeedFolder.new(self, seed)
def seed_folder(seed, database = nil)
SeedFolder.new(self, seed, database)
end

end # class DataFolder
Expand Down
64 changes: 50 additions & 14 deletions lib/db_agent/seed_folder.rb
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,9 @@ module DbAgent
class SeedFolder
include SeedUtils

def initialize(data_folder, seed = 'empty')
def initialize(data_folder, seed = 'empty', database = nil)
@data_folder = data_folder
@database = database
@seed = seed
end
attr_reader :data_folder, :seed
Expand All @@ -12,13 +13,41 @@ def db_handler
data_folder.db_handler
end

def metadata
@metadata ||= (folder(seed)/"metadata.json").load
end

def folder(seed = self.seed)
db_handler.data_folder/seed
if @database
db_handler.data_folder/seed/@database
else
db_handler.data_folder/seed
end
end

def parent
@parent ||= if inherits = metadata["inherits"]
SeedFolder.new(data_folder, inherits, @database)
else
NullObject.new(data_folder)
end
end

def before_seeding_files
f = (folder/'before_seeding.sql')
fs = f.file? ? [f] : []
parent.before_seeding_files + fs
end

def after_seeding_files
f = (folder/'after_seeding.sql')
fs = f.file? ? [f] : []
parent.after_seeding_files + fs
end

# Returns a Hash[Sequel.qualify(table_name) => Path]
def seed_files_per_table
pairs = _seed_files_per_table(seed)
pairs = _seed_files_per_table
pairs
.keys
.sort{|p1,p2|
Expand All @@ -29,23 +58,30 @@ def seed_files_per_table
end
end

def _seed_files_per_table(seed)
def _seed_files_per_table
folder = self.folder(seed)
data = {}

# load metadata and install parent dataset if any
metadata = (folder/"metadata.json").load
if parent = metadata["inherits"]
data = _seed_files_per_table(parent)
end
map = parent._seed_files_per_table

seed_files(folder).each do |f|
data[file2table(f)] = f
map[file2table(f)] = f
end

data
map
end
private :_seed_files_per_table
protected :_seed_files_per_table

class NullObject < SeedFolder
def _seed_files_per_table
{}
end

def before_seeding_files
[]
end

def after_seeding_files
[]
end
end
end # class SeedFolder
end # module DbAgent
25 changes: 12 additions & 13 deletions lib/db_agent/seeder.rb
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,11 @@ def initialize(handler)
attr_reader :handler, :data_folder

def install(from)
handler.sequel_db.transaction do
before_seeding!
seed_folder = data_folder.seed_folder(from)
seed_files = seed_folder.seed_files_per_table

seed_folder = data_folder.seed_folder(from)
seed_files = seed_folder.seed_files_per_table
handler.sequel_db.transaction do
before_seeding!(seed_folder)

# Truncate tables
seed_files.keys.reverse.each do |table|
Expand Down Expand Up @@ -111,17 +111,16 @@ def each_seed(install = true)

private

def before_seeding!
file = handler.data_folder/"before_seeding.sql"
return unless file.exists?

handler.sequel_db.execute(file.read)
def before_seeding!(seed_folder)
seed_folder.before_seeding_files.each do |file|
handler.sequel_db.execute(file.read)
end
end

def after_seeding!(seed_folder, folder = seed_folder.folder)
file = folder/"after_seeding.sql"
handler.sequel_db.execute(file.read) if file.exists?
after_seeding!(seed_folder, folder.parent) unless folder == handler.data_folder
def after_seeding!(seed_folder)
seed_folder.after_seeding_files.each do |file|
handler.sequel_db.execute(file.read)
end
end

def viewpoint
Expand Down
Loading