From 586dc2b18611efd28988f476cb260a8649d80864 Mon Sep 17 00:00:00 2001 From: Gary Mardell Date: Wed, 28 Jun 2023 14:36:20 -0700 Subject: [PATCH 1/3] Support for bulk importing operations --- spec/avram/insert_spec.cr | 4 +- src/avram/bulk_insert.cr | 75 +++++++++++++++++++++++++++++++++++++ src/avram/insert.cr | 22 +++++++---- src/avram/save_operation.cr | 8 +++- 4 files changed, 97 insertions(+), 12 deletions(-) create mode 100644 src/avram/bulk_insert.cr diff --git a/spec/avram/insert_spec.cr b/spec/avram/insert_spec.cr index c66501fcc..789329c3f 100644 --- a/spec/avram/insert_spec.cr +++ b/spec/avram/insert_spec.cr @@ -4,14 +4,14 @@ describe Avram::Insert do describe "inserting" do it "inserts with a hash of String" do params = {:first_name => "Paul", :last_name => "Smith"} - insert = Avram::Insert.new(table: :users, params: params) + insert = Avram::Insert.new(table: :users, params: [params]) insert.statement.should eq "insert into users(first_name, last_name) values($1, $2) returning *" insert.args.should eq ["Paul", "Smith"] end it "inserts with a hash of Nil" do params = {:first_name => nil} - insert = Avram::Insert.new(table: :users, params: params) + insert = Avram::Insert.new(table: :users, params: [params]) insert.statement.should eq "insert into users(first_name) values($1) returning *" insert.args.should eq [nil] end diff --git a/src/avram/bulk_insert.cr b/src/avram/bulk_insert.cr new file mode 100644 index 000000000..594590d7b --- /dev/null +++ b/src/avram/bulk_insert.cr @@ -0,0 +1,75 @@ +module Avram::BulkInsert(T) + macro included + define_import + + macro inherited + define_import + end + end + + macro define_import + def self.import(operations : Array(self)) + operations.each(&.before_save) + + if operations.all?(&.valid?) + now = Time.utc + + insert_values = operations.map do |operation| + operation.created_at.value ||= now if operation.responds_to?(:created_at) + operation.updated_at.value ||= now if operation.responds_to?(:updated_at) + operation.values + end + + insert_sql = Avram::Insert.new(T.table_name, insert_values, T.column_names) + + transaction_committed = T.database.transaction do + T.database.query insert_sql.statement, args: insert_sql.args do |rs| + T.from_rs(rs).each_with_index do |record, index| + operation = operations[index] + operation.record = record + operation.after_save(record) + end + end + + true + end + + if transaction_committed + operations.each do |operation| + operation.save_status = OperationStatus::Saved + operation.after_commit(operation.record.as(T)) + + Avram::Events::SaveSuccessEvent.publish( + operation_class: self.class.name, + attributes: operation.generic_attributes + ) + end + + true + else + operations.each do |operation| + operation.mark_as_failed + + Avram::Events::SaveFailedEvent.publish( + operation_class: self.class.name, + attributes: operation.generic_attributes + ) + end + + false + end + else + operations.each do |operation| + operation.mark_as_failed + + Avram::Events::SaveFailedEvent.publish( + operation_class: self.class.name, + attributes: operation.generic_attributes + ) + end + + false + end + end + end +end \ No newline at end of file diff --git a/src/avram/insert.cr b/src/avram/insert.cr index c7cb6c923..16a77e87b 100644 --- a/src/avram/insert.cr +++ b/src/avram/insert.cr @@ -1,11 +1,11 @@ class Avram::Insert - alias Params = Hash(Symbol, String) | Hash(Symbol, String?) | Hash(Symbol, Nil) + alias Params = Array(Hash(Symbol, String)) | Array(Hash(Symbol, String?)) | Array(Hash(Symbol, Nil)) def initialize(@table : TableName, @params : Params, @column_names : Array(Symbol) = [] of Symbol) end def statement - "insert into #{@table}(#{fields}) values(#{values_placeholders}) returning #{returning}" + "insert into #{@table}(#{fields}) values #{values_sql_fragment} returning #{returning}" end private def returning : String @@ -17,16 +17,22 @@ class Avram::Insert end def args - @params.values + @params.flat_map(&.values) end private def fields - @params.keys.join(", ") + @params.first.keys.join(", ") end - private def values_placeholders - @params.values.map_with_index do |_value, index| - "$#{index + 1}" - end.join(", ") + private def values_sql_fragment + @params.map_with_index { |params, offset| values_placeholders(params, offset * params.size) }.join(", ") + end + + private def values_placeholders(params, offset = 0) + String.build do |io| + io << "(" + io << params.values.map_with_index { |_v, index| "${offset + index + 1}" }.join(", ") + io << ")" + end end end diff --git a/src/avram/save_operation.cr b/src/avram/save_operation.cr index 26d20e08c..3b38036b6 100644 --- a/src/avram/save_operation.cr +++ b/src/avram/save_operation.cr @@ -23,6 +23,7 @@ abstract class Avram::SaveOperation(T) include Avram::InheritColumnAttributes include Avram::Upsert include Avram::AddColumnAttributes + include Avram::BulkInsert(T) enum OperationStatus Saved @@ -196,6 +197,10 @@ abstract class Avram::SaveOperation(T) attributes_to_hash(column_attributes.select(&.changed?)) end + def values : Hash(Symbol, String?) + attributes_to_hash(column_attributes) + end + macro add_cast_value_methods(columns) private def cast_value(value : Nil) nil @@ -307,8 +312,7 @@ abstract class Avram::SaveOperation(T) end private def insert_sql - insert_values = attributes_to_hash(column_attributes).compact - Avram::Insert.new(table_name, insert_values, T.column_names) + Avram::Insert.new(table_name, [values.compact], T.column_names) end private def attributes_to_hash(attributes) : Hash(Symbol, String?) From 29127adecd7311602e094f816259a58319611993 Mon Sep 17 00:00:00 2001 From: Gary Mardell Date: Wed, 28 Jun 2023 14:53:12 -0700 Subject: [PATCH 2/3] Properly interpolate index --- src/avram/insert.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/avram/insert.cr b/src/avram/insert.cr index 16a77e87b..e62418882 100644 --- a/src/avram/insert.cr +++ b/src/avram/insert.cr @@ -31,7 +31,7 @@ class Avram::Insert private def values_placeholders(params, offset = 0) String.build do |io| io << "(" - io << params.values.map_with_index { |_v, index| "${offset + index + 1}" }.join(", ") + io << params.values.map_with_index { |_v, index| "$#{offset + index + 1}" }.join(", ") io << ")" end end From 17fa3dea31823e9b3e0a07a9cba92aef6577bfdc Mon Sep 17 00:00:00 2001 From: Gary Mardell Date: Wed, 28 Jun 2023 16:59:38 -0700 Subject: [PATCH 3/3] Fix specs with a space between values --- spec/avram/insert_spec.cr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spec/avram/insert_spec.cr b/spec/avram/insert_spec.cr index 789329c3f..9d50d4f09 100644 --- a/spec/avram/insert_spec.cr +++ b/spec/avram/insert_spec.cr @@ -5,14 +5,14 @@ describe Avram::Insert do it "inserts with a hash of String" do params = {:first_name => "Paul", :last_name => "Smith"} insert = Avram::Insert.new(table: :users, params: [params]) - insert.statement.should eq "insert into users(first_name, last_name) values($1, $2) returning *" + insert.statement.should eq "insert into users(first_name, last_name) values ($1, $2) returning *" insert.args.should eq ["Paul", "Smith"] end it "inserts with a hash of Nil" do params = {:first_name => nil} insert = Avram::Insert.new(table: :users, params: [params]) - insert.statement.should eq "insert into users(first_name) values($1) returning *" + insert.statement.should eq "insert into users(first_name) values ($1) returning *" insert.args.should eq [nil] end end