From 5359d97556f573fe888571de7ee1b0075c055b71 Mon Sep 17 00:00:00 2001 From: Daniel Woelfel Date: Tue, 21 Jan 2025 15:27:10 -0800 Subject: [PATCH 1/7] use merge-into --- server/src/instant/db/model/triple.clj | 26 ++++-- server/src/instant/jdbc/sql.clj | 110 +++++++++++++++++++++++++ 2 files changed, 129 insertions(+), 7 deletions(-) diff --git a/server/src/instant/db/model/triple.clj b/server/src/instant/db/model/triple.clj index 84e1008425..4836f32846 100644 --- a/server/src/instant/db/model/triple.clj +++ b/server/src/instant/db/model/triple.clj @@ -269,6 +269,13 @@ :select :entity-id :from :ea-index-inserts}] (sql/do-execute! conn (hsql/format q)))) +;; Helper for :when-not-matched in `insert-multi +(def triple-t-values + (zipmap triple-cols + (map (fn [col] + (keyword (str "t." (name col)))) + triple-cols))) + (defn insert-multi! "Given a set of raw triples, we enhance each triple with metadata based on the triple's underlying attr and then insert these enhanced triples into @@ -430,13 +437,18 @@ [:remaining-triples {:select :* :from :enhanced-triples :where [:not :ea]}] [:ea-index-inserts - {:insert-into [[:triples triple-cols] - {:select triple-cols - :from :ea-triples-distinct}] - :on-conflict [:app-id :entity-id :attr-id {:where [:= :ea true]}] - :do-update-set {:value :excluded.value - :value-md5 :excluded.value-md5} - :returning :entity-id}] + {:merge-into :triples + :using [[:ea-triples-distinct :t]] + :on [:and + [:= :triples.app-id :t.app-id] + [:= :triples.entity-id :t.entity-id] + [:= :triples.attr-id :t.attr-id]] + :when-not-matched {:insert {:values [triple-t-values]}} + :when-matched [[[:= :triples.value-md5 :t.value-md5] + :do-nothing] + {:update {:set {:value :t.value + :value-md5 :t.value-md5}}}] + :returning :triples.entity-id}] [:remaining-inserts {:insert-into [[:triples triple-cols] {:select triple-cols diff --git a/server/src/instant/jdbc/sql.clj b/server/src/instant/jdbc/sql.clj index a9a4061c74..0baed23c73 100644 --- a/server/src/instant/jdbc/sql.clj +++ b/server/src/instant/jdbc/sql.clj @@ -3,6 +3,7 @@ [clojure.string :as string] ;; load all pg-ops for hsql [honey.sql.pg-ops] + [honey.sql] [instant.util.exception :as ex] [instant.util.io :as io] [instant.util.json :refer [->json <-json]] @@ -19,6 +20,115 @@ (java.time Instant LocalDate LocalDateTime) (org.postgresql.util PGobject PSQLException))) +(defn format-match-cond [expr] + (let [[sql & params] (honey.sql/format-expr expr)] + (when-not (string/blank? sql) + (into [(str "AND " sql)] params)))) + +(defn matched-merge-action? [x] + (case x + :do-nothing :do-nothing + :delete :delete + (and (map? x) + (or (:update x) + (:insert x))))) + +(defn format-matched-merge-action [x] + (case x + :do-nothing ["DO NOTHING"] + :delete ["DELETE"] + (cond (:update x) + (let [[sql & params] (honey.sql/format-dsl (:update x))] + (into [(str "UPDATE " sql)] params)) + + (:insert x) + (let [[sql & params] (honey.sql/format-dsl (:insert x))] + (into [(str "INSERT " sql)] params))))) + +(defn matched-cases [exp] + (cond + ;; just a single action with no condition + (matched-merge-action? exp) + [exp] + + ;; single action with condition + (and (vector? exp) + (= 2 (count exp)) + (not (vector? (first exp))) + (not (matched-merge-action? (first exp))) + (matched-merge-action? (second exp))) + [exp] + + ;; A list of condition+action + (vector? exp) + (do (println "vec") + exp) + + :else + (throw (Exception. "Unknown match clauses for :when-matched/:when-not-matched")))) + +(honey.sql/register-clause! + :when-not-matched + (fn [clause exp] + (let [exprs (matched-cases exp)] + (reduce (fn [[sql & params] expr] + (if (matched-merge-action? expr) + (let [[expr-sql & expr-params] (format-matched-merge-action expr)] + (into [(str sql " WHEN NOT MATCHED THEN " expr-sql)] + (into params expr-params))) + (do + (assert (and (vector? expr) + (matched-merge-action? (second expr))) + "Invalid match clause for :when-not-matched") + (let [[cond-expr action-expr] expr] + (let [[case-sql & case-params] (format-matched-cond cond-expr) + [expr-sql & expr-params] (format-matched-merge-action action-expr)] + (into [(str sql " WHEN NOT MATCHED " case-sql " THEN " expr-sql)] + (concat params case-params expr-params))))) + + )) + [""] + exprs))) + ;; Get behind of :using + :join-by) + +(honey.sql/register-clause! + :when-matched + (fn [clause exp] + (let [exprs (matched-cases exp)] + (reduce (fn [[sql & params] expr] + (if (matched-merge-action? expr) + (let [[expr-sql & expr-params] (format-matched-merge-action expr)] + (into [(str sql " WHEN MATCHED THEN " expr-sql)] + (into params expr-params))) + (do + (assert (and (vector? expr) + (matched-merge-action? (second expr))) + "Invalid match clause for :when-matched") + (let [[cond-expr action-expr] expr] + (let [[case-sql & case-params] (format-matched-cond cond-expr) + [expr-sql & expr-params] (format-matched-merge-action action-expr)] + (into [(str sql " WHEN MATCHED " case-sql " THEN " expr-sql)] + (concat params case-params expr-params))))) + + )) + [""] + exprs))) + :when-not-matched) + +(honey.sql/register-clause! + :on + (fn [clause expr] + (let [[sql & params] (honey.sql/format-expr expr)] + (into [(str "ON " sql)] params))) + :when-matched) + +(honey.sql/register-clause! + :merge-into + (fn [clause tbl] + [(str "MERGE INTO " (honey.sql/format-entity tbl))]) + :using) + (defn ->pg-text-array "Formats as text[] in pg, i.e. {item-1, item-2, item3}" [col] From 4ee9f88cc44582ddbeca3336429d6bc65773f5f5 Mon Sep 17 00:00:00 2001 From: Daniel Woelfel Date: Tue, 21 Jan 2025 15:35:15 -0800 Subject: [PATCH 2/7] fix lint --- server/src/instant/jdbc/sql.clj | 93 ++++++++++++++++++--------------- 1 file changed, 50 insertions(+), 43 deletions(-) diff --git a/server/src/instant/jdbc/sql.clj b/server/src/instant/jdbc/sql.clj index 0baed23c73..bf6cc3bda4 100644 --- a/server/src/instant/jdbc/sql.clj +++ b/server/src/instant/jdbc/sql.clj @@ -20,7 +20,13 @@ (java.time Instant LocalDate LocalDateTime) (org.postgresql.util PGobject PSQLException))) -(defn format-match-cond [expr] + +;; -------------------- +;; honey.sql extensions + +;; :merge-into + +(defn format-matched-cond [expr] (let [[sql & params] (honey.sql/format-expr expr)] (when-not (string/blank? sql) (into [(str "AND " sql)] params)))) @@ -69,66 +75,67 @@ (honey.sql/register-clause! :when-not-matched - (fn [clause exp] - (let [exprs (matched-cases exp)] - (reduce (fn [[sql & params] expr] - (if (matched-merge-action? expr) - (let [[expr-sql & expr-params] (format-matched-merge-action expr)] - (into [(str sql " WHEN NOT MATCHED THEN " expr-sql)] - (into params expr-params))) - (do - (assert (and (vector? expr) - (matched-merge-action? (second expr))) - "Invalid match clause for :when-not-matched") - (let [[cond-expr action-expr] expr] - (let [[case-sql & case-params] (format-matched-cond cond-expr) - [expr-sql & expr-params] (format-matched-merge-action action-expr)] - (into [(str sql " WHEN NOT MATCHED " case-sql " THEN " expr-sql)] - (concat params case-params expr-params))))) - - )) - [""] - exprs))) + (fn [_clause exp] + (reduce (fn [[sql & params] expr] + (if (matched-merge-action? expr) + (let [[expr-sql & expr-params] (format-matched-merge-action expr)] + (into [(str sql " WHEN NOT MATCHED THEN " expr-sql)] + (into params expr-params))) + (do + (assert (and (vector? expr) + (matched-merge-action? (second expr))) + "Invalid match clause for :when-not-matched") + (let [[cond-expr action-expr] expr + [case-sql & case-params] (format-matched-cond cond-expr) + [expr-sql & expr-params] (format-matched-merge-action action-expr)] + (into [(str sql " WHEN NOT MATCHED " case-sql " THEN " expr-sql)] + (concat params case-params expr-params)))) + + )) + [""] + (matched-cases exp))) ;; Get behind of :using :join-by) (honey.sql/register-clause! :when-matched - (fn [clause exp] - (let [exprs (matched-cases exp)] - (reduce (fn [[sql & params] expr] - (if (matched-merge-action? expr) - (let [[expr-sql & expr-params] (format-matched-merge-action expr)] - (into [(str sql " WHEN MATCHED THEN " expr-sql)] - (into params expr-params))) - (do - (assert (and (vector? expr) - (matched-merge-action? (second expr))) - "Invalid match clause for :when-matched") - (let [[cond-expr action-expr] expr] - (let [[case-sql & case-params] (format-matched-cond cond-expr) - [expr-sql & expr-params] (format-matched-merge-action action-expr)] - (into [(str sql " WHEN MATCHED " case-sql " THEN " expr-sql)] - (concat params case-params expr-params))))) - - )) - [""] - exprs))) + (fn [_clause exp] + (reduce (fn [[sql & params] expr] + (if (matched-merge-action? expr) + (let [[expr-sql & expr-params] (format-matched-merge-action expr)] + (into [(str sql " WHEN MATCHED THEN " expr-sql)] + (into params expr-params))) + (do + (assert (and (vector? expr) + (matched-merge-action? (second expr))) + "Invalid match clause for :when-matched") + (let [[cond-expr action-expr] expr + [case-sql & case-params] (format-matched-cond cond-expr) + [expr-sql & expr-params] (format-matched-merge-action action-expr)] + (into [(str sql " WHEN MATCHED " case-sql " THEN " expr-sql)] + (concat params case-params expr-params)))) + + )) + [""] + (matched-cases exp))) :when-not-matched) (honey.sql/register-clause! :on - (fn [clause expr] + (fn [_clause expr] (let [[sql & params] (honey.sql/format-expr expr)] (into [(str "ON " sql)] params))) :when-matched) (honey.sql/register-clause! :merge-into - (fn [clause tbl] + (fn [_clause tbl] [(str "MERGE INTO " (honey.sql/format-entity tbl))]) :using) +;; -------------------- +;; next.jdbc extensions + (defn ->pg-text-array "Formats as text[] in pg, i.e. {item-1, item-2, item3}" [col] From 4fb633839234b7cc8a98d47c40158e9ef0464255 Mon Sep 17 00:00:00 2001 From: Daniel Woelfel Date: Tue, 21 Jan 2025 15:39:29 -0800 Subject: [PATCH 3/7] postgres 17 --- .github/workflows/clojure.yml | 2 +- server/dev-postgres/Dockerfile | 4 ++-- server/dev-postgres/Makefile | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/clojure.yml b/.github/workflows/clojure.yml index cb0d72650e..717093284d 100644 --- a/.github/workflows/clojure.yml +++ b/.github/workflows/clojure.yml @@ -42,7 +42,7 @@ jobs: services: postgres: - image: public.ecr.aws/z9j8u5b3/instant-public:postgres-16 + image: public.ecr.aws/z9j8u5b3/instant-public:postgres-17 ports: - 5432:5432 env: diff --git a/server/dev-postgres/Dockerfile b/server/dev-postgres/Dockerfile index bab53ecfc2..2a66a75f07 100644 --- a/server/dev-postgres/Dockerfile +++ b/server/dev-postgres/Dockerfile @@ -1,9 +1,9 @@ -FROM postgres:13 +FROM postgres:17 RUN apt-get update && \ apt-get install -y \ build-essential \ - postgresql-server-dev-13 \ + postgresql-server-dev-17 \ wget && \ mkdir build && cd build && \ wget https://github.com/eulerto/wal2json/archive/refs/tags/wal2json_2_6.tar.gz && \ diff --git a/server/dev-postgres/Makefile b/server/dev-postgres/Makefile index 08af5496c9..8545ddccfc 100644 --- a/server/dev-postgres/Makefile +++ b/server/dev-postgres/Makefile @@ -6,4 +6,4 @@ ecr-login: build-and-push-image: docker buildx build --platform linux/amd64,linux/arm64 --push \ - -t public.ecr.aws/z9j8u5b3/instant-public:postgres-16 . + -t public.ecr.aws/z9j8u5b3/instant-public:postgres-17 . From f028f8b4c56af9a960ff581b48e969280d898037 Mon Sep 17 00:00:00 2001 From: Daniel Woelfel Date: Tue, 21 Jan 2025 15:53:09 -0800 Subject: [PATCH 4/7] add an easy test --- server/test/instant/jdbc/sql_test.clj | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/server/test/instant/jdbc/sql_test.clj b/server/test/instant/jdbc/sql_test.clj index 644548fe95..3dcfb61400 100644 --- a/server/test/instant/jdbc/sql_test.clj +++ b/server/test/instant/jdbc/sql_test.clj @@ -1,5 +1,6 @@ (ns instant.jdbc.sql-test - (:require [instant.jdbc.aurora :as aurora] + (:require [honey.sql] + [instant.jdbc.aurora :as aurora] [instant.jdbc.sql :as sql] [instant.util.test :refer [wait-for]] [clojure.test :refer [deftest testing is are]])) @@ -33,3 +34,24 @@ (binding [sql/*in-progress-stmts* in-progress] (let [query (sql/select (aurora/conn-pool) ["select 1"])] (is (= 0 (count @(:stmts in-progress)))))))) + +(deftest merge-into + (testing "smoke test" + (is (= ["MERGE INTO test USING a b ON (b.id = test.id) AND (b.v = test.v) WHEN MATCHED AND test.a = b.a THEN DELETE WHEN MATCHED AND test.b = b.b THEN UPDATE SET a = ? WHEN MATCHED AND test.c = b.c THEN INSERT (a) VALUES (?), (?) WHEN MATCHED THEN DO NOTHING WHEN NOT MATCHED THEN DO NOTHING RETURNING \"\".*" + 1 + 2 + 3] + (honey.sql/format {:merge-into :test + :using [[:a :b]] + :on [:and + [:= :b.id :test.id] + [:= :b.v :test.v]] + :when-matched [[[:= :test.a :b.a] + :delete] + [[:= :test.b :b.b] + {:update {:set {:a 1}}}] + [[:= :test.c :b.c] + {:insert {:values [{:a 2} {:a 3}]}}] + :do-nothing] + :when-not-matched :do-nothing + :returning :.*}))))) From 25664396fed194a4b4360b61e99b1c088c357bcc Mon Sep 17 00:00:00 2001 From: Daniel Woelfel Date: Tue, 21 Jan 2025 15:58:00 -0800 Subject: [PATCH 5/7] remove debug --- server/src/instant/jdbc/sql.clj | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/src/instant/jdbc/sql.clj b/server/src/instant/jdbc/sql.clj index bf6cc3bda4..27ea927b0d 100644 --- a/server/src/instant/jdbc/sql.clj +++ b/server/src/instant/jdbc/sql.clj @@ -67,8 +67,7 @@ ;; A list of condition+action (vector? exp) - (do (println "vec") - exp) + exp :else (throw (Exception. "Unknown match clauses for :when-matched/:when-not-matched")))) From 1657e3ef3f5d56714eb5969e82452c8a5a06e1c5 Mon Sep 17 00:00:00 2001 From: Daniel Woelfel Date: Tue, 21 Jan 2025 16:01:50 -0800 Subject: [PATCH 6/7] 2 workers helps with flaky tests? --- server/test/instant/reactive/session_test.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/test/instant/reactive/session_test.clj b/server/test/instant/reactive/session_test.clj index ba26f035ea..7856f79af5 100644 --- a/server/test/instant/reactive/session_test.clj +++ b/server/test/instant/reactive/session_test.clj @@ -45,7 +45,7 @@ {receive-q :grouped-queue} (grouped-queue/start-grouped-queue-with-workers - {:max-workers 1 + {:max-workers 2 :group-fn session/group-fn :reserve-fn session/receive-worker-reserve-fn :process-fn (partial session/process-fn store-conn)}) From e0e710ea155f68de9cabc3043d8bbca4ef3d1fb6 Mon Sep 17 00:00:00 2001 From: Daniel Woelfel Date: Fri, 16 Jan 2026 16:54:33 -0800 Subject: [PATCH 7/7] fix merge conflict --- server/src/instant/db/model/triple.clj | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/server/src/instant/db/model/triple.clj b/server/src/instant/db/model/triple.clj index d5dfa28d03..cc3e470ecd 100644 --- a/server/src/instant/db/model/triple.clj +++ b/server/src/instant/db/model/triple.clj @@ -401,6 +401,13 @@ (def value-lookup-error-prefix "missing-lookup-value") +;; Helper for :when-not-matched in `insert-multi +(def triple-t-values + (zipmap triple-cols + (map (fn [col] + (keyword (str "t." (name col)))) + triple-cols))) + (defn insert-multi! "Given a set of raw triples, we enhance each triple with metadata based on the triple's underlying attr and then insert these enhanced triples into