From f0806ca53343197690d41912111f4021f3e133f4 Mon Sep 17 00:00:00 2001 From: Daniel Woelfel Date: Mon, 30 Jun 2025 18:18:53 -0700 Subject: [PATCH 1/7] put uuids in the eav index --- .../resources/migrations/73_vae_uuids.up.sql | 15 +++++++++++ server/src/instant/db/datalog.clj | 27 ++++++++++--------- 2 files changed, 29 insertions(+), 13 deletions(-) create mode 100644 server/resources/migrations/73_vae_uuids.up.sql diff --git a/server/resources/migrations/73_vae_uuids.up.sql b/server/resources/migrations/73_vae_uuids.up.sql new file mode 100644 index 0000000000..4d201da9fe --- /dev/null +++ b/server/resources/migrations/73_vae_uuids.up.sql @@ -0,0 +1,15 @@ +create or replace function public.json_uuid_to_uuid(v jsonb) returns uuid + language sql immutable + as $$ + select (v->>0)::uuid + end; +$$; + +-- Create this index concurrently before running the migration +create index if not exists vae_uuid_index + on triples(app_id, public.json_uuid_to_uuid(value), attr_id, entity_id) + where vae; + +create unique index if not exists eav_uuid_index + on triples(app_id, entity_id, attr_id, public.json_uuid_to_uuid(value)) + where eav; diff --git a/server/src/instant/db/datalog.clj b/server/src/instant/db/datalog.clj index 66b9f15c5d..4d66c5e812 100644 --- a/server/src/instant/db/datalog.clj +++ b/server/src/instant/db/datalog.clj @@ -523,20 +523,20 @@ (string/replace (name x) "-" "_")) (defn- match-table-cols - "Every match table returns entity-id, attr-id, value-blob, value-uuid, + "Every match table returns entity-id, attr-id, value-blob, is-ref-val, and created-at columns. This is a quick helper to generate the column names" [table-name] [(kw table-name :-entity-id) (kw table-name :-attr-id) (kw table-name :-value-blob) - (kw table-name :-value-uuid) + (kw table-name :-is-ref-val) (kw table-name :-created-at)]) (defn- match-table-select "This generates the select portion of the match table. " [table-name] (map vector [:entity-id :attr-id :value - [:case :eav [:cast [:->> :value :0] :uuid] :else :null] + :eav :created-at] (match-table-cols table-name))) @@ -808,7 +808,7 @@ For example: [1 [:v :v]] => [:value :match-1-value-blob] - [1 [:e :v]] => [:entity-id :match-1-value-uuid] + [1 [:e :v]] => [:entity-id [:json_uuid_to_uuid :match-1-value-blob]] [1 [:v :a]] => [:value [:to_jsonb :match-1-attr-id]]" [prefix dest-idx [origin-ctype dest-ctype]] @@ -821,7 +821,7 @@ [:value [:to_jsonb (dest-col (component-type->col-name dest-ctype))]] (= :v dest-ctype) - [(component-type->col-name origin-ctype) (dest-col :value-uuid)] + [(component-type->col-name origin-ctype) [:json_uuid_to_uuid (dest-col :value-blob)]] :else [(component-type->col-name origin-ctype) @@ -863,7 +863,7 @@ The second part joins the first on - [[:= :entity-id :match-0-value-uuid]]" + [[:= :entity-id [:json_uuid_to_uuid :match-0-value-blob]]]" [prefix symbol-map named-p] (->> named-p variable-components @@ -1747,20 +1747,21 @@ (parse-uuid x))) (defn- sql-row->triple - "Converts the sql result, which returns value in either the - value-uuid or value-blob col, into our triple format. + "Converts the sql result into our triple format. Optionally parses uuids, when handling batched results that return JSON" - [row [e-col a-col v-blob-col v-uuid-col t-col] coerce-uuids?] + [row [e-col a-col v-blob-col v-is-ref-val-col t-col] coerce-uuids?] (if coerce-uuids? [(safe-parse-uuid (get row e-col)) (safe-parse-uuid (get row a-col)) - (or (safe-parse-uuid (get row v-uuid-col)) - (get row v-blob-col)) + (if (get row v-is-ref-val-col) + (safe-parse-uuid (get row v-blob-col)) + (get row v-blob-col)) (get row t-col)] [(get row e-col) (get row a-col) - (or (get row v-uuid-col) - (get row v-blob-col)) + (if (get row v-is-ref-val-col) + (safe-parse-uuid (get row v-blob-col)) + (get row v-blob-col)) (get row t-col)])) (defn- ensure-default-symbol-values [symbol-fields symbol-values] From 7f704e466a2b0ab7e53f02c46c610c162e95d951 Mon Sep 17 00:00:00 2001 From: Daniel Woelfel Date: Mon, 30 Jun 2025 20:17:08 -0700 Subject: [PATCH 2/7] fix test --- server/test/instant/db/datalog_test.clj | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/test/instant/db/datalog_test.clj b/server/test/instant/db/datalog_test.clj index dd55f843e1..c932f327a0 100644 --- a/server/test/instant/db/datalog_test.clj +++ b/server/test/instant/db/datalog_test.clj @@ -80,9 +80,9 @@ (deftest joins (testing "join conditions bind symbols to the matching previous patterns" (is (= - '(([:= :entity-id :match-0-value-uuid]) + '(([:= :entity-id [:json_uuid_to_uuid :match-0-value-blob]]) ([:= :entity-id :match-0-attr-id] - [:= :attr-id :match-0-value-uuid] + [:= :attr-id [:json_uuid_to_uuid :match-0-value-blob]] [:= :attr-id :match-1-entity-id])) (raw-pats->join-conds '[[:eav ?a ?b ?c] [:ea ?c ?d ?e] @@ -99,10 +99,10 @@ (is (= '(([:= :value [:to_jsonb :match-0-attr-id]])) (raw-pats->join-conds '[[:av _ ?a] [:eav _ _ ?a]])))) (testing "join conditions matches entities to coerced values" - (is (= '(([:= :entity-id :match-0-value-uuid])) + (is (= '(([:= :entity-id [:json_uuid_to_uuid :match-0-value-blob]])) (raw-pats->join-conds '[[:eav _ _ ?a] [:vae ?a]])))) (testing "join conditions matches attrs to coerced values" - (is (= '(([:= :attr-id :match-0-value-uuid])) + (is (= '(([:= :attr-id [:json_uuid_to_uuid :match-0-value-blob]])) (raw-pats->join-conds '[[:eav _ _ ?a] [:av _ ?a]])))) (testing "join conditions matches entities to attrs" (is (= '(([:= :entity-id :match-0-attr-id])) From a6d7755244a8644c635579fbf15d456539f2cd53 Mon Sep 17 00:00:00 2001 From: Daniel Woelfel Date: Wed, 2 Jul 2025 08:48:15 -0700 Subject: [PATCH 3/7] use vae instead of eav --- server/resources/migrations/73_vae_uuids.down.sql | 5 +++++ server/resources/migrations/73_vae_uuids.up.sql | 5 ++++- server/src/instant/db/datalog.clj | 4 ++-- server/src/instant/db/instaql.clj | 10 ++++++++++ server/src/instant/db/model/attr_pat.clj | 2 +- 5 files changed, 22 insertions(+), 4 deletions(-) create mode 100644 server/resources/migrations/73_vae_uuids.down.sql diff --git a/server/resources/migrations/73_vae_uuids.down.sql b/server/resources/migrations/73_vae_uuids.down.sql new file mode 100644 index 0000000000..d82e2b7bce --- /dev/null +++ b/server/resources/migrations/73_vae_uuids.down.sql @@ -0,0 +1,5 @@ +drop function public.json_uuid_to_uuid(v jsonb); + +drop index vae_uuid_index; + +drop index eav_uuid_index; diff --git a/server/resources/migrations/73_vae_uuids.up.sql b/server/resources/migrations/73_vae_uuids.up.sql index 4d201da9fe..217c4e4ea3 100644 --- a/server/resources/migrations/73_vae_uuids.up.sql +++ b/server/resources/migrations/73_vae_uuids.up.sql @@ -1,5 +1,7 @@ create or replace function public.json_uuid_to_uuid(v jsonb) returns uuid - language sql immutable + language sql + parallel safe + immutable as $$ select (v->>0)::uuid end; @@ -10,6 +12,7 @@ create index if not exists vae_uuid_index on triples(app_id, public.json_uuid_to_uuid(value), attr_id, entity_id) where vae; +-- Create this index concurrently before running the migration create unique index if not exists eav_uuid_index on triples(app_id, entity_id, attr_id, public.json_uuid_to_uuid(value)) where eav; diff --git a/server/src/instant/db/datalog.clj b/server/src/instant/db/datalog.clj index 4d66c5e812..6e0e452df7 100644 --- a/server/src/instant/db/datalog.clj +++ b/server/src/instant/db/datalog.clj @@ -722,14 +722,14 @@ {:name :ea_index :cols [:e :a] :idx-key :ea} - {:name :eav_index + {:name :eav_uuid_index :cols [:e :a :v] :idx-key :eav} {:name :triples_string_trgm_gist_idx :cols [:a :v] :idx-key :ave :data-type :string} - {:name :vae_index + {:name :vae_uuid_index :cols [:v :a :e] :idx-key :vae} {:name :triples_created_at_idx diff --git a/server/src/instant/db/instaql.clj b/server/src/instant/db/instaql.clj index 7795b4f19e..bee6ddcf9f 100644 --- a/server/src/instant/db/instaql.clj +++ b/server/src/instant/db/instaql.clj @@ -1188,6 +1188,16 @@ patterns)] (collect-query-results ctx (:data datalog-result) forms))))) +(defn explain + "Generates a nested datalog query, then runs explain." + [ctx o] + (let [query-hash (forms-hash o) + explain-fn (or (:datalog-explain-fn ctx) + d/explain)] + (let [{:keys [patterns forms]} (instaql-query->patterns ctx o)] + (explain-fn (assoc ctx :query-hash query-hash) + patterns)))) + ;; BYOP InstaQL (defn safe-table diff --git a/server/src/instant/db/model/attr_pat.clj b/server/src/instant/db/model/attr_pat.clj index f634f310aa..9664a26086 100644 --- a/server/src/instant/db/model/attr_pat.clj +++ b/server/src/instant/db/model/attr_pat.clj @@ -33,7 +33,7 @@ checking-data-type?]} v-actualized?] (let [ref? (= value-type :ref) - e-idx (if ref? :eav :ea) + e-idx (if ref? :vae :ea) v-idx (cond (and index? (not indexing?) From 775c8fb863b0a98feeb95131b3a7f215223665a1 Mon Sep 17 00:00:00 2001 From: Daniel Woelfel Date: Wed, 2 Jul 2025 08:50:02 -0700 Subject: [PATCH 4/7] fix lint --- server/src/instant/db/instaql.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/instant/db/instaql.clj b/server/src/instant/db/instaql.clj index bee6ddcf9f..c971b7b4c1 100644 --- a/server/src/instant/db/instaql.clj +++ b/server/src/instant/db/instaql.clj @@ -1194,7 +1194,7 @@ (let [query-hash (forms-hash o) explain-fn (or (:datalog-explain-fn ctx) d/explain)] - (let [{:keys [patterns forms]} (instaql-query->patterns ctx o)] + (let [{:keys [patterns]} (instaql-query->patterns ctx o)] (explain-fn (assoc ctx :query-hash query-hash) patterns)))) From 961d33d69676e5d6b0bd0e07ad7a9bea5a009b8c Mon Sep 17 00:00:00 2001 From: Daniel Woelfel Date: Wed, 2 Jul 2025 09:25:59 -0700 Subject: [PATCH 5/7] fix lint --- server/src/instant/db/instaql.clj | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/src/instant/db/instaql.clj b/server/src/instant/db/instaql.clj index c971b7b4c1..5378184189 100644 --- a/server/src/instant/db/instaql.clj +++ b/server/src/instant/db/instaql.clj @@ -1193,10 +1193,10 @@ [ctx o] (let [query-hash (forms-hash o) explain-fn (or (:datalog-explain-fn ctx) - d/explain)] - (let [{:keys [patterns]} (instaql-query->patterns ctx o)] - (explain-fn (assoc ctx :query-hash query-hash) - patterns)))) + d/explain) + {:keys [patterns]} (instaql-query->patterns ctx o)] + (explain-fn (assoc ctx :query-hash query-hash) + patterns))) ;; BYOP InstaQL From 70c3eba28a9cb98b94d9e4c867c7ad1030b83af8 Mon Sep 17 00:00:00 2001 From: Daniel Woelfel Date: Wed, 2 Jul 2025 10:15:18 -0700 Subject: [PATCH 6/7] fix tests --- server/src/instant/db/datalog.clj | 2 +- server/test/instant/db/instaql_test.clj | 57 ++++++++++++------------- 2 files changed, 29 insertions(+), 30 deletions(-) diff --git a/server/src/instant/db/datalog.clj b/server/src/instant/db/datalog.clj index 6e0e452df7..96cfe129f8 100644 --- a/server/src/instant/db/datalog.clj +++ b/server/src/instant/db/datalog.clj @@ -1061,7 +1061,7 @@ (if (named-variable? x) (assoc acc pat-idx {:sym sym :ref-value? (and (= :v component) - (= :eav (idx-key (:idx named-p))))}) + (= :vae (idx-key (:idx named-p))))}) acc))) {} [[:e 0] [:a 1] [:v 2]])) diff --git a/server/test/instant/db/instaql_test.clj b/server/test/instant/db/instaql_test.clj index 6bbbc66588..474b43e182 100644 --- a/server/test/instant/db/instaql_test.clj +++ b/server/test/instant/db/instaql_test.clj @@ -187,7 +187,7 @@ This checks equality strictly based on the set of topics and triples in the result" - [pretty-a pretty-b] + [pretty-actual pretty-expected] (let [{:keys [line column]} (meta &form) file *file*] `(if-let [operations# (:operations ~'*snapshot-replacements*)] @@ -195,18 +195,18 @@ {:file ~file :line ~line :column ~column} - ~pretty-a) - (let [pretty-a# ~pretty-a - pretty-b# ~pretty-b] + ~pretty-actual) + (let [pretty-expected# ~pretty-expected + pretty-actual# ~pretty-actual] (testing "(topics is-pretty-eq?)" - (is (= (set (mapcat :topics pretty-a#)) - (set (mapcat :topics pretty-b#))))) + (is (= (set (mapcat :topics pretty-expected#)) + (set (mapcat :topics pretty-actual#))))) (testing "(triples is-pretty-eq?)" - (is (= (set (mapcat :triples pretty-a#)) - (set (mapcat :triples pretty-b#))))) + (is (= (set (mapcat :triples pretty-expected#)) + (set (mapcat :triples pretty-actual#))))) (testing "(aggregate is-pretty-eq?)" - (is (= (set (remove nil? (mapcat :aggregate pretty-a#))) - (set (remove nil? (mapcat :aggregate pretty-b#)))))))))) + (is (= (set (remove nil? (mapcat :aggregate pretty-expected#))) + (set (remove nil? (mapcat :aggregate pretty-actual#)))))))))) (deftest validations (with-zeneca-app @@ -1477,7 +1477,7 @@ #{:users/createdAt :users/email :users/id :users/fullName :users/handle} _] -- - [:eav #{"eid-alex"} #{:users/bookshelves} _] + [:vae #{"eid-alex"} #{:users/bookshelves} _] [:ea _ #{:bookshelves/name} #{"Nonfiction"}] [:ave #{"eid-nonfiction"} #{:bookshelves/order} #{1}] -- @@ -2412,8 +2412,8 @@ '({:topics ([:ave _ #{:books/title} #{"The Count of Monte Cristo"}] [:vae _ #{:bookshelves/books} #{"eid-the-count-of-monte-cristo"}] [:vae _ #{:users/bookshelves} #{"eid-the-way-of-the-gentleman"}] - [:eav #{"eid-stepan-parunashvili"} #{:users/bookshelves} _] - [:eav _ #{:bookshelves/books} _] + [:vae #{"eid-stepan-parunashvili"} #{:users/bookshelves} _] + [:vae _ #{:bookshelves/books} _] [:ave _ #{:books/title} #{"Musashi"}] [:av #{"eid-stepan-parunashvili"} #{:users/email} #{"stopa@instantdb.com"}] @@ -2734,17 +2734,16 @@ {:users {:$not (resolvers/->uuid r "eid-joe-averbukh")}}]} :order {:order "desc"} :limit 1}}}) - '({:topics ([:eav {:$not "eid-stepan-parunashvili"} #{:users/bookshelves} _] + '({:topics ([:vae {:$not "eid-stepan-parunashvili"} #{:users/bookshelves} _] [:ea _ #{:bookshelves/id} _] [:ea _ #{:users/bookshelves} _] [:vae {:$not "eid-nicole"} #{:users/bookshelves} _] [:ea _ #{:bookshelves/id} _] [:ea _ #{:users/bookshelves} _] - [:vae {:$not "eid-joe-averbukh"} #{:users/bookshelves} - #{"eid-nonfiction"}] - [:ea #{"eid-nonfiction"} #{:bookshelves/id} _] + [:vae {:$not "eid-joe-averbukh"} #{:users/bookshelves} _] + [:ea _ #{:bookshelves/id} _] [:ea _ #{:users/bookshelves} _] - [:ave #{"eid-nonfiction"} #{:bookshelves/order} _] + [:ave _ #{:bookshelves/order} _] -- [:ea #{"eid-nonfiction"} #{:bookshelves/desc :bookshelves/name :bookshelves/order @@ -3030,7 +3029,7 @@ #{:users/createdAt :users/email :users/id :users/fullName :users/handle} _] -- - [:eav #{"eid-alex"} #{:users/bookshelves} _] + [:vae #{"eid-alex"} #{:users/bookshelves} _] -- [:ea #{"eid-nonfiction"} #{:bookshelves/desc :bookshelves/name :bookshelves/order @@ -3100,7 +3099,7 @@ #{:users/createdAt :users/email :users/id :users/fullName :users/handle} _] -- - [:eav #{"eid-alex"} #{:users/bookshelves} _] + [:vae #{"eid-alex"} #{:users/bookshelves} _] [:ea _ #{:bookshelves/name} #{"Nonfiction"}] -- [:ea #{"eid-nonfiction"} @@ -3135,7 +3134,7 @@ #{:users/createdAt :users/email :users/id :users/fullName :users/handle} _] -- - [:eav #{"eid-alex"} #{:users/bookshelves} _] + [:vae #{"eid-alex"} #{:users/bookshelves} _] [:ea _ #{:bookshelves/name} #{"Fiction" "Nonfiction"}] -- [:ea #{"eid-nonfiction"} @@ -3170,7 +3169,7 @@ #{:users/createdAt :users/email :users/id :users/fullName :users/handle} _] -- - [:eav #{"eid-alex"} #{:users/bookshelves} _] + [:vae #{"eid-alex"} #{:users/bookshelves} _] [:ea _ #{:bookshelves/name} #{"Nonfiction"}] [:ave #{"eid-nonfiction"} #{:bookshelves/order} #{1}] -- @@ -3343,7 +3342,7 @@ (testing "reverse works on link name" (is-pretty-eq? (query-pretty {:bookshelves {:$ {:where {:users (resolvers/->uuid r "eid-alex")}}}}) - '({:topics ([:eav #{"eid-alex"} #{:users/bookshelves} _] + '({:topics ([:vae #{"eid-alex"} #{:users/bookshelves} _] -- [:ea #{"eid-nonfiction"} #{:bookshelves/desc :bookshelves/name :bookshelves/order @@ -3730,8 +3729,8 @@ :bookshelves.books.title "The Count of Monte Cristo"}} :bookshelves {}}})] (is (= '[[:av _ #{:users/handle} #{"stopa" "joe"} _] - [:eav _ #{:users/bookshelves} _ _] - [:eav _ #{:bookshelves/books} _ _] + [:vae _ #{:users/bookshelves} _ _] + [:vae _ #{:bookshelves/books} _ _] [:ave _ #{:books/title} #{"The Count of Monte Cristo"} _] [:ea _ @@ -3742,7 +3741,7 @@ :users/handle} _ _] - [:eav _ #{:users/bookshelves} _ _] + [:vae _ #{:users/bookshelves} _ _] [:ea _ #{:bookshelves/desc @@ -3982,7 +3981,7 @@ r1 {:$users {:$ {:where {"books.title" "Sum"}}}}) '({:topics ([:ave _ #{:books/title} #{"Sum"}] - [:eav #{"eid-sum"} #{:books/$user-creator} _] + [:vae #{"eid-sum"} #{:books/$user-creator} _] -- [:ea #{"eid-alex"} #{:$users/email :$users/id} _]) :triples (("eid-sum" :books/$user-creator "eid-alex") @@ -4255,12 +4254,12 @@ -- [:ea #{"eid-alex"} #{:users/id :users/fullName} _] -- - [:eav #{"eid-alex"} #{:users/bookshelves} _] + [:vae #{"eid-alex"} #{:users/bookshelves} _] [:ea _ #{:bookshelves/name} #{"Nonfiction"}] -- [:ea #{"eid-nonfiction"} #{:bookshelves/order :bookshelves/id} _] -- - [:eav #{"eid-nonfiction"} #{:bookshelves/books} _] + [:vae #{"eid-nonfiction"} #{:bookshelves/books} _] [:ave _ #{:books/title} #{"Catch and Kill"}] -- [:ea #{"eid-catch-and-kill"} #{:books/id :books/title} _]) From dc137c9298bb3f4c5ce8ed7cf5678c57b4307d25 Mon Sep 17 00:00:00 2001 From: Daniel Woelfel Date: Wed, 2 Jul 2025 10:46:29 -0700 Subject: [PATCH 7/7] fix tests --- server/test/instant/db/datalog_test.clj | 26 ++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/server/test/instant/db/datalog_test.clj b/server/test/instant/db/datalog_test.clj index c932f327a0..adcfe0d051 100644 --- a/server/test/instant/db/datalog_test.clj +++ b/server/test/instant/db/datalog_test.clj @@ -282,10 +282,10 @@ (let [movie-title-aid (resolvers/->uuid r :movie/title) movie-director-aid (resolvers/->uuid r :movie/director) person-name-aid (resolvers/->uuid r :person/name)] - (is (= '{:topics [[:ea _ #{:movie/title} #{"Predator"}] - [:eav #{"eid-predator"} #{:movie/director} _] - [:ea _ #{:person/name} _]] - + (is (= '{:topics + [[:ea _ #{:movie/title} #{"Predator"}] + [:vae #{"eid-predator"} #{:movie/director} _] + [:ea _ #{:person/name} _]] :symbol-values {?e #{"eid-predator"}, ?director #{"eid-john-mctiernan"}, ?name #{"John McTiernan"}}, @@ -296,7 +296,7 @@ (query-pretty [[:ea '?e movie-title-aid "Predator"] - [:eav '?e movie-director-aid '?director] + [:vae '?e movie-director-aid '?director] [:ea '?director person-name-aid '?name]]))))) (testing "refs jump vae" @@ -344,16 +344,16 @@ [[:ea '?director person-name-aid "John McTiernan"] [:vae '?movie movie-director-aid '?director] [:ea '?movie movie-title-aid '?title]]) - % - (resolvers/walk-friendly r %) - (drop-join-rows-created-at %))) + % + (resolvers/walk-friendly r %) + (drop-join-rows-created-at %))) q2 (future (as-> (d/query ctx [[:ea '?e movie-title-aid "Predator"] - [:eav '?e movie-director-aid '?director] + [:vae '?e movie-director-aid '?director] [:ea '?director person-name-aid '?name]]) - % - (resolvers/walk-friendly r %) - (drop-join-rows-created-at %)))] + % + (resolvers/walk-friendly r %) + (drop-join-rows-created-at %)))] ;; Wait for queries to batch (loop [i 0] @@ -387,7 +387,7 @@ @q1)) (is (= '{:topics [[:ea _ #{:movie/title} #{"Predator"}] - [:eav #{"eid-predator"} #{:movie/director} _] + [:vae #{"eid-predator"} #{:movie/director} _] [:ea _ #{:person/name} _]] :symbol-values {?e #{"eid-predator"},