diff --git a/Cargo.lock b/Cargo.lock index edf2d2f1c..0b7f2ebec 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -253,19 +253,40 @@ version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3f15b4c6b148206ff3a2b35002e08929c2462467b62b9c02036d9c34f9ef994" dependencies = [ - "arrow-arith", - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-csv", - "arrow-data", - "arrow-ipc", - "arrow-json", - "arrow-ord", - "arrow-row", - "arrow-schema", - "arrow-select", - "arrow-string", + "arrow-arith 55.2.0", + "arrow-array 55.2.0", + "arrow-buffer 55.2.0", + "arrow-cast 55.2.0", + "arrow-csv 55.2.0", + "arrow-data 55.2.0", + "arrow-ipc 55.2.0", + "arrow-json 55.2.0", + "arrow-ord 55.2.0", + "arrow-row 55.2.0", + "arrow-schema 55.2.0", + "arrow-select 55.2.0", + "arrow-string 55.2.0", +] + +[[package]] +name = "arrow" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e833808ff2d94ed40d9379848a950d995043c7fb3e81a30b383f4c6033821cc" +dependencies = [ + "arrow-arith 56.2.0", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-cast 56.2.0", + "arrow-csv 56.2.0", + "arrow-data 56.2.0", + "arrow-ipc 56.2.0", + "arrow-json 56.2.0", + "arrow-ord 56.2.0", + "arrow-row 56.2.0", + "arrow-schema 56.2.0", + "arrow-select 56.2.0", + "arrow-string 56.2.0", ] [[package]] @@ -274,10 +295,24 @@ version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "30feb679425110209ae35c3fbf82404a39a4c0436bb3ec36164d8bffed2a4ce4" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 55.2.0", + "arrow-buffer 55.2.0", + "arrow-data 55.2.0", + "arrow-schema 55.2.0", + "chrono", + "num", +] + +[[package]] +name = "arrow-arith" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad08897b81588f60ba983e3ca39bda2b179bdd84dced378e7df81a5313802ef8" +dependencies = [ + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", "chrono", "num", ] @@ -289,9 +324,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70732f04d285d49054a48b72c54f791bb3424abae92d27aafdf776c98af161c8" dependencies = [ "ahash 0.8.12", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-buffer 55.2.0", + "arrow-data 55.2.0", + "arrow-schema 55.2.0", "chrono", "chrono-tz", "half", @@ -299,6 +334,23 @@ dependencies = [ "num", ] +[[package]] +name = "arrow-array" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8548ca7c070d8db9ce7aa43f37393e4bfcf3f2d3681df278490772fd1673d08d" +dependencies = [ + "ahash 0.8.12", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", + "chrono", + "chrono-tz", + "half", + "hashbrown 0.16.0", + "num", +] + [[package]] name = "arrow-buffer" version = "55.2.0" @@ -310,17 +362,49 @@ dependencies = [ "num", ] +[[package]] +name = "arrow-buffer" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e003216336f70446457e280807a73899dd822feaf02087d31febca1363e2fccc" +dependencies = [ + "bytes", + "half", + "num", +] + [[package]] name = "arrow-cast" version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e4f12eccc3e1c05a766cafb31f6a60a46c2f8efec9b74c6e0648766d30686af8" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", + "arrow-array 55.2.0", + "arrow-buffer 55.2.0", + "arrow-data 55.2.0", + "arrow-schema 55.2.0", + "arrow-select 55.2.0", + "atoi", + "base64 0.22.1", + "chrono", + "comfy-table", + "half", + "lexical-core", + "num", + "ryu", +] + +[[package]] +name = "arrow-cast" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "919418a0681298d3a77d1a315f625916cb5678ad0d74b9c60108eb15fd083023" +dependencies = [ + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", + "arrow-select 56.2.0", "atoi", "base64 0.22.1", "chrono", @@ -337,9 +421,24 @@ version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "012c9fef3f4a11573b2c74aec53712ff9fdae4a95f4ce452d1bbf088ee00f06b" dependencies = [ - "arrow-array", - "arrow-cast", - "arrow-schema", + "arrow-array 55.2.0", + "arrow-cast 55.2.0", + "arrow-schema 55.2.0", + "chrono", + "csv", + "csv-core", + "regex", +] + +[[package]] +name = "arrow-csv" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa9bf02705b5cf762b6f764c65f04ae9082c7cfc4e96e0c33548ee3f67012eb" +dependencies = [ + "arrow-array 56.2.0", + "arrow-cast 56.2.0", + "arrow-schema 56.2.0", "chrono", "csv", "csv-core", @@ -352,8 +451,20 @@ version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8de1ce212d803199684b658fc4ba55fb2d7e87b213de5af415308d2fee3619c2" dependencies = [ - "arrow-buffer", - "arrow-schema", + "arrow-buffer 55.2.0", + "arrow-schema 55.2.0", + "half", + "num", +] + +[[package]] +name = "arrow-data" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5c64fff1d142f833d78897a772f2e5b55b36cb3e6320376f0961ab0db7bd6d0" +dependencies = [ + "arrow-buffer 56.2.0", + "arrow-schema 56.2.0", "half", "num", ] @@ -364,26 +475,62 @@ version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9ea5967e8b2af39aff5d9de2197df16e305f47f404781d3230b2dc672da5d92" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 55.2.0", + "arrow-buffer 55.2.0", + "arrow-data 55.2.0", + "arrow-schema 55.2.0", "flatbuffers", "lz4_flex", "zstd", ] +[[package]] +name = "arrow-ipc" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d3594dcddccc7f20fd069bc8e9828ce37220372680ff638c5e00dea427d88f5" +dependencies = [ + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", + "arrow-select 56.2.0", + "flatbuffers", +] + [[package]] name = "arrow-json" version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5709d974c4ea5be96d900c01576c7c0b99705f4a3eec343648cb1ca863988a9c" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", - "arrow-schema", + "arrow-array 55.2.0", + "arrow-buffer 55.2.0", + "arrow-cast 55.2.0", + "arrow-data 55.2.0", + "arrow-schema 55.2.0", + "chrono", + "half", + "indexmap 2.11.0", + "lexical-core", + "memchr", + "num", + "serde", + "serde_json", + "simdutf8", +] + +[[package]] +name = "arrow-json" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88cf36502b64a127dc659e3b305f1d993a544eab0d48cce704424e62074dc04b" +dependencies = [ + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-cast 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", "chrono", "half", "indexmap 2.11.0", @@ -401,11 +548,24 @@ version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6506e3a059e3be23023f587f79c82ef0bcf6d293587e3272d20f2d30b969b5a7" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", + "arrow-array 55.2.0", + "arrow-buffer 55.2.0", + "arrow-data 55.2.0", + "arrow-schema 55.2.0", + "arrow-select 55.2.0", +] + +[[package]] +name = "arrow-ord" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c8f82583eb4f8d84d4ee55fd1cb306720cddead7596edce95b50ee418edf66f" +dependencies = [ + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", + "arrow-select 56.2.0", ] [[package]] @@ -414,10 +574,23 @@ version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52bf7393166beaf79b4bed9bfdf19e97472af32ce5b6b48169d321518a08cae2" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 55.2.0", + "arrow-buffer 55.2.0", + "arrow-data 55.2.0", + "arrow-schema 55.2.0", + "half", +] + +[[package]] +name = "arrow-row" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d07ba24522229d9085031df6b94605e0f4b26e099fb7cdeec37abd941a73753" +dependencies = [ + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", "half", ] @@ -427,11 +600,20 @@ version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af7686986a3bf2254c9fb130c623cdcb2f8e1f15763e7c71c310f0834da3d292" dependencies = [ - "bitflags 2.9.4", "serde", "serde_json", ] +[[package]] +name = "arrow-schema" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3aa9e59c611ebc291c28582077ef25c97f1975383f1479b12f3b9ffee2ffabe" +dependencies = [ + "bitflags 2.9.4", + "serde", +] + [[package]] name = "arrow-select" version = "55.2.0" @@ -439,10 +621,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd2b45757d6a2373faa3352d02ff5b54b098f5e21dccebc45a21806bc34501e5" dependencies = [ "ahash 0.8.12", - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 55.2.0", + "arrow-buffer 55.2.0", + "arrow-data 55.2.0", + "arrow-schema 55.2.0", + "num", +] + +[[package]] +name = "arrow-select" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c41dbbd1e97bfcaee4fcb30e29105fb2c75e4d82ae4de70b792a5d3f66b2e7a" +dependencies = [ + "ahash 0.8.12", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", "num", ] @@ -452,11 +648,28 @@ version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0377d532850babb4d927a06294314b316e23311503ed580ec6ce6a0158f49d40" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", + "arrow-array 55.2.0", + "arrow-buffer 55.2.0", + "arrow-data 55.2.0", + "arrow-schema 55.2.0", + "arrow-select 55.2.0", + "memchr", + "num", + "regex", + "regex-syntax", +] + +[[package]] +name = "arrow-string" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53f5183c150fbc619eede22b861ea7c0eebed8eaac0333eaa7f6da5205fd504d" +dependencies = [ + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", + "arrow-select 56.2.0", "memchr", "num", "regex", @@ -594,9 +807,9 @@ dependencies = [ [[package]] name = "aws-credential-types" -version = "1.2.6" +version = "1.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d025db5d9f52cbc413b167136afb3d8aeea708c0d8884783cf6253be5e22f6f2" +checksum = "faf26925f4a5b59eb76722b63c2892b1d70d06fa053c72e4a100ec308c1d47bc" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -629,9 +842,9 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.5.10" +version = "1.5.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c034a1bc1d70e16e7f4e4caf7e9f7693e4c9c24cd91cf17c2a0b21abaebc7c8b" +checksum = "bfa006bb32360ed90ac51203feafb9d02e3d21046e1fd3a450a404b90ea73e5d" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -651,6 +864,28 @@ dependencies = [ "uuid", ] +[[package]] +name = "aws-sdk-dynamodb" +version = "1.95.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "699bab2e1f9da570071889b674a43df1184b6ec42bf4770ef8758a8f3e02f38f" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "regex-lite", + "tracing", +] + [[package]] name = "aws-sdk-glue" version = "1.119.0" @@ -719,9 +954,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.85.0" +version = "1.88.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "410309ad0df4606bc721aff0d89c3407682845453247213a0ccc5ff8801ee107" +checksum = "d30990923f4f675523c51eb1c0dec9b752fb267b36a61e83cbc219c9d86da715" dependencies = [ "aws-credential-types", "aws-runtime", @@ -742,9 +977,9 @@ dependencies = [ [[package]] name = "aws-sigv4" -version = "1.3.4" +version = "1.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "084c34162187d39e3740cb635acd73c4e3a551a36146ad6fe8883c929c9f876c" +checksum = "bffc03068fbb9c8dd5ce1c6fb240678a5cffb86fb2b7b1985c999c4b83c8df68" dependencies = [ "aws-credential-types", "aws-smithy-http", @@ -764,9 +999,9 @@ dependencies = [ [[package]] name = "aws-smithy-async" -version = "1.2.5" +version = "1.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e190749ea56f8c42bf15dd76c65e14f8f765233e6df9b0506d9d934ebef867c" +checksum = "127fcfad33b7dfc531141fda7e1c402ac65f88aca5511a4d31e2e3d2cd01ce9c" dependencies = [ "futures-util", "pin-project-lite", @@ -775,9 +1010,9 @@ dependencies = [ [[package]] name = "aws-smithy-http" -version = "0.62.3" +version = "0.62.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c4dacf2d38996cf729f55e7a762b30918229917eca115de45dfa8dfb97796c9" +checksum = "3feafd437c763db26aa04e0cc7591185d0961e64c61885bece0fb9d50ceac671" dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", @@ -795,9 +1030,9 @@ dependencies = [ [[package]] name = "aws-smithy-http-client" -version = "1.1.1" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "147e8eea63a40315d704b97bf9bc9b8c1402ae94f89d5ad6f7550d963309da1b" +checksum = "1053b5e587e6fa40ce5a79ea27957b04ba660baa02b28b7436f64850152234f1" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -825,27 +1060,27 @@ dependencies = [ [[package]] name = "aws-smithy-json" -version = "0.61.5" +version = "0.61.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eaa31b350998e703e9826b2104dd6f63be0508666e1aba88137af060e8944047" +checksum = "cff418fc8ec5cadf8173b10125f05c2e7e1d46771406187b2c878557d4503390" dependencies = [ "aws-smithy-types", ] [[package]] name = "aws-smithy-observability" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9364d5989ac4dd918e5cc4c4bdcc61c9be17dcd2586ea7f69e348fc7c6cab393" +checksum = "2d1881b1ea6d313f9890710d65c158bdab6fb08c91ea825f74c1c8c357baf4cc" dependencies = [ "aws-smithy-runtime-api", ] [[package]] name = "aws-smithy-query" -version = "0.60.7" +version = "0.60.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2fbd61ceb3fe8a1cb7352e42689cec5335833cd9f94103a61e98f9bb61c64bb" +checksum = "d28a63441360c477465f80c7abac3b9c4d075ca638f982e605b7dc2a2c7156c9" dependencies = [ "aws-smithy-types", "urlencoding", @@ -853,9 +1088,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" -version = "1.9.1" +version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3946acbe1ead1301ba6862e712c7903ca9bb230bdf1fbd1b5ac54158ef2ab1f" +checksum = "40ab99739082da5347660c556689256438defae3bcefd66c52b095905730e404" dependencies = [ "aws-smithy-async", "aws-smithy-http", @@ -877,9 +1112,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime-api" -version = "1.9.0" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07f5e0fc8a6b3f2303f331b94504bbf754d85488f402d6f1dd7a6080f99afe56" +checksum = "3683c5b152d2ad753607179ed71988e8cfd52964443b4f74fd8e552d0bbfeb46" dependencies = [ "aws-smithy-async", "aws-smithy-types", @@ -894,9 +1129,9 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.3.2" +version = "1.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d498595448e43de7f4296b7b7a18a8a02c61ec9349128c80a368f7c3b4ab11a8" +checksum = "9f5b3a7486f6690ba25952cabf1e7d75e34d69eaff5081904a47bc79074d6457" dependencies = [ "base64-simd", "bytes", @@ -920,18 +1155,18 @@ dependencies = [ [[package]] name = "aws-smithy-xml" -version = "0.60.10" +version = "0.60.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3db87b96cb1b16c024980f133968d52882ca0daaee3a086c6decc500f6c99728" +checksum = "e9c34127e8c624bc2999f3b657e749c1393bedc9cd97b92a804db8ced4d2e163" dependencies = [ "xmlparser", ] [[package]] name = "aws-types" -version = "1.3.8" +version = "1.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b069d19bf01e46298eaedd7c6f283fe565a59263e53eebec945f3e6398f42390" +checksum = "e2fd329bf0e901ff3f60425691410c69094dc2a1f34b331f37bfc4e9ac1565a1" dependencies = [ "aws-credential-types", "aws-smithy-async", @@ -1585,12 +1820,14 @@ checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" [[package]] name = "comfy-table" -version = "7.1.4" +version = "7.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a65ebfec4fb190b6f90e944a817d60499ee0744e582530e2c9900a22e591d9a" +checksum = "e0d05af1e006a2407bedef5af410552494ce5be9090444dbbcb57258c1af3d56" dependencies = [ - "crossterm", - "unicode-segmentation", + "crossterm 0.27.0", + "crossterm 0.28.1", + "strum 0.26.3", + "strum_macros 0.26.4", "unicode-width", ] @@ -1868,6 +2105,19 @@ version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +[[package]] +name = "crossterm" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f476fe445d41c9e991fd07515a6f463074b782242ccf4a5b7b1d1012e70824df" +dependencies = [ + "bitflags 2.9.4", + "crossterm_winapi", + "libc", + "parking_lot", + "winapi", +] + [[package]] name = "crossterm" version = "0.28.1" @@ -1875,10 +2125,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6" dependencies = [ "bitflags 2.9.4", - "crossterm_winapi", "parking_lot", "rustix 0.38.44", - "winapi", ] [[package]] @@ -2023,9 +2271,9 @@ version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69dfeda1633bf8ec75b068d9f6c27cdc392ffcf5ff83128d5dbab65b73c1fd02" dependencies = [ - "arrow", - "arrow-ipc", - "arrow-schema", + "arrow 55.2.0", + "arrow-ipc 55.2.0", + "arrow-schema 55.2.0", "async-trait", "bytes", "bzip2 0.6.0", @@ -2061,7 +2309,7 @@ dependencies = [ "log", "object_store", "parking_lot", - "parquet", + "parquet 55.2.0", "rand 0.9.2", "regex", "sqlparser 0.55.0", @@ -2079,7 +2327,7 @@ version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2848fd1e85e2953116dab9cc2eb109214b0888d7bbd2230e30c07f1794f642c0" dependencies = [ - "arrow", + "arrow 55.2.0", "async-trait", "dashmap", "datafusion-common", @@ -2105,7 +2353,7 @@ version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "051a1634628c2d1296d4e326823e7536640d87a118966cdaff069b68821ad53b" dependencies = [ - "arrow", + "arrow 55.2.0", "async-trait", "datafusion-catalog", "datafusion-common", @@ -2128,7 +2376,7 @@ version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5966ac4973bf66cf5c189046b1b29de90db9ba059ce7193137dd089e277495de" dependencies = [ - "arrow", + "arrow 55.2.0", "async-trait", "aws-config", "aws-credential-types", @@ -2141,7 +2389,7 @@ dependencies = [ "mimalloc", "object_store", "parking_lot", - "parquet", + "parquet 55.2.0", "regex", "rustyline", "tokio", @@ -2156,8 +2404,8 @@ checksum = "765e4ad4ef7a4500e389a3f1e738791b71ff4c29fd00912c2f541d62b25da096" dependencies = [ "ahash 0.8.12", "apache-avro 0.17.0", - "arrow", - "arrow-ipc", + "arrow 55.2.0", + "arrow-ipc 55.2.0", "base64 0.22.1", "chrono", "half", @@ -2167,7 +2415,7 @@ dependencies = [ "libc", "log", "object_store", - "parquet", + "parquet 55.2.0", "paste", "recursive", "sqlparser 0.55.0", @@ -2192,7 +2440,7 @@ version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90cd841a77f378bc1a5c4a1c37345e1885a9203b008203f9f4b3a769729bf330" dependencies = [ - "arrow", + "arrow 55.2.0", "async-compression", "async-trait", "bytes", @@ -2212,7 +2460,7 @@ dependencies = [ "itertools 0.14.0", "log", "object_store", - "parquet", + "parquet 55.2.0", "rand 0.9.2", "tempfile", "tokio", @@ -2229,7 +2477,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3cba1696aa919da9517d29164d45f5902d6cc281f718e8d3bfe98bd52cd1142c" dependencies = [ "apache-avro 0.17.0", - "arrow", + "arrow 55.2.0", "async-trait", "bytes", "chrono", @@ -2253,7 +2501,7 @@ version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77f4a2c64939c6f0dd15b246723a699fa30d59d0133eb36a86e8ff8c6e2a8dc6" dependencies = [ - "arrow", + "arrow 55.2.0", "async-trait", "bytes", "datafusion-catalog", @@ -2278,7 +2526,7 @@ version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "11387aaf931b2993ad9273c63ddca33f05aef7d02df9b70fb757429b4b71cdae" dependencies = [ - "arrow", + "arrow 55.2.0", "async-trait", "bytes", "datafusion-catalog", @@ -2303,7 +2551,7 @@ version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "028f430c5185120bf806347848b8d8acd9823f4038875b3820eeefa35f2bb4a2" dependencies = [ - "arrow", + "arrow 55.2.0", "async-trait", "bytes", "datafusion-catalog", @@ -2325,7 +2573,7 @@ dependencies = [ "log", "object_store", "parking_lot", - "parquet", + "parquet 55.2.0", "rand 0.9.2", "tokio", ] @@ -2342,7 +2590,7 @@ version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "042ea192757d1b2d7dcf71643e7ff33f6542c7704f00228d8b85b40003fd8e0f" dependencies = [ - "arrow", + "arrow 55.2.0", "dashmap", "datafusion-common", "datafusion-expr", @@ -2361,7 +2609,7 @@ version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "025222545d6d7fab71e2ae2b356526a1df67a2872222cbae7535e557a42abd2e" dependencies = [ - "arrow", + "arrow 55.2.0", "async-trait", "chrono", "datafusion-common", @@ -2383,7 +2631,7 @@ version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d5c267104849d5fa6d81cf5ba88f35ecd58727729c5eb84066c25227b644ae2" dependencies = [ - "arrow", + "arrow 55.2.0", "datafusion-common", "indexmap 2.11.0", "itertools 0.14.0", @@ -2396,8 +2644,8 @@ version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c620d105aa208fcee45c588765483314eb415f5571cfd6c1bae3a59c5b4d15bb" dependencies = [ - "arrow", - "arrow-buffer", + "arrow 55.2.0", + "arrow-buffer 55.2.0", "base64 0.22.1", "blake2", "blake3", @@ -2426,7 +2674,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35f61d5198a35ed368bf3aacac74f0d0fa33de7a7cb0c57e9f68ab1346d2f952" dependencies = [ "ahash 0.8.12", - "arrow", + "arrow 55.2.0", "datafusion-common", "datafusion-doc", "datafusion-execution", @@ -2447,7 +2695,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13efdb17362be39b5024f6da0d977ffe49c0212929ec36eec550e07e2bc7812f" dependencies = [ "ahash 0.8.12", - "arrow", + "arrow 55.2.0", "datafusion-common", "datafusion-expr-common", "datafusion-physical-expr-common", @@ -2459,8 +2707,8 @@ version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9187678af567d7c9e004b72a0b6dc5b0a00ebf4901cb3511ed2db4effe092e66" dependencies = [ - "arrow", - "arrow-ord", + "arrow 55.2.0", + "arrow-ord 55.2.0", "datafusion-common", "datafusion-doc", "datafusion-execution", @@ -2481,7 +2729,7 @@ version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ecf156589cc21ef59fe39c7a9a841b4a97394549643bbfa88cc44e8588cf8fe5" dependencies = [ - "arrow", + "arrow 55.2.0", "async-trait", "datafusion-catalog", "datafusion-common", @@ -2497,7 +2745,7 @@ version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "edcb25e3e369f1366ec9a261456e45b5aad6ea1c0c8b4ce546587207c501ed9e" dependencies = [ - "arrow", + "arrow 55.2.0", "datafusion-common", "datafusion-doc", "datafusion-expr", @@ -2536,7 +2784,7 @@ version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c9fa98671458254928af854e5f6c915e66b860a8bde505baea0ff2892deab74d" dependencies = [ - "arrow", + "arrow 55.2.0", "chrono", "datafusion-common", "datafusion-expr", @@ -2557,7 +2805,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3515d51531cca5f7b5a6f3ea22742b71bb36fc378b465df124ff9a2fa349b002" dependencies = [ "ahash 0.8.12", - "arrow", + "arrow 55.2.0", "datafusion-common", "datafusion-expr", "datafusion-expr-common", @@ -2579,7 +2827,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24485475d9c618a1d33b2a3dad003d946dc7a7bbf0354d125301abc0a5a79e3e" dependencies = [ "ahash 0.8.12", - "arrow", + "arrow 55.2.0", "datafusion-common", "datafusion-expr-common", "hashbrown 0.14.5", @@ -2592,7 +2840,7 @@ version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9da411a0a64702f941a12af2b979434d14ec5d36c6f49296966b2c7639cbb3a" dependencies = [ - "arrow", + "arrow 55.2.0", "datafusion-common", "datafusion-execution", "datafusion-expr", @@ -2613,9 +2861,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6d168282bb7b54880bb3159f89b51c047db4287f5014d60c3ef4c6e1468212b" dependencies = [ "ahash 0.8.12", - "arrow", - "arrow-ord", - "arrow-schema", + "arrow 55.2.0", + "arrow-ord 55.2.0", + "arrow-schema 55.2.0", "async-trait", "chrono", "datafusion-common", @@ -2642,8 +2890,8 @@ version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "391a457b9d23744c53eeb89edd1027424cba100581488d89800ed841182df905" dependencies = [ - "arrow", - "arrow-schema", + "arrow 55.2.0", + "arrow-schema 55.2.0", "datafusion-common", "datafusion-datasource", "datafusion-expr-common", @@ -2660,7 +2908,7 @@ version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "053201c2bb729c7938f85879034df2b5a52cfaba16f1b3b66ab8505c81b2aad3" dependencies = [ - "arrow", + "arrow 55.2.0", "async-trait", "dashmap", "datafusion-common", @@ -2684,7 +2932,7 @@ version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9082779be8ce4882189b229c0cff4393bd0808282a7194130c9f32159f185e25" dependencies = [ - "arrow", + "arrow 55.2.0", "bigdecimal", "datafusion-common", "datafusion-expr", @@ -2706,11 +2954,11 @@ dependencies = [ [[package]] name = "delta_kernel" -version = "0.15.2" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a889d7dae18cc716c0ea061e8d199e9c60b8b8fd9c15a52be6aee0eff1e7c6fe" +checksum = "cb6b80fa39021744edf13509bbdd7caef94c1bf101e384990210332dbddddf44" dependencies = [ - "arrow", + "arrow 56.2.0", "bytes", "chrono", "comfy-table", @@ -2719,7 +2967,7 @@ dependencies = [ "indexmap 2.11.0", "itertools 0.14.0", "object_store", - "parquet", + "parquet 56.2.0", "reqwest", "roaring", "rustc_version", @@ -2736,9 +2984,9 @@ dependencies = [ [[package]] name = "delta_kernel_derive" -version = "0.15.2" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52d6f7437b6725b3fb184ce0cbe89c9d1fad8782235bf5ea1e8a13fc69bc5cca" +checksum = "ae1d02d9f5d886ae8bb7fc3f7a3cb8f1b75cd0f5c95f9b5f45bba308f1a0aa58" dependencies = [ "proc-macro2", "quote", @@ -2747,31 +2995,59 @@ dependencies = [ [[package]] name = "deltalake" -version = "0.28.1" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19aee5cc7555b3ea96c7fa552ce21c05332db55235e7f4a8a9d1400157c61030" +checksum = "e5ae322ff91049738d9e6509f6ccb7e773d0cf06e672458c88a975f99b7078fe" dependencies = [ "delta_kernel", + "deltalake-aws", "deltalake-core", + "deltalake-gcp", ] [[package]] -name = "deltalake-core" -version = "0.28.1" +name = "deltalake-aws" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589ff38c0c6ce383bd83f0a618d76940d53404593210093ee5e2a9b1cff9d579" -dependencies = [ - "arrow", - "arrow-arith", - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-ipc", - "arrow-json", - "arrow-ord", - "arrow-row", - "arrow-schema", - "arrow-select", +checksum = "8bb6fa32f92bd7393a67f45a27b61c1a6eb90c184e22771f87ee2186d33fc101" +dependencies = [ + "async-trait", + "aws-config", + "aws-credential-types", + "aws-sdk-dynamodb", + "aws-sdk-sts", + "aws-smithy-runtime-api", + "backon", + "bytes", + "chrono", + "deltalake-core", + "futures", + "object_store", + "regex", + "thiserror 2.0.16", + "tokio", + "tracing", + "url", + "uuid", +] + +[[package]] +name = "deltalake-core" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ab08e274529d6f80cb3a638ed13632012bd8fda07dcdbe022bed7e6fe9312a" +dependencies = [ + "arrow 56.2.0", + "arrow-arith 56.2.0", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-cast 56.2.0", + "arrow-ipc 56.2.0", + "arrow-json 56.2.0", + "arrow-ord 56.2.0", + "arrow-row 56.2.0", + "arrow-schema 56.2.0", + "arrow-select 56.2.0", "async-trait", "bytes", "cfg-if", @@ -2785,18 +3061,18 @@ dependencies = [ "humantime", "indexmap 2.11.0", "itertools 0.14.0", - "maplit", "num_cpus", "object_store", "parking_lot", - "parquet", + "parquet 56.2.0", "percent-encoding", "percent-encoding-rfc3986", + "pin-project-lite", "rand 0.8.5", "regex", "serde", "serde_json", - "sqlparser 0.56.0", + "sqlparser 0.59.0", "strum 0.27.2", "thiserror 2.0.16", "tokio", @@ -2808,9 +3084,9 @@ dependencies = [ [[package]] name = "deltalake-derive" -version = "0.28.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "751cfe39c31f065104f3c2238d0e423849a4e4f2e2b8adf923d8276a59da7f3a" +checksum = "1a785b4702c2d1b6ff286075f375fb2fd52dfbb2fadf17b9233f4d5eea35c6ec" dependencies = [ "convert_case", "itertools 0.14.0", @@ -2819,6 +3095,23 @@ dependencies = [ "syn 2.0.106", ] +[[package]] +name = "deltalake-gcp" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df2708856cfa92e8309141fda1ee8d41e174b0868f26e557d0b2ea8d30fb92e" +dependencies = [ + "async-trait", + "bytes", + "deltalake-core", + "futures", + "object_store", + "thiserror 2.0.16", + "tokio", + "tracing", + "url", +] + [[package]] name = "der" version = "0.7.10" @@ -3557,6 +3850,12 @@ dependencies = [ "foldhash", ] +[[package]] +name = "hashbrown" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5419bdc4f6a9207fbeba6d11b604d481addf78ecd10c11ad51e76c2f6482748d" + [[package]] name = "hashlink" version = "0.10.0" @@ -3866,14 +4165,14 @@ dependencies = [ "anyhow", "apache-avro 0.20.0", "array-init", - "arrow-arith", - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-ord", - "arrow-schema", - "arrow-select", - "arrow-string", + "arrow-arith 55.2.0", + "arrow-array 55.2.0", + "arrow-buffer 55.2.0", + "arrow-cast 55.2.0", + "arrow-ord 55.2.0", + "arrow-schema 55.2.0", + "arrow-select 55.2.0", + "arrow-string 55.2.0", "as-any", "async-trait", "backon", @@ -3892,7 +4191,7 @@ dependencies = [ "once_cell", "opendal", "ordered-float 4.6.0", - "parquet", + "parquet 55.2.0", "rand 0.8.5", "reqsign", "reqwest", @@ -4547,12 +4846,6 @@ dependencies = [ "pkg-config", ] -[[package]] -name = "maplit" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" - [[package]] name = "matchers" version = "0.2.0" @@ -4695,9 +4988,9 @@ version = "0.0.1" dependencies = [ "ahash 0.8.12", "anyhow", - "arrow", - "arrow-array", - "arrow-schema", + "arrow 55.2.0", + "arrow-array 55.2.0", + "arrow-schema 55.2.0", "async-stream", "async-trait", "backon", @@ -4733,7 +5026,7 @@ dependencies = [ "opentelemetry", "opentelemetry-otlp", "opentelemetry_sdk", - "parquet", + "parquet 55.2.0", "pico-args", "pprof", "prost 0.14.1", @@ -4761,9 +5054,9 @@ name = "moonlink_backend" version = "0.0.1" dependencies = [ "apache-avro 0.20.0", - "arrow", - "arrow-array", - "arrow-schema", + "arrow 55.2.0", + "arrow-array 55.2.0", + "arrow-schema 55.2.0", "futures", "iceberg", "moonlink", @@ -4774,7 +5067,7 @@ dependencies = [ "more-asserts", "native-tls", "nix 0.30.1", - "parquet", + "parquet 55.2.0", "postgres-native-tls", "rand 0.9.2", "roaring", @@ -4796,9 +5089,9 @@ version = "0.0.1" dependencies = [ "anyhow", "apache-avro 0.20.0", - "arrow", - "arrow-array", - "arrow-schema", + "arrow 55.2.0", + "arrow-array 55.2.0", + "arrow-schema 55.2.0", "async-stream", "async-trait", "bigdecimal", @@ -4814,7 +5107,7 @@ dependencies = [ "more-asserts", "native-tls", "num-traits", - "parquet", + "parquet 55.2.0", "pg_escape", "pin-project-lite", "postgres-native-tls", @@ -4835,8 +5128,8 @@ dependencies = [ name = "moonlink_datafusion" version = "0.0.1" dependencies = [ - "arrow", - "arrow-ipc", + "arrow 55.2.0", + "arrow-ipc 55.2.0", "async-trait", "bincode", "clap", @@ -4846,7 +5139,7 @@ dependencies = [ "moonlink_rpc", "moonlink_table_metadata", "object_store", - "parquet", + "parquet 55.2.0", "roaring", "tempfile", "thiserror 2.0.16", @@ -4915,11 +5208,11 @@ version = "0.0.1" dependencies = [ "anyhow", "apache-avro 0.20.0", - "arrow", - "arrow-array", - "arrow-buffer", - "arrow-ipc", - "arrow-schema", + "arrow 55.2.0", + "arrow-array 55.2.0", + "arrow-buffer 55.2.0", + "arrow-ipc 55.2.0", + "arrow-schema 55.2.0", "async-recursion", "axum 0.8.4", "bytes", @@ -4940,7 +5233,7 @@ dependencies = [ "opentelemetry-proto", "opentelemetry-stdout", "opentelemetry_sdk", - "parquet", + "parquet 55.2.0", "prost 0.14.1", "reqwest", "serde", @@ -5506,13 +5799,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b17da4150748086bd43352bc77372efa9b6e3dbd06a04831d2a98c041c225cfa" dependencies = [ "ahash 0.8.12", - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", - "arrow-ipc", - "arrow-schema", - "arrow-select", + "arrow-array 55.2.0", + "arrow-buffer 55.2.0", + "arrow-cast 55.2.0", + "arrow-data 55.2.0", + "arrow-ipc 55.2.0", + "arrow-schema 55.2.0", + "arrow-select 55.2.0", "base64 0.22.1", "brotli", "bytes", @@ -5536,6 +5829,42 @@ dependencies = [ "zstd", ] +[[package]] +name = "parquet" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0dbd48ad52d7dccf8ea1b90a3ddbfaea4f69878dd7683e51c507d4bc52b5b27" +dependencies = [ + "ahash 0.8.12", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-cast 56.2.0", + "arrow-data 56.2.0", + "arrow-ipc 56.2.0", + "arrow-schema 56.2.0", + "arrow-select 56.2.0", + "base64 0.22.1", + "brotli", + "bytes", + "chrono", + "flate2", + "futures", + "half", + "hashbrown 0.16.0", + "lz4_flex", + "num", + "num-bigint", + "object_store", + "paste", + "seq-macro", + "simdutf8", + "snap", + "thrift", + "tokio", + "twox-hash", + "zstd", +] + [[package]] name = "paste" version = "1.0.15" @@ -7375,9 +7704,9 @@ dependencies = [ [[package]] name = "sqlparser" -version = "0.56.0" +version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e68feb51ffa54fc841e086f58da543facfe3d7ae2a60d69b0a8cbbd30d16ae8d" +checksum = "4591acadbcf52f0af60eafbb2c003232b2b4cd8de5f0e9437cb8b1b59046cc0f" dependencies = [ "log", "recursive", diff --git a/Cargo.toml b/Cargo.toml index 975a36065..06c110f2c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,7 +40,7 @@ const_format = "0.2" crc32fast = "1" datafusion = "49" datafusion-cli = "49" -deltalake = "0.28" +deltalake = { version = "0.29", features = ["deltalake-aws", "deltalake-gcp"] } fastbloom = "0.12" futures = { version = "0.3", default-features = false } hashbrown = "0.15" diff --git a/src/moonlink/src/storage/mooncake_table/table_creation_test_utils.rs b/src/moonlink/src/storage/mooncake_table/table_creation_test_utils.rs index 8c9f93cf0..ed4ba2472 100644 --- a/src/moonlink/src/storage/mooncake_table/table_creation_test_utils.rs +++ b/src/moonlink/src/storage/mooncake_table/table_creation_test_utils.rs @@ -168,6 +168,41 @@ pub(crate) fn create_iceberg_table_config(warehouse_uri: String) -> IcebergTable } } +/// Test util function to create delta table config. +pub(crate) fn create_delta_table_config(warehouse_uri: String) -> DeltalakeTableConfig { + let accessor_config = if warehouse_uri.starts_with("s3://") { + #[cfg(feature = "storage-s3")] + { + s3_test_utils::create_s3_storage_config(&warehouse_uri) + } + #[cfg(not(feature = "storage-s3"))] + { + panic!("S3 support not enabled. Enable `storage-s3` feature."); + } + } else if warehouse_uri.starts_with("gs://") { + #[cfg(feature = "storage-gcs")] + { + gcs_test_utils::create_gcs_storage_config(&warehouse_uri) + } + #[cfg(not(feature = "storage-gcs"))] + { + panic!("GCS support not enabled. Enable `storage-gcs` feature."); + } + } else { + let storage_config = StorageConfig::FileSystem { + root_directory: warehouse_uri.clone(), + atomic_write_dir: None, + }; + AccessorConfig::new_with_storage_config(storage_config) + }; + + DeltalakeTableConfig { + table_name: DELTA_TEST_TABLE.to_string(), + location: warehouse_uri, + data_accessor_config: accessor_config, + } +} + /// Test util function to create arrow schema. pub(crate) fn create_test_arrow_schema() -> Arc { Arc::new(ArrowSchema::new(vec![ diff --git a/src/moonlink/src/storage/table/deltalake/test.rs b/src/moonlink/src/storage/table/deltalake/test.rs index 6f0ebd37d..5a0fd0450 100644 --- a/src/moonlink/src/storage/table/deltalake/test.rs +++ b/src/moonlink/src/storage/table/deltalake/test.rs @@ -2,8 +2,17 @@ use std::collections::{HashMap, HashSet}; use std::sync::Arc; use tempfile::TempDir; +#[cfg(feature = "storage-gcs")] +use crate::storage::filesystem::gcs::gcs_test_utils; +#[cfg(feature = "storage-gcs")] +use crate::storage::filesystem::gcs::test_guard::TestGuard as GcsTestGuard; +#[cfg(feature = "storage-s3")] +use crate::storage::filesystem::s3::s3_test_utils; +#[cfg(feature = "storage-s3")] +use crate::storage::filesystem::s3::test_guard::TestGuard as S3TestGuard; +use crate::storage::filesystem::accessor::factory::create_filesystem_accessor; use crate::storage::mooncake_table::table_creation_test_utils::{ - create_test_table_metadata, get_delta_table_config, + create_delta_table_config, create_test_table_metadata, get_delta_table_config }; use crate::storage::mooncake_table::table_operation_test_utils::create_local_parquet_file; use crate::storage::mooncake_table::{ @@ -12,17 +21,16 @@ use crate::storage::mooncake_table::{ }; use crate::storage::table::common::table_manager::TableManager; use crate::storage::table::common::table_manager::{PersistenceFileParams, PersistenceResult}; +use crate::storage::table::deltalake::deltalake_table_config::DeltalakeTableConfig; use crate::storage::table::deltalake::deltalake_table_manager::DeltalakeTableManager; use crate::{create_data_file, FileSystemAccessor, ObjectStorageCache}; -#[tokio::test] -async fn test_basic_store_and_load() { +async fn test_basic_store_and_load_impl(delta_table_config: DeltalakeTableConfig) { let temp_dir = TempDir::new().unwrap(); - let table_path = temp_dir.path().to_str().unwrap().to_string(); + let table_path = delta_table_config.location.clone(); let mooncake_table_metadata = create_test_table_metadata(table_path.clone()); - let filesystem_accessor = FileSystemAccessor::default_for_test(&temp_dir); - let delta_table_config = get_delta_table_config(&temp_dir); - + let filesystem_accessor = + create_filesystem_accessor(delta_table_config.data_accessor_config.clone()); let mut delta_table_manager = DeltalakeTableManager::new( mooncake_table_metadata.clone(), Arc::new(ObjectStorageCache::default_for_test(&temp_dir)), // Use independent object storage cache. @@ -145,3 +153,33 @@ async fn test_basic_store_and_load() { let dir_exists = tokio::fs::try_exists(table_path).await.unwrap(); assert!(!dir_exists); } + +#[tokio::test] +async fn test_basic_store_and_load() { + let temp_dir = TempDir::new().unwrap(); + let delta_table_config = get_delta_table_config(&temp_dir); + + test_basic_store_and_load_impl(delta_table_config).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[cfg(feature = "storage-s3")] +async fn test_basic_store_and_load_with_s3() { + let (bucket, warehouse_uri) = s3_test_utils::get_test_s3_bucket_and_warehouse(); + let _test_guard = S3TestGuard::new(bucket.clone()).await; + let delta_table_config = create_delta_table_config(warehouse_uri); + + test_basic_store_and_load_impl(delta_table_config).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[cfg(feature = "storage-gcs")] +async fn test_basic_store_and_load_with_gcs() { + let (bucket, warehouse_uri) = gcs_test_utils::get_test_gcs_bucket_and_warehouse(); + let _test_guard = GcsTestGuard::new(bucket.clone()).await; + let delta_table_config = create_delta_table_config(warehouse_uri); + + // Common testing logic. + test_basic_store_and_load_impl(delta_table_config.clone()).await; +} + diff --git a/src/moonlink/src/storage/table/deltalake/utils.rs b/src/moonlink/src/storage/table/deltalake/utils.rs index fab39bf23..5db291189 100644 --- a/src/moonlink/src/storage/table/deltalake/utils.rs +++ b/src/moonlink/src/storage/table/deltalake/utils.rs @@ -1,13 +1,84 @@ use deltalake::kernel::engine::arrow_conversion::TryFromArrow; +use deltalake::logstore::StorageConfig as DeltaStorageConfig; +use deltalake::open_table_with_storage_options; use deltalake::{open_table, operations::create::CreateBuilder, DeltaTable}; +use deltalake::arrow::datatypes::{DataType, Field, Schema as DeltaSchema}; +use std::collections::HashMap; use std::sync::Arc; +use url::Url; use crate::storage::filesystem::accessor::base_filesystem_accessor::BaseFileSystemAccess; use crate::storage::mooncake_table::TableMetadata as MooncakeTableMetadata; use crate::storage::table::deltalake::deltalake_table_config::DeltalakeTableConfig; +use crate::storage::StorageConfig as MoonlinkStorgaeConfig; use crate::CacheTrait; use crate::Result; +/// Get storage option to access deltalake table. +fn get_storage_option(storage_config: &MoonlinkStorgaeConfig) -> HashMap { + let mut storage_options = HashMap::new(); + + match storage_config { + #[cfg(feature = "storage-s3")] + MoonlinkStorgaeConfig::S3 { + access_key_id, + secret_access_key, + region, + bucket: _, + endpoint, + } => { + storage_options.insert("AWS_ACCESS_KEY_ID".into(), access_key_id.clone()); + storage_options.insert("AWS_SECRET_ACCESS_KEY".into(), secret_access_key.clone()); + storage_options.insert("AWS_REGION".into(), region.clone()); + + if let Some(endpoint) = endpoint { + storage_options.insert("AWS_ENDPOINT_URL".into(), endpoint.clone()); + } + } + #[cfg(feature = "storage-gcs")] + MoonlinkStorgaeConfig::Gcs { + project, + region, + bucket: _, + access_key_id, + secret_access_key, + endpoint, + disable_auth, + write_option: _, + } => { + storage_options.insert("GOOGLE_SERVICE_ACCOUNT".into(), project.clone()); + storage_options.insert("GOOGLE_REGION".into(), region.clone()); + storage_options.insert("GOOGLE_ACCESS_KEY_ID".into(), access_key_id.clone()); + storage_options.insert("GOOGLE_SECRET_ACCESS_KEY".into(), secret_access_key.clone()); + + if let Some(endpoint) = endpoint { + storage_options.insert("GOOGLE_ENDPOINT_URL".into(), endpoint.clone()); + } + if *disable_auth { + storage_options.insert("GOOGLE_DISABLE_AUTH".into(), "true".into()); + } + } + _ => {} + } + + storage_options +} + +/// Util function to convert arrow schema to delta schema. +fn to_deltalake_arrow_schema(schema: &arrow_schema::Schema) -> DeltaSchema { + DeltaSchema::new( + schema + .fields() + .iter() + .map(|f| { + // Clone the name, datatype, and nullability. + // You can extend this later for metadata or dict ids if needed. + Field::new(f.name(), f.data_type().clone(), f.is_nullable()) + }) + .collect::>(), + ) +} + /// Get or create a Delta table at the given location. /// /// - If the table doesn't exist → create a new one using the Arrow schema. @@ -20,11 +91,17 @@ pub(crate) async fn get_or_create_deltalake_table( _filesystem_accessor: Arc, config: DeltalakeTableConfig, ) -> Result { - match open_table(config.location.clone()).await { + let storage_options = get_storage_option(&config.data_accessor_config.storage_config); + + println!("storage option = {:?}", storage_options); + println!("location = {:?}", config.location); + + let table_uri = Url::parse(&config.location).unwrap(); + match open_table_with_storage_options(table_uri.clone(), storage_options).await { Ok(existing_table) => Ok(existing_table), Err(_) => { let arrow_schema = mooncake_table_metadata.schema.as_ref(); - let delta_schema_struct = deltalake::kernel::Schema::try_from_arrow(arrow_schema)?; + let delta_schema_struct = deltalake::kernel::Schema::try_from_arrow(arrow_schema).unwrap(); let delta_schema_fields = delta_schema_struct .fields .iter() @@ -32,7 +109,7 @@ pub(crate) async fn get_or_create_deltalake_table( .collect::>(); let table = CreateBuilder::new() - .with_location(config.location.clone()) + .with_location(table_uri) .with_columns(delta_schema_fields) .with_save_mode(deltalake::protocol::SaveMode::ErrorIfExists) .await?;