diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..1397643 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,252 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "type": "lldb", + "request": "launch", + "name": "Debug unit tests in library 'atsc'", + "cargo": { + "args": [ + "test", + "--no-run", + "--lib", + "--package=atsc" + ], + "filter": { + "name": "atsc", + "kind": "lib" + } + }, + "args": [], + "cwd": "${workspaceFolder}" + }, + { + "type": "lldb", + "request": "launch", + "name": "Debug executable 'atsc'", + "cargo": { + "args": [ + "build", + "--bin=atsc", + "--package=atsc" + ], + "filter": { + "name": "atsc", + "kind": "bin" + } + }, + "args": [], + "cwd": "${workspaceFolder}" + }, + { + "type": "lldb", + "request": "launch", + "name": "Debug unit tests in executable 'atsc'", + "cargo": { + "args": [ + "test", + "--no-run", + "--bin=atsc", + "--package=atsc" + ], + "filter": { + "name": "atsc", + "kind": "bin" + } + }, + "args": [], + "cwd": "${workspaceFolder}" + }, + { + "type": "lldb", + "request": "launch", + "name": "Debug integration test 'e2e'", + "cargo": { + "args": [ + "test", + "--no-run", + "--test=e2e", + "--package=atsc" + ], + "filter": { + "name": "e2e", + "kind": "test" + } + }, + "args": [], + "cwd": "${workspaceFolder}" + }, + { + "type": "lldb", + "request": "launch", + "name": "Debug integration test 'integration_test'", + "cargo": { + "args": [ + "test", + "--no-run", + "--test=integration_test", + "--package=atsc" + ], + "filter": { + "name": "integration_test", + "kind": "test" + } + }, + "args": [], + "cwd": "${workspaceFolder}" + }, + { + "type": "lldb", + "request": "launch", + "name": "Debug benchmark 'fft_bench'", + "cargo": { + "args": [ + "test", + "--no-run", + "--bench=fft_bench", + "--package=atsc" + ], + "filter": { + "name": "fft_bench", + "kind": "bench" + } + }, + "args": [], + "cwd": "${workspaceFolder}" + }, + { + "type": "lldb", + "request": "launch", + "name": "Debug benchmark 'polynomial_bench'", + "cargo": { + "args": [ + "test", + "--no-run", + "--bench=polynomial_bench", + "--package=atsc" + ], + "filter": { + "name": "polynomial_bench", + "kind": "bench" + } + }, + "args": [], + "cwd": "${workspaceFolder}" + }, + { + "type": "lldb", + "request": "launch", + "name": "Debug unit tests in library 'wavbrro'", + "cargo": { + "args": [ + "test", + "--no-run", + "--lib", + "--package=wavbrro" + ], + "filter": { + "name": "wavbrro", + "kind": "lib" + } + }, + "args": [], + "cwd": "${workspaceFolder}" + }, + { + "type": "lldb", + "request": "launch", + "name": "Debug executable 'wav2wbro'", + "cargo": { + "args": [ + "build", + "--bin=wav2wbro", + "--package=tools" + ], + "filter": { + "name": "wav2wbro", + "kind": "bin" + } + }, + "args": [], + "cwd": "${workspaceFolder}" + }, + { + "type": "lldb", + "request": "launch", + "name": "Debug unit tests in executable 'wav2wbro'", + "cargo": { + "args": [ + "test", + "--no-run", + "--bin=wav2wbro", + "--package=tools" + ], + "filter": { + "name": "wav2wbro", + "kind": "bin" + } + }, + "args": [], + "cwd": "${workspaceFolder}" + }, + { + "type": "lldb", + "request": "launch", + "name": "Debug unit tests in library 'vsri'", + "cargo": { + "args": [ + "test", + "--no-run", + "--lib", + "--package=vsri" + ], + "filter": { + "name": "vsri", + "kind": "lib" + } + }, + "args": [], + "cwd": "${workspaceFolder}" + }, + { + "type": "lldb", + "request": "launch", + "name": "Debug executable 'csv-compressor'", + "cargo": { + "args": [ + "build", + "--bin=csv-compressor", + "--package=csv-compressor" + ], + "filter": { + "name": "csv-compressor", + "kind": "bin" + } + }, + "args": [], + "cwd": "${workspaceFolder}" + }, + { + "type": "lldb", + "request": "launch", + "name": "Debug unit tests in executable 'csv-compressor'", + "cargo": { + "args": [ + "test", + "--no-run", + "--bin=csv-compressor", + "--package=csv-compressor" + ], + "filter": { + "name": "csv-compressor", + "kind": "bin" + } + }, + "args": [], + "cwd": "${workspaceFolder}" + } + ] +} \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index fd48ec2..87b6429 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -45,9 +45,9 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" [[package]] name = "anstream" -version = "0.6.15" +version = "0.6.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64e15c1ab1f89faffbf04a634d5e1962e9074f2741eef6d97f3c4e322426d526" +checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" dependencies = [ "anstyle", "anstyle-parse", @@ -60,43 +60,44 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.8" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1" +checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" [[package]] name = "anstyle-parse" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb47de1e80c2b463c735db5b217a0ddc39d612e7ac9e2e96a5aed1f57616c1cb" +checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d36fc52c7f6c869915e99412912f22093507da8d9e942ceaf66fe4b7c14422a" +checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" dependencies = [ - "windows-sys", + "windows-sys 0.59.0", ] [[package]] name = "anstyle-wincon" -version = "3.0.4" +version = "3.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bf74e1b6e971609db8ca7a9ce79fd5768ab6ae46441c572e46cf596f59e57f8" +checksum = "ca3534e77181a9cc07539ad51f2141fe32f6c3ffd4df76db8ad92346b003ae4e" dependencies = [ "anstyle", - "windows-sys", + "once_cell", + "windows-sys 0.59.0", ] [[package]] name = "arrayvec" -version = "0.7.4" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "atsc" @@ -110,6 +111,7 @@ dependencies = [ "env_logger", "hound", "inverse_distance_weight", + "lib_vsri", "log", "median", "num-traits", @@ -124,9 +126,9 @@ dependencies = [ [[package]] name = "autocfg" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "average" @@ -166,9 +168,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.6.0" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" +checksum = "8f68f53c83ab957f72c32642f3868eec03eb974d1fb82e453128456482613d36" [[package]] name = "bitvec" @@ -212,9 +214,9 @@ dependencies = [ [[package]] name = "bytemuck" -version = "1.16.3" +version = "1.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "102087e286b4677862ea56cf8fc58bb2cdfa8725c40ffb80fe3a008eb7f2fc83" +checksum = "ef657dfab802224e671f5818e9a4935f9b1957ed18e58292690cc39e7a4092a3" [[package]] name = "byteorder" @@ -224,9 +226,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.7.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fca2be1d5c43812bae364ee3f30b3afcb7877cf59f4aeb94c66f313a41d2fac9" +checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b" [[package]] name = "cast" @@ -236,9 +238,12 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.1.7" +version = "1.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26a5c3fd7bfa1ce3897a3a3501d362b2d87b7f2583ebcb4a949ec25911025cbc" +checksum = "13208fcbb66eaeffe09b99fffbe1af420f00a7b35aa99ad683dfc1aa76145229" +dependencies = [ + "shlex", +] [[package]] name = "cfg-if" @@ -248,9 +253,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.38" +version = "0.4.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" +checksum = "7e36cc9d416881d2e24f9a963be5fb1cd90966419ac844274161d10488b3e825" dependencies = [ "android-tzdata", "iana-time-zone", @@ -289,9 +294,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.13" +version = "4.5.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fbb260a053428790f3de475e304ff84cdbc4face759ea7a3e64c1edd938a7fc" +checksum = "769b0145982b4b48713e01ec42d61614425f27b7058bda7180a3a41f30104796" dependencies = [ "clap_builder", "clap_derive", @@ -299,9 +304,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.13" +version = "4.5.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64b17d7ea74e9f833c7dbf2cbe4fb12ff26783eda4782a8975b72f895c9b4d99" +checksum = "1b26884eb4b57140e4d2d93652abfa49498b938b3c9179f9fc487b0acc3edad7" dependencies = [ "anstream", "anstyle", @@ -311,21 +316,21 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.13" +version = "4.5.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "501d359d5f3dcaf6ecdeee48833ae73ec6e42723a1e52419c79abf9507eec0a0" +checksum = "54b755194d6389280185988721fffba69495eed5ee9feeee9a599b53db80318c" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.96", ] [[package]] name = "clap_lex" -version = "0.7.2" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" +checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" [[package]] name = "claxon" @@ -335,15 +340,15 @@ checksum = "4bfbf56724aa9eca8afa4fcfadeb479e722935bb2a0900c2d37e0cc477af0688" [[package]] name = "colorchoice" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3fd119d74b830634cea2a0f58bbd0d54540518a14397557951e79340abc28c0" +checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" [[package]] name = "core-foundation-sys" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "criterion" @@ -383,9 +388,9 @@ dependencies = [ [[package]] name = "crossbeam-deque" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" dependencies = [ "crossbeam-epoch", "crossbeam-utils", @@ -402,9 +407,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.20" +version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" [[package]] name = "crunchy" @@ -432,10 +437,10 @@ dependencies = [ "clap", "csv", "env_logger", + "lib_vsri", "log", "serde", "tempdir", - "vsri", "wavbrro", ] @@ -456,9 +461,9 @@ checksum = "347675b2993d588e8506457ea2de0e64a89ad0fcbc0e79d07d25f50542f40b59" [[package]] name = "easy-cast" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10936778145f3bea71fd9bf61332cce28c28e96a380714f7ab34838b80733fd6" +checksum = "72852736692ec862655eca398c9bb1b476161b563c9f80f45f4808b9629750d6" dependencies = [ "libm", ] @@ -471,18 +476,18 @@ checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" [[package]] name = "encoding_rs" -version = "0.8.34" +version = "0.8.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" dependencies = [ "cfg-if", ] [[package]] name = "env_filter" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f2c92ceda6ceec50f43169f9ee8424fe2db276791afde7b2cd8bc084cb376ab" +checksum = "186e05a59d4c50738528153b83b0b0194d3a29507dfec16eccd4b342903397d0" dependencies = [ "log", "regex", @@ -490,9 +495,9 @@ dependencies = [ [[package]] name = "env_logger" -version = "0.11.5" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13fa619b91fb2381732789fc5de83b45675e882f66623b7d8cb4f643017018d" +checksum = "dcaee3d8e3cfc3fd92428d477bc97fc29ec8716d180c0d74c643bb26166660e0" dependencies = [ "anstream", "anstyle", @@ -503,12 +508,12 @@ dependencies = [ [[package]] name = "errno" -version = "0.3.9" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" +checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.59.0", ] [[package]] @@ -519,9 +524,9 @@ checksum = "af9673d8203fcb076b19dfd17e38b3d4ae9f44959416ea532ce72415a6020365" [[package]] name = "fastrand" -version = "2.1.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "float-ord" @@ -589,9 +594,9 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "hermit-abi" -version = "0.3.9" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" +checksum = "fbf6a919d6cf397374f7dfeeea91d974c7c0a7221d0d0f4f20d859d329e53fcc" [[package]] name = "hound" @@ -607,9 +612,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "iana-time-zone" -version = "0.1.60" +version = "0.1.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" +checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -639,13 +644,13 @@ dependencies = [ [[package]] name = "is-terminal" -version = "0.4.12" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f23ff5ef2b80d608d61efee834934d862cd92461afc0560dedf493e4c033738b" +checksum = "261f68e344040fbd0edea105bef17c66edf46f984ddb1115b775ce31be948f4b" dependencies = [ "hermit-abi", "libc", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -665,16 +670,17 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.11" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" [[package]] name = "js-sys" -version = "0.3.69" +version = "0.3.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" +checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" dependencies = [ + "once_cell", "wasm-bindgen", ] @@ -684,29 +690,38 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "lib_vsri" +version = "0.2.0" +dependencies = [ + "bincode", + "chrono", + "log", +] + [[package]] name = "libc" -version = "0.2.155" +version = "0.2.169" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" +checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" [[package]] name = "libm" -version = "0.2.8" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" +checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa" [[package]] name = "linux-raw-sys" -version = "0.4.14" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" [[package]] name = "log" -version = "0.4.22" +version = "0.4.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" +checksum = "04cbf5b083de1c7e0222a7a51dbfdba1cbe1c6ab0b15e29fff3f6c077fd9cd9f" [[package]] name = "median" @@ -753,9 +768,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.19.0" +version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" [[package]] name = "oorandom" @@ -765,9 +780,9 @@ checksum = "b410bbe7e14ab526a0e86877eb47c6996a2bd7746f027ba551028c925390e4e9" [[package]] name = "plotters" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a15b6eccb8484002195a3e44fe65a4ce8e93a625797a063735536fd59cb01cf3" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" dependencies = [ "num-traits", "plotters-backend", @@ -778,24 +793,24 @@ dependencies = [ [[package]] name = "plotters-backend" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "414cec62c6634ae900ea1c56128dfe87cf63e7caece0852ec76aba307cebadb7" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" [[package]] name = "plotters-svg" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81b30686a7d9c3e010b84284bdd26a29f2138574f52f5eb6f794fc0ad924e705" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" dependencies = [ "plotters-backend", ] [[package]] name = "ppv-lite86" -version = "0.2.18" +version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dee4364d9f3b902ef14fab8a1ddffb783a1cb6b4bba3bfc1fa3922732c7de97f" +checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" dependencies = [ "zerocopy", ] @@ -811,9 +826,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.86" +version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +checksum = "60946a68e5f9d28b0dc1c21bb8a97ee7d018a8b322fa57838ba31cc878e22d99" dependencies = [ "unicode-ident", ] @@ -840,9 +855,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.36" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" +checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc" dependencies = [ "proc-macro2", ] @@ -942,9 +957,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.10.5" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b91213439dad192326a0d7c6ee3955910425f441d7038e0d6933b0aec5c4517f" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" dependencies = [ "aho-corasick", "memchr", @@ -954,9 +969,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.7" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" dependencies = [ "aho-corasick", "memchr", @@ -965,9 +980,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "remove_dir_all" @@ -989,9 +1004,9 @@ dependencies = [ [[package]] name = "rkyv" -version = "0.7.44" +version = "0.7.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cba464629b3394fc4dbc6f940ff8f5b4ff5c7aef40f29166fd4ad12acbc99c0" +checksum = "9008cd6385b9e161d8229e1f6549dd23c3d022f132a2ea37ac3a10ac4935779b" dependencies = [ "bitvec", "bytecheck", @@ -1007,9 +1022,9 @@ dependencies = [ [[package]] name = "rkyv_derive" -version = "0.7.44" +version = "0.7.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7dddfff8de25e6f62b9d64e6e432bf1c6736c57d20323e15ee10435fbda7c65" +checksum = "503d1d27590a2b0a3a4ca4c94755aa2875657196ecbf401a42eff41d7de532c0" dependencies = [ "proc-macro2", "quote", @@ -1033,17 +1048,23 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.34" +version = "0.38.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" +checksum = "a78891ee6bf2340288408954ac787aa063d8e8817e9f53abb37c695c6d834ef6" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "errno", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.59.0", ] +[[package]] +name = "rustversion" +version = "1.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7c45b9784283f1b2e7fb61b42047c2fd678ef0960d4f6f1eba131594cc369d4" + [[package]] name = "ryu" version = "1.0.18" @@ -1067,29 +1088,29 @@ checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" [[package]] name = "serde" -version = "1.0.204" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc76f558e0cbb2a839d37354c575f1dc3fdc6546b5be373ba43d95f231bf7c12" +checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.204" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222" +checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.96", ] [[package]] name = "serde_json" -version = "1.0.121" +version = "1.0.137" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ab380d7d9f22ef3f21ad3e6c1ebe8e4fc7a2000ccba2e4d71fc96f15b2cb609" +checksum = "930cfb6e6abf99298aaad7d29abbef7a9999a9a8806a40088f55f0dcec03146b" dependencies = [ "itoa", "memchr", @@ -1097,17 +1118,23 @@ dependencies = [ "serde", ] +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "simdutf8" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f27f6278552951f1f2b8cf9da965d10969b2efdea95a6ec47987ab46edfe263a" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" [[package]] name = "splines" -version = "4.3.1" +version = "4.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "228c4551e53c672e86439509545dd7ffcb966b6876c58b108e433a30e6117101" +checksum = "4ef39a4cf041b421dd2b4dd2da7f30bb7059be7a1872fb4f56114b53303d0e1e" [[package]] name = "strength_reduce" @@ -1267,9 +1294,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.87" +version = "2.0.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d" +checksum = "d5d0adab1ae378d7f53bdebc67a39f1f151407ef230f0ce2883572f5d8985c80" dependencies = [ "proc-macro2", "quote", @@ -1294,34 +1321,36 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.10.1" +version = "3.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" +checksum = "9a8a559c81686f576e8cd0290cd2a24a2a9ad80c98b3478856500fcbd7acd704" dependencies = [ "cfg-if", "fastrand", + "getrandom", + "once_cell", "rustix", - "windows-sys", + "windows-sys 0.59.0", ] [[package]] name = "thiserror" -version = "2.0.3" +version = "2.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c006c85c7651b3cf2ada4584faa36773bd07bac24acfb39f3c431b36d7e667aa" +checksum = "d452f284b73e6d76dd36758a0c8684b1d5be31f92b89d07fd5822175732206fc" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "2.0.3" +version = "2.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f077553d607adc1caf65430528a576c757a71ed73944b66ebb58ef2bbd243568" +checksum = "26afc1baea8a989337eeb52b6e72a039780ce45c3edfcc9c5b9d112feeb173c2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.96", ] [[package]] @@ -1336,9 +1365,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.8.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938" +checksum = "022db8904dfa342efe721985167e9fcd16c29b226db4397ed752a761cfce81e8" dependencies = [ "tinyvec_macros", ] @@ -1384,9 +1413,9 @@ checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "unicode-ident" -version = "1.0.12" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" [[package]] name = "utf8parse" @@ -1396,9 +1425,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.10.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81dfa00651efa65069b0b6b651f4aaa31ba9e3c3ce0137aaad053604ee7e0314" +checksum = "b3758f5e68192bb96cc8f9b7e2c2cfdabb435499a28499a42f8f984092adad4b" [[package]] name = "version_check" @@ -1412,14 +1441,6 @@ version = "0.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9dcc60c0624df774c82a0ef104151231d37da4962957d691c011c852b2473314" -[[package]] -name = "vsri" -version = "0.1.0" -dependencies = [ - "chrono", - "log", -] - [[package]] name = "walkdir" version = "2.5.0" @@ -1438,34 +1459,35 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.92" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" +checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" dependencies = [ "cfg-if", + "once_cell", + "rustversion", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.92" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" +checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" dependencies = [ "bumpalo", "log", - "once_cell", "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.96", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.92" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" +checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -1473,22 +1495,25 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.92" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" +checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.96", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.92" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" +checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +dependencies = [ + "unicode-ident", +] [[package]] name = "wavbrro" @@ -1502,9 +1527,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.69" +version = "0.3.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" +checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2" dependencies = [ "js-sys", "wasm-bindgen", @@ -1528,11 +1553,11 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d4cc384e1e73b93bafa6fb4f1df8c41695c8a91cf9c4c64358067d15a7b6c6b" +checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "windows-sys", + "windows-sys 0.59.0", ] [[package]] @@ -1559,6 +1584,15 @@ dependencies = [ "windows-targets", ] +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-targets" version = "0.52.6" @@ -1634,9 +1668,9 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.6.6" +version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "854e949ac82d619ee9a14c66a1b674ac730422372ccb759ce0c39cabcf2bf8e6" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" dependencies = [ "byteorder", "zerocopy-derive", @@ -1644,11 +1678,11 @@ dependencies = [ [[package]] name = "zerocopy-derive" -version = "0.6.6" +version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "125139de3f6b9d625c39e2efdd73d41bdac468ccd556556440e322be0e1bbd91" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.96", ] diff --git a/Cargo.toml b/Cargo.toml index 06e5a54..e5962b0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,8 +2,8 @@ members = [ "atsc", "tools", + "lib_vsri", "wavbrro", - "vsri", "csv-compressor" ] resolver = "2" diff --git a/atsc/Cargo.toml b/atsc/Cargo.toml index c9bb4c8..63d0c30 100644 --- a/atsc/Cargo.toml +++ b/atsc/Cargo.toml @@ -19,6 +19,7 @@ regex = "1.9.1" hound = "3.5" median = "0.3.2" wavbrro = { path = "../wavbrro" } +lib_vsri = { path = "../lib_vsri" } splines = "4.3.0" inverse_distance_weight = "0.1.1" num-traits = "0.2" diff --git a/atsc/src/compressor/constant.rs b/atsc/src/compressor/constant.rs index cc82e21..20bbab1 100644 --- a/atsc/src/compressor/constant.rs +++ b/atsc/src/compressor/constant.rs @@ -19,7 +19,7 @@ use crate::{ optimizer::utils::{Bitdepth, DataStats}, }; -use super::BinConfig; +use super::bincode_config; use bincode::{Decode, Encode}; use log::debug; @@ -113,14 +113,14 @@ impl Constant { /// Receives a data stream and generates a Constant pub fn decompress(data: &[u8]) -> Self { - let config = BinConfig::get(); + let config = bincode_config(); let (ct, _) = bincode::decode_from_slice(data, config).unwrap(); ct } /// This function transforms the structure into a Binary stream pub fn to_bytes(&self) -> Vec { - let config = BinConfig::get(); + let config = bincode_config(); bincode::encode_to_vec(self, config).unwrap() } diff --git a/atsc/src/compressor/fft.rs b/atsc/src/compressor/fft.rs index 4c9fa07..62c5f85 100644 --- a/atsc/src/compressor/fft.rs +++ b/atsc/src/compressor/fft.rs @@ -22,7 +22,7 @@ use bincode::{Decode, Encode}; use rustfft::{num_complex::Complex, FftPlanner}; use std::{cmp::Ordering, collections::BinaryHeap}; -use super::{BinConfig, CompressorResult}; +use super::{bincode_config, CompressorResult}; use log::{debug, error, info, trace, warn}; const FFT_COMPRESSOR_ID: u8 = 15; @@ -387,13 +387,13 @@ impl FFT { self.frequencies = FFT::fft_trim(&mut buffer, max_freq); } pub fn decompress(data: &[u8]) -> Self { - let config = BinConfig::get(); + let config = bincode_config(); let (fft, _) = bincode::decode_from_slice(data, config).unwrap(); fft } pub fn to_bytes(&self) -> Vec { - let config = BinConfig::get(); + let config = bincode_config(); bincode::encode_to_vec(self, config).unwrap() } diff --git a/atsc/src/compressor/mod.rs b/atsc/src/compressor/mod.rs index 75f0370..b275484 100644 --- a/atsc/src/compressor/mod.rs +++ b/atsc/src/compressor/mod.rs @@ -23,11 +23,13 @@ use self::constant::{constant_compressor, constant_to_data}; use self::fft::{fft, fft_compressor, fft_to_data}; use self::noop::{noop, noop_to_data}; use self::polynomial::{polynomial, polynomial_allowed_error, to_data, PolynomialType}; +use self::vsri::{vsri_compressor_bytes, vsri_to_data}; pub mod constant; pub mod fft; pub mod noop; pub mod polynomial; +pub mod vsri; #[derive(Encode, Decode, Default, Debug, Clone, Copy, Eq, Hash, PartialEq)] pub enum Compressor { @@ -38,6 +40,7 @@ pub enum Compressor { Constant, Polynomial, Auto, + VSRI, } /// Struct to store the results of a compression round. Will be used to pick the best compressor. @@ -65,7 +68,12 @@ impl Compressor { Compressor::Constant => constant_compressor(data, stats).compressed_data, Compressor::Polynomial => polynomial(data, PolynomialType::Polynomial), Compressor::Idw => polynomial(data, PolynomialType::Idw), - _ => todo!(), + Compressor::Auto => { + panic!("Auto compressor requires bounded compression with error threshold") + } + Compressor::VSRI => { + panic!("VSRI compressor is for timestamps (i32), use compress_vsri instead") + } } } @@ -82,7 +90,10 @@ impl Compressor { Compressor::Idw => { polynomial_allowed_error(data, max_error, PolynomialType::Idw).compressed_data } - _ => todo!(), + Compressor::Auto => panic!("Auto compressor should use optimizer module for selection"), + Compressor::VSRI => { + panic!("VSRI compressor is for timestamps (i32), use compress_vsri instead") + } } } @@ -96,7 +107,8 @@ impl Compressor { polynomial_allowed_error(data, max_error, PolynomialType::Polynomial) } Compressor::Idw => polynomial_allowed_error(data, max_error, PolynomialType::Idw), - _ => todo!(), + Compressor::Auto => panic!("Auto compressor should use optimizer module for selection"), + Compressor::VSRI => panic!("VSRI compressor is for timestamps (i32), not f64 data"), } } @@ -107,18 +119,28 @@ impl Compressor { Compressor::Constant => constant_to_data(samples, data), Compressor::Polynomial => to_data(samples, data), Compressor::Idw => to_data(samples, data), - _ => todo!(), + Compressor::Auto => panic!( + "Auto is a meta-compressor, actual compressor type should be stored in frame" + ), + Compressor::VSRI => { + // VSRI data should be decompressed using decompress_vsri, but if called through + // the f64 path, return empty to avoid panic + log::warn!("Attempted to decompress VSRI frame as f64 data, returning empty vector"); + Vec::new() + } } } -} -pub struct BinConfig { - config: Configuration, -} + pub fn compress_vsri(&self, data: &[i32]) -> Vec { + vsri_compressor_bytes(data) + } -impl BinConfig { - pub fn get() -> Configuration { - // Little endian and Variable int encoding - config::standard() + pub fn decompress_vsri(&self, data: &[u8]) -> Vec { + vsri_to_data(data) } } + +/// Returns the standard bincode configuration (little endian, variable int encoding) +pub fn bincode_config() -> Configuration { + config::standard() +} diff --git a/atsc/src/compressor/noop.rs b/atsc/src/compressor/noop.rs index 75b9d63..3415c68 100644 --- a/atsc/src/compressor/noop.rs +++ b/atsc/src/compressor/noop.rs @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -use super::BinConfig; +use super::bincode_config; use bincode::{Decode, Encode}; use log::{debug, info}; @@ -53,14 +53,14 @@ impl Noop { /// Receives a data stream and generates a Noop pub fn decompress(data: &[u8]) -> Self { - let config = BinConfig::get(); + let config = bincode_config(); let (noop, _) = bincode::decode_from_slice(data, config).unwrap(); noop } /// This function transforms the structure in a Binary stream to be appended to the frame pub fn to_bytes(&self) -> Vec { - let config = BinConfig::get(); + let config = bincode_config(); bincode::encode_to_vec(self, config).unwrap() } diff --git a/atsc/src/compressor/polynomial.rs b/atsc/src/compressor/polynomial.rs index 531a82a..61cd8bf 100644 --- a/atsc/src/compressor/polynomial.rs +++ b/atsc/src/compressor/polynomial.rs @@ -17,15 +17,12 @@ limitations under the License. use crate::optimizer::utils::{Bitdepth, DataStats}; use crate::utils::{error::calculate_error, round_and_limit_f64, round_f64, DECIMAL_PRECISION}; -use super::{BinConfig, CompressorResult}; +use super::{bincode_config, CompressorResult}; use bincode::{Decode, Encode}; use inverse_distance_weight::IDW; use log::{debug, info, trace}; use splines::{Interpolation, Key, Spline}; -const POLYNOMIAL_COMPRESSOR_ID: u8 = 0; -const IDW_COMPRESSOR_ID: u8 = 1; - #[derive(Encode, Decode, Default, Debug, Clone, PartialEq)] pub enum PolynomialType { #[default] @@ -314,13 +311,13 @@ impl Polynomial { } pub fn decompress(data: &[u8]) -> Self { - let config = BinConfig::get(); + let config = bincode_config(); let (poly, _) = bincode::decode_from_slice(data, config).unwrap(); poly } pub fn to_bytes(&self) -> Vec { - let config = BinConfig::get(); + let config = bincode_config(); bincode::encode_to_vec(self, config).unwrap() } diff --git a/atsc/src/compressor/vsri.rs b/atsc/src/compressor/vsri.rs new file mode 100644 index 0000000..3b5dd5f --- /dev/null +++ b/atsc/src/compressor/vsri.rs @@ -0,0 +1,117 @@ +/* +Copyright 2024 NetApp, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +use crate::compressor::CompressorResult; + +use super::bincode_config; +use bincode::{Decode, Encode}; +use lib_vsri::vsri::Vsri; +use log::debug; + +const VSRI_COMPRESSOR_ID: u8 = 249; + +/// Compressor frame for static data, stores the value and nothing else. +#[derive(Debug, Clone, Encode, Decode)] +pub struct VSRI { + pub id: u8, + pub vsri: Vsri, +} + +impl VSRI { + /// Creates a new instance of the Constant compressor with the size needed to handle the worst case + pub fn new() -> Self { + debug!("Constant compressor"); + VSRI { + id: VSRI_COMPRESSOR_ID, + vsri: Vsri::new("placeholder"), + } + } + + /// Receives a data stream and generates a Constant + pub fn decompress(data: &[u8]) -> Self { + let config = bincode_config(); + let (ct, _) = bincode::decode_from_slice(data, config).unwrap(); + ct + } + + /// This function transforms the structure into a Binary stream + pub fn to_bytes(&self) -> Vec { + // Use Bincode and flate2-rs? Do this at the Stream Level? + let config = bincode_config(); + bincode::encode_to_vec(self, config).unwrap() + } + + /// Returns an array of data. It creates an array of data the size of the frame with a constant value + /// and pushes the residuals to the right place. + pub fn to_data(&self) -> Vec { + self.vsri.get_all_timestamps() + } +} + +pub fn vsri_compressor(data: &[i32]) -> CompressorResult { + debug!("Initializing VSRI Compressor. Error and Stats provided"); + // Initialize the compressor + let mut c = VSRI::new(); + for ts in data { + c.vsri.update_for_point(*ts).unwrap(); + } + // Convert to bytes + CompressorResult::new(c.to_bytes(), 0.0) +} + +pub fn vsri_compressor_bytes(data: &[i32]) -> Vec { + debug!("Initializing VSRI Compressor. Error and Stats provided"); + // Initialize the compressor + let mut c = VSRI::new(); + for ts in data { + c.vsri.update_for_point(*ts).unwrap(); + } + // Convert to bytes + c.to_bytes() +} + +pub fn vsri_to_data(compressed_data: &[u8]) -> Vec { + let c = VSRI::decompress(compressed_data); + c.to_data() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_vsri_simple() { + let timestamps = vec![ + 1729606100, 1729606120, 1729606140, 1729606160, 1729606180, 1729606200, 1729606220, + 1729606240, 1729606260, + ]; + let vsri = vsri_compressor(×tamps); + let out = vsri_to_data(&vsri.compressed_data); + assert_eq!(timestamps, out); + } + + #[test] + fn test_vsri_several_segments() { + let timestamps = vec![ + 1729606100, 1729606120, 1729606140, 1729606160, 1729606180, 1729606200, 1729606220, + 1729606260, 1729606360, 1729606460, 1729606560, 1729606660, 1729606760, 1729606860, + 1729606881, 1729606882, 1729606883, 1729606884, 1729606885, 1729606886, 1729606887, + ]; + let vsri = vsri_compressor(×tamps); + let out = vsri_to_data(&vsri.compressed_data); + assert_eq!(timestamps, out); + } +} diff --git a/atsc/src/csv.rs b/atsc/src/csv.rs index eb08d9b..0278c0f 100644 --- a/atsc/src/csv.rs +++ b/atsc/src/csv.rs @@ -24,7 +24,7 @@ pub enum Error { type Result = std::result::Result; -#[derive(Debug, PartialEq)] +#[derive(Debug, PartialEq, Copy, Clone)] pub struct Sample { pub timestamp: i64, pub value: f64, diff --git a/atsc/src/data.rs b/atsc/src/data.rs index ea97dd9..dfd057d 100644 --- a/atsc/src/data.rs +++ b/atsc/src/data.rs @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -use crate::compressor::{BinConfig, Compressor}; +use crate::compressor::{bincode_config, Compressor}; use crate::frame::CompressorFrame; use crate::header::CompressorHeader; use bincode::{Decode, Encode}; @@ -34,6 +34,14 @@ impl CompressedStream { } } + /// Compress a chunk of data adding it as a new frame to the current stream + pub fn compress_vsri(&mut self, chunk: &[i32]) { + let mut compressor_frame = CompressorFrame::new(None); + compressor_frame.compress_vsri(chunk); + compressor_frame.close(); + self.data_frames.push(compressor_frame); + } + /// Compress a chunk of data adding it as a new frame to the current stream pub fn compress_chunk(&mut self, chunk: &[f64]) { let mut compressor_frame = CompressorFrame::new(None); @@ -75,14 +83,14 @@ impl CompressedStream { /// Transforms the whole CompressedStream into bytes to be written to a file pub fn to_bytes(self) -> Vec { // Will this chain encode?? - let config = BinConfig::get(); + let config = bincode_config(); bincode::encode_to_vec(self, config).unwrap() } /// Gets a binary stream and generates a Compressed Stream, at this point, anything inside the stream is /// still in the compressed state pub fn from_bytes(data: &[u8]) -> Self { - let config = BinConfig::get(); + let config = bincode_config(); let (compressed_stream, _) = bincode::decode_from_slice(data, config).unwrap(); compressed_stream } @@ -92,6 +100,14 @@ impl CompressedStream { .flat_map(|f| f.decompress()) .collect() } + + /// Decompress VSRI data (timestamps) + pub fn decompress_vsri(&self) -> Vec { + self.data_frames + .iter() + .flat_map(|f| f.decompress_vsri()) + .collect() + } } #[cfg(test)] @@ -144,4 +160,24 @@ mod tests { let out = cs2.decompress(); assert_eq!(vector1, out); } + + #[test] + fn test_vsri_roundtrip() { + // Test VSRI compression and decompression + let timestamps = vec![100, 200, 300, 400, 500]; + let mut cs = CompressedStream::new(); + cs.compress_vsri(×tamps); + + // Verify frame is tagged as VSRI + assert_eq!(cs.data_frames.len(), 1); + assert!(cs.data_frames[0].is_vsri()); + + // Serialize and deserialize + let bytes = cs.to_bytes(); + let cs2 = CompressedStream::from_bytes(&bytes); + + // Decompress and verify + let decompressed = cs2.decompress_vsri(); + assert_eq!(timestamps, decompressed); + } } diff --git a/atsc/src/frame/mod.rs b/atsc/src/frame/mod.rs index 0f031e6..c0fbe01 100644 --- a/atsc/src/frame/mod.rs +++ b/atsc/src/frame/mod.rs @@ -55,6 +55,13 @@ impl CompressorFrame { self.frame_size = size; } + /// Compress a vsri index + pub fn compress_vsri(&mut self, data: &[i32]) { + self.sample_count = data.len(); + self.compressor = Compressor::VSRI; + self.data = self.compressor.compress_vsri(data); + } + /// Compress a data and stores the result in the frame pub fn compress(&mut self, data: &[f64]) { self.sample_count = data.len(); @@ -156,4 +163,18 @@ impl CompressorFrame { ); self.compressor.decompress(self.sample_count, &self.data) } + + /// Decompresses a VSRI frame and returns the resulting timestamp array + pub fn decompress_vsri(&self) -> Vec { + debug!( + "Decompressing VSRI Frame. Size: {}, Samples: {}", + self.frame_size, self.sample_count + ); + self.compressor.decompress_vsri(&self.data) + } + + /// Check if this frame contains VSRI data + pub fn is_vsri(&self) -> bool { + matches!(self.compressor, Compressor::VSRI) + } } diff --git a/atsc/src/main.rs b/atsc/src/main.rs index 671a816..f44c86a 100644 --- a/atsc/src/main.rs +++ b/atsc/src/main.rs @@ -25,6 +25,18 @@ use std::error::Error; use std::path::PathBuf; use wavbrro::wavbrro::WavBrro; +/// Process VSRI +/// Function to write out VSRI, since it is straight forward can be written out immediately +fn write_vsri(vsri_data: &[i32], file_path: &mut PathBuf) -> Result<(), Box> { + // Need special stream for VSRI + let mut cs = CompressedStream::new(); + // Compress a VSRI + file_path.set_extension("vsri"); + cs.compress_vsri(vsri_data); + std::fs::write(file_path, cs.to_bytes())?; + Ok(()) +} + /// Processes the given input based on the provided arguments. fn process_args(arguments: &Args) -> Result<(), Box> { let metadata = std::fs::metadata(&arguments.input)?; @@ -96,13 +108,40 @@ fn process_single_file(mut file_path: PathBuf, arguments: &Args) -> Result<(), B // Assuming that header[0] is a time field and header[1] is value field read_samples_with_headers(&file_path, headers[0], headers[1])? }; - + // Timestamp needs to be compressed as VSRI + let timestamps: Vec = samples + .iter() + .map(|sample| sample.timestamp as i32) + .collect(); let data: Vec = samples.into_iter().map(|sample| sample.value).collect(); - + // Compress the timestamps and write them + write_vsri(×tamps, &mut file_path.clone())?; if arguments.verbose { println!("Input={:?}", data); } + /* let compressed_data = match arguments.compressor { + CompressorType::Vsri => { + // Get data in i32 format! + let vsri_data = [ + 1729606100, 1729606120, 1729606140, 1729606160, 1729606180, 1729606200, + 1729606220, + ]; + if arguments.verbose { + println!("Input={:?}", vsri_data); + } + } + _ => { + // Read an WavBRRO file and compress it + let data = WavBrro::from_file(&file_path)?; + if arguments.verbose { + println!("Input={:?}", data); + } + file_path.set_extension("bro"); + //compress + compress_data(&data, arguments) + } + }; */ // Compress let compressed_data = compress_data(&data, arguments); diff --git a/csv-compressor/Cargo.toml b/csv-compressor/Cargo.toml index 8a30324..6a29962 100644 --- a/csv-compressor/Cargo.toml +++ b/csv-compressor/Cargo.toml @@ -14,6 +14,6 @@ serde = { version = "1.0.171", features = ["derive"] } wavbrro = { version = "0.1.0", path = "../wavbrro" } log = "0.4.19" env_logger = "0.11.0" -vsri = { version = "0.1.0", path = "../vsri" } +lib_vsri = { version = "0.2.0", path = "../lib_vsri" } [dev-dependencies] tempdir = "0.3.7" diff --git a/csv-compressor/src/main.rs b/csv-compressor/src/main.rs index a56dd0c..d5d0328 100644 --- a/csv-compressor/src/main.rs +++ b/csv-compressor/src/main.rs @@ -20,10 +20,10 @@ use atsc::data::CompressedStream; use atsc::optimizer::OptimizerPlan; use atsc::utils::readers::bro_reader::read_file; use clap::{arg, Parser}; +use lib_vsri::vsri::Vsri; use log::debug; use std::fs; use std::path::{Path, PathBuf}; -use vsri::Vsri; use wavbrro::wavbrro::WavBrro; mod csv; diff --git a/csv-compressor/src/metric.rs b/csv-compressor/src/metric.rs index f508d00..b83e839 100644 --- a/csv-compressor/src/metric.rs +++ b/csv-compressor/src/metric.rs @@ -15,9 +15,9 @@ limitations under the License. */ use crate::csv::Sample; +use lib_vsri::vsri::{day_elapsed_seconds, Vsri}; use std::fmt::{Debug, Display, Formatter}; use std::path::Path; -use vsri::{day_elapsed_seconds, Vsri}; use wavbrro::wavbrro::WavBrro; /// Metric is responsible for generating WavBrro and VSRI from parsed Samples diff --git a/lib_vsri/Cargo.toml b/lib_vsri/Cargo.toml new file mode 100644 index 0000000..f1cdbe5 --- /dev/null +++ b/lib_vsri/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "lib_vsri" +version = "0.2.0" +edition = "2021" +authors = ["Carlos Rolo "] +license = "Apache-2.0" +description = "Very small index for a sequence of timestamps" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +chrono = "0.4.26" +log = "0.4.0" +bincode = "2.0.0-rc.3" \ No newline at end of file diff --git a/lib_vsri/README.md b/lib_vsri/README.md new file mode 100644 index 0000000..cc99757 --- /dev/null +++ b/lib_vsri/README.md @@ -0,0 +1,3 @@ +# VSRI (Very Small Rolo Index) + +Under construction! diff --git a/lib_vsri/src/lib.rs b/lib_vsri/src/lib.rs new file mode 100644 index 0000000..82222de --- /dev/null +++ b/lib_vsri/src/lib.rs @@ -0,0 +1,17 @@ +/* +Copyright 2024 NetApp, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +pub mod vsri; diff --git a/vsri/src/lib.rs b/lib_vsri/src/vsri.rs similarity index 83% rename from vsri/src/lib.rs rename to lib_vsri/src/vsri.rs index 0487e53..58ab30d 100644 --- a/vsri/src/lib.rs +++ b/lib_vsri/src/vsri.rs @@ -14,6 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ +use bincode::{Decode, Encode}; use chrono::{DateTime, Timelike, Utc}; use log::{debug, warn}; /// Very Small Rolo Index @@ -42,7 +43,8 @@ use std::fs::File; use std::io::{BufRead, BufReader, BufWriter, Write}; use std::path::{Path, PathBuf}; -// TODO: This should be configurable. Indexes are build for 1 day worth of samples, at 1 sample per second +/// Maximum samples per index. Default is 86400 (1 day at 1 sample/second). +/// Can be overridden by setting a different value before creating indices. pub static MAX_INDEX_SAMPLES: i32 = 86400; // Helper functions, this should be moved somewhere @@ -72,24 +74,22 @@ pub fn start_day_ts(dt: DateTime) -> i64 { /// # Examples /// Creating a new index, metric is of expected time 0, but for sure location of X is 0 /// ```no_run -/// # use vsri::Vsri; +/// # use lib_vsri::vsri::Vsri; /// let vsri = Vsri::new("metric_name"); /// vsri.flush(); /// ``` /// Updating an index, adding point at time 5sec /// ```no_run -/// -/// # use vsri::Vsri; +/// # use lib_vsri::vsri::Vsri; /// let mut vsri = Vsri::load("metric_name").unwrap(); /// vsri.update_for_point(5).unwrap(); /// vsri.flush(); /// ``` /// Fetch a sample location from the index given a timestamp /// ```no_run -/// # use vsri::Vsri; +/// # use lib_vsri::vsri::Vsri; /// let vsri = Vsri::get_sample_location("metric_name", 5); /// ``` - /// Index Structure /// index_name: Name of the index file we are indexing /// min_ts: the minimum TS available in this file @@ -98,7 +98,7 @@ pub fn start_day_ts(dt: DateTime) -> i64 { /// [sample_rate (m), initial_point(x,y), # of samples(length)] /// Each segments describes a line with the form of mX + B that has a length /// of # of samples. -#[derive(Debug, Default)] +#[derive(Debug, Default, Clone, Encode, Decode)] pub struct Vsri { index_file: String, min_ts: i32, @@ -317,10 +317,15 @@ impl Vsri { let segment_end_y = y0 + (sample_rate * (num_samples - 1)); if y >= y0 && y <= segment_end_y { - // x = (y - b)/ m - // TODO: This can return floats! - let x_value = (y - self.calculate_b(segment)) / sample_rate; - return Some(x_value); + // x = (y - b) / m + // Check if y aligns exactly with the sample rate (no fractional sample) + let b = self.calculate_b(segment); + let numerator = y - b; + if sample_rate != 0 && numerator % sample_rate == 0 { + return Some(numerator / sample_rate); + } + // y doesn't align with any sample in this segment + return None; } } None // No matching segment found for the given Y value @@ -403,12 +408,20 @@ impl Vsri { fn fits_segment(&self, y: i32) -> bool { let last_segment = self.current_segment(); let b = self.calculate_b(&last_segment); + let sample_rate = last_segment[0]; // What we have to check, is with the given y, calculate x. // Then check if x fits the interval for the current line // and it has to be the next one in the line - // x = (y - b)/ m - // TODO: Can return float, watch out - let x_value = (y - b) / last_segment[0]; + // x = (y - b) / m + // Check for exact division (no fractional sample) + if sample_rate == 0 { + return false; + } + let numerator = y - b; + if numerator % sample_rate != 0 { + return false; + } + let x_value = numerator / sample_rate; debug!( "[INDEX] Fit Calculation (Segment {:?}). b: {}, x: {}, calculated x: {}", last_segment, @@ -453,7 +466,6 @@ impl Vsri { } /// Reads an index file and loads the content into the structure - /// TODO: Add error control (Unwrap hell) pub fn load(filename: &str) -> Result { debug!("[INDEX] Load existing index"); let file = File::open(filename)?; @@ -466,18 +478,27 @@ impl Vsri { let line = line?; match i { 1 => { - min_ts = line.trim().parse::().unwrap(); + min_ts = line.trim().parse::().map_err(|e| { + std::io::Error::new(std::io::ErrorKind::InvalidData, format!("Invalid min_ts: {}", e)) + })?; } 2 => { - max_ts = line.trim().parse::().unwrap(); + max_ts = line.trim().parse::().map_err(|e| { + std::io::Error::new(std::io::ErrorKind::InvalidData, format!("Invalid max_ts: {}", e)) + })?; } _ => { - let values = line + let values: Vec = line .split(',') .map(|value| value.trim().parse::()) .collect::, _>>() - .unwrap(); - segments.push(values.try_into().unwrap()); + .map_err(|e| { + std::io::Error::new(std::io::ErrorKind::InvalidData, format!("Invalid segment: {}", e)) + })?; + let segment: [i32; 4] = values.try_into().map_err(|_| { + std::io::Error::new(std::io::ErrorKind::InvalidData, "Segment must have exactly 4 values") + })?; + segments.push(segment); } } i += 1; @@ -495,3 +516,51 @@ impl Vsri { pub enum Error { UpdateIndexForPointError, } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_vsri_simple() { + let timestamps = vec![ + 1729606100, 1729606120, 1729606140, 1729606160, 1729606180, 1729606200, 1729606220, + 1729606240, 1729606260, + ]; + let mut vsri = Vsri::new("test"); + for ts in ×tamps { + vsri.update_for_point(*ts).unwrap(); + } + let out_vsri = vsri.get_all_timestamps(); + assert_eq!(timestamps, out_vsri); + } + + #[test] + fn test_vsri_several_segments() { + let timestamps = vec![ + 1729606100, 1729606120, 1729606140, 1729606160, 1729606180, 1729606200, 1729606220, + 1729606260, 1729606360, 1729606460, 1729606560, 1729606660, 1729606760, 1729606860, + 1729606881, 1729606882, 1729606883, 1729606884, 1729606885, 1729606886, 1729606887, + ]; + let mut vsri = Vsri::new("test"); + for ts in ×tamps { + vsri.update_for_point(*ts).unwrap(); + } + let out_vsri = vsri.get_all_timestamps(); + assert_eq!(timestamps, out_vsri); + } + + #[test] + fn test_point_in_past() { + let timestamps = vec![ + 1729606100, 1729606120, 1729606140, 1729606160, 1729606180, 1729606200, 1729606220, + 1729606240, 1729606260, + ]; + let mut vsri = Vsri::new("test"); + for ts in ×tamps { + vsri.update_for_point(*ts).unwrap(); + } + let result = vsri.update_for_point(1729605260); + assert!(result.is_err()); + } +} diff --git a/vsri/Cargo.toml b/vsri/Cargo.toml deleted file mode 100644 index b5bbaa0..0000000 --- a/vsri/Cargo.toml +++ /dev/null @@ -1,10 +0,0 @@ -[package] -name = "vsri" -version = "0.1.0" -edition = "2021" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -chrono = "0.4.26" -log = "0.4.0" \ No newline at end of file