diff --git a/Cargo.lock b/Cargo.lock index 5c5a6f6..cc56da4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -84,9 +84,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.21" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" dependencies = [ "anstyle", "anstyle-parse", @@ -99,15 +99,15 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.13" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" [[package]] name = "anstyle-parse" -version = "0.2.7" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" dependencies = [ "utf8parse", ] @@ -381,7 +381,8 @@ name = "asap_planner" version = "0.1.0" dependencies = [ "anyhow", - "clap 4.5.60", + "chrono", + "clap 4.6.0", "indexmap 2.13.0", "pretty_assertions", "promql-parser", @@ -586,16 +587,16 @@ dependencies = [ [[package]] name = "blake3" -version = "1.8.3" +version = "1.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2468ef7d57b3fb7e16b576e8377cdbde2320c60e1491e961d11da40fc4f02a2d" +checksum = "4d2d5991425dfd0785aed03aedcf0b321d61975c9b5b3689c774a2610ae0b51e" dependencies = [ "arrayref", "arrayvec", "cc", "cfg-if", "constant_time_eq", - "cpufeatures", + "cpufeatures 0.3.0", ] [[package]] @@ -700,9 +701,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.2.56" +version = "1.2.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aebf35691d1bfb0ac386a69bac2fde4dd276fb618cf8bf4f5318fe285e821bb2" +checksum = "e1e928d4b69e3077709075a938a05ffbedfa53a84c8f766efbf8220bb1ff60e1" dependencies = [ "find-msvc-tools", "jobserver", @@ -799,9 +800,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.60" +version = "4.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2797f34da339ce31042b27d23607e051786132987f595b02ba4f6a6dffb7030a" +checksum = "b193af5b67834b676abd72466a96c1024e6a6ad978a1f484bd90b85c94041351" dependencies = [ "clap_builder", "clap_derive", @@ -809,9 +810,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.60" +version = "4.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24a241312cea5059b13574bb9b3861cabf758b879c15190b37b6d6fd63ab6876" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" dependencies = [ "anstream", "anstyle", @@ -821,9 +822,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.55" +version = "4.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a92793da1a46a5f2a02a6f4c46c6496b28c43638adea8306fcb0caa1634f24e5" +checksum = "1110bd8a634a1ab8cb04345d8d878267d57c3cf1b38d91b71af6686408bbca6a" dependencies = [ "heck 0.5.0", "proc-macro2", @@ -833,9 +834,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "1.0.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a822ea5bc7590f9d40f1ba12c0dc3c2760f3482c6984db1573ad11031420831" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" [[package]] name = "codespan-reporting" @@ -850,9 +851,9 @@ dependencies = [ [[package]] name = "colorchoice" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" [[package]] name = "comfy-table" @@ -925,6 +926,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -943,7 +953,7 @@ dependencies = [ "anes", "cast", "ciborium", - "clap 4.5.60", + "clap 4.6.0", "criterion-plot", "is-terminal", "itertools 0.10.5", @@ -1087,7 +1097,7 @@ version = "1.0.194" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0956799fa8678d4c50eed028f2de1c0552ae183c76e976cf7ca8c4e36a7c328" dependencies = [ - "clap 4.5.60", + "clap 4.6.0", "codespan-reporting", "indexmap 2.13.0", "proc-macro2", @@ -1459,7 +1469,7 @@ dependencies = [ "itertools 0.13.0", "log", "paste", - "petgraph", + "petgraph 0.6.5", ] [[package]] @@ -1711,6 +1721,12 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + [[package]] name = "flatbuffers" version = "24.12.23" @@ -2447,9 +2463,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.17" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "jiff" @@ -2504,10 +2520,12 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.91" +version = "0.3.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b49715b7073f385ba4bc528e5747d02e66cb39c6146efb66b781f131f0fb399c" +checksum = "cc4c90f45aa2e6eacbe8645f77fdea542ac97a494bcd117a67df9ff4d611f995" dependencies = [ + "cfg-if", + "futures-util", "once_cell", "wasm-bindgen", ] @@ -2605,9 +2623,9 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" [[package]] name = "libredox" -version = "0.1.14" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1744e39d1d6a9948f4f388969627434e31128196de472883b39f148769bfe30a" +checksum = "7ddbf48fd451246b1f8c2610bd3b4ac0cc6e149d89832867093ab69a17194f08" dependencies = [ "bitflags 2.11.0", "libc", @@ -2628,9 +2646,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.24" +version = "1.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4735e9cbde5aac84a5ce588f6b23a90b9b0b528f6c5a8db8a4aff300463a0839" +checksum = "d52f4c29e2a68ac30c9087e1b772dc9f44a2b66ed44edf2266cf2be9b03dafc1" dependencies = [ "cc", "libc", @@ -2730,9 +2748,9 @@ dependencies = [ [[package]] name = "lz4_flex" -version = "0.11.5" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08ab2867e3eeeca90e844d1940eab391c9dc5228783db2ed999acbc0a9ed375a" +checksum = "373f5eceeeab7925e0c1098212f2fbc4d416adec9d35051a6ab251e824c1854a" dependencies = [ "twox-hash 2.1.2", ] @@ -2797,9 +2815,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.1.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" +checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" dependencies = [ "libc", "wasi", @@ -2873,9 +2891,9 @@ dependencies = [ [[package]] name = "num-conv" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf97ec579c3c42f953ef76dbf8d55ac91fb219dde70e49aa4a6b7d74e9919050" +checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967" [[package]] name = "num-integer" @@ -2930,9 +2948,9 @@ dependencies = [ [[package]] name = "num_enum" -version = "0.7.5" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1207a7e20ad57b847bbddc6776b968420d38292bbfe2089accff5e19e82454c" +checksum = "5d0bca838442ec211fa11de3a8b0e0e8f3a4522575b5c4c06ed722e005036f26" dependencies = [ "num_enum_derive", "rustversion", @@ -2940,9 +2958,9 @@ dependencies = [ [[package]] name = "num_enum_derive" -version = "0.7.5" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff32365de1b6743cb203b710788263c44a03de03802daf96092f2da4fe6ba4d7" +checksum = "680998035259dcfcafe653688bf2aa6d3e2dc05e98be6ab46afb089dc84f1df8" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -2991,9 +3009,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.21.3" +version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "once_cell_polyfill" @@ -3009,9 +3027,9 @@ checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" [[package]] name = "openssl" -version = "0.10.75" +version = "0.10.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08838db121398ad17ab8531ce9de97b244589089e290a384c900cb9ff7434328" +checksum = "951c002c75e16ea2c65b8c7e4d3d51d5530d8dfa7d060b4776828c88cfb18ecf" dependencies = [ "bitflags 2.11.0", "cfg-if", @@ -3041,9 +3059,9 @@ checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" [[package]] name = "openssl-sys" -version = "0.9.111" +version = "0.9.112" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82cab2d520aa75e3c58898289429321eb788c3106963d0dc886ec7a5f4adc321" +checksum = "57d55af3b3e226502be1526dfdba67ab0e9c96fc293004e79576b2b9edb0dbdb" dependencies = [ "cc", "libc", @@ -3185,7 +3203,7 @@ checksum = "acea383beda9652270f3c9678d83aa58cbfc16880343cae0c0c8c7d6c0974132" dependencies = [ "jiff", "num-traits", - "winnow", + "winnow 0.7.15", ] [[package]] @@ -3221,7 +3239,17 @@ version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ - "fixedbitset", + "fixedbitset 0.4.2", + "indexmap 2.13.0", +] + +[[package]] +name = "petgraph" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" +dependencies = [ + "fixedbitset 0.5.7", "indexmap 2.13.0", ] @@ -3323,9 +3351,9 @@ checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" [[package]] name = "portable-atomic-util" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a9db96d7fa8782dd8c15ce32ffe8680bbd1e978a43bf51a34d39483540495f5" +checksum = "091397be61a01d4be58e7841595bd4bfedb15f1cd54977d79b8271e94ed799a3" dependencies = [ "portable-atomic", ] @@ -3475,11 +3503,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" dependencies = [ "heck 0.5.0", - "itertools 0.10.5", + "itertools 0.14.0", "log", "multimap", "once_cell", - "petgraph", + "petgraph 0.7.1", "prettyplease", "prost", "prost-types", @@ -3537,7 +3565,7 @@ dependencies = [ "base64 0.21.7", "bincode", "chrono", - "clap 4.5.60", + "clap 4.6.0", "criterion", "ctor", "dashmap 5.5.3", @@ -3914,9 +3942,9 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.28" +version = "0.1.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" dependencies = [ "windows-sys 0.61.2", ] @@ -4063,7 +4091,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest", ] @@ -4094,9 +4122,9 @@ dependencies = [ [[package]] name = "simd-adler32" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" [[package]] name = "siphasher" @@ -4108,7 +4136,7 @@ checksum = "b2aa850e253778c88a04c3d7323b043aeda9d3e30d5971937c1855769763678e" name = "sketch-core" version = "0.1.0" dependencies = [ - "clap 4.5.60", + "clap 4.6.0", "ctor", "dsrs", "rmp-serde", @@ -4122,7 +4150,7 @@ name = "sketch_db_common" version = "0.1.0" dependencies = [ "anyhow", - "clap 4.5.60", + "clap 4.6.0", "promql_utilities", "serde", "serde_json", @@ -4136,7 +4164,7 @@ version = "0.1.0" source = "git+https://github.com/ProjectASAP/sketchlib-rust?rev=440427438fdaf3ac2298b53ee148f9e12a64ffcc#440427438fdaf3ac2298b53ee148f9e12a64ffcc" dependencies = [ "bytes", - "clap 4.5.60", + "clap 4.6.0", "pcap", "prost", "prost-build", @@ -4417,9 +4445,9 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.26.0" +version = "3.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82a72c767771b47409d2345987fda8628641887d5466101319899796367354a0" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", "getrandom 0.4.2", @@ -4649,32 +4677,32 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "1.0.0+spec-1.1.0" +version = "1.1.0+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32c2555c699578a4f59f0cc68e5116c8d7cabbd45e1409b989d4be085b53f13e" +checksum = "97251a7c317e03ad83774a8752a7e81fb6067740609f75ea2b585b569a59198f" dependencies = [ "serde_core", ] [[package]] name = "toml_edit" -version = "0.25.4+spec-1.1.0" +version = "0.25.8+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7193cbd0ce53dc966037f54351dbbcf0d5a642c7f0038c382ef9e677ce8c13f2" +checksum = "16bff38f1d86c47f9ff0647e6838d7bb362522bdf44006c7068c2b1e606f1f3c" dependencies = [ "indexmap 2.13.0", "toml_datetime", "toml_parser", - "winnow", + "winnow 1.0.1", ] [[package]] name = "toml_parser" -version = "1.0.9+spec-1.1.0" +version = "1.1.0+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "702d4415e08923e7e1ef96cd5727c0dfed80b4d2fa25db9647fe5eb6f7c5a4c4" +checksum = "2334f11ee363607eb04df9b8fc8a13ca1715a72ba8662a26ac285c98aabb4011" dependencies = [ - "winnow", + "winnow 1.0.1", ] [[package]] @@ -4813,9 +4841,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.22" +version = "0.3.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f30143827ddab0d256fd843b7a66d164e9f271cfa0dde49142c5ca0ca291f1e" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" dependencies = [ "matchers", "nu-ansi-term", @@ -4868,9 +4896,9 @@ checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" [[package]] name = "unicode-segmentation" -version = "1.12.0" +version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" +checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c" [[package]] name = "unicode-width" @@ -4928,9 +4956,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.22.0" +version = "1.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a68d3c8f01c0cfa54a75291d83601161799e4a89a39e0929f4b0354d88757a37" +checksum = "5ac8b6f42ead25368cf5b098aeb3dc8a1a2c05a3eee8a9a1a68c640edbfc79d9" dependencies = [ "getrandom 0.4.2", "js-sys", @@ -5027,9 +5055,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.114" +version = "0.2.115" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6532f9a5c1ece3798cb1c2cfdba640b9b3ba884f5db45973a6f442510a87d38e" +checksum = "6523d69017b7633e396a89c5efab138161ed5aafcbc8d3e5c5a42ae38f50495a" dependencies = [ "cfg-if", "once_cell", @@ -5040,23 +5068,19 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.64" +version = "0.4.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9c5522b3a28661442748e09d40924dfb9ca614b21c00d3fd135720e48b67db8" +checksum = "2d1faf851e778dfa54db7cd438b70758eba9755cb47403f3496edd7c8fc212f0" dependencies = [ - "cfg-if", - "futures-util", "js-sys", - "once_cell", "wasm-bindgen", - "web-sys", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.114" +version = "0.2.115" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18a2d50fcf105fb33bb15f00e7a77b772945a2ee45dcf454961fd843e74c18e6" +checksum = "4e3a6c758eb2f701ed3d052ff5737f5bfe6614326ea7f3bbac7156192dc32e67" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -5064,9 +5088,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.114" +version = "0.2.115" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03ce4caeaac547cdf713d280eda22a730824dd11e6b8c3ca9e42247b25c631e3" +checksum = "921de2737904886b52bcbb237301552d05969a6f9c40d261eb0533c8b055fedf" dependencies = [ "bumpalo", "proc-macro2", @@ -5077,9 +5101,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.114" +version = "0.2.115" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75a326b8c223ee17883a4251907455a2431acc2791c98c26279376490c378c16" +checksum = "a93e946af942b58934c604527337bad9ae33ba1d5c6900bbb41c2c07c2364a93" dependencies = [ "unicode-ident", ] @@ -5120,9 +5144,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.91" +version = "0.3.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "854ba17bb104abfb26ba36da9729addc7ce7f06f5c0f90f3c391f8461cca21f9" +checksum = "84cde8507f4d7cfcb1185b8cb5890c494ffea65edbe1ba82cfd63661c805ed94" dependencies = [ "js-sys", "wasm-bindgen", @@ -5427,6 +5451,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "winnow" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09dac053f1cd375980747450bfc7250c264eaae0583872e845c0c7cd578872b5" +dependencies = [ + "memchr", +] + [[package]] name = "winreg" version = "0.50.0" @@ -5577,18 +5610,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.42" +version = "0.8.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2578b716f8a7a858b7f02d5bd870c14bf4ddbbcf3a4c05414ba6503640505e3" +checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.42" +version = "0.8.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e6cc098ea4d3bd6246687de65af3f920c430e236bee1e3bf2e441463f08a02f" +checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" dependencies = [ "proc-macro2", "quote", diff --git a/asap-planner-rs/Cargo.toml b/asap-planner-rs/Cargo.toml index 5c74e72..8e7288b 100644 --- a/asap-planner-rs/Cargo.toml +++ b/asap-planner-rs/Cargo.toml @@ -25,6 +25,7 @@ tracing.workspace = true tracing-subscriber.workspace = true clap.workspace = true indexmap.workspace = true +chrono.workspace = true promql-parser = "0.5.0" [dev-dependencies] diff --git a/asap-planner-rs/src/config/input.rs b/asap-planner-rs/src/config/input.rs index 07226ca..431bc77 100644 --- a/asap-planner-rs/src/config/input.rs +++ b/asap-planner-rs/src/config/input.rs @@ -13,10 +13,17 @@ pub struct QueryGroup { pub id: Option, pub queries: Vec, pub repetition_delay: u64, + #[serde(default)] pub controller_options: ControllerOptions, + /// Per-group step override (seconds). Falls back to `RuntimeOptions::step` when None. + #[serde(default)] + pub step: Option, + /// Per-group range_duration override (seconds). Falls back to `RuntimeOptions::range_duration` when None. + #[serde(default)] + pub range_duration: Option, } -#[derive(Debug, Clone, Deserialize)] +#[derive(Debug, Clone, Deserialize, Default)] pub struct ControllerOptions { pub accuracy_sla: f64, pub latency_sla: f64, diff --git a/asap-planner-rs/src/lib.rs b/asap-planner-rs/src/lib.rs index 3b8753c..744e754 100644 --- a/asap-planner-rs/src/lib.rs +++ b/asap-planner-rs/src/lib.rs @@ -2,6 +2,7 @@ pub mod config; pub mod error; pub mod output; pub mod planner; +pub mod query_log; use serde_yaml::Value as YamlValue; use std::path::Path; @@ -291,6 +292,30 @@ impl Controller { }) } + /// Build a `Controller` from a Prometheus query log file and a metrics config YAML. + /// + /// - `log_path`: newline-delimited JSON query log (Prometheus `--query.log-file` output) + /// - `metrics_path`: YAML file with a `metrics:` section listing metric names and labels + pub fn from_query_log( + log_path: &Path, + metrics_path: &Path, + opts: RuntimeOptions, + ) -> Result { + let entries = query_log::parse_log_file(log_path)?; + let (instants, ranges) = + query_log::infer_queries(&entries, opts.prometheus_scrape_interval); + + let metrics_yaml = std::fs::read_to_string(metrics_path)?; + let metrics_config: query_log::MetricsConfig = serde_yaml::from_str(&metrics_yaml)?; + + let config = query_log::to_controller_config(instants, ranges, metrics_config.metrics); + + Ok(Self { + config, + options: opts, + }) + } + pub fn generate(&self) -> Result { let output = output::generator::generate_plan(&self.config, &self.options)?; Ok(PlannerOutput { diff --git a/asap-planner-rs/src/main.rs b/asap-planner-rs/src/main.rs index 0fca987..79a54c0 100644 --- a/asap-planner-rs/src/main.rs +++ b/asap-planner-rs/src/main.rs @@ -6,8 +6,17 @@ use std::path::PathBuf; #[derive(Parser, Debug)] #[command(name = "asap-planner", about = "ASAP Query Planner")] struct Args { - #[arg(long = "input_config")] - input_config: PathBuf, + /// Path to a hand-authored YAML workload config. Mutually exclusive with --query-log. + #[arg(long = "input_config", conflicts_with = "query_log")] + input_config: Option, + + /// Path to a Prometheus query log file (newline-delimited JSON). Mutually exclusive with --input_config. + #[arg(long = "query-log", conflicts_with = "input_config")] + query_log: Option, + + /// Path to a metrics config YAML (required when using --query-log). + #[arg(long = "metrics-config", requires = "query_log")] + metrics_config: Option, #[arg(long = "output_dir")] output_dir: PathBuf, @@ -71,20 +80,33 @@ fn main() -> anyhow::Result<()> { range_duration: args.range_duration, step: args.step, }; - let controller = Controller::from_file(&args.input_config, opts)?; + let controller = match (args.input_config, args.query_log) { + (Some(config_path), None) => Controller::from_file(&config_path, opts)?, + (None, Some(log_path)) => { + let metrics_path = args + .metrics_config + .expect("--metrics-config is required when using --query-log"); + Controller::from_query_log(&log_path, &metrics_path, opts)? + } + _ => anyhow::bail!( + "exactly one of --input_config or --query-log must be provided for PromQL mode" + ), + }; controller.generate_to_dir(&args.output_dir)?; } QueryLanguage::sql | QueryLanguage::elastic_sql => { let interval = args.data_ingestion_interval.ok_or_else(|| { anyhow::anyhow!("--data-ingestion-interval is required for SQL mode") })?; + let config_path = args + .input_config + .ok_or_else(|| anyhow::anyhow!("--input_config is required for SQL mode"))?; let opts = SQLRuntimeOptions { streaming_engine: engine, query_evaluation_time: None, data_ingestion_interval: interval, }; - SQLController::from_file(&args.input_config, opts)? - .generate_to_dir(&args.output_dir)?; + SQLController::from_file(&config_path, opts)?.generate_to_dir(&args.output_dir)?; } QueryLanguage::elastic_querydsl => { anyhow::bail!("ElasticQueryDSL is not yet supported"); diff --git a/asap-planner-rs/src/output/generator.rs b/asap-planner-rs/src/output/generator.rs index 75f51fd..efc5122 100644 --- a/asap-planner-rs/src/output/generator.rs +++ b/asap-planner-rs/src/output/generator.rs @@ -57,8 +57,8 @@ pub fn generate_plan( metric_schema.clone(), opts.streaming_engine, controller_config.sketch_parameters.clone(), - opts.range_duration, - opts.step, + qg.range_duration.unwrap_or(opts.range_duration), + qg.step.unwrap_or(opts.step), cleanup_policy, ); @@ -73,14 +73,25 @@ pub fn generate_plan( } if should_process { - let (configs, cleanup_param) = processor.get_streaming_aggregation_configs()?; - let mut keys_for_query = Vec::new(); - for config in configs { - let key = config.identifying_key(); - keys_for_query.push((key.clone(), cleanup_param)); - dedup_map.entry(key).or_insert(config); + match processor.get_streaming_aggregation_configs() { + Ok((configs, cleanup_param)) => { + let mut keys_for_query = Vec::new(); + for config in configs { + let key = config.identifying_key(); + keys_for_query.push((key.clone(), cleanup_param)); + dedup_map.entry(key).or_insert(config); + } + query_keys_map.insert(query_string.clone(), keys_for_query); + } + Err(ControllerError::UnknownMetric(ref metric)) => { + tracing::warn!( + query = %query_string, + metric = %metric, + "skipping query referencing unknown metric" + ); + } + Err(e) => return Err(e), } - query_keys_map.insert(query_string.clone(), keys_for_query); } } } diff --git a/asap-planner-rs/src/query_log/converter.rs b/asap-planner-rs/src/query_log/converter.rs new file mode 100644 index 0000000..5703d04 --- /dev/null +++ b/asap-planner-rs/src/query_log/converter.rs @@ -0,0 +1,55 @@ +use serde::Deserialize; + +use crate::config::input::{ + AggregateCleanupConfig, ControllerConfig, MetricDefinition, QueryGroup, +}; + +use super::frequency::{InstantQueryInfo, RangeQueryInfo}; + +/// Subset of ControllerConfig used when loading from a metrics-only YAML file. +#[derive(Deserialize)] +pub struct MetricsConfig { + pub metrics: Vec, +} + +/// Build a `ControllerConfig` from extracted instant and range queries plus a metrics definition. +/// +/// Each query becomes its own `QueryGroup` (one query per group, no SLA fields needed). +pub fn to_controller_config( + instants: Vec, + ranges: Vec, + metrics: Vec, +) -> ControllerConfig { + let mut query_groups: Vec = Vec::new(); + + for info in instants { + query_groups.push(QueryGroup { + id: None, + queries: vec![info.query], + repetition_delay: info.repetition_delay, + controller_options: Default::default(), + step: None, + range_duration: None, + }); + } + + for info in ranges { + query_groups.push(QueryGroup { + id: None, + queries: vec![info.query], + repetition_delay: info.repetition_delay, + controller_options: Default::default(), + step: Some(info.step), + range_duration: Some(info.range_duration), + }); + } + + ControllerConfig { + query_groups, + metrics, + sketch_parameters: None, + aggregate_cleanup: Some(AggregateCleanupConfig { + policy: Some("read_based".to_string()), + }), + } +} diff --git a/asap-planner-rs/src/query_log/frequency.rs b/asap-planner-rs/src/query_log/frequency.rs new file mode 100644 index 0000000..90877a8 --- /dev/null +++ b/asap-planner-rs/src/query_log/frequency.rs @@ -0,0 +1,277 @@ +use chrono::{DateTime, Utc}; +use std::collections::HashMap; + +use super::parser::LogEntry; + +/// A single (query_string, step) variant paired with all its log entries. +type QueryVariant<'a> = ((String, u64), Vec<&'a LogEntry>); + +#[derive(Debug, Clone, PartialEq)] +pub struct InstantQueryInfo { + pub query: String, + /// Median inter-arrival time rounded to nearest scrape interval (seconds). + pub repetition_delay: u64, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct RangeQueryInfo { + pub query: String, + /// Median inter-arrival time rounded to nearest scrape interval (seconds). + pub repetition_delay: u64, + /// Step from params.step (seconds). + pub step: u64, + /// Median of (end − start) across occurrences (seconds). + pub range_duration: u64, +} + +/// Infer query repetition delays from a slice of parsed log entries. +/// +/// Returns `(instant_queries, range_queries)`. +/// +/// Rules: +/// - Entries with step == 0 are instant; others are range. +/// - Group by (query_string, step). +/// - If the same query string appears under multiple step values, keep the +/// variant with the most occurrences and warn about discarded variants. +/// - Groups with only 1 occurrence are skipped with a warning. +/// - `repetition_delay` = median inter-arrival time, rounded to nearest +/// `scrape_interval` second. +/// - If `|raw_median − rounded| / scrape_interval ≥ 0.1` a warning is emitted. +pub fn infer_queries( + entries: &[LogEntry], + scrape_interval: u64, +) -> (Vec, Vec) { + // Group by (query_string, step) + let mut groups: HashMap<(String, u64), Vec<&LogEntry>> = HashMap::new(); + for entry in entries { + groups + .entry((entry.query.clone(), entry.step)) + .or_default() + .push(entry); + } + + // Per query string, if it appears under multiple steps, keep the most frequent. + let mut by_query: HashMap> = HashMap::new(); + for ((query, step), entries) in groups { + by_query + .entry(query.clone()) + .or_default() + .push(((query, step), entries)); + } + + let mut instant_results: Vec = Vec::new(); + let mut range_results: Vec = Vec::new(); + + for (_query_str, mut variants) in by_query { + // Sort descending by count so [0] is the most frequent variant. + variants.sort_by(|a, b| b.1.len().cmp(&a.1.len())); + + if variants.len() > 1 { + tracing::warn!( + query = %variants[0].0.0, + kept_step = variants[0].0.1, + kept_count = variants[0].1.len(), + "query appears with multiple step values; keeping most-frequent variant" + ); + } + + let ((query, step), variant_entries) = variants.remove(0); + + if variant_entries.len() < 2 { + tracing::warn!(%query, "query appears only once in log; skipping"); + continue; + } + + let repetition_delay = infer_repetition_delay(&variant_entries, scrape_interval, &query); + + if step == 0 { + instant_results.push(InstantQueryInfo { + query, + repetition_delay, + }); + } else { + let range_duration = median_range_duration(&variant_entries); + range_results.push(RangeQueryInfo { + query, + repetition_delay, + step, + range_duration, + }); + } + } + + (instant_results, range_results) +} + +/// Compute median inter-arrival time from timestamps and round to nearest scrape interval. +fn infer_repetition_delay(entries: &[&LogEntry], scrape_interval: u64, query: &str) -> u64 { + let mut timestamps: Vec> = entries.iter().map(|e| e.ts).collect(); + timestamps.sort(); + + let deltas: Vec = timestamps + .windows(2) + .map(|w| (w[1] - w[0]).num_seconds() as f64) + .collect(); + + let raw_median = median_f64(&deltas); + let rounded = round_to_nearest(raw_median, scrape_interval); + + let misalignment = (raw_median - rounded as f64).abs() / scrape_interval as f64; + if misalignment >= 0.1 { + tracing::warn!( + %query, + raw_median_secs = raw_median, + rounded_secs = rounded, + misalignment_pct = misalignment * 100.0, + "inferred repetition_delay is poorly aligned with scrape_interval; result may be inaccurate" + ); + } + + rounded +} + +/// Median of (end − start) durations in seconds. +fn median_range_duration(entries: &[&LogEntry]) -> u64 { + let mut durations: Vec = entries + .iter() + .map(|e| (e.end - e.start).num_seconds() as f64) + .collect(); + durations.sort_by(|a, b| a.partial_cmp(b).unwrap()); + median_f64(&durations) as u64 +} + +/// Median of a sorted-or-unsorted slice of f64 values. +fn median_f64(values: &[f64]) -> f64 { + assert!(!values.is_empty()); + let mut sorted = values.to_vec(); + sorted.sort_by(|a, b| a.partial_cmp(b).unwrap()); + let mid = sorted.len() / 2; + if sorted.len().is_multiple_of(2) { + (sorted[mid - 1] + sorted[mid]) / 2.0 + } else { + sorted[mid] + } +} + +/// Round `value` to the nearest multiple of `interval`, minimum 1 interval. +fn round_to_nearest(value: f64, interval: u64) -> u64 { + let interval_f = interval as f64; + let rounded = (value / interval_f).round() as u64; + rounded.max(1) * interval +} + +// ── unit tests ──────────────────────────────────────────────────────────────── + +#[cfg(test)] +mod tests { + use super::*; + use chrono::TimeZone; + + fn make_instant_entries( + base_ts: DateTime, + interval_secs: i64, + count: u32, + ) -> Vec { + (0..count) + .map(|i| LogEntry { + query: "rate(http_requests_total[5m])".to_string(), + start: base_ts + chrono::Duration::seconds(interval_secs * i as i64), + end: base_ts + chrono::Duration::seconds(interval_secs * i as i64), + step: 0, + ts: base_ts + chrono::Duration::seconds(interval_secs * i as i64), + }) + .collect() + } + + fn make_range_entries( + base_ts: DateTime, + interval_secs: i64, + range_secs: i64, + step: u64, + count: u32, + ) -> Vec { + (0..count) + .map(|i| { + let start = base_ts + chrono::Duration::seconds(interval_secs * i as i64); + LogEntry { + query: "rate(http_requests_total[5m])".to_string(), + start, + end: start + chrono::Duration::seconds(range_secs), + step, + ts: base_ts + chrono::Duration::seconds(interval_secs * i as i64), + } + }) + .collect() + } + + fn base_ts() -> DateTime { + Utc.with_ymd_and_hms(2025, 12, 2, 18, 0, 0).unwrap() + } + + #[test] + fn single_occurrence_skipped() { + let entries = make_instant_entries(base_ts(), 60, 1); + let (instants, ranges) = infer_queries(&entries, 15); + assert!(instants.is_empty()); + assert!(ranges.is_empty()); + } + + #[test] + fn median_inter_arrival_odd_count() { + // 5 entries at exactly 60s apart → 4 deltas all 60s → median=60 → rounded=60 + let entries = make_instant_entries(base_ts(), 60, 5); + let (instants, _) = infer_queries(&entries, 15); + assert_eq!(instants.len(), 1); + assert_eq!(instants[0].repetition_delay, 60); + } + + #[test] + fn median_inter_arrival_even_count() { + // 4 entries at 60s apart → 3 deltas all 60s → median=60 → rounded=60 + let entries = make_instant_entries(base_ts(), 60, 4); + let (instants, _) = infer_queries(&entries, 15); + assert_eq!(instants.len(), 1); + assert_eq!(instants[0].repetition_delay, 60); + } + + #[test] + fn round_down_to_nearest_scrape() { + // raw=16s, scrape=15 → nearest=15 (|16-15|/15=6.7% < 10%, no warn) + assert_eq!(round_to_nearest(16.0, 15), 15); + } + + #[test] + fn round_up_to_nearest_scrape() { + // raw=23s, scrape=15 → nearest=30 (23 is closer to 30 than 15) + assert_eq!(round_to_nearest(23.0, 15), 30); + } + + #[test] + fn misaligned_still_returns_result() { + // raw=22s, scrape=15 → rounds to 15, but |22-15|/15=46.7% ≥ 10% → warn emitted + // We only verify the value is returned (warning is logged, not returned) + assert_eq!(round_to_nearest(22.0, 15), 15); + } + + #[test] + fn range_duration_from_start_end() { + let entries = make_range_entries(base_ts(), 60, 3600, 30, 5); + let (_, ranges) = infer_queries(&entries, 15); + assert_eq!(ranges.len(), 1); + assert_eq!(ranges[0].range_duration, 3600); + assert_eq!(ranges[0].step, 30); + } + + #[test] + fn same_query_multiple_steps_keeps_most_frequent() { + let mut entries = make_instant_entries(base_ts(), 60, 3); // step=0, 3 occurrences + let range_entries = make_range_entries(base_ts(), 60, 3600, 30, 2); // step=30, 2 occurrences + entries.extend(range_entries); + + let (instants, ranges) = infer_queries(&entries, 15); + // step=0 variant has more occurrences → kept as instant + assert_eq!(instants.len(), 1); + // step=30 variant discarded + assert!(ranges.is_empty()); + } +} diff --git a/asap-planner-rs/src/query_log/mod.rs b/asap-planner-rs/src/query_log/mod.rs new file mode 100644 index 0000000..d198741 --- /dev/null +++ b/asap-planner-rs/src/query_log/mod.rs @@ -0,0 +1,7 @@ +pub mod converter; +pub mod frequency; +pub mod parser; + +pub use converter::{to_controller_config, MetricsConfig}; +pub use frequency::{infer_queries, InstantQueryInfo, RangeQueryInfo}; +pub use parser::{parse_log_file, LogEntry}; diff --git a/asap-planner-rs/src/query_log/parser.rs b/asap-planner-rs/src/query_log/parser.rs new file mode 100644 index 0000000..7a14a05 --- /dev/null +++ b/asap-planner-rs/src/query_log/parser.rs @@ -0,0 +1,114 @@ +use chrono::{DateTime, Utc}; +use serde::Deserialize; + +#[derive(Debug, Clone, PartialEq)] +pub struct LogEntry { + pub query: String, + pub start: DateTime, + pub end: DateTime, + /// Step in seconds; 0 means instant query. + pub step: u64, + pub ts: DateTime, +} + +// ── raw serde shapes ────────────────────────────────────────────────────────── + +#[derive(Deserialize)] +struct RawEntry { + params: RawParams, + ts: DateTime, +} + +#[derive(Deserialize)] +struct RawParams { + query: String, + start: DateTime, + end: DateTime, + step: u64, +} + +// ── public API ──────────────────────────────────────────────────────────────── + +/// Parse a Prometheus query log file (newline-delimited JSON). +/// Malformed or incomplete lines are skipped with a warning; they never cause a panic. +pub fn parse_log_file(path: &std::path::Path) -> Result, std::io::Error> { + let contents = std::fs::read_to_string(path)?; + Ok(parse_log_str(&contents)) +} + +/// Parse a Prometheus query log from an in-memory string. +pub fn parse_log_str(input: &str) -> Vec { + input + .lines() + .enumerate() + .filter_map(|(line_no, line)| { + let line = line.trim(); + if line.is_empty() { + return None; + } + match serde_json::from_str::(line) { + Ok(raw) => Some(LogEntry { + query: raw.params.query, + start: raw.params.start, + end: raw.params.end, + step: raw.params.step, + ts: raw.ts, + }), + Err(e) => { + tracing::warn!(line = line_no + 1, error = %e, "skipping malformed query log line"); + None + } + } + }) + .collect() +} + +// ── unit tests ──────────────────────────────────────────────────────────────── + +#[cfg(test)] +mod tests { + use super::*; + + const INSTANT_LINE: &str = r#"{"params":{"end":"2025-12-02T18:00:00.000Z","query":"rate(http_requests_total[5m])","start":"2025-12-02T18:00:00.000Z","step":0},"ts":"2025-12-02T18:00:00.001Z"}"#; + const RANGE_LINE: &str = r#"{"params":{"end":"2025-12-02T19:00:00.000Z","query":"rate(http_requests_total[5m])","start":"2025-12-02T18:00:00.000Z","step":30},"ts":"2025-12-02T18:00:00.001Z"}"#; + + #[test] + fn parse_valid_instant_entry() { + let entries = parse_log_str(INSTANT_LINE); + assert_eq!(entries.len(), 1); + let e = &entries[0]; + assert_eq!(e.query, "rate(http_requests_total[5m])"); + assert_eq!(e.step, 0); + assert_eq!(e.start, e.end); + } + + #[test] + fn parse_valid_range_entry() { + let entries = parse_log_str(RANGE_LINE); + assert_eq!(entries.len(), 1); + let e = &entries[0]; + assert_eq!(e.step, 30); + assert_ne!(e.start, e.end); + let duration = (e.end - e.start).num_seconds(); + assert_eq!(duration, 3600); + } + + #[test] + fn malformed_json_returns_empty() { + let entries = parse_log_str("not valid json at all"); + assert!(entries.is_empty()); + } + + #[test] + fn missing_params_field_skipped() { + let entries = parse_log_str(r#"{"ts":"2025-12-02T18:00:00.000Z"}"#); + assert!(entries.is_empty()); + } + + #[test] + fn mixed_lines_skips_bad() { + let input = format!("{}\nnot json\n{}", INSTANT_LINE, INSTANT_LINE); + let entries = parse_log_str(&input); + assert_eq!(entries.len(), 2); + } +} diff --git a/asap-planner-rs/tests/comparison/test_data/metrics/http_requests.yaml b/asap-planner-rs/tests/comparison/test_data/metrics/http_requests.yaml new file mode 100644 index 0000000..dc7ec91 --- /dev/null +++ b/asap-planner-rs/tests/comparison/test_data/metrics/http_requests.yaml @@ -0,0 +1,3 @@ +metrics: + - metric: http_requests_total + labels: [instance, job, method, status] diff --git a/asap-planner-rs/tests/comparison/test_data/query_logs/instant_only.log b/asap-planner-rs/tests/comparison/test_data/query_logs/instant_only.log new file mode 100644 index 0000000..c94846d --- /dev/null +++ b/asap-planner-rs/tests/comparison/test_data/query_logs/instant_only.log @@ -0,0 +1,10 @@ +{"params":{"end":"2025-12-02T18:00:00.000Z","query":"rate(http_requests_total[5m])","start":"2025-12-02T18:00:00.000Z","step":0},"ts":"2025-12-02T18:00:00.001Z"} +{"params":{"end":"2025-12-02T18:01:00.000Z","query":"rate(http_requests_total[5m])","start":"2025-12-02T18:01:00.000Z","step":0},"ts":"2025-12-02T18:01:00.001Z"} +{"params":{"end":"2025-12-02T18:02:00.000Z","query":"rate(http_requests_total[5m])","start":"2025-12-02T18:02:00.000Z","step":0},"ts":"2025-12-02T18:02:00.001Z"} +{"params":{"end":"2025-12-02T18:03:00.000Z","query":"rate(http_requests_total[5m])","start":"2025-12-02T18:03:00.000Z","step":0},"ts":"2025-12-02T18:03:00.001Z"} +{"params":{"end":"2025-12-02T18:04:00.000Z","query":"rate(http_requests_total[5m])","start":"2025-12-02T18:04:00.000Z","step":0},"ts":"2025-12-02T18:04:00.001Z"} +{"params":{"end":"2025-12-02T18:00:00.000Z","query":"sum(http_requests_total)","start":"2025-12-02T18:00:00.000Z","step":0},"ts":"2025-12-02T18:00:00.002Z"} +{"params":{"end":"2025-12-02T18:01:00.000Z","query":"sum(http_requests_total)","start":"2025-12-02T18:01:00.000Z","step":0},"ts":"2025-12-02T18:01:00.002Z"} +{"params":{"end":"2025-12-02T18:02:00.000Z","query":"sum(http_requests_total)","start":"2025-12-02T18:02:00.000Z","step":0},"ts":"2025-12-02T18:02:00.002Z"} +{"params":{"end":"2025-12-02T18:03:00.000Z","query":"sum(http_requests_total)","start":"2025-12-02T18:03:00.000Z","step":0},"ts":"2025-12-02T18:03:00.002Z"} +{"params":{"end":"2025-12-02T18:04:00.000Z","query":"sum(http_requests_total)","start":"2025-12-02T18:04:00.000Z","step":0},"ts":"2025-12-02T18:04:00.002Z"} diff --git a/asap-planner-rs/tests/comparison/test_data/query_logs/range_only.log b/asap-planner-rs/tests/comparison/test_data/query_logs/range_only.log new file mode 100644 index 0000000..bb1107b --- /dev/null +++ b/asap-planner-rs/tests/comparison/test_data/query_logs/range_only.log @@ -0,0 +1,5 @@ +{"params":{"end":"2025-12-02T19:00:00.000Z","query":"rate(http_requests_total[5m])","start":"2025-12-02T18:00:00.000Z","step":30},"ts":"2025-12-02T18:00:00.001Z"} +{"params":{"end":"2025-12-02T19:01:00.000Z","query":"rate(http_requests_total[5m])","start":"2025-12-02T18:01:00.000Z","step":30},"ts":"2025-12-02T18:01:00.001Z"} +{"params":{"end":"2025-12-02T19:02:00.000Z","query":"rate(http_requests_total[5m])","start":"2025-12-02T18:02:00.000Z","step":30},"ts":"2025-12-02T18:02:00.001Z"} +{"params":{"end":"2025-12-02T19:03:00.000Z","query":"rate(http_requests_total[5m])","start":"2025-12-02T18:03:00.000Z","step":30},"ts":"2025-12-02T18:03:00.001Z"} +{"params":{"end":"2025-12-02T19:04:00.000Z","query":"rate(http_requests_total[5m])","start":"2025-12-02T18:04:00.000Z","step":30},"ts":"2025-12-02T18:04:00.001Z"} diff --git a/asap-planner-rs/tests/comparison/test_data/query_logs/single_occurrence.log b/asap-planner-rs/tests/comparison/test_data/query_logs/single_occurrence.log new file mode 100644 index 0000000..5857715 --- /dev/null +++ b/asap-planner-rs/tests/comparison/test_data/query_logs/single_occurrence.log @@ -0,0 +1,2 @@ +{"params":{"end":"2025-12-02T18:00:00.000Z","query":"rate(http_requests_total[5m])","start":"2025-12-02T18:00:00.000Z","step":0},"ts":"2025-12-02T18:00:00.001Z"} +{"params":{"end":"2025-12-02T18:00:00.000Z","query":"sum(http_requests_total)","start":"2025-12-02T18:00:00.000Z","step":0},"ts":"2025-12-02T18:00:00.002Z"} diff --git a/asap-planner-rs/tests/comparison/test_data/query_logs/with_malformed.log b/asap-planner-rs/tests/comparison/test_data/query_logs/with_malformed.log new file mode 100644 index 0000000..1c5c2cf --- /dev/null +++ b/asap-planner-rs/tests/comparison/test_data/query_logs/with_malformed.log @@ -0,0 +1,7 @@ +{"params":{"end":"2025-12-02T18:00:00.000Z","query":"rate(http_requests_total[5m])","start":"2025-12-02T18:00:00.000Z","step":0},"ts":"2025-12-02T18:00:00.001Z"} +not valid json at all +{"params":{"end":"2025-12-02T18:01:00.000Z","query":"rate(http_requests_total[5m])","start":"2025-12-02T18:01:00.000Z","step":0},"ts":"2025-12-02T18:01:00.001Z"} +{"missing_params": true, "ts": "2025-12-02T18:01:30.000Z"} +{"params":{"end":"2025-12-02T18:02:00.000Z","query":"rate(http_requests_total[5m])","start":"2025-12-02T18:02:00.000Z","step":0},"ts":"2025-12-02T18:02:00.001Z"} +{"params":{"end":"2025-12-02T18:03:00.000Z","query":"rate(http_requests_total[5m])","start":"2025-12-02T18:03:00.000Z","step":0},"ts":"2025-12-02T18:03:00.001Z"} +{"params":{"end":"2025-12-02T18:04:00.000Z","query":"rate(http_requests_total[5m])","start":"2025-12-02T18:04:00.000Z","step":0},"ts":"2025-12-02T18:04:00.001Z"} diff --git a/asap-planner-rs/tests/integration.rs b/asap-planner-rs/tests/integration.rs index d51db0d..a7f39ac 100644 --- a/asap-planner-rs/tests/integration.rs +++ b/asap-planner-rs/tests/integration.rs @@ -1,6 +1,73 @@ use asap_planner::{Controller, ControllerError, RuntimeOptions, StreamingEngine}; use std::path::Path; +// ─── query_log integration tests ───────────────────────────────────────────── + +#[test] +fn query_log_instant_produces_valid_configs() { + let c = Controller::from_query_log( + Path::new("tests/comparison/test_data/query_logs/instant_only.log"), + Path::new("tests/comparison/test_data/metrics/http_requests.yaml"), + arroyo_opts(), + ) + .unwrap(); + let out = c.generate().unwrap(); + assert!(out.streaming_aggregation_count() > 0); + assert!(out.inference_query_count() > 0); +} + +#[test] +fn query_log_range_produces_valid_configs() { + let c = Controller::from_query_log( + Path::new("tests/comparison/test_data/query_logs/range_only.log"), + Path::new("tests/comparison/test_data/metrics/http_requests.yaml"), + arroyo_opts(), + ) + .unwrap(); + let out = c.generate().unwrap(); + // range_only.log has step=30, so effective window size must be ≤ 30 + assert!(out.all_tumbling_window_sizes_leq(30)); +} + +#[test] +fn query_log_single_occurrence_excluded() { + let c = Controller::from_query_log( + Path::new("tests/comparison/test_data/query_logs/single_occurrence.log"), + Path::new("tests/comparison/test_data/metrics/http_requests.yaml"), + arroyo_opts(), + ) + .unwrap(); + let out = c.generate().unwrap(); + assert_eq!(out.inference_query_count(), 0); +} + +#[test] +fn query_log_malformed_lines_skipped() { + // with_malformed.log has 5 valid entries for rate() interspersed with bad lines + let c = Controller::from_query_log( + Path::new("tests/comparison/test_data/query_logs/with_malformed.log"), + Path::new("tests/comparison/test_data/metrics/http_requests.yaml"), + arroyo_opts(), + ) + .unwrap(); + let out = c.generate().unwrap(); + assert!(out.inference_query_count() > 0); +} + +#[test] +fn query_log_output_files_written() { + let dir = tempfile::tempdir().unwrap(); + let c = Controller::from_query_log( + Path::new("tests/comparison/test_data/query_logs/instant_only.log"), + Path::new("tests/comparison/test_data/metrics/http_requests.yaml"), + arroyo_opts(), + ) + .unwrap(); + c.generate_to_dir(dir.path()).unwrap(); + assert!(dir.path().join("streaming_config.yaml").exists()); + assert!(dir.path().join("inference_config.yaml").exists()); +} + fn arroyo_opts() -> RuntimeOptions { RuntimeOptions { prometheus_scrape_interval: 15, @@ -276,7 +343,8 @@ metrics: } #[test] -fn query_referencing_unknown_metric_returns_error() { +fn query_referencing_unknown_metric_is_skipped_with_warning() { + // Unknown metric no longer aborts the run; the query is silently skipped. let yaml = r#" query_groups: - id: 1 @@ -291,10 +359,9 @@ metrics: labels: ["instance"] "#; let c = Controller::from_yaml(yaml, arroyo_opts()).unwrap(); - assert!(matches!( - c.generate(), - Err(ControllerError::UnknownMetric(_)) - )); + let out = c.generate().unwrap(); + assert_eq!(out.inference_query_count(), 0); + assert_eq!(out.streaming_aggregation_count(), 0); } #[test] diff --git a/docs/03-how-to-guides/operations/bootstrap-config-from-query-log.md b/docs/03-how-to-guides/operations/bootstrap-config-from-query-log.md new file mode 100644 index 0000000..0bc3dfb --- /dev/null +++ b/docs/03-how-to-guides/operations/bootstrap-config-from-query-log.md @@ -0,0 +1,49 @@ +# Bootstrap Config from Prometheus Query Log + +Generate sketch configs from real query traffic instead of hand-authoring a workload YAML. + +## Steps + +### 1. Enable the Prometheus query log + +Add to your Prometheus startup flags: +``` +--query.log-file=/var/log/prometheus/query.log +``` + +Let it run for a representative period (hours to days). Each line is a JSON entry: +```json +{"params":{"query":"rate(http_requests_total[5m])","start":"2025-12-02T18:00:00Z","end":"2025-12-02T18:00:00Z","step":0},"ts":"2025-12-02T18:00:00.001Z"} +``` + +### 2. Create a metrics config + +List the metrics you want ASAP to sketch: +```yaml +# metrics.yaml +metrics: + - metric: http_requests_total + labels: [instance, job, method, status] + - metric: node_cpu_seconds_total + labels: [instance, mode] +``` + +### 3. Run the planner + +```bash +asap-planner \ + --query-log /var/log/prometheus/query.log \ + --metrics-config metrics.yaml \ + --output_dir ./configs \ + --prometheus_scrape_interval 15 \ + --streaming_engine arroyo +``` + +This writes `streaming_config.yaml` and `inference_config.yaml` to `./configs/`. + +## Notes + +- Queries appearing only once are skipped (need frequency to infer repeat interval) +- Queries referencing metrics not in `metrics.yaml` are skipped with a warning +- Unsupported PromQL patterns (e.g. `absent`, complex multi-level aggregations) are skipped +- Repeat interval is inferred from median inter-arrival time, rounded to the nearest scrape interval diff --git a/docs/README.md b/docs/README.md index 6cb5a7b..4e424d5 100644 --- a/docs/README.md +++ b/docs/README.md @@ -37,6 +37,7 @@ Task-oriented guides for common operations: ### Operations Tasks - [Manual Stack Run for Prometheus](03-how-to-guides/operations/manual-stack-run-prometheus.md) - Run ASAP components manually to accelerate Prometheus +- [Bootstrap Config from Query Log](03-how-to-guides/operations/bootstrap-config-from-query-log.md) - Auto-generate sketch configs from Prometheus query traffic - [Manual Stack Run for Clickhouse](03-how-to-guides/operations/manual-stack-run-clickhouse.md) - Run ASAP components manually to accelerate Clickhouse - [Deploy to CloudLab](03-how-to-guides/operations/deploy-cloudlab.md) - Deployment guide - [Troubleshooting](03-how-to-guides/operations/troubleshooting.md) - Common issues & solutions