diff --git a/Cargo.lock b/Cargo.lock index a613e53..879ed17 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -50,7 +50,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "rand 0.9.2", - "sha1", + "sha1 0.10.6", "smallvec", "tokio", "tokio-util", @@ -358,6 +358,44 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +[[package]] +name = "aws-config" +version = "1.8.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e33f815b73a3899c03b380d543532e5865f230dce9678d108dc10732a8682275" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sdk-sts", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 1.4.0", + "time", + "tokio", + "tracing", + "url", +] + +[[package]] +name = "aws-credential-types" +version = "1.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f20799b373a1be121fe3005fba0c2090af9411573878f224df44b42727fcaf7" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "zeroize", +] + [[package]] name = "aws-lc-rs" version = "1.17.0" @@ -380,6 +418,360 @@ dependencies = [ "fs_extra", ] +[[package]] +name = "aws-runtime" +version = "1.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c9b9de216a988dd54b754a82a7660cfe14cee4f6782ae4524470972fa0ccb39" +dependencies = [ + "aws-credential-types", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "bytes-utils", + "fastrand", + "http 0.2.12", + "http 1.4.0", + "http-body 0.4.6", + "http-body 1.0.1", + "percent-encoding", + "pin-project-lite", + "tracing", + "uuid", +] + +[[package]] +name = "aws-sdk-s3" +version = "1.137.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2dd7213994e2ff9382ff100403b78c30d1b74cdfcd8fa9d0d1dc3a94a5c4874" +dependencies = [ + "arc-swap", + "aws-credential-types", + "aws-runtime", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-checksums", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "bytes", + "fastrand", + "hex", + "hmac", + "http 0.2.12", + "http 1.4.0", + "http-body 1.0.1", + "lru 0.16.4", + "percent-encoding", + "regex-lite", + "sha2", + "tracing", + "url", +] + +[[package]] +name = "aws-sdk-sts" +version = "1.107.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d0d328ba962af23ecfa3c9f23b98d3d35e325fa218d7f13d17a6bf522f8a560" +dependencies = [ + "arc-swap", + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-query", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "fastrand", + "http 0.2.12", + "http 1.4.0", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sigv4" +version = "1.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bae38512beae0ffee7010fc24e7a8a123c53efdfef42a61e80fda4882418dc71" +dependencies = [ + "aws-credential-types", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "form_urlencoded", + "hex", + "hmac", + "http 0.2.12", + "http 1.4.0", + "percent-encoding", + "sha2", + "time", + "tracing", +] + +[[package]] +name = "aws-smithy-async" +version = "1.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ffcaf626bdda484571968400c326a244598634dc75fd451325a54ad1a59acfc" +dependencies = [ + "futures-util", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "aws-smithy-checksums" +version = "0.64.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9e8e65f4f81fcccdeb6c3eca2af17ac21d421a1786a26a394aecf421d616d3a" +dependencies = [ + "aws-smithy-http", + "aws-smithy-types", + "bytes", + "crc-fast", + "hex", + "http 1.4.0", + "http-body 1.0.1", + "http-body-util", + "md-5", + "pin-project-lite", + "sha1 0.11.0", + "sha2", + "tracing", +] + +[[package]] +name = "aws-smithy-eventstream" +version = "0.60.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78d8391e65fcea47c586a22e1a41f173b38615b112b2c6b7a44e80cec3e6b706" +dependencies = [ + "aws-smithy-types", + "bytes", + "crc32fast", +] + +[[package]] +name = "aws-smithy-http" +version = "0.63.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba1ab2dc1c2c3749ead27180d333c42f11be8b0e934058fb4b2258ee8dbe5231" +dependencies = [ + "aws-smithy-eventstream", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "bytes-utils", + "futures-core", + "futures-util", + "http 1.4.0", + "http-body 1.0.1", + "http-body-util", + "percent-encoding", + "pin-project-lite", + "pin-utils", + "tracing", +] + +[[package]] +name = "aws-smithy-http-client" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3ef8931ad1c98aa6a55b4256f847f3116090819844e0dd41ea682cac5dd2d3" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "h2 0.3.27", + "h2 0.4.13", + "http 0.2.12", + "http 1.4.0", + "http-body 0.4.6", + "hyper 0.14.32", + "hyper 1.8.1", + "hyper-rustls 0.24.2", + "hyper-rustls 0.27.7", + "hyper-util", + "pin-project-lite", + "rustls 0.21.12", + "rustls 0.23.36", + "rustls-native-certs", + "rustls-pki-types", + "tokio", + "tokio-rustls 0.26.4", + "tower 0.5.3", + "tracing", +] + +[[package]] +name = "aws-smithy-json" +version = "0.62.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "701a947f4797e52a911e114a898667c746c39feea467bbd1abd7b3721f702ffa" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", +] + +[[package]] +name = "aws-smithy-observability" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06c2315d173edbf1920da8ba3a7189695827002e4c0fc961973ab1c54abca9c" +dependencies = [ + "aws-smithy-runtime-api", +] + +[[package]] +name = "aws-smithy-query" +version = "0.60.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a56d79744fb3edb5d722ef79d86081e121d3b9422cb209eb03aea6aa4f21ebd" +dependencies = [ + "aws-smithy-types", + "urlencoding", +] + +[[package]] +name = "aws-smithy-runtime" +version = "1.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e6f5caf6fea86f8c2206541ab5857cfcda9013426cdbe8fa0098b9e2d32182" +dependencies = [ + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-http-client", + "aws-smithy-observability", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "bytes", + "fastrand", + "http 0.2.12", + "http 1.4.0", + "http-body 0.4.6", + "http-body 1.0.1", + "http-body-util", + "pin-project-lite", + "pin-utils", + "tokio", + "tracing", +] + +[[package]] +name = "aws-smithy-runtime-api" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9db177daa6ba8afb9ee1aefcf548c907abcf52065e394ee11a92780057fe0e8c" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api-macros", + "aws-smithy-types", + "bytes", + "http 0.2.12", + "http 1.4.0", + "pin-project-lite", + "tokio", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-runtime-api-macros" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d7396fd9500589e62e460e987ecb671bad374934e55ec3b5f498cc7a8a8a7b7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "aws-smithy-schema" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7442cb268338f0eb8278140a107c046756aa01093d8ef5e99628d34ae09c94f5" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-types", + "http 1.4.0", +] + +[[package]] +name = "aws-smithy-types" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32b42fcf341259d85ca10fac9a2f6448a8ec691c6955a18e45bc3b71a85fab85" +dependencies = [ + "base64-simd", + "bytes", + "bytes-utils", + "futures-core", + "http 0.2.12", + "http 1.4.0", + "http-body 0.4.6", + "http-body 1.0.1", + "http-body-util", + "itoa", + "num-integer", + "pin-project-lite", + "pin-utils", + "ryu", + "serde", + "time", + "tokio", + "tokio-util", +] + +[[package]] +name = "aws-smithy-xml" +version = "0.60.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce02add1aa3677d022f8adf81dcbe3046a95f17a1b1e8979c145cd21d3d22b3" +dependencies = [ + "xmlparser", +] + +[[package]] +name = "aws-types" +version = "1.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d16bf10b03a3c01e6b3b7d47cd964e873ffe9e7d4e80fad16bd4c077cb068531" +dependencies = [ + "aws-credential-types", + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "rustc_version", + "tracing", +] + [[package]] name = "axum" version = "0.7.9" @@ -391,9 +783,9 @@ dependencies = [ "bytes", "futures-util", "http 1.4.0", - "http-body", + "http-body 1.0.1", "http-body-util", - "hyper", + "hyper 1.8.1", "hyper-util", "itoa", "matchit 0.7.3", @@ -425,9 +817,9 @@ dependencies = [ "form_urlencoded", "futures-util", "http 1.4.0", - "http-body", + "http-body 1.0.1", "http-body-util", - "hyper", + "hyper 1.8.1", "hyper-util", "itoa", "matchit 0.8.4", @@ -457,7 +849,7 @@ dependencies = [ "bytes", "futures-util", "http 1.4.0", - "http-body", + "http-body 1.0.1", "http-body-util", "mime", "pin-project-lite", @@ -477,7 +869,7 @@ dependencies = [ "bytes", "futures-core", "http 1.4.0", - "http-body", + "http-body 1.0.1", "http-body-util", "mime", "pin-project-lite", @@ -493,6 +885,16 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" +dependencies = [ + "outref", + "vsimd", +] + [[package]] name = "bit-set" version = "0.8.0" @@ -534,7 +936,7 @@ dependencies = [ "cc", "cfg-if", "constant_time_eq", - "cpufeatures", + "cpufeatures 0.2.17", ] [[package]] @@ -546,6 +948,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + [[package]] name = "brotli" version = "8.0.2" @@ -597,6 +1008,16 @@ version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +[[package]] +name = "bytes-utils" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" +dependencies = [ + "bytes", + "either", +] + [[package]] name = "bytestring" version = "1.5.0" @@ -738,6 +1159,12 @@ dependencies = [ "cc", ] +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + [[package]] name = "codespan-reporting" version = "0.13.1" @@ -761,6 +1188,12 @@ version = "0.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea0095f6103c2a8b44acd6fd15960c801dafebf02e21940360833e0673f48ba7" +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + [[package]] name = "constant_time_eq" version = "0.4.2" @@ -822,6 +1255,25 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + +[[package]] +name = "crc-fast" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e75b2483e97a5a7da73ac68a05b629f9c53cff58d8ed1c77866079e18b00dba5" +dependencies = [ + "digest 0.10.7", + "spin 0.10.0", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -949,6 +1401,24 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "ctutils" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +dependencies = [ + "cmov", +] + [[package]] name = "cxx" version = "1.0.194" @@ -1064,8 +1534,20 @@ version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", - "crypto-common", + "block-buffer 0.10.4", + "crypto-common 0.1.7", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "const-oid", + "crypto-common 0.2.2", + "ctutils", ] [[package]] @@ -1212,7 +1694,7 @@ version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e139bc46ca777eb5efaf62df0ab8cc5fd400866427e56c68b22e414e53bd3be" dependencies = [ - "spin", + "spin 0.9.8", ] [[package]] @@ -1490,6 +1972,11 @@ name = "hashbrown" version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] [[package]] name = "heck" @@ -1503,6 +1990,21 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest 0.11.3", +] + [[package]] name = "htmlescape" version = "0.3.1" @@ -1530,6 +2032,17 @@ dependencies = [ "itoa", ] +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +dependencies = [ + "bytes", + "http 0.2.12", + "pin-project-lite", +] + [[package]] name = "http-body" version = "1.0.1" @@ -1549,7 +2062,7 @@ dependencies = [ "bytes", "futures-core", "http 1.4.0", - "http-body", + "http-body 1.0.1", "pin-project-lite", ] @@ -1571,6 +2084,39 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "hybrid-array" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3944cf8cf766b40e2a1a333ee5e9b563f854d5fa49d6a8ca2764e97c6eddb214" +dependencies = [ + "typenum", +] + +[[package]] +name = "hyper" +version = "0.14.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2 0.3.27", + "http 0.2.12", + "http-body 0.4.6", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2 0.5.10", + "tokio", + "tower-service", + "tracing", + "want", +] + [[package]] name = "hyper" version = "1.8.1" @@ -1583,7 +2129,7 @@ dependencies = [ "futures-core", "h2 0.4.13", "http 1.4.0", - "http-body", + "http-body 1.0.1", "httparse", "httpdate", "itoa", @@ -1594,6 +2140,21 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-rustls" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" +dependencies = [ + "futures-util", + "http 0.2.12", + "hyper 0.14.32", + "log", + "rustls 0.21.12", + "tokio", + "tokio-rustls 0.24.1", +] + [[package]] name = "hyper-rustls" version = "0.27.7" @@ -1601,12 +2162,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" dependencies = [ "http 1.4.0", - "hyper", + "hyper 1.8.1", "hyper-util", - "rustls", + "rustls 0.23.36", + "rustls-native-certs", "rustls-pki-types", "tokio", - "tokio-rustls", + "tokio-rustls 0.26.4", "tower-service", "webpki-roots", ] @@ -1617,7 +2179,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" dependencies = [ - "hyper", + "hyper 1.8.1", "hyper-util", "pin-project-lite", "tokio", @@ -1632,7 +2194,7 @@ checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ "bytes", "http-body-util", - "hyper", + "hyper 1.8.1", "hyper-util", "native-tls", "tokio", @@ -1651,8 +2213,8 @@ dependencies = [ "futures-channel", "futures-util", "http 1.4.0", - "http-body", - "hyper", + "http-body 1.0.1", + "hyper 1.8.1", "ipnet", "libc", "percent-encoding", @@ -2062,6 +2624,15 @@ dependencies = [ "hashbrown 0.15.5", ] +[[package]] +name = "lru" +version = "0.16.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f66e8d5d03f609abc3a39e6f08e4164ebf1447a732906d39eb9b99b7919ef39" +dependencies = [ + "hashbrown 0.16.1", +] + [[package]] name = "lru-slab" version = "0.1.2" @@ -2120,6 +2691,16 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" +[[package]] +name = "md-5" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" +dependencies = [ + "cfg-if", + "digest 0.11.3", +] + [[package]] name = "measure_time" version = "0.8.3" @@ -2243,6 +2824,15 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf97ec579c3c42f953ef76dbf8d55ac91fb219dde70e49aa4a6b7d74e9919050" +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -2331,6 +2921,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "outref" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" + [[package]] name = "ownedbytes" version = "0.7.0" @@ -2597,7 +3193,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash 2.1.1", - "rustls", + "rustls 0.23.36", "socket2 0.6.2", "thiserror 2.0.18", "tokio", @@ -2617,7 +3213,7 @@ dependencies = [ "rand 0.9.2", "ring", "rustc-hash 2.1.1", - "rustls", + "rustls 0.23.36", "rustls-pki-types", "slab", "thiserror 2.0.18", @@ -2835,10 +3431,10 @@ dependencies = [ "futures-util", "h2 0.4.13", "http 1.4.0", - "http-body", + "http-body 1.0.1", "http-body-util", - "hyper", - "hyper-rustls", + "hyper 1.8.1", + "hyper-rustls 0.27.7", "hyper-tls", "hyper-util", "js-sys", @@ -2848,7 +3444,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls", + "rustls 0.23.36", "rustls-pki-types", "serde", "serde_json", @@ -2856,7 +3452,7 @@ dependencies = [ "sync_wrapper", "tokio", "tokio-native-tls", - "tokio-rustls", + "tokio-rustls 0.26.4", "tower 0.5.3", "tower-http 0.6.8", "tower-service", @@ -2948,6 +3544,18 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "rustls" +version = "0.21.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" +dependencies = [ + "log", + "ring", + "rustls-webpki 0.101.7", + "sct", +] + [[package]] name = "rustls" version = "0.23.36" @@ -2959,7 +3567,7 @@ dependencies = [ "once_cell", "ring", "rustls-pki-types", - "rustls-webpki", + "rustls-webpki 0.103.9", "subtle", "zeroize", ] @@ -2995,6 +3603,16 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-webpki" +version = "0.101.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "rustls-webpki" version = "0.103.9" @@ -3061,6 +3679,16 @@ version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d68f2ec51b097e4c1a75b681a8bec621909b5e91f15bb7b840c4f2f7b01148b2" +[[package]] +name = "sct" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "security-framework" version = "3.7.0" @@ -3195,8 +3823,30 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", - "cpufeatures", - "digest", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha1" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", ] [[package]] @@ -3280,6 +3930,12 @@ dependencies = [ "lock_api", ] +[[package]] +name = "spin" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5fe4ccb98d9c292d56fec89a5e07da7fc4cf0dc11e156b41793132775d3e591" + [[package]] name = "stable_deref_trait" version = "1.2.1" @@ -3372,7 +4028,7 @@ dependencies = [ "itertools 0.12.1", "levenshtein_automata", "log", - "lru", + "lru 0.12.5", "lz4_flex", "measure_time", "memmap2", @@ -3571,13 +4227,13 @@ dependencies = [ "criterion", "prost", "rcgen", - "rustls", + "rustls 0.23.36", "rustls-pemfile", "tempfile", "thiserror 2.0.18", "tidaldb", "tokio", - "tokio-rustls", + "tokio-rustls 0.26.4", "tokio-stream", "tonic", "tonic-build", @@ -3607,7 +4263,7 @@ dependencies = [ "tidal-net", "tidaldb", "tokio", - "tokio-rustls", + "tokio-rustls 0.26.4", "tower 0.5.3", "tower-http 0.6.8", "tracing", @@ -3635,10 +4291,15 @@ dependencies = [ name = "tidalctl" version = "0.1.0" dependencies = [ + "aws-config", + "aws-credential-types", + "aws-sdk-s3", "blake3", "serde", "serde_json", + "tempfile", "tidaldb", + "tokio", ] [[package]] @@ -3654,7 +4315,7 @@ dependencies = [ "dashmap", "fjall", "fs4", - "lru", + "lru 0.12.5", "proptest", "rand 0.9.2", "roaring", @@ -3775,13 +4436,23 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-rustls" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +dependencies = [ + "rustls 0.21.12", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" dependencies = [ - "rustls", + "rustls 0.23.36", "tokio", ] @@ -3823,9 +4494,9 @@ dependencies = [ "bytes", "h2 0.4.13", "http 1.4.0", - "http-body", + "http-body 1.0.1", "http-body-util", - "hyper", + "hyper 1.8.1", "hyper-timeout", "hyper-util", "percent-encoding", @@ -3835,7 +4506,7 @@ dependencies = [ "rustls-pemfile", "socket2 0.5.10", "tokio", - "tokio-rustls", + "tokio-rustls 0.26.4", "tokio-stream", "tower 0.4.13", "tower-layer", @@ -3904,7 +4575,7 @@ dependencies = [ "bytes", "futures-util", "http 1.4.0", - "http-body", + "http-body 1.0.1", "http-body-util", "http-range-header", "httpdate", @@ -3929,7 +4600,7 @@ dependencies = [ "bytes", "futures-util", "http 1.4.0", - "http-body", + "http-body 1.0.1", "iri-string", "pin-project-lite", "tokio", @@ -4086,6 +4757,12 @@ dependencies = [ "serde", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "usearch" version = "2.24.0" @@ -4174,6 +4851,12 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "vsimd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" + [[package]] name = "wait-timeout" version = "0.2.1" @@ -4699,6 +5382,12 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" +[[package]] +name = "xmlparser" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" + [[package]] name = "xxhash-rust" version = "0.8.15" diff --git a/k8s/cluster/statefulset.yaml b/k8s/cluster/statefulset.yaml index 87451e9..925c118 100644 --- a/k8s/cluster/statefulset.yaml +++ b/k8s/cluster/statefulset.yaml @@ -84,7 +84,7 @@ spec: mountPath: /data containers: - name: tidaldb - image: registry.threesix.ai/tidal/server@sha256:2e1f7a0e2603fc135f6e79a8c376ca265f8ad74724904b1922df310e8e6da306 # m12-rc11 (= rc10 + corrected Bug 6: only force the re-baselining snapshot when frontier > baseline (divergent); at/below baseline keep frontier+1 so a caught-up node does NOT needlessly reseed-cascade) + image: registry.threesix.ai/tidal/server@sha256:ee0a8d8226c88102b7009605cb48768bda329ac3915ec68868e16cc74ca35d9a # m12-rc13 (= rc12 + WAL_RETENTION_SEGMENTS 4->16: a follower briefly down across a rolling restart stream-catches-up from WAL instead of forcing a snapshot reseed-on-rejoin; per-shard catch-up window 64MiB->256MiB, worst-case 768MiB/pod retained WAL) imagePullPolicy: IfNotPresent # The image ENTRYPOINT is the bare binary. We override the command with # a tiny /bin/sh wrapper (the bookworm-slim runtime HAS a shell) so we @@ -236,7 +236,14 @@ spec: cpu: "500m" memory: 1Gi limits: - cpu: "2" + # m12 read-SLA fix: 2->3. The cgroup cpu quota is what the engine's + # available_parallelism() reads (SEARCH_GATE / worker_threads sizing). + # At "2" a cross-shard search burst starved the async reactor + the + # election/heartbeat/apply loops (reads hung to the 30s route timeout; + # the starved control plane churned elections -> reseed self-exit). + # "3" leaves ~1 core for kubelet/system on the 4-core nodes; requests + # stay at 500m so the pod still schedules (server nodes alloc=3). + cpu: "3" memory: 4Gi # m12/1536: 100k×1536 HNSW load peaks ~1.9Gi; 1M needs more headroom (nodes have 13Gi allocatable) securityContext: allowPrivilegeEscalation: false diff --git a/tidal-server/src/cluster/node.rs b/tidal-server/src/cluster/node.rs index 74978e1..a85fa0c 100644 --- a/tidal-server/src/cluster/node.rs +++ b/tidal-server/src/cluster/node.rs @@ -6546,26 +6546,43 @@ async fn broadcast_to_peers( /// pre-m11p6 single read. The cross-group merge keeps each group's own diversity /// pass but does NOT re-diversify across groups (a cross-shard re-rank is the L4 /// follow-up; disjoint groups make the score-merge sound for cardinality). -fn scatter_merge( - dbs: &[Arc], +async fn scatter_merge( + dbs: Vec>, limit: usize, - score: impl Fn(&T) -> f64, + score: impl Fn(&T) -> f64 + Send, per_db: F, ) -> std::result::Result<(Vec, usize), ServerError> where - F: Fn(&TidalDb) -> std::result::Result<(Vec, usize), ServerError>, + T: Send + 'static, + // `per_db` runs on a `spawn_blocking` thread, once per shard, possibly + // concurrently — it must be `Clone + Send + Sync + 'static` and own a clone + // of its captured query state. Each invocation gets its own `Arc`. + F: Fn(Arc) -> crate::offload::ShardSearch + Clone + Send + Sync + 'static, { - // S=1: the one group's result is already ranked + limited by the engine. - // A single group's error IS the read's error — there is nothing to degrade to. - if let [only] = dbs { - return per_db(only); + // S=1: the one group's result is already ranked + limited by the engine. A + // single group's error IS the read's error — nothing to degrade to. Still + // gated + offloaded (one search, one permit) so it cannot pin the reactor. + if let [only] = dbs.as_slice() { + let only = Arc::clone(only); + let per_db = per_db.clone(); + return crate::offload::offload_search(move || per_db(only)).await; } + // Fan every hosted group's search out CONCURRENTLY. `offload_search` gates + // each on the process-wide SEARCH_GATE (~= core count), so a wide fan-out + // never oversubscribes the cores; a shed shard returns Backpressure here and + // is degraded over exactly like any other per-shard error. + let futures = dbs.into_iter().map(|db| { + let per_db = per_db.clone(); + crate::offload::offload_search(move || per_db(db)) + }); + let results = futures_util::future::join_all(futures).await; + let mut merged: Vec = Vec::new(); let mut total = 0usize; let mut ok_groups = 0usize; let mut last_err: Option = None; - for (idx, db) in dbs.iter().enumerate() { - match per_db(db) { + for (idx, result) in results.into_iter().enumerate() { + match result { Ok((items, candidates)) => { total = total.saturating_add(candidates); merged.extend(items); @@ -6931,20 +6948,19 @@ pub async fn feed( // m11p6: scatter the corpus-wide read over the LOCALLY hosted shard groups // and merge (see `scatter_merge`). - let retrieve = build_retrieve()?; + let retrieve = std::sync::Arc::new(build_retrieve()?); let dbs = node.hosted_dbs(); - let (local_items, local_total) = offload_region_read(move || { - scatter_merge( - &dbs, - limit, - |it: &tidaldb::query::RetrieveResult| it.score, - |db| { - let r = db.retrieve(&retrieve).map_err(ServerError::Tidal)?; - Ok((r.items, r.total_candidates)) - }, - ) - }) - .await?; + let (local_items, local_total) = scatter_merge( + dbs, + limit, + |it: &tidaldb::query::RetrieveResult| it.score, + move |db: Arc| { + let r = db.retrieve(&retrieve).map_err(ServerError::Tidal)?; + Ok((r.items, r.total_candidates)) + }, + ) + .await + .map_err(ClusterAppError)?; // m12p4: full placement / `S=1` → the local scatter already covers the whole // corpus; return it UNCHANGED (byte-for-byte the pre-m12p4 response). Under @@ -7126,21 +7142,20 @@ pub async fn search( } // m11p6: scatter the search over the LOCALLY hosted shard groups and merge. - let search_query = build_search()?; + let search_query = std::sync::Arc::new(build_search()?); let dbs = node.hosted_dbs(); - let (local_items, local_total) = offload_region_read(move || { - scatter_merge( - &dbs, - limit as usize, - |it: &tidaldb::query::SearchResultItem| it.score, - |db| { - db.reload_text_index().map_err(ServerError::Tidal)?; - let r = db.search(&search_query).map_err(ServerError::Tidal)?; - Ok((r.items, r.total_candidates)) - }, - ) - }) - .await?; + let (local_items, local_total) = scatter_merge( + dbs, + limit as usize, + |it: &tidaldb::query::SearchResultItem| it.score, + move |db: Arc| { + db.reload_text_index().map_err(ServerError::Tidal)?; + let r = db.search(&search_query).map_err(ServerError::Tidal)?; + Ok((r.items, r.total_candidates)) + }, + ) + .await + .map_err(ClusterAppError)?; // m12p4: full placement / `S=1` → unchanged. Partial placement → fan out. let missing = node.missing_groups(); @@ -7268,25 +7283,24 @@ pub async fn vector_search( } // m11p6: scatter the probe over the LOCALLY hosted shard groups and merge. - let vector = req.vector.clone(); + let vector = std::sync::Arc::new(req.vector.clone()); let dbs = node.hosted_dbs(); - let (local_items, local_total) = offload_region_read(move || { - scatter_merge( - &dbs, - k, - // Distance is "lower = better"; scatter_merge ranks by "higher = - // better", so the merge key is the negated distance. - |r: &tidaldb::storage::vector::VectorSearchResult| -f64::from(r.distance), - |db| { - let r = db - .vector_search_items(&vector, k, ef_search) - .map_err(ServerError::Tidal)?; - let n = r.len(); - Ok((r, n)) - }, - ) - }) - .await?; + let (local_items, local_total) = scatter_merge( + dbs, + k, + // Distance is "lower = better"; scatter_merge ranks by "higher = + // better", so the merge key is the negated distance. + |r: &tidaldb::storage::vector::VectorSearchResult| -f64::from(r.distance), + move |db: Arc| { + let r = db + .vector_search_items(&vector, k, ef_search) + .map_err(ServerError::Tidal)?; + let n = r.len(); + Ok((r, n)) + }, + ) + .await + .map_err(ClusterAppError)?; // m12p4: full placement / `S=1` → unchanged. Partial placement → fan out the // probe to each missing group (POST body carrying `shard=g`) and merge by diff --git a/tidal-server/src/main.rs b/tidal-server/src/main.rs index aa1cb56..02c6adb 100644 --- a/tidal-server/src/main.rs +++ b/tidal-server/src/main.rs @@ -109,12 +109,44 @@ struct StandaloneArgs { metrics: Option, } -#[tokio::main] -async fn main() { - if let Err(err) = run().await { - eprintln!("error: {err}"); - std::process::exit(1); - } +/// Entry point. +/// +/// We build the multi-thread runtime EXPLICITLY rather than via `#[tokio::main]` +/// so the reactor's worker-thread floor is decoupled from the container's CPU +/// cgroup quota. `#[tokio::main]` sizes `worker_threads` from +/// `available_parallelism()`, which is quota-aware: with `limits.cpu="2"` it +/// yields only 2 async workers, and a burst of CPU-bound shard searches on the +/// blocking pool can then starve the reactor + the election/heartbeat/apply +/// loops (the read-SLA collapse). Pinning `worker_threads` keeps the control +/// plane responsive independent of the quota. +/// +/// `max_blocking_threads` is DELIBERATELY left at tokio's default (512): the +/// CPU-bound search admission is bounded by the `offload::SEARCH_GATE` semaphore, +/// not by starving the blocking pool. Capping the blocking pool low is the one +/// change that could DEADLOCK (a `spawn_blocking` task parked in a nested +/// blocking wait would have no thread to make progress), so it is never lowered. +fn main() { + // Floor the reactor at 4 workers (one per physical core on the 4-vCPU + // nodes), regardless of the cgroup quota. `enable_all()` matches + // `#[tokio::main]`'s I/O + time driver setup. + let worker_threads = std::thread::available_parallelism() + .map_or(4, std::num::NonZeroUsize::get) + .max(4); + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(worker_threads) + // max_blocking_threads intentionally left at the 512 default — see above. + .enable_all() + .build() + .unwrap_or_else(|e| { + eprintln!("error: build tokio runtime: {e}"); + std::process::exit(1); + }); + runtime.block_on(async { + if let Err(err) = run().await { + eprintln!("error: {err}"); + std::process::exit(1); + } + }); } async fn run() -> Result<()> { diff --git a/tidal-server/src/offload.rs b/tidal-server/src/offload.rs index ec98ea0..69bfa8a 100644 --- a/tidal-server/src/offload.rs +++ b/tidal-server/src/offload.rs @@ -47,13 +47,51 @@ const READ_BACKPRESSURE_RETRY_AFTER_MS: u64 = 50; /// generous headroom, capped well below the blocking-pool limit. static READ_GATE: OnceLock = OnceLock::new(); +/// Process-wide cap on CONCURRENT single-db blocking reads admitted by +/// [`offload_read`] (standalone reads + the single-group internal `?shard` hops). +/// Each such request runs exactly ONE engine search, so a per-request permit is +/// the right unit here. Re-sized DOWN from the m12p6 `*16` fan-out (which let a +/// 2-quota pod admit 32 reads that, multiplied by the 3-shard serial scatter, +/// became ~96 concurrent CPU-bound searches and a multi-second p99): a small +/// multiple of the core count buffers a brief burst, then sheds as a fast 429. fn read_inflight_limit() -> usize { - std::thread::available_parallelism() - .map_or(8, std::num::NonZeroUsize::get) - .saturating_mul(16) - .clamp(16, 256) + core_parallelism() + .saturating_mul(READ_INFLIGHT_PER_CORE) + .clamp(MIN_READ_INFLIGHT, MAX_READ_INFLIGHT) } +/// Buffer factor for [`read_inflight_limit`]: a few queued single-db reads per +/// core absorb a burst without letting the blocking pool climb into a backlog. +const READ_INFLIGHT_PER_CORE: usize = 4; +const MIN_READ_INFLIGHT: usize = 4; +const MAX_READ_INFLIGHT: usize = 64; + +/// The quota-aware core count. `available_parallelism` honours the cgroup CPU +/// quota (e.g. `limits.cpu="3"` -> 3 even on a 4-core node), so this is the +/// number of CPU-bound shard-searches the pod may run truly in parallel. +fn core_parallelism() -> usize { + std::thread::available_parallelism().map_or(2, std::num::NonZeroUsize::get) +} + +/// Process-wide cap on CONCURRENT CPU-bound shard-searches across ALL in-flight +/// cross-shard reads. Sized to the core count (with a +1 of slack so one +/// in-flight search blocked on the embedding-registry read-lock cannot idle a +/// core), so the async reactor and the election/heartbeat/apply loops always +/// retain CPU. EVERY per-shard search in [`offload_search`] acquires one +/// permit; a 3-shard read therefore consumes up to 3 permits, and the gate — not +/// a per-read counter — is what bounds total search parallelism. Excess sheds as +/// a fast 429 (see `SEARCH_ADMIT_TIMEOUT_MS`). +static SEARCH_GATE: OnceLock = OnceLock::new(); + +fn search_inflight_limit() -> usize { + core_parallelism().saturating_add(1).clamp(2, 16) +} + +/// Max time a per-shard search waits for a [`SEARCH_GATE`] permit before shedding +/// as backpressure. Same 50ms budget as the read/write gates: a search drains in +/// worker-thread time, so a quick retry beats parking the request for seconds. +const SEARCH_ADMIT_TIMEOUT_MS: u64 = READ_BACKPRESSURE_RETRY_AFTER_MS; + /// Run a blocking READ-only query (RETRIEVE / SEARCH / text-index reload) on /// tokio's blocking pool, off the async reactor, and await its result. /// @@ -105,6 +143,57 @@ where // `_permit` drops here, releasing the read slot. } +/// Outcome of one shard's CPU-bound search inside a cross-shard scatter. +/// +/// Either the engine's `(items, total_candidates)` slice, or a per-shard error +/// to be degraded over (NOT to abort the whole read). A 429 here is a real +/// per-shard shed, surfaced like any other shard error so the merge serves +/// survivors. +pub type ShardSearch = std::result::Result<(Vec, usize), ServerError>; + +/// Run one CPU-bound per-shard search under the process-wide [`SEARCH_GATE`]. +/// +/// Off the reactor: acquires a permit (50ms timeout -> fast `Backpressure` 429, +/// never a hang), then `spawn_blocking`s the closure. The permit is moved INTO +/// the blocking closure and dropped only when the search finishes, so the gate +/// reflects searches actually burning a core, not merely admitted ones. +/// +/// # Errors +/// * `TidalError::Backpressure` (429) when the gate is saturated for 50ms. +/// * `ServerError::Cluster` (500) if the gate is closed (defensive; the static +/// lives for the process) or the blocking task panicked. +pub async fn offload_search(f: F) -> Result +where + F: FnOnce() -> Result + Send + 'static, + T: Send + 'static, +{ + let gate = SEARCH_GATE.get_or_init(|| Semaphore::new(search_inflight_limit())); + let permit = match tokio::time::timeout( + std::time::Duration::from_millis(SEARCH_ADMIT_TIMEOUT_MS), + gate.acquire(), + ) + .await + { + Ok(Ok(permit)) => permit, + Ok(Err(_closed)) => { + return Err(ServerError::Cluster("search admission gate closed".into())); + } + Err(_elapsed) => { + return Err(ServerError::Tidal(TidalError::Backpressure { + retry_after_ms: SEARCH_ADMIT_TIMEOUT_MS, + })); + } + }; + // Move the permit into the blocking task so it is held for the search's whole + // CPU lifetime and released on the blocking thread when the search returns. + tokio::task::spawn_blocking(move || { + let _permit = permit; + f() + }) + .await + .map_err(|e| ServerError::Cluster(format!("blocking search worker failed: {e}")))? +} + /// Configuration for the cluster write worker pool. /// /// Defaults are derived once at startup: `workers` from diff --git a/tidal-server/tests/cluster_reseed.rs b/tidal-server/tests/cluster_reseed.rs index c954131..92bdf4d 100644 --- a/tidal-server/tests/cluster_reseed.rs +++ b/tidal-server/tests/cluster_reseed.rs @@ -86,15 +86,17 @@ const FAST_ELECTION_YAML: &str = "election:\n heartbeat_interval_ms: 100\n ele /// follower's frontier (a single segment is never deleted; the snapshot path /// only triggers once history is genuinely compacted away). /// -/// Sizing (m12p6): the graceful-shutdown compaction now RETAINS the -/// `WAL_RETENTION_SEGMENTS` (= 4) most-recent sealed segments so a briefly-down -/// follower can stream-catch-up instead of reseeding. For this exit-gate to STILL -/// force the snapshot-required path, the offline batch must compact the follower's -/// frontier segment away DESPITE retention — i.e. produce more than -/// `WAL_RETENTION_SEGMENTS + 1` segments. At ~56 KiB/item the 16 MiB segment holds -/// ~292 items, so 1800 items ≈ 6 segments ⇒ the follower's frontier segment is -/// well past the 4-segment retention window and is genuinely deleted. -const OFFLINE_ITEMS: u64 = 1800; +/// Sizing (m12 rolling-restart tuning): the graceful-shutdown compaction now +/// RETAINS the `WAL_RETENTION_SEGMENTS` (= 16) most-recent sealed segments so a +/// briefly-down follower can stream-catch-up instead of reseeding. For this +/// exit-gate to STILL force the snapshot-required path, the offline batch must +/// compact the follower's frontier segment away DESPITE retention — i.e. produce +/// more than `WAL_RETENTION_SEGMENTS + 1` = 17 segments. At ~56 KiB/item the +/// 16 MiB segment holds ~292 items, so 5600 items ≈ 19 segments ⇒ the follower's +/// frontier segment is well past the 16-segment retention window and is genuinely +/// deleted. (Sized off the constant + a 3-segment margin so a future `N` bump +/// never silently un-sizes this gate.) +const OFFLINE_ITEMS: u64 = 5600; /// The engine caps item metadata at 8 KiB per VALUE and 64 KiB TOTAL per item /// (a hard query-index-integrity invariant — never to be weakened). So the WAL @@ -586,10 +588,18 @@ fn mp_quarantined_node_reseeds_without_wipe() { convergence_budget() + Duration::from_secs(10), "the restarted old leader quarantines on its divergent suffix", ); - assert!( - status_bool(&cluster, LEADER, "reseed_required"), - "the quarantine latch must also write the reseed marker (m11p5 §2.4): {:?}", - cluster.local_status(LEADER) + // The quarantine flag (election_driver `quarantined.store`) and the reseed + // marker (`latch_reseed_marker`) are latched SEQUENTIALLY, not atomically, so + // there is a brief window where status reports quarantined=true before the + // marker surfaces. Poll for the marker (matching the quarantined check above) + // rather than a single-shot read, which races under concurrent-test load. + await_status_bool( + &cluster, + LEADER, + "reseed_required", + true, + convergence_budget() + Duration::from_secs(10), + "the quarantine latch must also write the reseed marker (m11p5 §2.4)", ); println!("[quarantine] old leader quarantined AND latched reseed_required (no wipe)"); diff --git a/tidal-stress/k8s/recall-rc12-spread-job.yaml b/tidal-stress/k8s/recall-rc12-spread-job.yaml new file mode 100644 index 0000000..c7a391b --- /dev/null +++ b/tidal-stress/k8s/recall-rc12-spread-job.yaml @@ -0,0 +1,102 @@ +# rc12 read-SLA gate — true p99 + recall@10 vs a brute-force cosine oracle, +# with read load SPREAD across all 3 nodes (full placement => any node serves any +# read). recall.rs round-robins each probe across the --target list, so three +# per-pod DNS targets give an even 1/3-per-node split (deterministic, unlike a +# single ClusterIP Service which leans on kube-proxy connection balancing). +# All three per-pod DNS names are cert SANs (wildcard *.tidaldb-peers + -0/-1/-2). +# +# Skip-seed: the 100k/1536 corpus already lives on the PVCs (a rolling restart +# does not wipe them); the harness regenerates the SAME deterministic base +# vectors locally to build the brute-force ground-truth oracle. +# +# Apply: kubectl apply -f tidal-stress/k8s/recall-rc12-spread-job.yaml +# Watch: kubectl logs -f job/tidal-recall-rc12 -n tidaldb-cluster +apiVersion: batch/v1 +kind: Job +metadata: + name: tidal-recall-rc12 + namespace: tidaldb-cluster + labels: + app.kubernetes.io/name: tidal-stress + app.kubernetes.io/part-of: tidaldb +spec: + backoffLimit: 0 + ttlSecondsAfterFinished: 7200 + template: + metadata: + labels: + app.kubernetes.io/name: tidal-stress + app.kubernetes.io/part-of: tidaldb + spec: + restartPolicy: Never + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + seccompProfile: + type: RuntimeDefault + containers: + - name: stress + image: registry.threesix.ai/tidal/stress:m12-rc7-seedretry + imagePullPolicy: IfNotPresent + args: + - --target + - https://tidaldb-0.tidaldb-peers.tidaldb-cluster.svc.cluster.local:9500 + - --target + - https://tidaldb-1.tidaldb-peers.tidaldb-cluster.svc.cluster.local:9500 + - --target + - https://tidaldb-2.tidaldb-peers.tidaldb-cluster.svc.cluster.local:9500 + - --ca-cert + - /etc/tidaldb/tls/ca.crt + - --verify-recall + - --skip-seed + - --seed-concurrency + - "32" + - --corpus + - "100000" + - --embedding-dim + - "1536" + - --recall-k + - "10" + - --recall-queries + - "1000" + - --read-p99-target-ms + - "10" + - --recall-target + - "0.95" + - --recall-ef-search + - "64" + - --ramp + - "100:30,200:30,300:30,500:30" + env: + - name: TIDAL_API_KEY + valueFrom: + secretKeyRef: + name: tidaldb-credentials + key: TIDAL_API_KEY + - name: TIDAL_STRESS_LOG + value: warn + resources: + requests: + cpu: 500m + memory: 512Mi + limits: + cpu: "3" + memory: 2Gi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + volumeMounts: + - name: cluster-tls + mountPath: /etc/tidaldb/tls + readOnly: true + volumes: + - name: cluster-tls + secret: + secretName: tidaldb-cluster-tls + items: + - key: ca.crt + path: ca.crt diff --git a/tidal-stress/k8s/soak-monitor.yaml b/tidal-stress/k8s/soak-monitor.yaml new file mode 100644 index 0000000..3c23ffc --- /dev/null +++ b/tidal-stress/k8s/soak-monitor.yaml @@ -0,0 +1,157 @@ +# Soak monitor — the always-on in-cluster observer for the 30-night soak. +# +# Two jobs, both writing to the SAME durable RWX result PVC the nightly CronJob +# writes its JSON summaries to, so EVERYTHING the operator needs to judge the +# 30-night streak lives in one place and survives the laptop session ending: +# +# 1. Restart watch: every 5 min, snapshot the tidaldb-{0,1,2} pod restart +# counts + phase into /results/restarts.tsv. The GA bar is not just +# "30 green soak verdicts" — it is ZERO unrecovered failures over the +# window. An under-load pod restart during a soak night must be visible +# even if the soak Job itself still passed, so we record it independently. +# +# 2. HTTP read surface: serve /results over HTTP on :8080 so the operator can +# `kubectl port-forward deploy/tidal-soak-monitor 8080:8080 -n tidaldb-cluster` +# and read the ledger / nightly summaries / restart log from a browser at +# any time, from any machine, without exec'ing into a pod. +# +# The monitor does NOT generate load and does NOT gate anything — it is a passive +# recorder. The pass/fail signal is the CronJob's Job exit codes; this just makes +# the 30-night picture observable and durable. +# +# Apply: kubectl apply -f tidal-stress/k8s/soak-monitor.yaml +# Read: kubectl port-forward deploy/tidal-soak-monitor 8080:8080 -n tidaldb-cluster +# then open http://localhost:8080/ledger.tsv (and /restarts.tsv, /) +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: tidal-soak-monitor + namespace: tidaldb-cluster + labels: + app.kubernetes.io/name: tidal-soak + app.kubernetes.io/part-of: tidaldb +automountServiceAccountToken: true +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: tidal-soak-monitor + namespace: tidaldb-cluster + labels: + app.kubernetes.io/name: tidal-soak + app.kubernetes.io/part-of: tidaldb +rules: + # Read-only: pod restart counts + phase, and the nightly soak Job verdicts. + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch"] + - apiGroups: ["batch"] + resources: ["jobs"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: tidal-soak-monitor + namespace: tidaldb-cluster + labels: + app.kubernetes.io/name: tidal-soak + app.kubernetes.io/part-of: tidaldb +subjects: + - kind: ServiceAccount + name: tidal-soak-monitor + namespace: tidaldb-cluster +roleRef: + kind: Role + name: tidal-soak-monitor + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: tidal-soak-monitor + namespace: tidaldb-cluster + labels: + app.kubernetes.io/name: tidal-soak + app.kubernetes.io/part-of: tidaldb +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: tidal-soak-monitor + template: + metadata: + labels: + app.kubernetes.io/name: tidal-soak-monitor + app.kubernetes.io/part-of: tidaldb + spec: + serviceAccountName: tidal-soak-monitor + securityContext: + runAsNonRoot: true + runAsUser: 1001 + runAsGroup: 1001 + fsGroup: 1001 + seccompProfile: + type: RuntimeDefault + containers: + # ── Restart-watch sidecar: kubectl snapshot loop ────────────────────── + - name: restart-watch + image: bitnami/kubectl:latest + imagePullPolicy: IfNotPresent + command: ["/bin/sh", "-c"] + args: + - | + echo "restart-watch up $(date -u +%FT%TZ)" + # Header once (only if the file is new/empty). + if [ ! -s /results/restarts.tsv ]; then + printf 'ts_utc\tpod\trestarts\tphase\tready\n' >> /results/restarts.tsv + fi + while true; do + TS="$(date -u +%FT%TZ)" + kubectl get pods -n tidaldb-cluster \ + -l app.kubernetes.io/name=tidaldb \ + -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.containerStatuses[0].restartCount}{"\t"}{.status.phase}{"\t"}{.status.containerStatuses[0].ready}{"\n"}{end}' 2>/dev/null \ + | while IFS="$(printf '\t')" read -r POD RC PH RD; do + [ -n "$POD" ] && printf '%s\t%s\t%s\t%s\t%s\n' "$TS" "$POD" "$RC" "$PH" "$RD" >> /results/restarts.tsv + done + sleep 300 + done + resources: + requests: { cpu: 10m, memory: 32Mi } + limits: { cpu: 100m, memory: 128Mi } + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: { drop: ["ALL"] } + volumeMounts: + - name: results + mountPath: /results + # ── HTTP read surface: serve the durable result dir ─────────────────── + - name: http + image: busybox:1.36 + imagePullPolicy: IfNotPresent + # busybox httpd: one-shot static file server rooted at /results. + command: ["/bin/sh", "-c"] + args: + - | + echo "http surface up $(date -u +%FT%TZ) on :8080 serving /results" + exec httpd -f -p 8080 -h /results + ports: + - name: http + containerPort: 8080 + resources: + requests: { cpu: 10m, memory: 16Mi } + limits: { cpu: 100m, memory: 64Mi } + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: { drop: ["ALL"] } + volumeMounts: + - name: results + mountPath: /results + readOnly: true + volumes: + - name: results + persistentVolumeClaim: + claimName: tidal-soak-results diff --git a/tidal-stress/k8s/soak-nightly-cronjob.yaml b/tidal-stress/k8s/soak-nightly-cronjob.yaml new file mode 100644 index 0000000..d33e488 --- /dev/null +++ b/tidal-stress/k8s/soak-nightly-cronjob.yaml @@ -0,0 +1,162 @@ +# 30-NIGHT SOAK — the calendar half of the M11 GA bar. +# +# GA exit gate (docs/planning/milestone-11/phase-9.md §"Exit gate", +# docs/roadmap-to-cluster.md:115, guarantee-traceability.md G-C): +# "Nightly suite green 30 consecutive days before GA." +# +# This CronJob IS that nightly. It fires once per night and runs a REAL sustained +# mixed read+write soak against the live 3-node cluster (round-robin across all +# three pods — full placement, every pod serves every shard group), with the +# m11p9 regression gates ARMED so a perf/correctness regression on any night +# makes the Job FAIL (non-zero exit) — that night is NOT green and the 30-night +# streak resets. +# +# PASS CRITERIA PER NIGHT (each must hold for the Job to exit 0): +# --fail-on-knee : no stage breached the built-in SLO (error rate >1% OR +# feed p99 >150ms — the capacity knee). +# --max-p99-ms 150 : no stage's worst-op p99 exceeded 150ms (cluster network- +# hop SLO; in-process SLA is 50ms, the read-recall G1 is +# 10ms — 150 is the soak regression tripwire, well above +# the measured 9.28ms read p99 / ~16ms cross-shard p99). +# --max-error-pct 1 : no stage's error rate exceeded 1%. +# The streak passes when 30 CONSECUTIVE nightly Jobs have exited 0 with zero +# under-load pod restarts (the monitor records restarts; see soak-monitor.yaml). +# +# SUSTAINED RATE: 500 rps mixed (peach: feed/search/view/like/skip/item/embed) — +# deliberately BELOW the SLA knee. The rc12 read-SLA gate sustained 500 rps at +# p99 9.28ms / recall 0.9989 / 0% err / 0 under-load restarts, so 500 rps is a +# safe endurance rate that exercises BOTH read and write paths without parking +# the cluster at its ceiling (a soak proves stability over time, not peak — the +# 1-hour 100k-DAU peak run is the separate `--ramp 3900:3600` throughput gate). +# +# WHY A CRONJOB, NOT ONE LONG JOB: the GA bar is literally "30 CONSECUTIVE DAYS" +# of an independent nightly verdict — a CronJob produces exactly that audit trail +# (30 dated Job objects, each PASS/FAIL), survives node reboots/evictions, and +# each night re-pulls fresh cluster state (a single 30-day Job would mask a +# mid-window regression and die on any one eviction). Each night is ~70 min of +# sustained load (a 1h hold + warm-up), bounded by activeDeadlineSeconds. +# +# Apply: kubectl apply -f tidal-stress/k8s/soak-nightly-cronjob.yaml +# Watch tonight's run: kubectl get jobs -n tidaldb-cluster -l app.kubernetes.io/name=tidal-soak +# Read a night's verdict: kubectl logs -n tidaldb-cluster job/ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: tidal-soak-nightly + namespace: tidaldb-cluster + labels: + app.kubernetes.io/name: tidal-soak + app.kubernetes.io/part-of: tidaldb +spec: + # 02:00 cluster-local nightly. Off-peak; one hour hold finishes well before any + # morning activity. concurrencyPolicy Forbid: never overlap two soaks (they + # would contend for the same 3 nodes and mutually depress p99 → false FAIL). + schedule: "0 2 * * *" + concurrencyPolicy: Forbid + startingDeadlineSeconds: 3600 + successfulJobsHistoryLimit: 30 # keep all 30 nights of PASS verdicts + failedJobsHistoryLimit: 30 # keep every FAIL for streak-reset forensics + jobTemplate: + metadata: + labels: + app.kubernetes.io/name: tidal-soak + app.kubernetes.io/part-of: tidaldb + spec: + backoffLimit: 0 # a failed night is a FAIL — do not silently retry + activeDeadlineSeconds: 5400 # 90 min hard cap (1h hold + warm-up + margin) + ttlSecondsAfterFinished: 2678400 # keep finished Job pods 31 days (full window) + template: + metadata: + labels: + app.kubernetes.io/name: tidal-soak + app.kubernetes.io/part-of: tidaldb + spec: + restartPolicy: Never + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 # so the soak user can write the RWX result PVC + seccompProfile: + type: RuntimeDefault + containers: + - name: soak + image: registry.threesix.ai/tidal/stress:m12-rc7-seedretry + imagePullPolicy: IfNotPresent + # entrypoint is tidal-stress; wrap it so we can stamp the result file + # name with the date and append a ledger line regardless of verdict. + command: ["/bin/sh", "-c"] + args: + - | + set -u + DATE="$(date -u +%Y-%m-%d)" + OUT="/results/soak-$DATE.json" + echo "soak $DATE start $(date -u +%H:%M:%SZ) target=cluster image=m12-rc7-seedretry" + tidal-stress \ + --target https://tidaldb-0.tidaldb-peers.tidaldb-cluster.svc.cluster.local:9500 \ + --target https://tidaldb-1.tidaldb-peers.tidaldb-cluster.svc.cluster.local:9500 \ + --target https://tidaldb-2.tidaldb-peers.tidaldb-cluster.svc.cluster.local:9500 \ + --ca-cert /etc/tidaldb/tls/ca.crt \ + --skip-seed \ + --corpus 100000 \ + --embedding-dim 1536 \ + --mix peach \ + --users 50000 \ + --ramp "500:3600" \ + --json-summary "$OUT" \ + --max-p99-ms 150 \ + --max-error-pct 1 \ + --fail-on-knee + RC=$? + # Extract the verdict + headline p99/error from the JSON summary + # for a one-line ledger entry (grep, no jq dependency in the image). + # Fields are the real summary.rs keys: top-level "passed" (bool), + # per-stage "overall_p99_ms" (ms) and "error_rate" (FRACTION 0-1). + # Worst stage = max p99 over the run; we report the LAST stage's + # numbers (the sustained-rate stage, since the soak ramp is single- + # stage 500:3600 anyway) and trust "passed" for the verdict. + VERD="PASS"; [ "$RC" -ne 0 ] && VERD="FAIL" + PASSED="$(grep -o '"passed"[: ]*[a-z]*' "$OUT" 2>/dev/null | head -1 | grep -o '[a-z]*$')" + P99="$(grep -o '"overall_p99_ms"[: ]*[0-9.]*' "$OUT" 2>/dev/null | tail -1 | grep -o '[0-9.]*$')" + ERRF="$(grep -o '"error_rate"[: ]*[0-9.]*' "$OUT" 2>/dev/null | tail -1 | grep -o '[0-9.]*$')" + printf '%s\t%s\trc=%s\tpassed=%s\tp99_ms=%s\terr_frac=%s\timage=m12-rc7-seedretry\n' \ + "$DATE" "$VERD" "$RC" "${PASSED:-?}" "${P99:-?}" "${ERRF:-?}" >> /results/ledger.tsv + echo "soak $DATE end $(date -u +%H:%M:%SZ) verdict=$VERD rc=$RC p99=${P99:-?} err=${ERR:-?}" + exit "$RC" + env: + - name: TIDAL_API_KEY + valueFrom: + secretKeyRef: + name: tidaldb-credentials + key: TIDAL_API_KEY + - name: TIDAL_STRESS_LOG + value: warn + resources: + requests: + cpu: 500m + memory: 512Mi + limits: + cpu: "2" + memory: 2Gi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + volumeMounts: + - name: cluster-tls + mountPath: /etc/tidaldb/tls + readOnly: true + - name: results + mountPath: /results + volumes: + - name: cluster-tls + secret: + secretName: tidaldb-cluster-tls + items: + - key: ca.crt + path: ca.crt + - name: results + persistentVolumeClaim: + claimName: tidal-soak-results diff --git a/tidal-stress/k8s/soak-results-pvc.yaml b/tidal-stress/k8s/soak-results-pvc.yaml new file mode 100644 index 0000000..8dd3e44 --- /dev/null +++ b/tidal-stress/k8s/soak-results-pvc.yaml @@ -0,0 +1,25 @@ +# Durable result sink for the 30-night soak (GA bar: nightly suite green 30 +# consecutive days — docs/planning/milestone-11/phase-9.md "Exit gate"). +# +# longhorn-rwx: RWX so the nightly soak CronJob pods (writers) and the always-on +# soak-monitor (reader) can mount it concurrently, and Retain reclaim policy so +# the 30 nights of JSON summaries survive a PVC delete / job churn / operator +# session ending. Each night writes /results/soak-YYYY-MM-DD.json + appends one +# line to /results/ledger.tsv (dateverdictp99err% image). +# +# Apply: kubectl apply -f tidal-stress/k8s/soak-results-pvc.yaml +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: tidal-soak-results + namespace: tidaldb-cluster + labels: + app.kubernetes.io/name: tidal-soak + app.kubernetes.io/part-of: tidaldb +spec: + accessModes: + - ReadWriteMany + storageClassName: longhorn-rwx + resources: + requests: + storage: 2Gi diff --git a/tidal-stress/k8s/t5-readtput-job.yaml b/tidal-stress/k8s/t5-readtput-job.yaml new file mode 100644 index 0000000..d2e1d26 --- /dev/null +++ b/tidal-stress/k8s/t5-readtput-job.yaml @@ -0,0 +1,67 @@ +# T5 read-throughput (the measurable half): how many /vector_search ops/s the +# 3-node cluster sustains within SLA, spread round-robin across all 3 pods. +# (T5-as-written's 2.5x WRITE-scaling is structurally impossible on 3-node RF3 +# full placement — proven in docs/profiling/m12p4-t5-sharded-throughput.md.) +apiVersion: batch/v1 +kind: Job +metadata: + name: tidal-t5-readtput + namespace: tidaldb-cluster + labels: { app.kubernetes.io/name: tidal-stress, app.kubernetes.io/part-of: tidaldb } +spec: + backoffLimit: 0 + ttlSecondsAfterFinished: 7200 + template: + metadata: + labels: { app.kubernetes.io/name: tidal-stress, app.kubernetes.io/part-of: tidaldb } + spec: + restartPolicy: Never + automountServiceAccountToken: false + securityContext: { runAsNonRoot: true, runAsUser: 1000, runAsGroup: 1000, seccompProfile: { type: RuntimeDefault } } + containers: + - name: stress + image: registry.threesix.ai/tidal/stress:m12-rc7-seedretry + imagePullPolicy: IfNotPresent + args: + - --target + - https://tidaldb-0.tidaldb-peers.tidaldb-cluster.svc.cluster.local:9500 + - --target + - https://tidaldb-1.tidaldb-peers.tidaldb-cluster.svc.cluster.local:9500 + - --target + - https://tidaldb-2.tidaldb-peers.tidaldb-cluster.svc.cluster.local:9500 + - --ca-cert + - /etc/tidaldb/tls/ca.crt + - --verify-recall + - --skip-seed + - --corpus + - "100000" + - --embedding-dim + - "1536" + - --recall-k + - "10" + - --recall-queries + - "1000" + - --read-p99-target-ms + - "10" + - --recall-target + - "0.95" + - --recall-ef-search + - "64" + - --ramp + - "1000:20,2000:20,3000:20,3800:20" + env: + - name: TIDAL_API_KEY + valueFrom: { secretKeyRef: { name: tidaldb-credentials, key: TIDAL_API_KEY } } + - name: TIDAL_STRESS_LOG + value: warn + resources: + requests: { cpu: "1", memory: 512Mi } + limits: { cpu: "3", memory: 2Gi } + securityContext: { allowPrivilegeEscalation: false, readOnlyRootFilesystem: true, capabilities: { drop: ["ALL"] } } + volumeMounts: + - { name: cluster-tls, mountPath: /etc/tidaldb/tls, readOnly: true } + volumes: + - name: cluster-tls + secret: + secretName: tidaldb-cluster-tls + items: [ { key: ca.crt, path: ca.crt } ] diff --git a/tidal/src/wal/compaction.rs b/tidal/src/wal/compaction.rs index 4b2f4cd..ba05edc 100644 --- a/tidal/src/wal/compaction.rs +++ b/tidal/src/wal/compaction.rs @@ -79,7 +79,26 @@ pub fn compact_wal(wal_dir: &Path, checkpoint_seq: u64) -> Result 16) +/// +/// A 1536-dim embedding insert rides the WAL as one kind-2 blob ≈ 64 B header + +/// 6144 B vector + ~64 B meta ≈ 6.27 KiB/seqno; a packed signal-event batch is +/// 32 B/event up to 256 events/batch. At `N = 4` the per-shard catch-up window is +/// `4 * 16 MiB = 64 MiB` ≈ only ~10.4 k embedding-seqnos, so a follower that fell +/// a few thousand inserts behind (or was down across a rolling-restart) crossed +/// the window and was forced into a full snapshot reseed. `N = 16` gives +/// `16 * 16 MiB = 256 MiB` ≈ ~42.8 k embedding-seqnos (or ~8.4 M batched +/// event-seqnos) of stream-catch-up headroom per shard. +/// +/// # Disk bound +/// +/// Retention is per-shard. With 3 shards hosted per pod under one 5 GiB PVC, the +/// worst-case retained-WAL floor is `3 * 16 * 16 MiB = 768 MiB` (≈ 15 % of the +/// PVC), bounded and self-trimming as segments age past `N`. The remaining +/// ~4.2 GiB stays free for the signal ledger, HNSW graph, snapshots, and the live +/// (uncompacted) segment per shard. +pub const WAL_RETENTION_SEGMENTS: usize = 16; /// Lower `floor` so the `WAL_RETENTION_SEGMENTS` most-recent (highest-`first_seq`) /// segments always survive, regardless of the checkpoint. `segments` is sorted diff --git a/tidalctl/Cargo.toml b/tidalctl/Cargo.toml index 3ed7610..6eb7b35 100644 --- a/tidalctl/Cargo.toml +++ b/tidalctl/Cargo.toml @@ -36,6 +36,26 @@ serde_json = "1" # WAL segments and snapshot artifacts with, so a tidalctl-written manifest and an # engine-verified one agree. blake3 = "1" +# Object-store (S3-compatible) export/import for the DR gate (m11p8 R2). Chosen +# over `rust-s3` because the AWS SDK shares the workspace's existing +# hyper/rustls/tokio tree (no native-tls / second TLS stack), and R2 is handled +# with `.endpoint_url(.r2.cloudflarestorage.com)` + `.force_path_style`. +# `default-features = false` + `rustls` keeps the TLS stack aligned with the rest +# of the workspace (reqwest is already `rustls-tls`); `behavior-version-latest` +# pins the SDK behavior contract so an SDK minor bump can't silently change it. +aws-config = { version = "1.8", default-features = false, features = ["behavior-version-latest", "rustls"] } +# `rt-tokio` exposes `ByteStream::from_path` (stream a file body straight off +# disk on `put_object` — no whole-file read into memory). +aws-sdk-s3 = { version = "1.137", default-features = false, features = ["behavior-version-latest", "rustls", "rt-tokio"] } +# `hardcoded-credentials` exposes `Credentials::from_keys` so the env-var creds +# (AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY) become a static SigV4 provider. +aws-credential-types = { version = "1.2", features = ["hardcoded-credentials"] } +# S3 transfers run on a locally-built current-thread runtime ONLY when an --s3-* +# flag is given; the local-dir backup/restore path stays fully synchronous. +tokio = { version = "1", default-features = false, features = ["rt", "macros"] } +# Fresh staging dir for an S3 import (downloaded prefix -> temp dir -> the +# UNCHANGED verified restore reads it). Reaped when the restore returns. +tempfile = "3" [dev-dependencies] tidaldb = { path = "../tidal", features = ["test-utils"] } diff --git a/tidalctl/src/commands/backup.rs b/tidalctl/src/commands/backup.rs index 02c7237..5cc2464 100644 --- a/tidalctl/src/commands/backup.rs +++ b/tidalctl/src/commands/backup.rs @@ -35,6 +35,7 @@ use std::{ use serde::{Deserialize, Serialize}; +use crate::commands::s3::S3Target; use crate::{CliError, wal_state}; /// Manifest filename written at the root of every backup. @@ -80,7 +81,18 @@ const MANIFEST_VERSION: u32 = 1; /// `tidalctl backup --path --out `: copy the data dir to `dest` /// and write a BLAKE3 manifest. Refuses to overwrite a non-empty destination so a /// backup never clobbers an existing one. -pub(crate) fn run_backup(src: &Path, out: &Path, pretty: bool) -> Result<(String, i32), CliError> { +/// +/// When `s3` is `Some`, the export is ADDITIVE: after the local `out` dir and its +/// `BACKUP_MANIFEST.json` are written and fsynced (the consistency barrier), +/// every file under `out` is mirrored into the bucket with the manifest uploaded +/// LAST as the export's atomicity marker. With `s3 == None` the behavior is +/// byte-identical to the original local-only backup. +pub(crate) fn run_backup( + src: &Path, + out: &Path, + s3: Option<&S3Target>, + pretty: bool, +) -> Result<(String, i32), CliError> { if !src.is_dir() { return Err(CliError::new(format!( "source data dir does not exist: {}", @@ -167,24 +179,71 @@ pub(crate) fn run_backup(src: &Path, out: &Path, pretty: bool) -> Result<(String fsync_dirs_recursive(out) .map_err(|e| CliError::new(format!("fsync backup dirs {}: {e}", out.display())))?; - let summary = serde_json::json!({ + // ADDITIVE S3/R2 export. The local artifact above is the source of truth and + // is already durable; the export walks it and `put_object`s every file under + // the prefix, uploading BACKUP_MANIFEST.json LAST so the manifest's presence + // in the bucket signals a complete export (the object-store analogue of the + // local fsync barrier). A pure local backup (`s3 == None`) skips this entirely. + let mut summary = serde_json::json!({ "backed_up": src.display().to_string(), "destination": out.display().to_string(), "checkpoint_seq": manifest.checkpoint_seq, "file_count": manifest.file_count, "total_bytes": manifest.total_bytes, }); + if let Some(target) = s3 { + let uploaded = crate::commands::s3::export_dir(out, target)?; + summary["s3_export"] = serde_json::json!({ + "endpoint": target.endpoint, + "bucket": target.bucket, + "prefix": target.prefix, + "objects_uploaded": uploaded, + "atomicity_marker": BACKUP_MANIFEST, + }); + } Ok((render(&summary, pretty)?, 0)) } /// `tidalctl restore --from --path `: verify the backup's BLAKE3 /// manifest, then copy every file into `target`. Refuses a non-empty target so a /// restore never overwrites a live data dir (the destructive-op guard). +/// +/// When `s3` is `Some`, the import is ADDITIVE: the prefix is downloaded into a +/// FRESH temp staging dir and then the UNCHANGED verified restore runs on that +/// staging dir (BLAKE3 verification, `safe_join`, fresh-target placement). The +/// `--from` flag is not required in S3 mode (the staging dir replaces it); with +/// `s3 == None` the behavior is byte-identical to the original local restore. pub(crate) fn run_restore( target: &Path, - from: &Path, + from: Option<&Path>, + s3: Option<&S3Target>, pretty: bool, ) -> Result<(String, i32), CliError> { + // Resolve the directory the verified restore reads from. In S3 mode, download + // the prefix into a fresh temp staging dir and restore from THAT, unchanged; + // the TempDir guard is held for the whole restore so the staging bytes survive + // BLAKE3 verification + copy, then are reaped on return. + let staging = match s3 { + Some(target_s3) => { + let dir = tempfile::Builder::new() + .prefix("tidalctl-s3-restore-") + .tempdir() + .map_err(|e| CliError::new(format!("create S3 staging dir: {e}")))?; + let downloaded = crate::commands::s3::import_to_dir(dir.path(), target_s3)?; + Some((dir, downloaded)) + } + None => None, + }; + let (from, s3_downloaded): (&Path, Option) = match (&staging, from) { + (Some((dir, n)), _) => (dir.path(), Some(*n)), + (None, Some(local)) => (local, None), + (None, None) => { + return Err(CliError::new( + "restore requires --from (or --s3-endpoint/--s3-bucket for an object-store import)", + )); + } + }; + let manifest_path = from.join(BACKUP_MANIFEST); let manifest_bytes = std::fs::read(&manifest_path).map_err(|e| { CliError::new(format!( @@ -264,13 +323,25 @@ pub(crate) fn run_restore( fsync_dirs_recursive(target) .map_err(|e| CliError::new(format!("fsync target dirs {}: {e}", target.display())))?; - let summary = serde_json::json!({ + let mut summary = serde_json::json!({ "restored": target.display().to_string(), "from": from.display().to_string(), "checkpoint_seq": manifest.checkpoint_seq, "files_verified_and_restored": restored, "note": "point a stopped node at this dir; under ack=quorum followers catch up via the live stream", }); + if let (Some(s3_target), Some(downloaded)) = (s3, s3_downloaded) { + summary["s3_import"] = serde_json::json!({ + "endpoint": s3_target.endpoint, + "bucket": s3_target.bucket, + "prefix": s3_target.prefix, + "objects_downloaded": downloaded, + "staged_then_verified": true, + }); + } + // The TempDir staging guard (if any) is dropped here, after the verified + // restore copied every byte into `target` — the staged download is reaped. + drop(staging); Ok((render(&summary, pretty)?, 0)) } diff --git a/tidalctl/src/commands/mod.rs b/tidalctl/src/commands/mod.rs index 3e7df89..2d00179 100644 --- a/tidalctl/src/commands/mod.rs +++ b/tidalctl/src/commands/mod.rs @@ -9,5 +9,6 @@ pub(crate) mod backup; pub(crate) mod diagnostics; pub(crate) mod paths; pub(crate) mod recover; +pub(crate) mod s3; pub(crate) mod scope_stats; pub(crate) mod status; diff --git a/tidalctl/src/commands/s3.rs b/tidalctl/src/commands/s3.rs new file mode 100644 index 0000000..e2445cc --- /dev/null +++ b/tidalctl/src/commands/s3.rs @@ -0,0 +1,522 @@ +//! Object-store (S3-compatible, targeting Cloudflare R2) export/import for the DR +//! gate (m11p8). +//! +//! This is an ADDITIVE layer over the local-dir backup/restore in +//! [`crate::commands::backup`]: it never reimplements the consistency-proven +//! backup engine. EXPORT runs AFTER the local artifact (data files + +//! `BACKUP_MANIFEST.json`) is written and fsynced, then mirrors that exact tree +//! into the bucket. IMPORT downloads the prefix into a temp staging dir and hands +//! it to the UNCHANGED verified restore (BLAKE3 + `safe_join` + fresh-target). +//! +//! ## Atomicity marker +//! +//! `BACKUP_MANIFEST.json` is uploaded LAST. The restore reads the manifest first +//! and verifies every listed file's BLAKE3 against the bytes it downloaded, so a +//! partial export (process died mid-upload) leaves the manifest absent and the +//! import fails closed with "is the prefix a tidalctl backup?" rather than +//! restoring a truncated tree. The manifest's presence is therefore the +//! commit point of the whole export, exactly as the local fsync barrier is for +//! the on-disk artifact. +//! +//! ## R2 compatibility +//! +//! R2 is S3-v4-signature compatible behind a custom endpoint +//! (`https://.r2.cloudflarestorage.com`). We pass that endpoint via +//! `--s3-endpoint`, force path-style addressing (R2 does not do virtual-hosted +//! `.` buckets), and read credentials from the standard +//! `AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY` env vars. The region is a +//! placeholder (`auto`) R2 ignores but the `SigV4` signer requires. + +use std::path::{Path, PathBuf}; + +use aws_credential_types::Credentials; +use aws_sdk_s3::config::{BehaviorVersion, Region}; +use aws_sdk_s3::primitives::ByteStream; +use aws_sdk_s3::Client; + +use crate::CliError; + +/// The manifest filename — duplicated from [`crate::commands::backup`] (it is a +/// private const there) so the export can guarantee it is the LAST object put. +const BACKUP_MANIFEST: &str = "BACKUP_MANIFEST.json"; + +/// R2 ignores the `SigV4` region but the signer requires a non-empty one. +const R2_PLACEHOLDER_REGION: &str = "auto"; + +/// The S3/R2 target for an export or import: a bucket plus an optional key +/// prefix, against a custom endpoint. Built from the `--s3-endpoint/-bucket/ +/// -prefix` flags; `None` for a pure local-dir operation (the additive S3 layer +/// is skipped entirely, the synchronous local path is byte-identical to before). +#[derive(Debug, Clone)] +pub(crate) struct S3Target { + pub endpoint: String, + pub bucket: String, + /// Forward-slash key prefix, normalized to no leading/trailing slash. + pub prefix: String, +} + +impl S3Target { + /// Build a target from the three flags, requiring all-or-nothing: an S3 + /// export/import needs an endpoint AND a bucket (the prefix may be empty, a + /// bucket-root export). A partial set is a usage error, not a silent + /// local-only fallback — an operator who passed `--s3-bucket` but forgot + /// `--s3-endpoint` must hear about it, not get a local-only backup they think + /// went to R2. + pub(crate) fn from_flags( + endpoint: Option<&str>, + bucket: Option<&str>, + prefix: Option<&str>, + ) -> Result, CliError> { + match (endpoint, bucket) { + (None, None) => { + if prefix.is_some() { + return Err(CliError::new( + "--s3-prefix requires --s3-endpoint and --s3-bucket", + )); + } + Ok(None) + } + (Some(endpoint), Some(bucket)) => Ok(Some(Self { + endpoint: endpoint.to_string(), + bucket: bucket.to_string(), + prefix: normalize_prefix(prefix.unwrap_or("")), + })), + (Some(_), None) => Err(CliError::new("--s3-endpoint given without --s3-bucket")), + (None, Some(_)) => Err(CliError::new("--s3-bucket given without --s3-endpoint")), + } + } +} + +/// Strip leading/trailing slashes and collapse the empty-prefix case to `""`, so +/// the prefix joins cleanly with a relative path via [`object_key`]. +fn normalize_prefix(prefix: &str) -> String { + prefix.trim_matches('/').to_string() +} + +/// Map a backup-relative file path to its object key under the prefix. +/// +/// PURE function (the unit-tested core of the key mapping): joins the normalized +/// prefix with the relative path using a single `/`, always forward-slash +/// separated regardless of host path separator, and never emits a leading slash +/// or a doubled `//`. An empty prefix yields the bare relative path. The relative +/// path is itself already forward-slash form (it is the manifest's `path`, or a +/// path derived from one via the same `rel_to_slash` rendering the backup writes). +/// +/// This is the inverse of [`relative_from_key`]: `relative_from_key(prefix, +/// object_key(prefix, rel)) == rel` for any prefix and any non-empty rel — the +/// round-trip the export/import pair depends on. +pub(crate) fn object_key(prefix: &str, rel: &str) -> String { + let prefix = prefix.trim_matches('/'); + let rel = rel.trim_start_matches('/'); + if prefix.is_empty() { + rel.to_string() + } else { + format!("{prefix}/{rel}") + } +} + +/// Recover the backup-relative path from a listed object key, given the prefix +/// used to export it. The inverse of [`object_key`]. +/// +/// Returns `None` if `key` is not under `prefix` (a stray object the list +/// returned that was not part of this backup) — the caller skips it rather than +/// downloading something outside the artifact. +pub(crate) fn relative_from_key(prefix: &str, key: &str) -> Option { + let prefix = prefix.trim_matches('/'); + if prefix.is_empty() { + return Some(key.trim_start_matches('/').to_string()); + } + let with_slash = format!("{prefix}/"); + key.strip_prefix(&with_slash) + .filter(|rest| !rest.is_empty()) + .map(ToString::to_string) +} + +/// Build an S3 client for an R2-compatible endpoint with path-style addressing +/// and `SigV4` creds from the environment. Returns a `CliError` (mapped to exit 1) +/// rather than panicking if creds are absent, so an operator who forgot to export +/// `AWS_ACCESS_KEY_ID` gets a clean message, not a stack trace. +async fn build_client(target: &S3Target) -> Result { + let access_key = std::env::var("AWS_ACCESS_KEY_ID").map_err(|_| { + CliError::new( + "AWS_ACCESS_KEY_ID is not set (required for the S3/R2 export; set it plus \ + AWS_SECRET_ACCESS_KEY to your R2 token credentials)", + ) + })?; + let secret_key = std::env::var("AWS_SECRET_ACCESS_KEY") + .map_err(|_| CliError::new("AWS_SECRET_ACCESS_KEY is not set (required for the S3/R2 export)"))?; + + let creds = Credentials::from_keys(access_key, secret_key, None); + + let conf = aws_config::defaults(BehaviorVersion::latest()) + .region(Region::new(R2_PLACEHOLDER_REGION)) + .endpoint_url(&target.endpoint) + .credentials_provider(creds) + .load() + .await; + + let s3_conf = aws_sdk_s3::config::Builder::from(&conf) + // R2 buckets are addressed path-style (`//`), not + // virtual-hosted (`.`), so force it. + .force_path_style(true) + .build(); + Ok(Client::from_conf(s3_conf)) +} + +/// Run an async S3 closure to completion on a local current-thread runtime. +/// +/// tidalctl is otherwise fully synchronous; we build the runtime ONLY on the +/// S3 path (when `--s3-*` is given) so the local-dir backup/restore stays a plain +/// blocking call with zero runtime overhead. +fn block_on(fut: F) -> Result +where + F: std::future::Future>, +{ + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|e| CliError::new(format!("failed to build S3 transfer runtime: {e}")))?; + rt.block_on(fut) +} + +/// EXPORT: mirror the just-written local backup dir into the bucket under the +/// prefix, uploading `BACKUP_MANIFEST.json` LAST as the completion marker. +/// +/// Called by `run_backup` after the local artifact is fully written and fsynced. +/// `local_dir` is that local artifact dir; every regular file under it is +/// `put_object`ed, manifest last. +pub(crate) fn export_dir(local_dir: &Path, target: &S3Target) -> Result { + // Enumerate every regular file (relative paths), then partition the manifest + // out so it is uploaded strictly last. + let mut rel_files = Vec::new(); + collect_rel_files(local_dir, Path::new(""), &mut rel_files) + .map_err(|e| CliError::new(format!("walk local backup dir: {e}")))?; + + let manifest_present = rel_files.iter().any(|r| rel_to_slash(r) == BACKUP_MANIFEST); + if !manifest_present { + return Err(CliError::new(format!( + "local backup dir {} has no {BACKUP_MANIFEST}; refusing to export an artifact \ + with no atomicity marker", + local_dir.display() + ))); + } + + block_on(async move { + let client = build_client(target).await?; + + let mut uploaded = 0u64; + // Data files first; manifest deferred to the end. + for rel in &rel_files { + let rel_slash = rel_to_slash(rel); + if rel_slash == BACKUP_MANIFEST { + continue; + } + put_file(&client, target, local_dir, &rel_slash).await?; + uploaded += 1; + } + // The atomicity marker LAST: its presence signals a complete export. + put_file(&client, target, local_dir, BACKUP_MANIFEST).await?; + uploaded += 1; + Ok(uploaded) + }) +} + +/// IMPORT: download every object under the prefix into `staging_dir` (which must +/// be a fresh temp dir), reconstructing the backup tree, then return so the +/// caller hands `staging_dir` to the UNCHANGED verified restore. +pub(crate) fn import_to_dir(staging_dir: &Path, target: &S3Target) -> Result { + block_on(async move { + let client = build_client(target).await?; + + let keys = list_keys(&client, target).await?; + if keys.is_empty() { + return Err(CliError::new(format!( + "no objects under s3://{}/{} (is the prefix a tidalctl backup?)", + target.bucket, target.prefix + ))); + } + + let mut manifest_seen = false; + let mut downloaded = 0u64; + for key in &keys { + let Some(rel) = relative_from_key(&target.prefix, key) else { + continue; // stray object outside the prefix (defensive) + }; + if rel == BACKUP_MANIFEST { + manifest_seen = true; + } + get_to_file(&client, target, key, staging_dir, &rel).await?; + downloaded += 1; + } + + if !manifest_seen { + return Err(CliError::new(format!( + "prefix s3://{}/{} has no {BACKUP_MANIFEST}; the export is incomplete or this \ + is not a tidalctl backup (refusing to restore a partial artifact)", + target.bucket, target.prefix + ))); + } + Ok(downloaded) + }) +} + +/// PUT one file from the local dir to its object key. +async fn put_file( + client: &Client, + target: &S3Target, + local_dir: &Path, + rel_slash: &str, +) -> Result<(), CliError> { + let key = object_key(&target.prefix, rel_slash); + let abs = local_dir.join(rel_to_native(rel_slash)); + let body = ByteStream::from_path(&abs) + .await + .map_err(|e| CliError::new(format!("open {} for upload: {e}", abs.display())))?; + client + .put_object() + .bucket(&target.bucket) + .key(&key) + .body(body) + .send() + .await + .map_err(|e| CliError::new(format!("put_object {key}: {}", s3_err(&e))))?; + Ok(()) +} + +/// GET one object into the staging dir at its relative path. +async fn get_to_file( + client: &Client, + target: &S3Target, + key: &str, + staging_dir: &Path, + rel: &str, +) -> Result<(), CliError> { + let resp = client + .get_object() + .bucket(&target.bucket) + .key(key) + .send() + .await + .map_err(|e| CliError::new(format!("get_object {key}: {}", s3_err(&e))))?; + let bytes = resp + .body + .collect() + .await + .map_err(|e| CliError::new(format!("read object body {key}: {e}")))? + .into_bytes(); + + let dest = staging_dir.join(rel_to_native(rel)); + if let Some(parent) = dest.parent() { + std::fs::create_dir_all(parent) + .map_err(|e| CliError::new(format!("mkdir {}: {e}", parent.display())))?; + } + std::fs::write(&dest, &bytes) + .map_err(|e| CliError::new(format!("write staged {}: {e}", dest.display())))?; + Ok(()) +} + +/// List every object key under the prefix, following continuation tokens so a +/// backup with more than 1000 files (the page cap) lists fully. +async fn list_keys(client: &Client, target: &S3Target) -> Result, CliError> { + let list_prefix = if target.prefix.is_empty() { + String::new() + } else { + format!("{}/", target.prefix) + }; + + let mut keys = Vec::new(); + let mut continuation: Option = None; + loop { + let mut req = client.list_objects_v2().bucket(&target.bucket); + if !list_prefix.is_empty() { + req = req.prefix(&list_prefix); + } + if let Some(token) = &continuation { + req = req.continuation_token(token); + } + let resp = req + .send() + .await + .map_err(|e| CliError::new(format!("list_objects_v2: {}", s3_err(&e))))?; + + for obj in resp.contents() { + if let Some(key) = obj.key() { + keys.push(key.to_string()); + } + } + if resp.is_truncated().unwrap_or(false) { + continuation = resp.next_continuation_token().map(ToString::to_string); + if continuation.is_none() { + break; + } + } else { + break; + } + } + Ok(keys) +} + +/// Render an AWS SDK error to a flat operator-readable string (the SDK's nested +/// `Display` is verbose; `DisplayErrorContext` flattens the source chain). +fn s3_err(e: &aws_sdk_s3::error::SdkError) -> String { + format!("{}", aws_sdk_s3::error::DisplayErrorContext(e)) +} + +/// Recursively collect file paths relative to `root` (forward-slash form is +/// applied later by `rel_to_slash`). Mirrors `backup::collect_files` but keeps +/// the manifest IN (it is uploaded, last) — the only difference from the backup +/// walk, which skips it. +fn collect_rel_files(root: &Path, rel: &Path, out: &mut Vec) -> std::io::Result<()> { + let dir = root.join(rel); + for entry in std::fs::read_dir(&dir)? { + let entry = entry?; + let child_rel = rel.join(entry.file_name()); + let file_type = entry.file_type()?; + if file_type.is_dir() { + collect_rel_files(root, &child_rel, out)?; + } else if file_type.is_file() { + out.push(child_rel); + } + } + Ok(()) +} + +/// Render a relative path with forward slashes (portable object keys), matching +/// the backup manifest's `rel_to_slash`. +fn rel_to_slash(rel: &Path) -> String { + rel.components() + .map(|c| c.as_os_str().to_string_lossy().into_owned()) + .collect::>() + .join("/") +} + +/// Turn a forward-slash relative key path back into a host-native `PathBuf` +/// (the inverse of `rel_to_slash` for the local filesystem). +fn rel_to_native(rel_slash: &str) -> PathBuf { + rel_slash.split('/').collect() +} + +#[cfg(test)] +#[allow(clippy::unwrap_used)] +mod tests { + use super::*; + + /// The pure key mapping: prefix + relative path -> object key, with no leading + /// slash and no doubled separator, for the prefix shapes an operator can pass + /// (`backups/node-A`, a trailing-slash form, the bucket-root empty prefix). + #[test] + fn object_key_joins_prefix_and_relative_path() { + assert_eq!( + object_key("backups/node-A", "wal/seg-1.seg"), + "backups/node-A/wal/seg-1.seg" + ); + assert_eq!( + object_key("backups/node-A", "BACKUP_MANIFEST.json"), + "backups/node-A/BACKUP_MANIFEST.json" + ); + // An empty prefix is a bucket-root export: the key is the bare rel path. + assert_eq!(object_key("", "wal/seg-1.seg"), "wal/seg-1.seg"); + // Leading/trailing slashes on the prefix are normalized away (no `//`). + assert_eq!( + object_key("/backups/node-A/", "data.db"), + "backups/node-A/data.db" + ); + // A stray leading slash on the rel path never produces `prefix//rel`. + assert_eq!(object_key("p", "/data.db"), "p/data.db"); + } + + /// `relative_from_key` is the exact inverse of `object_key`: it recovers the + /// backup-relative path from a listed key, and rejects a key outside the + /// prefix. + #[test] + fn relative_from_key_inverts_object_key() { + assert_eq!( + relative_from_key("backups/node-A", "backups/node-A/wal/seg-1.seg") + .as_deref(), + Some("wal/seg-1.seg") + ); + // Empty prefix: the whole key is the relative path. + assert_eq!( + relative_from_key("", "BACKUP_MANIFEST.json").as_deref(), + Some("BACKUP_MANIFEST.json") + ); + // A key NOT under the prefix is rejected (stray object the list returned). + assert_eq!( + relative_from_key("backups/node-A", "backups/node-B/data.db"), + None + ); + // The prefix "directory marker" itself (key == prefix, no child) is None. + assert_eq!(relative_from_key("backups/node-A", "backups/node-A/"), None); + } + + /// The full round-trip the export/import pair depends on: for every + /// (prefix, rel) an export produces a key, and importing that key recovers the + /// SAME rel — across the prefix shapes and a nested path. + #[test] + fn object_key_round_trips_through_relative_from_key() { + let cases = [ + ("backups/node-A", "wal/seg-1.seg"), + ("backups/node-A", "BACKUP_MANIFEST.json"), + ("", "data.db"), + ("p", "a/b/c/deep.bin"), + ("dr/2026-06-17", "signals/keyspace/000123.fjall"), + ]; + for (prefix, rel) in cases { + let key = object_key(prefix, rel); + let recovered = relative_from_key(prefix, &key); + assert_eq!( + recovered.as_deref(), + Some(rel), + "round-trip failed for prefix={prefix:?} rel={rel:?} (key={key:?})" + ); + } + } + + /// `normalize_prefix` strips slashes so `from_flags` stores a clean prefix. + #[test] + fn normalize_prefix_strips_surrounding_slashes() { + assert_eq!(normalize_prefix("/backups/node-A/"), "backups/node-A"); + assert_eq!(normalize_prefix("backups/node-A"), "backups/node-A"); + assert_eq!(normalize_prefix(""), ""); + assert_eq!(normalize_prefix("/"), ""); + } + + /// `from_flags` is all-or-nothing on endpoint+bucket: a partial set is a usage + /// error (never a silent local-only fallback), and the no-flags case is `None` + /// (pure local backup). Assertions avoid `Result::unwrap` so the production + /// `CliError` need not derive `Debug`. + #[test] + fn from_flags_requires_endpoint_and_bucket_together() { + // No S3 flags: local-only, no target. + assert!(matches!(S3Target::from_flags(None, None, None), Ok(None))); + // Full set: a target with a normalized prefix. + let Ok(Some(t)) = S3Target::from_flags( + Some("https://acct.r2.cloudflarestorage.com"), + Some("dr"), + Some("/node-A/"), + ) else { + panic!("full flag set must build a target"); + }; + assert_eq!(t.bucket, "dr"); + assert_eq!(t.prefix, "node-A"); + // Empty prefix is allowed (bucket-root export). + let Ok(Some(t)) = S3Target::from_flags(Some("https://e"), Some("b"), None) else { + panic!("endpoint+bucket with no prefix must build a target"); + }; + assert_eq!(t.prefix, ""); + // Partial sets are usage errors. + assert!(S3Target::from_flags(Some("https://e"), None, None).is_err()); + assert!(S3Target::from_flags(None, Some("b"), None).is_err()); + assert!(S3Target::from_flags(None, None, Some("p")).is_err()); + } + + /// `rel_to_native` is the inverse of `rel_to_slash` on a multi-segment path, + /// so a key listed from S3 lands at the right place in the staging tree. + #[test] + fn rel_to_native_round_trips_rel_to_slash() { + let p = PathBuf::from("wal").join("seg-1.seg"); + let slash = rel_to_slash(&p); + assert_eq!(slash, "wal/seg-1.seg"); + assert_eq!(rel_to_native(&slash), p); + } +} diff --git a/tidalctl/src/main.rs b/tidalctl/src/main.rs index abc07f6..6b0805d 100644 --- a/tidalctl/src/main.rs +++ b/tidalctl/src/main.rs @@ -77,6 +77,13 @@ struct CliArgs { out: Option, /// Restore source (`restore --from `). from: Option, + /// Object-store (S3/R2) endpoint URL (`--s3-endpoint`), e.g. + /// `https://.r2.cloudflarestorage.com`. Additive on backup/restore. + s3_endpoint: Option, + /// Object-store bucket (`--s3-bucket`). + s3_bucket: Option, + /// Object-store key prefix (`--s3-prefix`); empty ⇒ bucket-root. + s3_prefix: Option, } enum Command { @@ -143,6 +150,9 @@ fn parse_args(args: &[String]) -> Result { let mut path: Option = None; let mut out: Option = None; let mut from: Option = None; + let mut s3_endpoint: Option = None; + let mut s3_bucket: Option = None; + let mut s3_prefix: Option = None; let mut pretty = false; let mut verify_only = false; let mut i = 2; @@ -170,6 +180,27 @@ fn parse_args(args: &[String]) -> Result { } from = Some(PathBuf::from(&args[i])); } + "--s3-endpoint" => { + i += 1; + if i >= args.len() { + return Err(CliError::new("--s3-endpoint requires a value")); + } + s3_endpoint = Some(args[i].clone()); + } + "--s3-bucket" => { + i += 1; + if i >= args.len() { + return Err(CliError::new("--s3-bucket requires a value")); + } + s3_bucket = Some(args[i].clone()); + } + "--s3-prefix" => { + i += 1; + if i >= args.len() { + return Err(CliError::new("--s3-prefix requires a value")); + } + s3_prefix = Some(args[i].clone()); + } "--pretty" => { pretty = true; } @@ -192,11 +223,15 @@ fn parse_args(args: &[String]) -> Result { verify_only, out, from, + s3_endpoint, + s3_bucket, + s3_prefix, }) } fn usage() -> String { - "Usage: tidalctl --path [--out ] [--from ] [--pretty]\n\n\ + "Usage: tidalctl --path [--out ] [--from ] [--pretty]\n \ + [--s3-endpoint --s3-bucket [--s3-prefix

]]\n\n\ Commands:\n \ status Report WAL state, checkpoint, and directory layout\n \ paths Report resolved directory paths and existence\n \ @@ -208,6 +243,15 @@ fn usage() -> String { Backup/restore operate on a data dir AT REST (a stopped/drained node):\n \ tidalctl backup --path /data/node --out /backups/node-A\n \ tidalctl restore --from /backups/node-A --path /data/node-new\n\n\ + Object-store (S3/R2) export/import is ADDITIVE — pass --s3-endpoint +\n \ + --s3-bucket on backup to ALSO mirror the artifact to the bucket (manifest\n \ + uploaded last as the completion marker), or on restore to download the prefix\n \ + into a temp dir and run the verified restore on it. Creds from env\n \ + AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY:\n \ + AWS_ACCESS_KEY_ID=... AWS_SECRET_ACCESS_KEY=... \\\n \ + tidalctl backup --path /data/node --out /backups/node-A \\\n \ + --s3-endpoint https://.r2.cloudflarestorage.com \\\n \ + --s3-bucket tidaldb-dr --s3-prefix node-A\n\n\ Exit codes: 0 = ok/empty, 1 = usage/internal error, 2 = degraded/unreadable\n \ (WAL, checkpoint, or a derived index exists but could not be read)." .to_string() @@ -231,14 +275,25 @@ fn run(args: &[String]) -> Result<(String, i32), CliError> { .out .as_ref() .ok_or_else(|| CliError::new("backup requires --out "))?; - commands::backup::run_backup(&cli.path, out, cli.pretty) + // The S3 target (if any) is built from the three flags; all-or-nothing + // on endpoint+bucket so a half-set is a usage error, not a silent + // local-only backup the operator thinks went to R2. + let s3 = commands::s3::S3Target::from_flags( + cli.s3_endpoint.as_deref(), + cli.s3_bucket.as_deref(), + cli.s3_prefix.as_deref(), + )?; + commands::backup::run_backup(&cli.path, out, s3.as_ref(), cli.pretty) } Command::Restore => { - let from = cli - .from - .as_ref() - .ok_or_else(|| CliError::new("restore requires --from "))?; - commands::backup::run_restore(&cli.path, from, cli.pretty) + // --from is required ONLY for a local restore; an S3 import stages the + // prefix into a temp dir and restores from that instead. + let s3 = commands::s3::S3Target::from_flags( + cli.s3_endpoint.as_deref(), + cli.s3_bucket.as_deref(), + cli.s3_prefix.as_deref(), + )?; + commands::backup::run_restore(&cli.path, cli.from.as_deref(), s3.as_ref(), cli.pretty) } } }