diff --git a/.github/workflows/postgres-metastore.yaml b/.github/workflows/postgres-metastore.yaml new file mode 100644 index 000000000..1a554aa81 --- /dev/null +++ b/.github/workflows/postgres-metastore.yaml @@ -0,0 +1,106 @@ +name: PostgreSQL metastore +on: + pull_request: + push: + branches: [main] + workflow_dispatch: +permissions: + contents: read +concurrency: + group: postgres-metastore-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} +jobs: + contract-and-smoke: + runs-on: ubuntu-24.04 + timeout-minutes: 90 + services: + postgres: + image: postgres:17-alpine + env: + POSTGRES_PASSWORD: contract + POSTGRES_DB: contract + ports: + - 55439:5432 + options: >- + --health-cmd "pg_isready -U postgres" + --health-interval 5s --health-timeout 5s --health-retries 12 + env: + P_TEST_REVIEW_S3_URL: http://127.0.0.1:59087 + P_TEST_POSTGRES_URL: postgresql://postgres:contract@127.0.0.1:55439/contract?sslmode=disable + CARGO_BUILD_JOBS: 2 + P_BUILD_NO_UI: 1 + steps: + - uses: actions/checkout@v4 + - name: Prepare Linux build resources + run: | + sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc /opt/hostedtoolcache/CodeQL + if sudo fallocate -l 16G /swap.build && sudo chmod 600 /swap.build \ + && sudo mkswap /swap.build && sudo swapon /swap.build; then + echo "Added 16G swap" + else + sudo rm -f /swap.build + echo "::warning title=Swap::Could not add swap; continuing without it" + fi + free -h + df -h / + - uses: ./.github/actions/setup-rust-ci + with: + cache-key: postgres-metastore + tools: nextest + - name: Start isolated MinIO + run: | + docker run -d --name migration-minio -p 127.0.0.1:59087:9000 -e MINIO_ROOT_USER=minioadmin -e MINIO_ROOT_PASSWORD=minioadmin quay.io/parseablehq/minio:edge server /data + docker exec migration-minio mkdir -p /data/review /data/migration-unit + curl --fail --retry 30 --retry-all-errors --retry-delay 1 http://127.0.0.1:59087/minio/health/ready + - name: Migration and regressions on PostgreSQL 17 and MinIO + run: 'cargo test --locked --lib metastore:: -- --nocapture --test-threads=1' + - name: Full library suite with PostgreSQL + run: cargo test --locked --lib -- --test-threads=1 + - name: Full library suite without PostgreSQL + run: env -u P_TEST_POSTGRES_URL -u P_TEST_REVIEW_S3_URL cargo test --locked --lib -- --nocapture --test-threads=1 + - name: LocalFS migration and legacy locators + run: env -u P_TEST_REVIEW_S3_URL cargo test --locked --lib migration::tests::migration_round_trip_postgres -- --nocapture + - name: Deadline against paused PostgreSQL + env: + P_TEST_POSTGRES_CONTAINER: ${{ job.services.postgres.id }} + run: cargo test --locked --lib postgres_health_deadline_regression -- --nocapture --test-threads=1 + - name: Shared contract and SQL fault tests (must execute) + shell: bash + run: | + set -o pipefail + cargo test --locked --lib contract_tests::object_store_contract -- --nocapture 2>&1 | tee postgres-contract.log + cargo test --locked --lib contract_tests::postgres_contract -- --ignored --nocapture 2>&1 | tee -a postgres-contract.log + cargo test --locked --lib contract_tests::postgres_startup_cleanup -- --ignored --nocapture 2>&1 | tee -a postgres-contract.log + python3 - <<'PY' + from pathlib import Path + text = Path('postgres-contract.log').read_text() + assert 'postgres_contract ... ok' in text, text + assert 'postgres_startup_cleanup ... ok' in text, text + assert '0 failed; 0 ignored;' in text, text + assert 'skipping PostgreSQL' not in text, text + PY + - name: Build smoke binary + run: cargo build --locked + - name: Object store, SQL plaintext and verified TLS end to end + run: python3 scripts/postgres-metastore-smoke.py + - name: Offline migration end to end + run: python3 scripts/postgres-metastore-migration-smoke.py + - name: Offline migration with verified PostgreSQL TLS + env: + P_MIGRATION_TLS: 1 + run: python3 scripts/postgres-metastore-migration-smoke.py + - name: Remove isolated MinIO + if: always() + run: docker rm -f migration-minio + - uses: actions/upload-artifact@v4 + if: always() + with: + name: postgres-metastore-evidence + path: | + postgres-contract.log + .postgres-smoke/results.json + .postgres-smoke/runs/*.json + docs/validation/postgres-metastore-smoke.json + docs/validation/postgres-migration*.json + .postgres-smoke/*.log + if-no-files-found: error diff --git a/.gitignore b/.gitignore index 034f0d47e..9b84ba6e7 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,6 @@ cache # Local credentials and review notes *.pem pr-*.md + +# Isolated PostgreSQL smoke artifacts +.postgres-smoke/ diff --git a/Cargo.lock b/Cargo.lock index 002c99c52..677747710 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1592,7 +1592,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6139a8597ed92cf816dfb33f5dd6cf0bb93a6adc938f11039f371bc5bcd26c3" dependencies = [ "chrono", - "phf", + "phf 0.12.1", ] [[package]] @@ -1725,6 +1725,12 @@ dependencies = [ "serde_core", ] +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + [[package]] name = "const-oid" version = "0.10.2" @@ -2760,6 +2766,62 @@ dependencies = [ "stacker", ] +[[package]] +name = "deadpool" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e98a7e119cd347f4201e1159b19831029e203e2d8b790547708e8157b4acf1e" +dependencies = [ + "deadpool-runtime", + "tokio", +] + +[[package]] +name = "deadpool-postgres" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65a536565624b97fc19f758cd01b15d12908d3344425066efc8162236fbd3749" +dependencies = [ + "async-trait", + "deadpool", + "getrandom 0.4.2", + "tokio", + "tokio-postgres", + "tracing", +] + +[[package]] +name = "deadpool-runtime" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2657f61fb1dd8bf37a8d51093cc7cee4e77125b22f7753f49b289f831bec2bae" +dependencies = [ + "tokio", +] + +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid 0.9.6", + "der_derive", + "flagset", + "zeroize", +] + +[[package]] +name = "der_derive" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8034092389675178f570469e6c3b0465d3d30b4505c294a6550db47f3c17ad18" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "deranged" version = "0.5.5" @@ -2876,7 +2938,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4850db49bf08e663084f7fb5c87d202ef91a3907271aff24a94eb97ff039153c" dependencies = [ "block-buffer 0.12.0", - "const-oid", + "const-oid 0.10.2", "crypto-common 0.2.1", "ctutils", ] @@ -2986,6 +3048,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "fallible-iterator" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" + [[package]] name = "fastrand" version = "1.9.0" @@ -3023,6 +3091,12 @@ version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" +[[package]] +name = "flagset" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7ac824320a75a52197e8f2d787f6a38b6718bb6897a35142d749af3c0e8f4fe" + [[package]] name = "flatbuffers" version = "25.12.19" @@ -3875,11 +3949,12 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.83" +version = "0.3.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "464a3709c7f55f1f721e5389aa6ea4e3bc6aba669353300af094b29ffbdde1d8" +checksum = "8dc09d2154098bbf4ef54eab6b117a3c24cbbffe94046b16ee6d6d4510584d3a" dependencies = [ - "once_cell", + "cfg-if", + "futures-util", "wasm-bindgen", ] @@ -4394,6 +4469,15 @@ dependencies = [ "objc2-core-foundation", ] +[[package]] +name = "objc2-system-configuration" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7216bd11cbda54ccabcab84d523dc93b858ec75ecfb3a7d89513fa22464da396" +dependencies = [ + "objc2-core-foundation", +] + [[package]] name = "object" version = "0.32.2" @@ -4738,6 +4822,7 @@ dependencies = [ "dashmap", "datafusion", "datafusion-proto", + "deadpool-postgres", "derive_more 1.0.0", "erased-serde", "fs_extra", @@ -4798,6 +4883,8 @@ dependencies = [ "tikv-jemalloc-ctl", "tikv-jemallocator", "tokio", + "tokio-postgres", + "tokio-postgres-rustls", "tokio-stream", "tokio-util", "tonic", @@ -4813,6 +4900,7 @@ dependencies = [ "url", "uuid", "vergen-gitcl", + "webpki-roots", "xxhash-rust", "zstd", ] @@ -4885,7 +4973,17 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "913273894cec178f401a31ec4b656318d95473527be05c0752cc41cdc32be8b7" dependencies = [ - "phf_shared", + "phf_shared 0.12.1", +] + +[[package]] +name = "phf" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" +dependencies = [ + "phf_shared 0.13.1", + "serde", ] [[package]] @@ -4897,6 +4995,15 @@ dependencies = [ "siphasher", ] +[[package]] +name = "phf_shared" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project" version = "1.1.10" @@ -4941,6 +5048,35 @@ version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" +[[package]] +name = "postgres-protocol" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08808e3c483c46e999108051c78334f473d5adb59d78bb80a1268c7e6aa6c514" +dependencies = [ + "base64 0.22.1", + "byteorder", + "bytes", + "fallible-iterator", + "hmac", + "md-5 0.11.0", + "memchr", + "rand 0.10.1", + "sha2 0.11.0", + "stringprep", +] + +[[package]] +name = "postgres-types" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "851ca9db4932932d69f3ea811b1abe63087a0f740a47692619dd40d4899b68be" +dependencies = [ + "bytes", + "fallible-iterator", + "postgres-protocol", +] + [[package]] name = "potential_utf" version = "0.1.4" @@ -6151,6 +6287,16 @@ dependencies = [ "vob 3.0.6", ] +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + [[package]] name = "sqlparser" version = "0.62.0" @@ -6220,6 +6366,17 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + [[package]] name = "strsim" version = "0.11.1" @@ -6491,6 +6648,27 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" +[[package]] +name = "tls_codec" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de2e01245e2bb89d6f05801c564fa27624dbd7b1846859876c7dad82e90bf6b" +dependencies = [ + "tls_codec_derive", + "zeroize", +] + +[[package]] +name = "tls_codec_derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d2e76690929402faae40aebdda620a2c0e25dd6d3b9afe48867dfd95991f4bd" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "tokio" version = "1.52.3" @@ -6519,6 +6697,46 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "tokio-postgres" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a528f7d280f6d5b9cd149635c8705b0dd049754bc67d81d31fa25169a93809d3" +dependencies = [ + "async-trait", + "byteorder", + "bytes", + "fallible-iterator", + "futures-channel", + "futures-util", + "log", + "parking_lot", + "percent-encoding", + "phf 0.13.1", + "pin-project-lite", + "postgres-protocol", + "postgres-types", + "rand 0.10.1", + "socket2 0.6.4", + "tokio", + "tokio-util", + "whoami", +] + +[[package]] +name = "tokio-postgres-rustls" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c2ad44aa0ae96db89c4742212ed41645b2f597311ff6e1945542a4d9fadc2fb" +dependencies = [ + "rustls", + "sha2 0.11.0", + "tokio", + "tokio-postgres", + "tokio-rustls", + "x509-cert", +] + [[package]] name = "tokio-rustls" version = "0.26.4" @@ -6844,12 +7062,33 @@ version = "2.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539" +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + [[package]] name = "unicode-ident" version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" +[[package]] +name = "unicode-normalization" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75cd3fb03d5288b3ba9a1ee103ac7e9cbb59f0b03d64f8ddfcc04dd42758a85d" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "993ad287eb20e8ba755312801cfe1413e0f25d604075863acaddca10b04c1833" + [[package]] name = "unicode-segmentation" version = "1.12.0" @@ -7096,6 +7335,15 @@ version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" +[[package]] +name = "wasi" +version = "0.14.7+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "883478de20367e224c0090af9cf5f9fa85bed63a95c1abf3afc5c083ebc06e8c" +dependencies = [ + "wasip2", +] + [[package]] name = "wasip2" version = "1.0.1+wasi-0.2.4" @@ -7114,14 +7362,22 @@ dependencies = [ "wit-bindgen 0.51.0", ] +[[package]] +name = "wasite" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fe902b4a6b8028a753d5424909b764ccf79b7a209eac9bf97e59cda9f71a42" +dependencies = [ + "wasi 0.14.7+wasi-0.2.4", +] + [[package]] name = "wasm-bindgen" -version = "0.2.106" +version = "0.2.130" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d759f433fa64a2d763d1340820e46e111a7a5ab75f993d1852d70b03dbb80fd" +checksum = "a0181c14ddd6c7b8b0b0132a5f1ea2f32be0d893cd3ea97b6e3b28886b8a3d1c" dependencies = [ "cfg-if", - "once_cell", "rustversion", "wasm-bindgen-macro", "wasm-bindgen-shared", @@ -7129,22 +7385,20 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.56" +version = "0.4.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "836d9622d604feee9e5de25ac10e3ea5f2d65b41eac0d9ce72eb5deae707ce7c" +checksum = "5e14e0327cb5368ad60ccb22d04a9827a693c95a926d8d125e9fa39e0baec501" dependencies = [ - "cfg-if", "js-sys", - "once_cell", + "tokio", "wasm-bindgen", - "web-sys", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.106" +version = "0.2.130" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48cb0d2638f8baedbc542ed444afc0644a29166f1595371af4fecf8ce1e7eeb3" +checksum = "6e931a02a39abdbf66966fc35902b5ed58344750a70885bed4ebb3d8eaf28ed9" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -7152,22 +7406,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.106" +version = "0.2.130" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cefb59d5cd5f92d9dcf80e4683949f15ca4b511f4ac0a6e14d4e1ac60c6ecd40" +checksum = "2ca4b54280e455bd831e33a6b8e734baf432fe63fbc35c42ac14ce67a043a53a" dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.117", + "syn 3.0.4", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.106" +version = "0.2.130" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbc538057e648b67f72a982e708d485b2efa771e1ac05fec311f9f63e5800db4" +checksum = "3b1f9c3798725016c6ee5f2f57869ec0831e8bbc0ab0310a86e787e97f41389e" dependencies = [ "unicode-ident", ] @@ -7221,9 +7475,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.83" +version = "0.3.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b32828d774c412041098d182a8b38b16ea816958e07cf40eec2bc080ae137ac" +checksum = "d7d179596ff5ec0f6de4db64d54011bde6289cc49603d3a6e4ce3f329207ab88" dependencies = [ "js-sys", "wasm-bindgen", @@ -7248,6 +7502,19 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "whoami" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "626c4bac6755d76ffc12cb01b2eac751db1996b9e0041de9aa02c8c211ddc82c" +dependencies = [ + "libc", + "libredox", + "objc2-system-configuration", + "wasite", + "web-sys", +] + [[package]] name = "winapi" version = "0.3.9" @@ -7754,6 +8021,18 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" +[[package]] +name = "x509-cert" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1301e935010a701ae5f8655edc0ad17c44bad3ac5ce8c39185f75453b720ae94" +dependencies = [ + "const-oid 0.9.6", + "der", + "spki", + "tls_codec", +] + [[package]] name = "xmlparser" version = "0.13.6" @@ -7835,6 +8114,20 @@ name = "zeroize" version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] [[package]] name = "zerotrie" diff --git a/Cargo.toml b/Cargo.toml index 87d216f30..b62d95ff0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,10 @@ categories = ["logs", "observability", "metrics", "traces"] build = "build.rs" [dependencies] +tokio-postgres = "0.7" +deadpool-postgres = "0.14" +tokio-postgres-rustls = "0.14" +webpki-roots = "1" # Arrow and DataFusion ecosystem arrow = "59.2.0" arrow-array = "59.2.0" diff --git a/docs/metastore-boundary-audit.md b/docs/metastore-boundary-audit.md index 087ae05e2..1a0a3c850 100644 --- a/docs/metastore-boundary-audit.md +++ b/docs/metastore-boundary-audit.md @@ -144,3 +144,17 @@ Final validation on this branch: **1,379 passed, 0 failed, 4 ignored** (1,383 discovered), 79.65 seconds. - Both source-guard tests and both LocalFS compatibility fixtures passed. - `cargo fmt` and `git diff --check` passed. Cargo target remained in this worktree. + + +## Combined PostgreSQL lifecycle implementation + +The original inventory above records the routing branch. In the combined branch, +its seven missing methods are implemented by PostgreSQL as documented in +[postgres-metastore.md](postgres-metastore.md#lifecycle-and-metadata-boundary). +Backend primitives moved to `document_metastore.rs` and `postgres_documents.rs` +are allowed by the source guard. The real MinIO lifecycle smoke supplements +that source policy with bucket key assertions. All A operations now select the +configured backend. B/C/D entries remain data plane, local recovery state or +unused compatibility primitives. No additional active shared metadata bypass +was found. PostgreSQL stream listing suppresses durable deletion markers, and +HTTP DELETE permits explicit retry of such a stream after restart. diff --git a/docs/postgres-metastore.md b/docs/postgres-metastore.md new file mode 100644 index 000000000..7b0837b36 --- /dev/null +++ b/docs/postgres-metastore.md @@ -0,0 +1,768 @@ +# PostgreSQL document metastore + +Opt in using `--metastore-postgres-url` / `P_METASTORE_POSTGRES_URL` on any +storage subcommand. No setting is required for the default object-store backend. +For an isolated local PostgreSQL, for example: + +``` +P_METASTORE_POSTGRES_URL='postgresql://user:password@127.0.0.1:55432/parseable?sslmode=disable' \ +P_BUILD_NO_UI=1 target/debug/parseable s3-store +``` + +Keep the URL in a secret. Options Debug output redacts it. All participants +(standalone, ingest, query and prism) must select the same backend/database. +Neither startup nor runtime falls back to bucket metadata after selecting SQL. +Builds use runtime queries and need no `DATABASE_URL` or live database. + +| Option suffix (also `P_METASTORE_POSTGRES_`) | Default | +| --- | --- | +| URL | unset; object-store metadata | +| POOL_SIZE | 16 connections per process | +| CONNECT_TIMEOUT_MS | 5000; also bounds pool wait/create/recycle | +| STATEMENT_TIMEOUT_MS | 10000; also bounds SQL lock waits | +| ID | unset; optionally pin the backend UUID | +| CA | unset; optionally trust only a PEM CA bundle | + +CLI spellings use `--metastore-postgres-` plus lowercase hyphenated suffix. +TLS defaults to required with certificate and hostname verification against +webpki roots. URL `sslmode=disable` explicitly opts out. `prefer` and `allow` are +upgraded to required, so TLS cannot silently downgrade. Client certificates are +not implemented in this slice. + +## Pairing and migrations + +The first initialization creates a UUID and records the configured storage +endpoint/bucket identity. After committing SQL and releasing its connection, it +conditionally creates `.parseable/postgres-metastore-id` in that bucket. Existing +markers must contain the database UUID. Concurrent starts use a transactional +PostgreSQL advisory lock and atomic bucket creation. Partial initialization is +retryable, with no metadata service admitted until both checks succeed. +A preexisting marker paired to another database fails startup. A database bound +to another bucket/endpoint also fails. `ID` can pin an expected existing UUID. +Endpoint aliases are intentionally conservative: changing an endpoint spelling +requires an explicit operator pairing migration, even for the same bucket. +Never delete the marker to bypass a mismatch. Back up the database and marker +together; restoring a fresh empty database is not a metadata recovery procedure. +The marker is identity protection, not fencing against old binaries. + +`parseable_backend` records singleton UUID, bucket identity, schema and protocol +version. Embedded migrations `001_documents.sql` and `002_review_fixes.sql` are forward-only, transactional, +and serialized across starting nodes. Unknown versions fail closed. +`parseable_documents` stores `(tenant, key)` primary keys, an advisory `kind`, +faithful BYTEA bodies, BIGINT version, and creation/update timestamps. Keys retain +object-store path spelling including tenant prefixes. The default tenant uses +an empty tenant column, exactly as root bucket paths do. Versions come from a +backend-wide sequence, avoiding delete/recreate ABA; gaps are expected. Atomic +INSERT/UPDATE/DELETE preconditions implement dashboard versions. + +The documents table and broad `document` kind are interim. Manifests remain +compressed documents, with existing whole-document read/modify/write behavior. +This is not atomic catalog publication, retention, compaction, reader pinning or +multi-document transactions. Future normalized catalog tables can coexist with +the document keyspace without changing existing document versions. + +## Compatibility and data plane + +The implementations share the existing metastore path/encoding/behavior code. +SQL primitive operations preserve raw bytes and sort keys with C collation. +Not-found GETs use the existing NoSuchKey error; optional getters retain their +existing conversions. Default filesystem behavior is retained for object-store +metastore. SQL listing always uses its document inventory, including local-store +standalone. Local filesystem multitenancy is still unsupported product policy. + +Existing ambiguous behavior is retained: default alert-state listing uses the +literal `DEFAULT_TENANT` directory although state writes use root paths; global +outbound-policy maps use the empty-string default key unlike other maps; manifest +inventory completion order is unspecified. SQL directory enumeration returns +immediate child names; S3's current `list_dirs_relative` flattens ancestor path +components as well, an apparent implementation accident that does not define +useful directory semantics. SQL cannot see externally written bucket metadata. + +Parquet, local staging/WAL, hot-tier data and object-store discovery remain in +the data plane. Lifecycle routing is included in this branch. Existing buckets need a separate +import before opting in. Do not switch a populated deployment in place. + +An import tool needs the original tenant and exact path key, original bytes +(including compressed manifest bytes), a validated kind, and object-store mode +and owner naming context. It should allocate fresh native versions, validate +schemas/JSON and cross-document manifest references, quiesce old writers, and +pair/verify the bucket UUID before admission. Upstream ETags cannot be reused as +SQL versions. Offline import/export is described below; normalized file catalog conversion is +separate work. + +## Validation + +Use `PATH=$HOME/.cargo/bin:$PATH P_BUILD_NO_UI=1 CARGO_BUILD_JOBS=4` and keep +`target` in the checkout. Run one Cargo compilation at a time. + +``` +cargo test --lib contract_tests -- --nocapture +P_TEST_POSTGRES_URL='postgresql://postgres:password@127.0.0.1:55432/test?sslmode=disable' \ + cargo test --lib contract_tests -- --nocapture +cargo build +python3 scripts/postgres-metastore-smoke.py +cargo clippy --all-targets +``` + +Use an empty disposable database for the SQL contract. With no test URL the SQL +test reports a skip and succeeds. The smoke script creates PostgreSQL 17 Alpine (including a verified TLS run) +and MinIO containers on randomly selected spare ports, isolated buckets and +staging, and removes its containers in `finally`. It runs stream creation, +ingestion, upload, retention configuration/startup execution, dashboard creation, +restart, query/ingest cold loading, repeated queries, deletion/recreation and +dependency outages in both modes. A separate SQL database exercises a self-signed +CA through `P_METASTORE_POSTGRES_CA`; an unrelated CA must reject startup. `.postgres-smoke/results.json` records key inventory, SQL rows and +storage metric deltas (background storage activity can contribute to deltas). + +## Lifecycle and metadata boundary + +All active mutable shared metadata operations now enter `Metastore`, including +health probes, cold stream discovery, stream/partition deletion and tenant +startup discovery. The companion audit is in [metastore-boundary-audit.md](metastore-boundary-audit.md). +The shared document layer retains encodings and existing CRUD behavior; the +seven lifecycle methods are supplied by each backend. + +* `stream_exists`: SQL inventory of canonical/ingestor stream JSON under the + tenant's stream prefix; no bucket metadata lookup. +* `is_stream_deleting`: durable SQL `.stream/.deleting` document, visible across + processes and restarts, checked before lazy loading. +* `delete_stream`: commit the SQL marker, delete the bucket data prefix, then + delete all stream rows (including the marker) in one SQL statement. Before + data deletion a crash leaves hidden metadata and recoverable data; afterwards + it leaves hidden metadata with missing data. Retrying completes either case. + Once complete, recreation starts without old schemas/manifests/settings. +* `delete_stream_partition`: the caller first publishes snapshot removal, then + the backend deletes partition data, then SQL rows under that date prefix. + A crash can leave unreferenced data or unreferenced manifest rows; retry + removes them. No snapshot ever gets republished here. +* `list_tenant_candidates`: union of nonempty SQL tenant columns and nonhidden + root bucket prefixes. SQL-only tenants are retained when their deployment + metadata validates; data-only/invalid candidates follow existing startup + cleanup, deleting both data and rows. With multitenancy disabled these are + physical candidates only; cloud object stores remain the supported product + configuration for multitenancy. +* `check_health`: checks SQL `SELECT 1` and reads the bucket pairing marker on + every probe. Readiness returns 503 if either fails. Liveness retains its + process/shutdown semantics. Pool waits/connects and SQL statements are bounded + by the configured timeouts; there is no metadata fallback. +* `read_latency`: elapsed combined SQL and bucket probe, labeled as such in the + banner; probe errors are logged. It is not a standalone SQL latency measure. + +The bucket's only metadata object in PostgreSQL mode is the immutable pairing +id `.parseable/postgres-metastore-id`. Its read is an intentional health probe. +LocalFS conditional marker creation additionally uses a private filesystem lock +file; this is local coordination, not a second metadata authority. +Parquet reads/writes, date/hour/minute discovery for bounds and retention, +legacy file listing, hot-tier planning and physical occupancy remain bucket +operations. Dormant tombstone/legacy provider helpers remain unused by +application code. Staging/WAL and hot-tier local recovery files remain local. +No other active metadata bypass was found in the combined inventories. + +Deletion markers reject new stream metadata writes in PostgreSQL mode; deletion does not fully fence concurrent writers. Existing whole-document read/modify/ +write races, retention/publication races and check-then-load races remain. +Automatic resumption of interrupted deletion is not implemented: retry the +operation. Semantic catalog APIs, atomic publication, reader pins, compaction, +conditional writes beyond dashboards remain future work. + +## Driver cost + +The driver is `tokio-postgres` with `deadpool-postgres` and +`tokio-postgres-rustls`. The existing repository already uses Tokio and rustls +0.23; this avoids a second TLS stack and compile-time database introspection. +See the [client documentation](https://docs.rs/tokio-postgres/latest/tokio_postgres/) +and [pool documentation](https://docs.rs/deadpool-postgres/latest/deadpool_postgres/). +The lockfile adds 34 package/version entries, including target-specific entries +and resolution changes. The six core PostgreSQL/pool debug rlibs total about +16 MB. The first full debug build on this checkout took 4m17s after `cargo check` +had populated metadata artifacts; that is not an isolated driver compile-time +measurement. The incremental footprint does not justify a backend feature gate +relative to the existing Arrow/DataFusion build. + +The shared contract calls every existing trait method against filesystem-backed +object metadata and real PostgreSQL. It covers both tenant namespaces, missing +objects, global getters, schema union/refresh, manifest encoding/listing, +Ingest-only listing and invalid mode selection, filter reconciliation, alert +state/history, node registration, and conditional dashboard create/update/delete. +It also tests concurrent initialization, concurrent create/update winners, +delete/recreate versions, bounded pool exhaustion and statement timeouts. +Filesystem delimiter listing is unsupported and its stream listing rejects +unrelated root metadata directories; these preexisting differences are explicit +expectations. The SQL path uses bucket-style listing semantics. Empty-string and absent +default tenant schema lookups are equivalent. The smoke covers query/ingest cold loading; broader prism and multi-tenant cluster validation remains future work. + +## Recorded combined validation + +2026-11-11 UTC, PostgreSQL 17 Alpine and MinIO on isolated ephemeral host ports. +No live service or bucket was used. All temporary containers were removed. +Debug build passed. Real PostgreSQL and LocalFS contracts: 2 passed. Full library +suite without a database URL: 1,381 passed, 4 ignored. Integration: 1 passed. +Plain `cargo clippy --all-targets` passed with 21 existing library-test warnings +and no new warnings. Source guard and compatibility fixtures passed unchanged +in meaning. SQL contracts also check physical data before/after injected +deletion failures, retry, tenant isolation, partition deletion, hidden deletion +markers, recreation, and absence of deleted stream rows. + +The smoke passed in object-store, PostgreSQL plaintext and verified TLS modes: +create, ingest/upload, retention settings, dashboard, restart as query node +(running startup retention), ingestor publication of a new stream and cold query, +stream deletion/recreation, upload, and query of the new value (7 rather than 42). +Parquet visibility precedes manifest/snapshot publication; the harness permits +that existing pipeline to settle before cold queries. Normal requests have a +30-second client limit; cold/recreated queries allow 90 seconds. Database outage +requests retain the 30-second limit. + +Both SQL configurations observed readiness 503 and a failed metadata request +when PostgreSQL stopped (combined probes/requests completed in under 7 ms on +this local run), then successful readiness and queries after recovery. All +three configurations observed readiness 503 when MinIO stopped. TLS used a +self-signed CA through `P_METASTORE_POSTGRES_CA`; an unrelated CA rejected startup. +Future schema versions, wrong pinned UUIDs and unreachable databases also +rejected startup. + +Five repeated queries measured the query process's counters after cold loading, +with an ingestor also running: + +| Backend | Metadata GETs/query | LIST + LIST_DELIM/query | Parquet GET_RANGES/query | Total/query | +| --- | ---: | ---: | ---: | ---: | +| Object store | 6 | 2 | 1 | 9 | +| PostgreSQL plaintext | 0 | 0 | 1 | 1 | +| PostgreSQL verified TLS | 0 | 0 | 1 | 1 | + +These are node-local request counts, not latency or deployment-wide totals; +background activity can contribute. The earlier standalone measurement was 4 +versus 1 and used a different topology. Both SQL buckets contained only the +pairing marker and Parquet, including after recreation. Before recreation, the +deleted stream had zero SQL rows. Dashboard, node registry, schema, stream and +manifest documents remained exclusively SQL. Exact inventories, counter deltas, +outage outcomes and test summary are in [the evidence](validation/postgres-metastore-smoke.json). + +Remaining risks are unconditional document update races, publication/retention +races and lack of fencing against concurrent writers or resident readers during +deletion. Interrupted deletion requires explicit retry. Automatic recovery/GC, +reader pins, multi-tenant cluster/prism smoke and client-certificate +TLS remain future work. + +## Offline migration and rollback + +Use this build on **every** node before attempting a transfer. The offline command +shares each storage subcommand's provider configuration and PostgreSQL TLS/pool +options. It does not start HTTP, ingest, retention or background sync services: + +``` +parseable s3-store --metadata-migration verify --acknowledge-server-stopped +parseable s3-store --metadata-migration import --acknowledge-server-stopped --dry-run +parseable s3-store --metadata-migration import --acknowledge-server-stopped +parseable s3-store --metadata-migration export --acknowledge-server-stopped +``` + +`local-store`, `blob-store` and `gcs-store` accept the same flags. LocalFS is for +single-tenant use only. Configure `P_METASTORE_POSTGRES_URL` even for verify and +export, together with exactly the original storage endpoint/bucket. Keep the URL +out of shell history. An empty disposable database is required for a first import. +A successful import cannot be applied again to a populated database. An interrupted +import with its matching migration marker can resume, checking every body again. + +Single-node procedure: + +1. Back up the entire bucket (including hidden keys), the server configuration, + staging directory and any existing PostgreSQL database. Preserve object versions + if available. Record the deployed binary version. Test restoration separately. +2. Let uploads finish, then stop the server and disable automatic restarts. Stop + scripts or other writers too. Supply the original `P_STAGING_DIR`; the command + rejects `.arrow`, `.arrows`, `.part` and `.parquet` staging buffers. Resolve unsynced + buffers by restarting the old mode and allowing upload, then stopping again. +3. Set the destination PostgreSQL URL, leaving all storage settings unchanged. + Run verify; differences and a nonzero exit are expected before first import. + Save its per-key SHA-256 output. Run import with `--dry-run`, inspect + the counts per tenant/kind, bytes and recent-write warnings. This creates no + SQL tables, identity or bucket objects. +4. Run import without dry-run. It validates stream JSON, schemas, compressed and + plain manifests, and every snapshot manifest reference before publishing SQL. + It rereads SQL bodies and compares SHA-256 digests, checks the bucket inventory + for changes, commits the complete SQL inventory, then establishes pairing. +5. Run verify again: require zero differences. Start the server with the same + PostgreSQL URL. Confirm readiness, stream list, schemas, stats, first/latest + event bounds, RBAC, alerts, dashboards and filters. Query known time ranges and + compare row counts/values against saved results before admitting ingest. +6. Keep the original bucket metadata and backups until the rollback window ends. + PostgreSQL writes do not update those original objects. + +Rollback/downgrade: + +1. Back up PostgreSQL, the paired bucket, staging and configuration again. Drain + uploads and stop all writers and automatic restarts as above. +2. Run verify and retain the differences. SQL changes since import normally mean differing bucket objects. Export compares the current bucket against per-key import digests; ordinary SQL changes and deletions require no force flag. +3. Review the differences; export normally to replace imported bodies and remove imported bucket-only metadata. If a bucket writer changed metadata after import, export refuses; after comparing backups explicitly use `--overwrite-bucket-changes-since-import` to accept overwriting that writer’s changes. Parquet is unaffected. Export rereads + each written body, validates all stream references, then removes pairing. +4. Run verify and require zero differences. Remove the PostgreSQL URL and start + object-store mode. Repeat readiness, metadata and known-query checks, including + rows ingested while PostgreSQL was active. Keep the SQL backup. The exported + database is marked exported and refuses normal SQL startup; reimport into a + fresh empty database to switch back later. + +The shared document layer inventories exact stored keys rather than rebuilding +paths. All hidden document namespaces (deployment/RBAC, users, dashboard bodies, +locators/titles, filters, alerts/state/history, correlations, targets/settings, +node registry, stream JSON and schemas) and date-partition manifests, including +per-ingestor variants and tenant prefixes, are copied without reserialization, except explicit default-tenant snapshot locators that require canonicalization (described below). +SQL versions are freshly allocated. Unfinished deletion markers are rejected. +Parquet, staging/WAL, hot-tier/cache files and private LocalFS lock files are not +metadata documents and are excluded, as are the pairing and transfer controls. +Inventories retain keys/timestamps only; bodies are read sequentially and SQL +export pages contain at most 32 bodies. Progress/digests go to stdout/stderr; +protect these reports because keys may contain user/tenant identifiers. + +Crash recovery: + +* Before the durable `.parseable/metastore-migration` marker, failures leave the + source authoritative and no committed import rows. SQL bootstrap may already + have committed an empty identity; bucket mode remains available. +* During import, an open SQL transaction rolls back on process death. The marker + blocks both server modes. Rerun the same import/configuration to restart. +* After import commit but before pairing/cleanup, all SQL rows already exist and + were verified. Rerunning validates them against the bucket, completes pairing + and removes the transfer marker. A changed source fails closed. +* During export, the pairing and transfer marker block object-store startup; + the SQL transfer state also blocks SQL startup, even after pairing removal. + Rerun export with the same overwrite flag to finish writing and verification. +* After export pairing removal but before marker cleanup, rerun export. Never + manually remove a transfer marker to force startup. + +These controls are **not fencing**. The stopped-server acknowledgement is mandatory +also for verify/dry-run. Recent metadata writes (including registry updates) cause +loud warnings; staging buffers cause refusal. Neither check proves that a remote +writer or resident process is stopped, and a quiet writer can wake at any moment. +The staging check only examines the path supplied to the command. Disable writers +operationally and do not run two migration tools simultaneously. This build refuses +object-store startup on a paired bucket. `P_ALLOW_PAIRED_BUCKET=` explicitly +bypasses pairing for disaster recovery and can expose stale metadata; it never +bypasses an incomplete transfer. Older binaries cannot recognize either marker; +do not start them until a verified export completes. + +### Migration validation evidence + +The migration tests passed with PostgreSQL 17 Alpine on an ephemeral host port: +1,384 library tests passed, 4 ignored. The fixture uses a separate disposable +SQL database, checks both default and named tenant document scopes, plain and +compressed manifests, snapshot references, missing/invalid stream documents, +nonempty refusal, committed-import resumption, overwrite refusal, exported SQL +startup refusal and exact byte equality after the round trip. With no +`P_TEST_POSTGRES_URL`, the SQL fixture explicitly skips. Debug build and plain +`cargo clippy --all-targets` passed; Clippy retains the 21 preexisting library-test +warnings and the preexisting staging integration-test warning. + +Run `python3 scripts/postgres-metastore-migration-smoke.py` after building. This +sibling of the existing smoke creates its own PostgreSQL 17 and MinIO containers, +three datasets (including custom partitions), uploaded manifests, a user/role, +dashboard, saved filter and alert. It kills import and export after the durable +transfer marker becomes visible, reruns each, checks both server modes reject +an incomplete import, checks nonempty and paired-bucket refusal, compares SQL +queries and metadata, ingests a new SQL-mode row, exports, and checks rollback +queries and metadata. Response ordering and the generated stats response timestamp +are excluded from comparisons; persisted stats and event bounds are compared. +The MinIO scenario passed, including the final metadata comparisons after rollback. +All containers are removed in `finally`; the script uses only spare host ports. + +This does not establish production-scale throughput or multi-node fencing. +Legacy objects that the current provider cannot reread because of historical +path/encoding problems fail validation; the command does not guess replacement +keys. Resolve those anomalies with backups before migrating. + +## Review fixes and operating changes + +An empty or whitespace-only PostgreSQL URL selects object-store mode. Pool and timeout tuning is ignored when that backend is inactive. +When selected, numeric tuning must contain positive integers. The object-store metastore and +bulk storage client retain separate request budgets: sharing them would also +make metadata reads compete with Parquet traffic, altering default-mode request +scheduling. Lifecycle prefix deletes currently use the metadata client's budget. + +Migration `002_review_fixes.sql` changes the key column to `COLLATE "C"` and +rebuilds its primary key transactionally under the same startup advisory lock. +On a large database, budget startup time for that rebuild and its exclusive table +lock. Prefix reads and deletes use explicit component-bounded ranges; they do not +require PostgreSQL 15's `starts_with` optimization. Listings are tenant-scoped +and key-only; manifest query planning uses bounded SQL batches rather than one pooled +connection per manifest. SQL manifest locators are bucket-relative on every +provider; old absolute LocalFS locators are accepted on read. No other SQL +caller uses the provider-absolute locator form. Existing cloud +locators retain their spelling, including percent escapes; SQL delimiter listings +also preserve exact stored keys instead of escaping them a second time. Parquet paths remain provider-specific. + +Stream document writes and deletion markers serialize through a database trigger. +While a deletion marker exists, stream creation and schema/stream updates fail +with a distinct deletion-pending error (HTTP 409 on metastore error paths). +Retry DELETE to finish cleanup before recreating. Default object-store mode still +has its existing check/create behavior; interrupted-deletion fencing there is +not changed. This admission rule is not full fencing of Parquet publication or +resident writers. Retention unions bucket dates with SQL dates, so SQL-only +manifest leftovers after physical deletion are discoverable on the next run. +Tenant cleanup excludes root streams, including streams being deleted, and +requires a single tenant path component. Absent, empty and `DEFAULT_TENANT` +tenant identifiers use the same empty SQL tenant column. Stream lifecycle keys +follow the existing root-path normalization. + +Server SQL operations, including transaction start/commit and health, have a +client deadline of statement timeout + 250 ms. Startup/schema migrations use +the separate migration budget described below. Offline SQL validation can also +use that budget; import commits and aborts use the lock session's normal +statement deadline. +Canceled/timed-out connections are discarded instead of recycled. Readiness has +an overall deadline of that same duration, including pool acquisition and bucket +probe. TCP keepalive starts after one second with one-second probes and three +retries; TCP user timeout follows statement timeout. Connections target read-write +servers, and connections reporting read-only errors are discarded for failover. +Database failures use a dedicated unavailable error (503 through metastore HTTP +error mapping). URL startup `options` are preserved before timeout options are +appended; timeout settings take precedence. PgBouncer must accept startup +`options` (configure its ignore-startup-parameters policy as appropriate). +The database role needs schema/table/sequence creation and alteration privileges, +plus PL/pgSQL function and trigger creation for migration 002, and normal document +CRUD privileges afterward. + +A configured PEM CA bundle **replaces** public webpki trust roots. Supply managed +database private CAs with `P_METASTORE_POSTGRES_CA`; the operating system trust +store is not loaded. An empty CA bundle is rejected at configuration time. CA plus `sslmode=disable` emits a warning and sends plaintext. + +Normal startup refuses an existing database identity, including one with zero documents, when the bucket has no pairing marker. +Restore the matching marker from backup. For deliberate recovery only, after +verifying the bucket and database identity, set +`P_METASTORE_POSTGRES_ALLOW_UNPAIRED=` to allow creation of the missing marker. +This override never permits mismatched identities or incomplete offline transfers. +The import tool's committed-import recovery continues to establish pairing itself +under its durable transfer marker, and migrates schema versions 1 and 2 to 3. Exported +or incomplete databases still refuse server admission. + +Conditional-create lost responses remain ambiguous, as on object storage: a retry +returns conflict. Comparing bytes cannot distinguish another writer that wrote +the same body, so no success is synthesized from body equality. + +Reproduce the populated-table plans with +`scripts/postgres-metastore-review-explain.sql` in a disposable database after +applying migrations. Captured PostgreSQL 17 output (200,400 rows, four tenants) +is in [postgres-review-explain.txt](validation/postgres-review-explain.txt). + +Default-mode follow-up: bucket tombstones also make lazy loading report an absent +stream, while `ObjectStorage::create_stream` unconditionally writes schema and +stream JSON. Recreation while a tombstone remains is therefore the same admission +gap in object-store mode. It is reported here and intentionally unchanged in this +PostgreSQL production fix. The two new SQL HTTP errors also preserve their status +when stream creation wraps them in a serialized metastore error detail. + +### Review validation + +The review regression module is separate from the protected existing contracts. +It checks two manifest publications through the returned locator on LocalFS and +an S3-style provider; exact percent-key spelling; overlapping tenant listings; +`DEFAULT_TENANT`; marker/create races including a blocked SQL writer; a 200-key +batch with a two-connection pool; injected failure after physical partition +removal and subsequent SQL cleanup; missing pairing-marker refusal; URL startup +options; read-only connection discard; CA-only trust; and HTTP error propagation. +A TCP proxy stops forwarding without resetting sockets. Four consecutive +readiness calls each return within two seconds with a 300 ms statement timeout, +and a healthy probe succeeds after forwarding resumes. + +Full library runs with and without the PostgreSQL test URL each passed 1,389 +tests with four ignored. SQL skips are printed in the no-database run. Debug +build and plain `cargo clippy --all-targets` passed; Clippy retains the same 21 +library-test warnings and one existing integration-test warning, with no new +warnings. PostgreSQL was `postgres:17-alpine` on a spare port; no live database, +service or bucket was used. The populated collation migration preserved all +200,400 rows; its resulting key collation and query plans are recorded above. + +The offline smoke now waits for staging to remain drained before stopping a +writer, and polls published event bounds before capturing the export comparison. +This avoids stopping with an unsynced custom-partition file or comparing a cached +pre-export bound to a freshly loaded rollback bound. It passed interrupted import +and export, SQL-mode query/metadata equality, new ingestion, verified export and +object-store rollback. Existing Rust test modules and expectations are unchanged. +That historical review smoke rejected schema version 3 and restored version 2, +which was current with migration 002. Current validation rejects version 999 +and restores version 3. + +The three-mode smoke passed object-store, verified TLS PostgreSQL and plaintext +PostgreSQL scenarios, including lifecycle, dependency outages, CA rejection and +future-schema/pinned-identity rejection. Raw results are in +[postgres-review-smoke.json](validation/postgres-review-smoke.json). +One complete run passed with the default SQL timeout. The final rebuilt-binary +confirmation used `P_METASTORE_POSTGRES_STATEMENT_TIMEOUT_MS=60000` because this +shared machine produced PostgreSQL filesystem syncs over 25 seconds; an earlier +confirmation correctly returned 503 at the default client deadline. The proxy +blackhole regression retained its 300 ms statement timeout. A first smoke attempt +also exceeded the existing auxiliary-process shutdown wait; a subsequent complete +run passed without changing that protected cleanup logic. All started containers +were removed. Large migrations or slow storage may require increasing the +statement timeout; the readiness/client deadline increases with that setting. + +## Hardened offline procedure + +**Older binaries ignore pairing and transfer markers and can write stale bucket +metadata. Disable their restart policies and prevent redeployment for the entire +PostgreSQL interval.** Digests detect bucket changes during rollback; they do not +fence writers. Never run these commands against a running deployment. + +1. Save binary version, environment and `.parseable.env`, working directory, + staging, a PostgreSQL `pg_dump`, and all bucket metadata: hidden namespaces, + every manifest variant, and pairing/transfer markers. Preserve object versions. +2. Record stream list, schemas, RBAC, alerts, targets, dashboards, filters, known + query results and row counts, plus first/latest timestamps. +3. Drain staging, stop the single server and all other writers, and disable + restarts. Explicitly set `P_STAGING_DIR` to the server's actual directory. + The tool loads `.parseable.env` with the server's loader. If buffers can never + sync, retain a complete staging backup, restore normal mode and repair the + stream/storage problem. Alternatively move the stranded buffers to a separate + preserved recovery directory outside staging, accepting that those events + require later replay; never delete them to satisfy the check. +4. Configure the exact bucket and destination PostgreSQL URL. Run import + `--dry-run --acknowledge-server-stopped`. Database identity, pin, pairing and + transfer ownership are checked read-only before inventory. Review counts, + skipped keys, digests and recent-write warnings. +5. Run import with the same configuration and acknowledgement. Run verify and + require zero differences and SQL document count equal to the dry-run count. +6. Start PostgreSQL mode. Compare all observations from step 2 before ingest. + Preserve both backups until rollback closes. + +Rollback uses the same backups, drain and stopped-writer steps. Run verify and +save differences, then export with the acknowledgement and **no force flag**. +Imported bucket objects still matching their import digest can be overwritten or +deleted safely. Unknown or changed bucket objects require the narrow +`--overwrite-bucket-changes-since-import` flag after reviewing the risk of losing +bucket-writer metadata. `--migration-force` is rejected with an explanation of its replacement. +After successful export, verify, remove the PostgreSQL URL, start bucket mode +and compare all saved results, including events ingested in PostgreSQL mode. + +The command surface remains `--metadata-migration import|verify|export`. +`--abort` is accepted with import or export. Import abort is safe before its SQL +commit; export abort is safe before export state commits. Once irreversible +export writes can have begun, abort refuses and explains that export must finish. +`--allow-dangling-snapshot-references` reports an explicit override during import; +it preserves the existing dangling reference. Without it, missing references +fail validation. Migration modifiers require `--metadata-migration` and cannot +be silently accepted by a normal server start. + +| Failure boundary | Recovery command (same settings and stopped-writer acknowledgement) | +| --- | --- | +| Before marker | Retry import/export; original mode remains available | +| Import marker, no SQL commit | Retry import, or import --abort then start bucket mode | +| Import SQL committed | Retry import to establish pairing and clear marker | +| Import paired, marker remains | Retry import | +| Export validation failed before marker | Correct the SQL metadata in PostgreSQL mode, then retry export | +| Export marker, state still ready | Export --abort restores PostgreSQL startup | +| Export state committed / any destination write | Retry export; abort refuses | +| Export pairing removed, marker remains | Retry export to finish verification and marker cleanup | +| Complete export | Start bucket mode; SQL startup refuses exported state | + +Default startup performs two marker GETs. Not-found permits normal startup. +Other probe errors identify the marker and ask for retry after bucket access is +restored; they do not write a permanent refusal. A paired bucket requires export +or `P_ALLOW_PAIRED_BUCKET=`; incomplete transfers cannot be +bypassed. + +Inventory lists root/tenant/stream/date directories, skips hour and minute +prefixes, and uses flat paginated listings within hidden metadata prefixes. +Bodies are spooled in the command's working directory and imported transactionally; +the inventory recheck and all remote bucket I/O run outside SQL transactions. +For a single-tenant deployment with S streams, D dates per stream and one +`.stream` metadata page per stream, a walk needs approximately +1 + 2S + SD plus global hidden-metadata pages, rather than +1 + 2S + SD(1 + 24 + 1440). For S=20, D=30, that is about 641 versus +879,041 LIST requests per walk, or 1,282 versus 1,758,082 for two walks, +excluding global metadata pagination. GET costs scale with metadata documents, +not Parquet count. Large hidden namespaces add one request per object-store page. + +Four default-backend bugs also present on origin/main are corrected: target +serialization now writes tenantId and accepts legacy tenant on load; alert +listing excludes mttr.json; LocalFS stream listing uses the selected instance; +startup cleanup waits for the entire discovery/validation pass to succeed before +deleting any invalid tenant. + +The merged CRUD contract and boundary guard kept their assertions. The smoke's +future-schema test now rejects 3 and restores 2 (migration 002 is supported). +The migration smoke no longer expects ordinary rollback to fail without force; +it interrupts and resumes an unforced export. The migration fixture still +rejects a bucket-writer edit, restores its import baseline, then proves that SQL +edits and SQL deletions export without force. + +### Import-tool review disposition + +| Finding | Disposition | +| --- | --- | +| B1 inventory cost | Fixed: metadata-only hidden-prefix pages and date-level listing; hour/minute subtrees excluded; source recheck outside SQL transaction. | +| B2 late export validation | Fixed: validate authoritative SQL rows and SQL snapshot references before marker/state/destination writes; abort available before irreversible boundaries. | +| M1 encoding | Fixed: decode one object-store path encoding layer; real MinIO inventory and SQL round trip cover café and auth0\|é. | +| M2 late preflight | Fixed: read-only database, identity, pin and marker checks precede inventory, including dry-run. | +| M3 old binaries | Mitigated as directed: prominent operational prohibition plus import digests; no mutation of metadata older binaries consume. | +| M4 broad force | Fixed: import digests distinguish SQL updates/deletions from bucket-writer changes; normal export requires no force; explicit narrow override handles the latter. | +| M5 staging/env | Fixed: server env-file loader, explicit staging argument/environment, .part refusal, documented repair/quarantine path. | +| m1 skipped/unknown keys | Fixed: skipped-key report/count; unknown hidden namespaces abort; dashboard locations/titles and LocalFS lock artifacts recognized. | +| m2 legacy tenant scope | Fixed: legacy root stream/schema keys use default tenant and are excluded from SQL tenant cleanup candidates. | +| m3 dangling/legacy snapshot | Fixed: explicit reported dangling-reference override; legacy v1–v3 stream references validated without rewriting source bytes. | +| m4 duplicated bootstrap | Fixed: offline tool and server call the same transactional bootstrap and forward migrations. | +| m5 missing abort | Fixed: safe precommit abort commands, with unsafe abort refusal and rerun instructions. | +| m6 startup probes | Documented: two marker GETs; missing permits startup, transient probe failure requests retry and writes no permanent refusal. | +| m7 broad pairing override | Fixed: P_ALLOW_PAIRED_BUCKET must equal the exact pairing id. | +| m8 duplicate validation | Fixed: import validates in pre-pass; second read spools source bytes and verifies SQL digests rather than validating again. | +| m9 writer fencing | Documented limitation: all writers/restarts must remain stopped; markers/digests are not writer fencing. | + +The old --migration-force flag is explicitly rejected. Modifier flags require +--metadata-migration; normal help hides them, while invoking migration help shows +them. Keys and digest reports can contain tenant/user identifiers and must be +protected like metadata backups. + +The review regression fixture now permits 2 seconds for SQL work and 10 seconds +for database creation, to avoid conflating filesystem setup latency with the +network deadline test. Its proxy blackhole still uses a 300 ms statement timeout, +four probes, and the original under-two-second assertions. + +### Hardening validation + +The combined implementation was validated locally with disposable PostgreSQL 17 +and MinIO, with all task containers removed afterward. Full library runs with +and without the database URL each passed 1,405 tests (seven ignored); SQL tests +print explicit skips without prerequisites. The PostgreSQL contract and startup +cleanup fixtures were then run explicitly and passed, as did the paused-server +deadline test. Focused metastore runs passed 37 tests against LocalFS/PostgreSQL +and MinIO/PostgreSQL; follow-up migration checks cover legacy absolute locators, +both legacy tenant scopes, safe/unsafe aborts, and early pin refusal. + +Both end-to-end scripts passed: the main smoke in object-store, PostgreSQL +plaintext and verified TLS modes; offline migration in plaintext and verified +TLS, including interrupted import/export, .part refusal, new SQL-mode ingestion +and unforced object-store rollback. The smoke generated its own +[raw evidence](validation/postgres-metastore-smoke.json). +[Round-two validation commands and results](validation/postgres-round2-checks.json) +record the latest checks; [earlier results](validation/postgres-hardening-checks.json) +remain as historical evidence. Plain cargo clippy --all-targets and the debug build +passed; Clippy retained 21 existing library-test warnings and one existing +integration-test warning, with no new warnings. The first hosted workflow run +[failed during runner setup](https://github.com/VibeEngineeringAI/parseable/actions/runs/43037438052) +because the Ubuntu package mirror could not be resolved; compilation and tests +never started. The expanded workflow has not been pushed or run remotely. + +Counts in the inventory chart are a model of request growth, not a throughput +benchmark. No concurrent-writer fencing, automatic resolution of an interrupted +stream deletion, or production-scale throughput claim is made. + +Review nits: migration modifiers are scoped in parsing/help and reports document +their identifier privacy risk. Deduplicating the object-safe storage guard and +SQL startup guard adapters is deferred: both enforce the same tested controls, +and consolidating their different error adapters adds no recovery capability. +The required real PostgreSQL/MinIO and injected-boundary coverage is implemented; +the expanded hosted checks remain unrun under the no-push constraint. + +### Review hardening and operational limits + +LocalFS lookups try the exact document key first, then the old root-prefixed +locator compatibility form. If both rows exist, the exact row wins and a warning +is logged. Cloud locators retain their existing interpretation. Empty tenants and +`DEFAULT_TENANT` use the same metadata paths as an absent tenant. + +Schema version 3 admits manifests under the same transaction advisory lock as +stream documents and deletion markers. Startup migrations have a separate +`P_METASTORE_POSTGRES_MIGRATION_TIMEOUT_MS` budget (default 1800000, 30 minutes). +Migration 002 takes an exclusive table lock while changing collation and rebuilding +indexes; stop old writers before upgrading a large populated database. Concurrent +new initializers serialize on the startup advisory lock. Increase the migration +budget for a large table; ordinary queries retain the statement deadline. + +When deletion is confirmed pending, the uploader drops staged parquet and schema +files for that stream. If deletion wins after upload began, it first removes all +parquet objects uploaded by that batch, then retires those staged files. A bucket +cleanup failure preserves staging for retry. Publication delayed beyond the final +SQL row deletion remains possible: an incarnation token is required to fence that +window (issue 142). Do not recreate a stream while old uploaders are still active. +Tenant deletion refuses a tenant name that collides with a known default-tenant +stream, including SQL-only cleanup candidates. Rename or resolve that collision +before retrying cleanup. + +A client SQL timeout sends a separately bounded PostgreSQL cancel request and +discards the connection. Cancellation or its response can be lost, and a write or +COMMIT may already have completed. Such failures have an ambiguous outcome: inspect +state before retrying; cancellation is not a rollback guarantee. + +Offline transfers, including abort, hold a session advisory lock for the entire +operation; a second invocation fails immediately. Import releases its ordinary +database connection during spooling and the second inventory walk. The dedicated +lock session remains connected. Bucket reads and inventory walks use concurrency +16. No bucket I/O occurs inside the SQL import transaction. + +The tool requires an existing explicit staging directory, resolves it to an +absolute path, and prints it together with the current directory's environment +file (or that none was loaded). Run from the deployment's configuration directory; +the tool does not search other directories for an environment file. + +Import validates all JSON documents before committing. The dangling snapshot +reference override applies to both import and export, including export's final +validation. Export dry-run reads SQL and bucket state, checks references and +reports bucket conflicts without bootstrapping SQL or writing bucket markers. +An imported key deleted from the bucket is a conflict. New SQL keys absent from +the bucket are normal. Conflicts are counted, with the first 100 keys printed, +before requiring `--overwrite-bucket-changes-since-import`. This flag also permits +removing bucket-only metadata; back up the bucket first. Databases imported by an +older tool without digests can export equal documents without an override; changed +documents require it. Import abort may clear committed SQL only when the matching +Import marker still exists and no pairing marker has ever been published. + +The disaster recovery setting `P_METASTORE_POSTGRES_ALLOW_UNPAIRED` must equal the +backend UUID, rather than a boolean. Remove it after recovery. + +Spools contain sensitive metadata, including RBAC. They live below a private +(mode 0700) `.parseable-migration-spool-` directory in the working +directory. Normal exit removes the temporary body directory; SIGKILL can leave +it behind. The next import with the same backend id and exclusive transfer lock +removes that stale spool before reading metadata again. + +For 20 streams × 30 days, assuming 650 LIST requests per walk, one manifest per +stream/day, and two stream documents, import makes approximately 3180 bucket +requests: two inventories (1300 LIST), two reads of 640 documents, and 600 +snapshot-reference reads. For 1000 streams the corresponding estimate is 159000 +requests. At 150 ms/request and concurrency 16 the latency-only lower bounds are +about 30 seconds and 25 minutes respectively; shallow discovery rounds, pagination, +SQL verification, body sizes and network throughput add time. These are estimates, +not measured throughput. The committed literal EXPLAIN output does not demonstrate +prepared generic-plan behavior. + +Default object-store mode retains query storage-error HTTP 400 responses. +Explicit-stream ingest in distributed mode propagates storage lookup failures +instead of reporting a missing stream (404); actual missing streams remain 404. +Shared +metadata path helpers now normalize empty and explicit default tenants; malformed +percent-encoded inventory keys return errors instead of panicking, `.keystone/` +is accepted by the offline metadata inventory, and inventory +discovery uses bounded concurrency. The uploader retirement policy applies to +confirmed PostgreSQL deletion conflicts. + +Default-tenant alert-state enumeration now uses the canonical `.alerts/` +directory, consistent with writes and single-state reads. It therefore includes +states previously missed by the default-tenant listing. Per-date manifest +inventory also normalizes explicit default-tenant aliases. + +Import canonicalizes explicit `DEFAULT_TENANT/` metadata keys and snapshot +manifest locators. It retains the original source digests for rollback conflict +checks, and verify compares the corresponding canonical representation. Other +metadata bodies, including legacy absolute locators without that alias, retain +their bytes. This avoids exporting a snapshot that points to an alias key +removed during rollback; unknown JSON fields are retained when locators change. + +Offline commands require a direct PostgreSQL connection or session pooling: the +whole-transfer advisory lock must stay on one database session. Transaction +pooling cannot provide that lock lifetime. + +Manifest planning consults the cache for both documents and known misses. Cold +reads select at most 32 locators (64 KiB of locator text) per sizing batch and +transfer at most 8 MiB of bodies per chunk, decoding before fetching the next +chunk. The body statement rechecks the limit to handle concurrent growth. One +larger manifest is returned alone. Missing manifests still produce individual +warnings. + +`export --abort` before export starts is a safe no-op, including databases +initialized directly in PostgreSQL mode without a transfer row. + +Import commits and abort SQL execute on the session owning the transfer lock. +If that session is lost during spooling, the command fails without reconnecting +and committing its old work; rerun or abort under a new lock. Functional smoke +fixtures use a 30-second SQL budget on shared build machines; separate timeout +regressions retain their short deadlines. + +Failed PostgreSQL upload batches wait for sibling uploads to settle before +compensation. If deletion cannot be confirmed, staging remains available for +retry. Default object-store batches retain their existing error handling. diff --git a/docs/validation/postgres-hardening-checks.json b/docs/validation/postgres-hardening-checks.json new file mode 100644 index 000000000..df134384c --- /dev/null +++ b/docs/validation/postgres-hardening-checks.json @@ -0,0 +1,213 @@ +{ + "commands": [ + { + "name": "library-without-url", + "argv": [ + "cargo", + "test", + "--lib", + "--", + "--nocapture" + ], + "exit_code": 0, + "log": "/.postgres-smoke/library-without-url.log", + "summaries": [ + "test result: ok. 1398 passed; 0 failed; 7 ignored; 0 measured; 0 filtered out; finished in 75.00s" + ] + }, + { + "name": "library-with-url", + "argv": [ + "cargo", + "test", + "--lib", + "--", + "--test-threads=1", + "--nocapture" + ], + "exit_code": 0, + "log": "/.postgres-smoke/library-with-url.log", + "summaries": [ + "test result: ok. 1398 passed; 0 failed; 7 ignored; 0 measured; 0 filtered out; finished in 137.63s" + ] + }, + { + "name": "contracts", + "argv": [ + "cargo", + "test", + "--lib", + "contract_tests", + "--", + "--nocapture", + "--test-threads=1" + ], + "exit_code": 0, + "log": "/.postgres-smoke/contracts.log", + "summaries": [ + "test result: ok. 10 passed; 0 failed; 3 ignored; 0 measured; 1392 filtered out; finished in 1.77s" + ] + }, + { + "name": "postgres-contract", + "argv": [ + "cargo", + "test", + "--lib", + "contract_tests::postgres_contract", + "--", + "--ignored", + "--nocapture" + ], + "exit_code": 0, + "log": "/.postgres-smoke/postgres-contract.log", + "summaries": [ + "test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 1404 filtered out; finished in 5.51s" + ] + }, + { + "name": "startup-cleanup", + "argv": [ + "cargo", + "test", + "--lib", + "contract_tests::postgres_startup_cleanup", + "--", + "--ignored", + "--nocapture" + ], + "exit_code": 0, + "log": "/.postgres-smoke/startup-cleanup.log", + "summaries": [ + "test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 1404 filtered out; finished in 0.75s" + ] + }, + { + "name": "paused-health", + "argv": [ + "cargo", + "test", + "--lib", + "postgres_health_deadline_regression", + "--", + "--nocapture" + ], + "exit_code": 0, + "log": "/.postgres-smoke/paused-health.log", + "summaries": [ + "test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 1404 filtered out; finished in 0.94s" + ] + }, + { + "name": "clippy", + "argv": [ + "cargo", + "clippy", + "--all-targets" + ], + "exit_code": 0, + "log": "/.postgres-smoke/clippy.log", + "summaries": [] + }, + { + "name": "focused-localfs", + "exit_code": 0, + "summaries": [ + "test result: ok. 34 passed; 0 failed; 3 ignored; 0 measured; 1368 filtered out; finished in 9.08s" + ] + }, + { + "name": "focused-minio", + "argv": [ + "cargo", + "test", + "--lib", + "metastore::", + "--", + "--nocapture", + "--test-threads=1" + ], + "exit_code": 0, + "log": "/.postgres-smoke/focused-minio.log", + "summaries": [ + "test result: ok. 34 passed; 0 failed; 3 ignored; 0 measured; 1368 filtered out; finished in 10.77s" + ] + }, + { + "name": "final-build", + "argv": [ + "cargo", + "build" + ], + "exit_code": 0, + "log": "/.postgres-smoke/final-build.log", + "summaries": [] + }, + { + "name": "cli-scope-and-help", + "exit_code": 0, + "flags": 5 + }, + { + "name": "legacy-layouts-and-minio", + "exit_code": 0, + "summaries": [ + "test result: ok. 34 passed; 0 failed; 3 ignored; 0 measured; 1368 filtered out; finished in 10.64s" + ], + "log_sha256": "ff9958a1ad0394e52c04f10db1143cdb29bba5f57076c20da99b5f1090a72841" + }, + { + "name": "abort-and-pinned-preflight", + "exit_code": 0, + "summaries": [ + "test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 1404 filtered out; finished in 1.90s" + ], + "log_sha256": "7dbd844df0d6555df0b6c4393a0f3fdf68f680c3e396ec8de8abcadda8bfbdf8" + }, + { + "name": "final-clippy", + "exit_code": 0, + "log_sha256": "c36993dc1ec5f5e4a472a4a8c99619da26e0e30fbc46d259016bbd857db75179" + }, + { + "name": "verified-build", + "exit_code": 0, + "log_sha256": "b7a890fd5f3d3fbfff9c8b0a146f9880bf12076a1ebe4aef82460c7ffe8e22c3" + } + ], + "success": true, + "end_to_end": { + "main_modes": [ + "object-store", + "postgres-tls", + "postgres" + ], + "main_success": true, + "migration_plaintext_success": true, + "migration_verified_tls_success": true + }, + "migration_end_to_end": [ + { + "mode": "plaintext", + "success": true, + "log_sha256": "1dda51d6b4c53a068158afd8d7af090dff8145b58d278ac68f2a51ac552054a4" + }, + { + "mode": "verified-tls", + "success": true, + "log_sha256": "1dda51d6b4c53a068158afd8d7af090dff8145b58d278ac68f2a51ac552054a4" + } + ], + "failure_injection": { + "import-marker-spool-before-commit": "rerun import or import --abort; object-store guard admits startup after abort", + "import-commit-paired": "rerun import; abort refuses committed documents", + "export-marker-ready": "export --abort; restarted PostgreSQL backend admits startup", + "export-state-object-write-bucket-delete-pair-removed": "rerun export; abort refuses; exported bucket guard admits startup and PostgreSQL startup refuses" + }, + "clippy_warnings": { + "library_test": 21, + "integration_test": 1, + "new": 0 + }, + "workflow_yaml_parsed": true +} diff --git a/docs/validation/postgres-import-list-requests.html b/docs/validation/postgres-import-list-requests.html new file mode 100644 index 000000000..0c09912a9 --- /dev/null +++ b/docs/validation/postgres-import-list-requests.html @@ -0,0 +1,13 @@ + + +

Estimated LIST requests for one inventory: 20 streams × 30 days

+ + + + + +PreviousHardened +879,041641 +110010,0001,000,000 + +Log scale. Excludes global hidden-metadata pages. Two import walks: 1,758,082 → 1,282 requests. GET costs depend on metadata count; hour/minute prefixes and Parquet files are excluded. diff --git a/docs/validation/postgres-metastore-requests.html b/docs/validation/postgres-metastore-requests.html new file mode 100644 index 000000000..9015b1657 --- /dev/null +++ b/docs/validation/postgres-metastore-requests.html @@ -0,0 +1 @@ +
Object store
9
PostgreSQL
1
PostgreSQL TLS
1

Bucket requests per query, five repeated queries. Query process counters with an ingestor running. PostgreSQL: one Parquet range request. Object store: six metadata GETs, two listings and one Parquet range request.

diff --git a/docs/validation/postgres-metastore-smoke.json b/docs/validation/postgres-metastore-smoke.json new file mode 100644 index 000000000..fb90c93cf --- /dev/null +++ b/docs/validation/postgres-metastore-smoke.json @@ -0,0 +1,7896 @@ +{ + "runs": [ + { + "requested_modes": [ + "object-store", + "postgres-tls", + "postgres" + ], + "source_commit": "49924e2ed81ff4e1ce9bccbe6cdf69d585c3cd31", + "binary_sha256": "bb0af87e40744d6798700aa269afa3083570bf89ed5613a67ec550df68353653", + "source_diff_sha256": "e810e0ac8997bc3c815ddf95d21838593d8cb1d97520b113e05cd154ed04acbb", + "image_digests": { + "postgres:17-alpine": [ + "postgres@sha256:b0f9560a2de083e2cc7382e75f808c7381a32852a7ec49117deedb300e552b24" + ], + "quay.io/parseablehq/minio:edge": [ + "quay.io/parseablehq/minio@sha256:88276d0a028d758ffaf80d65dd80071386e91b4114251a3dda101eadf000faeb" + ] + }, + "measurement": "Node-local storage counter delta during five queries, including background traffic; not isolated requests per query.", + "object-store": { + "lazy_reload": true, + "query": "[{\"count\":1}]", + "dashboard": { + "version": "v1", + "title": "Postgres smoke", + "author": "8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918", + "dashboardId": "01M75NFJS7A10VN0ZFSBBY6ZAT", + "created": "2026-11-11T16:01:30.919827850+00:00", + "modified": "2026-11-11T16:01:30.919827850+00:00", + "tags": null, + "isFavorite": false, + "dashboardType": "Dashboard", + "tiles": [], + "tenantId": null, + "revision": "0d4c1de651e8d0cb27704f511e0becd54950ddaf09051115b487084b2d3e7b80" + }, + "keys": [ + ".parseable/.parseable.json", + ".parseable/ingestor.ca453944eff07df.json", + ".parseable/querier.6e37ab554650b70.json", + ".users/.dashboard-locations/01M75NFJS7A10VN0ZFSBBY6ZAT.json", + ".users/.dashboard-titles/571a2cefe2d0d1a9a26dab4810a5556875c7d7531d4f558a8f7b28ab486e2395.json", + ".users/8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918/dashboards/01M75NFJS7A10VN0ZFSBBY6ZAT.json", + "lazy/.stream/.ingestor.ca453944eff07df.schema", + "lazy/.stream/.ingestor.ca453944eff07df.stream.json", + "lazy/.stream/.schema", + "lazy/.stream/.stream.json", + "lazy/date=2026-11-11/hour=16/minute=02/ca453944eff07df.data.01M75NJAP4C59XEDHX8HVK5QBE.parquet", + "lazy/date=2026-11-11/ingestor..ca453944eff07df.manifest.json", + "pstats/.stream/.ingestor.ca453944eff07df.schema", + "pstats/.stream/.ingestor.ca453944eff07df.stream.json", + "pstats/.stream/.schema", + "pstats/.stream/.stream.json", + "pstats/date=2026-11-11/.manifest.json", + "pstats/date=2026-11-11/hour=16/minute=00/dataset_name=smoke/.data.01M75NGFXMDMRDEPRWCR78SP2S.parquet", + "smoke/.stream/.ingestor.ca453944eff07df.schema", + "smoke/.stream/.ingestor.ca453944eff07df.stream.json", + "smoke/.stream/.schema", + "smoke/.stream/.stream.json", + "smoke/date=2026-11-11/.manifest.json", + "smoke/date=2026-11-11/hour=16/minute=00/.data.01M75NENAKWMP0MHNV6WWF9ZCD.parquet" + ], + "non_parquet_keys": [ + ".parseable/.parseable.json", + ".parseable/ingestor.ca453944eff07df.json", + ".parseable/querier.6e37ab554650b70.json", + ".users/.dashboard-locations/01M75NFJS7A10VN0ZFSBBY6ZAT.json", + ".users/.dashboard-titles/571a2cefe2d0d1a9a26dab4810a5556875c7d7531d4f558a8f7b28ab486e2395.json", + ".users/8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918/dashboards/01M75NFJS7A10VN0ZFSBBY6ZAT.json", + "lazy/.stream/.ingestor.ca453944eff07df.schema", + "lazy/.stream/.ingestor.ca453944eff07df.stream.json", + "lazy/.stream/.schema", + "lazy/.stream/.stream.json", + "lazy/date=2026-11-11/ingestor..ca453944eff07df.manifest.json", + "pstats/.stream/.ingestor.ca453944eff07df.schema", + "pstats/.stream/.ingestor.ca453944eff07df.stream.json", + "pstats/.stream/.schema", + "pstats/.stream/.stream.json", + "pstats/date=2026-11-11/.manifest.json", + "smoke/.stream/.ingestor.ca453944eff07df.schema", + "smoke/.stream/.ingestor.ca453944eff07df.stream.json", + "smoke/.stream/.schema", + "smoke/.stream/.stream.json", + "smoke/date=2026-11-11/.manifest.json" + ], + "request_delta": { + "parseable_storage_request_response_time_count{method=\"GET\",provider=\"s3\",status=\"200\"}": 30.0, + "parseable_storage_request_response_time_count{method=\"GET_RANGES\",provider=\"s3\",status=\"200\"}": 5.0, + "parseable_storage_request_response_time_count{method=\"LIST\",provider=\"s3\",status=\"200\"}": 5.0, + "parseable_storage_request_response_time_count{method=\"LIST_DELIM\",provider=\"s3\",status=\"200\"}": 5.0 + }, + "observed_requests_during_window_divided_by_queries": 9.0, + "bucket_outage_readiness": 503, + "delete_recreate": true, + "final_keys": [ + ".parseable/.parseable.json", + ".parseable/ingestor.ca453944eff07df.json", + ".parseable/querier.6e37ab554650b70.json", + ".users/.dashboard-locations/01M75NFJS7A10VN0ZFSBBY6ZAT.json", + ".users/.dashboard-titles/571a2cefe2d0d1a9a26dab4810a5556875c7d7531d4f558a8f7b28ab486e2395.json", + ".users/8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918/dashboards/01M75NFJS7A10VN0ZFSBBY6ZAT.json", + "lazy/.stream/.ingestor.ca453944eff07df.schema", + "lazy/.stream/.ingestor.ca453944eff07df.stream.json", + "lazy/.stream/.schema", + "lazy/.stream/.stream.json", + "lazy/date=2026-11-11/hour=16/minute=02/ca453944eff07df.data.01M75NJAP4C59XEDHX8HVK5QBE.parquet", + "lazy/date=2026-11-11/ingestor..ca453944eff07df.manifest.json", + "pstats/.stream/.ingestor.ca453944eff07df.schema", + "pstats/.stream/.ingestor.ca453944eff07df.stream.json", + "pstats/.stream/.schema", + "pstats/.stream/.stream.json", + "pstats/date=2026-11-11/.manifest.json", + "pstats/date=2026-11-11/hour=16/minute=00/dataset_name=smoke/.data.01M75NGFXMDMRDEPRWCR78SP2S.parquet", + "pstats/date=2026-11-11/hour=16/minute=02/dataset_name=lazy/ca453944eff07df.data.01M75NM594TPMV98RWN5XAZY56.parquet", + "pstats/date=2026-11-11/ingestor..ca453944eff07df.manifest.json", + "smoke/.stream/.ingestor.ca453944eff07df.schema", + "smoke/.stream/.ingestor.ca453944eff07df.stream.json", + "smoke/.stream/.schema", + "smoke/.stream/.stream.json", + "smoke/date=2026-11-11/hour=16/minute=03/ca453944eff07df.data.01M75NM5946JWKF95XGEZEP8Z2.parquet", + "smoke/date=2026-11-11/ingestor..ca453944eff07df.manifest.json" + ] + }, + "postgres-tls": { + "trusted_ca_restart": true, + "untrusted_ca_rejected": true, + "untrusted_ca_output": "Error: ObjectStorageError: PostgreSQL metastore unavailable: Error occurred while creating a new object: error performing TLS handshake\n\nCaused by:\n PostgreSQL metastore unavailable: Error occurred while creating a new object: error performing TLS handshake\n", + "untrusted_ca_server_alert": "2026-11-11 16:04:49.430 UTC [125] LOG: could not accept SSL connection: tlsv1 alert unknown ca\n" + }, + "commands": [ + { + "argv": [ + "docker", + "run", + "-d", + "--name", + "parseable-pg-smoke-1181462-0", + "-e", + "POSTGRES_PASSWORD=smoke", + "-e", + "POSTGRES_DB=parseable", + "-p", + "127.0.0.1:44333:5432", + "postgres:17-alpine" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "run", + "-d", + "--name", + "parseable-pg-smoke-1181462-1", + "-e", + "MINIO_ROOT_USER=minioadmin", + "-e", + "MINIO_ROOT_PASSWORD=minioadmin", + "-p", + "127.0.0.1:37943:9000", + "quay.io/parseablehq/minio:edge", + "server", + "/data" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1181462-0", + "pg_isready", + "-h", + "127.0.0.1", + "-U", + "postgres" + ], + "exit_code": 2 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1181462-0", + "pg_isready", + "-h", + "127.0.0.1", + "-U", + "postgres" + ], + "exit_code": 2 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1181462-0", + "pg_isready", + "-h", + "127.0.0.1", + "-U", + "postgres" + ], + "exit_code": 0 + }, + { + "argv": [ + "openssl", + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-keyout", + ".postgres-smoke/tls/ca.key", + "-out", + ".postgres-smoke/tls/ca.crt", + "-days", + "2", + "-subj", + "/CN=Smoke CA" + ], + "exit_code": 0 + }, + { + "argv": [ + "openssl", + "req", + "-newkey", + "rsa:2048", + "-nodes", + "-keyout", + ".postgres-smoke/tls/server.key", + "-out", + ".postgres-smoke/tls/server.csr", + "-subj", + "/CN=localhost" + ], + "exit_code": 0 + }, + { + "argv": [ + "openssl", + "x509", + "-req", + "-in", + ".postgres-smoke/tls/server.csr", + "-CA", + ".postgres-smoke/tls/ca.crt", + "-CAkey", + ".postgres-smoke/tls/ca.key", + "-CAcreateserial", + "-out", + ".postgres-smoke/tls/server.crt", + "-days", + "2", + "-extfile", + ".postgres-smoke/tls/extensions" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "cp", + ".postgres-smoke/tls/server.key", + "parseable-pg-smoke-1181462-0:/var/lib/postgresql/data/server.key" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "cp", + ".postgres-smoke/tls/server.crt", + "parseable-pg-smoke-1181462-0:/var/lib/postgresql/data/server.crt" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1181462-0", + "sh", + "-c", + "chown postgres:postgres /var/lib/postgresql/data/server.* && chmod 600 /var/lib/postgresql/data/server.key" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1181462-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-c", + "ALTER SYSTEM SET ssl='on';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1181462-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-c", + "SELECT pg_reload_conf();" + ], + "exit_code": 0 + }, + { + "argv": [ + "git", + "rev-parse", + "HEAD" + ], + "exit_code": 0 + }, + { + "argv": [ + "git", + "diff", + "HEAD" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "image", + "inspect", + "postgres:17-alpine" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "image", + "inspect", + "quay.io/parseablehq/minio:edge" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1181462-1", + "mkdir", + "-p", + "/data/smoke-object-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "stop", + "parseable-pg-smoke-1181462-1" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "start", + "parseable-pg-smoke-1181462-1" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1181462-1", + "mkdir", + "-p", + "/data/smoke-postgres-tls" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1181462-0", + "psql", + "-U", + "postgres", + "-c", + "CREATE DATABASE tls;" + ], + "exit_code": 0 + }, + { + "argv": [ + "openssl", + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-keyout", + ".postgres-smoke/tls/wrong.key", + "-out", + ".postgres-smoke/tls/wrong.crt", + "-days", + "2", + "-subj", + "/CN=Wrong CA" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "logs", + "parseable-pg-smoke-1181462-0" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "logs", + "parseable-pg-smoke-1181462-0" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1181462-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1181462-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1181462-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1181462-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1181462-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1181462-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1181462-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1181462-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1181462-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1181462-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1181462-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1181462-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1181462-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1181462-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1181462-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1181462-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "rm", + "-f", + "parseable-pg-smoke-1181462-1" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "rm", + "-f", + "parseable-pg-smoke-1181462-0" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 0 + } + ], + "success": false, + "failure": "Traceback (most recent call last):\n File \"scripts/postgres-metastore-smoke.py\", line 344, in \n request('/logstream/smoke/retention', 'PUT', [{'description': 'smoke retention', 'action': 'delete', 'duration': '30d'}])\n ~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"scripts/postgres-metastore-smoke.py\", line 283, in request\n with urllib.request.urlopen(req, timeout=timeout) as response:\n ~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/lib/python3.14/urllib/request.py\", line 187, in urlopen\n return opener.open(url, data, timeout)\n ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^\n File \"/usr/lib/python3.14/urllib/request.py\", line 493, in open\n response = meth(req, response)\n File \"/usr/lib/python3.14/urllib/request.py\", line 602, in http_response\n response = self.parent.error(\n 'http', request, response, code, msg, hdrs)\n File \"/usr/lib/python3.14/urllib/request.py\", line 531, in error\n return self._call_chain(*args)\n ~~~~~~~~~~~~~~~~^^^^^^^\n File \"/usr/lib/python3.14/urllib/request.py\", line 464, in _call_chain\n result = func(*args)\n File \"/usr/lib/python3.14/urllib/request.py\", line 611, in http_error_default\n raise HTTPError(req.full_url, code, msg, hdrs, fp)\nurllib.error.HTTPError: HTTP Error 503: Service Unavailable\nStorage Error MetastoreError: MetastoreErrorDetail { operation: \"ObjectStorageError\", message: \"PostgreSQL metastore unavailable: PostgreSQL metastore unavailable: SQL client deadline exceeded\", stream_name: None, file_path: None, timestamp: Some(2026-11-11T16:06:02.469332470Z), metadata: {}, status_code: 503 }\n", + "http_requests": [ + { + "path": "/liveness", + "method": "GET", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "PUT", + "status": 200 + }, + { + "path": "/ingest", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/logstream/smoke/retention", + "method": "PUT", + "status": 200 + }, + { + "path": "/dashboards", + "method": "POST", + "status": 200 + }, + { + "path": "/liveness", + "method": "GET", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/dashboards", + "method": "GET", + "status": 200 + }, + { + "path": "/metrics", + "method": "GET", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/metrics", + "method": "GET", + "status": 200 + }, + { + "path": "/readiness", + "method": "GET", + "status": 503 + }, + { + "path": "/readiness", + "method": "GET", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "DELETE", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "PUT", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/liveness", + "method": "GET", + "status": 200 + }, + { + "path": "/liveness", + "method": "GET", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "PUT", + "status": 200 + }, + { + "path": "/ingest", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/logstream/smoke/retention", + "method": "PUT", + "status": 503 + } + ] + }, + { + "requested_modes": [ + "object-store", + "postgres-tls", + "postgres" + ], + "source_commit": "30bd6ce316fb6689ffc6371f8afb58dd361fcc66", + "binary_sha256": "76cb03a095ab04cb1645eb95032c9da94aeab4486009cb5589c66463b5e329e9", + "source_diff_sha256": "2977ab72da8198c1e8ff5f455b2ad9052e26adfbb080738fed2a130407a3b28d", + "image_digests": { + "postgres:17-alpine": [ + "postgres@sha256:b0f9560a2de083e2cc7382e75f808c7381a32852a7ec49117deedb300e552b24" + ], + "quay.io/parseablehq/minio:edge": [ + "quay.io/parseablehq/minio@sha256:88276d0a028d758ffaf80d65dd80071386e91b4114251a3dda101eadf000faeb" + ] + }, + "measurement": "Node-local storage counter delta during five queries, including background traffic; not isolated requests per query.", + "object-store": { + "lazy_reload": true, + "query": "[{\"count\":1}]", + "dashboard": { + "version": "v1", + "title": "Postgres smoke", + "author": "8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918", + "dashboardId": "01M75Q46A1TPT0FMVP1V702SN4", + "created": "2026-11-11T16:30:14.849272674+00:00", + "modified": "2026-11-11T16:30:14.849272674+00:00", + "tags": null, + "isFavorite": false, + "dashboardType": "Dashboard", + "tiles": [], + "tenantId": null, + "revision": "4712ade4f49952d0bdeb55a6356bec2aff0c0535ffb9ddcb59ecdc1c4013e52b" + }, + "keys": [ + ".parseable/.parseable.json", + ".parseable/ingestor.5f166cb2f1e1418.json", + ".parseable/querier.04822b872795bce.json", + ".users/.dashboard-locations/01M75Q46A1TPT0FMVP1V702SN4.json", + ".users/.dashboard-titles/571a2cefe2d0d1a9a26dab4810a5556875c7d7531d4f558a8f7b28ab486e2395.json", + ".users/8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918/dashboards/01M75Q46A1TPT0FMVP1V702SN4.json", + "lazy/.stream/.ingestor.5f166cb2f1e1418.schema", + "lazy/.stream/.ingestor.5f166cb2f1e1418.stream.json", + "lazy/.stream/.schema", + "lazy/.stream/.stream.json", + "lazy/date=2026-11-11/hour=16/minute=31/5f166cb2f1e1418.data.01M75Q7DTBPJJWJCBH8AE4YN8S.parquet", + "lazy/date=2026-11-11/ingestor..5f166cb2f1e1418.manifest.json", + "pstats/.stream/.ingestor.5f166cb2f1e1418.schema", + "pstats/.stream/.ingestor.5f166cb2f1e1418.stream.json", + "pstats/.stream/.schema", + "pstats/.stream/.stream.json", + "pstats/date=2026-11-11/.manifest.json", + "pstats/date=2026-11-11/hour=16/minute=28/dataset_name=smoke/.data.01M75Q5K2P76MWJ3C5H3MPQJGR.parquet", + "smoke/.stream/.ingestor.5f166cb2f1e1418.schema", + "smoke/.stream/.ingestor.5f166cb2f1e1418.stream.json", + "smoke/.stream/.schema", + "smoke/.stream/.stream.json", + "smoke/date=2026-11-11/.manifest.json", + "smoke/date=2026-11-11/hour=16/minute=28/.data.01M75Q1XWPVJ100X36PKXWZ2BR.parquet" + ], + "non_parquet_keys": [ + ".parseable/.parseable.json", + ".parseable/ingestor.5f166cb2f1e1418.json", + ".parseable/querier.04822b872795bce.json", + ".users/.dashboard-locations/01M75Q46A1TPT0FMVP1V702SN4.json", + ".users/.dashboard-titles/571a2cefe2d0d1a9a26dab4810a5556875c7d7531d4f558a8f7b28ab486e2395.json", + ".users/8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918/dashboards/01M75Q46A1TPT0FMVP1V702SN4.json", + "lazy/.stream/.ingestor.5f166cb2f1e1418.schema", + "lazy/.stream/.ingestor.5f166cb2f1e1418.stream.json", + "lazy/.stream/.schema", + "lazy/.stream/.stream.json", + "lazy/date=2026-11-11/ingestor..5f166cb2f1e1418.manifest.json", + "pstats/.stream/.ingestor.5f166cb2f1e1418.schema", + "pstats/.stream/.ingestor.5f166cb2f1e1418.stream.json", + "pstats/.stream/.schema", + "pstats/.stream/.stream.json", + "pstats/date=2026-11-11/.manifest.json", + "smoke/.stream/.ingestor.5f166cb2f1e1418.schema", + "smoke/.stream/.ingestor.5f166cb2f1e1418.stream.json", + "smoke/.stream/.schema", + "smoke/.stream/.stream.json", + "smoke/date=2026-11-11/.manifest.json" + ], + "request_delta": { + "parseable_storage_request_response_time_count{method=\"GET\",provider=\"s3\",status=\"200\"}": 30.0, + "parseable_storage_request_response_time_count{method=\"GET_RANGES\",provider=\"s3\",status=\"200\"}": 5.0, + "parseable_storage_request_response_time_count{method=\"LIST\",provider=\"s3\",status=\"200\"}": 5.0, + "parseable_storage_request_response_time_count{method=\"LIST_DELIM\",provider=\"s3\",status=\"200\"}": 5.0 + }, + "observed_requests_during_window_divided_by_queries": 9.0, + "bucket_outage_readiness": 503, + "delete_recreate": true, + "final_keys": [ + ".parseable/.parseable.json", + ".parseable/ingestor.5f166cb2f1e1418.json", + ".parseable/querier.04822b872795bce.json", + ".users/.dashboard-locations/01M75Q46A1TPT0FMVP1V702SN4.json", + ".users/.dashboard-titles/571a2cefe2d0d1a9a26dab4810a5556875c7d7531d4f558a8f7b28ab486e2395.json", + ".users/8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918/dashboards/01M75Q46A1TPT0FMVP1V702SN4.json", + "lazy/.stream/.ingestor.5f166cb2f1e1418.schema", + "lazy/.stream/.ingestor.5f166cb2f1e1418.stream.json", + "lazy/.stream/.schema", + "lazy/.stream/.stream.json", + "lazy/date=2026-11-11/hour=16/minute=31/5f166cb2f1e1418.data.01M75Q7DTBPJJWJCBH8AE4YN8S.parquet", + "lazy/date=2026-11-11/ingestor..5f166cb2f1e1418.manifest.json", + "pstats/.stream/.ingestor.5f166cb2f1e1418.schema", + "pstats/.stream/.ingestor.5f166cb2f1e1418.stream.json", + "pstats/.stream/.schema", + "pstats/.stream/.stream.json", + "pstats/date=2026-11-11/.manifest.json", + "pstats/date=2026-11-11/hour=16/minute=28/dataset_name=smoke/.data.01M75Q5K2P76MWJ3C5H3MPQJGR.parquet", + "pstats/date=2026-11-11/hour=16/minute=31/dataset_name=lazy/5f166cb2f1e1418.data.01M75Q98DCXTRQG16AV1VMJQQ0.parquet", + "pstats/date=2026-11-11/ingestor..5f166cb2f1e1418.manifest.json", + "smoke/.stream/.ingestor.5f166cb2f1e1418.schema", + "smoke/.stream/.ingestor.5f166cb2f1e1418.stream.json", + "smoke/.stream/.schema", + "smoke/.stream/.stream.json", + "smoke/date=2026-11-11/hour=16/minute=32/5f166cb2f1e1418.data.01M75Q98DCX5GHSZ2DT9V938W5.parquet", + "smoke/date=2026-11-11/ingestor..5f166cb2f1e1418.manifest.json" + ] + }, + "postgres-tls": { + "trusted_ca_restart": true, + "untrusted_ca_rejected": true, + "untrusted_ca_output": "Error: ObjectStorageError: PostgreSQL metastore unavailable: Error occurred while creating a new object: error performing TLS handshake\n\nCaused by:\n PostgreSQL metastore unavailable: Error occurred while creating a new object: error performing TLS handshake\n", + "untrusted_ca_server_alert": "2026-11-11 16:33:48.704 UTC [134] LOG: could not accept SSL connection: tlsv1 alert unknown ca\n", + "lazy_reload": true, + "query": "[{\"count\":1}]", + "dashboard": { + "version": "v1", + "title": "Postgres smoke", + "author": "8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918", + "dashboardId": "01M75QC0CEB0PTBM5878FAVP92", + "created": "2026-11-11T16:34:30.926914794+00:00", + "modified": "2026-11-11T16:34:30.926914794+00:00", + "tags": null, + "isFavorite": false, + "dashboardType": "Dashboard", + "tiles": [], + "tenantId": null, + "revision": "6ec933632984267e6241812ea5543f0673ce168cff1fd35e5c9a1f850ace5cf4" + }, + "keys": [ + ".parseable/postgres-metastore-id", + "lazy/date=2026-11-11/hour=16/minute=35/6b0aca88cac3b75.data.01M75QEQZRYCNSWYRX3T9X8J74.parquet", + "pstats/date=2026-11-11/hour=16/minute=33/dataset_name=smoke/.data.01M75QCXHFRZYMYAYTQMD83EC3.parquet", + "smoke/date=2026-11-11/hour=16/minute=33/.data.01M75QB2YFJG5RWDGDQ99WAZK6.parquet" + ], + "non_parquet_keys": [ + ".parseable/postgres-metastore-id" + ], + "request_delta": { + "parseable_storage_request_response_time_count{method=\"GET_RANGES\",provider=\"s3\",status=\"200\"}": 5.0 + }, + "observed_requests_during_window_divided_by_queries": 1.0, + "database_outage": { + "readiness": 503, + "request_failed": true, + "seconds": 0.005427954951301217 + }, + "explicit_stream_errors": { + "pending_deletion": 409, + "database_outage": 503 + }, + "bucket_outage_readiness": 503, + "deleted_stream_rows": 0, + "delete_recreate": true, + "final_keys": [ + ".parseable/postgres-metastore-id", + "lazy/date=2026-11-11/hour=16/minute=35/6b0aca88cac3b75.data.01M75QEQZRYCNSWYRX3T9X8J74.parquet", + "pstats/date=2026-11-11/hour=16/minute=33/dataset_name=smoke/.data.01M75QCXHFRZYMYAYTQMD83EC3.parquet", + "pstats/date=2026-11-11/hour=16/minute=35/dataset_name=lazy/6b0aca88cac3b75.data.01M75QGJJR3RGNQQ96RGNFQK0N.parquet", + "smoke/date=2026-11-11/hour=16/minute=36/6b0aca88cac3b75.data.01M75QGJJRRTFFK2GHBC5J9WXQ.parquet" + ], + "interrupted_deletion_restart": [ + { + "step": "before-data", + "hidden": true, + "cold_load_rejected": true, + "http_retry": true, + "recreation": true + }, + { + "step": "after-data", + "hidden": true, + "cold_load_rejected": true, + "http_retry": true, + "recreation": true + } + ] + }, + "postgres": { + "lazy_reload": true, + "query": "[{\"count\":1}]", + "dashboard": { + "version": "v1", + "title": "Postgres smoke", + "author": "8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918", + "dashboardId": "01M75QN9GBN0VPHXE5CC4GK4ZY", + "created": "2026-11-11T16:39:35.179279671+00:00", + "modified": "2026-11-11T16:39:35.179279671+00:00", + "tags": null, + "isFavorite": false, + "dashboardType": "Dashboard", + "tiles": [], + "tenantId": null, + "revision": "3b457f579bb60bc8eecc0709e7b33bc818ffc2e96df084430487d7f090f5101a" + }, + "keys": [ + ".parseable/postgres-metastore-id", + "lazy/date=2026-11-11/hour=16/minute=40/028b583f6797003.data.01M75QQX8G7JYT2K06T248BANP.parquet", + "pstats/date=2026-11-11/hour=16/minute=38/dataset_name=smoke/.data.01M75QP21YY11Q5KTHGP8XN94H.parquet", + "smoke/date=2026-11-11/hour=16/minute=38/.data.01M75QM7EZ784610Z48E8M9VXF.parquet" + ], + "non_parquet_keys": [ + ".parseable/postgres-metastore-id" + ], + "request_delta": { + "parseable_storage_request_response_time_count{method=\"GET_RANGES\",provider=\"s3\",status=\"200\"}": 5.0 + }, + "observed_requests_during_window_divided_by_queries": 1.0, + "database_outage": { + "readiness": 503, + "request_failed": true, + "seconds": 0.004398432793095708 + }, + "explicit_stream_errors": { + "pending_deletion": 409, + "database_outage": 503 + }, + "bucket_outage_readiness": 503, + "deleted_stream_rows": 0, + "delete_recreate": true, + "final_keys": [ + ".parseable/postgres-metastore-id", + "lazy/date=2026-11-11/hour=16/minute=40/028b583f6797003.data.01M75QQX8G7JYT2K06T248BANP.parquet", + "pstats/date=2026-11-11/hour=16/minute=38/dataset_name=smoke/.data.01M75QP21YY11Q5KTHGP8XN94H.parquet", + "pstats/date=2026-11-11/hour=16/minute=40/dataset_name=lazy/028b583f6797003.data.01M75QSQVGB4N665G100P7TE8F.parquet", + "smoke/date=2026-11-11/hour=16/minute=43/028b583f6797003.data.01M75QXD1G8Z00Q5V5QP6NQXBH.parquet" + ], + "interrupted_deletion_restart": [ + { + "step": "before-data", + "hidden": true, + "cold_load_rejected": true, + "http_retry": true, + "recreation": true + }, + { + "step": "after-data", + "hidden": true, + "cold_load_rejected": true, + "http_retry": true, + "recreation": true + } + ] + }, + "startup_rejections": { + "future-schema": "Error: ObjectStorageError: PostgreSQL metastore unavailable: unsupported schema/protocol version; upgrade this server\n\nCaused by:\n PostgreSQL metastore unavailable: unsupported schema/protocol version; upgrade this server\n", + "wrong-id": "Error: ObjectStorageError: PostgreSQL metastore unavailable: backend/bucket identity mismatch\n\nCaused by:\n PostgreSQL metastore unavailable: backend/bucket identity mismatch\n", + "unreachable": "Error: ObjectStorageError: PostgreSQL metastore unavailable: Error occurred while creating a new object: error connecting to server\n\nCaused by:\n PostgreSQL metastore unavailable: Error occurred while creating a new object: error connecting to server\n" + }, + "postgres_rows": "tenant | kind | key | version | bytes \n--------+----------+--------------------------------------------------------------------------------------------------------------------+---------+-------\n | document | .parseable/.parseable.json | 150 | 384\n | document | .parseable/ingestor.028b583f6797003.json | 39 | 241\n | document | .parseable/querier.ec99af452c30213.json | 152 | 240\n | document | .users/.dashboard-locations/01M75QN9GBN0VPHXE5CC4GK4ZY.json | 16 | 119\n | document | .users/.dashboard-titles/571a2cefe2d0d1a9a26dab4810a5556875c7d7531d4f558a8f7b28ab486e2395.json | 18 | 170\n | document | .users/8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918/dashboards/01M75QN9GBN0VPHXE5CC4GK4ZY.json | 17 | 545\n | schema | lazy/.stream/.ingestor.028b583f6797003.schema | 56 | 585\n | stream | lazy/.stream/.ingestor.028b583f6797003.stream.json | 54 | 853\n | schema | lazy/.stream/.schema | 157 | 585\n | stream | lazy/.stream/.stream.json | 57 | 597\n | manifest | lazy/date=2026-11-11/ingestor..028b583f6797003.manifest.json | 52 | 420\n | schema | pstats/.stream/.ingestor.028b583f6797003.schema | 113 | 1117\n | stream | pstats/.stream/.ingestor.028b583f6797003.stream.json | 140 | 895\n | schema | pstats/.stream/.schema | 160 | 1117\n | stream | pstats/.stream/.stream.json | 23 | 867\n | manifest | pstats/date=2026-11-11/.manifest.json | 21 | 551\n | manifest | pstats/date=2026-11-11/ingestor..028b583f6797003.manifest.json | 138 | 671\n | schema | smoke/.stream/.ingestor.028b583f6797003.schema | 170 | 27\n | stream | smoke/.stream/.ingestor.028b583f6797003.stream.json | 171 | 597\n | schema | smoke/.stream/.schema | 168 | 27\n | stream | smoke/.stream/.stream.json | 169 | 597\n(21 rows)", + "commands": [ + { + "argv": [ + "docker", + "run", + "-d", + "--name", + "parseable-pg-smoke-1309689-0", + "-e", + "POSTGRES_PASSWORD=smoke", + "-e", + "POSTGRES_DB=parseable", + "-p", + "127.0.0.1:50913:5432", + "postgres:17-alpine" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "run", + "-d", + "--name", + "parseable-pg-smoke-1309689-1", + "-e", + "MINIO_ROOT_USER=minioadmin", + "-e", + "MINIO_ROOT_PASSWORD=minioadmin", + "-p", + "127.0.0.1:56891:9000", + "quay.io/parseablehq/minio:edge", + "server", + "/data" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "pg_isready", + "-h", + "127.0.0.1", + "-U", + "postgres" + ], + "exit_code": 2 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "pg_isready", + "-h", + "127.0.0.1", + "-U", + "postgres" + ], + "exit_code": 2 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "pg_isready", + "-h", + "127.0.0.1", + "-U", + "postgres" + ], + "exit_code": 2 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "pg_isready", + "-h", + "127.0.0.1", + "-U", + "postgres" + ], + "exit_code": 0 + }, + { + "argv": [ + "openssl", + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-keyout", + ".postgres-smoke/tls/ca.key", + "-out", + ".postgres-smoke/tls/ca.crt", + "-days", + "2", + "-subj", + "/CN=Smoke CA" + ], + "exit_code": 0 + }, + { + "argv": [ + "openssl", + "req", + "-newkey", + "rsa:2048", + "-nodes", + "-keyout", + ".postgres-smoke/tls/server.key", + "-out", + ".postgres-smoke/tls/server.csr", + "-subj", + "/CN=localhost" + ], + "exit_code": 0 + }, + { + "argv": [ + "openssl", + "x509", + "-req", + "-in", + ".postgres-smoke/tls/server.csr", + "-CA", + ".postgres-smoke/tls/ca.crt", + "-CAkey", + ".postgres-smoke/tls/ca.key", + "-CAcreateserial", + "-out", + ".postgres-smoke/tls/server.crt", + "-days", + "2", + "-extfile", + ".postgres-smoke/tls/extensions" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "cp", + ".postgres-smoke/tls/server.key", + "parseable-pg-smoke-1309689-0:/var/lib/postgresql/data/server.key" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "cp", + ".postgres-smoke/tls/server.crt", + "parseable-pg-smoke-1309689-0:/var/lib/postgresql/data/server.crt" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "sh", + "-c", + "chown postgres:postgres /var/lib/postgresql/data/server.* && chmod 600 /var/lib/postgresql/data/server.key" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-c", + "ALTER SYSTEM SET ssl='on';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-c", + "SELECT pg_reload_conf();" + ], + "exit_code": 0 + }, + { + "argv": [ + "git", + "rev-parse", + "HEAD" + ], + "exit_code": 0 + }, + { + "argv": [ + "git", + "diff", + "HEAD" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "image", + "inspect", + "postgres:17-alpine" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "image", + "inspect", + "quay.io/parseablehq/minio:edge" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-1", + "mkdir", + "-p", + "/data/smoke-object-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "stop", + "parseable-pg-smoke-1309689-1" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "start", + "parseable-pg-smoke-1309689-1" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-1", + "mkdir", + "-p", + "/data/smoke-postgres-tls" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-c", + "CREATE DATABASE tls;" + ], + "exit_code": 0 + }, + { + "argv": [ + "openssl", + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-keyout", + ".postgres-smoke/tls/wrong.key", + "-out", + ".postgres-smoke/tls/wrong.crt", + "-days", + "2", + "-subj", + "/CN=Wrong CA" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "logs", + "parseable-pg-smoke-1309689-0" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "logs", + "parseable-pg-smoke-1309689-0" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-v", + "ON_ERROR_STOP=1", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "INSERT INTO parseable_documents(tenant,key,kind,body) VALUES('','deletion-explicit/.stream/.deleting','document',''::bytea);" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-v", + "ON_ERROR_STOP=1", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "DELETE FROM parseable_documents WHERE tenant='' AND key='deletion-explicit/.stream/.deleting';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "stop", + "parseable-pg-smoke-1309689-0" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "start", + "parseable-pg-smoke-1309689-0" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "stop", + "parseable-pg-smoke-1309689-1" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "start", + "parseable-pg-smoke-1309689-1" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT count(*) FROM parseable_documents WHERE starts_with(key,'smoke/');" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-v", + "ON_ERROR_STOP=1", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "INSERT INTO parseable_documents(tenant,key,kind,body) VALUES('','smoke/.stream/.deleting','document',''::bytea);" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-v", + "ON_ERROR_STOP=1", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT count(*) FROM parseable_documents WHERE starts_with(key,'smoke/');" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-v", + "ON_ERROR_STOP=1", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "CREATE FUNCTION smoke_delete_fault() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'injected row deletion failure'; END $$; CREATE TRIGGER smoke_delete_fault BEFORE DELETE ON parseable_documents FOR EACH ROW EXECUTE FUNCTION smoke_delete_fault();" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-v", + "ON_ERROR_STOP=1", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "DROP TRIGGER smoke_delete_fault ON parseable_documents; DROP FUNCTION smoke_delete_fault();" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-v", + "ON_ERROR_STOP=1", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT count(*) FROM parseable_documents WHERE starts_with(key,'smoke/');" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-1", + "mkdir", + "-p", + "/data/smoke-postgres" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-v", + "ON_ERROR_STOP=1", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "INSERT INTO parseable_documents(tenant,key,kind,body) VALUES('','deletion-explicit/.stream/.deleting','document',''::bytea);" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-v", + "ON_ERROR_STOP=1", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "DELETE FROM parseable_documents WHERE tenant='' AND key='deletion-explicit/.stream/.deleting';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "stop", + "parseable-pg-smoke-1309689-0" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "start", + "parseable-pg-smoke-1309689-0" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "stop", + "parseable-pg-smoke-1309689-1" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "start", + "parseable-pg-smoke-1309689-1" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT count(*) FROM parseable_documents WHERE starts_with(key,'smoke/');" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-v", + "ON_ERROR_STOP=1", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "INSERT INTO parseable_documents(tenant,key,kind,body) VALUES('','smoke/.stream/.deleting','document',''::bytea);" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-v", + "ON_ERROR_STOP=1", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT count(*) FROM parseable_documents WHERE starts_with(key,'smoke/');" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-v", + "ON_ERROR_STOP=1", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "CREATE FUNCTION smoke_delete_fault() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'injected row deletion failure'; END $$; CREATE TRIGGER smoke_delete_fault BEFORE DELETE ON parseable_documents FOR EACH ROW EXECUTE FUNCTION smoke_delete_fault();" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-v", + "ON_ERROR_STOP=1", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "DROP TRIGGER smoke_delete_fault ON parseable_documents; DROP FUNCTION smoke_delete_fault();" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-v", + "ON_ERROR_STOP=1", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT count(*) FROM parseable_documents WHERE starts_with(key,'smoke/');" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-c", + "UPDATE parseable_backend SET schema_version=999;" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-c", + "UPDATE parseable_backend SET schema_version=3;" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-c", + "SELECT tenant,kind,key,version,octet_length(body) AS bytes FROM parseable_documents ORDER BY tenant,key;" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "rm", + "-f", + "parseable-pg-smoke-1309689-1" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "rm", + "-f", + "parseable-pg-smoke-1309689-0" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": -9 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1 + } + ], + "success": true, + "http_requests": [ + { + "path": "/liveness", + "method": "GET", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "PUT", + "status": 200 + }, + { + "path": "/ingest", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/logstream/smoke/retention", + "method": "PUT", + "status": 200 + }, + { + "path": "/dashboards", + "method": "POST", + "status": 200 + }, + { + "path": "/liveness", + "method": "GET", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/dashboards", + "method": "GET", + "status": 200 + }, + { + "path": "/metrics", + "method": "GET", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/metrics", + "method": "GET", + "status": 200 + }, + { + "path": "/readiness", + "method": "GET", + "status": 503 + }, + { + "path": "/readiness", + "method": "GET", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "DELETE", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "PUT", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/liveness", + "method": "GET", + "status": 200 + }, + { + "path": "/liveness", + "method": "GET", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "PUT", + "status": 200 + }, + { + "path": "/ingest", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/logstream/smoke/retention", + "method": "PUT", + "status": 200 + }, + { + "path": "/dashboards", + "method": "POST", + "status": 200 + }, + { + "path": "/liveness", + "method": "GET", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/dashboards", + "method": "GET", + "status": 200 + }, + { + "path": "/metrics", + "method": "GET", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/metrics", + "method": "GET", + "status": 200 + }, + { + "path": "/readiness", + "method": "GET", + "status": 503 + }, + { + "path": "/logstream/outage", + "method": "PUT", + "status": 503 + }, + { + "path": "/readiness", + "method": "GET", + "status": 503 + }, + { + "path": "/readiness", + "method": "GET", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/readiness", + "method": "GET", + "status": 503 + }, + { + "path": "/readiness", + "method": "GET", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "DELETE", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "PUT", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/liveness", + "method": "GET", + "status": 200 + }, + { + "path": "/logstream", + "method": "GET", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 400 + }, + { + "path": "/logstream/smoke", + "method": "DELETE", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "PUT", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "DELETE", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "PUT", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "DELETE", + "status": 503 + }, + { + "path": "/liveness", + "method": "GET", + "status": 200 + }, + { + "path": "/logstream", + "method": "GET", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 400 + }, + { + "path": "/logstream/smoke", + "method": "DELETE", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "PUT", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "DELETE", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "PUT", + "status": 200 + }, + { + "path": "/liveness", + "method": "GET", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "PUT", + "status": 200 + }, + { + "path": "/ingest", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/logstream/smoke/retention", + "method": "PUT", + "status": 200 + }, + { + "path": "/dashboards", + "method": "POST", + "status": 200 + }, + { + "path": "/liveness", + "method": "GET", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/dashboards", + "method": "GET", + "status": 200 + }, + { + "path": "/metrics", + "method": "GET", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/metrics", + "method": "GET", + "status": 200 + }, + { + "path": "/readiness", + "method": "GET", + "status": 503 + }, + { + "path": "/logstream/outage", + "method": "PUT", + "status": 503 + }, + { + "path": "/readiness", + "method": "GET", + "status": 503 + }, + { + "path": "/readiness", + "method": "GET", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/readiness", + "method": "GET", + "status": 503 + }, + { + "path": "/readiness", + "method": "GET", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "DELETE", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "PUT", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/liveness", + "method": "GET", + "status": 200 + }, + { + "path": "/logstream", + "method": "GET", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 400 + }, + { + "path": "/logstream/smoke", + "method": "DELETE", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "PUT", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "DELETE", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "PUT", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "DELETE", + "status": 503 + }, + { + "path": "/liveness", + "method": "GET", + "status": 200 + }, + { + "path": "/logstream", + "method": "GET", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 400 + }, + { + "path": "/logstream/smoke", + "method": "DELETE", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "PUT", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "DELETE", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "PUT", + "status": 200 + } + ] + }, + { + "requested_modes": [ + "object-store", + "postgres-tls", + "postgres" + ], + "source_commit": "b96259b05238bae99ca48d5bee3cee9943b399d5", + "binary_sha256": "b30aa28e2dac81b076dbe41a11bc8d686c3b9ea3c6490d3471682a507def0d39", + "source_diff_sha256": "03b1e883545353e632d87207d72bb644a88731f4761ca4494fa9c40e47277cec", + "image_digests": { + "postgres:17-alpine": [ + "postgres@sha256:b0f9560a2de083e2cc7382e75f808c7381a32852a7ec49117deedb300e552b24" + ], + "quay.io/parseablehq/minio:edge": [ + "quay.io/parseablehq/minio@sha256:88276d0a028d758ffaf80d65dd80071386e91b4114251a3dda101eadf000faeb" + ] + }, + "measurement": "Node-local storage counter delta during five queries, including background traffic; not isolated requests per query.", + "object-store": { + "lazy_reload": true, + "query": "[{\"count\":1}]", + "dashboard": { + "version": "v1", + "title": "Postgres smoke", + "author": "8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918", + "dashboardId": "01M75JNVK6AYE1WXADG6JA3DQZ", + "created": "2026-11-11T15:12:30.822910300+00:00", + "modified": "2026-11-11T15:12:30.822910300+00:00", + "tags": null, + "isFavorite": false, + "dashboardType": "Dashboard", + "tiles": [], + "tenantId": null, + "revision": "2e2d1ea944667dfde6dab5d44c0d29b0768305f9059cbed7133c0406f4bea6c4" + }, + "keys": [ + ".parseable/.parseable.json", + ".parseable/ingestor.a878d4a37bf1656.json", + ".parseable/querier.c1530d7252e7658.json", + ".users/.dashboard-locations/01M75JNVK6AYE1WXADG6JA3DQZ.json", + ".users/.dashboard-titles/571a2cefe2d0d1a9a26dab4810a5556875c7d7531d4f558a8f7b28ab486e2395.json", + ".users/8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918/dashboards/01M75JNVK6AYE1WXADG6JA3DQZ.json", + "lazy/.stream/.ingestor.a878d4a37bf1656.schema", + "lazy/.stream/.ingestor.a878d4a37bf1656.stream.json", + "lazy/.stream/.schema", + "lazy/.stream/.stream.json", + "lazy/date=2026-11-11/hour=15/minute=13/a878d4a37bf1656.data.01M75JRJYKT089B5QZ24C9WCDG.parquet", + "lazy/date=2026-11-11/ingestor..a878d4a37bf1656.manifest.json", + "pstats/.stream/.ingestor.a878d4a37bf1656.schema", + "pstats/.stream/.ingestor.a878d4a37bf1656.stream.json", + "pstats/.stream/.schema", + "pstats/.stream/.stream.json", + "pstats/date=2026-11-11/.manifest.json", + "pstats/date=2026-11-11/hour=15/minute=11/dataset_name=smoke/.data.01M75JPRNSZ6BDJG6PFGB2QWT2.parquet", + "smoke/.stream/.ingestor.a878d4a37bf1656.schema", + "smoke/.stream/.ingestor.a878d4a37bf1656.stream.json", + "smoke/.stream/.schema", + "smoke/.stream/.stream.json", + "smoke/date=2026-11-11/.manifest.json", + "smoke/date=2026-11-11/hour=15/minute=11/.data.01M75JMY2SDDNVZR0S50MDPK9Y.parquet" + ], + "non_parquet_keys": [ + ".parseable/.parseable.json", + ".parseable/ingestor.a878d4a37bf1656.json", + ".parseable/querier.c1530d7252e7658.json", + ".users/.dashboard-locations/01M75JNVK6AYE1WXADG6JA3DQZ.json", + ".users/.dashboard-titles/571a2cefe2d0d1a9a26dab4810a5556875c7d7531d4f558a8f7b28ab486e2395.json", + ".users/8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918/dashboards/01M75JNVK6AYE1WXADG6JA3DQZ.json", + "lazy/.stream/.ingestor.a878d4a37bf1656.schema", + "lazy/.stream/.ingestor.a878d4a37bf1656.stream.json", + "lazy/.stream/.schema", + "lazy/.stream/.stream.json", + "lazy/date=2026-11-11/ingestor..a878d4a37bf1656.manifest.json", + "pstats/.stream/.ingestor.a878d4a37bf1656.schema", + "pstats/.stream/.ingestor.a878d4a37bf1656.stream.json", + "pstats/.stream/.schema", + "pstats/.stream/.stream.json", + "pstats/date=2026-11-11/.manifest.json", + "smoke/.stream/.ingestor.a878d4a37bf1656.schema", + "smoke/.stream/.ingestor.a878d4a37bf1656.stream.json", + "smoke/.stream/.schema", + "smoke/.stream/.stream.json", + "smoke/date=2026-11-11/.manifest.json" + ], + "request_delta": { + "parseable_storage_request_response_time_count{method=\"GET\",provider=\"s3\",status=\"200\"}": 30.0, + "parseable_storage_request_response_time_count{method=\"GET_RANGES\",provider=\"s3\",status=\"200\"}": 5.0, + "parseable_storage_request_response_time_count{method=\"LIST\",provider=\"s3\",status=\"200\"}": 5.0, + "parseable_storage_request_response_time_count{method=\"LIST_DELIM\",provider=\"s3\",status=\"200\"}": 5.0 + }, + "observed_requests_during_window_divided_by_queries": 9.0, + "bucket_outage_readiness": 503, + "delete_recreate": true, + "final_keys": [ + ".parseable/.parseable.json", + ".parseable/ingestor.a878d4a37bf1656.json", + ".parseable/querier.c1530d7252e7658.json", + ".users/.dashboard-locations/01M75JNVK6AYE1WXADG6JA3DQZ.json", + ".users/.dashboard-titles/571a2cefe2d0d1a9a26dab4810a5556875c7d7531d4f558a8f7b28ab486e2395.json", + ".users/8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918/dashboards/01M75JNVK6AYE1WXADG6JA3DQZ.json", + "lazy/.stream/.ingestor.a878d4a37bf1656.schema", + "lazy/.stream/.ingestor.a878d4a37bf1656.stream.json", + "lazy/.stream/.schema", + "lazy/.stream/.stream.json", + "lazy/date=2026-11-11/hour=15/minute=13/a878d4a37bf1656.data.01M75JRJYKT089B5QZ24C9WCDG.parquet", + "lazy/date=2026-11-11/ingestor..a878d4a37bf1656.manifest.json", + "pstats/.stream/.ingestor.a878d4a37bf1656.schema", + "pstats/.stream/.ingestor.a878d4a37bf1656.stream.json", + "pstats/.stream/.schema", + "pstats/.stream/.stream.json", + "pstats/date=2026-11-11/.manifest.json", + "pstats/date=2026-11-11/hour=15/minute=11/dataset_name=smoke/.data.01M75JPRNSZ6BDJG6PFGB2QWT2.parquet", + "pstats/date=2026-11-11/hour=15/minute=13/dataset_name=lazy/a878d4a37bf1656.data.01M75JTDHJA1V5QTMHR68NFYFX.parquet", + "pstats/date=2026-11-11/ingestor..a878d4a37bf1656.manifest.json", + "smoke/.stream/.ingestor.a878d4a37bf1656.schema", + "smoke/.stream/.ingestor.a878d4a37bf1656.stream.json", + "smoke/.stream/.schema", + "smoke/.stream/.stream.json", + "smoke/date=2026-11-11/hour=15/minute=14/a878d4a37bf1656.data.01M75JTDHJCXR3RTGPN9ARG161.parquet", + "smoke/date=2026-11-11/ingestor..a878d4a37bf1656.manifest.json" + ] + }, + "postgres-tls": { + "trusted_ca_restart": true, + "untrusted_ca_rejected": true, + "untrusted_ca_output": "Error: ObjectStorageError: PostgreSQL metastore unavailable: Error occurred while creating a new object: error performing TLS handshake\n\nCaused by:\n PostgreSQL metastore unavailable: Error occurred while creating a new object: error performing TLS handshake\n", + "untrusted_ca_server_alert": "2026-11-11 15:15:48.818 UTC [131] LOG: could not accept SSL connection: tlsv1 alert unknown ca\n", + "lazy_reload": true, + "query": "[{\"count\":1}]", + "dashboard": { + "version": "v1", + "title": "Postgres smoke", + "author": "8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918", + "dashboardId": "01M75JX776RMF67VDZC03QBV80", + "created": "2026-11-11T15:16:32.102431645+00:00", + "modified": "2026-11-11T15:16:32.102431645+00:00", + "tags": null, + "isFavorite": false, + "dashboardType": "Dashboard", + "tiles": [], + "tenantId": null, + "revision": "f4a0456e421a611f22c150d81cf0b2350c36772618d218293ffe1d12c8c6f5ec" + }, + "keys": [ + ".parseable/postgres-metastore-id", + "lazy/date=2026-11-11/hour=15/minute=18/8ed27ce105d0786.data.01M75K1RM89ZBQZG7NXGCMKY59.parquet", + "pstats/date=2026-11-11/hour=15/minute=15/dataset_name=smoke/.data.01M75JY3CJR1NXRYQPC06KFWWT.parquet", + "smoke/date=2026-11-11/hour=15/minute=15/.data.01M75JW8SK0YDV8R9GHVTGQPAZ.parquet" + ], + "non_parquet_keys": [ + ".parseable/postgres-metastore-id" + ], + "request_delta": { + "parseable_storage_request_response_time_count{method=\"GET_RANGES\",provider=\"s3\",status=\"200\"}": 5.0 + }, + "observed_requests_during_window_divided_by_queries": 1.0, + "database_outage": { + "readiness": 503, + "request_failed": true, + "seconds": 0.006427645916119218 + }, + "bucket_outage_readiness": 503, + "deleted_stream_rows": 0, + "delete_recreate": true, + "final_keys": [ + ".parseable/postgres-metastore-id", + "lazy/date=2026-11-11/hour=15/minute=18/8ed27ce105d0786.data.01M75K1RM89ZBQZG7NXGCMKY59.parquet", + "pstats/date=2026-11-11/hour=15/minute=15/dataset_name=smoke/.data.01M75JY3CJR1NXRYQPC06KFWWT.parquet", + "pstats/date=2026-11-11/hour=15/minute=18/dataset_name=lazy/8ed27ce105d0786.data.01M75K3K788KV2CVEGMCSFSZF5.parquet", + "smoke/date=2026-11-11/hour=15/minute=19/8ed27ce105d0786.data.01M75K3K78967A00SJ62H7ZNT4.parquet" + ], + "interrupted_deletion_restart": [ + { + "step": "before-data", + "hidden": true, + "cold_load_rejected": true, + "http_retry": true, + "recreation": true + }, + { + "step": "after-data", + "hidden": true, + "cold_load_rejected": true, + "http_retry": true, + "recreation": true + } + ] + }, + "postgres": { + "lazy_reload": true, + "query": "[{\"count\":1}]", + "dashboard": { + "version": "v1", + "title": "Postgres smoke", + "author": "8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918", + "dashboardId": "01M75KA0K4HTDY1R3F6TS6F4XM", + "created": "2026-11-11T15:23:31.300938938+00:00", + "modified": "2026-11-11T15:23:31.300938938+00:00", + "tags": null, + "isFavorite": false, + "dashboardType": "Dashboard", + "tiles": [], + "tenantId": null, + "revision": "d3245c653cbb43fc182dcb0fcd1f52857cc95f9557bd80ee994c5504fdfb5bb9" + }, + "keys": [ + ".parseable/postgres-metastore-id", + "lazy/date=2026-11-11/hour=15/minute=24/da8518511ee84c0.data.01M75KCQV2HKAMSQCGK79BRZTR.parquet", + "pstats/date=2026-11-11/hour=15/minute=22/dataset_name=smoke/.data.01M75KAX7FTN48CQVS8XXY0M31.parquet", + "smoke/date=2026-11-11/hour=15/minute=22/.data.01M75K92MFMGVRDR8V49HY99V6.parquet" + ], + "non_parquet_keys": [ + ".parseable/postgres-metastore-id" + ], + "request_delta": { + "parseable_storage_request_response_time_count{method=\"GET_RANGES\",provider=\"s3\",status=\"200\"}": 5.0 + }, + "observed_requests_during_window_divided_by_queries": 1.0, + "database_outage": { + "readiness": 503, + "request_failed": true, + "seconds": 0.006476623937487602 + }, + "bucket_outage_readiness": 503, + "deleted_stream_rows": 0, + "delete_recreate": true, + "final_keys": [ + ".parseable/postgres-metastore-id", + "lazy/date=2026-11-11/hour=15/minute=24/da8518511ee84c0.data.01M75KCQV2HKAMSQCGK79BRZTR.parquet", + "pstats/date=2026-11-11/hour=15/minute=22/dataset_name=smoke/.data.01M75KAX7FTN48CQVS8XXY0M31.parquet", + "pstats/date=2026-11-11/hour=15/minute=24/dataset_name=lazy/da8518511ee84c0.data.01M75KEMZ0DJ8K16GNDD13EVJ1.parquet", + "smoke/date=2026-11-11/hour=15/minute=27/da8518511ee84c0.data.01M75KJ7M27Q4HM890XEA3QRMX.parquet" + ], + "interrupted_deletion_restart": [ + { + "step": "before-data", + "hidden": true, + "cold_load_rejected": true, + "http_retry": true, + "recreation": true + }, + { + "step": "after-data", + "hidden": true, + "cold_load_rejected": true, + "http_retry": true, + "recreation": true + } + ] + }, + "startup_rejections": { + "future-schema": "Error: ObjectStorageError: PostgreSQL metastore unavailable: unsupported schema/protocol version; upgrade this server\n\nCaused by:\n PostgreSQL metastore unavailable: unsupported schema/protocol version; upgrade this server\n", + "wrong-id": "Error: ObjectStorageError: PostgreSQL metastore unavailable: backend/bucket identity mismatch\n\nCaused by:\n PostgreSQL metastore unavailable: backend/bucket identity mismatch\n", + "unreachable": "Error: ObjectStorageError: PostgreSQL metastore unavailable: Error occurred while creating a new object: error connecting to server\n\nCaused by:\n PostgreSQL metastore unavailable: Error occurred while creating a new object: error connecting to server\n" + }, + "postgres_rows": "tenant | kind | key | version | bytes \n--------+----------+--------------------------------------------------------------------------------------------------------------------+---------+-------\n | document | .parseable/.parseable.json | 121 | 383\n | document | .parseable/ingestor.da8518511ee84c0.json | 39 | 241\n | document | .parseable/querier.d960519648dca85.json | 123 | 240\n | document | .users/.dashboard-locations/01M75KA0K4HTDY1R3F6TS6F4XM.json | 16 | 119\n | document | .users/.dashboard-titles/571a2cefe2d0d1a9a26dab4810a5556875c7d7531d4f558a8f7b28ab486e2395.json | 18 | 170\n | document | .users/8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918/dashboards/01M75KA0K4HTDY1R3F6TS6F4XM.json | 17 | 545\n | schema | lazy/.stream/.ingestor.da8518511ee84c0.schema | 56 | 585\n | stream | lazy/.stream/.ingestor.da8518511ee84c0.stream.json | 54 | 853\n | schema | lazy/.stream/.schema | 131 | 585\n | stream | lazy/.stream/.stream.json | 57 | 597\n | manifest | lazy/date=2026-11-11/ingestor..da8518511ee84c0.manifest.json | 52 | 418\n | schema | pstats/.stream/.ingestor.da8518511ee84c0.schema | 148 | 1117\n | stream | pstats/.stream/.ingestor.da8518511ee84c0.stream.json | 146 | 898\n | schema | pstats/.stream/.schema | 127 | 1117\n | stream | pstats/.stream/.stream.json | 23 | 867\n | manifest | pstats/date=2026-11-11/.manifest.json | 21 | 563\n | manifest | pstats/date=2026-11-11/ingestor..da8518511ee84c0.manifest.json | 144 | 790\n | schema | smoke/.stream/.ingestor.da8518511ee84c0.schema | 141 | 27\n | stream | smoke/.stream/.ingestor.da8518511ee84c0.stream.json | 142 | 597\n | schema | smoke/.stream/.schema | 139 | 27\n | stream | smoke/.stream/.stream.json | 140 | 597\n(21 rows)", + "commands": [ + { + "argv": [ + "docker", + "run", + "-d", + "--name", + "parseable-pg-smoke-804785-0", + "-e", + "POSTGRES_PASSWORD=smoke", + "-e", + "POSTGRES_DB=parseable", + "-p", + "127.0.0.1:49281:5432", + "postgres:17-alpine" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "run", + "-d", + "--name", + "parseable-pg-smoke-804785-1", + "-e", + "MINIO_ROOT_USER=minioadmin", + "-e", + "MINIO_ROOT_PASSWORD=minioadmin", + "-p", + "127.0.0.1:38415:9000", + "quay.io/parseablehq/minio:edge", + "server", + "/data" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "pg_isready", + "-h", + "127.0.0.1", + "-U", + "postgres" + ], + "exit_code": 2 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "pg_isready", + "-h", + "127.0.0.1", + "-U", + "postgres" + ], + "exit_code": 2 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "pg_isready", + "-h", + "127.0.0.1", + "-U", + "postgres" + ], + "exit_code": 2 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "pg_isready", + "-h", + "127.0.0.1", + "-U", + "postgres" + ], + "exit_code": 0 + }, + { + "argv": [ + "openssl", + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-keyout", + ".postgres-smoke/tls/ca.key", + "-out", + ".postgres-smoke/tls/ca.crt", + "-days", + "2", + "-subj", + "/CN=Smoke CA" + ], + "exit_code": 0 + }, + { + "argv": [ + "openssl", + "req", + "-newkey", + "rsa:2048", + "-nodes", + "-keyout", + ".postgres-smoke/tls/server.key", + "-out", + ".postgres-smoke/tls/server.csr", + "-subj", + "/CN=localhost" + ], + "exit_code": 0 + }, + { + "argv": [ + "openssl", + "x509", + "-req", + "-in", + ".postgres-smoke/tls/server.csr", + "-CA", + ".postgres-smoke/tls/ca.crt", + "-CAkey", + ".postgres-smoke/tls/ca.key", + "-CAcreateserial", + "-out", + ".postgres-smoke/tls/server.crt", + "-days", + "2", + "-extfile", + ".postgres-smoke/tls/extensions" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "cp", + ".postgres-smoke/tls/server.key", + "parseable-pg-smoke-804785-0:/var/lib/postgresql/data/server.key" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "cp", + ".postgres-smoke/tls/server.crt", + "parseable-pg-smoke-804785-0:/var/lib/postgresql/data/server.crt" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "sh", + "-c", + "chown postgres:postgres /var/lib/postgresql/data/server.* && chmod 600 /var/lib/postgresql/data/server.key" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-c", + "ALTER SYSTEM SET ssl='on';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-c", + "SELECT pg_reload_conf();" + ], + "exit_code": 0 + }, + { + "argv": [ + "git", + "rev-parse", + "HEAD" + ], + "exit_code": 0 + }, + { + "argv": [ + "git", + "diff", + "HEAD" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "image", + "inspect", + "postgres:17-alpine" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "image", + "inspect", + "quay.io/parseablehq/minio:edge" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-1", + "mkdir", + "-p", + "/data/smoke-object-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "stop", + "parseable-pg-smoke-804785-1" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "start", + "parseable-pg-smoke-804785-1" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-1", + "mkdir", + "-p", + "/data/smoke-postgres-tls" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-c", + "CREATE DATABASE tls;" + ], + "exit_code": 0 + }, + { + "argv": [ + "openssl", + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-keyout", + ".postgres-smoke/tls/wrong.key", + "-out", + ".postgres-smoke/tls/wrong.crt", + "-days", + "2", + "-subj", + "/CN=Wrong CA" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "logs", + "parseable-pg-smoke-804785-0" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "logs", + "parseable-pg-smoke-804785-0" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "stop", + "parseable-pg-smoke-804785-0" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "start", + "parseable-pg-smoke-804785-0" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "stop", + "parseable-pg-smoke-804785-1" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "start", + "parseable-pg-smoke-804785-1" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT count(*) FROM parseable_documents WHERE starts_with(key,'smoke/');" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-v", + "ON_ERROR_STOP=1", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "INSERT INTO parseable_documents(tenant,key,kind,body) VALUES('','smoke/.stream/.deleting','document',''::bytea);" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-v", + "ON_ERROR_STOP=1", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT count(*) FROM parseable_documents WHERE starts_with(key,'smoke/');" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-v", + "ON_ERROR_STOP=1", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "CREATE FUNCTION smoke_delete_fault() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'injected row deletion failure'; END $$; CREATE TRIGGER smoke_delete_fault BEFORE DELETE ON parseable_documents FOR EACH ROW EXECUTE FUNCTION smoke_delete_fault();" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-v", + "ON_ERROR_STOP=1", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "DROP TRIGGER smoke_delete_fault ON parseable_documents; DROP FUNCTION smoke_delete_fault();" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-v", + "ON_ERROR_STOP=1", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT count(*) FROM parseable_documents WHERE starts_with(key,'smoke/');" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-1", + "mkdir", + "-p", + "/data/smoke-postgres" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "stop", + "parseable-pg-smoke-804785-0" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "start", + "parseable-pg-smoke-804785-0" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "stop", + "parseable-pg-smoke-804785-1" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "start", + "parseable-pg-smoke-804785-1" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT count(*) FROM parseable_documents WHERE starts_with(key,'smoke/');" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-v", + "ON_ERROR_STOP=1", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "INSERT INTO parseable_documents(tenant,key,kind,body) VALUES('','smoke/.stream/.deleting','document',''::bytea);" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-v", + "ON_ERROR_STOP=1", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT count(*) FROM parseable_documents WHERE starts_with(key,'smoke/');" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-v", + "ON_ERROR_STOP=1", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "CREATE FUNCTION smoke_delete_fault() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'injected row deletion failure'; END $$; CREATE TRIGGER smoke_delete_fault BEFORE DELETE ON parseable_documents FOR EACH ROW EXECUTE FUNCTION smoke_delete_fault();" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-v", + "ON_ERROR_STOP=1", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "DROP TRIGGER smoke_delete_fault ON parseable_documents; DROP FUNCTION smoke_delete_fault();" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-v", + "ON_ERROR_STOP=1", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT count(*) FROM parseable_documents WHERE starts_with(key,'smoke/');" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-c", + "UPDATE parseable_backend SET schema_version=999;" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-c", + "UPDATE parseable_backend SET schema_version=3;" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-c", + "SELECT tenant,kind,key,version,octet_length(body) AS bytes FROM parseable_documents ORDER BY tenant,key;" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "rm", + "-f", + "parseable-pg-smoke-804785-1" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "rm", + "-f", + "parseable-pg-smoke-804785-0" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1 + } + ], + "success": true, + "http_requests": [ + { + "path": "/liveness", + "method": "GET", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "PUT", + "status": 200 + }, + { + "path": "/ingest", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/logstream/smoke/retention", + "method": "PUT", + "status": 200 + }, + { + "path": "/dashboards", + "method": "POST", + "status": 200 + }, + { + "path": "/liveness", + "method": "GET", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/dashboards", + "method": "GET", + "status": 200 + }, + { + "path": "/metrics", + "method": "GET", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/metrics", + "method": "GET", + "status": 200 + }, + { + "path": "/readiness", + "method": "GET", + "status": 503 + }, + { + "path": "/readiness", + "method": "GET", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "DELETE", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "PUT", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/liveness", + "method": "GET", + "status": 200 + }, + { + "path": "/liveness", + "method": "GET", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "PUT", + "status": 200 + }, + { + "path": "/ingest", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/logstream/smoke/retention", + "method": "PUT", + "status": 200 + }, + { + "path": "/dashboards", + "method": "POST", + "status": 200 + }, + { + "path": "/liveness", + "method": "GET", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/dashboards", + "method": "GET", + "status": 200 + }, + { + "path": "/metrics", + "method": "GET", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/metrics", + "method": "GET", + "status": 200 + }, + { + "path": "/readiness", + "method": "GET", + "status": 503 + }, + { + "path": "/logstream/outage", + "method": "PUT", + "status": 503 + }, + { + "path": "/readiness", + "method": "GET", + "status": 503 + }, + { + "path": "/readiness", + "method": "GET", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/readiness", + "method": "GET", + "status": 503 + }, + { + "path": "/readiness", + "method": "GET", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "DELETE", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "PUT", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/liveness", + "method": "GET", + "status": 200 + }, + { + "path": "/logstream", + "method": "GET", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 400 + }, + { + "path": "/logstream/smoke", + "method": "DELETE", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "PUT", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "DELETE", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "PUT", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "DELETE", + "status": 503 + }, + { + "path": "/liveness", + "method": "GET", + "status": 200 + }, + { + "path": "/logstream", + "method": "GET", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 400 + }, + { + "path": "/logstream/smoke", + "method": "DELETE", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "PUT", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "DELETE", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "PUT", + "status": 200 + }, + { + "path": "/liveness", + "method": "GET", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "PUT", + "status": 200 + }, + { + "path": "/ingest", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/logstream/smoke/retention", + "method": "PUT", + "status": 200 + }, + { + "path": "/dashboards", + "method": "POST", + "status": 200 + }, + { + "path": "/liveness", + "method": "GET", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/dashboards", + "method": "GET", + "status": 200 + }, + { + "path": "/metrics", + "method": "GET", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/metrics", + "method": "GET", + "status": 200 + }, + { + "path": "/readiness", + "method": "GET", + "status": 503 + }, + { + "path": "/logstream/outage", + "method": "PUT", + "status": 503 + }, + { + "path": "/readiness", + "method": "GET", + "status": 503 + }, + { + "path": "/readiness", + "method": "GET", + "status": 503 + }, + { + "path": "/readiness", + "method": "GET", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/readiness", + "method": "GET", + "status": 503 + }, + { + "path": "/readiness", + "method": "GET", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "DELETE", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "PUT", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/liveness", + "method": "GET", + "status": 200 + }, + { + "path": "/logstream", + "method": "GET", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 400 + }, + { + "path": "/logstream/smoke", + "method": "DELETE", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "PUT", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "DELETE", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "PUT", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "DELETE", + "status": 503 + }, + { + "path": "/liveness", + "method": "GET", + "status": 200 + }, + { + "path": "/logstream", + "method": "GET", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 400 + }, + { + "path": "/logstream/smoke", + "method": "DELETE", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "PUT", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "DELETE", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "PUT", + "status": 200 + } + ] + } + ] +} diff --git a/docs/validation/postgres-migration-tls.json b/docs/validation/postgres-migration-tls.json new file mode 100644 index 000000000..f203d7a1e --- /dev/null +++ b/docs/validation/postgres-migration-tls.json @@ -0,0 +1,1062 @@ +{ + "success": true, + "tls": true, + "commands": [ + { + "argv": [ + "docker", + "run", + "-d", + "--name", + "parseable-pg-smoke-1476557-0", + "-e", + "POSTGRES_PASSWORD=smoke", + "-e", + "POSTGRES_DB=migration", + "-p", + "127.0.0.1:34747:5432", + "postgres:17-alpine" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "run", + "-d", + "--name", + "parseable-pg-smoke-1476557-1", + "-e", + "MINIO_ROOT_USER=minioadmin", + "-e", + "MINIO_ROOT_PASSWORD=minioadmin", + "-p", + "127.0.0.1:40655:9000", + "quay.io/parseablehq/minio:edge", + "server", + "/data" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1476557-1", + "mkdir", + "-p", + "/data/migration" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1476557-0", + "pg_isready", + "-h", + "127.0.0.1", + "-U", + "postgres" + ], + "exit_code": 2 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1476557-0", + "pg_isready", + "-h", + "127.0.0.1", + "-U", + "postgres" + ], + "exit_code": 0 + }, + { + "argv": [ + "openssl", + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-keyout", + ".postgres-smoke/migration-tls/ca.key", + "-out", + ".postgres-smoke/migration-tls/ca.crt", + "-days", + "2", + "-subj", + "/CN=Smoke CA" + ], + "exit_code": 0 + }, + { + "argv": [ + "openssl", + "req", + "-newkey", + "rsa:2048", + "-nodes", + "-keyout", + ".postgres-smoke/migration-tls/server.key", + "-out", + ".postgres-smoke/migration-tls/server.csr", + "-subj", + "/CN=localhost" + ], + "exit_code": 0 + }, + { + "argv": [ + "openssl", + "x509", + "-req", + "-in", + ".postgres-smoke/migration-tls/server.csr", + "-CA", + ".postgres-smoke/migration-tls/ca.crt", + "-CAkey", + ".postgres-smoke/migration-tls/ca.key", + "-CAcreateserial", + "-out", + ".postgres-smoke/migration-tls/server.crt", + "-days", + "2", + "-extfile", + ".postgres-smoke/migration-tls/extensions" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "cp", + ".postgres-smoke/migration-tls/server.key", + "parseable-pg-smoke-1476557-0:/var/lib/postgresql/data/server.key" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "cp", + ".postgres-smoke/migration-tls/server.crt", + "parseable-pg-smoke-1476557-0:/var/lib/postgresql/data/server.crt" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1476557-0", + "sh", + "-c", + "chown postgres:postgres /var/lib/postgresql/data/server.* && chmod 600 /var/lib/postgresql/data/server.key" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1476557-0", + "psql", + "-U", + "postgres", + "-d", + "migration", + "-c", + "ALTER SYSTEM SET ssl='on';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1476557-0", + "psql", + "-U", + "postgres", + "-d", + "migration", + "-c", + "SELECT pg_reload_conf();" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store", + "--metadata-migration", + "import", + "--acknowledge-server-stopped", + "--dry-run" + ], + "exit_code": 1, + "expected_success": false + }, + { + "argv": [ + "target/debug/parseable", + "s3-store", + "--metadata-migration", + "verify", + "--acknowledge-server-stopped" + ], + "exit_code": 1, + "expected_success": false + }, + { + "argv": [ + "target/debug/parseable", + "s3-store", + "--metadata-migration", + "import", + "--dry-run", + "--acknowledge-server-stopped" + ], + "cwd": ".postgres-smoke/migration-env-fixture", + "exit_code": 0, + "env_file_checked": true + }, + { + "argv": [ + "target/debug/parseable", + "s3-store", + "--metadata-migration", + "import", + "--acknowledge-server-stopped", + "--dry-run" + ], + "exit_code": 0, + "expected_success": true + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1, + "refusal_state": "import-before-commit", + "postgres": false + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1, + "refusal_state": "import-before-commit", + "postgres": true + }, + { + "argv": [ + "target/debug/parseable", + "s3-store", + "--metadata-migration", + "import", + "--acknowledge-server-stopped" + ], + "exit_code": 0, + "expected_success": true + }, + { + "argv": [ + "target/debug/parseable", + "s3-store", + "--metadata-migration", + "verify", + "--acknowledge-server-stopped" + ], + "exit_code": 0, + "expected_success": true + }, + { + "argv": [ + "target/debug/parseable", + "s3-store", + "--metadata-migration", + "export", + "--acknowledge-server-stopped", + "--dry-run" + ], + "exit_code": 0, + "expected_success": true + }, + { + "argv": [ + "target/debug/parseable", + "s3-store", + "--metadata-migration", + "import", + "--acknowledge-server-stopped" + ], + "exit_code": 1, + "expected_success": false + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1476557-0", + "psql", + "-U", + "postgres", + "-d", + "migration", + "-qAt", + "-c", + "SELECT id FROM parseable_backend" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1, + "refusal_state": "import-committed-unpaired", + "postgres": false + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1, + "refusal_state": "import-committed-unpaired", + "postgres": true + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1, + "refusal_state": "import-paired-before-marker-cleanup", + "postgres": false + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1, + "refusal_state": "import-paired-before-marker-cleanup", + "postgres": true + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1, + "refusal_state": "paired-boolean-override", + "postgres": false + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1, + "refusal_state": "unpaired-boolean-override", + "postgres": true + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1, + "refusal_state": "export-marker-before-state", + "postgres": false + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1, + "refusal_state": "export-marker-before-state", + "postgres": true + }, + { + "argv": [ + "target/debug/parseable", + "s3-store", + "--metadata-migration", + "export", + "--acknowledge-server-stopped", + "--abort" + ], + "exit_code": 0, + "expected_success": true + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1476557-0", + "psql", + "-U", + "postgres", + "-d", + "migration", + "-qAt", + "-c", + "UPDATE parseable_transfer SET state='exported'" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1, + "refusal_state": "export-state-before-object-writes", + "postgres": false + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1, + "refusal_state": "export-state-before-object-writes", + "postgres": true + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1476557-0", + "psql", + "-U", + "postgres", + "-d", + "migration", + "-qAt", + "-c", + "UPDATE parseable_transfer SET state='ready'" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1, + "refusal_state": "interrupted-export", + "postgres": false + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1, + "refusal_state": "interrupted-export", + "postgres": true + }, + { + "argv": [ + "target/debug/parseable", + "s3-store", + "--metadata-migration", + "export", + "--acknowledge-server-stopped" + ], + "exit_code": 0, + "expected_success": true + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1, + "refusal_state": "export-pair-removed-before-marker-cleanup", + "postgres": false + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1, + "refusal_state": "export-pair-removed-before-marker-cleanup", + "postgres": true + }, + { + "argv": [ + "target/debug/parseable", + "s3-store", + "--metadata-migration", + "export", + "--acknowledge-server-stopped" + ], + "exit_code": 0, + "expected_success": true + }, + { + "argv": [ + "target/debug/parseable", + "s3-store", + "--metadata-migration", + "verify", + "--acknowledge-server-stopped" + ], + "exit_code": 0, + "expected_success": true + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1, + "refusal_state": "exported-database-without-marker", + "postgres": true + } + ], + "bucket_keys": [ + ".alerts/01M75RG3APZZ2T0DVYD48GB62H.json", + ".alerts/alert_state_01M75RG3APZZ2T0DVYD48GB62H.json", + ".parseable/.parseable.json", + ".settings/migration-fixtures/0.json", + ".settings/migration-fixtures/1.json", + ".settings/migration-fixtures/10.json", + ".settings/migration-fixtures/100.json", + ".settings/migration-fixtures/101.json", + ".settings/migration-fixtures/102.json", + ".settings/migration-fixtures/103.json", + ".settings/migration-fixtures/104.json", + ".settings/migration-fixtures/105.json", + ".settings/migration-fixtures/106.json", + ".settings/migration-fixtures/107.json", + ".settings/migration-fixtures/108.json", + ".settings/migration-fixtures/109.json", + ".settings/migration-fixtures/11.json", + ".settings/migration-fixtures/110.json", + ".settings/migration-fixtures/111.json", + ".settings/migration-fixtures/112.json", + ".settings/migration-fixtures/113.json", + ".settings/migration-fixtures/114.json", + ".settings/migration-fixtures/115.json", + ".settings/migration-fixtures/116.json", + ".settings/migration-fixtures/117.json", + ".settings/migration-fixtures/118.json", + ".settings/migration-fixtures/119.json", + ".settings/migration-fixtures/12.json", + ".settings/migration-fixtures/120.json", + ".settings/migration-fixtures/121.json", + ".settings/migration-fixtures/122.json", + ".settings/migration-fixtures/123.json", + ".settings/migration-fixtures/124.json", + ".settings/migration-fixtures/125.json", + ".settings/migration-fixtures/126.json", + ".settings/migration-fixtures/127.json", + ".settings/migration-fixtures/128.json", + ".settings/migration-fixtures/129.json", + ".settings/migration-fixtures/13.json", + ".settings/migration-fixtures/130.json", + ".settings/migration-fixtures/131.json", + ".settings/migration-fixtures/132.json", + ".settings/migration-fixtures/133.json", + ".settings/migration-fixtures/134.json", + ".settings/migration-fixtures/135.json", + ".settings/migration-fixtures/136.json", + ".settings/migration-fixtures/137.json", + ".settings/migration-fixtures/138.json", + ".settings/migration-fixtures/139.json", + ".settings/migration-fixtures/14.json", + ".settings/migration-fixtures/140.json", + ".settings/migration-fixtures/141.json", + ".settings/migration-fixtures/142.json", + ".settings/migration-fixtures/143.json", + ".settings/migration-fixtures/144.json", + ".settings/migration-fixtures/145.json", + ".settings/migration-fixtures/146.json", + ".settings/migration-fixtures/147.json", + ".settings/migration-fixtures/148.json", + ".settings/migration-fixtures/149.json", + ".settings/migration-fixtures/15.json", + ".settings/migration-fixtures/150.json", + ".settings/migration-fixtures/151.json", + ".settings/migration-fixtures/152.json", + ".settings/migration-fixtures/153.json", + ".settings/migration-fixtures/154.json", + ".settings/migration-fixtures/155.json", + ".settings/migration-fixtures/156.json", + ".settings/migration-fixtures/157.json", + ".settings/migration-fixtures/158.json", + ".settings/migration-fixtures/159.json", + ".settings/migration-fixtures/16.json", + ".settings/migration-fixtures/160.json", + ".settings/migration-fixtures/161.json", + ".settings/migration-fixtures/162.json", + ".settings/migration-fixtures/163.json", + ".settings/migration-fixtures/164.json", + ".settings/migration-fixtures/165.json", + ".settings/migration-fixtures/166.json", + ".settings/migration-fixtures/167.json", + ".settings/migration-fixtures/168.json", + ".settings/migration-fixtures/169.json", + ".settings/migration-fixtures/17.json", + ".settings/migration-fixtures/170.json", + ".settings/migration-fixtures/171.json", + ".settings/migration-fixtures/172.json", + ".settings/migration-fixtures/173.json", + ".settings/migration-fixtures/174.json", + ".settings/migration-fixtures/175.json", + ".settings/migration-fixtures/176.json", + ".settings/migration-fixtures/177.json", + ".settings/migration-fixtures/178.json", + ".settings/migration-fixtures/179.json", + ".settings/migration-fixtures/18.json", + ".settings/migration-fixtures/180.json", + ".settings/migration-fixtures/181.json", + ".settings/migration-fixtures/182.json", + ".settings/migration-fixtures/183.json", + ".settings/migration-fixtures/184.json", + ".settings/migration-fixtures/185.json", + ".settings/migration-fixtures/186.json", + ".settings/migration-fixtures/187.json", + ".settings/migration-fixtures/188.json", + ".settings/migration-fixtures/189.json", + ".settings/migration-fixtures/19.json", + ".settings/migration-fixtures/190.json", + ".settings/migration-fixtures/191.json", + ".settings/migration-fixtures/192.json", + ".settings/migration-fixtures/193.json", + ".settings/migration-fixtures/194.json", + ".settings/migration-fixtures/195.json", + ".settings/migration-fixtures/196.json", + ".settings/migration-fixtures/197.json", + ".settings/migration-fixtures/198.json", + ".settings/migration-fixtures/199.json", + ".settings/migration-fixtures/2.json", + ".settings/migration-fixtures/20.json", + ".settings/migration-fixtures/200.json", + ".settings/migration-fixtures/201.json", + ".settings/migration-fixtures/202.json", + ".settings/migration-fixtures/203.json", + ".settings/migration-fixtures/204.json", + ".settings/migration-fixtures/205.json", + ".settings/migration-fixtures/206.json", + ".settings/migration-fixtures/207.json", + ".settings/migration-fixtures/208.json", + ".settings/migration-fixtures/209.json", + ".settings/migration-fixtures/21.json", + ".settings/migration-fixtures/210.json", + ".settings/migration-fixtures/211.json", + ".settings/migration-fixtures/212.json", + ".settings/migration-fixtures/213.json", + ".settings/migration-fixtures/214.json", + ".settings/migration-fixtures/215.json", + ".settings/migration-fixtures/216.json", + ".settings/migration-fixtures/217.json", + ".settings/migration-fixtures/218.json", + ".settings/migration-fixtures/219.json", + ".settings/migration-fixtures/22.json", + ".settings/migration-fixtures/220.json", + ".settings/migration-fixtures/221.json", + ".settings/migration-fixtures/222.json", + ".settings/migration-fixtures/223.json", + ".settings/migration-fixtures/224.json", + ".settings/migration-fixtures/225.json", + ".settings/migration-fixtures/226.json", + ".settings/migration-fixtures/227.json", + ".settings/migration-fixtures/228.json", + ".settings/migration-fixtures/229.json", + ".settings/migration-fixtures/23.json", + ".settings/migration-fixtures/230.json", + ".settings/migration-fixtures/231.json", + ".settings/migration-fixtures/232.json", + ".settings/migration-fixtures/233.json", + ".settings/migration-fixtures/234.json", + ".settings/migration-fixtures/235.json", + ".settings/migration-fixtures/236.json", + ".settings/migration-fixtures/237.json", + ".settings/migration-fixtures/238.json", + ".settings/migration-fixtures/239.json", + ".settings/migration-fixtures/24.json", + ".settings/migration-fixtures/240.json", + ".settings/migration-fixtures/241.json", + ".settings/migration-fixtures/242.json", + ".settings/migration-fixtures/243.json", + ".settings/migration-fixtures/244.json", + ".settings/migration-fixtures/245.json", + ".settings/migration-fixtures/246.json", + ".settings/migration-fixtures/247.json", + ".settings/migration-fixtures/248.json", + ".settings/migration-fixtures/249.json", + ".settings/migration-fixtures/25.json", + ".settings/migration-fixtures/250.json", + ".settings/migration-fixtures/251.json", + ".settings/migration-fixtures/252.json", + ".settings/migration-fixtures/253.json", + ".settings/migration-fixtures/254.json", + ".settings/migration-fixtures/255.json", + ".settings/migration-fixtures/256.json", + ".settings/migration-fixtures/257.json", + ".settings/migration-fixtures/258.json", + ".settings/migration-fixtures/259.json", + ".settings/migration-fixtures/26.json", + ".settings/migration-fixtures/260.json", + ".settings/migration-fixtures/261.json", + ".settings/migration-fixtures/262.json", + ".settings/migration-fixtures/263.json", + ".settings/migration-fixtures/264.json", + ".settings/migration-fixtures/265.json", + ".settings/migration-fixtures/266.json", + ".settings/migration-fixtures/267.json", + ".settings/migration-fixtures/268.json", + ".settings/migration-fixtures/269.json", + ".settings/migration-fixtures/27.json", + ".settings/migration-fixtures/270.json", + ".settings/migration-fixtures/271.json", + ".settings/migration-fixtures/272.json", + ".settings/migration-fixtures/273.json", + ".settings/migration-fixtures/274.json", + ".settings/migration-fixtures/275.json", + ".settings/migration-fixtures/276.json", + ".settings/migration-fixtures/277.json", + ".settings/migration-fixtures/278.json", + ".settings/migration-fixtures/279.json", + ".settings/migration-fixtures/28.json", + ".settings/migration-fixtures/280.json", + ".settings/migration-fixtures/281.json", + ".settings/migration-fixtures/282.json", + ".settings/migration-fixtures/283.json", + ".settings/migration-fixtures/284.json", + ".settings/migration-fixtures/285.json", + ".settings/migration-fixtures/286.json", + ".settings/migration-fixtures/287.json", + ".settings/migration-fixtures/288.json", + ".settings/migration-fixtures/289.json", + ".settings/migration-fixtures/29.json", + ".settings/migration-fixtures/290.json", + ".settings/migration-fixtures/291.json", + ".settings/migration-fixtures/292.json", + ".settings/migration-fixtures/293.json", + ".settings/migration-fixtures/294.json", + ".settings/migration-fixtures/295.json", + ".settings/migration-fixtures/296.json", + ".settings/migration-fixtures/297.json", + ".settings/migration-fixtures/298.json", + ".settings/migration-fixtures/299.json", + ".settings/migration-fixtures/3.json", + ".settings/migration-fixtures/30.json", + ".settings/migration-fixtures/300.json", + ".settings/migration-fixtures/301.json", + ".settings/migration-fixtures/302.json", + ".settings/migration-fixtures/303.json", + ".settings/migration-fixtures/304.json", + ".settings/migration-fixtures/305.json", + ".settings/migration-fixtures/306.json", + ".settings/migration-fixtures/307.json", + ".settings/migration-fixtures/308.json", + ".settings/migration-fixtures/309.json", + ".settings/migration-fixtures/31.json", + ".settings/migration-fixtures/310.json", + ".settings/migration-fixtures/311.json", + ".settings/migration-fixtures/312.json", + ".settings/migration-fixtures/313.json", + ".settings/migration-fixtures/314.json", + ".settings/migration-fixtures/315.json", + ".settings/migration-fixtures/316.json", + ".settings/migration-fixtures/317.json", + ".settings/migration-fixtures/318.json", + ".settings/migration-fixtures/319.json", + ".settings/migration-fixtures/32.json", + ".settings/migration-fixtures/320.json", + ".settings/migration-fixtures/321.json", + ".settings/migration-fixtures/322.json", + ".settings/migration-fixtures/323.json", + ".settings/migration-fixtures/324.json", + ".settings/migration-fixtures/325.json", + ".settings/migration-fixtures/326.json", + ".settings/migration-fixtures/327.json", + ".settings/migration-fixtures/328.json", + ".settings/migration-fixtures/329.json", + ".settings/migration-fixtures/33.json", + ".settings/migration-fixtures/330.json", + ".settings/migration-fixtures/331.json", + ".settings/migration-fixtures/332.json", + ".settings/migration-fixtures/333.json", + ".settings/migration-fixtures/334.json", + ".settings/migration-fixtures/335.json", + ".settings/migration-fixtures/336.json", + ".settings/migration-fixtures/337.json", + ".settings/migration-fixtures/338.json", + ".settings/migration-fixtures/339.json", + ".settings/migration-fixtures/34.json", + ".settings/migration-fixtures/340.json", + ".settings/migration-fixtures/341.json", + ".settings/migration-fixtures/342.json", + ".settings/migration-fixtures/343.json", + ".settings/migration-fixtures/344.json", + ".settings/migration-fixtures/345.json", + ".settings/migration-fixtures/346.json", + ".settings/migration-fixtures/347.json", + ".settings/migration-fixtures/348.json", + ".settings/migration-fixtures/349.json", + ".settings/migration-fixtures/35.json", + ".settings/migration-fixtures/350.json", + ".settings/migration-fixtures/351.json", + ".settings/migration-fixtures/352.json", + ".settings/migration-fixtures/353.json", + ".settings/migration-fixtures/354.json", + ".settings/migration-fixtures/355.json", + ".settings/migration-fixtures/356.json", + ".settings/migration-fixtures/357.json", + ".settings/migration-fixtures/358.json", + ".settings/migration-fixtures/359.json", + ".settings/migration-fixtures/36.json", + ".settings/migration-fixtures/360.json", + ".settings/migration-fixtures/361.json", + ".settings/migration-fixtures/362.json", + ".settings/migration-fixtures/363.json", + ".settings/migration-fixtures/364.json", + ".settings/migration-fixtures/365.json", + ".settings/migration-fixtures/366.json", + ".settings/migration-fixtures/367.json", + ".settings/migration-fixtures/368.json", + ".settings/migration-fixtures/369.json", + ".settings/migration-fixtures/37.json", + ".settings/migration-fixtures/370.json", + ".settings/migration-fixtures/371.json", + ".settings/migration-fixtures/372.json", + ".settings/migration-fixtures/373.json", + ".settings/migration-fixtures/374.json", + ".settings/migration-fixtures/375.json", + ".settings/migration-fixtures/376.json", + ".settings/migration-fixtures/377.json", + ".settings/migration-fixtures/378.json", + ".settings/migration-fixtures/379.json", + ".settings/migration-fixtures/38.json", + ".settings/migration-fixtures/380.json", + ".settings/migration-fixtures/381.json", + ".settings/migration-fixtures/382.json", + ".settings/migration-fixtures/383.json", + ".settings/migration-fixtures/384.json", + ".settings/migration-fixtures/385.json", + ".settings/migration-fixtures/386.json", + ".settings/migration-fixtures/387.json", + ".settings/migration-fixtures/388.json", + ".settings/migration-fixtures/389.json", + ".settings/migration-fixtures/39.json", + ".settings/migration-fixtures/390.json", + ".settings/migration-fixtures/391.json", + ".settings/migration-fixtures/392.json", + ".settings/migration-fixtures/393.json", + ".settings/migration-fixtures/394.json", + ".settings/migration-fixtures/395.json", + ".settings/migration-fixtures/396.json", + ".settings/migration-fixtures/397.json", + ".settings/migration-fixtures/398.json", + ".settings/migration-fixtures/399.json", + ".settings/migration-fixtures/4.json", + ".settings/migration-fixtures/40.json", + ".settings/migration-fixtures/400.json", + ".settings/migration-fixtures/401.json", + ".settings/migration-fixtures/402.json", + ".settings/migration-fixtures/403.json", + ".settings/migration-fixtures/404.json", + ".settings/migration-fixtures/405.json", + ".settings/migration-fixtures/406.json", + ".settings/migration-fixtures/407.json", + ".settings/migration-fixtures/408.json", + ".settings/migration-fixtures/409.json", + ".settings/migration-fixtures/41.json", + ".settings/migration-fixtures/410.json", + ".settings/migration-fixtures/411.json", + ".settings/migration-fixtures/412.json", + ".settings/migration-fixtures/413.json", + ".settings/migration-fixtures/414.json", + ".settings/migration-fixtures/415.json", + ".settings/migration-fixtures/416.json", + ".settings/migration-fixtures/417.json", + ".settings/migration-fixtures/418.json", + ".settings/migration-fixtures/419.json", + ".settings/migration-fixtures/42.json", + ".settings/migration-fixtures/420.json", + ".settings/migration-fixtures/421.json", + ".settings/migration-fixtures/422.json", + ".settings/migration-fixtures/423.json", + ".settings/migration-fixtures/424.json", + ".settings/migration-fixtures/425.json", + ".settings/migration-fixtures/426.json", + ".settings/migration-fixtures/427.json", + ".settings/migration-fixtures/428.json", + ".settings/migration-fixtures/429.json", + ".settings/migration-fixtures/43.json", + ".settings/migration-fixtures/430.json", + ".settings/migration-fixtures/431.json", + ".settings/migration-fixtures/432.json", + ".settings/migration-fixtures/433.json", + ".settings/migration-fixtures/434.json", + ".settings/migration-fixtures/435.json", + ".settings/migration-fixtures/436.json", + ".settings/migration-fixtures/437.json", + ".settings/migration-fixtures/438.json", + ".settings/migration-fixtures/439.json", + ".settings/migration-fixtures/44.json", + ".settings/migration-fixtures/440.json", + ".settings/migration-fixtures/441.json", + ".settings/migration-fixtures/442.json", + ".settings/migration-fixtures/443.json", + ".settings/migration-fixtures/444.json", + ".settings/migration-fixtures/445.json", + ".settings/migration-fixtures/446.json", + ".settings/migration-fixtures/447.json", + ".settings/migration-fixtures/448.json", + ".settings/migration-fixtures/449.json", + ".settings/migration-fixtures/45.json", + ".settings/migration-fixtures/450.json", + ".settings/migration-fixtures/451.json", + ".settings/migration-fixtures/452.json", + ".settings/migration-fixtures/453.json", + ".settings/migration-fixtures/454.json", + ".settings/migration-fixtures/455.json", + ".settings/migration-fixtures/456.json", + ".settings/migration-fixtures/457.json", + ".settings/migration-fixtures/458.json", + ".settings/migration-fixtures/459.json", + ".settings/migration-fixtures/46.json", + ".settings/migration-fixtures/460.json", + ".settings/migration-fixtures/461.json", + ".settings/migration-fixtures/462.json", + ".settings/migration-fixtures/463.json", + ".settings/migration-fixtures/464.json", + ".settings/migration-fixtures/465.json", + ".settings/migration-fixtures/466.json", + ".settings/migration-fixtures/467.json", + ".settings/migration-fixtures/468.json", + ".settings/migration-fixtures/469.json", + ".settings/migration-fixtures/47.json", + ".settings/migration-fixtures/470.json", + ".settings/migration-fixtures/471.json", + ".settings/migration-fixtures/472.json", + ".settings/migration-fixtures/473.json", + ".settings/migration-fixtures/474.json", + ".settings/migration-fixtures/475.json", + ".settings/migration-fixtures/476.json", + ".settings/migration-fixtures/477.json", + ".settings/migration-fixtures/478.json", + ".settings/migration-fixtures/479.json", + ".settings/migration-fixtures/48.json", + ".settings/migration-fixtures/480.json", + ".settings/migration-fixtures/481.json", + ".settings/migration-fixtures/482.json", + ".settings/migration-fixtures/483.json", + ".settings/migration-fixtures/484.json", + ".settings/migration-fixtures/485.json", + ".settings/migration-fixtures/486.json", + ".settings/migration-fixtures/487.json", + ".settings/migration-fixtures/488.json", + ".settings/migration-fixtures/489.json", + ".settings/migration-fixtures/49.json", + ".settings/migration-fixtures/490.json", + ".settings/migration-fixtures/491.json", + ".settings/migration-fixtures/492.json", + ".settings/migration-fixtures/493.json", + ".settings/migration-fixtures/494.json", + ".settings/migration-fixtures/495.json", + ".settings/migration-fixtures/496.json", + ".settings/migration-fixtures/497.json", + ".settings/migration-fixtures/498.json", + ".settings/migration-fixtures/499.json", + ".settings/migration-fixtures/5.json", + ".settings/migration-fixtures/50.json", + ".settings/migration-fixtures/51.json", + ".settings/migration-fixtures/52.json", + ".settings/migration-fixtures/53.json", + ".settings/migration-fixtures/54.json", + ".settings/migration-fixtures/55.json", + ".settings/migration-fixtures/56.json", + ".settings/migration-fixtures/57.json", + ".settings/migration-fixtures/58.json", + ".settings/migration-fixtures/59.json", + ".settings/migration-fixtures/6.json", + ".settings/migration-fixtures/60.json", + ".settings/migration-fixtures/61.json", + ".settings/migration-fixtures/62.json", + ".settings/migration-fixtures/63.json", + ".settings/migration-fixtures/64.json", + ".settings/migration-fixtures/65.json", + ".settings/migration-fixtures/66.json", + ".settings/migration-fixtures/67.json", + ".settings/migration-fixtures/68.json", + ".settings/migration-fixtures/69.json", + ".settings/migration-fixtures/7.json", + ".settings/migration-fixtures/70.json", + ".settings/migration-fixtures/71.json", + ".settings/migration-fixtures/72.json", + ".settings/migration-fixtures/73.json", + ".settings/migration-fixtures/74.json", + ".settings/migration-fixtures/75.json", + ".settings/migration-fixtures/76.json", + ".settings/migration-fixtures/77.json", + ".settings/migration-fixtures/78.json", + ".settings/migration-fixtures/79.json", + ".settings/migration-fixtures/8.json", + ".settings/migration-fixtures/80.json", + ".settings/migration-fixtures/81.json", + ".settings/migration-fixtures/82.json", + ".settings/migration-fixtures/83.json", + ".settings/migration-fixtures/84.json", + ".settings/migration-fixtures/85.json", + ".settings/migration-fixtures/86.json", + ".settings/migration-fixtures/87.json", + ".settings/migration-fixtures/88.json", + ".settings/migration-fixtures/89.json", + ".settings/migration-fixtures/9.json", + ".settings/migration-fixtures/90.json", + ".settings/migration-fixtures/91.json", + ".settings/migration-fixtures/92.json", + ".settings/migration-fixtures/93.json", + ".settings/migration-fixtures/94.json", + ".settings/migration-fixtures/95.json", + ".settings/migration-fixtures/96.json", + ".settings/migration-fixtures/97.json", + ".settings/migration-fixtures/98.json", + ".settings/migration-fixtures/99.json", + ".users/.dashboard-locations/01M75RG39Z4NKWZK8SA8NFW8Q6.json", + ".users/.dashboard-titles/2b9aa3aaf18aa9aa60fcc45d30833be66dc5907be190dfff57197c19883297e4.json", + ".users/8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918/dashboards/01M75RG39Z4NKWZK8SA8NFW8Q6.json", + ".users/8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918/filters/first/01M75RG3AG7WMBR9T8RWFJMBRQ.json", + "first/.stream/.schema", + "first/.stream/.stream.json", + "first/date=2026-11-11/.manifest.json", + "first/date=2026-11-11/hour=16/minute=54/.data.01M75RHH4MMRY8P8Z2VV37G1GJ.parquet", + "first/date=2026-11-11/hour=16/minute=56/.data.01M75RN67QPP2693W1AFKVZ62Z.parquet", + "partitioned/.stream/.schema", + "partitioned/.stream/.stream.json", + "partitioned/date=2026-11-11/.manifest.json", + "partitioned/date=2026-11-11/hour=16/minute=54/region=west/.data.01M75RHH4MPPMHAPHBNY12A63B.parquet", + "pstats/.stream/.schema", + "pstats/.stream/.stream.json", + "pstats/date=2026-11-11/.manifest.json", + "pstats/date=2026-11-11/hour=16/minute=54/dataset_name=first/.data.01M75RKBQMSRE7XKV9SQSKSVDA.parquet", + "pstats/date=2026-11-11/hour=16/minute=54/dataset_name=partitioned/.data.01M75RKBQMSRE7XKV9SQSKSVDA.parquet", + "pstats/date=2026-11-11/hour=16/minute=54/dataset_name=second/.data.01M75RKBQMSRE7XKV9SQSKSVDA.parquet", + "pstats/date=2026-11-11/hour=16/minute=56/dataset_name=first/.data.01M75RQ0TPPX471FAN826ARJ2V.parquet", + "second/.stream/.schema", + "second/.stream/.stream.json", + "second/date=2026-11-11/.manifest.json", + "second/date=2026-11-11/hour=16/minute=54/.data.01M75RHH4MF5M9DZ6MTG796EK5.parquet" + ] +} diff --git a/docs/validation/postgres-migration.json b/docs/validation/postgres-migration.json new file mode 100644 index 000000000..025e9ca67 --- /dev/null +++ b/docs/validation/postgres-migration.json @@ -0,0 +1,960 @@ +{ + "success": true, + "tls": false, + "commands": [ + { + "argv": [ + "docker", + "run", + "-d", + "--name", + "parseable-pg-smoke-1458428-0", + "-e", + "POSTGRES_PASSWORD=smoke", + "-e", + "POSTGRES_DB=migration", + "-p", + "127.0.0.1:42245:5432", + "postgres:17-alpine" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "run", + "-d", + "--name", + "parseable-pg-smoke-1458428-1", + "-e", + "MINIO_ROOT_USER=minioadmin", + "-e", + "MINIO_ROOT_PASSWORD=minioadmin", + "-p", + "127.0.0.1:59295:9000", + "quay.io/parseablehq/minio:edge", + "server", + "/data" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1458428-1", + "mkdir", + "-p", + "/data/migration" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1458428-0", + "pg_isready", + "-h", + "127.0.0.1", + "-U", + "postgres" + ], + "exit_code": 2 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1458428-0", + "pg_isready", + "-h", + "127.0.0.1", + "-U", + "postgres" + ], + "exit_code": 2 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1458428-0", + "pg_isready", + "-h", + "127.0.0.1", + "-U", + "postgres" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store", + "--metadata-migration", + "import", + "--acknowledge-server-stopped", + "--dry-run" + ], + "exit_code": 1, + "expected_success": false + }, + { + "argv": [ + "target/debug/parseable", + "s3-store", + "--metadata-migration", + "verify", + "--acknowledge-server-stopped" + ], + "exit_code": 1, + "expected_success": false + }, + { + "argv": [ + "target/debug/parseable", + "s3-store", + "--metadata-migration", + "import", + "--dry-run", + "--acknowledge-server-stopped" + ], + "cwd": ".postgres-smoke/migration-env-fixture", + "exit_code": 0, + "env_file_checked": true + }, + { + "argv": [ + "target/debug/parseable", + "s3-store", + "--metadata-migration", + "import", + "--acknowledge-server-stopped", + "--dry-run" + ], + "exit_code": 0, + "expected_success": true + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1, + "refusal_state": "import-before-commit", + "postgres": false + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1, + "refusal_state": "import-before-commit", + "postgres": true + }, + { + "argv": [ + "target/debug/parseable", + "s3-store", + "--metadata-migration", + "import", + "--acknowledge-server-stopped" + ], + "exit_code": 0, + "expected_success": true + }, + { + "argv": [ + "target/debug/parseable", + "s3-store", + "--metadata-migration", + "verify", + "--acknowledge-server-stopped" + ], + "exit_code": 0, + "expected_success": true + }, + { + "argv": [ + "target/debug/parseable", + "s3-store", + "--metadata-migration", + "export", + "--acknowledge-server-stopped", + "--dry-run" + ], + "exit_code": 0, + "expected_success": true + }, + { + "argv": [ + "target/debug/parseable", + "s3-store", + "--metadata-migration", + "import", + "--acknowledge-server-stopped" + ], + "exit_code": 1, + "expected_success": false + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1458428-0", + "psql", + "-U", + "postgres", + "-d", + "migration", + "-qAt", + "-c", + "SELECT id FROM parseable_backend" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1, + "refusal_state": "import-committed-unpaired", + "postgres": false + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1, + "refusal_state": "import-committed-unpaired", + "postgres": true + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1, + "refusal_state": "import-paired-before-marker-cleanup", + "postgres": false + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1, + "refusal_state": "import-paired-before-marker-cleanup", + "postgres": true + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1, + "refusal_state": "paired-boolean-override", + "postgres": false + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1, + "refusal_state": "unpaired-boolean-override", + "postgres": true + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1, + "refusal_state": "export-marker-before-state", + "postgres": false + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1, + "refusal_state": "export-marker-before-state", + "postgres": true + }, + { + "argv": [ + "target/debug/parseable", + "s3-store", + "--metadata-migration", + "export", + "--acknowledge-server-stopped", + "--abort" + ], + "exit_code": 0, + "expected_success": true + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1458428-0", + "psql", + "-U", + "postgres", + "-d", + "migration", + "-qAt", + "-c", + "UPDATE parseable_transfer SET state='exported'" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1, + "refusal_state": "export-state-before-object-writes", + "postgres": false + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1, + "refusal_state": "export-state-before-object-writes", + "postgres": true + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1458428-0", + "psql", + "-U", + "postgres", + "-d", + "migration", + "-qAt", + "-c", + "UPDATE parseable_transfer SET state='ready'" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1, + "refusal_state": "interrupted-export", + "postgres": false + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1, + "refusal_state": "interrupted-export", + "postgres": true + }, + { + "argv": [ + "target/debug/parseable", + "s3-store", + "--metadata-migration", + "export", + "--acknowledge-server-stopped" + ], + "exit_code": 0, + "expected_success": true + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1, + "refusal_state": "export-pair-removed-before-marker-cleanup", + "postgres": false + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1, + "refusal_state": "export-pair-removed-before-marker-cleanup", + "postgres": true + }, + { + "argv": [ + "target/debug/parseable", + "s3-store", + "--metadata-migration", + "export", + "--acknowledge-server-stopped" + ], + "exit_code": 0, + "expected_success": true + }, + { + "argv": [ + "target/debug/parseable", + "s3-store", + "--metadata-migration", + "verify", + "--acknowledge-server-stopped" + ], + "exit_code": 0, + "expected_success": true + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1, + "refusal_state": "exported-database-without-marker", + "postgres": true + } + ], + "bucket_keys": [ + ".alerts/01M75R7GG5CH5F1F34AJWY19TV.json", + ".alerts/alert_state_01M75R7GG5CH5F1F34AJWY19TV.json", + ".parseable/.parseable.json", + ".settings/migration-fixtures/0.json", + ".settings/migration-fixtures/1.json", + ".settings/migration-fixtures/10.json", + ".settings/migration-fixtures/100.json", + ".settings/migration-fixtures/101.json", + ".settings/migration-fixtures/102.json", + ".settings/migration-fixtures/103.json", + ".settings/migration-fixtures/104.json", + ".settings/migration-fixtures/105.json", + ".settings/migration-fixtures/106.json", + ".settings/migration-fixtures/107.json", + ".settings/migration-fixtures/108.json", + ".settings/migration-fixtures/109.json", + ".settings/migration-fixtures/11.json", + ".settings/migration-fixtures/110.json", + ".settings/migration-fixtures/111.json", + ".settings/migration-fixtures/112.json", + ".settings/migration-fixtures/113.json", + ".settings/migration-fixtures/114.json", + ".settings/migration-fixtures/115.json", + ".settings/migration-fixtures/116.json", + ".settings/migration-fixtures/117.json", + ".settings/migration-fixtures/118.json", + ".settings/migration-fixtures/119.json", + ".settings/migration-fixtures/12.json", + ".settings/migration-fixtures/120.json", + ".settings/migration-fixtures/121.json", + ".settings/migration-fixtures/122.json", + ".settings/migration-fixtures/123.json", + ".settings/migration-fixtures/124.json", + ".settings/migration-fixtures/125.json", + ".settings/migration-fixtures/126.json", + ".settings/migration-fixtures/127.json", + ".settings/migration-fixtures/128.json", + ".settings/migration-fixtures/129.json", + ".settings/migration-fixtures/13.json", + ".settings/migration-fixtures/130.json", + ".settings/migration-fixtures/131.json", + ".settings/migration-fixtures/132.json", + ".settings/migration-fixtures/133.json", + ".settings/migration-fixtures/134.json", + ".settings/migration-fixtures/135.json", + ".settings/migration-fixtures/136.json", + ".settings/migration-fixtures/137.json", + ".settings/migration-fixtures/138.json", + ".settings/migration-fixtures/139.json", + ".settings/migration-fixtures/14.json", + ".settings/migration-fixtures/140.json", + ".settings/migration-fixtures/141.json", + ".settings/migration-fixtures/142.json", + ".settings/migration-fixtures/143.json", + ".settings/migration-fixtures/144.json", + ".settings/migration-fixtures/145.json", + ".settings/migration-fixtures/146.json", + ".settings/migration-fixtures/147.json", + ".settings/migration-fixtures/148.json", + ".settings/migration-fixtures/149.json", + ".settings/migration-fixtures/15.json", + ".settings/migration-fixtures/150.json", + ".settings/migration-fixtures/151.json", + ".settings/migration-fixtures/152.json", + ".settings/migration-fixtures/153.json", + ".settings/migration-fixtures/154.json", + ".settings/migration-fixtures/155.json", + ".settings/migration-fixtures/156.json", + ".settings/migration-fixtures/157.json", + ".settings/migration-fixtures/158.json", + ".settings/migration-fixtures/159.json", + ".settings/migration-fixtures/16.json", + ".settings/migration-fixtures/160.json", + ".settings/migration-fixtures/161.json", + ".settings/migration-fixtures/162.json", + ".settings/migration-fixtures/163.json", + ".settings/migration-fixtures/164.json", + ".settings/migration-fixtures/165.json", + ".settings/migration-fixtures/166.json", + ".settings/migration-fixtures/167.json", + ".settings/migration-fixtures/168.json", + ".settings/migration-fixtures/169.json", + ".settings/migration-fixtures/17.json", + ".settings/migration-fixtures/170.json", + ".settings/migration-fixtures/171.json", + ".settings/migration-fixtures/172.json", + ".settings/migration-fixtures/173.json", + ".settings/migration-fixtures/174.json", + ".settings/migration-fixtures/175.json", + ".settings/migration-fixtures/176.json", + ".settings/migration-fixtures/177.json", + ".settings/migration-fixtures/178.json", + ".settings/migration-fixtures/179.json", + ".settings/migration-fixtures/18.json", + ".settings/migration-fixtures/180.json", + ".settings/migration-fixtures/181.json", + ".settings/migration-fixtures/182.json", + ".settings/migration-fixtures/183.json", + ".settings/migration-fixtures/184.json", + ".settings/migration-fixtures/185.json", + ".settings/migration-fixtures/186.json", + ".settings/migration-fixtures/187.json", + ".settings/migration-fixtures/188.json", + ".settings/migration-fixtures/189.json", + ".settings/migration-fixtures/19.json", + ".settings/migration-fixtures/190.json", + ".settings/migration-fixtures/191.json", + ".settings/migration-fixtures/192.json", + ".settings/migration-fixtures/193.json", + ".settings/migration-fixtures/194.json", + ".settings/migration-fixtures/195.json", + ".settings/migration-fixtures/196.json", + ".settings/migration-fixtures/197.json", + ".settings/migration-fixtures/198.json", + ".settings/migration-fixtures/199.json", + ".settings/migration-fixtures/2.json", + ".settings/migration-fixtures/20.json", + ".settings/migration-fixtures/200.json", + ".settings/migration-fixtures/201.json", + ".settings/migration-fixtures/202.json", + ".settings/migration-fixtures/203.json", + ".settings/migration-fixtures/204.json", + ".settings/migration-fixtures/205.json", + ".settings/migration-fixtures/206.json", + ".settings/migration-fixtures/207.json", + ".settings/migration-fixtures/208.json", + ".settings/migration-fixtures/209.json", + ".settings/migration-fixtures/21.json", + ".settings/migration-fixtures/210.json", + ".settings/migration-fixtures/211.json", + ".settings/migration-fixtures/212.json", + ".settings/migration-fixtures/213.json", + ".settings/migration-fixtures/214.json", + ".settings/migration-fixtures/215.json", + ".settings/migration-fixtures/216.json", + ".settings/migration-fixtures/217.json", + ".settings/migration-fixtures/218.json", + ".settings/migration-fixtures/219.json", + ".settings/migration-fixtures/22.json", + ".settings/migration-fixtures/220.json", + ".settings/migration-fixtures/221.json", + ".settings/migration-fixtures/222.json", + ".settings/migration-fixtures/223.json", + ".settings/migration-fixtures/224.json", + ".settings/migration-fixtures/225.json", + ".settings/migration-fixtures/226.json", + ".settings/migration-fixtures/227.json", + ".settings/migration-fixtures/228.json", + ".settings/migration-fixtures/229.json", + ".settings/migration-fixtures/23.json", + ".settings/migration-fixtures/230.json", + ".settings/migration-fixtures/231.json", + ".settings/migration-fixtures/232.json", + ".settings/migration-fixtures/233.json", + ".settings/migration-fixtures/234.json", + ".settings/migration-fixtures/235.json", + ".settings/migration-fixtures/236.json", + ".settings/migration-fixtures/237.json", + ".settings/migration-fixtures/238.json", + ".settings/migration-fixtures/239.json", + ".settings/migration-fixtures/24.json", + ".settings/migration-fixtures/240.json", + ".settings/migration-fixtures/241.json", + ".settings/migration-fixtures/242.json", + ".settings/migration-fixtures/243.json", + ".settings/migration-fixtures/244.json", + ".settings/migration-fixtures/245.json", + ".settings/migration-fixtures/246.json", + ".settings/migration-fixtures/247.json", + ".settings/migration-fixtures/248.json", + ".settings/migration-fixtures/249.json", + ".settings/migration-fixtures/25.json", + ".settings/migration-fixtures/250.json", + ".settings/migration-fixtures/251.json", + ".settings/migration-fixtures/252.json", + ".settings/migration-fixtures/253.json", + ".settings/migration-fixtures/254.json", + ".settings/migration-fixtures/255.json", + ".settings/migration-fixtures/256.json", + ".settings/migration-fixtures/257.json", + ".settings/migration-fixtures/258.json", + ".settings/migration-fixtures/259.json", + ".settings/migration-fixtures/26.json", + ".settings/migration-fixtures/260.json", + ".settings/migration-fixtures/261.json", + ".settings/migration-fixtures/262.json", + ".settings/migration-fixtures/263.json", + ".settings/migration-fixtures/264.json", + ".settings/migration-fixtures/265.json", + ".settings/migration-fixtures/266.json", + ".settings/migration-fixtures/267.json", + ".settings/migration-fixtures/268.json", + ".settings/migration-fixtures/269.json", + ".settings/migration-fixtures/27.json", + ".settings/migration-fixtures/270.json", + ".settings/migration-fixtures/271.json", + ".settings/migration-fixtures/272.json", + ".settings/migration-fixtures/273.json", + ".settings/migration-fixtures/274.json", + ".settings/migration-fixtures/275.json", + ".settings/migration-fixtures/276.json", + ".settings/migration-fixtures/277.json", + ".settings/migration-fixtures/278.json", + ".settings/migration-fixtures/279.json", + ".settings/migration-fixtures/28.json", + ".settings/migration-fixtures/280.json", + ".settings/migration-fixtures/281.json", + ".settings/migration-fixtures/282.json", + ".settings/migration-fixtures/283.json", + ".settings/migration-fixtures/284.json", + ".settings/migration-fixtures/285.json", + ".settings/migration-fixtures/286.json", + ".settings/migration-fixtures/287.json", + ".settings/migration-fixtures/288.json", + ".settings/migration-fixtures/289.json", + ".settings/migration-fixtures/29.json", + ".settings/migration-fixtures/290.json", + ".settings/migration-fixtures/291.json", + ".settings/migration-fixtures/292.json", + ".settings/migration-fixtures/293.json", + ".settings/migration-fixtures/294.json", + ".settings/migration-fixtures/295.json", + ".settings/migration-fixtures/296.json", + ".settings/migration-fixtures/297.json", + ".settings/migration-fixtures/298.json", + ".settings/migration-fixtures/299.json", + ".settings/migration-fixtures/3.json", + ".settings/migration-fixtures/30.json", + ".settings/migration-fixtures/300.json", + ".settings/migration-fixtures/301.json", + ".settings/migration-fixtures/302.json", + ".settings/migration-fixtures/303.json", + ".settings/migration-fixtures/304.json", + ".settings/migration-fixtures/305.json", + ".settings/migration-fixtures/306.json", + ".settings/migration-fixtures/307.json", + ".settings/migration-fixtures/308.json", + ".settings/migration-fixtures/309.json", + ".settings/migration-fixtures/31.json", + ".settings/migration-fixtures/310.json", + ".settings/migration-fixtures/311.json", + ".settings/migration-fixtures/312.json", + ".settings/migration-fixtures/313.json", + ".settings/migration-fixtures/314.json", + ".settings/migration-fixtures/315.json", + ".settings/migration-fixtures/316.json", + ".settings/migration-fixtures/317.json", + ".settings/migration-fixtures/318.json", + ".settings/migration-fixtures/319.json", + ".settings/migration-fixtures/32.json", + ".settings/migration-fixtures/320.json", + ".settings/migration-fixtures/321.json", + ".settings/migration-fixtures/322.json", + ".settings/migration-fixtures/323.json", + ".settings/migration-fixtures/324.json", + ".settings/migration-fixtures/325.json", + ".settings/migration-fixtures/326.json", + ".settings/migration-fixtures/327.json", + ".settings/migration-fixtures/328.json", + ".settings/migration-fixtures/329.json", + ".settings/migration-fixtures/33.json", + ".settings/migration-fixtures/330.json", + ".settings/migration-fixtures/331.json", + ".settings/migration-fixtures/332.json", + ".settings/migration-fixtures/333.json", + ".settings/migration-fixtures/334.json", + ".settings/migration-fixtures/335.json", + ".settings/migration-fixtures/336.json", + ".settings/migration-fixtures/337.json", + ".settings/migration-fixtures/338.json", + ".settings/migration-fixtures/339.json", + ".settings/migration-fixtures/34.json", + ".settings/migration-fixtures/340.json", + ".settings/migration-fixtures/341.json", + ".settings/migration-fixtures/342.json", + ".settings/migration-fixtures/343.json", + ".settings/migration-fixtures/344.json", + ".settings/migration-fixtures/345.json", + ".settings/migration-fixtures/346.json", + ".settings/migration-fixtures/347.json", + ".settings/migration-fixtures/348.json", + ".settings/migration-fixtures/349.json", + ".settings/migration-fixtures/35.json", + ".settings/migration-fixtures/350.json", + ".settings/migration-fixtures/351.json", + ".settings/migration-fixtures/352.json", + ".settings/migration-fixtures/353.json", + ".settings/migration-fixtures/354.json", + ".settings/migration-fixtures/355.json", + ".settings/migration-fixtures/356.json", + ".settings/migration-fixtures/357.json", + ".settings/migration-fixtures/358.json", + ".settings/migration-fixtures/359.json", + ".settings/migration-fixtures/36.json", + ".settings/migration-fixtures/360.json", + ".settings/migration-fixtures/361.json", + ".settings/migration-fixtures/362.json", + ".settings/migration-fixtures/363.json", + ".settings/migration-fixtures/364.json", + ".settings/migration-fixtures/365.json", + ".settings/migration-fixtures/366.json", + ".settings/migration-fixtures/367.json", + ".settings/migration-fixtures/368.json", + ".settings/migration-fixtures/369.json", + ".settings/migration-fixtures/37.json", + ".settings/migration-fixtures/370.json", + ".settings/migration-fixtures/371.json", + ".settings/migration-fixtures/372.json", + ".settings/migration-fixtures/373.json", + ".settings/migration-fixtures/374.json", + ".settings/migration-fixtures/375.json", + ".settings/migration-fixtures/376.json", + ".settings/migration-fixtures/377.json", + ".settings/migration-fixtures/378.json", + ".settings/migration-fixtures/379.json", + ".settings/migration-fixtures/38.json", + ".settings/migration-fixtures/380.json", + ".settings/migration-fixtures/381.json", + ".settings/migration-fixtures/382.json", + ".settings/migration-fixtures/383.json", + ".settings/migration-fixtures/384.json", + ".settings/migration-fixtures/385.json", + ".settings/migration-fixtures/386.json", + ".settings/migration-fixtures/387.json", + ".settings/migration-fixtures/388.json", + ".settings/migration-fixtures/389.json", + ".settings/migration-fixtures/39.json", + ".settings/migration-fixtures/390.json", + ".settings/migration-fixtures/391.json", + ".settings/migration-fixtures/392.json", + ".settings/migration-fixtures/393.json", + ".settings/migration-fixtures/394.json", + ".settings/migration-fixtures/395.json", + ".settings/migration-fixtures/396.json", + ".settings/migration-fixtures/397.json", + ".settings/migration-fixtures/398.json", + ".settings/migration-fixtures/399.json", + ".settings/migration-fixtures/4.json", + ".settings/migration-fixtures/40.json", + ".settings/migration-fixtures/400.json", + ".settings/migration-fixtures/401.json", + ".settings/migration-fixtures/402.json", + ".settings/migration-fixtures/403.json", + ".settings/migration-fixtures/404.json", + ".settings/migration-fixtures/405.json", + ".settings/migration-fixtures/406.json", + ".settings/migration-fixtures/407.json", + ".settings/migration-fixtures/408.json", + ".settings/migration-fixtures/409.json", + ".settings/migration-fixtures/41.json", + ".settings/migration-fixtures/410.json", + ".settings/migration-fixtures/411.json", + ".settings/migration-fixtures/412.json", + ".settings/migration-fixtures/413.json", + ".settings/migration-fixtures/414.json", + ".settings/migration-fixtures/415.json", + ".settings/migration-fixtures/416.json", + ".settings/migration-fixtures/417.json", + ".settings/migration-fixtures/418.json", + ".settings/migration-fixtures/419.json", + ".settings/migration-fixtures/42.json", + ".settings/migration-fixtures/420.json", + ".settings/migration-fixtures/421.json", + ".settings/migration-fixtures/422.json", + ".settings/migration-fixtures/423.json", + ".settings/migration-fixtures/424.json", + ".settings/migration-fixtures/425.json", + ".settings/migration-fixtures/426.json", + ".settings/migration-fixtures/427.json", + ".settings/migration-fixtures/428.json", + ".settings/migration-fixtures/429.json", + ".settings/migration-fixtures/43.json", + ".settings/migration-fixtures/430.json", + ".settings/migration-fixtures/431.json", + ".settings/migration-fixtures/432.json", + ".settings/migration-fixtures/433.json", + ".settings/migration-fixtures/434.json", + ".settings/migration-fixtures/435.json", + ".settings/migration-fixtures/436.json", + ".settings/migration-fixtures/437.json", + ".settings/migration-fixtures/438.json", + ".settings/migration-fixtures/439.json", + ".settings/migration-fixtures/44.json", + ".settings/migration-fixtures/440.json", + ".settings/migration-fixtures/441.json", + ".settings/migration-fixtures/442.json", + ".settings/migration-fixtures/443.json", + ".settings/migration-fixtures/444.json", + ".settings/migration-fixtures/445.json", + ".settings/migration-fixtures/446.json", + ".settings/migration-fixtures/447.json", + ".settings/migration-fixtures/448.json", + ".settings/migration-fixtures/449.json", + ".settings/migration-fixtures/45.json", + ".settings/migration-fixtures/450.json", + ".settings/migration-fixtures/451.json", + ".settings/migration-fixtures/452.json", + ".settings/migration-fixtures/453.json", + ".settings/migration-fixtures/454.json", + ".settings/migration-fixtures/455.json", + ".settings/migration-fixtures/456.json", + ".settings/migration-fixtures/457.json", + ".settings/migration-fixtures/458.json", + ".settings/migration-fixtures/459.json", + ".settings/migration-fixtures/46.json", + ".settings/migration-fixtures/460.json", + ".settings/migration-fixtures/461.json", + ".settings/migration-fixtures/462.json", + ".settings/migration-fixtures/463.json", + ".settings/migration-fixtures/464.json", + ".settings/migration-fixtures/465.json", + ".settings/migration-fixtures/466.json", + ".settings/migration-fixtures/467.json", + ".settings/migration-fixtures/468.json", + ".settings/migration-fixtures/469.json", + ".settings/migration-fixtures/47.json", + ".settings/migration-fixtures/470.json", + ".settings/migration-fixtures/471.json", + ".settings/migration-fixtures/472.json", + ".settings/migration-fixtures/473.json", + ".settings/migration-fixtures/474.json", + ".settings/migration-fixtures/475.json", + ".settings/migration-fixtures/476.json", + ".settings/migration-fixtures/477.json", + ".settings/migration-fixtures/478.json", + ".settings/migration-fixtures/479.json", + ".settings/migration-fixtures/48.json", + ".settings/migration-fixtures/480.json", + ".settings/migration-fixtures/481.json", + ".settings/migration-fixtures/482.json", + ".settings/migration-fixtures/483.json", + ".settings/migration-fixtures/484.json", + ".settings/migration-fixtures/485.json", + ".settings/migration-fixtures/486.json", + ".settings/migration-fixtures/487.json", + ".settings/migration-fixtures/488.json", + ".settings/migration-fixtures/489.json", + ".settings/migration-fixtures/49.json", + ".settings/migration-fixtures/490.json", + ".settings/migration-fixtures/491.json", + ".settings/migration-fixtures/492.json", + ".settings/migration-fixtures/493.json", + ".settings/migration-fixtures/494.json", + ".settings/migration-fixtures/495.json", + ".settings/migration-fixtures/496.json", + ".settings/migration-fixtures/497.json", + ".settings/migration-fixtures/498.json", + ".settings/migration-fixtures/499.json", + ".settings/migration-fixtures/5.json", + ".settings/migration-fixtures/50.json", + ".settings/migration-fixtures/51.json", + ".settings/migration-fixtures/52.json", + ".settings/migration-fixtures/53.json", + ".settings/migration-fixtures/54.json", + ".settings/migration-fixtures/55.json", + ".settings/migration-fixtures/56.json", + ".settings/migration-fixtures/57.json", + ".settings/migration-fixtures/58.json", + ".settings/migration-fixtures/59.json", + ".settings/migration-fixtures/6.json", + ".settings/migration-fixtures/60.json", + ".settings/migration-fixtures/61.json", + ".settings/migration-fixtures/62.json", + ".settings/migration-fixtures/63.json", + ".settings/migration-fixtures/64.json", + ".settings/migration-fixtures/65.json", + ".settings/migration-fixtures/66.json", + ".settings/migration-fixtures/67.json", + ".settings/migration-fixtures/68.json", + ".settings/migration-fixtures/69.json", + ".settings/migration-fixtures/7.json", + ".settings/migration-fixtures/70.json", + ".settings/migration-fixtures/71.json", + ".settings/migration-fixtures/72.json", + ".settings/migration-fixtures/73.json", + ".settings/migration-fixtures/74.json", + ".settings/migration-fixtures/75.json", + ".settings/migration-fixtures/76.json", + ".settings/migration-fixtures/77.json", + ".settings/migration-fixtures/78.json", + ".settings/migration-fixtures/79.json", + ".settings/migration-fixtures/8.json", + ".settings/migration-fixtures/80.json", + ".settings/migration-fixtures/81.json", + ".settings/migration-fixtures/82.json", + ".settings/migration-fixtures/83.json", + ".settings/migration-fixtures/84.json", + ".settings/migration-fixtures/85.json", + ".settings/migration-fixtures/86.json", + ".settings/migration-fixtures/87.json", + ".settings/migration-fixtures/88.json", + ".settings/migration-fixtures/89.json", + ".settings/migration-fixtures/9.json", + ".settings/migration-fixtures/90.json", + ".settings/migration-fixtures/91.json", + ".settings/migration-fixtures/92.json", + ".settings/migration-fixtures/93.json", + ".settings/migration-fixtures/94.json", + ".settings/migration-fixtures/95.json", + ".settings/migration-fixtures/96.json", + ".settings/migration-fixtures/97.json", + ".settings/migration-fixtures/98.json", + ".settings/migration-fixtures/99.json", + ".users/.dashboard-locations/01M75R7GF09K6AAC2SFQVDMCY3.json", + ".users/.dashboard-titles/2b9aa3aaf18aa9aa60fcc45d30833be66dc5907be190dfff57197c19883297e4.json", + ".users/8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918/dashboards/01M75R7GF09K6AAC2SFQVDMCY3.json", + ".users/8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918/filters/first/01M75R7GFTZ5J1962A6DH7F76H.json", + "first/.stream/.schema", + "first/.stream/.stream.json", + "first/date=2026-11-11/.manifest.json", + "first/date=2026-11-11/hour=16/minute=49/.data.01M75R8CC3NZ3FYFDNS7SYHNC4.parquet", + "first/date=2026-11-11/hour=16/minute=51/.data.01M75RC1EGSNZ3A495S8YYPXQS.parquet", + "partitioned/.stream/.schema", + "partitioned/.stream/.stream.json", + "partitioned/date=2026-11-11/.manifest.json", + "partitioned/date=2026-11-11/hour=16/minute=49/region=west/.data.01M75R8CC3M7ENK36HXV3K5Z40.parquet", + "pstats/.stream/.schema", + "pstats/.stream/.stream.json", + "pstats/date=2026-11-11/.manifest.json", + "pstats/date=2026-11-11/hour=16/minute=49/dataset_name=first/.data.01M75RA6Z3CTNEVGMEAQ4YX6MZ.parquet", + "pstats/date=2026-11-11/hour=16/minute=49/dataset_name=partitioned/.data.01M75RA6Z3CTNEVGMEAQ4YX6MZ.parquet", + "pstats/date=2026-11-11/hour=16/minute=49/dataset_name=second/.data.01M75RA6Z3CTNEVGMEAQ4YX6MZ.parquet", + "pstats/date=2026-11-11/hour=16/minute=51/dataset_name=first/.data.01M75RDW1HJ9H57GDBS6TY1PHN.parquet", + "second/.stream/.schema", + "second/.stream/.stream.json", + "second/date=2026-11-11/.manifest.json", + "second/date=2026-11-11/hour=16/minute=49/.data.01M75R8CC3AGT026GHYTM8M2CJ.parquet" + ] +} diff --git a/docs/validation/postgres-review-explain.txt b/docs/validation/postgres-review-explain.txt new file mode 100644 index 000000000..c7cdbc081 --- /dev/null +++ b/docs/validation/postgres-review-explain.txt @@ -0,0 +1,133 @@ +CREATE TABLE +CREATE SEQUENCE +CREATE TABLE +ALTER TABLE +CREATE INDEX +CREATE FUNCTION +DROP TRIGGER +CREATE TRIGGER +UPDATE 0 +INSERT 0 200000 +INSERT 0 400 +ANALYZE + count | count +--------+------- + 200400 | 4 +(1 row) + + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Sort (cost=16.88..16.88 rows=2 width=43) (actual time=8.587..8.656 rows=2000 loops=1) + Sort Key: key COLLATE "C" + Sort Method: quicksort Memory: 49kB + Buffers: shared hit=974 read=1064 written=961 + -> Bitmap Heap Scan on parseable_documents (cost=8.88..16.87 rows=2 width=43) (actual time=0.765..7.578 rows=2000 loops=1) + Recheck Cond: (((tenant = 'tenant-1'::text) AND (key = 'tenant-1/logs/date=001'::text)) OR ((tenant = 'tenant-1'::text) AND (key >= 'tenant-1/logs/date=001/'::text) AND (key < 'tenant-1/logs/date=0010'::text))) + Heap Blocks: exact=2000 + Buffers: shared hit=974 read=1064 written=961 + -> BitmapOr (cost=8.88..8.88 rows=2 width=0) (actual time=0.520..0.522 rows=0 loops=1) + Buffers: shared hit=25 read=13 + -> Bitmap Index Scan on parseable_documents_pkey (cost=0.00..4.43 rows=1 width=0) (actual time=0.009..0.009 rows=0 loops=1) + Index Cond: ((tenant = 'tenant-1'::text) AND (key = 'tenant-1/logs/date=001'::text)) + Buffers: shared hit=3 + -> Bitmap Index Scan on parseable_documents_pkey (cost=0.00..4.45 rows=2 width=0) (actual time=0.511..0.511 rows=2000 loops=1) + Index Cond: ((tenant = 'tenant-1'::text) AND (key >= 'tenant-1/logs/date=001/'::text) AND (key < 'tenant-1/logs/date=0010'::text)) + Buffers: shared hit=22 read=13 + Planning: + Buffers: shared hit=11 + Planning Time: 0.200 ms + Execution Time: 8.808 ms +(20 rows) + + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Sort (cost=16.88..16.88 rows=2 width=1068) (actual time=3.188..3.240 rows=2000 loops=1) + Sort Key: key COLLATE "C" + Sort Method: quicksort Memory: 2174kB + Buffers: shared hit=2038 + -> Bitmap Heap Scan on parseable_documents (cost=8.88..16.87 rows=2 width=1068) (actual time=0.645..1.645 rows=2000 loops=1) + Recheck Cond: (((tenant = 'tenant-1'::text) AND (key = 'tenant-1/logs/date=001'::text)) OR ((tenant = 'tenant-1'::text) AND (key >= 'tenant-1/logs/date=001/'::text) AND (key < 'tenant-1/logs/date=0010'::text))) + Heap Blocks: exact=2000 + Buffers: shared hit=2038 + -> BitmapOr (cost=8.88..8.88 rows=2 width=0) (actual time=0.410..0.411 rows=0 loops=1) + Buffers: shared hit=38 + -> Bitmap Index Scan on parseable_documents_pkey (cost=0.00..4.43 rows=1 width=0) (actual time=0.009..0.009 rows=0 loops=1) + Index Cond: ((tenant = 'tenant-1'::text) AND (key = 'tenant-1/logs/date=001'::text)) + Buffers: shared hit=3 + -> Bitmap Index Scan on parseable_documents_pkey (cost=0.00..4.45 rows=2 width=0) (actual time=0.400..0.400 rows=2000 loops=1) + Index Cond: ((tenant = 'tenant-1'::text) AND (key >= 'tenant-1/logs/date=001/'::text) AND (key < 'tenant-1/logs/date=0010'::text)) + Buffers: shared hit=35 + Planning: + Buffers: shared hit=3 + Planning Time: 0.157 ms + Execution Time: 3.436 ms +(20 rows) + + QUERY PLAN +----------------------------------------------------------------------------------------------------------------------------------------------------------- + Index Scan using parseable_documents_pkey on parseable_documents (cost=0.42..12.87 rows=1 width=1068) (actual time=0.050..0.053 rows=2 loops=1) + Index Cond: ((tenant = 'tenant-1'::text) AND (key = ANY ('{tenant-1/logs/date=001/1.manifest.json,tenant-1/logs/date=001/101.manifest.json}'::text[]))) + Buffers: shared hit=5 + Planning Time: 0.075 ms + Execution Time: 0.070 ms +(5 rows) + + QUERY PLAN +--------------------------------------------------------------------------------------------------------------------------------------------------------- + Index Only Scan using parseable_stream_inventory on parseable_documents (cost=0.27..23.76 rows=5 width=43) (actual time=0.044..0.076 rows=100 loops=1) + Index Cond: (tenant = 'tenant-1'::text) + Heap Fetches: 100 + Buffers: shared hit=41 + Planning: + Buffers: shared hit=2 + Planning Time: 0.194 ms + Execution Time: 0.093 ms +(8 rows) + +BEGIN + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Delete on parseable_documents (cost=8.88..16.87 rows=0 width=0) (actual time=2.070..2.071 rows=0 loops=1) + Buffers: shared hit=4038 dirtied=1054 + -> Bitmap Heap Scan on parseable_documents (cost=8.88..16.87 rows=2 width=6) (actual time=0.465..1.629 rows=2000 loops=1) + Recheck Cond: (((tenant = 'tenant-1'::text) AND (key = 'tenant-1/logs/date=001'::text)) OR ((tenant = 'tenant-1'::text) AND (key >= 'tenant-1/logs/date=001/'::text) AND (key < 'tenant-1/logs/date=0010'::text))) + Heap Blocks: exact=2000 + Buffers: shared hit=2038 + -> BitmapOr (cost=8.88..8.88 rows=2 width=0) (actual time=0.300..0.301 rows=0 loops=1) + Buffers: shared hit=38 + -> Bitmap Index Scan on parseable_documents_pkey (cost=0.00..4.43 rows=1 width=0) (actual time=0.006..0.006 rows=0 loops=1) + Index Cond: ((tenant = 'tenant-1'::text) AND (key = 'tenant-1/logs/date=001'::text)) + Buffers: shared hit=3 + -> Bitmap Index Scan on parseable_documents_pkey (cost=0.00..4.45 rows=2 width=0) (actual time=0.294..0.294 rows=2000 loops=1) + Index Cond: ((tenant = 'tenant-1'::text) AND (key >= 'tenant-1/logs/date=001/'::text) AND (key < 'tenant-1/logs/date=0010'::text)) + Buffers: shared hit=35 + Planning Time: 0.061 ms + Execution Time: 2.156 ms +(16 rows) + +ROLLBACK + rows_after_populated_collation_migration +------------------------------------------ + 200400 +(1 row) + + attcollation +-------------- + "C" +(1 row) + + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------ + Result (cost=18.65..18.66 rows=1 width=1) (actual time=0.047..0.048 rows=1 loops=1) + Buffers: shared hit=2 read=2 + InitPlan 1 + -> Index Only Scan using parseable_documents_pkey on parseable_documents (cost=0.42..18.65 rows=1 width=0) (actual time=0.046..0.046 rows=1 loops=1) + Index Cond: ((tenant = 'tenant-1'::text) AND (key >= 'tenant-1/stream-1/.stream/'::text) AND (key < 'tenant-1/stream-1/.stream0'::text)) + Filter: (key ~~ '%stream.json'::text) + Heap Fetches: 0 + Buffers: shared hit=2 read=2 + Planning: + Buffers: shared hit=102 read=2 + Planning Time: 0.772 ms + Execution Time: 0.101 ms +(12 rows) diff --git a/docs/validation/postgres-review-smoke.json b/docs/validation/postgres-review-smoke.json new file mode 100644 index 000000000..006f48ef9 --- /dev/null +++ b/docs/validation/postgres-review-smoke.json @@ -0,0 +1,205 @@ +{ + "object-store": { + "lazy_reload": true, + "query": "[{\"count\":1}]", + "dashboard": { + "version": "v1", + "title": "Postgres smoke", + "author": "8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918", + "dashboardId": "01M74QAKS40EBX60MNDX4ZJRJH", + "created": "2026-11-11T07:14:30.820382797+00:00", + "modified": "2026-11-11T07:14:30.820382797+00:00", + "tags": null, + "isFavorite": false, + "dashboardType": "Dashboard", + "tiles": [], + "tenantId": null, + "revision": "410760f4f6ce9417312ed35635650d0ed5d2121c45bd21069d7bea97a6214f64" + }, + "keys": [ + ".parseable/.parseable.json", + ".parseable/ingestor.991bf8aa66a3be0.json", + ".parseable/querier.ce5fd278a14f75c.json", + ".users/.dashboard-locations/01M74QAKS40EBX60MNDX4ZJRJH.json", + ".users/.dashboard-titles/571a2cefe2d0d1a9a26dab4810a5556875c7d7531d4f558a8f7b28ab486e2395.json", + ".users/8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918/dashboards/01M74QAKS40EBX60MNDX4ZJRJH.json", + "lazy/.stream/.ingestor.991bf8aa66a3be0.schema", + "lazy/.stream/.ingestor.991bf8aa66a3be0.stream.json", + "lazy/.stream/.schema", + "lazy/.stream/.stream.json", + "lazy/date=2026-11-11/hour=07/minute=14/991bf8aa66a3be0.data.01M74QBH6TVHY0D3F6775WMZDN.parquet", + "lazy/date=2026-11-11/ingestor..991bf8aa66a3be0.manifest.json", + "pstats/.stream/.ingestor.991bf8aa66a3be0.schema", + "pstats/.stream/.ingestor.991bf8aa66a3be0.stream.json", + "pstats/.stream/.schema", + "pstats/.stream/.stream.json", + "pstats/date=2026-11-11/.manifest.json", + "pstats/date=2026-11-11/hour=07/minute=13/dataset_name=smoke/.data.01M74QARNVN5BPQ7Z8CJJJYE04.parquet", + "smoke/.stream/.ingestor.991bf8aa66a3be0.schema", + "smoke/.stream/.ingestor.991bf8aa66a3be0.stream.json", + "smoke/.stream/.schema", + "smoke/.stream/.stream.json", + "smoke/date=2026-11-11/.manifest.json", + "smoke/date=2026-11-11/hour=07/minute=13/.data.01M74Q9NS8KCT5185X617N4M6Y.parquet" + ], + "non_parquet_keys": [ + ".parseable/.parseable.json", + ".parseable/ingestor.991bf8aa66a3be0.json", + ".parseable/querier.ce5fd278a14f75c.json", + ".users/.dashboard-locations/01M74QAKS40EBX60MNDX4ZJRJH.json", + ".users/.dashboard-titles/571a2cefe2d0d1a9a26dab4810a5556875c7d7531d4f558a8f7b28ab486e2395.json", + ".users/8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918/dashboards/01M74QAKS40EBX60MNDX4ZJRJH.json", + "lazy/.stream/.ingestor.991bf8aa66a3be0.schema", + "lazy/.stream/.ingestor.991bf8aa66a3be0.stream.json", + "lazy/.stream/.schema", + "lazy/.stream/.stream.json", + "lazy/date=2026-11-11/ingestor..991bf8aa66a3be0.manifest.json", + "pstats/.stream/.ingestor.991bf8aa66a3be0.schema", + "pstats/.stream/.ingestor.991bf8aa66a3be0.stream.json", + "pstats/.stream/.schema", + "pstats/.stream/.stream.json", + "pstats/date=2026-11-11/.manifest.json", + "smoke/.stream/.ingestor.991bf8aa66a3be0.schema", + "smoke/.stream/.ingestor.991bf8aa66a3be0.stream.json", + "smoke/.stream/.schema", + "smoke/.stream/.stream.json", + "smoke/date=2026-11-11/.manifest.json" + ], + "request_delta": { + "parseable_storage_request_response_time_count{method=\"GET\",provider=\"s3\",status=\"200\"}": 30.0, + "parseable_storage_request_response_time_count{method=\"GET_RANGES\",provider=\"s3\",status=\"200\"}": 5.0, + "parseable_storage_request_response_time_count{method=\"LIST\",provider=\"s3\",status=\"200\"}": 5.0, + "parseable_storage_request_response_time_count{method=\"LIST_DELIM\",provider=\"s3\",status=\"200\"}": 5.0 + }, + "requests_per_query": 9.0, + "bucket_outage_readiness": 503, + "delete_recreate": true, + "final_keys": [ + ".parseable/.parseable.json", + ".parseable/ingestor.991bf8aa66a3be0.json", + ".parseable/querier.ce5fd278a14f75c.json", + ".users/.dashboard-locations/01M74QAKS40EBX60MNDX4ZJRJH.json", + ".users/.dashboard-titles/571a2cefe2d0d1a9a26dab4810a5556875c7d7531d4f558a8f7b28ab486e2395.json", + ".users/8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918/dashboards/01M74QAKS40EBX60MNDX4ZJRJH.json", + "lazy/.stream/.ingestor.991bf8aa66a3be0.schema", + "lazy/.stream/.ingestor.991bf8aa66a3be0.stream.json", + "lazy/.stream/.schema", + "lazy/.stream/.stream.json", + "lazy/date=2026-11-11/hour=07/minute=14/991bf8aa66a3be0.data.01M74QBH6TVHY0D3F6775WMZDN.parquet", + "lazy/date=2026-11-11/ingestor..991bf8aa66a3be0.manifest.json", + "pstats/.stream/.ingestor.991bf8aa66a3be0.schema", + "pstats/.stream/.ingestor.991bf8aa66a3be0.stream.json", + "pstats/.stream/.schema", + "pstats/.stream/.stream.json", + "pstats/date=2026-11-11/.manifest.json", + "pstats/date=2026-11-11/hour=07/minute=13/dataset_name=smoke/.data.01M74QARNVN5BPQ7Z8CJJJYE04.parquet", + "pstats/date=2026-11-11/hour=07/minute=14/dataset_name=lazy/991bf8aa66a3be0.data.01M74QDBSTHC3RTVX7DSBDG4R6.parquet", + "pstats/date=2026-11-11/ingestor..991bf8aa66a3be0.manifest.json", + "smoke/.stream/.ingestor.991bf8aa66a3be0.schema", + "smoke/.stream/.ingestor.991bf8aa66a3be0.stream.json", + "smoke/.stream/.schema", + "smoke/.stream/.stream.json", + "smoke/date=2026-11-11/hour=07/minute=15/991bf8aa66a3be0.data.01M74QDBSTPQCEJ9QBFKANH753.parquet", + "smoke/date=2026-11-11/ingestor..991bf8aa66a3be0.manifest.json" + ] + }, + "postgres-tls": { + "lazy_reload": true, + "query": "[{\"count\":1}]", + "dashboard": { + "version": "v1", + "title": "Postgres smoke", + "author": "8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918", + "dashboardId": "01M74QG3P3MHGHPFMD9S98MPTQ", + "created": "2026-11-11T07:17:30.947201084+00:00", + "modified": "2026-11-11T07:17:30.947201084+00:00", + "tags": null, + "isFavorite": false, + "dashboardType": "Dashboard", + "tiles": [], + "tenantId": null, + "revision": "4d9eab070b7637613a929c14f1303c15ac4e4f9ad1e9c8066863366625de52d4" + }, + "keys": [ + ".parseable/postgres-metastore-id", + "lazy/date=2026-11-11/hour=07/minute=17/0c643310e0de82d.data.01M74QH0C4PR4JWSPKVY2FQYD9.parquet", + "pstats/date=2026-11-11/hour=07/minute=16/dataset_name=smoke/.data.01M74QG8JNMAZ9Z6MJWE1BE3SK.parquet", + "smoke/date=2026-11-11/hour=07/minute=16/.data.01M74QF5SMF7ND9BTQE3139FZ6.parquet" + ], + "non_parquet_keys": [ + ".parseable/postgres-metastore-id" + ], + "request_delta": { + "parseable_storage_request_response_time_count{method=\"GET_RANGES\",provider=\"s3\",status=\"200\"}": 5.0 + }, + "requests_per_query": 1.0, + "database_outage": { + "readiness": 503, + "request_failed": true, + "seconds": 0.004562790039926767 + }, + "bucket_outage_readiness": 503, + "deleted_stream_rows": 0, + "delete_recreate": true, + "final_keys": [ + ".parseable/postgres-metastore-id", + "lazy/date=2026-11-11/hour=07/minute=17/0c643310e0de82d.data.01M74QH0C4PR4JWSPKVY2FQYD9.parquet", + "pstats/date=2026-11-11/hour=07/minute=16/dataset_name=smoke/.data.01M74QG8JNMAZ9Z6MJWE1BE3SK.parquet", + "pstats/date=2026-11-11/hour=07/minute=17/dataset_name=lazy/0c643310e0de82d.data.01M74QJTZ4XZF0G5S9QQYCKVXT.parquet", + "smoke/date=2026-11-11/hour=07/minute=18/0c643310e0de82d.data.01M74QJTZ4TK4Y4VW9DNG33BXF.parquet" + ], + "untrusted_ca_rejected": true + }, + "postgres": { + "lazy_reload": true, + "query": "[{\"count\":1}]", + "dashboard": { + "version": "v1", + "title": "Postgres smoke", + "author": "8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918", + "dashboardId": "01M74QNKSP83NNEMXQKJET4Q55", + "created": "2026-11-11T07:20:31.286622700+00:00", + "modified": "2026-11-11T07:20:31.286622700+00:00", + "tags": null, + "isFavorite": false, + "dashboardType": "Dashboard", + "tiles": [], + "tenantId": null, + "revision": "0fff761b383dd11e360a71fdb08cf1df18bb871cab49903853a19b1aa608d99c" + }, + "keys": [ + ".parseable/postgres-metastore-id", + "lazy/date=2026-11-11/hour=07/minute=20/b98cdf90ef5f01d.data.01M74QPGD6ZHAMA3M61E0P12J1.parquet", + "pstats/date=2026-11-11/hour=07/minute=19/dataset_name=smoke/.data.01M74QNRP8KGH60YDSV2PMSBRQ.parquet", + "smoke/date=2026-11-11/hour=07/minute=19/.data.01M74QMNWYNPWNFQVWVVKRYPFH.parquet" + ], + "non_parquet_keys": [ + ".parseable/postgres-metastore-id" + ], + "request_delta": { + "parseable_storage_request_response_time_count{method=\"GET_RANGES\",provider=\"s3\",status=\"200\"}": 5.0 + }, + "requests_per_query": 1.0, + "database_outage": { + "readiness": 503, + "request_failed": true, + "seconds": 0.003669349942356348 + }, + "bucket_outage_readiness": 503, + "deleted_stream_rows": 0, + "delete_recreate": true, + "final_keys": [ + ".parseable/postgres-metastore-id", + "lazy/date=2026-11-11/hour=07/minute=20/b98cdf90ef5f01d.data.01M74QPGD6ZHAMA3M61E0P12J1.parquet", + "pstats/date=2026-11-11/hour=07/minute=19/dataset_name=smoke/.data.01M74QNRP8KGH60YDSV2PMSBRQ.parquet", + "pstats/date=2026-11-11/hour=07/minute=20/dataset_name=lazy/b98cdf90ef5f01d.data.01M74QRB07K2NWMWJCFB4WYB7G.parquet", + "smoke/date=2026-11-11/hour=07/minute=21/b98cdf90ef5f01d.data.01M74QRB07D45VQGFJ851YWQ8R.parquet" + ] + }, + "startup_rejections": { + "future-schema": "Error: ObjectStorageError: PostgreSQL metastore unavailable: unsupported schema/protocol version; upgrade this server\n\nCaused by:\n PostgreSQL metastore unavailable: unsupported schema/protocol version; upgrade this server\n", + "wrong-id": "Error: ObjectStorageError: PostgreSQL metastore unavailable: backend/bucket identity mismatch\n\nCaused by:\n PostgreSQL metastore unavailable: backend/bucket identity mismatch\n", + "unreachable": "Error: ObjectStorageError: PostgreSQL metastore unavailable: Error occurred while creating a new object: error connecting to server\n\nCaused by:\n PostgreSQL metastore unavailable: Error occurred while creating a new object: error connecting to server\n" + }, + "postgres_rows": "tenant | kind | key | version | bytes \n--------+----------+--------------------------------------------------------------------------------------------------------------------+---------+-------\n | document | .parseable/.parseable.json | 27 | 384\n | document | .parseable/ingestor.b98cdf90ef5f01d.json | 37 | 241\n | document | .parseable/querier.e89f2bd5be65301.json | 28 | 240\n | document | .users/.dashboard-locations/01M74QNKSP83NNEMXQKJET4Q55.json | 16 | 119\n | document | .users/.dashboard-titles/571a2cefe2d0d1a9a26dab4810a5556875c7d7531d4f558a8f7b28ab486e2395.json | 18 | 170\n | document | .users/8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918/dashboards/01M74QNKSP83NNEMXQKJET4Q55.json | 17 | 545\n | schema | lazy/.stream/.ingestor.b98cdf90ef5f01d.schema | 54 | 585\n | stream | lazy/.stream/.ingestor.b98cdf90ef5f01d.stream.json | 52 | 853\n | schema | lazy/.stream/.schema | 56 | 585\n | stream | lazy/.stream/.stream.json | 55 | 597\n | manifest | lazy/date=2026-11-11/ingestor..b98cdf90ef5f01d.manifest.json | 50 | 423\n | schema | pstats/.stream/.ingestor.b98cdf90ef5f01d.schema | 81 | 1117\n | stream | pstats/.stream/.ingestor.b98cdf90ef5f01d.stream.json | 79 | 895\n | schema | pstats/.stream/.schema | 32 | 1117\n | stream | pstats/.stream/.stream.json | 21 | 867\n | manifest | pstats/date=2026-11-11/.manifest.json | 19 | 559\n | manifest | pstats/date=2026-11-11/ingestor..b98cdf90ef5f01d.manifest.json | 77 | 697\n | schema | smoke/.stream/.ingestor.b98cdf90ef5f01d.schema | 73 | 585\n | stream | smoke/.stream/.ingestor.b98cdf90ef5f01d.stream.json | 71 | 854\n | schema | smoke/.stream/.schema | 75 | 585\n | stream | smoke/.stream/.stream.json | 59 | 597\n | manifest | smoke/date=2026-11-11/ingestor..b98cdf90ef5f01d.manifest.json | 69 | 412\n(22 rows)" +} \ No newline at end of file diff --git a/docs/validation/postgres-round2-checks.json b/docs/validation/postgres-round2-checks.json new file mode 100644 index 000000000..63560960a --- /dev/null +++ b/docs/validation/postgres-round2-checks.json @@ -0,0 +1,155 @@ +{ + "success": true, + "base_commit": "b96259b05238bae99ca48d5bee3cee9943b399d5", + "checks": [ + { + "name": "full-library-postgres-minio", + "command": "cargo test --locked --lib -- --test-threads=1", + "passed": 1405, + "failed": 0, + "ignored": 7 + }, + { + "name": "full-library-no-database", + "command": "cargo test --locked --lib -- --nocapture --test-threads=1", + "prerequisites_unset": [ + "P_TEST_POSTGRES_URL", + "P_TEST_REVIEW_S3_URL" + ], + "visible_skips": true, + "passed": 1405, + "failed": 0, + "ignored": 7 + }, + { + "name": "contracts-and-startup", + "command": "cargo test --locked --lib contract_tests:: -- --include-ignored --skip startup_cleanup_child --nocapture --test-threads=1", + "passed": 13, + "failed": 0, + "ignored": 0 + }, + { + "name": "focused-metastore", + "command": "cargo test --locked --lib metastore:: -- --nocapture --test-threads=1", + "passed": 37, + "failed": 0, + "ignored": 3 + }, + { + "name": "localfs-migration", + "command": "cargo test --lib migration::tests::migration_round_trip_postgres -- --nocapture", + "prerequisites_unset": [ + "P_TEST_REVIEW_S3_URL" + ], + "passed": 1, + "failed": 0, + "ignored": 0 + }, + { + "name": "paused-postgres-deadline", + "command": "cargo test --lib postgres_health_deadline_regression -- --nocapture --test-threads=1", + "passed": 1, + "failed": 0, + "ignored": 0 + }, + { + "name": "upload-sibling-drain", + "passed": 1, + "failed": 0, + "ignored": 0 + }, + { + "name": "build", + "command": "cargo build", + "exit_code": 0 + }, + { + "name": "smoke", + "command": "python3 scripts/postgres-metastore-smoke.py", + "modes": [ + "object-store", + "postgres", + "postgres-tls" + ], + "exit_code": 0, + "evidence": "docs/validation/postgres-metastore-smoke.json" + }, + { + "name": "migration-smoke", + "command": "python3 scripts/postgres-metastore-migration-smoke.py", + "modes": [ + "plaintext", + "verified-tls" + ], + "exit_code": 0, + "evidence": [ + "docs/validation/postgres-migration.json", + "docs/validation/postgres-migration-tls.json" + ] + }, + { + "name": "clippy", + "command": "cargo clippy --all-targets", + "exit_code": 0, + "new_warnings": 0, + "existing_library_test_warnings": 21 + }, + { + "name": "script-syntax-and-workflow-yaml", + "exit_code": 0 + }, + { + "name": "diff-whitespace", + "command": "git diff --check b96259b0", + "exit_code": 0 + } + ], + "first_hosted_ci": { + "conclusion": "failure", + "headSha": "b96259b05238bae99ca48d5bee3cee9943b399d5", + "status": "completed", + "url": "https://github.com/VibeEngineeringAI/parseable/actions/runs/43037438052", + "reason": "Runner setup could not resolve Ubuntu package mirror; compilation and tests never started.", + "code_failure": false, + "expanded_workflow_pushed": false + }, + "regression_fixture_note": "Functional metadata SQL tests use a 30-second budget after intermittent shared-host I/O timeouts. Dedicated short deadline and cancellation assertions retain their original budgets.", + "import_estimates": { + "assumptions": [ + "30 days", + "one manifest per stream/day", + "two stream documents per stream", + "two inventories", + "two metadata-body reads", + "snapshot-reference validation reads", + "150 milliseconds per request", + "16 concurrent reads" + ], + "deployments": [ + { + "streams": 20, + "requests": 3180, + "latency_only_lower_bound_seconds": 29.8125 + }, + { + "streams": 1000, + "requests": 159000, + "latency_only_lower_bound_seconds": 1490.625 + } + ], + "limitations": "Estimates, not benchmarks. Discovery dependencies, pagination, SQL, disk and body transfer add time." + }, + "uploader_policy": "Only after confirming PostgreSQL deletion pending: settle upload siblings, compensate remote parquet for the batch, then retire staged parquet and schema. Retain staging when deletion is unconfirmed or cleanup fails.", + "remaining_limits": [ + "Publication delayed beyond final SQL deletion requires stream incarnation fencing (issue 142).", + "SQL cancellation is best effort; timed-out writes and COMMIT can have ambiguous outcomes.", + "Transfers require all writers stopped and a direct or session-pooled PostgreSQL connection.", + "Literal EXPLAIN does not establish generic prepared-plan performance." + ], + "redaction": { + "home_paths": 0, + "checkout_paths": 0, + "hostname_occurrences": 0 + }, + "containers_removed": true +} diff --git a/scripts/postgres-metastore-migration-smoke.py b/scripts/postgres-metastore-migration-smoke.py new file mode 100755 index 000000000..ad205f24a --- /dev/null +++ b/scripts/postgres-metastore-migration-smoke.py @@ -0,0 +1,267 @@ +#!/usr/bin/env python3 +"""Isolated MinIO/PostgreSQL offline migration smoke. No live deployment access. +Uses the existing smoke fixture's helpers; every created container is removed. +""" +from pathlib import Path +import os +import subprocess as sp +import time +import json +import base64 +import datetime as dt +import urllib.request +import signal + +# Load only reusable definitions, never the existing smoke's executable driver. +namespace = {'__file__': str(Path(__file__).with_name('postgres-metastore-smoke.py'))} +source = Path(namespace['__file__']).read_text().split('\ntry:\n pgport, s3port')[0] +exec(compile(source, namespace['__file__'], 'exec'), namespace) +run, port, container, wait_for, list_keys = [namespace[k] for k in ('run','port','container','wait_for','list_keys')] +root, out = namespace['ROOT'], namespace['OUT'] +binary = str(Path(os.environ.get('CARGO_TARGET_DIR', root/'target'))/'debug/parseable') +server = None +transfer_processes = [] +log = open(out / ('migration-tls.log' if os.environ.get('P_MIGRATION_TLS') == '1' else 'migration-plain.log'), 'w') +def stop(): + global server + if server: + server.terminate() + try: server.wait(timeout=30) + except sp.TimeoutExpired: server.kill(); server.wait() + server = None +signal.signal(signal.SIGTERM, namespace['interrupted']) +signal.signal(signal.SIGINT, namespace['interrupted']) +try: + pgport, s3port, http = port(), port(), port() + pg, cmd = container('postgres:17-alpine',POSTGRES_PASSWORD='smoke',POSTGRES_DB='migration') + run(*cmd,'-p',f'127.0.0.1:{pgport}:5432','postgres:17-alpine') + mini, cmd = container('quay.io/parseablehq/minio:edge',MINIO_ROOT_USER='minioadmin',MINIO_ROOT_PASSWORD='minioadmin') + run(*cmd,'-p',f'127.0.0.1:{s3port}:9000','quay.io/parseablehq/minio:edge','server','/data') + run('docker','exec',mini,'mkdir','-p','/data/migration') + wait_for(lambda:'accepting connections' in run('docker','exec',pg,'pg_isready','-h','127.0.0.1','-U','postgres')) + wait_for(lambda: urllib.request.urlopen(f'http://127.0.0.1:{s3port}/minio/health/ready', timeout=3).status == 200) + env = dict(os.environ,P_ADDR=f'127.0.0.1:{http}',P_GRPC_PORT=str(port()),P_FLIGHT_PORT=str(port()), + P_STAGING_DIR=str(out/f'migration-staging-{os.getpid()}'),P_USERNAME='admin',P_PASSWORD='smoke-pass', + P_S3_URL=f'http://127.0.0.1:{s3port}',P_S3_BUCKET='migration',P_S3_REGION='us-east-1', + P_S3_ACCESS_KEY='minioadmin',P_S3_SECRET_KEY='minioadmin',P_S3_PATH_STYLE='true',RUST_LOG='warn', + P_METASTORE_POSTGRES_STATEMENT_TIMEOUT_MS='30000') + env.pop('P_METASTORE_POSTGRES_URL',None) + pgurl = f'postgresql://postgres:smoke@127.0.0.1:{pgport}/migration?sslmode=disable' + if os.environ.get('P_MIGRATION_TLS') == '1': + tls = out / 'migration-tls' + tls.mkdir(exist_ok=True) + run('openssl', 'req', '-x509', '-newkey', 'rsa:2048', '-nodes', '-keyout', str(tls/'ca.key'), '-out', str(tls/'ca.crt'), '-days', '2', '-subj', '/CN=Smoke CA') + run('openssl', 'req', '-newkey', 'rsa:2048', '-nodes', '-keyout', str(tls/'server.key'), '-out', str(tls/'server.csr'), '-subj', '/CN=localhost') + (tls/'extensions').write_text('subjectAltName=DNS:localhost,IP:127.0.0.1\nextendedKeyUsage=serverAuth\n') + run('openssl', 'x509', '-req', '-in', str(tls/'server.csr'), '-CA', str(tls/'ca.crt'), '-CAkey', str(tls/'ca.key'), '-CAcreateserial', '-out', str(tls/'server.crt'), '-days', '2', '-extfile', str(tls/'extensions')) + for name in ['server.key', 'server.crt']: + run('docker', 'cp', str(tls/name), pg+':/var/lib/postgresql/data/'+name) + run('docker', 'exec', pg, 'sh', '-c', 'chown postgres:postgres /var/lib/postgresql/data/server.* && chmod 600 /var/lib/postgresql/data/server.key') + run('docker', 'exec', pg, 'psql', '-U', 'postgres', '-d', 'migration', '-c', "ALTER SYSTEM SET ssl='on';") + run('docker', 'exec', pg, 'psql', '-U', 'postgres', '-d', 'migration', '-c', 'SELECT pg_reload_conf();') + + env['P_METASTORE_POSTGRES_CA'] = str(tls/'ca.crt') + pgurl = pgurl.replace('sslmode=disable', 'sslmode=require') + + def request(path,method='GET',body=None,headers=None): + req = urllib.request.Request(f'http://127.0.0.1:{http}/api/v1'+path,method=method, + data=None if body is None else json.dumps(body).encode(),headers={ + 'Authorization':'Basic '+base64.b64encode(b'admin:smoke-pass').decode(), + 'Content-Type':'application/json',**(headers or {})}) + with urllib.request.urlopen(req,timeout=90) as response: return response.read().decode() + def start(): + global server + server = sp.Popen([binary,'s3-store'],env=env,stdout=log,stderr=log) + def ready(): + if server.poll() is not None: raise RuntimeError('server exited: '+str(log.name)) + request('/liveness'); return True + wait_for(ready) + def wait_for_staging_drain(): + quiet_since = None + def drained(): + nonlocal quiet_since + pending = any(p.is_file() and p.suffix in {'.arrow', '.arrows', '.part', '.parquet'} + for p in Path(env['P_STAGING_DIR']).rglob('*')) + if pending: + quiet_since = None + return False + if quiet_since is None: + quiet_since = time.monotonic() + return time.monotonic() - quiet_since >= 2 + wait_for(drained, 300) + def migrate(action,*flags,expected=0): + result = sp.run([binary,'s3-store','--metadata-migration',action,'--acknowledge-server-stopped',*flags], + env=dict(env,P_METASTORE_POSTGRES_URL=pgurl),stdout=log,stderr=log,timeout=180) + namespace['commands'].append(dict(argv=result.args, exit_code=result.returncode, expected_success=expected == 0)) + assert (result.returncode == 0) == (expected == 0), (action,result.returncode) + def sql(statement): + return run('docker','exec',pg,'psql','-U','postgres','-d','migration','-qAt','-c',statement) + def refuse(label, postgres, overrides=None): + attempt_env = dict(env) + attempt_env.pop('P_METASTORE_POSTGRES_URL', None) + if postgres: attempt_env['P_METASTORE_POSTGRES_URL'] = pgurl + attempt_env.update(overrides or {}) + rejected = sp.run([binary,'s3-store'],env=attempt_env,text=True,stdout=sp.PIPE,stderr=sp.STDOUT,timeout=30) + log.write(rejected.stdout); log.flush() + assert rejected.returncode != 0, (label, postgres) + assert any(word in rejected.stdout.lower() for word in ['migration', 'exported', 'paired']), rejected.stdout + namespace['commands'].append(dict(argv=rejected.args, exit_code=rejected.returncode, refusal_state=label, postgres=postgres)) + def refuse_both(label): + for postgres in [False, True]: refuse(label, postgres) + def object_write(key, body): + return namespace['get_key'](env['P_S3_URL'],'migration',key,body=body) + def object_delete(key): + return namespace['get_key'](env['P_S3_URL'],'migration',key,method='DELETE') + start() + for stream in ['first','second','partitioned']: + request('/logstream/'+stream,'PUT',headers={'X-P-Custom-Partition':'region'} if stream=='partitioned' else None) + request('/ingest','POST',[{'message':'migration','value':42,'region':'west'}],{'X-P-Stream':stream}) + request('/role/migration-reader','PUT',{'roleType':'user','actions':[{'privilege':'reader','resource':{'stream':'first'}}]}) + request('/user/migration-user','POST',['migration-reader']) + request('/dashboards','POST',{'title':'Migration dashboard','tiles':[]}) + request('/filters','POST',{'stream_name':'first','filter_name':'Migration filter','query':{'filter_type':'sql','filter_query':'SELECT * FROM first','filter_builder':None},'time_filter':None}) + request('/alerts','POST',{'severity':'high','title':'Migration alert','alertType':'threshold', + 'queryType':'code','query':'select count(*) as count from first','thresholdConfig':{'operator':'>','value':1000}, + 'evalConfig':{'rollingWindow':{'evalStart':'10 minutes','evalEnd':'now','evalFrequency':10}}, + 'targets':[],'notificationConfig':{'interval':1}}) + wait_for(lambda:sum(k.endswith('manifest.json') for k in list_keys(env['P_S3_URL'],'migration'))>=3,240) + query={'query':'select sum(value) as value from first', + 'startTime':(dt.datetime.now(dt.timezone.utc)-dt.timedelta(hours=1)).isoformat(), + 'endTime':(dt.datetime.now(dt.timezone.utc)+dt.timedelta(hours=1)).isoformat()} + before=request('/query','POST',query) + def normalized(raw): + def ordered(value): + if isinstance(value,dict): return {k:ordered(v) for k,v in sorted(value.items())} + if isinstance(value,list): return sorted([ordered(v) for v in value],key=lambda v:json.dumps(v,sort_keys=True)) + return value + return ordered(json.loads(raw)) + def snapshot(path): + value=normalized(request(path)) + if path.endswith('/stats'): value.pop('time',None) # response generation time + return value + # Query statistics are created asynchronously; include the internal stream in both snapshots. + wait_for(lambda: any(item['name']=='pstats' for item in json.loads(request('/logstream'))),120) + wait_for_staging_drain() + saved={p:snapshot(p) for p in ['/logstream','/logstream/first/schema','/logstream/first/info','/logstream/first/stats','/dashboards','/filters','/user','/alerts']} + wait_for_staging_drain() + stop() + stuck = Path(env['P_STAGING_DIR'])/'stranded.part' + stuck.write_bytes(b'unsynced') + migrate('import','--dry-run',expected=1) + stuck.unlink() + + print('object-store fixture uploaded; starting offline import',flush=True) + migrate('verify',expected=1) + missing_env = dict(env, P_METASTORE_POSTGRES_URL=pgurl, P_STAGING_DIR='missing-relative-staging') + attempt = sp.run([binary, 's3-store', '--metadata-migration', 'import', '--dry-run', '--acknowledge-server-stopped'], + cwd=out, env=missing_env, text=True, stdout=sp.PIPE, stderr=sp.STDOUT, timeout=30) + assert attempt.returncode != 0 and 'staging directory must exist' in attempt.stdout, attempt.stdout + config_dir = out / 'migration-env-fixture' + config_dir.mkdir(exist_ok=True) + (config_dir / 'staging').mkdir(exist_ok=True) + (config_dir / '.parseable.env').write_text('P_STAGING_DIR=./staging\n') + fixture_env = dict(env, P_METASTORE_POSTGRES_URL=pgurl) + fixture_env.pop('P_STAGING_DIR') + attempt = sp.run([binary, 's3-store', '--metadata-migration', 'import', '--dry-run', '--acknowledge-server-stopped'], + cwd=config_dir, env=fixture_env, text=True, stdout=sp.PIPE, stderr=sp.STDOUT, timeout=30) + assert attempt.returncode == 0, attempt.stdout + assert 'Resolved staging directory: ' + str((config_dir / 'staging').resolve()) in attempt.stdout, attempt.stdout + assert 'Environment file: ' + str(config_dir / '.parseable.env') in attempt.stdout, attempt.stdout + namespace['commands'].append(dict(argv=attempt.args, cwd=str(config_dir), exit_code=attempt.returncode, env_file_checked=True)) + migrate('import','--dry-run') + # Upload actual S3 metadata objects so interruption is observable on MinIO's XL layout. + for i in range(500): + namespace['get_key'](env['P_S3_URL'], 'migration', f'.settings/migration-fixtures/{i}.json', body=b'{}') + interrupted=sp.Popen([binary,'s3-store','--metadata-migration','import','--acknowledge-server-stopped'], + env=dict(env,P_METASTORE_POSTGRES_URL=pgurl),stdout=log,stderr=log) + transfer_processes.append(interrupted) + deadline=time.monotonic()+120 + while '.parseable/metastore-migration' not in list_keys(env['P_S3_URL'],'migration'): + assert interrupted.poll() is None, 'import completed before interruption' + assert time.monotonic()= expected_latest + wait_for(bounds_published) + latest={p:snapshot(p) for p in saved if p != '/alerts'} + stop() + print('new PostgreSQL rows ingested; starting offline export',flush=True) + object_write(busy_key, ('Export:' + backend_id).encode()) + refuse_both('export-marker-before-state') + migrate('export','--abort') + object_write(busy_key, ('Export:' + backend_id).encode()) + sql("UPDATE parseable_transfer SET state='exported'") + refuse_both('export-state-before-object-writes') + sql("UPDATE parseable_transfer SET state='ready'") + object_delete(busy_key) + export=sp.Popen([binary,'s3-store','--metadata-migration','export','--acknowledge-server-stopped'],env=env,stdout=log,stderr=log) + transfer_processes.append(export) + deadline=time.monotonic()+120 + while '.parseable/metastore-migration' not in list_keys(env['P_S3_URL'],'migration'): + assert export.poll() is None, 'export completed before interruption' + assert time.monotonic()= 'tenant-1/logs/date=001/' AND key < 'tenant-1/logs/date=0010')) ORDER BY key; +EXPLAIN (ANALYZE, BUFFERS) SELECT key,body FROM parseable_documents WHERE tenant='tenant-1' AND (key='tenant-1/logs/date=001' OR (key >= 'tenant-1/logs/date=001/' AND key < 'tenant-1/logs/date=0010')) ORDER BY key; +EXPLAIN (ANALYZE, BUFFERS) SELECT key,body FROM parseable_documents WHERE tenant='tenant-1' AND key=ANY(ARRAY['tenant-1/logs/date=001/1.manifest.json','tenant-1/logs/date=001/101.manifest.json']); +EXPLAIN (ANALYZE, BUFFERS) SELECT key FROM parseable_documents WHERE tenant='tenant-1' AND key LIKE '%/.stream/%'; +BEGIN; +EXPLAIN (ANALYZE, BUFFERS) DELETE FROM parseable_documents WHERE tenant='tenant-1' AND (key='tenant-1/logs/date=001' OR (key >= 'tenant-1/logs/date=001/' AND key < 'tenant-1/logs/date=0010')); +ROLLBACK; +EXPLAIN (ANALYZE, BUFFERS) SELECT EXISTS(SELECT 1 FROM parseable_documents WHERE tenant='tenant-1' AND key >= 'tenant-1/stream-1/.stream/' AND key < 'tenant-1/stream-1/.stream0' AND key LIKE '%stream.json'); diff --git a/scripts/postgres-metastore-smoke.py b/scripts/postgres-metastore-smoke.py new file mode 100755 index 000000000..d39ef90c0 --- /dev/null +++ b/scripts/postgres-metastore-smoke.py @@ -0,0 +1,516 @@ +#!/usr/bin/env python3 +"""Isolated MinIO/PostgreSQL smoke and request-count comparison (no live services). +Run after P_BUILD_NO_UI=1 CARGO_BUILD_JOBS=4 cargo build. +Artifacts stay in .postgres-smoke/; all started processes/containers are removed. +""" +import atexit +import signal +import sys +import traceback +import base64 +import datetime as dt +import json +import hashlib +import hmac +import xml.etree.ElementTree as ET +import os +from pathlib import Path +import socket +import subprocess as sp +import time +import urllib.request +import urllib.error + +ROOT = Path(__file__).resolve().parents[1] +OUT = ROOT / '.postgres-smoke' +OUT.mkdir(exist_ok=True) +ARCHIVE = OUT / 'runs' +ARCHIVE.mkdir(exist_ok=True) +if (OUT/'results.json').exists(): + previous = (OUT/'results.json').read_bytes() + (ARCHIVE/(hashlib.sha256(previous).hexdigest()+'.json')).write_bytes(previous) +selection = os.environ.get('P_SMOKE_MODE', 'all') +assert selection in ['all', 'postgres', 'postgres-tls'], selection +modes = ['object-store', 'postgres-tls', 'postgres'] if selection == 'all' else [selection] +containers = [] +server = None +auxiliary = None + +commands = [] +processes = [] +results = {"requested_modes": modes} +http_requests = [] +def redacted_json(value): + return redacted_text(json.dumps(value, indent=2)) + +def redacted_text(text): + replacements = [(str(ROOT) + '/', ''), (str(ROOT), '.'), + (str(Path.home()), '')] + hostname = socket.gethostname() + host_forms = {hostname, ''.join(c for c in hostname if c.isalnum() or c in '-_')} + replacements.extend((host, '') for host in sorted(host_forms, key=len, reverse=True) if host) + target = os.environ.get('CARGO_TARGET_DIR') + if target: + replacements.insert(0, (target, 'target')) + for original, replacement in replacements: + text = text.replace(original, replacement) + return text + +def run(*args): + attempt = sp.run(args, text=True, stdout=sp.PIPE, stderr=sp.PIPE) + commands.append(dict(argv=list(args), exit_code=attempt.returncode)) + attempt.check_returncode() + return attempt.stdout.strip() + +def postgres_logs(name): + args = ['docker', 'logs', name] + attempt = sp.run(args, text=True, stdout=sp.PIPE, stderr=sp.STDOUT) + commands.append(dict(argv=args, exit_code=attempt.returncode)) + attempt.check_returncode() + return attempt.stdout + +def scalar(response, column, expected): + value = json.loads(response) + assert value == [{column: expected}], value + return True + +def wait_for_staging_drain(staging): + quiet_since = None + def drained(): + nonlocal quiet_since + pending = any(p.is_file() and p.suffix in {'.arrow', '.arrows', '.part', '.parquet'} + for p in Path(staging).rglob('*')) + if pending: + quiet_since = None + return False + if quiet_since is None: + quiet_since = time.monotonic() + return time.monotonic() - quiet_since >= 2 + wait_for(drained, 300) + +def cleanup(): + global auxiliary + try: + if auxiliary is not None: + auxiliary.terminate() + try: + auxiliary.wait(timeout=15) + except sp.TimeoutExpired: + auxiliary.kill() + auxiliary.wait() + auxiliary = None + stop() + finally: + if sys.exc_info()[0] is not None: + results['container_logs'] = {} + for name in containers: + attempt = sp.run(['docker','logs',name],text=True,stdout=sp.PIPE,stderr=sp.STDOUT,timeout=10) + results['container_logs'][name] = attempt.stdout + for name in reversed(containers): + attempt = sp.run(['docker', 'rm', '-fv', name], stdout=sp.DEVNULL, stderr=sp.DEVNULL) + commands.append(dict(argv=['docker', 'rm', '-fv', name], exit_code=attempt.returncode)) + containers.clear() + for process in processes: + commands.append(dict(argv=process.args, exit_code=process.poll())) + processes.clear() + for path in OUT.glob('*.log'): + path.write_text(redacted_text(path.read_text(errors='replace'))) + results['commands'] = commands + results['success'] = sys.exc_info()[0] is None + if sys.exc_info()[0] is not None: + results['failure'] = traceback.format_exc() + if (OUT/'checks.json').exists(): + results['validation'] = json.loads((OUT/'checks.json').read_text()) + results['http_requests'] = http_requests + evidence = redacted_json(results) + '\n' + (OUT/'results.json').write_text(evidence) + (ARCHIVE/(hashlib.sha256(evidence.encode()).hexdigest()+'.json')).write_text(evidence) + runs = [json.loads(path.read_text()) for path in sorted(ARCHIVE.glob('*.json'))] + (ROOT/'docs/validation/postgres-metastore-smoke.json').write_text(redacted_json({'runs': runs})+'\n') + +def interrupted(signum, frame): + raise SystemExit(128 + signum) +if globals().get('__name__') == '__main__': + atexit.register(cleanup) + signal.signal(signal.SIGTERM, interrupted) + signal.signal(signal.SIGINT, interrupted) + +def startup_attempt(environment, timeout): + process = sp.Popen([str(Path(os.environ.get('CARGO_TARGET_DIR', ROOT/'target'))/'debug/parseable'), 's3-store'], + env=environment, cwd=ROOT, stdout=sp.PIPE, stderr=sp.STDOUT, text=True) + processes.append(process) + try: + output, _ = process.communicate(timeout=timeout) + except sp.TimeoutExpired: + process.kill() + process.communicate() + raise + return sp.CompletedProcess(process.args, process.returncode, output) + +def port(): + with socket.socket() as s: + s.bind(('127.0.0.1', 0)) + return s.getsockname()[1] + +def container(image, *args, **env): + name = 'parseable-pg-smoke-' + str(os.getpid()) + '-' + str(len(containers)) + command = ['docker', 'run', '-d', '--name', name] + for key, value in env.items(): + command += ['-e', f'{key}={value}'] + containers.append(name) + return name, command + +def list_keys(endpoint, bucket): + """List real S3 keys with SigV4; no client package or image utilities needed.""" + now = dt.datetime.now(dt.timezone.utc) + stamp, day = now.strftime('%Y%m%dT%H%M%SZ'), now.strftime('%Y%m%d') + host = endpoint.removeprefix('http://') + payload = hashlib.sha256(b'').hexdigest() + headers = f'host:{host}\nx-amz-content-sha256:{payload}\nx-amz-date:{stamp}\n' + signed = 'host;x-amz-content-sha256;x-amz-date' + canonical = f'GET\n/{bucket}\nlist-type=2\n{headers}\n{signed}\n{payload}' + scope = f'{day}/us-east-1/s3/aws4_request' + message = f'AWS4-HMAC-SHA256\n{stamp}\n{scope}\n{hashlib.sha256(canonical.encode()).hexdigest()}' + key = b'AWS4minioadmin' + for value in [day, 'us-east-1', 's3', 'aws4_request']: + key = hmac.new(key, value.encode(), hashlib.sha256).digest() + signature = hmac.new(key, message.encode(), hashlib.sha256).hexdigest() + authorization = f'AWS4-HMAC-SHA256 Credential=minioadmin/{scope}, SignedHeaders={signed}, Signature={signature}' + req = urllib.request.Request(f'{endpoint}/{bucket}?list-type=2', headers={ + 'Authorization': authorization, 'X-Amz-Date': stamp, 'X-Amz-Content-Sha256': payload}) + with urllib.request.urlopen(req, timeout=10) as response: + root = ET.fromstring(response.read()) + assert root.findtext('{*}IsTruncated') != 'true', 'smoke fixture exceeded a listing page' + return sorted(node.text for node in root.findall('{*}Contents/{*}Key')) + +def get_key(endpoint, bucket, key, body=None, method=None): + """Read publication documents with SigV4 without an SDK dependency.""" + now = dt.datetime.now(dt.timezone.utc) + stamp, day = now.strftime('%Y%m%dT%H%M%SZ'), now.strftime('%Y%m%d') + host = endpoint.removeprefix('http://') + method = method or ('GET' if body is None else 'PUT') + payload = hashlib.sha256(body or b'').hexdigest() + headers = f'host:{host}\nx-amz-content-sha256:{payload}\nx-amz-date:{stamp}\n' + signed = 'host;x-amz-content-sha256;x-amz-date' + canonical = f'{method}\n/{bucket}/{key}\n\n{headers}\n{signed}\n{payload}' + scope = f'{day}/us-east-1/s3/aws4_request' + message = f'AWS4-HMAC-SHA256\n{stamp}\n{scope}\n{hashlib.sha256(canonical.encode()).hexdigest()}' + signing_key = b'AWS4minioadmin' + for value in [day, 'us-east-1', 's3', 'aws4_request']: + signing_key = hmac.new(signing_key, value.encode(), hashlib.sha256).digest() + signature = hmac.new(signing_key, message.encode(), hashlib.sha256).hexdigest() + authorization = f'AWS4-HMAC-SHA256 Credential=minioadmin/{scope}, SignedHeaders={signed}, Signature={signature}' + req = urllib.request.Request(f'{endpoint}/{bucket}/{key}', data=body, method=method, headers={ + 'Authorization': authorization, 'X-Amz-Date': stamp, 'X-Amz-Content-Sha256': payload}) + with urllib.request.urlopen(req, timeout=10) as response: + return response.read() + +def stop(): + global server + if server: + server.terminate() + try: + server.wait(timeout=15) + except sp.TimeoutExpired: + server.kill() + server.wait() + server = None + +def wait_for(check, seconds=180): + deadline = time.monotonic() + seconds + last_error = None + while time.monotonic() < deadline: + try: + result = check() + if result: + return result + except RuntimeError: + raise + except Exception as error: + last_error = error + time.sleep(1) + raise RuntimeError('timed out waiting for smoke fixture') from last_error + +try: + pgport, s3port = port(), port() + pg, cmd = container('postgres:17-alpine', POSTGRES_PASSWORD='smoke', POSTGRES_DB='parseable') + run(*cmd, '-p', f'127.0.0.1:{pgport}:5432', 'postgres:17-alpine') + mini, cmd = container('quay.io/parseablehq/minio:edge', MINIO_ROOT_USER='minioadmin', MINIO_ROOT_PASSWORD='minioadmin') + run(*cmd, '-p', f'127.0.0.1:{s3port}:9000', 'quay.io/parseablehq/minio:edge', 'server', '/data') + wait_for(lambda: 'accepting connections' in run('docker', 'exec', pg, 'pg_isready', '-h', '127.0.0.1', '-U', 'postgres')) + wait_for(lambda: urllib.request.urlopen(f'http://127.0.0.1:{s3port}/minio/health/ready', timeout=3).status == 200) + # An isolated CA and signed localhost server certificate; no host config changes. + tls = OUT / 'tls' + tls.mkdir(exist_ok=True) + run('openssl', 'req', '-x509', '-newkey', 'rsa:2048', '-nodes', '-keyout', str(tls/'ca.key'), '-out', str(tls/'ca.crt'), '-days', '2', '-subj', '/CN=Smoke CA') + run('openssl', 'req', '-newkey', 'rsa:2048', '-nodes', '-keyout', str(tls/'server.key'), '-out', str(tls/'server.csr'), '-subj', '/CN=localhost') + (tls/'extensions').write_text('subjectAltName=DNS:localhost,IP:127.0.0.1\nextendedKeyUsage=serverAuth\n') + run('openssl', 'x509', '-req', '-in', str(tls/'server.csr'), '-CA', str(tls/'ca.crt'), '-CAkey', str(tls/'ca.key'), '-CAcreateserial', '-out', str(tls/'server.crt'), '-days', '2', '-extfile', str(tls/'extensions')) + for name in ['server.key', 'server.crt']: + run('docker', 'cp', str(tls/name), pg+':/var/lib/postgresql/data/'+name) + run('docker', 'exec', pg, 'sh', '-c', 'chown postgres:postgres /var/lib/postgresql/data/server.* && chmod 600 /var/lib/postgresql/data/server.key') + run('docker', 'exec', pg, 'psql', '-U', 'postgres', '-d', 'parseable', '-c', "ALTER SYSTEM SET ssl='on';") + run('docker', 'exec', pg, 'psql', '-U', 'postgres', '-d', 'parseable', '-c', 'SELECT pg_reload_conf();') + results['source_commit'] = run('git', 'rev-parse', 'HEAD') + results['binary_sha256'] = hashlib.sha256((Path(os.environ.get('CARGO_TARGET_DIR', ROOT/'target'))/'debug/parseable').read_bytes()).hexdigest() + results['source_diff_sha256'] = hashlib.sha256(run('git', 'diff', 'HEAD').encode()).hexdigest() + results['image_digests'] = {image: json.loads(run('docker', 'image', 'inspect', image))[0]['RepoDigests'] + for image in ['postgres:17-alpine', 'quay.io/parseablehq/minio:edge']} + results['measurement'] = 'Node-local storage counter delta during five queries, including background traffic; not isolated requests per query.' + for mode in modes: + print('starting '+mode, flush=True) + bucket = 'smoke-' + mode + run('docker', 'exec', mini, 'mkdir', '-p', '/data/' + bucket) + http, grpc, flight = port(), port(), port() + base = f'http://127.0.0.1:{http}/api/v1' + env = dict(os.environ, P_ADDR=f'127.0.0.1:{http}', P_GRPC_PORT=str(grpc), P_FLIGHT_PORT=str(flight), + P_STAGING_DIR=str(OUT / (mode + '-staging-' + str(os.getpid()))), P_USERNAME='admin', P_PASSWORD='smoke-pass', + P_S3_URL=f'http://127.0.0.1:{s3port}', P_S3_BUCKET=bucket, P_S3_REGION='us-east-1', + P_S3_ACCESS_KEY='minioadmin', P_S3_SECRET_KEY='minioadmin', P_S3_PATH_STYLE='true', RUST_LOG='warn', + P_METASTORE_POSTGRES_STATEMENT_TIMEOUT_MS='30000') + env.pop('P_METASTORE_POSTGRES_URL', None) + if mode.startswith('postgres'): + env['P_METASTORE_POSTGRES_URL'] = f'postgresql://postgres:smoke@127.0.0.1:{pgport}/parseable?sslmode=disable' + if mode == 'postgres-tls': + run('docker', 'exec', pg, 'psql', '-U', 'postgres', '-c', 'CREATE DATABASE tls;') + env['P_METASTORE_POSTGRES_URL'] = f'postgresql://postgres:smoke@localhost:{pgport}/tls?sslmode=require' + env['P_METASTORE_POSTGRES_CA'] = str(tls/'ca.crt') + log = open(OUT / (mode + '.log'), 'w') + def start(): + global server + server = sp.Popen([str(Path(os.environ.get('CARGO_TARGET_DIR', ROOT / 'target')) / 'debug/parseable'), 's3-store'], env=env, cwd=ROOT, stdout=log, stderr=log) + processes.append(server) + def request(path, method='GET', body=None, headers=None, timeout=90): + data = None if body is None else json.dumps(body).encode() + req = urllib.request.Request(base + path, data=data, method=method, + headers={'Authorization': 'Basic ' + base64.b64encode(b'admin:smoke-pass').decode(), + 'Content-Type': 'application/json', **(headers or {})}) + try: + with urllib.request.urlopen(req, timeout=timeout) as response: + http_requests.append(dict(path=path, method=method, status=response.status)) + return response.read().decode() + except urllib.error.HTTPError as error: + http_requests.append(dict(path=path, method=method, status=error.code)) + error.add_note(error.read().decode()) + raise + def status(path, method='GET', body=None): + try: + request(path, method, body) + return 200 + except urllib.error.HTTPError as error: + return error.code + def ready(): + if server.poll() is not None: + raise RuntimeError('server exited; inspect ' + str(log.name)) + request('/liveness') + return True + results[mode] = {} + start() + wait_for(ready) + if mode == 'postgres-tls': + stop() + rejected = env.copy() + run('openssl', 'req', '-x509', '-newkey', 'rsa:2048', '-nodes', '-keyout', str(tls/'wrong.key'), '-out', str(tls/'wrong.crt'), '-days', '2', '-subj', '/CN=Wrong CA') + rejected['P_METASTORE_POSTGRES_CA'] = str(tls/'wrong.crt') + prior_logs = postgres_logs(pg) + attempt = startup_attempt(rejected, 30) + rejection_logs = postgres_logs(pg)[len(prior_logs):] + commands.append(dict(argv=attempt.args, exit_code=attempt.returncode)) + assert attempt.returncode != 0 + assert 'error performing tls handshake' in attempt.stdout.lower(), attempt.stdout + assert 'unknown ca' in rejection_logs.lower(), (attempt.stdout, rejection_logs) + start() + wait_for(ready) + results[mode]['trusted_ca_restart'] = True + results[mode]['untrusted_ca_rejected'] = True + results[mode]['untrusted_ca_output'] = attempt.stdout + results[mode]['untrusted_ca_server_alert'] = rejection_logs + request('/logstream/smoke', 'PUT') + request('/ingest', 'POST', [{'message': 'postgres smoke', 'value': 42}], {'X-P-Stream': 'smoke'}) + database = 'tls' if mode == 'postgres-tls' else 'parseable' + def published(stream): + if mode.startswith('postgres'): + raw = run('docker', 'exec', pg, 'psql', '-U', 'postgres', '-d', database, '-Atc', + f"SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'{stream}/.stream/') AND right(key,11)='stream.json';") + documents = [json.loads(line) for line in raw.splitlines()] + else: + documents = [json.loads(get_key(env['P_S3_URL'], bucket, key)) + for key in list_keys(env['P_S3_URL'], bucket) + if key.startswith(stream+'/.stream/') and key.endswith('stream.json')] + return any(document.get('snapshot', {}).get('manifest_list') for document in documents) + def uploaded(): + return any(key.startswith('smoke/') and key.endswith('.parquet') + for key in list_keys(env['P_S3_URL'], bucket)) and published('smoke') + wait_for(uploaded, 240) + query = {'query': 'select count(*) as count from smoke', + 'startTime': (dt.datetime.now(dt.timezone.utc)-dt.timedelta(hours=1)).isoformat(), + 'endTime': (dt.datetime.now(dt.timezone.utc)+dt.timedelta(minutes=5)).isoformat()} + first = request('/query', 'POST', query) + scalar(first, 'count', 1) + request('/logstream/smoke/retention', 'PUT', [{'description': 'smoke retention', 'action': 'delete', 'duration': '30d'}]) + dashboard = request('/dashboards', 'POST', {'title': 'Postgres smoke', 'tiles': []}) + wait_for_staging_drain(env['P_STAGING_DIR']) + stop() + env['P_MODE'] = 'query' + env['P_QUERIER_ENDPOINT'] = f'127.0.0.1:{http}' + start() + wait_for(ready) + # Publish a stream from another process after this process restarted, + # so its first query must enter the cold-load path. + auxiliary_env = env.copy() + auxiliary_env['P_MODE'] = 'ingest' + auxiliary_port = port() + auxiliary_env.update(P_INGESTOR_ENDPOINT=f'127.0.0.1:{auxiliary_port}', P_ADDR=f'127.0.0.1:{auxiliary_port}', P_GRPC_PORT=str(port()), P_FLIGHT_PORT=str(port()), + P_STAGING_DIR=str(OUT/(mode+'-aux-'+str(os.getpid())))) + auxiliary = sp.Popen([str(Path(os.environ.get('CARGO_TARGET_DIR', ROOT/'target'))/'debug/parseable'), 's3-store'], env=auxiliary_env, stdout=log, stderr=log) + processes.append(auxiliary) + def aux_request(path, method='GET', body=None, headers=None): + original_base = base + # request() closes over base; temporarily use the other node. + return auxiliary_request(original_base.replace(str(http), str(auxiliary_port)), path, method, body, headers) + def auxiliary_request(node_base, path, method, body, headers): + req = urllib.request.Request(node_base+path, method=method, + data=None if body is None else json.dumps(body).encode(), headers={ + 'Authorization': 'Basic '+base64.b64encode(b'admin:smoke-pass').decode(), + 'Content-Type': 'application/json', **(headers or {})}) + with urllib.request.urlopen(req, timeout=30) as response: + return response.read().decode() + wait_for(lambda: aux_request('/liveness') == '') + aux_request('/ingest', 'POST', [{'message':'lazy reload', 'value':9}], {'X-P-Stream':'lazy'}) + wait_for(lambda: published('lazy'), 240) + # Snapshot publication is polled before the first cold-load request. + lazy_query = dict(query, query='select count(*) as count from lazy') + wait_for(lambda: scalar(request('/query', 'POST', lazy_query, timeout=90), 'count', 1), 240) + print(mode+': cold load passed', flush=True) + second = request('/query', 'POST', query) + assert json.loads(first) == json.loads(second), (first, second) + assert any(item.get('title') == 'Postgres smoke' for item in json.loads(request('/dashboards'))), request('/dashboards') + def counts(): + raw = request('/metrics') + return {line.split()[0]:float(line.split()[1]) for line in raw.splitlines() + if line.startswith('parseable_storage_request_response_time_count')} + before = counts() + for _ in range(5): + assert json.loads(request('/query', 'POST', query)) == json.loads(second) + after = counts() + delta = {key: value-before.get(key,0) for key,value in after.items() if value-before.get(key,0)} + keys = list_keys(env['P_S3_URL'], bucket) + results[mode].update(dict(lazy_reload=True, query=second,dashboard=json.loads(dashboard),keys=keys, + non_parquet_keys=[k for k in keys if not k.endswith('.parquet')], + request_delta=delta, observed_requests_during_window_divided_by_queries=sum(delta.values())/5)) + if mode.startswith('postgres'): + assert results[mode]['non_parquet_keys'] == ['.parseable/postgres-metastore-id'] + database = 'tls' if mode == 'postgres-tls' else 'parseable' + sql = lambda text: run('docker', 'exec', pg, 'psql', '-v', 'ON_ERROR_STOP=1', '-U', 'postgres', '-d', database, '-Atc', text) + sql("INSERT INTO parseable_documents(tenant,key,kind,body) VALUES('','deletion-explicit/.stream/.deleting','document',''::bytea);") + try: + aux_request('/logstream/deletion-explicit', 'POST', [{'value':1}]) + except urllib.error.HTTPError as error: + assert error.code == 409, error.code + else: + raise AssertionError('explicit-stream ingest admitted a pending deletion') + sql("DELETE FROM parseable_documents WHERE tenant='' AND key='deletion-explicit/.stream/.deleting';") + run('docker', 'stop', pg) + started = time.monotonic() + assert status('/readiness') == 503 + assert status('/logstream/outage', 'PUT') >= 400 + elapsed = time.monotonic()-started + assert elapsed < 10, elapsed + results[mode]['database_outage'] = dict(readiness=503, request_failed=True, seconds=elapsed) + try: + aux_request('/logstream/outage-explicit', 'POST', [{'value':1}]) + except urllib.error.HTTPError as error: + assert error.code == 503, error.code + else: + raise AssertionError('explicit-stream ingest hid a database outage') + results[mode]['explicit_stream_errors'] = dict(pending_deletion=409, database_outage=503) + run('docker', 'start', pg) + wait_for(lambda: status('/readiness') == 200) + assert json.loads(request('/query', 'POST', query)) == json.loads(second) + run('docker', 'stop', mini) + assert status('/readiness') == 503 + results[mode]['bucket_outage_readiness'] = 503 + run('docker', 'start', mini) + wait_for(lambda: status('/readiness') == 200) + request('/logstream/smoke', 'DELETE') + if mode.startswith('postgres'): + database = 'tls' if mode == 'postgres-tls' else 'parseable' + remaining = run('docker', 'exec', pg, 'psql', '-U', 'postgres', '-d', database, '-Atc', "SELECT count(*) FROM parseable_documents WHERE starts_with(key,'smoke/');") + assert remaining == '0', remaining + results[mode]['deleted_stream_rows'] = 0 + assert not any(k.startswith('smoke/') for k in list_keys(env['P_S3_URL'], bucket)) + request('/logstream/smoke', 'PUT') + aux_request('/ingest', 'POST', [{'message': 'recreated', 'value': 7}], {'X-P-Stream': 'smoke'}) + wait_for(uploaded, 240) + wait_for(lambda: scalar(request('/query', 'POST', query, timeout=90), 'count', 1), 240) + recreated_query = dict(query, query='select sum(value) as value from smoke') + wait_for(lambda: scalar(request('/query', 'POST', recreated_query, timeout=90), 'value', 7), 240) + results[mode]['delete_recreate'] = True + print(mode+': lifecycle and outages passed', flush=True) + final_keys = list_keys(env['P_S3_URL'], bucket) + results[mode]['final_keys'] = final_keys + if mode.startswith('postgres'): + assert [k for k in final_keys if not k.endswith('.parquet')] == ['.parseable/postgres-metastore-id'] + if mode.startswith('postgres'): + restart_faults = [] + for fault in ['before-data', 'after-data']: + aux_request('/ingest', 'POST', [{'value': 7}], {'X-P-Stream': 'smoke'}) + wait_for(uploaded, 240) + sql = lambda text: run('docker', 'exec', pg, 'psql', '-v', 'ON_ERROR_STOP=1', '-U', 'postgres', '-d', database, '-Atc', text) + if fault == 'before-data': + sql("INSERT INTO parseable_documents(tenant,key,kind,body) VALUES('','smoke/.stream/.deleting','document',''::bytea);") + else: + sql("CREATE FUNCTION smoke_delete_fault() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'injected row deletion failure'; END $$; CREATE TRIGGER smoke_delete_fault BEFORE DELETE ON parseable_documents FOR EACH ROW EXECUTE FUNCTION smoke_delete_fault();") + assert status('/logstream/smoke', 'DELETE') >= 400 + sql('DROP TRIGGER smoke_delete_fault ON parseable_documents; DROP FUNCTION smoke_delete_fault();') + stop() + start() + wait_for(ready) + listing = json.loads(request('/logstream')) + assert all(item.get('name') != 'smoke' for item in listing), listing + assert status('/query', 'POST', query) >= 400, 'cold loader admitted deleting stream' + request('/logstream/smoke', 'DELETE') + assert sql("SELECT count(*) FROM parseable_documents WHERE starts_with(key,'smoke/');") == '0' + assert not any(k.startswith('smoke/') for k in list_keys(env['P_S3_URL'], bucket)) + request('/logstream/smoke', 'PUT') + request('/logstream/smoke', 'DELETE') + restart_faults.append(dict(step=fault, hidden=True, cold_load_rejected=True, http_retry=True, recreation=True)) + request('/logstream/smoke', 'PUT') + results[mode]['interrupted_deletion_restart'] = restart_faults + stop() + auxiliary.terminate() + try: + auxiliary.wait(timeout=15) + except sp.TimeoutExpired: + auxiliary.kill() + auxiliary.wait() + auxiliary = None + log.close() + failures = {} + # The final loop environment is PostgreSQL mode. Assert startup rejects a + # future schema and a pinned UUID mismatch before any HTTP listener starts. + for case in ['future-schema', 'wrong-id', 'unreachable']: + rejected_env = env.copy() + if case == 'future-schema': + run('docker', 'exec', pg, 'psql', '-U', 'postgres', '-d', database, '-c', + 'UPDATE parseable_backend SET schema_version=999;') + elif case == 'wrong-id': + rejected_env['P_METASTORE_POSTGRES_ID'] = 'wrong-backend' + else: + rejected_env['P_METASTORE_POSTGRES_URL'] = f'postgresql://postgres:smoke@127.0.0.1:{port()}/parseable?sslmode=disable' + attempt = startup_attempt(rejected_env, 20) + commands.append(dict(argv=attempt.args, exit_code=attempt.returncode)) + assert attempt.returncode != 0, (case, attempt.stdout) + assert 'PostgreSQL' in attempt.stdout, (case, attempt.stdout) + failures[case] = attempt.stdout + if case == 'future-schema': + run('docker', 'exec', pg, 'psql', '-U', 'postgres', '-d', database, '-c', + 'UPDATE parseable_backend SET schema_version=3;') + results['startup_rejections'] = failures + results['postgres_rows'] = run('docker','exec',pg,'psql','-U','postgres','-d',database,'-c', + 'SELECT tenant,kind,key,version,octet_length(body) AS bytes FROM parseable_documents ORDER BY tenant,key;') + (OUT/'results.json').write_text(redacted_json(results)) + print(redacted_json(results)) +finally: + cleanup() + atexit.unregister(cleanup) diff --git a/src/alerts/target.rs b/src/alerts/target.rs index 3cc13286d..17b688744 100644 --- a/src/alerts/target.rs +++ b/src/alerts/target.rs @@ -208,6 +208,7 @@ pub struct Target { pub target: TargetType, #[serde(default = "Ulid::new")] pub id: Ulid, + #[serde(rename = "tenantId")] pub tenant: Option, } @@ -523,6 +524,7 @@ pub struct TargetVerifier { pub notification_config: Option, #[serde(default = "Ulid::new")] pub id: Ulid, + #[serde(alias = "tenant")] pub tenant_id: Option, } diff --git a/src/banner.rs b/src/banner.rs index 53cf9dfac..f022a0c77 100644 --- a/src/banner.rs +++ b/src/banner.rs @@ -109,11 +109,17 @@ async fn storage_info(config: &Parseable) { ); } + let probe = if config.options.postgres.enabled() { + "PostgreSQL + data bucket probe" + } else { + "latency" + }; eprintln!( "\ - {:8}Store: \"{}\", (latency: {:?})", + {:8}Store: \"{}\", ({}: {:?})", "", storage.get_endpoint(), + probe, latency ); } diff --git a/src/catalog/mod.rs b/src/catalog/mod.rs index 5aac2ef03..8bab4d2d7 100644 --- a/src/catalog/mod.rs +++ b/src/catalog/mod.rs @@ -585,7 +585,10 @@ pub fn partition_path( upper_bound: DateTime, tenant_id: &Option, ) -> RelativePathBuf { - let root = tenant_id.as_deref().unwrap_or(""); + let root = tenant_id + .as_deref() + .filter(|t| !t.is_empty() && *t != crate::parseable::DEFAULT_TENANT) + .unwrap_or(""); let lower = lower_bound.date_naive().format("%Y-%m-%d").to_string(); let upper = upper_bound.date_naive().format("%Y-%m-%d").to_string(); if lower == upper { diff --git a/src/cli.rs b/src/cli.rs index 2f1ea395d..4e2a5a7f6 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -137,6 +137,11 @@ pub struct GcsStoreArgs { #[derive(Parser, Debug, Default)] pub struct Options { + #[command(flatten)] + pub migration: crate::metastore::metastores::migration::MigrationOptions, + #[command(flatten)] + pub postgres: crate::metastore::metastores::postgres_metastore::PostgresOptions, + // Authentication #[arg(long, env = "P_USERNAME", help = "Admin username to be set for this Parseable server", default_value = DEFAULT_USERNAME)] pub username: String, diff --git a/src/handlers/http/ingest.rs b/src/handlers/http/ingest.rs index 31b75d819..ea8242371 100644 --- a/src/handlers/http/ingest.rs +++ b/src/handlers/http/ingest.rs @@ -400,7 +400,8 @@ pub async fn post_event( .await { Ok(true) => {} - Ok(false) | Err(_) => return Err(StreamNotFound(stream_name.clone()).into()), + Ok(false) => return Err(StreamNotFound(stream_name.clone()).into()), + Err(e) => return Err(e.into()), } } else { return Err(StreamNotFound(stream_name.clone()).into()); @@ -582,15 +583,16 @@ impl actix_web::ResponseError for PostError { JsonFlattenError(e) if e.is_limit_exceeded() => StatusCode::BAD_REQUEST, - Event(_) - | CreateStream(_) - | CustomError(_) - | NetworkError(_) - | ObjectStorageError(_) - | DashboardError(_) - | FiltersError(_) - | StreamError(_) - | JsonFlattenError(_) => StatusCode::INTERNAL_SERVER_ERROR, + ObjectStorageError(e) => e.http_status(), + CreateStream(CreateStreamError::Storage { err, .. }) => err.http_status(), + StreamError(super::logstream::error::StreamError::Storage( + e @ (crate::storage::ObjectStorageError::MetastoreUnavailable(_) + | crate::storage::ObjectStorageError::StreamDeleting), + )) => e.http_status(), + Event(_) | CreateStream(_) | CustomError(_) | NetworkError(_) | DashboardError(_) + | FiltersError(_) | StreamError(_) | JsonFlattenError(_) => { + StatusCode::INTERNAL_SERVER_ERROR + } StreamNotFound(_) => StatusCode::NOT_FOUND, diff --git a/src/handlers/http/logstream.rs b/src/handlers/http/logstream.rs index 5b4e42987..7105f572f 100644 --- a/src/handlers/http/logstream.rs +++ b/src/handlers/http/logstream.rs @@ -59,9 +59,17 @@ pub async fn delete( let stream_name = logstream.into_inner(); // Error out if stream doesn't exist in memory, or in the case of query node, in storage as well let tenant_id = get_tenant_id_from_request(&req)?; - if !PARSEABLE - .check_or_load_stream(&stream_name, &tenant_id) - .await + // A crashed PostgreSQL deletion is deliberately absent from stream loading, + // but an explicit DELETE must still be able to finish it after restart. + let pending_delete = PARSEABLE.options.postgres.enabled() + && PARSEABLE + .metastore + .is_stream_deleting(&stream_name, &tenant_id) + .await?; + if !pending_delete + && !PARSEABLE + .check_or_load_stream(&stream_name, &tenant_id) + .await { return Err(StreamNotFound(stream_name).into()); } @@ -847,8 +855,8 @@ pub mod error { StreamError::CreateStream(CreateStreamError::StreamNameValidation(_)) => { StatusCode::BAD_REQUEST } - StreamError::CreateStream(CreateStreamError::Storage { .. }) => { - StatusCode::INTERNAL_SERVER_ERROR + StreamError::CreateStream(CreateStreamError::Storage { err, .. }) => { + err.http_status() } StreamError::CreateStream(CreateStreamError::Custom { .. }) => { StatusCode::INTERNAL_SERVER_ERROR @@ -860,7 +868,7 @@ pub mod error { StreamError::TenantNotFoundError(_) => StatusCode::NOT_FOUND, StreamError::Custom { status, .. } => *status, StreamError::UninitializedLogstream => StatusCode::METHOD_NOT_ALLOWED, - StreamError::Storage(_) => StatusCode::INTERNAL_SERVER_ERROR, + StreamError::Storage(e) => e.http_status(), StreamError::NoAlertsSet => StatusCode::NOT_FOUND, StreamError::BadAlertJson { .. } => StatusCode::BAD_REQUEST, StreamError::AlertValidation(_) => StatusCode::BAD_REQUEST, diff --git a/src/handlers/http/modal/query/querier_logstream.rs b/src/handlers/http/modal/query/querier_logstream.rs index 2655c9254..f51e8e671 100644 --- a/src/handlers/http/modal/query/querier_logstream.rs +++ b/src/handlers/http/modal/query/querier_logstream.rs @@ -69,7 +69,15 @@ pub async fn delete( // if the stream not found in memory map, //check if it exists in the storage //create stream and schema from storage - if !PARSEABLE.streams.contains(&stream_name, &tenant_id) + // A crashed PostgreSQL deletion is deliberately absent from stream loading, + // but an explicit DELETE must still be able to finish it after restart. + let pending_delete = PARSEABLE.options.postgres.enabled() + && PARSEABLE + .metastore + .is_stream_deleting(&stream_name, &tenant_id) + .await?; + if !pending_delete + && !PARSEABLE.streams.contains(&stream_name, &tenant_id) && !PARSEABLE .create_stream_and_schema_from_storage(&stream_name, &tenant_id) .await diff --git a/src/handlers/http/oidc.rs b/src/handlers/http/oidc.rs index 316e6b027..155168189 100644 --- a/src/handlers/http/oidc.rs +++ b/src/handlers/http/oidc.rs @@ -860,7 +860,7 @@ pub enum OIDCError { impl actix_web::ResponseError for OIDCError { fn status_code(&self) -> StatusCode { match self { - Self::ObjectStorageError(_) => StatusCode::INTERNAL_SERVER_ERROR, + Self::ObjectStorageError(e) => e.http_status(), Self::Serde(_) => StatusCode::INTERNAL_SERVER_ERROR, Self::BadRequest(_) => StatusCode::BAD_REQUEST, Self::Unauthorized => StatusCode::UNAUTHORIZED, diff --git a/src/handlers/http/query.rs b/src/handlers/http/query.rs index f23582fd1..246d14ba3 100644 --- a/src/handlers/http/query.rs +++ b/src/handlers/http/query.rs @@ -814,6 +814,11 @@ impl actix_web::ResponseError for QueryError { QueryError::JsonParse(_) => StatusCode::INTERNAL_SERVER_ERROR, QueryError::Execute(e) => e.status_code(), QueryError::MetastoreError(e) => e.status_code(), + QueryError::ObjectStorage(e) => match e { + ObjectStorageError::MetastoreUnavailable(_) + | ObjectStorageError::StreamDeleting => e.http_status(), + _ => StatusCode::BAD_REQUEST, + }, QueryError::Header(_) => StatusCode::BAD_REQUEST, QueryError::HeaderBuild(e) => actix_web::ResponseError::status_code(e), _ => StatusCode::BAD_REQUEST, diff --git a/src/handlers/http/rbac.rs b/src/handlers/http/rbac.rs index 1c107288a..d5b64be92 100644 --- a/src/handlers/http/rbac.rs +++ b/src/handlers/http/rbac.rs @@ -649,7 +649,7 @@ impl actix_web::ResponseError for RBACError { Self::UserDoesNotExist => StatusCode::NOT_FOUND, Self::SerdeError(_) => StatusCode::BAD_REQUEST, Self::ValidationError(_) => StatusCode::BAD_REQUEST, - Self::ObjectStorageError(_) => StatusCode::INTERNAL_SERVER_ERROR, + Self::ObjectStorageError(e) => e.http_status(), Self::Network(_) => StatusCode::BAD_GATEWAY, Self::Anyhow(_) => StatusCode::INTERNAL_SERVER_ERROR, Self::RoleValidationError => StatusCode::BAD_REQUEST, diff --git a/src/handlers/http/role.rs b/src/handlers/http/role.rs index 832b350d2..da5635f6b 100644 --- a/src/handlers/http/role.rs +++ b/src/handlers/http/role.rs @@ -338,7 +338,7 @@ impl actix_web::ResponseError for RoleError { fn status_code(&self) -> StatusCode { match self { Self::UnknownDefaultRole | Self::DefaultRoleInUse => StatusCode::BAD_REQUEST, - Self::ObjectStorageError(_) => StatusCode::INTERNAL_SERVER_ERROR, + Self::ObjectStorageError(e) => e.http_status(), Self::RoleInUse => StatusCode::BAD_REQUEST, Self::SuperAdminPrivilege => StatusCode::BAD_REQUEST, Self::ProtectedRole => StatusCode::BAD_REQUEST, diff --git a/src/handlers/http/users/dashboards.rs b/src/handlers/http/users/dashboards.rs index 225e01347..468cf5594 100644 --- a/src/handlers/http/users/dashboards.rs +++ b/src/handlers/http/users/dashboards.rs @@ -494,7 +494,8 @@ impl DashboardError { impl actix_web::ResponseError for DashboardError { fn status_code(&self) -> StatusCode { match self { - Self::ObjectStorage(_) | Self::Custom(_) => StatusCode::INTERNAL_SERVER_ERROR, + Self::ObjectStorage(e) => e.http_status(), + Self::Custom(_) => StatusCode::INTERNAL_SERVER_ERROR, Self::Serde(_) | Self::Metadata(_) | Self::InvalidQueryParameter | Self::Header(_) => { StatusCode::BAD_REQUEST } diff --git a/src/handlers/http/users/filters.rs b/src/handlers/http/users/filters.rs index a27438aba..af6b58a52 100644 --- a/src/handlers/http/users/filters.rs +++ b/src/handlers/http/users/filters.rs @@ -236,7 +236,7 @@ pub enum FiltersError { impl actix_web::ResponseError for FiltersError { fn status_code(&self) -> StatusCode { match self { - Self::ObjectStorage(_) => StatusCode::INTERNAL_SERVER_ERROR, + Self::ObjectStorage(e) => e.http_status(), Self::Serde(_) => StatusCode::BAD_REQUEST, Self::Metadata(_) => StatusCode::BAD_REQUEST, Self::UserDoesNotExist(_) => StatusCode::NOT_FOUND, diff --git a/src/interactive.rs b/src/interactive.rs index 0bac1e882..1a2ba98e5 100644 --- a/src/interactive.rs +++ b/src/interactive.rs @@ -26,6 +26,10 @@ const ENV_FILE_NAME: &str = ".parseable.env"; /// Guard to avoid parsing `.parseable.env` more than once per process. static ENV_FILE_LOADED: AtomicBool = AtomicBool::new(false); +static LOADED_ENV_PATH: std::sync::OnceLock = std::sync::OnceLock::new(); +pub(crate) fn loaded_env_path() -> Option<&'static std::path::Path> { + LOADED_ENV_PATH.get().map(PathBuf::as_path) +} /// A required or optional env var that the user may need to provide. struct EnvPrompt { @@ -127,6 +131,7 @@ pub fn load_env_file() { Err(_) => return, }; + let _ = LOADED_ENV_PATH.set(path); for line in io::BufReader::new(file).lines() { let line = match line { Ok(l) => l, diff --git a/src/main.rs b/src/main.rs index 9e42c4bb6..271f92357 100644 --- a/src/main.rs +++ b/src/main.rs @@ -58,6 +58,31 @@ async fn main() -> anyhow::Result<()> { warn!("Failed to install rustls crypto provider: {:?}", e); } + // Offline commands bypass server initialization and interactive configuration. + if std::env::args() + .any(|a| a == "--metadata-migration" || a.starts_with("--metadata-migration=")) + { + use clap::Parser; + use parseable::cli::{Cli, StorageOptions}; + use parseable::storage::ObjectStorageProvider; + parseable::interactive::load_env_file(); + let cli = Cli::parse(); + let (options, provider): (_, &dyn ObjectStorageProvider) = match &cli.storage { + StorageOptions::Local(a) => (&a.options, &a.storage), + StorageOptions::S3(a) => (&a.options, &a.storage), + StorageOptions::Blob(a) => (&a.options, &a.storage), + StorageOptions::Gcs(a) => (&a.options, &a.storage), + }; + if options.migration.metadata_migration.is_some() { + return parseable::metastore::metastores::postgres_metastore::run_migration( + provider, options, + ) + .await; + } + } + + PARSEABLE.metastore.initiate_connection().await?; + // these are empty ptrs so mem footprint should be minimal let server: Box = match &PARSEABLE.options.mode { Mode::Query => Box::new(QueryServer), diff --git a/src/metastore/metastore_traits.rs b/src/metastore/metastore_traits.rs index fa484a51b..128bbce26 100644 --- a/src/metastore/metastore_traits.rs +++ b/src/metastore/metastore_traits.rs @@ -49,6 +49,22 @@ pub use crate::storage::object_storage::{ /// need different methods for different kinds of metadata. #[async_trait] pub trait Metastore: std::fmt::Debug + Send + Sync { + /// Optional SQL batch; the default performs no requests. + async fn manifest_batch( + &self, + _paths: &[String], + _tenant: &Option, + ) -> Result>>, MetastoreError> { + Ok(None) + } + /// SQL-only partition inventory for retention recovery. + async fn metadata_dates( + &self, + _stream: &str, + _tenant: &Option, + ) -> Result, MetastoreError> { + Ok(Vec::new()) + } async fn initiate_connection(&self) -> Result<(), MetastoreError>; async fn get_objects( &self, diff --git a/src/metastore/metastores/document_metastore.rs b/src/metastore/metastores/document_metastore.rs new file mode 100644 index 000000000..f47b7d856 --- /dev/null +++ b/src/metastore/metastores/document_metastore.rs @@ -0,0 +1,1700 @@ +macro_rules! scoped_delimiter { + (true, $s:ident, $p:expr, $t:expr) => { + $s.storage.list_with_delimiter($p).await + }; + (false, $s:ident, $p:expr, $t:expr) => { + $s.storage.list_with_delimiter($p, $t).await + }; +} +macro_rules! postgres_list_streams { + (true, $s:ident, $t:expr, $body:block) => { + $body + }; + (false, $s:ident, $t:expr, $body:block) => { + Ok($s.storage.list_streams($t).await?) + }; +} +// Shared path, encoding and behavioral contract for document metastores. +macro_rules! document_metastore { +($name:ident, $storage:ty, $local_listing:tt, {$($lifecycle:tt)*}) => { +/* + * Parseable Server (C) 2022 - 2025 Parseable, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +use std::{ + collections::{BTreeMap, HashMap, HashSet}, + sync::Arc, +}; + +use actix_web::http::StatusCode; +use arrow_schema::Schema; +use bytes::Bytes; +use chrono::{DateTime, Utc}; +use dashmap::DashMap; +use futures::{StreamExt, TryStreamExt}; +use relative_path::{RelativePath, RelativePathBuf}; +use tonic::async_trait; +use tracing::{info, warn}; +use ulid::Ulid; + +use crate::{ + alerts::{ + alert_structs::{AlertStateEntry, MTTRHistory}, + outbound_http_policy::AlertTargetPolicyConfig, + target::Target, + }, + catalog::{ + manifest::{Manifest, decode_manifest_blocking, encode_manifest}, + partition_path, + }, + handlers::http::{ + modal::{Metadata, NodeMetadata, NodeType}, + users::USERS_ROOT_DIR, + }, + metastore::{ + MetastoreError, + metastore_traits::{ + ConditionalMetadataError, MetadataVersion, MetadataWriteCondition, Metastore, + MetastoreObject, VersionedMetadata, + }, + }, + option::Mode, + parseable::{DEFAULT_TENANT, PARSEABLE}, + storage::{ + ALERTS_ROOT_DIRECTORY, ObjectStorageError, PARSEABLE_ROOT_DIRECTORY, + SETTINGS_ROOT_DIRECTORY, STREAM_METADATA_FILE_NAME, STREAM_ROOT_DIRECTORY, + TARGETS_ROOT_DIRECTORY, + object_storage::{ + alert_json_path, alert_state_json_path, filter_path, manifest_path, mttr_json_path, + outbound_http_policy_json_path, parseable_json_path, schema_path, stream_json_path, + to_bytes, + }, + }, + users::filters::{Filter, FilterType, SQL_FILTER_STREAM_NAME, migrate_v1_v2}, +}; + +/// Using PARSEABLE's storage as a metastore (default) +#[derive(Debug)] +pub struct $name { + pub storage: Arc<$storage>, +} + +impl $name { + async fn delete_filter_path( + &self, + path: &RelativePathBuf, + tenant_id: &Option, + ) -> Result<(), MetastoreError> { + match self.storage.delete_object(path, tenant_id).await { + Ok(()) => Ok(()), + Err(error) if is_missing_optional_dir(&error) => Ok(()), + Err(error) => Err(error.into()), + } + } + + /// Enumerate only this owner's ID, never a filter belonging to another user. + async fn filter_aliases( + &self, + filter: &Filter, + tenant_id: &Option, + ) -> Result, MetastoreError> { + let owner = filter + .user_id + .as_deref() + .expect("stored filters have an owner"); + let id = filter + .filter_id + .as_deref() + .expect("stored filters have an ID"); + let root = filter_path(owner, "", "", tenant_id); + let mut copies = Vec::new(); + for stream in self.storage.list_dirs_relative(&root, tenant_id).await? { + let path = root.join(&stream).join(format!("{id}.json")); + let bytes = match self.storage.get_object(&path, tenant_id).await { + Ok(bytes) => bytes, + Err(error) if is_missing_optional_dir(&error) => continue, + Err(error) => return Err(error.into()), + }; + let copy: Filter = serde_json::from_slice(&bytes)?; + if copy.filter_id == filter.filter_id && copy.user_id == filter.user_id { + copies.push((path, copy)); + } + } + copies.sort_by(|a, b| a.0.cmp(&b.0)); + Ok(copies) + } + + async fn load_tenant_filters( + &self, + tenant_id: &Option, + ) -> Result, MetastoreError> { + let users_dir = + RelativePathBuf::from_iter([tenant_id.as_deref().filter(|t| !t.is_empty() && *t != crate::parseable::DEFAULT_TENANT).unwrap_or(""), USERS_ROOT_DIR]); + let mut grouped: BTreeMap<(String, String), Vec<(RelativePathBuf, Filter)>> = + BTreeMap::new(); + for user in self + .storage + .list_dirs_relative(&users_dir, tenant_id) + .await? + { + let root = users_dir.join(&user).join("filters"); + for stream in self.storage.list_dirs_relative(&root, tenant_id).await? { + let directory = root.join(&stream); + let objects = self + .storage + .get_objects( + Some(&directory), + Box::new(|name| name.ends_with(".json")), + tenant_id, + ) + .await?; + for bytes in objects { + let mut value: serde_json::Value = serde_json::from_slice(&bytes)?; + let is_v1 = value.get("version").and_then(|v| v.as_str()) == Some("v1"); + if is_v1 { + value = migrate_v1_v2(value, tenant_id); + } + let Ok(mut filter) = serde_json::from_value::(value) else { + continue; + }; + // The discovered tenant directory is authoritative, including + // default-tenant records whose payload predates tenant metadata. + filter.tenant_id = tenant_id.clone().filter(|tenant| !tenant.is_empty()); + let (Some(owner), Some(id)) = (&filter.user_id, &filter.filter_id) else { + continue; + }; + // Stored identity fields must remain single path components. + if !safe_filter_component(owner) + || !safe_filter_component(id) + || !safe_filter_component(&filter.stream_name) + || filter.stream_name != stream + || (!is_v1 && owner != &user) + { + warn!( + "Ignoring saved filter whose identity does not match its storage directory" + ); + continue; + } + let source = directory.join(format!("{id}.json")); + // get_objects returns bytes without filenames. Verify the inferred + // filename before any cleanup; never delete an unrelated object. + match self.storage.get_object(&source, tenant_id).await { + Ok(stored) if stored == bytes => {} + Ok(_) => continue, + Err(error) if is_missing_optional_dir(&error) => continue, + Err(error) => return Err(error.into()), + } + if is_v1 { + // Persist before removing the original so interrupted migration is safe. + self.put_filter(&filter, tenant_id).await?; + if source.as_str() != filter.get_object_path() { + self.delete_filter_path(&source, tenant_id).await?; + } + } + grouped + .entry((owner.clone(), id.clone())) + .or_default() + .push(( + if is_v1 { + RelativePathBuf::from(filter.get_object_path()) + } else { + source + }, + filter, + )); + } + } + } + let mut filters = Vec::new(); + for (_, mut copies) in grouped { + // A v1 migration may write a hashed-owner directory that is visited + // later in this same scan. It is still one physical object. + copies.sort_by(|a, b| a.0.cmp(&b.0)); + copies.dedup_by(|a, b| a.0 == b.0); + copies.sort_by(|a, b| { + let canonical = |filter: &Filter| { + filter.query.filter_type == FilterType::SQL + && filter.stream_name == SQL_FILTER_STREAM_NAME + }; + canonical(&b.1).cmp(&canonical(&a.1)).then(a.0.cmp(&b.0)) + }); + let (canonical_path, filter) = &copies[0]; + if filter.query.filter_type == FilterType::SQL + && filter.stream_name == SQL_FILTER_STREAM_NAME + { + for (path, alias) in copies.iter().skip(1) { + if path != canonical_path && alias.query.filter_type == FilterType::SQL { + // The canonical object is already durable. Retain it in memory + // even if cleanup is temporarily unavailable; retry on restart. + if let Err(error) = self.delete_filter_path(path, tenant_id).await { + warn!("Failed to clean saved SQL filter alias {path}: {error}"); + } + } else if alias.query.filter_type != FilterType::SQL { + warn!( + "Retaining non-SQL saved filter copy {path} with the same owner {:?} and ID {:?} as canonical SQL filter {canonical_path}", + filter.user_id, filter.filter_id + ); + } + } + } + if copies.len() > 1 + && !(filter.query.filter_type == FilterType::SQL + && filter.stream_name == SQL_FILTER_STREAM_NAME) + { + warn!( + "Multiple saved filter copies for owner {:?}, ID {:?}; using deterministic path {canonical_path}", + filter.user_id, filter.filter_id + ); + } + filters.push(filter.clone()); + } + Ok(filters) + } +} + +fn stored_filter_path(filter: &Filter, tenant_id: &Option) -> RelativePathBuf { + filter_path( + filter + .user_id + .as_deref() + .expect("stored filters have an owner"), + &filter.stream_name, + &format!( + "{}.json", + filter + .filter_id + .as_deref() + .expect("stored filters have an ID") + ), + tenant_id, + ) +} + +fn safe_filter_component(component: &str) -> bool { + !component.is_empty() + && component != "." + && component != ".." + && !component.contains(['/', '\\']) +} + +pub fn is_missing_optional_dir(err: &ObjectStorageError) -> bool { + match err { + ObjectStorageError::NoSuchKey(_) => true, + ObjectStorageError::IoError(err) => err.kind() == std::io::ErrorKind::NotFound, + _ => false, + } +} + +#[async_trait] +impl Metastore for $name { + /// Since Parseable already starts with a connection to an object store, no need to implement this + async fn initiate_connection(&self) -> Result<(), MetastoreError> { + if $local_listing { + self.storage.check_migration_guard().await?; + Ok(()) + } else { Ok(self.storage.check(&None).await?) } + } + + /// Fetch mutiple .json objects + async fn get_objects( + &self, + parent_path: &str, + tenant_id: &Option, + ) -> Result, MetastoreError> { + Ok(self + .storage + .get_objects( + Some(&RelativePathBuf::from(parent_path)), + Box::new(|file_name| file_name.ends_with(".json")), + tenant_id, + ) + .await?) + } + + /// This function fetches all the keystones from the underlying object store + async fn get_keystones(&self) -> Result>, MetastoreError> { + let base_paths = PARSEABLE.list_tenants().unwrap_or_else(|| vec!["".into()]); + let mut keystones = HashMap::new(); + for mut tenant in base_paths { + let keystone_path = RelativePathBuf::from_iter([&tenant, ".keystone"]); + let objs = self + .storage + .get_objects( + Some(&keystone_path), + Box::new(|file_name| { + file_name.ends_with(".json") && !file_name.starts_with("conv_") + }), + &Some(tenant.clone()), + ) + .await?; + if tenant.is_empty() { + tenant.clone_from(&DEFAULT_TENANT.to_string()); + } + keystones.insert(tenant, objs); + } + + Ok(keystones) + } + + /// This function puts a keystone in the object store at the given path + async fn put_keystone( + &self, + obj: &dyn MetastoreObject, + tenant_id: &Option, + ) -> Result<(), MetastoreError> { + let id = obj.get_object_id(); + let path = if let Some(tenant_id) = tenant_id.as_ref().filter(|t| !t.is_empty() && t.as_str() != crate::parseable::DEFAULT_TENANT) { + RelativePathBuf::from_iter([tenant_id, ".keystone", &format!("{id}.json")]) + } else { + RelativePathBuf::from_iter([".keystone", &format!("{id}.json")]) + }; + Ok(self + .storage + .put_object(&path, to_bytes(obj), tenant_id) + .await?) + } + + /// Delete a keystone + async fn delete_keystone( + &self, + obj: &dyn MetastoreObject, + tenant_id: &Option, + ) -> Result<(), MetastoreError> { + let id = obj.get_object_id(); + let path = if let Some(tenant_id) = tenant_id.as_ref().filter(|t| !t.is_empty() && t.as_str() != crate::parseable::DEFAULT_TENANT) { + RelativePathBuf::from_iter([tenant_id, ".keystone", &format!("{id}.json")]) + } else { + RelativePathBuf::from_iter([".keystone", &format!("{id}.json")]) + }; + Ok(self.storage.delete_object(&path, tenant_id).await?) + } + + /// This function fetches all the conversations from the underlying object store + async fn get_conversations(&self) -> Result>, MetastoreError> { + let base_paths = PARSEABLE.list_tenants().unwrap_or_else(|| vec!["".into()]); + let mut conversations = HashMap::new(); + for mut tenant in base_paths { + let conv_path = RelativePathBuf::from_iter([&tenant, ".keystone"]); + let objs = self + .storage + .get_objects( + Some(&conv_path), + Box::new(|file_name| { + file_name.ends_with(".json") && file_name.starts_with("conv_") + }), + &Some(tenant.clone()), + ) + .await?; + if tenant.is_empty() { + tenant.clone_from(&DEFAULT_TENANT.to_string()); + } + conversations.insert(tenant, objs); + } + + Ok(conversations) + } + + /// This function puts a conversation in the object store at the given path + async fn put_conversation( + &self, + obj: &dyn MetastoreObject, + tenant_id: &Option, + ) -> Result<(), MetastoreError> { + let id = obj.get_object_id(); + let path = if let Some(tenant_id) = tenant_id.as_ref().filter(|t| !t.is_empty() && t.as_str() != crate::parseable::DEFAULT_TENANT) { + RelativePathBuf::from_iter([tenant_id, ".keystone", &format!("conv_{id}.json")]) + } else { + RelativePathBuf::from_iter([".keystone", &format!("conv_{id}.json")]) + }; + Ok(self + .storage + .put_object(&path, to_bytes(obj), tenant_id) + .await?) + } + + /// Delete a conversation + async fn delete_conversation( + &self, + obj: &dyn MetastoreObject, + tenant_id: &Option, + ) -> Result<(), MetastoreError> { + let id = obj.get_object_id(); + let path = if let Some(tenant_id) = tenant_id.as_ref().filter(|t| !t.is_empty() && t.as_str() != crate::parseable::DEFAULT_TENANT) { + RelativePathBuf::from_iter([tenant_id, ".keystone", &format!("conv_{id}.json")]) + } else { + RelativePathBuf::from_iter([".keystone", &format!("conv_{id}.json")]) + }; + Ok(self.storage.delete_object(&path, tenant_id).await?) + } + + /// This function fetches all the alerts from the underlying object store + async fn get_alerts(&self) -> Result>, MetastoreError> { + let base_paths = PARSEABLE.list_tenants().unwrap_or_else(|| vec!["".into()]); + let mut all_alerts = HashMap::new(); + for mut tenant in base_paths { + let alerts_path = RelativePathBuf::from_iter([&tenant, ALERTS_ROOT_DIRECTORY]); + let alerts = match self + .storage + .get_objects( + Some(&alerts_path), + Box::new(|file_name| { + !file_name.starts_with("alert_state_") && file_name != "mttr.json" && file_name.ends_with(".json") + }), + &Some(tenant.clone()), + ) + .await + { + Ok(alerts) => alerts, + Err(err) if is_missing_optional_dir(&err) => Vec::new(), + Err(err) => return Err(MetastoreError::ObjectStorageError(err)), + }; + if tenant.is_empty() { + tenant.clone_from(&DEFAULT_TENANT.to_string()); + } + all_alerts.insert(tenant, alerts); + } + Ok(all_alerts) + } + + /// This function puts an alert in the object store at the given path + async fn put_alert( + &self, + obj: &dyn MetastoreObject, + tenant_id: &Option, + ) -> Result<(), MetastoreError> { + let id = Ulid::from_string(&obj.get_object_id()).map_err(|e| MetastoreError::Error { + status_code: StatusCode::BAD_REQUEST, + message: e.to_string(), + flow: "put_alert".into(), + })?; + let path = alert_json_path(id, tenant_id); + + Ok(self + .storage + .put_object(&path, to_bytes(obj), tenant_id) + .await?) + } + + /// Delete an alert + async fn delete_alert( + &self, + obj: &dyn MetastoreObject, + tenant_id: &Option, + ) -> Result<(), MetastoreError> { + let id = Ulid::from_string(&obj.get_object_id()).map_err(|e| MetastoreError::Error { + status_code: StatusCode::BAD_REQUEST, + message: e.to_string(), + flow: "delete_alert".into(), + })?; + let path = alert_json_path(id, tenant_id); + match self.storage.delete_object(&path, tenant_id).await { + Ok(()) => Ok(()), + Err(err) if is_missing_optional_dir(&err) => Ok(()), + Err(err) => Err(MetastoreError::ObjectStorageError(err)), + } + } + + /// alerts state + async fn get_alert_states( + &self, + tenant_id: &Option, + ) -> Result, MetastoreError> { + let tenant = tenant_id.as_deref().filter(|t| !t.is_empty() && *t != DEFAULT_TENANT).unwrap_or(""); + let base_path = RelativePathBuf::from_iter([tenant, ALERTS_ROOT_DIRECTORY]); + let alert_state_bytes = self + .storage + .get_objects( + Some(&base_path), + Box::new(|file_name| { + file_name.starts_with("alert_state_") && file_name.ends_with(".json") + }), + tenant_id, + ) + .await?; + + let mut alert_states = Vec::new(); + for bytes in alert_state_bytes { + if let Ok(entry) = serde_json::from_slice::(&bytes) { + alert_states.push(entry); + } + } + + Ok(alert_states) + } + + async fn get_alert_state_entry( + &self, + alert_id: &Ulid, + tenant_id: &Option, + ) -> Result, MetastoreError> { + let path = alert_state_json_path(*alert_id, tenant_id); + match self.storage.get_object(&path, tenant_id).await { + Ok(bytes) => { + if let Ok(entry) = serde_json::from_slice::(&bytes) { + Ok(Some(entry)) + } else { + Ok(None) + } + } + Err(ObjectStorageError::NoSuchKey(_)) => Ok(None), + Err(e) => Err(MetastoreError::ObjectStorageError(e)), + } + } + + async fn put_alert_state( + &self, + obj: &dyn MetastoreObject, + tenant_id: &Option, + ) -> Result<(), MetastoreError> { + let id = Ulid::from_string(&obj.get_object_id()).map_err(|e| MetastoreError::Error { + status_code: StatusCode::BAD_REQUEST, + message: e.to_string(), + flow: "put_alert_state".into(), + })?; + let path = alert_state_json_path(id, tenant_id); + + // Parse the new state entry from the MetastoreObject + let new_state_entry: AlertStateEntry = serde_json::from_slice(&to_bytes(obj))?; + let new_state = new_state_entry + .current_state() + .ok_or_else(|| MetastoreError::InvalidJsonStructure { + expected: "AlertStateEntry with at least one state".to_string(), + found: "AlertStateEntry with empty states".to_string(), + })? + .state; + + // Try to read and parse existing file + if let Ok(existing_bytes) = self.storage.get_object(&path, tenant_id).await { + // File exists - try to parse and update + if let Ok(mut existing_entry) = + serde_json::from_slice::(&existing_bytes) + { + // Update the state and only save if it actually changed + let state_changed = existing_entry.update_state(new_state); + + if state_changed { + let updated_bytes = serde_json::to_vec(&existing_entry) + .map_err(MetastoreError::JsonParseError)?; + + self.storage + .put_object(&path, updated_bytes.into(), tenant_id) + .await?; + } + return Ok(()); + } + } + + // Create and save new entry (either file didn't exist or parsing failed) + let new_entry = AlertStateEntry::new(id, new_state, tenant_id.clone()); + let new_bytes = serde_json::to_vec(&new_entry).map_err(MetastoreError::JsonParseError)?; + + self.storage + .put_object(&path, new_bytes.into(), tenant_id) + .await?; + + Ok(()) + } + + /// Delete an alert state file + async fn delete_alert_state( + &self, + obj: &dyn MetastoreObject, + tenant_id: &Option, + ) -> Result<(), MetastoreError> { + let id = Ulid::from_string(&obj.get_object_id()).map_err(|e| MetastoreError::Error { + status_code: StatusCode::BAD_REQUEST, + message: e.to_string(), + flow: "delete_alert_state".into(), + })?; + let path = alert_state_json_path(id, tenant_id); + match self.storage.delete_object(&path, tenant_id).await { + Ok(()) => Ok(()), + Err(err) if is_missing_optional_dir(&err) => Ok(()), + Err(err) => Err(MetastoreError::ObjectStorageError(err)), + } + } + + /// Get MTTR history from storage + async fn get_mttr_history( + &self, + tenant_id: &Option, + ) -> Result, MetastoreError> { + let path = mttr_json_path(tenant_id); + match self.storage.get_object(&path, tenant_id).await { + Ok(bytes) => { + if let Ok(mut history) = serde_json::from_slice::(&bytes) { + if let Some(tenant) = tenant_id.as_ref() { + history.tenant_id = Some(tenant.clone()); + } + Ok(Some(history)) + } else { + Ok(None) + } + } + Err(ObjectStorageError::NoSuchKey(_)) => Ok(None), + Err(e) => Err(MetastoreError::ObjectStorageError(e)), + } + } + + /// Put MTTR history to storage + async fn put_mttr_history( + &self, + obj: &dyn MetastoreObject, + tenant_id: &Option, + ) -> Result<(), MetastoreError> { + let path = RelativePathBuf::from(obj.get_object_path()); + Ok(self + .storage + .put_object(&path, to_bytes(obj), tenant_id) + .await?) + } + + /// This function fetches all the llmconfigs from the underlying object store + async fn get_llmconfigs(&self) -> Result>, MetastoreError> { + let base_paths = PARSEABLE.list_tenants().unwrap_or_else(|| vec!["".into()]); + let mut all_configs = HashMap::new(); + for mut tenant in base_paths { + let base_path = + RelativePathBuf::from_iter([&tenant, SETTINGS_ROOT_DIRECTORY, "llmconfigs"]); + let conf_bytes = self + .storage + .get_objects( + Some(&base_path), + Box::new(|file_name| file_name.ends_with(".json")), + &Some(tenant.clone()), + ) + .await?; + if tenant.is_empty() { + tenant.clone_from(&DEFAULT_TENANT.to_string()); + } + all_configs.insert(tenant, conf_bytes); + } + Ok(all_configs) + } + + /// This function puts an llmconfig in the object store at the given path + async fn put_llmconfig( + &self, + obj: &dyn MetastoreObject, + tenant_id: &Option, + ) -> Result<(), MetastoreError> { + let path = obj.get_object_path(); + + Ok(self + .storage + .put_object(&RelativePathBuf::from(path), to_bytes(obj), tenant_id) + .await?) + } + + /// Delete an llmconfig + async fn delete_llmconfig( + &self, + obj: &dyn MetastoreObject, + tenant_id: &Option, + ) -> Result<(), MetastoreError> { + let path = obj.get_object_path(); + Ok(self + .storage + .delete_object(&RelativePathBuf::from(path), tenant_id) + .await?) + } + + async fn get_dashboard_versioned( + &self, + path: &str, + tenant_id: &Option, + ) -> Result, ConditionalMetadataError> { + self.storage + .get_metadata_versioned(RelativePath::new(path), tenant_id) + .await + } + + async fn put_dashboard_conditional( + &self, + path: &str, + bytes: Bytes, + condition: MetadataWriteCondition, + tenant_id: &Option, + ) -> Result { + self.storage + .put_metadata_conditional(RelativePath::new(path), bytes, condition, tenant_id) + .await + } + + async fn delete_dashboard_conditional( + &self, + path: &str, + version: MetadataVersion, + tenant_id: &Option, + ) -> Result<(), ConditionalMetadataError> { + self.storage + .delete_metadata_conditional(RelativePath::new(path), version, tenant_id) + .await + } + + /// Fetch all dashboards; hidden service indexes are not dashboard owners. + async fn get_dashboards(&self) -> Result>, MetastoreError> { + let mut dashboards = HashMap::new(); + let tenants = PARSEABLE.list_tenants().unwrap_or_else(|| vec!["".into()]); + for tenant in tenants { + let tenant_id = if tenant.is_empty() { + None + } else { + Some(tenant.clone()) + }; + let bytes = self.get_tenant_dashboards(&tenant_id).await?; + dashboards.insert( + tenant_id.unwrap_or_else(|| DEFAULT_TENANT.to_string()), + bytes, + ); + } + Ok(dashboards) + } + + async fn get_tenant_dashboards( + &self, + tenant_id: &Option, + ) -> Result, MetastoreError> { + let users = + RelativePathBuf::from_iter([tenant_id.as_deref().filter(|t| !t.is_empty() && *t != crate::parseable::DEFAULT_TENANT).unwrap_or(""), USERS_ROOT_DIR]); + let mut dashboards = Vec::new(); + for owner in self.storage.list_dirs_relative(&users, tenant_id).await? { + if owner.starts_with('.') { + continue; + } + let path = users.join(owner).join("dashboards"); + match self + .storage + .get_objects( + Some(&path), + Box::new(|name| name.ends_with(".json")), + tenant_id, + ) + .await + { + Ok(bytes) => dashboards.extend(bytes), + Err(error) if is_missing_optional_dir(&error) => {} + Err(error) => return Err(error.into()), + } + } + Ok(dashboards) + } + + /// Save a dashboard + async fn put_dashboard( + &self, + obj: &dyn MetastoreObject, + tenant_id: &Option, + ) -> Result<(), MetastoreError> { + // we need the path to store in obj store + let path = obj.get_object_path(); + + Ok(self + .storage + .put_object(&RelativePathBuf::from(path), to_bytes(obj), tenant_id) + .await?) + } + + /// Delete a dashboard + async fn delete_dashboard( + &self, + obj: &dyn MetastoreObject, + tenant_id: &Option, + ) -> Result<(), MetastoreError> { + let path = obj.get_object_path(); + Ok(self + .storage + .delete_object(&RelativePathBuf::from(path), tenant_id) + .await?) + } + + /// Fetch all chats + async fn get_chats( + &self, + ) -> Result>>, MetastoreError> { + let base_paths = PARSEABLE.list_tenants().unwrap_or_else(|| vec!["".into()]); + let all_chats = DashMap::new(); + for tenant in base_paths { + let all_user_chats = DashMap::new(); + let tenant_id = if tenant.is_empty() { + None + } else { + Some(tenant.clone()) + }; + let users_dir = RelativePathBuf::from_iter([&tenant, USERS_ROOT_DIR]); + for user in self + .storage + .list_dirs_relative(&users_dir, &tenant_id) + .await? + { + if user.starts_with(".") { + continue; + } + let mut chats = Vec::new(); + let chats_path = users_dir.join(&user).join("chats"); + let user_chats = self + .storage + .get_objects( + Some(&chats_path), + Box::new(|file_name| file_name.ends_with(".json")), + &tenant_id, + ) + .await?; + for chat in user_chats { + chats.push(chat); + } + + all_user_chats.insert(user, chats); + } + all_chats.insert( + tenant_id.as_deref().unwrap_or(DEFAULT_TENANT).into(), + all_user_chats, + ); + } + Ok(all_chats) + } + + /// Save a chat + async fn put_chat( + &self, + obj: &dyn MetastoreObject, + tenant_id: &Option, + ) -> Result<(), MetastoreError> { + // we need the path to store in obj store + let path = obj.get_object_path(); + + Ok(self + .storage + .put_object(&RelativePathBuf::from(path), to_bytes(obj), tenant_id) + .await?) + } + + /// Delete a chat + async fn delete_chat( + &self, + obj: &dyn MetastoreObject, + tenant_id: &Option, + ) -> Result<(), MetastoreError> { + let path = obj.get_object_path(); + Ok(self + .storage + .delete_object(&RelativePathBuf::from(path), tenant_id) + .await?) + } + + // Reconcile duplicate SQL paths before exposing filters to the cache. + async fn get_filters(&self) -> Result>, MetastoreError> { + let mut all = HashMap::new(); + let tenants = PARSEABLE.list_tenants().unwrap_or_else(|| vec!["".into()]); + for tenant in tenants { + let tenant_id = Some(tenant.clone()); + let filters = self.load_tenant_filters(&tenant_id).await?; + all.insert( + if tenant.is_empty() { + DEFAULT_TENANT.to_owned() + } else { + tenant + }, + filters, + ); + } + Ok(all) + } + + /// Save a filter + async fn put_filter( + &self, + obj: &dyn MetastoreObject, + tenant_id: &Option, + ) -> Result<(), MetastoreError> { + // we need the path to store in obj store + let path = obj.get_object_path(); + + Ok(self + .storage + .put_object(&RelativePathBuf::from(path), to_bytes(obj), tenant_id) + .await?) + } + + async fn reconcile_filter( + &self, + filter: &Filter, + tenant_id: &Option, + ) -> Result<(), MetastoreError> { + let canonical = stored_filter_path(filter, tenant_id); + for (path, _) in self.filter_aliases(filter, tenant_id).await? { + if path != canonical { + self.delete_filter_path(&path, tenant_id).await?; + } + } + Ok(()) + } + + /// Delete aliases first. If any cleanup fails, the canonical copy remains + /// available for a retry instead of allowing an old alias to reappear. + async fn delete_filter( + &self, + filter: &Filter, + tenant_id: &Option, + ) -> Result<(), MetastoreError> { + self.reconcile_filter(filter, tenant_id).await?; + self.delete_filter_path(&stored_filter_path(filter, tenant_id), tenant_id) + .await + } + + /// Fetch an `ObjectStoreFormat` file + /// + /// If `get_base` is true, get the one at the base of the stream directory else depends on Mode + async fn get_stream_json( + &self, + stream_name: &str, + get_base: bool, + tenant_id: &Option, + _is_migration: bool, + ) -> Result { + let tenant = tenant_id.as_deref().filter(|t| !t.is_empty() && *t != crate::parseable::DEFAULT_TENANT).unwrap_or(""); + let path = if get_base { + RelativePathBuf::from_iter([ + tenant, + stream_name, + STREAM_ROOT_DIRECTORY, + STREAM_METADATA_FILE_NAME, + ]) + } else { + stream_json_path(stream_name, tenant_id) + }; + Ok(self.storage.get_object(&path, tenant_id).await?) + } + + /// Fetch all `ObjectStoreFormat` present in a stream folder + async fn get_all_stream_jsons( + &self, + stream_name: &str, + mode: Option, + tenant_id: &Option, + _is_migration: bool, + ) -> Result, MetastoreError> { + let root = tenant_id.as_deref().filter(|t| !t.is_empty() && *t != crate::parseable::DEFAULT_TENANT).unwrap_or(""); + let path = RelativePathBuf::from_iter([root, stream_name, STREAM_ROOT_DIRECTORY]); + if let Some(mode) = mode { + if mode.eq(&Mode::Ingest) { + Ok(self + .storage + .get_objects( + Some(&path), + Box::new(|file_name| { + file_name.starts_with(".ingestor") && file_name.ends_with("stream.json") + }), + tenant_id, + ) + .await?) + } else { + return Err(MetastoreError::Error { + status_code: StatusCode::BAD_REQUEST, + message: "Incorrect server mode passed as input. Only `Ingest` is allowed." + .into(), + flow: "get_all_streams with mode".into(), + }); + } + } else { + Ok(self + .storage + .get_objects( + Some(&path), + Box::new(|file_name| file_name.ends_with("stream.json")), + tenant_id, + ) + .await?) + } + } + + /// Save an `ObjectStoreFormat` file + async fn put_stream_json( + &self, + obj: &dyn MetastoreObject, + stream_name: &str, + tenant_id: &Option, + ) -> Result<(), MetastoreError> { + let path = stream_json_path(stream_name, tenant_id); + + Ok(self + .storage + .put_object(&path, to_bytes(obj), tenant_id) + .await?) + } + + /// Fetch manifest files for the given date keys in parallel. + /// + /// Per-date work runs concurrently via `futures::future::try_join_all` + /// so a slow LIST on one date does not block the others. Each date's + /// timing is logged at info level so any pathologically slow dates + /// (S3 retries, oversized prefixes) are visible without needing extra + /// span instrumentation. + async fn get_manifest_files_for_dates( + &self, + stream_name: &str, + tenant_id: &Option, + dates: &[String], + ) -> Result>, MetastoreError> { + let inventory_limit = Arc::new(tokio::sync::Semaphore::new(16)); + let date_futures = dates.iter().map(|date| { + let stream = stream_name.to_string(); + let tenant = tenant_id.clone(); + let date = date.clone(); + let inventory_limit = inventory_limit.clone(); + async move { + let t_start = std::time::Instant::now(); + let date_path = if let Some(tenant) = tenant.as_ref().filter(|t| !t.is_empty() && t.as_str() != DEFAULT_TENANT) { + object_store::path::Path::from(format!("{}/{}/{}", tenant, stream, date)) + } else { + object_store::path::Path::from(format!("{}/{}", stream, date)) + }; + + let t_list = std::time::Instant::now(); + let list_permit = inventory_limit + .clone() + .acquire_owned() + .await + .expect("inventory semaphore remains open"); + let list_result = scoped_delimiter!($local_listing, self, Some(date_path), &tenant); + drop(list_permit); + let resp = match list_result { + Ok(r) => r, + Err(ObjectStorageError::NoSuchKey(_)) => { + info!( + stream = %stream, + tenant = ?tenant, + date = %date, + list_ms = t_list.elapsed().as_millis() as u64, + "manifest fetch: date prefix not found" + ); + return Ok::<(String, Vec), MetastoreError>((date, Vec::new())); + } + Err(e) => return Err(e.into()), + }; + let list_ms = t_list.elapsed().as_millis() as u64; + + let manifest_paths: Vec = resp + .objects + .iter() + .filter(|name| name.location.filename().unwrap().ends_with("manifest.json")) + .map(|name| name.location.to_string()) + .collect(); + let manifest_count = manifest_paths.len(); + + let t_gets = std::time::Instant::now(); + let manifests = futures::stream::iter(manifest_paths) + .map(|path| { + let tenant = tenant.clone(); + let inventory_limit = inventory_limit.clone(); + async move { + let _permit = inventory_limit + .acquire_owned() + .await + .expect("inventory semaphore remains open"); + let bytes = self + .storage + .get_object(&RelativePathBuf::from(path), &tenant) + .await?; + Ok::(decode_manifest_blocking(bytes).await?) + } + }) + .buffer_unordered(16) + .try_collect::>() + .await?; + let gets_ms = t_gets.elapsed().as_millis() as u64; + let total_ms = t_start.elapsed().as_millis() as u64; + + if total_ms > 100 { + info!( + stream = %stream, + tenant = ?tenant, + date = %date, + list_ms, + gets_ms, + total_ms, + manifest_count, + "manifest fetch per-date timing" + ); + } + Ok((date, manifests)) + } + }); + + let results: Vec<(String, Vec)> = + futures::future::try_join_all(date_futures).await?; + + let mut result_file_list: BTreeMap> = BTreeMap::new(); + for (date, manifests) in results { + if !manifests.is_empty() { + result_file_list.insert(date, manifests); + } + } + + Ok(result_file_list) + } + + /// Fetch a specific `Manifest` file + async fn get_manifest( + &self, + stream_name: &str, + lower_bound: DateTime, + upper_bound: DateTime, + manifest_url: Option, + tenant_id: &Option, + ) -> Result, MetastoreError> { + let path = match manifest_url { + Some(url) => RelativePathBuf::from(url), + None => { + let path = partition_path(stream_name, lower_bound, upper_bound, tenant_id); + manifest_path(path.as_str()) + } + }; + match self.storage.get_object(&path, tenant_id).await { + Ok(bytes) => { + let manifest = decode_manifest_blocking(bytes).await?; + Ok(Some(manifest)) + } + Err(ObjectStorageError::NoSuchKey(_)) => Ok(None), + Err(err) => Err(MetastoreError::ObjectStorageError(err)), + } + } + + /// Get the path for a specific `Manifest` file + async fn get_manifest_path( + &self, + stream_name: &str, + lower_bound: DateTime, + upper_bound: DateTime, + tenant_id: &Option, + ) -> Result { + let path = partition_path(stream_name, lower_bound, upper_bound, tenant_id); + Ok(self + .storage + .absolute_url(&manifest_path(path.as_str())) + .to_string()) + } + + async fn put_manifest( + &self, + manifest: &Manifest, + stream_name: &str, + lower_bound: DateTime, + upper_bound: DateTime, + tenant_id: &Option, + ) -> Result<(), MetastoreError> { + let manifest_file_name = manifest_path("").to_string(); + let path = partition_path(stream_name, lower_bound, upper_bound, tenant_id) + .join(&manifest_file_name); + + Ok(self + .storage + .put_object(&path, encode_manifest(manifest)?, tenant_id) + .await?) + } + + async fn delete_manifest( + &self, + stream_name: &str, + lower_bound: DateTime, + upper_bound: DateTime, + tenant_id: &Option, + ) -> Result<(), MetastoreError> { + let manifest_file_name = manifest_path("").to_string(); + let path = partition_path(stream_name, lower_bound, upper_bound, tenant_id) + .join(&manifest_file_name); + Ok(self.storage.delete_object(&path, tenant_id).await?) + } + + /// targets + async fn get_targets(&self) -> Result>, MetastoreError> { + let base_paths = PARSEABLE.list_tenants().unwrap_or_else(|| vec!["".into()]); + let mut all_targets = HashMap::new(); + for mut tenant in base_paths { + let targets_path = RelativePathBuf::from_iter([ + &tenant, + SETTINGS_ROOT_DIRECTORY, + TARGETS_ROOT_DIRECTORY, + ]); + let target_bytes = match self + .storage + .get_objects( + Some(&targets_path), + Box::new(|file_name| file_name.ends_with(".json")), + &Some(tenant.clone()), + ) + .await + { + Ok(targets) => targets, + Err(err) if is_missing_optional_dir(&err) => Vec::new(), + Err(err) => return Err(MetastoreError::ObjectStorageError(err)), + }; + let targets = target_bytes + .iter() + .filter_map(|bytes| { + serde_json::from_slice(bytes) + .inspect_err(|err| warn!("Expected compatible json, error = {err}")) + .ok() + }) + .collect(); + if tenant.is_empty() { + tenant.clone_from(&DEFAULT_TENANT.to_string()); + } + all_targets.insert(tenant, targets); + } + Ok(all_targets) + } + + async fn put_target( + &self, + obj: &dyn MetastoreObject, + tenant_id: &Option, + ) -> Result<(), MetastoreError> { + // we need the path to store in obj store + let path = obj.get_object_path(); + + Ok(self + .storage + .put_object(&RelativePathBuf::from(path), to_bytes(obj), tenant_id) + .await?) + } + + async fn delete_target( + &self, + obj: &dyn MetastoreObject, + tenant_id: &Option, + ) -> Result<(), MetastoreError> { + // we need the path to store in obj store + let path = obj.get_object_path(); + + Ok(self + .storage + .delete_object(&RelativePathBuf::from(path), tenant_id) + .await?) + } + + async fn get_outbound_policies( + &self, + ) -> Result, MetastoreError> { + let mut all_policies = HashMap::new(); + + let base_paths = PARSEABLE.list_tenants().unwrap_or_else(|| vec!["".into()]); + + for tenant in base_paths { + let tenant_id = if tenant.is_empty() { + None + } else { + Some(tenant.clone()) + }; + let path = outbound_http_policy_json_path(&tenant_id); + let policy = match self.storage.get_object(&path, &tenant_id).await { + Ok(bytes) => match serde_json::from_slice::(&bytes) { + Ok(policy) => policy, + Err(err) => { + warn!( + "Failed to deserialize outbound HTTP policy for tenant {tenant}: {err}" + ); + continue; + } + }, + Err(err) if is_missing_optional_dir(&err) => continue, + Err(err) => return Err(MetastoreError::ObjectStorageError(err)), + }; + + all_policies.insert(tenant, policy); + } + + Ok(all_policies) + } + + async fn put_outbound_policy( + &self, + tenant_id: &Option, + policy: &AlertTargetPolicyConfig, + ) -> Result<(), MetastoreError> { + let path = outbound_http_policy_json_path(tenant_id); + Ok(self + .storage + .put_object(&path, to_bytes(policy), tenant_id) + .await?) + } + + async fn get_all_schemas( + &self, + stream_name: &str, + tenant_id: &Option, + ) -> Result, MetastoreError> { + let path_prefix = if let Some(tenant) = tenant_id.as_ref().filter(|t| !t.is_empty() && t.as_str() != crate::parseable::DEFAULT_TENANT) { + relative_path::RelativePathBuf::from(format!( + "{tenant}/{stream_name}/{STREAM_ROOT_DIRECTORY}" + )) + } else { + relative_path::RelativePathBuf::from(format!("{stream_name}/{STREAM_ROOT_DIRECTORY}")) + }; + Ok(self + .storage + .get_objects( + Some(&path_prefix), + Box::new(|file_name: String| file_name.contains(".schema")), + tenant_id, + ) + .await? + .iter() + // we should be able to unwrap as we know the data is valid schema + .map(|byte_obj| { + serde_json::from_slice(byte_obj) + .unwrap_or_else(|_| panic!("got an invalid schema for stream: {stream_name}")) + }) + .collect()) + } + + async fn refresh_schema( + &self, + stream_name: &str, + tenant_id: &Option, + schema_file: &str, + ) -> Result { + let objects = self.storage.schema_objects(stream_name, tenant_id).await?; + let mut schemas = Vec::with_capacity(objects.len()); + let mut canonical = None; + for (name, bytes) in objects { + let schema: Schema = serde_json::from_slice(&bytes) + .map_err(|e| ObjectStorageError::Custom(e.to_string()))?; + if name == schema_file { + canonical = Some(schema.clone()); + } + schemas.push(schema); + } + let merged = + Schema::try_merge(schemas).map_err(|e| ObjectStorageError::Custom(e.to_string()))?; + // Field order is immaterial, but nested types, nullability and field + // metadata all matter. Normalize only the top-level ordering. + let normalize = |schema: &Schema| { + let mut schema = schema.clone(); + let mut fields = schema.fields.to_vec(); + fields.sort_by(|a, b| a.name().cmp(b.name())); + schema.fields = fields.into(); + schema + }; + if canonical.as_ref().map(normalize) != Some(normalize(&merged)) { + let path = RelativePathBuf::from_iter([ + tenant_id.as_deref().filter(|t| !t.is_empty() && *t != crate::parseable::DEFAULT_TENANT).unwrap_or(""), + stream_name, + STREAM_ROOT_DIRECTORY, + schema_file, + ]); + self.storage + .put_object(&path, to_bytes(&merged), tenant_id) + .await?; + } + Ok(merged) + } + + async fn get_schema( + &self, + stream_name: &str, + tenant_id: &Option, + ) -> Result { + Ok(self + .storage + .get_object(&schema_path(stream_name, tenant_id), tenant_id) + .await?) + } + + async fn put_schema( + &self, + obj: Schema, + stream_name: &str, + tenant_id: &Option, + ) -> Result<(), MetastoreError> { + let path = schema_path(stream_name, tenant_id); + Ok(self + .storage + .put_object(&path, to_bytes(&obj), tenant_id) + .await?) + } + + async fn get_parseable_metadata( + &self, + tenant_id: &Option, + ) -> Result, MetastoreError> { + let path = if let Some(tenant_id) = tenant_id.as_ref().filter(|t| !t.is_empty() && t.as_str() != crate::parseable::DEFAULT_TENANT) { + RelativePathBuf::from_iter([tenant_id, ".parseable.json"]) + } else { + parseable_json_path() + }; + + let parseable_metadata: Option = + match self.storage.get_object(&path, tenant_id).await { + Ok(bytes) => Some(bytes), + Err(err) => { + if matches!(err, ObjectStorageError::NoSuchKey(_)) { + None + } else { + return Err(MetastoreError::ObjectStorageError(err)); + } + } + }; + + Ok(parseable_metadata) + } + + async fn delete_tenant(&self, tenant_id: &str) -> Result<(), MetastoreError> { + self.storage + .delete_prefix(&RelativePathBuf::from(tenant_id), &None) + .await + .map_err(MetastoreError::ObjectStorageError) + } + + async fn get_ingestor_metadata(&self) -> Result, MetastoreError> { + let base_path = RelativePathBuf::from(PARSEABLE_ROOT_DIRECTORY); + Ok(self + .storage + .get_objects( + Some(&base_path), + Box::new(|file_name| file_name.starts_with("ingestor")), + &None, + ) + .await?) + } + + async fn put_parseable_metadata( + &self, + obj: &dyn MetastoreObject, + tenant_id: &Option, + ) -> Result<(), MetastoreError> { + let path = if let Some(tenant_id) = tenant_id.as_ref().filter(|t| !t.is_empty() && t.as_str() != crate::parseable::DEFAULT_TENANT) { + RelativePathBuf::from_iter([tenant_id, ".parseable.json"]) + } else { + parseable_json_path() + }; + + self.storage + .put_object(&path, to_bytes(obj), tenant_id) + .await + .map_err(MetastoreError::ObjectStorageError) + } + + async fn get_node_metadata( + &self, + node_type: NodeType, + tenant_id: &Option, + ) -> Result, MetastoreError> { + let root_path = RelativePathBuf::from(PARSEABLE_ROOT_DIRECTORY); + let prefix_owned = node_type.to_string(); + + let metadata = self + .storage + .get_objects( + Some(&root_path), + Box::new(move |file_name| file_name.starts_with(&prefix_owned)), // Use the owned copy + tenant_id, + ) + .await? + .into_iter() + .collect(); + + Ok(metadata) + } + + async fn put_node_metadata(&self, obj: &dyn MetastoreObject) -> Result<(), MetastoreError> { + let path = obj.get_object_path(); + self.storage + .put_object(&RelativePathBuf::from(path), to_bytes(obj), &None) + .await?; + Ok(()) + } + + async fn delete_node_metadata( + &self, + domain_name: &str, + node_type: NodeType, + ) -> Result { + let metadatas = self + .storage + .get_objects( + Some(&RelativePathBuf::from(PARSEABLE_ROOT_DIRECTORY)), + Box::new(move |file_name| file_name.starts_with(&node_type.to_string())), + &None, + ) + .await?; + + let node_metadatas = metadatas + .iter() + .filter_map(|elem| match serde_json::from_slice::(elem) { + Ok(meta) if meta.domain_name() == domain_name => Some(meta), + _ => None, + }) + .collect::>(); + + if node_metadatas.is_empty() { + return Ok(false); + } + + let node_meta_filename = node_metadatas[0].file_path().to_string(); + let file = RelativePathBuf::from(&node_meta_filename); + + match self.storage.delete_object(&file, &None).await { + Ok(_) => Ok(true), + Err(err) => { + if matches!(err, ObjectStorageError::IoError(_)) { + Ok(false) + } else { + Err(MetastoreError::ObjectStorageError(err)) + } + } + } + } + + $($lifecycle)* + + async fn list_streams( + &self, + tenant_id: &Option, + ) -> Result, MetastoreError> { + postgres_list_streams!($local_listing, self, tenant_id, { + // using LocalFS list_streams because it doesn't implement list_with_delimiter + if $local_listing && self.storage.store_url().scheme() == "file" { + self.storage.list_streams() + .await + .map_err(MetastoreError::ObjectStorageError) + } else { + // not local-disk, object storage + let root = tenant_id + .as_ref() + .map(|tenant| object_store::path::Path::from_iter([tenant.clone()])); + + let resp = scoped_delimiter!($local_listing, self, root.clone(), tenant_id)?; + + let streams = resp + .common_prefixes + .iter() + .flat_map(|path| path.parts()) + .map(|name| name.as_ref().to_string()) + .filter(|name| { + name != PARSEABLE_ROOT_DIRECTORY + && name != USERS_ROOT_DIR + && name != SETTINGS_ROOT_DIRECTORY + && name != ALERTS_ROOT_DIRECTORY + && name != crate::storage::TOMBSTONE_ROOT_DIRECTORY + }) + .collect::>(); + let results = super::document_metastore::collect_dataset_probes( + streams.into_iter().map(|stream| { + let root = root.clone(); + async move { + if !$local_listing && self.is_stream_deleting(&stream, tenant_id).await? { + return Ok::<_, MetastoreError>(None); + } + let stream_path = if let Some(root) = root.as_ref() { + object_store::path::Path::from_iter([ + root.as_ref(), + &stream, + STREAM_ROOT_DIRECTORY, + ]) + } else { + object_store::path::Path::from(format!( + "{}/{}", + stream, STREAM_ROOT_DIRECTORY + )) + }; + let resp = + scoped_delimiter!($local_listing, self, Some(stream_path), tenant_id)?; + if resp + .objects + .iter() + .any(|name| name.location.filename().unwrap().ends_with("stream.json")) + { + Ok(Some(stream)) + } else { + Ok(None) + } + } + }), + ) + .await?; + Ok(results.into_iter().flatten().collect()) + } + }) + } +} + + +}; +} +pub(crate) use document_metastore; + +/// Runs dataset probes sixteen at a time. All-or-error collection: a failed +/// probe cannot publish a partial inventory. +pub(crate) async fn collect_dataset_probes( + probes: impl IntoIterator>>, +) -> Result, E> { + use futures::{StreamExt, TryStreamExt}; + futures::stream::iter(probes) + .buffer_unordered(16) + .try_collect() + .await +} + +/// Classification belongs to the shared document layer. Never reconstruct a key. +pub(crate) fn document_scope(key: &str) -> Option<(String, &'static str)> { + if key == super::migration::PAIR || key == super::migration::BUSY || key.ends_with(".lock") { + return None; + } + let parts: Vec<_> = key.split('/').collect(); + if key.ends_with("manifest.json") { + let date = parts.iter().position(|p| p.starts_with("date="))?; + return match date { + 1 => Some((String::new(), "manifest")), + 2 => Some(( + if parts[0] == crate::parseable::DEFAULT_TENANT { + String::new() + } else { + parts[0].to_owned() + }, + "manifest", + )), + _ => None, + }; + } + let hidden = parts.iter().position(|p| p.starts_with('.'))?; + let tenant = match hidden { + 0 => "", + 1 if parts[1] == crate::storage::STREAM_ROOT_DIRECTORY + || parts[1].ends_with("stream.json") + || parts[1].contains(".schema") => + { + "" + } + 1 => parts[0], + 2 if parts[2] == crate::storage::STREAM_ROOT_DIRECTORY + || parts[2].ends_with("stream.json") + || parts[2].contains(".schema") => + { + parts[0] + } + _ => return None, + }; + Some(( + if tenant == crate::parseable::DEFAULT_TENANT { + String::new() + } else { + tenant.to_owned() + }, + super::postgres_documents::kind(relative_path::RelativePath::new(key)), + )) +} diff --git a/src/metastore/metastores/migration.rs b/src/metastore/metastores/migration.rs new file mode 100644 index 000000000..0f8a6b672 --- /dev/null +++ b/src/metastore/metastores/migration.rs @@ -0,0 +1,1064 @@ +//! Offline document transfer. Bodies are processed one at a time; inventories hold keys only. +use super::postgres_documents::PostgresDocuments; +use crate::storage::{ObjectStorage, ObjectStorageError}; +use anyhow::{Context, ensure}; +use bytes::Bytes; +use relative_path::RelativePath; +use sha2::{Digest, Sha256}; +use std::{collections::BTreeMap, sync::Arc}; + +pub(crate) type InventoryDocument = ( + String, + (String, &'static str), + chrono::DateTime, +); + +pub const PAIR: &str = ".parseable/postgres-metastore-id"; +pub const BUSY: &str = ".parseable/metastore-migration"; + +fn migration_help_requested() -> bool { + std::env::args().any(|a| a == "--metadata-migration" || a.starts_with("--metadata-migration=")) +} + +#[derive(clap::Args, Debug, Default)] +pub struct MigrationOptions { + /// Offline operation; requires the server and all other writers to be stopped. + #[arg(long, value_enum)] + pub metadata_migration: Option, + #[arg(long, hide = !migration_help_requested(), requires = "metadata_migration")] + pub acknowledge_server_stopped: bool, + #[arg( + long = "dry-run", + visible_alias = "migration-dry-run", + hide = !migration_help_requested(), requires = "metadata_migration" + )] + pub migration_dry_run: bool, + /// Retired broad override; use the explicit bucket-change override instead. + #[arg(long, hide = !migration_help_requested(), requires = "metadata_migration")] + pub migration_force: bool, + /// Explicitly authorize overwriting metadata changed by bucket writers since import. + #[arg(long, hide = !migration_help_requested(), requires = "metadata_migration")] + pub overwrite_bucket_changes_since_import: bool, + #[arg( + long, + hide = !migration_help_requested(), requires = "metadata_migration", + conflicts_with = "migration_dry_run" + )] + pub abort: bool, + /// Permit missing snapshot manifests during import, retaining the dangling reference. + #[arg(long, hide = !migration_help_requested(), requires = "metadata_migration")] + pub allow_dangling_snapshot_references: bool, + #[cfg(test)] + #[arg(skip)] + pub injected_failure: Option<&'static str>, +} +#[derive(clap::ValueEnum, Debug, Clone, Copy, PartialEq, Eq)] +pub enum Action { + Import, + Export, + Verify, +} + +pub async fn optional(store: &dyn ObjectStorage, key: &str) -> anyhow::Result> { + match store.get_object(RelativePath::new(key), &None).await { + Ok(b) => Ok(Some(b)), + Err(ObjectStorageError::NoSuchKey(_)) => Ok(None), + Err(ObjectStorageError::IoError(e)) if e.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(e) => Err(e.into()), + } +} +pub(crate) fn recovery_override(value: Option<&str>, id: &[u8]) -> bool { + value.is_some_and(|v| v.as_bytes() == id) +} +pub async fn guard(store: &dyn ObjectStorage, postgres: bool) -> anyhow::Result<()> { + ensure!( + optional(store, BUSY).await?.is_none(), + "offline metadata migration incomplete; stop all writers and rerun the migration command" + ); + if let Some(id) = optional(store, PAIR).await? + && !postgres + { + ensure!( + recovery_override( + std::env::var("P_ALLOW_PAIRED_BUCKET").ok().as_deref(), + id.as_ref() + ), + "bucket paired with PostgreSQL; object-store metadata is stale. Export first, or explicitly set P_ALLOW_PAIRED_BUCKET= for disaster recovery" + ); + } + Ok(()) +} + +pub fn digest(body: &[u8]) -> String { + format!("{:x}", Sha256::digest(body)) +} + +pub(crate) use super::document_metastore::document_scope; + +pub(crate) fn stream_references(body: &[u8]) -> anyhow::Result> { + let value: serde_json::Value = serde_json::from_slice(body)?; + if matches!(value["version"].as_str(), Some("v1" | "v2" | "v3")) { + let list = value + .pointer("/snapshot/manifest_list") + .and_then(|v| v.as_array()); + return list + .into_iter() + .flatten() + .map(|v| { + v["manifest_path"] + .as_str() + .map(str::to_owned) + .context("legacy snapshot has invalid manifest path") + }) + .collect(); + } + let config: crate::storage::ObjectStoreFormat = serde_json::from_value(value)?; + Ok(config + .snapshot + .manifest_list + .into_iter() + .map(|m| m.manifest_path) + .collect()) +} + +/// An explicit default-tenant prefix is an alias only for tenant-shaped +/// metadata (not for a default stream actually named DEFAULT_TENANT). +pub(crate) fn canonical_key(key: &str) -> &str { + let Some(rest) = key.strip_prefix("DEFAULT_TENANT/") else { + return key; + }; + let parts: Vec<_> = rest.split('/').collect(); + if parts.first().is_some_and(|p| { + p.starts_with('.') && *p != ".stream" && *p != ".schema" && *p != ".stream.json" + }) || parts.get(1).is_some_and(|p| { + *p == ".stream" + || p.contains(".schema") + || p.ends_with("stream.json") + || p.starts_with("date=") + }) { + rest + } else { + key + } +} + +// Keep bytes unchanged unless a legacy explicit default-tenant locator must +// follow its document to the canonical namespace. Unknown JSON fields survive. +pub(crate) fn import_body( + store: &dyn ObjectStorage, + key: &str, + body: &Bytes, +) -> anyhow::Result { + if !key.ends_with("stream.json") { + return Ok(body.clone()); + } + let mut value: serde_json::Value = serde_json::from_slice(body)?; + let base = store.absolute_url(RelativePath::new("")); + let mut changed = false; + if let Some(items) = value + .pointer_mut("/snapshot/manifest_list") + .and_then(|v| v.as_array_mut()) + { + for item in items { + if let Some(path) = item.get("manifest_path").and_then(|v| v.as_str()) { + let relative = path + .strip_prefix(base.as_ref()) + .and_then(|s| s.strip_prefix('/')) + .unwrap_or(path); + let canonical = canonical_key(relative); + if canonical != relative { + let replacement = if relative != path { + format!("{base}/{canonical}") + } else { + canonical.to_owned() + }; + item["manifest_path"] = replacement.into(); + changed = true; + } + } + } + } + Ok(if changed { + Bytes::from(serde_json::to_vec(&value)?) + } else { + body.clone() + }) +} + +pub(crate) async fn validate( + store: &dyn ObjectStorage, + key: &str, + body: &Bytes, +) -> anyhow::Result<()> { + if key.ends_with("manifest.json") { + crate::catalog::manifest::decode_manifest(body)?; + } else if key.ends_with("stream.json") { + for manifest_path in stream_references(body).with_context(|| key.to_owned())? { + let bytes = store + .get_object(RelativePath::new(&manifest_path), &None) + .await + .with_context(|| format!("{key}: missing manifest {manifest_path}"))?; + crate::catalog::manifest::decode_manifest(&bytes)?; + } + } else if key.contains(".schema") { + serde_json::from_slice::(body).with_context(|| key.to_owned())?; + } else { + serde_json::from_slice::(body).with_context(|| key.to_owned())?; + } + ensure!( + !key.ends_with(".deleting") && !key.contains(".tombstones/"), + "unfinished deletion: {key}" + ); + Ok(()) +} + +pub async fn run( + store: Arc, + db: &PostgresDocuments, + opts: &MigrationOptions, +) -> anyhow::Result<()> { + let action = opts.metadata_migration.context("missing operation")?; + ensure!( + !opts.migration_force, + "--migration-force is retired: normal rollback needs no flag; to overwrite metadata changed by bucket writers use --overwrite-bucket-changes-since-import after backup" + ); + ensure!( + opts.acknowledge_server_stopped, + "requires --acknowledge-server-stopped; stop ALL servers and writers first" + ); + eprintln!("OFFLINE migration: all servers must remain stopped. No writer fencing exists."); + #[cfg(test)] + eprintln!( + "TEST transfer: action={action:?} dry_run={} abort={} injection={:?}", + opts.migration_dry_run, opts.abort, opts.injected_failure + ); + + let mut transfer_lock = db.transfer_lock().await?; + db.migration_preflight(store.as_ref(), opts).await?; + if opts.abort { + return db + .abort_migration(store.as_ref(), action, &transfer_lock) + .await; + } + let inventory = store.metadata_inventory().await?; + let mut skipped = 0; + for (key, _) in &inventory { + if document_scope(key).is_none() { + skipped += 1; + eprintln!("skipped key={key}"); + } + if key.starts_with(".parseable-metadata-locks/") && key.ends_with(".lock") { + continue; + } + for component in key.split('/').filter(|p| p.starts_with('.')) { + ensure!( + matches!( + component, + ".parseable" + | ".keystone" + | ".users" + | ".alerts" + | ".targets" + | ".settings" + | ".correlations" + | ".stream" + | ".dashboard-titles" + | ".dashboard-locators" + | ".dashboard-locations" + | ".parseable.json" + | ".schema" + | ".stream.json" + ) || component.starts_with(".ingestor.") + || component.starts_with(".querier."), + "unknown hidden metadata key: {key}" + ); + } + } + eprintln!("skipped keys: {skipped}"); + let keys: Vec<_> = inventory + .iter() + .filter_map(|(k, modified)| document_scope(k).map(|scope| (k.clone(), scope, *modified))) + .collect(); + let canonical: std::collections::HashSet<_> = + keys.iter().map(|(key, _, _)| canonical_key(key)).collect(); + ensure!( + action != Action::Import || canonical.len() == keys.len(), + "duplicate metadata keys after default-tenant normalization" + ); + let mut counts = BTreeMap::<(String, String), (usize, usize)>::new(); + use futures::{StreamExt, TryStreamExt}; + let statistics: Vec<_> = futures::stream::iter(&keys) + .map(|(key, (tenant, kind), modified)| { + let store = &store; + async move { + let body = store.get_object(RelativePath::new(key), &None).await?; + if action != Action::Export { + match validate(store.as_ref(), key, &body).await { + Err(e) + if opts.allow_dangling_snapshot_references + && e.to_string().contains("missing manifest") => + { + eprintln!("OVERRIDE dangling snapshot reference: {e:#}") + } + result => result?, + } + } + if chrono::Utc::now() + .signed_duration_since(*modified) + .num_seconds() + < 120 + { + eprintln!("WARNING recently modified metadata (possible active writer): {key}"); + } + println!( + "bucket tenant={tenant:?} kind={kind} key={key} bytes={} sha256={}", + body.len(), + digest(&body) + ); + Ok::<_, anyhow::Error>(((tenant.clone(), kind.to_string()), body.len())) + } + }) + .buffer_unordered(16) + .try_collect() + .await?; + for (scope, bytes) in statistics { + let count = counts.entry(scope).or_default(); + count.0 += 1; + count.1 += bytes; + } + eprintln!("counts by (tenant, kind): {counts:?}"); + // Dry-run never bootstraps SQL or writes a bucket marker. + if opts.migration_dry_run { + if action == Action::Export { + let client = db.client().await?; + db.export_preflight(&client, store.as_ref(), opts, &keys) + .await?; + } + transfer_lock.batch_execute("SELECT 1").await?; + return Ok(()); + } + transfer_lock.batch_execute("SELECT 1").await?; + db.migrate(store, opts, &keys, &mut transfer_lock).await +} + +pub(crate) fn failure_point(opts: &MigrationOptions, step: &str) -> anyhow::Result<()> { + #[cfg(test)] + anyhow::ensure!( + opts.injected_failure != Some(step), + "injected failure: {step}" + ); + #[cfg(not(test))] + let _ = (opts, step); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{metastore::metastores::postgres_metastore::PostgresOptions, storage::LocalFS}; + + #[test] + fn recovery_overrides_require_exact_backend_id() { + for value in [ + None, + Some("true"), + Some("false"), + Some(""), + Some("another-backend"), + ] { + assert!(!recovery_override(value, b"backend")); + } + assert!(recovery_override(Some("backend"), b"backend")); + } + #[test] + fn inventory_scope_preserves_all_document_namespaces() { + assert_eq!( + canonical_key("DEFAULT_TENANT/logs/.ingestor.node.schema"), + "logs/.ingestor.node.schema" + ); + assert_eq!( + canonical_key("DEFAULT_TENANT/logs/.ingestor.node.stream.json"), + "logs/.ingestor.node.stream.json" + ); + for (key, tenant, kind) in [ + (".parseable.json", "", "document"), + ("logs/.stream.json", "", "stream"), + ("logs/.schema", "", "schema"), + ("tenant/logs/.stream.json", "tenant", "stream"), + ("tenant/logs/.schema", "tenant", "schema"), + ("tenant/.parseable.json", "tenant", "document"), + ("logs/.stream/.ingestor.node.stream.json", "", "stream"), + ( + "tenant/logs/.stream/.ingestor.node.schema", + "tenant", + "schema", + ), + ( + "tenant/logs/date=2026-01-01/ingestor.node.manifest.json", + "tenant", + "manifest", + ), + ( + "logs/date=2026-01-01/ingestor.node.manifest.json", + "", + "manifest", + ), + ( + "tenant/.users/owner/.dashboard-titles/key.json", + "tenant", + "document", + ), + (".parseable/ingestor.node.json", "", "document"), + ] { + assert_eq!( + document_scope(key), + Some((tenant.to_owned(), kind)), + "{key}" + ); + } + for key in [PAIR, BUSY, "logs/date=2026-01-01/data.parquet"] { + assert!(document_scope(key).is_none()); + } + } + + #[tokio::test] + async fn invalid_stream_and_missing_snapshot_manifest_fail_before_writes() { + let root = tempfile::tempdir().unwrap(); + let store = LocalFS::new(root.path().to_owned()); + assert!( + validate( + &store, + "logs/.stream/.stream.json", + &Bytes::from_static(b"{}") + ) + .await + .is_err() + ); + let mut config = crate::storage::ObjectStoreFormat::default(); + config + .snapshot + .manifest_list + .push(crate::catalog::snapshot::ManifestItem { + manifest_path: "logs/date=2026-01-01/ingestor.missing.manifest.json".into(), + time_lower_bound: chrono::Utc::now(), + time_upper_bound: chrono::Utc::now(), + events_ingested: 0, + ingestion_size: 0, + storage_size: 0, + }); + assert!( + validate( + &store, + "logs/.stream/.stream.json", + &Bytes::from(serde_json::to_vec(&config).unwrap()) + ) + .await + .is_err() + ); + assert!( + validate( + &store, + ".users/owner/filters/broken.json", + &Bytes::from_static(b"not-json") + ) + .await + .is_err() + ); + assert!(optional(&store, BUSY).await.unwrap().is_none()); + } + + #[tokio::test] + async fn minio_metadata_inventory_encoding_and_scale() -> anyhow::Result<()> { + let Ok(endpoint) = std::env::var("P_TEST_REVIEW_S3_URL") else { + eprintln!("SKIP MinIO inventory: P_TEST_REVIEW_S3_URL unset"); + return Ok(()); + }; + if std::env::var_os("P_MINIO_INVENTORY_CHILD").is_none() { + let output=std::process::Command::new(std::env::current_exe()?).args(["--exact","metastore::metastores::migration::tests::minio_metadata_inventory_encoding_and_scale","--nocapture"]).env("P_MINIO_INVENTORY_CHILD","1").output()?; + anyhow::ensure!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stdout).to_string() + + &String::from_utf8_lossy(&output.stderr) + ); + return Ok(()); + } + use crate::storage::ObjectStorageProvider; + use clap::Parser; + #[derive(clap::Parser)] + struct Args { + #[command(flatten)] + storage: crate::storage::S3Config, + } + let provider = Args::parse_from([ + "test", + "--endpoint-url", + &endpoint, + "--region", + "us-east-1", + "--bucket-name", + "review", + "--access-key-id", + "minioadmin", + "--secret-key", + "minioadmin", + ]) + .storage; + let store = provider.construct_client(); + let date = format!("date=2026-01-01-{}", uuid::Uuid::new_v4().simple()); + let manifest_key = format!("café/{date}/manifest.json"); + for key in [ + "café/.stream/.schema", + ".users/auth0|é/filters/one.json", + manifest_key.as_str(), + ] { + store + .put_object(RelativePath::new(key), Bytes::from_static(b"{}"), &None) + .await?; + } + let counter = || { + crate::metrics::STORAGE_REQUEST_RESPONSE_TIME + .with_label_values(&["s3", "LIST_DELIM", "200"]) + .get_sample_count() + }; + let n = counter(); + let before = store.metadata_inventory().await?; + let requests = counter() - n; + for i in 0..100 { + store + .put_object( + RelativePath::new(&format!("café/{date}/hour=00/minute={i}/data.parquet")), + Bytes::new(), + &None, + ) + .await?; + } + let n = counter(); + let after = store.metadata_inventory().await?; + assert_eq!( + requests, + counter() - n, + "parquet files must not add inventory requests" + ); + assert_eq!(before, after); + for expected in [ + "café/.stream/.schema", + ".users/auth0|é/filters/one.json", + manifest_key.as_str(), + ] { + assert!( + after.iter().any(|(key, _)| key == expected), + "inventory missing {expected}" + ); + } + for (key, _) in after { + store.get_object(RelativePath::new(&key), &None).await?; + } + Ok(()) + } + + #[tokio::test] + async fn dangling_import_can_be_exported_with_the_same_override() -> anyhow::Result<()> { + let Ok(url) = std::env::var("P_TEST_POSTGRES_URL") else { + eprintln!("SKIP dangling import/export: P_TEST_POSTGRES_URL unset"); + return Ok(()); + }; + let root = tempfile::tempdir()?; + let storage: Arc = Arc::new(LocalFS::new(root.path().to_owned())); + let admin = PostgresDocuments::new( + &PostgresOptions { + metastore_postgres_url: Some(url.clone()), + ..Default::default() + }, + "dangling".into(), + storage.clone(), + )?; + let database = format!("dangling_{}", uuid::Uuid::new_v4().simple()); + admin + .client() + .await? + .batch_execute(&format!("CREATE DATABASE {database}")) + .await?; + let result = async { + let mut url = url::Url::parse(&url)?; + url.set_path(&database); + let db = PostgresDocuments::new(&PostgresOptions { metastore_postgres_url: Some(url.to_string()), ..Default::default() }, "dangling".into(), storage.clone())?; + let body = Bytes::from_static(br#"{"version":"v3","snapshot":{"manifest_list":[{"manifest_path":"logs/date=2026-01-01/missing.manifest.json"}]}}"#); + storage.put_object(RelativePath::new("logs/.stream/.stream.json"), body.clone(), &None).await?; + let mut command = MigrationOptions { metadata_migration: Some(Action::Import), acknowledge_server_stopped: true, ..Default::default() }; + assert!(run(storage.clone(), &db, &command).await.is_err()); + command.allow_dangling_snapshot_references = true; + run(storage.clone(), &db, &command).await?; + command.metadata_migration = Some(Action::Export); + command.migration_dry_run = true; + command.allow_dangling_snapshot_references = false; + assert!(run(storage.clone(), &db, &command).await.is_err()); + command.allow_dangling_snapshot_references = true; + run(storage.clone(), &db, &command).await?; + command.migration_dry_run = false; + run(storage.clone(), &db, &command).await?; + assert_eq!(storage.get_object(RelativePath::new("logs/.stream/.stream.json"), &None).await?, body); + storage.check_migration_guard().await?; + anyhow::Ok(()) + }.await; + admin + .client() + .await? + .batch_execute(&format!("DROP DATABASE {database} WITH (FORCE)")) + .await?; + result + } + + #[tokio::test] + async fn migration_round_trip_postgres() -> anyhow::Result<()> { + let Ok(url) = std::env::var("P_TEST_POSTGRES_URL") else { + eprintln!("skip migration PostgreSQL test: P_TEST_POSTGRES_URL unset"); + return Ok(()); + }; + let root = tempfile::tempdir()?; + let store: Arc = + if let Ok(endpoint) = std::env::var("P_TEST_REVIEW_S3_URL") { + use crate::storage::ObjectStorageProvider; + use clap::Parser; + #[derive(clap::Parser)] + struct Args { + #[command(flatten)] + storage: crate::storage::S3Config, + } + Args::parse_from([ + "test", + "--endpoint-url", + &endpoint, + "--region", + "us-east-1", + "--bucket-name", + "migration-unit", + "--access-key-id", + "minioadmin", + "--secret-key", + "minioadmin", + ]) + .storage + .construct_client() + } else { + eprintln!("SKIP MinIO round trip: P_TEST_REVIEW_S3_URL unset; exercising LocalFS"); + Arc::new(LocalFS::new(root.path().to_owned())) + }; + let options = PostgresOptions { + metastore_postgres_url: Some(url.clone()), + metastore_postgres_statement_timeout_ms: 30000, + ..Default::default() + }; + let admin = PostgresDocuments::new(&options, "test".into(), store.clone())?; + let schema = format!("migration_{}", uuid::Uuid::new_v4().simple()); + admin + .client() + .await? + .batch_execute(&format!("CREATE DATABASE {schema}")) + .await?; + let result = async { + let mut url = url::Url::parse(&url)?; + url.set_path(&format!("/{schema}")); + let opts = PostgresOptions { + metastore_postgres_url: Some(url.to_string()), + metastore_postgres_pool_size: 1, + metastore_postgres_statement_timeout_ms: 30000, + ..Default::default() + }; + let db = PostgresDocuments::new(&opts, "test".into(), store.clone())?; + let plain = Bytes::from(serde_json::to_vec( + &crate::catalog::manifest::Manifest::default(), + )?); + let compressed = crate::catalog::manifest::encode_manifest( + &crate::catalog::manifest::Manifest::default(), + )?; + let mut documents = vec![ + ( + ".parseable.json".to_owned(), + Bytes::from_static(b"{ \"users\":[], \"roles\":{} }\n"), + ), + ( + "café/date=2026-01-01/ingestor.one.manifest.json".into(), + plain, + ), + ( + "tenant/logs/date=2026-01-01/ingestor.two.manifest.json".into(), + compressed, + ), + ]; + let mut config = crate::storage::ObjectStoreFormat::default(); + config + .snapshot + .manifest_list + .push(crate::catalog::snapshot::ManifestItem { + manifest_path: if store.store_url().scheme() == "file" { + store + .absolute_url(RelativePath::new(&documents[1].0)) + .to_string() + } else { + documents[1].0.clone() + }, + time_lower_bound: chrono::Utc::now(), + time_upper_bound: chrono::Utc::now(), + events_ingested: 0, + ingestion_size: 0, + storage_size: 0, + }); + documents.push(( + "café/.stream/.ingestor.one.stream.json".into(), + Bytes::from(serde_json::to_vec(&config)?), + )); + for prefix in [ + ".users/auth0|é/dashboards", + ".users/owner/filters", + ".alerts", + ".targets", + ".settings", + ".parseable", + ".correlations", + ".keystone", + ] { + for tenant in ["", "tenant/"] { + documents.push(( + format!("{tenant}{prefix}/fixture.json"), + Bytes::from_static(b"{\"unknown\":true} \n"), + )); + } + } + documents.push(( + "DEFAULT_TENANT/alias/.stream/.stream.json".into(), + Bytes::from_static(br#"{"version":"v3","snapshot":{"manifest_list":[{"manifest_path":"DEFAULT_TENANT/alias/date=2026-01-01/manifest.json"}]}}"#), + )); + documents.push(( + "DEFAULT_TENANT/alias/date=2026-01-01/manifest.json".into(), + crate::catalog::manifest::encode_manifest( + &crate::catalog::manifest::Manifest::default(), + )?, + )); + for (key, body) in &documents { + store + .put_object(RelativePath::new(key), body.clone(), &None) + .await?; + } + let mut command = MigrationOptions { + metadata_migration: Some(Action::Import), + acknowledge_server_stopped: true, + migration_dry_run: true, + migration_force: false, + ..Default::default() + }; + let pinned_options = PostgresOptions { + metastore_postgres_id: Some("wrong-id".into()), + ..opts.clone() + }; + let pinned = PostgresDocuments::new(&pinned_options, "test".into(), store.clone())?; + assert!( + run(store.clone(), &pinned, &command) + .await + .unwrap_err() + .to_string() + .contains("pinned backend") + ); + let lock = db.transfer_lock().await?; + assert!( + run(store.clone(), &db, &command) + .await + .unwrap_err() + .to_string() + .contains("another offline") + ); + command.abort = true; + assert!(run(store.clone(), &db, &command).await.unwrap_err().to_string().contains("another offline")); + command.abort = false; + drop(lock); + run(store.clone(), &db, &command).await?; + assert!(optional(store.as_ref(), PAIR).await?.is_none()); + command.migration_dry_run = false; + command.injected_failure = Some("marker"); + assert!(run(store.clone(), &db, &command).await.is_err()); + command.abort = true; + run(store.clone(), &db, &command).await?; + store.check_migration_guard().await?; + command.abort = false; + for step in [ + "marker", + "spool", + "lock-loss", + "import-before-commit", + "import-commit", + "paired", + ] { + command.injected_failure = Some(step); + assert!(run(store.clone(), &db, &command).await.is_err(), "{step}"); + if step == "spool" { + let id: String = db.client().await?.query_one("SELECT id FROM parseable_backend", &[]).await?.get(0); + let spool = std::env::current_dir()?.join(format!(".parseable-migration-spool-{id}")); + #[cfg(unix)] { + use std::os::unix::fs::PermissionsExt; + assert_eq!(std::fs::metadata(&spool)?.permissions().mode() & 0o777, 0o700); + } + std::fs::write(spool.join("stale"), b"private RBAC metadata")?; + command.abort = true; + run(store.clone(), &db, &command).await?; + assert!(!spool.exists(), "abort must clean stale private spools under the transfer lock"); + command.abort = false; + } + if step == "lock-loss" { + assert_eq!(db.client().await?.query_one("SELECT count(*) FROM parseable_documents", &[]).await?.get::<_, i64>(0), 0); + assert!(optional(store.as_ref(), PAIR).await?.is_none()); + command.abort = true; + run(store.clone(), &db, &command).await?; + command.abort = false; + } + if step == "import-commit" { + command.abort = true; + run(store.clone(), &db, &command).await?; + assert_eq!( + db.client() + .await? + .query_one("SELECT count(*) FROM parseable_documents", &[]) + .await? + .get::<_, i64>(0), + 0 + ); + command.abort = false; + } + } + command.injected_failure = None; + command.abort = true; + assert!( + run(store.clone(), &db, &command).await.is_err(), + "committed import cannot abort" + ); + command.abort = false; + run(store.clone(), &db, &command).await?; + assert!(store.check_migration_guard().await.is_err()); + assert!(run(store.clone(), &db, &command).await.is_err()); + command.metadata_migration = Some(Action::Verify); + run(store.clone(), &db, &command).await?; + let alias = "DEFAULT_TENANT/alias/date=2026-01-01/manifest.json"; + assert!(db.get_object(RelativePath::new(alias), &None).await.is_ok()); + assert_eq!(db.manifest_batch(&[alias.to_owned()], &None).await?.len(), 1); + command.metadata_migration = Some(Action::Export); + command.migration_dry_run = true; + run(store.clone(), &db, &command).await?; + let original = store + .get_object(RelativePath::new(&documents[0].0), &None) + .await?; + store + .delete_object(RelativePath::new(&documents[0].0), &None) + .await?; + assert!( + run(store.clone(), &db, &command).await.is_err(), + "deleted imported key must conflict even in dry run" + ); + command.overwrite_bucket_changes_since_import = true; + run(store.clone(), &db, &command).await?; + assert!( + optional(store.as_ref(), &documents[0].0).await?.is_none(), + "dry run must not restore key" + ); + store + .put_object(RelativePath::new(&documents[0].0), original, &None) + .await?; + let extra = ".users/owner/filters/after-import.json"; + store + .put_object(RelativePath::new(extra), Bytes::from_static(b"{}"), &None) + .await?; + command.overwrite_bucket_changes_since_import = false; + assert!(run(store.clone(), &db, &command).await.is_err()); + command.overwrite_bucket_changes_since_import = true; + run(store.clone(), &db, &command).await?; + assert!(optional(store.as_ref(), extra).await?.is_some()); + store.delete_object(RelativePath::new(extra), &None).await?; + command.overwrite_bucket_changes_since_import = false; + db.client() + .await? + .batch_execute( + "ALTER TABLE parseable_import_digests RENAME TO saved_import_digests", + ) + .await?; + // Equal bodies need no baseline, but any changed body does. + db.put_object( + RelativePath::new(&documents[0].0), + Bytes::from_static(b"{}"), + &None, + ) + .await?; + assert!(run(store.clone(), &db, &command).await.is_err()); + command.overwrite_bucket_changes_since_import = true; + run(store.clone(), &db, &command).await?; + db.put_object( + RelativePath::new(&documents[0].0), + documents[0].1.clone(), + &None, + ) + .await?; + db.client() + .await? + .batch_execute( + "ALTER TABLE saved_import_digests RENAME TO parseable_import_digests", + ) + .await?; + command.overwrite_bucket_changes_since_import = false; + let stream_key = documents + .iter() + .find(|(key, _)| key.ends_with("stream.json")) + .unwrap() + .0 + .clone(); + let mut dangling = config.clone(); + dangling.snapshot.manifest_list[0].manifest_path = + "café/date=2026-01-01/missing.manifest.json".into(); + db.put_object( + RelativePath::new(&stream_key), + Bytes::from(serde_json::to_vec(&dangling)?), + &None, + ) + .await?; + assert!(run(store.clone(), &db, &command).await.is_err()); + command.allow_dangling_snapshot_references = true; + run(store.clone(), &db, &command).await?; + db.put_object( + RelativePath::new(&stream_key), + documents + .iter() + .find(|(k, _)| k == &stream_key) + .unwrap() + .1 + .clone(), + &None, + ) + .await?; + command.allow_dangling_snapshot_references = false; + command.migration_dry_run = false; + // Simulate crash after committed import, before marker cleanup. + let id = optional(store.as_ref(), PAIR).await?.unwrap(); + store + .put_object( + RelativePath::new(BUSY), + Bytes::from(format!("Import:{}", std::str::from_utf8(&id)?)), + &None, + ) + .await?; + command.metadata_migration = Some(Action::Import); + run(store.clone(), &db, &command).await?; + // Invalid SQL documents must fail before a marker or destination write. + db.put_object( + RelativePath::new("broken/.stream/.deleting"), + Bytes::new(), + &None, + ) + .await?; + command.metadata_migration = Some(Action::Export); + assert!(run(store.clone(), &db, &command).await.is_err()); + assert!(optional(store.as_ref(), BUSY).await?.is_none()); + db.delete_object(RelativePath::new("broken/.stream/.deleting"), &None) + .await?; + // Bucket writers require explicit overwrite acknowledgement on export. + store + .put_object( + RelativePath::new(&documents[0].0), + Bytes::from_static(b"changed"), + &None, + ) + .await?; + command.metadata_migration = Some(Action::Export); + assert!(run(store.clone(), &db, &command).await.is_err()); + // Restore the imported bucket body; SQL edits must then export without force. + store + .put_object( + RelativePath::new(&documents[0].0), + documents[0].1.clone(), + &None, + ) + .await?; + documents[0].1 = Bytes::from_static(b"{\"users\":[],\"roles\":{},\"updated\":true}"); + db.put_object( + RelativePath::new(&documents[0].0), + documents[0].1.clone(), + &None, + ) + .await?; + let deleted_index = documents.iter().position(|(key, _)| key == "tenant/.correlations/fixture.json").unwrap(); + let deleted = documents.remove(deleted_index).0; + let deleted_tenant = document_scope(&deleted).unwrap().0; + db.delete_object( + RelativePath::new(canonical_key(&deleted)), + &Some(deleted_tenant), + ) + .await?; + // Exercise the actual destructive override, with multiple conflicts. + store.put_object(RelativePath::new(extra), Bytes::from_static(b"{}"), &None).await?; + store.put_object(RelativePath::new(&documents[0].0), Bytes::from_static(b"{\"external\":true}"), &None).await?; + command.migration_dry_run = true; + let conflicts = run(store.clone(), &db, &command).await.unwrap_err(); + assert!(conflicts.to_string().contains("2 bucket conflicts"), "{conflicts:#}"); + command.migration_dry_run = false; + command.overwrite_bucket_changes_since_import = true; + for step in [ + "marker", + "export-state", + "object-write", + "bucket-delete", + "pair-removed", + ] { + command.injected_failure = Some(step); + assert!(run(store.clone(), &db, &command).await.is_err(), "{step}"); + command.abort = true; + if step == "marker" { + run(store.clone(), &db, &command).await?; + let restarted = PostgresDocuments::new(&opts, "test".into(), store.clone())?; + restarted.check(&None).await?; + } else { + assert!( + run(store.clone(), &db, &command).await.is_err(), + "export after {step} cannot abort" + ); + } + command.abort = false; + } + command.injected_failure = None; + run(store.clone(), &db, &command).await?; + assert!(optional(store.as_ref(), PAIR).await?.is_none()); + store.check_migration_guard().await?; + assert!(optional(store.as_ref(), &deleted).await?.is_none()); + assert!(optional(store.as_ref(), extra).await?.is_none(), "override must remove bucket-only keys"); + for (key, body) in documents { + assert_eq!( + store + .get_object(RelativePath::new(canonical_key(&key)), &None) + .await?, + import_body(store.as_ref(), &key, &body)?, + "byte-for-byte round trip (canonical default locator): {key}" + ); + } + command.metadata_migration = Some(Action::Verify); + run(store.clone(), &db, &command).await?; + assert!( + db.check(&None).await.is_err(), + "exported database must not auto-pair" + ); + anyhow::Ok(()) + } + .await; + // The CI workflow runs this fixture repeatedly against one isolated + // bucket. Remove both source aliases and canonical export keys. + let cleanup = async { + for (key, _) in store.metadata_inventory().await? { + store.delete_object(RelativePath::new(&key), &None).await?; + } + anyhow::Ok(()) + } + .await; + let dropped = admin + .client() + .await? + .batch_execute(&format!("DROP DATABASE {schema} WITH (FORCE)")) + .await; + result?; + cleanup?; + dropped?; + Ok(()) + } +} diff --git a/src/metastore/metastores/mod.rs b/src/metastore/metastores/mod.rs index 19663b9bb..ee711d152 100644 --- a/src/metastore/metastores/mod.rs +++ b/src/metastore/metastores/mod.rs @@ -16,4 +16,15 @@ * */ +#[macro_use] +mod document_metastore; pub mod object_store_metastore; +pub(crate) mod postgres_documents; +pub mod postgres_metastore; + +pub mod migration; + +mod postgres_client; + +#[cfg(test)] +mod review_fix_tests; diff --git a/src/metastore/metastores/object_store_metastore.rs b/src/metastore/metastores/object_store_metastore.rs index c85566b0b..d74ff6a6c 100644 --- a/src/metastore/metastores/object_store_metastore.rs +++ b/src/metastore/metastores/object_store_metastore.rs @@ -1,1681 +1,182 @@ -/* - * Parseable Server (C) 2022 - 2025 Parseable, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - */ - -use std::{ - collections::{BTreeMap, HashMap, HashSet}, - sync::Arc, -}; - -use actix_web::http::StatusCode; -use arrow_schema::Schema; -use bytes::Bytes; -use chrono::{DateTime, Utc}; -use dashmap::DashMap; -use futures::{StreamExt, TryStreamExt}; -use relative_path::{RelativePath, RelativePathBuf}; -use tonic::async_trait; -use tracing::{info, warn}; -use ulid::Ulid; - -use crate::{ - alerts::{ - alert_structs::{AlertStateEntry, MTTRHistory}, - outbound_http_policy::AlertTargetPolicyConfig, - target::Target, - }, - catalog::{ - manifest::{Manifest, decode_manifest_blocking, encode_manifest}, - partition_path, - }, - handlers::http::{ - modal::{Metadata, NodeMetadata, NodeType}, - users::USERS_ROOT_DIR, - }, - metastore::{ - MetastoreError, - metastore_traits::{ - ConditionalMetadataError, MetadataVersion, MetadataWriteCondition, Metastore, - MetastoreObject, VersionedMetadata, - }, - }, - option::Mode, - parseable::{DEFAULT_TENANT, PARSEABLE}, - storage::{ - ALERTS_ROOT_DIRECTORY, ObjectStorage, ObjectStorageError, PARSEABLE_ROOT_DIRECTORY, - SETTINGS_ROOT_DIRECTORY, STREAM_METADATA_FILE_NAME, STREAM_ROOT_DIRECTORY, - TARGETS_ROOT_DIRECTORY, TOMBSTONE_ROOT_DIRECTORY, - object_storage::{ - alert_json_path, alert_state_json_path, filter_path, manifest_path, mttr_json_path, - outbound_http_policy_json_path, parseable_json_path, schema_path, stream_json_path, - to_bytes, - }, - }, - users::filters::{Filter, FilterType, SQL_FILTER_STREAM_NAME, migrate_v1_v2}, -}; - -/// Using PARSEABLE's storage as a metastore (default) -#[derive(Debug)] -pub struct ObjectStoreMetastore { - pub storage: Arc, -} - -impl ObjectStoreMetastore { - async fn delete_filter_path( - &self, - path: &RelativePathBuf, - tenant_id: &Option, - ) -> Result<(), MetastoreError> { - match self.storage.delete_object(path, tenant_id).await { - Ok(()) => Ok(()), - Err(error) if is_missing_optional_dir(&error) => Ok(()), - Err(error) => Err(error.into()), - } - } - - /// Enumerate only this owner's ID, never a filter belonging to another user. - async fn filter_aliases( - &self, - filter: &Filter, - tenant_id: &Option, - ) -> Result, MetastoreError> { - let owner = filter - .user_id - .as_deref() - .expect("stored filters have an owner"); - let id = filter - .filter_id - .as_deref() - .expect("stored filters have an ID"); - let root = filter_path(owner, "", "", tenant_id); - let mut copies = Vec::new(); - for stream in self.storage.list_dirs_relative(&root, tenant_id).await? { - let path = root.join(&stream).join(format!("{id}.json")); - let bytes = match self.storage.get_object(&path, tenant_id).await { - Ok(bytes) => bytes, - Err(error) if is_missing_optional_dir(&error) => continue, - Err(error) => return Err(error.into()), - }; - let copy: Filter = serde_json::from_slice(&bytes)?; - if copy.filter_id == filter.filter_id && copy.user_id == filter.user_id { - copies.push((path, copy)); - } - } - copies.sort_by(|a, b| a.0.cmp(&b.0)); - Ok(copies) - } - - async fn load_tenant_filters( - &self, - tenant_id: &Option, - ) -> Result, MetastoreError> { - let users_dir = - RelativePathBuf::from_iter([tenant_id.as_deref().unwrap_or(""), USERS_ROOT_DIR]); - let mut grouped: BTreeMap<(String, String), Vec<(RelativePathBuf, Filter)>> = - BTreeMap::new(); - for user in self - .storage - .list_dirs_relative(&users_dir, tenant_id) - .await? - { - let root = users_dir.join(&user).join("filters"); - for stream in self.storage.list_dirs_relative(&root, tenant_id).await? { - let directory = root.join(&stream); - let objects = self - .storage - .get_objects( - Some(&directory), - Box::new(|name| name.ends_with(".json")), - tenant_id, - ) - .await?; - for bytes in objects { - let mut value: serde_json::Value = serde_json::from_slice(&bytes)?; - let is_v1 = value.get("version").and_then(|v| v.as_str()) == Some("v1"); - if is_v1 { - value = migrate_v1_v2(value, tenant_id); - } - let Ok(mut filter) = serde_json::from_value::(value) else { - continue; - }; - // The discovered tenant directory is authoritative, including - // default-tenant records whose payload predates tenant metadata. - filter.tenant_id = tenant_id.clone().filter(|tenant| !tenant.is_empty()); - let (Some(owner), Some(id)) = (&filter.user_id, &filter.filter_id) else { - continue; - }; - // Stored identity fields must remain single path components. - if !safe_filter_component(owner) - || !safe_filter_component(id) - || !safe_filter_component(&filter.stream_name) - || filter.stream_name != stream - || (!is_v1 && owner != &user) - { - warn!( - "Ignoring saved filter whose identity does not match its storage directory" - ); - continue; - } - let source = directory.join(format!("{id}.json")); - // get_objects returns bytes without filenames. Verify the inferred - // filename before any cleanup; never delete an unrelated object. - match self.storage.get_object(&source, tenant_id).await { - Ok(stored) if stored == bytes => {} - Ok(_) => continue, - Err(error) if is_missing_optional_dir(&error) => continue, - Err(error) => return Err(error.into()), - } - if is_v1 { - // Persist before removing the original so interrupted migration is safe. - self.put_filter(&filter, tenant_id).await?; - if source.as_str() != filter.get_object_path() { - self.delete_filter_path(&source, tenant_id).await?; - } - } - grouped - .entry((owner.clone(), id.clone())) - .or_default() - .push(( - if is_v1 { - RelativePathBuf::from(filter.get_object_path()) - } else { - source - }, - filter, - )); - } - } - } - let mut filters = Vec::new(); - for (_, mut copies) in grouped { - // A v1 migration may write a hashed-owner directory that is visited - // later in this same scan. It is still one physical object. - copies.sort_by(|a, b| a.0.cmp(&b.0)); - copies.dedup_by(|a, b| a.0 == b.0); - copies.sort_by(|a, b| { - let canonical = |filter: &Filter| { - filter.query.filter_type == FilterType::SQL - && filter.stream_name == SQL_FILTER_STREAM_NAME - }; - canonical(&b.1).cmp(&canonical(&a.1)).then(a.0.cmp(&b.0)) - }); - let (canonical_path, filter) = &copies[0]; - if filter.query.filter_type == FilterType::SQL - && filter.stream_name == SQL_FILTER_STREAM_NAME - { - for (path, alias) in copies.iter().skip(1) { - if path != canonical_path && alias.query.filter_type == FilterType::SQL { - // The canonical object is already durable. Retain it in memory - // even if cleanup is temporarily unavailable; retry on restart. - if let Err(error) = self.delete_filter_path(path, tenant_id).await { - warn!("Failed to clean saved SQL filter alias {path}: {error}"); - } - } else if alias.query.filter_type != FilterType::SQL { - warn!( - "Retaining non-SQL saved filter copy {path} with the same owner {:?} and ID {:?} as canonical SQL filter {canonical_path}", - filter.user_id, filter.filter_id - ); - } - } - } - if copies.len() > 1 - && !(filter.query.filter_type == FilterType::SQL - && filter.stream_name == SQL_FILTER_STREAM_NAME) - { - warn!( - "Multiple saved filter copies for owner {:?}, ID {:?}; using deterministic path {canonical_path}", - filter.user_id, filter.filter_id - ); - } - filters.push(filter.clone()); - } - Ok(filters) - } -} - -fn stored_filter_path(filter: &Filter, tenant_id: &Option) -> RelativePathBuf { - filter_path( - filter - .user_id - .as_deref() - .expect("stored filters have an owner"), - &filter.stream_name, - &format!( - "{}.json", - filter - .filter_id - .as_deref() - .expect("stored filters have an ID") - ), - tenant_id, - ) -} - -fn safe_filter_component(component: &str) -> bool { - !component.is_empty() - && component != "." - && component != ".." - && !component.contains(['/', '\\']) -} - -pub fn is_missing_optional_dir(err: &ObjectStorageError) -> bool { - match err { - ObjectStorageError::NoSuchKey(_) => true, - ObjectStorageError::IoError(err) => err.kind() == std::io::ErrorKind::NotFound, - _ => false, - } -} - -#[async_trait] -impl Metastore for ObjectStoreMetastore { - /// Since Parseable already starts with a connection to an object store, no need to implement this - async fn initiate_connection(&self) -> Result<(), MetastoreError> { - unimplemented!() - } - - /// Fetch mutiple .json objects - async fn get_objects( - &self, - parent_path: &str, - tenant_id: &Option, - ) -> Result, MetastoreError> { - Ok(self - .storage - .get_objects( - Some(&RelativePathBuf::from(parent_path)), - Box::new(|file_name| file_name.ends_with(".json")), - tenant_id, - ) - .await?) - } - - /// This function fetches all the keystones from the underlying object store - async fn get_keystones(&self) -> Result>, MetastoreError> { - let base_paths = PARSEABLE.list_tenants().unwrap_or_else(|| vec!["".into()]); - let mut keystones = HashMap::new(); - for mut tenant in base_paths { - let keystone_path = RelativePathBuf::from_iter([&tenant, ".keystone"]); - let objs = self - .storage - .get_objects( - Some(&keystone_path), - Box::new(|file_name| { - file_name.ends_with(".json") && !file_name.starts_with("conv_") - }), - &Some(tenant.clone()), - ) - .await?; - if tenant.is_empty() { - tenant.clone_from(&DEFAULT_TENANT.to_string()); - } - keystones.insert(tenant, objs); - } - - Ok(keystones) - } - - /// This function puts a keystone in the object store at the given path - async fn put_keystone( - &self, - obj: &dyn MetastoreObject, - tenant_id: &Option, - ) -> Result<(), MetastoreError> { - let id = obj.get_object_id(); - let path = if let Some(tenant_id) = tenant_id.as_ref() { - RelativePathBuf::from_iter([tenant_id, ".keystone", &format!("{id}.json")]) - } else { - RelativePathBuf::from_iter([".keystone", &format!("{id}.json")]) - }; - Ok(self - .storage - .put_object(&path, to_bytes(obj), tenant_id) - .await?) - } - - /// Delete a keystone - async fn delete_keystone( - &self, - obj: &dyn MetastoreObject, - tenant_id: &Option, - ) -> Result<(), MetastoreError> { - let id = obj.get_object_id(); - let path = if let Some(tenant_id) = tenant_id.as_ref() { - RelativePathBuf::from_iter([tenant_id, ".keystone", &format!("{id}.json")]) - } else { - RelativePathBuf::from_iter([".keystone", &format!("{id}.json")]) - }; - Ok(self.storage.delete_object(&path, tenant_id).await?) - } - - /// This function fetches all the conversations from the underlying object store - async fn get_conversations(&self) -> Result>, MetastoreError> { - let base_paths = PARSEABLE.list_tenants().unwrap_or_else(|| vec!["".into()]); - let mut conversations = HashMap::new(); - for mut tenant in base_paths { - let conv_path = RelativePathBuf::from_iter([&tenant, ".keystone"]); - let objs = self - .storage - .get_objects( - Some(&conv_path), - Box::new(|file_name| { - file_name.ends_with(".json") && file_name.starts_with("conv_") - }), - &Some(tenant.clone()), - ) - .await?; - if tenant.is_empty() { - tenant.clone_from(&DEFAULT_TENANT.to_string()); - } - conversations.insert(tenant, objs); - } - - Ok(conversations) - } - - /// This function puts a conversation in the object store at the given path - async fn put_conversation( - &self, - obj: &dyn MetastoreObject, - tenant_id: &Option, - ) -> Result<(), MetastoreError> { - let id = obj.get_object_id(); - let path = if let Some(tenant_id) = tenant_id.as_ref() { - RelativePathBuf::from_iter([tenant_id, ".keystone", &format!("conv_{id}.json")]) - } else { - RelativePathBuf::from_iter([".keystone", &format!("conv_{id}.json")]) - }; - Ok(self - .storage - .put_object(&path, to_bytes(obj), tenant_id) - .await?) - } - - /// Delete a conversation - async fn delete_conversation( - &self, - obj: &dyn MetastoreObject, - tenant_id: &Option, - ) -> Result<(), MetastoreError> { - let id = obj.get_object_id(); - let path = if let Some(tenant_id) = tenant_id.as_ref() { - RelativePathBuf::from_iter([tenant_id, ".keystone", &format!("conv_{id}.json")]) - } else { - RelativePathBuf::from_iter([".keystone", &format!("conv_{id}.json")]) - }; - Ok(self.storage.delete_object(&path, tenant_id).await?) - } - - /// This function fetches all the alerts from the underlying object store - async fn get_alerts(&self) -> Result>, MetastoreError> { - let base_paths = PARSEABLE.list_tenants().unwrap_or_else(|| vec!["".into()]); - let mut all_alerts = HashMap::new(); - for mut tenant in base_paths { - let alerts_path = RelativePathBuf::from_iter([&tenant, ALERTS_ROOT_DIRECTORY]); - let alerts = match self - .storage - .get_objects( - Some(&alerts_path), - Box::new(|file_name| { - !file_name.starts_with("alert_state_") && file_name.ends_with(".json") - }), - &Some(tenant.clone()), - ) - .await - { - Ok(alerts) => alerts, - Err(err) if is_missing_optional_dir(&err) => Vec::new(), - Err(err) => return Err(MetastoreError::ObjectStorageError(err)), - }; - if tenant.is_empty() { - tenant.clone_from(&DEFAULT_TENANT.to_string()); - } - all_alerts.insert(tenant, alerts); - } - Ok(all_alerts) - } - - /// This function puts an alert in the object store at the given path - async fn put_alert( - &self, - obj: &dyn MetastoreObject, - tenant_id: &Option, - ) -> Result<(), MetastoreError> { - let id = Ulid::from_string(&obj.get_object_id()).map_err(|e| MetastoreError::Error { - status_code: StatusCode::BAD_REQUEST, - message: e.to_string(), - flow: "put_alert".into(), - })?; - let path = alert_json_path(id, tenant_id); - - Ok(self - .storage - .put_object(&path, to_bytes(obj), tenant_id) - .await?) - } - - /// Delete an alert - async fn delete_alert( - &self, - obj: &dyn MetastoreObject, - tenant_id: &Option, - ) -> Result<(), MetastoreError> { - let id = Ulid::from_string(&obj.get_object_id()).map_err(|e| MetastoreError::Error { - status_code: StatusCode::BAD_REQUEST, - message: e.to_string(), - flow: "delete_alert".into(), - })?; - let path = alert_json_path(id, tenant_id); - match self.storage.delete_object(&path, tenant_id).await { - Ok(()) => Ok(()), - Err(err) if is_missing_optional_dir(&err) => Ok(()), - Err(err) => Err(MetastoreError::ObjectStorageError(err)), - } - } - - /// alerts state - async fn get_alert_states( - &self, - tenant_id: &Option, - ) -> Result, MetastoreError> { - let tenant = tenant_id.as_deref().unwrap_or(DEFAULT_TENANT); - let base_path = RelativePathBuf::from_iter([tenant, ALERTS_ROOT_DIRECTORY]); - let alert_state_bytes = self - .storage - .get_objects( - Some(&base_path), - Box::new(|file_name| { - file_name.starts_with("alert_state_") && file_name.ends_with(".json") - }), - tenant_id, - ) - .await?; - - let mut alert_states = Vec::new(); - for bytes in alert_state_bytes { - if let Ok(entry) = serde_json::from_slice::(&bytes) { - alert_states.push(entry); - } - } - - Ok(alert_states) - } - - async fn get_alert_state_entry( - &self, - alert_id: &Ulid, - tenant_id: &Option, - ) -> Result, MetastoreError> { - let path = alert_state_json_path(*alert_id, tenant_id); - match self.storage.get_object(&path, tenant_id).await { - Ok(bytes) => { - if let Ok(entry) = serde_json::from_slice::(&bytes) { - Ok(Some(entry)) - } else { - Ok(None) - } - } - Err(ObjectStorageError::NoSuchKey(_)) => Ok(None), - Err(e) => Err(MetastoreError::ObjectStorageError(e)), - } - } - - async fn put_alert_state( - &self, - obj: &dyn MetastoreObject, - tenant_id: &Option, - ) -> Result<(), MetastoreError> { - let id = Ulid::from_string(&obj.get_object_id()).map_err(|e| MetastoreError::Error { - status_code: StatusCode::BAD_REQUEST, - message: e.to_string(), - flow: "put_alert_state".into(), - })?; - let path = alert_state_json_path(id, tenant_id); - - // Parse the new state entry from the MetastoreObject - let new_state_entry: AlertStateEntry = serde_json::from_slice(&to_bytes(obj))?; - let new_state = new_state_entry - .current_state() - .ok_or_else(|| MetastoreError::InvalidJsonStructure { - expected: "AlertStateEntry with at least one state".to_string(), - found: "AlertStateEntry with empty states".to_string(), - })? - .state; - - // Try to read and parse existing file - if let Ok(existing_bytes) = self.storage.get_object(&path, tenant_id).await { - // File exists - try to parse and update - if let Ok(mut existing_entry) = - serde_json::from_slice::(&existing_bytes) - { - // Update the state and only save if it actually changed - let state_changed = existing_entry.update_state(new_state); - - if state_changed { - let updated_bytes = serde_json::to_vec(&existing_entry) - .map_err(MetastoreError::JsonParseError)?; - - self.storage - .put_object(&path, updated_bytes.into(), tenant_id) - .await?; - } - return Ok(()); - } - } - - // Create and save new entry (either file didn't exist or parsing failed) - let new_entry = AlertStateEntry::new(id, new_state, tenant_id.clone()); - let new_bytes = serde_json::to_vec(&new_entry).map_err(MetastoreError::JsonParseError)?; - - self.storage - .put_object(&path, new_bytes.into(), tenant_id) - .await?; - - Ok(()) - } - - /// Delete an alert state file - async fn delete_alert_state( - &self, - obj: &dyn MetastoreObject, - tenant_id: &Option, - ) -> Result<(), MetastoreError> { - let id = Ulid::from_string(&obj.get_object_id()).map_err(|e| MetastoreError::Error { - status_code: StatusCode::BAD_REQUEST, - message: e.to_string(), - flow: "delete_alert_state".into(), - })?; - let path = alert_state_json_path(id, tenant_id); - match self.storage.delete_object(&path, tenant_id).await { - Ok(()) => Ok(()), - Err(err) if is_missing_optional_dir(&err) => Ok(()), - Err(err) => Err(MetastoreError::ObjectStorageError(err)), - } - } - - /// Get MTTR history from storage - async fn get_mttr_history( - &self, - tenant_id: &Option, - ) -> Result, MetastoreError> { - let path = mttr_json_path(tenant_id); - match self.storage.get_object(&path, tenant_id).await { - Ok(bytes) => { - if let Ok(mut history) = serde_json::from_slice::(&bytes) { - if let Some(tenant) = tenant_id.as_ref() { - history.tenant_id = Some(tenant.clone()); - } - Ok(Some(history)) - } else { - Ok(None) - } - } - Err(ObjectStorageError::NoSuchKey(_)) => Ok(None), - Err(e) => Err(MetastoreError::ObjectStorageError(e)), - } - } - - /// Put MTTR history to storage - async fn put_mttr_history( - &self, - obj: &dyn MetastoreObject, - tenant_id: &Option, - ) -> Result<(), MetastoreError> { - let path = RelativePathBuf::from(obj.get_object_path()); - Ok(self - .storage - .put_object(&path, to_bytes(obj), tenant_id) - .await?) - } - - /// This function fetches all the llmconfigs from the underlying object store - async fn get_llmconfigs(&self) -> Result>, MetastoreError> { - let base_paths = PARSEABLE.list_tenants().unwrap_or_else(|| vec!["".into()]); - let mut all_configs = HashMap::new(); - for mut tenant in base_paths { - let base_path = - RelativePathBuf::from_iter([&tenant, SETTINGS_ROOT_DIRECTORY, "llmconfigs"]); - let conf_bytes = self - .storage - .get_objects( - Some(&base_path), - Box::new(|file_name| file_name.ends_with(".json")), - &Some(tenant.clone()), - ) - .await?; - if tenant.is_empty() { - tenant.clone_from(&DEFAULT_TENANT.to_string()); - } - all_configs.insert(tenant, conf_bytes); - } - Ok(all_configs) - } - - /// This function puts an llmconfig in the object store at the given path - async fn put_llmconfig( - &self, - obj: &dyn MetastoreObject, - tenant_id: &Option, - ) -> Result<(), MetastoreError> { - let path = obj.get_object_path(); - - Ok(self - .storage - .put_object(&RelativePathBuf::from(path), to_bytes(obj), tenant_id) - .await?) - } - - /// Delete an llmconfig - async fn delete_llmconfig( - &self, - obj: &dyn MetastoreObject, - tenant_id: &Option, - ) -> Result<(), MetastoreError> { - let path = obj.get_object_path(); - Ok(self - .storage - .delete_object(&RelativePathBuf::from(path), tenant_id) - .await?) - } - - async fn get_dashboard_versioned( - &self, - path: &str, - tenant_id: &Option, - ) -> Result, ConditionalMetadataError> { - self.storage - .get_metadata_versioned(RelativePath::new(path), tenant_id) - .await - } - - async fn put_dashboard_conditional( - &self, - path: &str, - bytes: Bytes, - condition: MetadataWriteCondition, - tenant_id: &Option, - ) -> Result { - self.storage - .put_metadata_conditional(RelativePath::new(path), bytes, condition, tenant_id) - .await - } - - async fn delete_dashboard_conditional( - &self, - path: &str, - version: MetadataVersion, - tenant_id: &Option, - ) -> Result<(), ConditionalMetadataError> { - self.storage - .delete_metadata_conditional(RelativePath::new(path), version, tenant_id) - .await - } - - /// Fetch all dashboards; hidden service indexes are not dashboard owners. - async fn get_dashboards(&self) -> Result>, MetastoreError> { - let mut dashboards = HashMap::new(); - let tenants = PARSEABLE.list_tenants().unwrap_or_else(|| vec!["".into()]); - for tenant in tenants { - let tenant_id = if tenant.is_empty() { - None - } else { - Some(tenant.clone()) - }; - let bytes = self.get_tenant_dashboards(&tenant_id).await?; - dashboards.insert( - tenant_id.unwrap_or_else(|| DEFAULT_TENANT.to_string()), - bytes, - ); - } - Ok(dashboards) - } - - async fn get_tenant_dashboards( - &self, - tenant_id: &Option, - ) -> Result, MetastoreError> { - let users = - RelativePathBuf::from_iter([tenant_id.as_deref().unwrap_or(""), USERS_ROOT_DIR]); - let mut dashboards = Vec::new(); - for owner in self.storage.list_dirs_relative(&users, tenant_id).await? { - if owner.starts_with('.') { - continue; - } - let path = users.join(owner).join("dashboards"); - match self - .storage - .get_objects( - Some(&path), - Box::new(|name| name.ends_with(".json")), - tenant_id, - ) - .await - { - Ok(bytes) => dashboards.extend(bytes), - Err(error) if is_missing_optional_dir(&error) => {} - Err(error) => return Err(error.into()), - } - } - Ok(dashboards) - } - - /// Save a dashboard - async fn put_dashboard( - &self, - obj: &dyn MetastoreObject, - tenant_id: &Option, - ) -> Result<(), MetastoreError> { - // we need the path to store in obj store - let path = obj.get_object_path(); - - Ok(self - .storage - .put_object(&RelativePathBuf::from(path), to_bytes(obj), tenant_id) - .await?) - } - - /// Delete a dashboard - async fn delete_dashboard( - &self, - obj: &dyn MetastoreObject, - tenant_id: &Option, - ) -> Result<(), MetastoreError> { - let path = obj.get_object_path(); - Ok(self - .storage - .delete_object(&RelativePathBuf::from(path), tenant_id) - .await?) - } - - /// Fetch all chats - async fn get_chats( - &self, - ) -> Result>>, MetastoreError> { - let base_paths = PARSEABLE.list_tenants().unwrap_or_else(|| vec!["".into()]); - let all_chats = DashMap::new(); - for tenant in base_paths { - let all_user_chats = DashMap::new(); - let tenant_id = if tenant.is_empty() { - None - } else { - Some(tenant.clone()) - }; - let users_dir = RelativePathBuf::from_iter([&tenant, USERS_ROOT_DIR]); - for user in self - .storage - .list_dirs_relative(&users_dir, &tenant_id) - .await? - { - if user.starts_with(".") { - continue; - } - let mut chats = Vec::new(); - let chats_path = users_dir.join(&user).join("chats"); - let user_chats = self - .storage - .get_objects( - Some(&chats_path), - Box::new(|file_name| file_name.ends_with(".json")), - &tenant_id, - ) - .await?; - for chat in user_chats { - chats.push(chat); - } - - all_user_chats.insert(user, chats); - } - all_chats.insert( - tenant_id.as_deref().unwrap_or(DEFAULT_TENANT).into(), - all_user_chats, - ); - } - Ok(all_chats) - } - - /// Save a chat - async fn put_chat( - &self, - obj: &dyn MetastoreObject, - tenant_id: &Option, - ) -> Result<(), MetastoreError> { - // we need the path to store in obj store - let path = obj.get_object_path(); - - Ok(self - .storage - .put_object(&RelativePathBuf::from(path), to_bytes(obj), tenant_id) - .await?) - } - - /// Delete a chat - async fn delete_chat( - &self, - obj: &dyn MetastoreObject, - tenant_id: &Option, - ) -> Result<(), MetastoreError> { - let path = obj.get_object_path(); - Ok(self - .storage - .delete_object(&RelativePathBuf::from(path), tenant_id) - .await?) - } - - // Reconcile duplicate SQL paths before exposing filters to the cache. - async fn get_filters(&self) -> Result>, MetastoreError> { - let mut all = HashMap::new(); - let tenants = PARSEABLE.list_tenants().unwrap_or_else(|| vec!["".into()]); - for tenant in tenants { - let tenant_id = Some(tenant.clone()); - let filters = self.load_tenant_filters(&tenant_id).await?; - all.insert( - if tenant.is_empty() { - DEFAULT_TENANT.to_owned() - } else { - tenant - }, - filters, - ); - } - Ok(all) - } - - /// Save a filter - async fn put_filter( - &self, - obj: &dyn MetastoreObject, - tenant_id: &Option, - ) -> Result<(), MetastoreError> { - // we need the path to store in obj store - let path = obj.get_object_path(); - - Ok(self - .storage - .put_object(&RelativePathBuf::from(path), to_bytes(obj), tenant_id) - .await?) - } - - async fn reconcile_filter( - &self, - filter: &Filter, - tenant_id: &Option, - ) -> Result<(), MetastoreError> { - let canonical = stored_filter_path(filter, tenant_id); - for (path, _) in self.filter_aliases(filter, tenant_id).await? { - if path != canonical { - self.delete_filter_path(&path, tenant_id).await?; - } - } - Ok(()) - } - - /// Delete aliases first. If any cleanup fails, the canonical copy remains - /// available for a retry instead of allowing an old alias to reappear. - async fn delete_filter( - &self, - filter: &Filter, - tenant_id: &Option, - ) -> Result<(), MetastoreError> { - self.reconcile_filter(filter, tenant_id).await?; - self.delete_filter_path(&stored_filter_path(filter, tenant_id), tenant_id) - .await - } - - /// Fetch an `ObjectStoreFormat` file - /// - /// If `get_base` is true, get the one at the base of the stream directory else depends on Mode - async fn get_stream_json( - &self, - stream_name: &str, - get_base: bool, - tenant_id: &Option, - _is_migration: bool, - ) -> Result { - let tenant = tenant_id.as_deref().unwrap_or(""); - let path = if get_base { - RelativePathBuf::from_iter([ - tenant, - stream_name, - STREAM_ROOT_DIRECTORY, - STREAM_METADATA_FILE_NAME, - ]) - } else { - stream_json_path(stream_name, tenant_id) - }; - Ok(self.storage.get_object(&path, tenant_id).await?) - } - - /// Fetch all `ObjectStoreFormat` present in a stream folder - async fn get_all_stream_jsons( - &self, - stream_name: &str, - mode: Option, - tenant_id: &Option, - _is_migration: bool, - ) -> Result, MetastoreError> { - let root = tenant_id.as_deref().unwrap_or(""); - let path = RelativePathBuf::from_iter([root, stream_name, STREAM_ROOT_DIRECTORY]); - if let Some(mode) = mode { - if mode.eq(&Mode::Ingest) { - Ok(self - .storage - .get_objects( - Some(&path), - Box::new(|file_name| { - file_name.starts_with(".ingestor") && file_name.ends_with("stream.json") - }), - tenant_id, - ) - .await?) - } else { - return Err(MetastoreError::Error { - status_code: StatusCode::BAD_REQUEST, - message: "Incorrect server mode passed as input. Only `Ingest` is allowed." - .into(), - flow: "get_all_streams with mode".into(), - }); - } - } else { - Ok(self - .storage - .get_objects( - Some(&path), - Box::new(|file_name| file_name.ends_with("stream.json")), - tenant_id, - ) - .await?) - } - } - - /// Save an `ObjectStoreFormat` file - async fn put_stream_json( - &self, - obj: &dyn MetastoreObject, - stream_name: &str, - tenant_id: &Option, - ) -> Result<(), MetastoreError> { - let path = stream_json_path(stream_name, tenant_id); - - Ok(self - .storage - .put_object(&path, to_bytes(obj), tenant_id) - .await?) - } - - /// Fetch manifest files for the given date keys in parallel. - /// - /// Per-date work runs concurrently via `futures::future::try_join_all` - /// so a slow LIST on one date does not block the others. Each date's - /// timing is logged at info level so any pathologically slow dates - /// (S3 retries, oversized prefixes) are visible without needing extra - /// span instrumentation. - async fn get_manifest_files_for_dates( - &self, - stream_name: &str, - tenant_id: &Option, - dates: &[String], - ) -> Result>, MetastoreError> { - let inventory_limit = Arc::new(tokio::sync::Semaphore::new(16)); - let date_futures = dates.iter().map(|date| { - let stream = stream_name.to_string(); - let tenant = tenant_id.clone(); - let date = date.clone(); - let inventory_limit = inventory_limit.clone(); - async move { - let t_start = std::time::Instant::now(); - let date_path = if let Some(tenant) = tenant.as_ref() { - object_store::path::Path::from(format!("{}/{}/{}", tenant, stream, date)) - } else { - object_store::path::Path::from(format!("{}/{}", stream, date)) - }; - - let t_list = std::time::Instant::now(); - let list_permit = inventory_limit - .clone() - .acquire_owned() - .await - .expect("inventory semaphore remains open"); - let list_result = self.storage.list_with_delimiter(Some(date_path)).await; - drop(list_permit); - let resp = match list_result { - Ok(r) => r, - Err(ObjectStorageError::NoSuchKey(_)) => { - info!( - stream = %stream, - tenant = ?tenant, - date = %date, - list_ms = t_list.elapsed().as_millis() as u64, - "manifest fetch: date prefix not found" - ); - return Ok::<(String, Vec), MetastoreError>((date, Vec::new())); - } - Err(e) => return Err(e.into()), - }; - let list_ms = t_list.elapsed().as_millis() as u64; - - let manifest_paths: Vec = resp - .objects - .iter() - .filter(|name| name.location.filename().unwrap().ends_with("manifest.json")) - .map(|name| name.location.to_string()) - .collect(); - let manifest_count = manifest_paths.len(); - - let t_gets = std::time::Instant::now(); - let manifests = futures::stream::iter(manifest_paths) - .map(|path| { - let tenant = tenant.clone(); - let inventory_limit = inventory_limit.clone(); - async move { - let _permit = inventory_limit - .acquire_owned() - .await - .expect("inventory semaphore remains open"); - let bytes = self - .storage - .get_object(&RelativePathBuf::from(path), &tenant) - .await?; - Ok::(decode_manifest_blocking(bytes).await?) - } - }) - .buffer_unordered(16) - .try_collect::>() - .await?; - let gets_ms = t_gets.elapsed().as_millis() as u64; - let total_ms = t_start.elapsed().as_millis() as u64; - - if total_ms > 100 { - info!( - stream = %stream, - tenant = ?tenant, - date = %date, - list_ms, - gets_ms, - total_ms, - manifest_count, - "manifest fetch per-date timing" - ); - } - Ok((date, manifests)) - } - }); - - let results: Vec<(String, Vec)> = - futures::future::try_join_all(date_futures).await?; - - let mut result_file_list: BTreeMap> = BTreeMap::new(); - for (date, manifests) in results { - if !manifests.is_empty() { - result_file_list.insert(date, manifests); - } - } - - Ok(result_file_list) - } - - /// Fetch a specific `Manifest` file - async fn get_manifest( - &self, - stream_name: &str, - lower_bound: DateTime, - upper_bound: DateTime, - manifest_url: Option, - tenant_id: &Option, - ) -> Result, MetastoreError> { - let path = match manifest_url { - Some(url) => RelativePathBuf::from(url), - None => { - let path = partition_path(stream_name, lower_bound, upper_bound, tenant_id); - manifest_path(path.as_str()) - } - }; - match self.storage.get_object(&path, tenant_id).await { - Ok(bytes) => { - let manifest = decode_manifest_blocking(bytes).await?; - Ok(Some(manifest)) - } - Err(ObjectStorageError::NoSuchKey(_)) => Ok(None), - Err(err) => Err(MetastoreError::ObjectStorageError(err)), - } - } - - /// Get the path for a specific `Manifest` file - async fn get_manifest_path( - &self, - stream_name: &str, - lower_bound: DateTime, - upper_bound: DateTime, - tenant_id: &Option, - ) -> Result { - let path = partition_path(stream_name, lower_bound, upper_bound, tenant_id); - Ok(self - .storage - .absolute_url(&manifest_path(path.as_str())) - .to_string()) - } - - async fn put_manifest( - &self, - manifest: &Manifest, - stream_name: &str, - lower_bound: DateTime, - upper_bound: DateTime, - tenant_id: &Option, - ) -> Result<(), MetastoreError> { - let manifest_file_name = manifest_path("").to_string(); - let path = partition_path(stream_name, lower_bound, upper_bound, tenant_id) - .join(&manifest_file_name); - - Ok(self - .storage - .put_object(&path, encode_manifest(manifest)?, tenant_id) - .await?) - } - - async fn delete_manifest( - &self, - stream_name: &str, - lower_bound: DateTime, - upper_bound: DateTime, - tenant_id: &Option, - ) -> Result<(), MetastoreError> { - let manifest_file_name = manifest_path("").to_string(); - let path = partition_path(stream_name, lower_bound, upper_bound, tenant_id) - .join(&manifest_file_name); - Ok(self.storage.delete_object(&path, tenant_id).await?) - } - - /// targets - async fn get_targets(&self) -> Result>, MetastoreError> { - let base_paths = PARSEABLE.list_tenants().unwrap_or_else(|| vec!["".into()]); - let mut all_targets = HashMap::new(); - for mut tenant in base_paths { - let targets_path = RelativePathBuf::from_iter([ - &tenant, - SETTINGS_ROOT_DIRECTORY, - TARGETS_ROOT_DIRECTORY, - ]); - let target_bytes = match self - .storage - .get_objects( - Some(&targets_path), - Box::new(|file_name| file_name.ends_with(".json")), - &Some(tenant.clone()), - ) - .await - { - Ok(targets) => targets, - Err(err) if is_missing_optional_dir(&err) => Vec::new(), - Err(err) => return Err(MetastoreError::ObjectStorageError(err)), - }; - let targets = target_bytes - .iter() - .filter_map(|bytes| { - serde_json::from_slice(bytes) - .inspect_err(|err| warn!("Expected compatible json, error = {err}")) - .ok() - }) - .collect(); - if tenant.is_empty() { - tenant.clone_from(&DEFAULT_TENANT.to_string()); - } - all_targets.insert(tenant, targets); - } - Ok(all_targets) - } - - async fn put_target( - &self, - obj: &dyn MetastoreObject, - tenant_id: &Option, - ) -> Result<(), MetastoreError> { - // we need the path to store in obj store - let path = obj.get_object_path(); - - Ok(self - .storage - .put_object(&RelativePathBuf::from(path), to_bytes(obj), tenant_id) - .await?) - } - - async fn delete_target( - &self, - obj: &dyn MetastoreObject, - tenant_id: &Option, - ) -> Result<(), MetastoreError> { - // we need the path to store in obj store - let path = obj.get_object_path(); - - Ok(self - .storage - .delete_object(&RelativePathBuf::from(path), tenant_id) - .await?) - } - - async fn get_outbound_policies( - &self, - ) -> Result, MetastoreError> { - let mut all_policies = HashMap::new(); - - let base_paths = PARSEABLE.list_tenants().unwrap_or_else(|| vec!["".into()]); - - for tenant in base_paths { - let tenant_id = if tenant.is_empty() { - None - } else { - Some(tenant.clone()) - }; - let path = outbound_http_policy_json_path(&tenant_id); - let policy = match self.storage.get_object(&path, &tenant_id).await { - Ok(bytes) => match serde_json::from_slice::(&bytes) { - Ok(policy) => policy, - Err(err) => { - warn!( - "Failed to deserialize outbound HTTP policy for tenant {tenant}: {err}" - ); - continue; - } - }, - Err(err) if is_missing_optional_dir(&err) => continue, - Err(err) => return Err(MetastoreError::ObjectStorageError(err)), - }; - - all_policies.insert(tenant, policy); - } - - Ok(all_policies) - } - - async fn put_outbound_policy( - &self, - tenant_id: &Option, - policy: &AlertTargetPolicyConfig, - ) -> Result<(), MetastoreError> { - let path = outbound_http_policy_json_path(tenant_id); - Ok(self - .storage - .put_object(&path, to_bytes(policy), tenant_id) - .await?) - } - - async fn get_all_schemas( - &self, - stream_name: &str, - tenant_id: &Option, - ) -> Result, MetastoreError> { - let path_prefix = if let Some(tenant) = tenant_id { - relative_path::RelativePathBuf::from(format!( - "{tenant}/{stream_name}/{STREAM_ROOT_DIRECTORY}" - )) - } else { - relative_path::RelativePathBuf::from(format!("{stream_name}/{STREAM_ROOT_DIRECTORY}")) - }; - Ok(self - .storage - .get_objects( - Some(&path_prefix), - Box::new(|file_name: String| file_name.contains(".schema")), - tenant_id, - ) - .await? - .iter() - // we should be able to unwrap as we know the data is valid schema - .map(|byte_obj| { - serde_json::from_slice(byte_obj) - .unwrap_or_else(|_| panic!("got an invalid schema for stream: {stream_name}")) - }) - .collect()) - } - - async fn refresh_schema( - &self, - stream_name: &str, - tenant_id: &Option, - schema_file: &str, - ) -> Result { - let objects = self.storage.schema_objects(stream_name, tenant_id).await?; - let mut schemas = Vec::with_capacity(objects.len()); - let mut canonical = None; - for (name, bytes) in objects { - let schema: Schema = serde_json::from_slice(&bytes) - .map_err(|e| ObjectStorageError::Custom(e.to_string()))?; - if name == schema_file { - canonical = Some(schema.clone()); - } - schemas.push(schema); - } - let merged = - Schema::try_merge(schemas).map_err(|e| ObjectStorageError::Custom(e.to_string()))?; - // Field order is immaterial, but nested types, nullability and field - // metadata all matter. Normalize only the top-level ordering. - let normalize = |schema: &Schema| { - let mut schema = schema.clone(); - let mut fields = schema.fields.to_vec(); - fields.sort_by(|a, b| a.name().cmp(b.name())); - schema.fields = fields.into(); - schema - }; - if canonical.as_ref().map(normalize) != Some(normalize(&merged)) { - let path = RelativePathBuf::from_iter([ - tenant_id.as_deref().unwrap_or(""), - stream_name, - STREAM_ROOT_DIRECTORY, - schema_file, - ]); - self.storage - .put_object(&path, to_bytes(&merged), tenant_id) - .await?; - } - Ok(merged) - } - - async fn get_schema( - &self, - stream_name: &str, - tenant_id: &Option, - ) -> Result { - Ok(self - .storage - .get_object(&schema_path(stream_name, tenant_id), tenant_id) - .await?) - } - - async fn put_schema( - &self, - obj: Schema, - stream_name: &str, - tenant_id: &Option, - ) -> Result<(), MetastoreError> { - let path = schema_path(stream_name, tenant_id); - Ok(self - .storage - .put_object(&path, to_bytes(&obj), tenant_id) - .await?) - } - - async fn get_parseable_metadata( - &self, - tenant_id: &Option, - ) -> Result, MetastoreError> { - let path = if let Some(tenant_id) = tenant_id.as_ref() { - RelativePathBuf::from_iter([tenant_id, ".parseable.json"]) - } else { - parseable_json_path() - }; - - let parseable_metadata: Option = - match self.storage.get_object(&path, tenant_id).await { - Ok(bytes) => Some(bytes), - Err(err) => { - if matches!(err, ObjectStorageError::NoSuchKey(_)) { - None - } else { - return Err(MetastoreError::ObjectStorageError(err)); - } - } - }; - - Ok(parseable_metadata) - } - - async fn delete_tenant(&self, tenant_id: &str) -> Result<(), MetastoreError> { - self.storage - .delete_prefix(&RelativePathBuf::from(tenant_id), &None) - .await - .map_err(MetastoreError::ObjectStorageError) - } - - async fn get_ingestor_metadata(&self) -> Result, MetastoreError> { - let base_path = RelativePathBuf::from(PARSEABLE_ROOT_DIRECTORY); - Ok(self - .storage - .get_objects( - Some(&base_path), - Box::new(|file_name| file_name.starts_with("ingestor")), - &None, - ) - .await?) - } - - async fn put_parseable_metadata( - &self, - obj: &dyn MetastoreObject, - tenant_id: &Option, - ) -> Result<(), MetastoreError> { - let path = if let Some(tenant_id) = tenant_id.as_ref() { - RelativePathBuf::from_iter([tenant_id, ".parseable.json"]) - } else { - parseable_json_path() - }; - - self.storage - .put_object(&path, to_bytes(obj), tenant_id) - .await - .map_err(MetastoreError::ObjectStorageError) - } - - async fn get_node_metadata( - &self, - node_type: NodeType, - tenant_id: &Option, - ) -> Result, MetastoreError> { - let root_path = RelativePathBuf::from(PARSEABLE_ROOT_DIRECTORY); - let prefix_owned = node_type.to_string(); - - let metadata = self - .storage - .get_objects( - Some(&root_path), - Box::new(move |file_name| file_name.starts_with(&prefix_owned)), // Use the owned copy +super::document_metastore::document_metastore!( + ObjectStoreMetastore, + dyn crate::storage::ObjectStorage, + true, + { + async fn check_health(&self, tenant_id: &Option) -> Result<(), MetastoreError> { + Ok(self.storage.check(tenant_id).await?) + } + + async fn read_latency(&self, tenant_id: &Option) -> std::time::Duration { + self.storage.get_latency(tenant_id).await + } + + async fn stream_exists( + &self, + stream_name: &str, + tenant_id: &Option, + ) -> Result { + Ok(self.storage.stream_exists(stream_name, tenant_id).await?) + } + + async fn is_stream_deleting( + &self, + stream_name: &str, + tenant_id: &Option, + ) -> Result { + Ok(crate::storage::object_storage::is_tombstoned( + self.storage.as_ref(), + stream_name, tenant_id, ) - .await? - .into_iter() - .collect(); - - Ok(metadata) - } - - async fn put_node_metadata(&self, obj: &dyn MetastoreObject) -> Result<(), MetastoreError> { - let path = obj.get_object_path(); - self.storage - .put_object(&RelativePathBuf::from(path), to_bytes(obj), &None) - .await?; - Ok(()) - } - - async fn delete_node_metadata( - &self, - domain_name: &str, - node_type: NodeType, - ) -> Result { - let metadatas = self - .storage - .get_objects( - Some(&RelativePathBuf::from(PARSEABLE_ROOT_DIRECTORY)), - Box::new(move |file_name| file_name.starts_with(&node_type.to_string())), - &None, - ) - .await?; - - let node_metadatas = metadatas - .iter() - .filter_map(|elem| match serde_json::from_slice::(elem) { - Ok(meta) if meta.domain_name() == domain_name => Some(meta), - _ => None, - }) - .collect::>(); - - if node_metadatas.is_empty() { - return Ok(false); + .await?) } - let node_meta_filename = node_metadatas[0].file_path().to_string(); - let file = RelativePathBuf::from(&node_meta_filename); - - match self.storage.delete_object(&file, &None).await { - Ok(_) => Ok(true), - Err(err) => { - if matches!(err, ObjectStorageError::IoError(_)) { - Ok(false) - } else { - Err(MetastoreError::ObjectStorageError(err)) - } - } + async fn delete_stream( + &self, + stream_name: &str, + tenant_id: &Option, + ) -> Result<(), MetastoreError> { + Ok(self.storage.delete_stream(stream_name, tenant_id).await?) } - } - - async fn check_health(&self, tenant_id: &Option) -> Result<(), MetastoreError> { - Ok(self.storage.check(tenant_id).await?) - } - - async fn read_latency(&self, tenant_id: &Option) -> std::time::Duration { - self.storage.get_latency(tenant_id).await - } - - async fn stream_exists( - &self, - stream_name: &str, - tenant_id: &Option, - ) -> Result { - Ok(self.storage.stream_exists(stream_name, tenant_id).await?) - } - async fn is_stream_deleting( - &self, - stream_name: &str, - tenant_id: &Option, - ) -> Result { - Ok(crate::storage::object_storage::is_tombstoned( - self.storage.as_ref(), - stream_name, - tenant_id, - ) - .await?) - } + async fn delete_stream_partition( + &self, + stream_name: &str, + date: &str, + tenant_id: &Option, + ) -> Result<(), MetastoreError> { + let path = + crate::storage::object_storage::stream_data_path(stream_name, &[date], tenant_id); + Ok(self.storage.delete_prefix(&path, tenant_id).await?) + } - async fn delete_stream( - &self, - stream_name: &str, - tenant_id: &Option, - ) -> Result<(), MetastoreError> { - Ok(self.storage.delete_stream(stream_name, tenant_id).await?) + async fn list_tenant_candidates(&self) -> Result, MetastoreError> { + Ok(self + .storage + .list_dirs_relative(&RelativePathBuf::from_iter([""]), &None) + .await? + .into_iter() + .filter(|d| !d.starts_with('.')) + .collect()) + } } +); +#[cfg(test)] +mod boundary_compatibility_tests { + use super::*; + use crate::storage::{LocalFS, ObjectStorage, object_storage::tombstone_path}; - async fn delete_stream_partition( - &self, - stream_name: &str, - date: &str, - tenant_id: &Option, - ) -> Result<(), MetastoreError> { - let path = - crate::storage::object_storage::stream_data_path(stream_name, &[date], tenant_id); - Ok(self.storage.delete_prefix(&path, tenant_id).await?) + #[tokio::test] + async fn partition_deletion_preserves_tenant_scope_and_adjacent_bytes() { + let dir = tempfile::tempdir().unwrap(); + let storage = Arc::new(LocalFS::new(dir.path().to_owned())); + let metastore = ObjectStoreMetastore { + storage: storage.clone(), + }; + let tenant = Some("tenant".to_owned()); + let keys = [ + "tenant/logs/date=2026-01-01/hour=00/data.parquet", + "tenant/logs/date=2026-01-01/ingestor.node.manifest.json", + "tenant/logs/date=2026-01-02/ingestor.node.manifest.json", + "logs/date=2026-01-01/ingestor.node.manifest.json", + ]; + for key in keys { + storage + .put_object( + RelativePath::new(key), + Bytes::from_static(b"original bytes"), + &tenant, + ) + .await + .unwrap(); + } + metastore + .delete_stream_partition("logs", "date=2026-01-01", &tenant) + .await + .unwrap(); + for key in &keys[..2] { + assert!( + storage + .get_object(RelativePath::new(key), &tenant) + .await + .is_err() + ); + } + for key in &keys[2..] { + assert_eq!( + storage + .get_object(RelativePath::new(key), &tenant) + .await + .unwrap(), + Bytes::from_static(b"original bytes") + ); + } } - async fn list_tenant_candidates(&self) -> Result, MetastoreError> { - Ok(self - .storage - .list_dirs_relative(&RelativePathBuf::from_iter([""]), &None) - .await? + #[tokio::test] + async fn lifecycle_seams_preserve_localfs_results_and_bytes() { + let dir = tempfile::tempdir().unwrap(); + let storage = Arc::new(LocalFS::new(dir.path().to_owned())); + let metastore = ObjectStoreMetastore { + storage: storage.clone(), + }; + assert_eq!( + metastore.check_health(&None).await.is_ok(), + storage.check(&None).await.is_ok() + ); + metastore.read_latency(&None).await; + assert!(!metastore.stream_exists("logs", &None).await.unwrap()); + assert!(!metastore.is_stream_deleting("logs", &None).await.unwrap()); + let key = RelativePath::new("logs/.stream/.stream.json"); + let bytes = Bytes::from_static(b"{\"version\":\"v7\",\"unknown\":true}"); + storage.put_object(key, bytes.clone(), &None).await.unwrap(); + assert_eq!( + metastore.stream_exists("logs", &None).await.unwrap(), + storage.stream_exists("logs", &None).await.unwrap() + ); + // Discovery/existence probes never normalize or rewrite stored JSON. + assert_eq!(storage.get_object(key, &None).await.unwrap(), bytes); + storage + .put_object( + &tombstone_path("logs", &None), + Bytes::from_static(b"null"), + &None, + ) + .await + .unwrap(); + assert!(metastore.is_stream_deleting("logs", &None).await.unwrap()); + let mut expected: Vec<_> = storage + .list_dirs_relative(RelativePath::new(""), &None) + .await + .unwrap() .into_iter() .filter(|d| !d.starts_with('.')) - .collect()) - } - - async fn list_streams( - &self, - tenant_id: &Option, - ) -> Result, MetastoreError> { - // using LocalFS list_streams because it doesn't implement list_with_delimiter - if PARSEABLE.storage.name() == "drive" { - PARSEABLE - .storage - .get_object_store() - .list_streams() - .await - .map_err(MetastoreError::ObjectStorageError) - } else { - // not local-disk, object storage - let root = tenant_id - .as_ref() - .map(|tenant| object_store::path::Path::from_iter([tenant.clone()])); - - let resp = self.storage.list_with_delimiter(root.clone()).await?; - - let streams = resp - .common_prefixes - .iter() - .flat_map(|path| path.parts()) - .map(|name| name.as_ref().to_string()) - .filter(|name| { - name != PARSEABLE_ROOT_DIRECTORY - && name != USERS_ROOT_DIR - && name != SETTINGS_ROOT_DIRECTORY - && name != ALERTS_ROOT_DIRECTORY - && name != TOMBSTONE_ROOT_DIRECTORY - }) - .collect::>(); - let results = collect_dataset_probes(streams.into_iter().map(|stream| { - let root = root.clone(); - async move { - let stream_path = if let Some(root) = root.as_ref() { - object_store::path::Path::from_iter([ - root.as_ref(), - &stream, - STREAM_ROOT_DIRECTORY, - ]) - } else { - object_store::path::Path::from(format!( - "{}/{}", - stream, STREAM_ROOT_DIRECTORY - )) - }; - let resp = self.storage.list_with_delimiter(Some(stream_path)).await?; - if resp - .objects - .iter() - .any(|name| name.location.filename().unwrap().ends_with("stream.json")) - { - Ok::<_, MetastoreError>(Some(stream)) - } else { - Ok(None) - } - } - })) - .await?; - Ok(results.into_iter().flatten().collect()) - } + .collect(); + let mut actual = metastore.list_tenant_candidates().await.unwrap(); + expected.sort(); + actual.sort(); + assert_eq!(actual, expected); + metastore.delete_stream("logs", &None).await.unwrap(); + assert!(!metastore.stream_exists("logs", &None).await.unwrap()); + // Historical prefix deletion leaves the separate marker untouched. + assert!(metastore.is_stream_deleting("logs", &None).await.unwrap()); + let raw_error = storage + .delete_stream("missing", &None) + .await + .unwrap_err() + .to_string(); + let trait_error = metastore.delete_stream("missing", &None).await.unwrap_err(); + assert!( + matches!(trait_error, MetastoreError::ObjectStorageError(error) if error.to_string() == raw_error) + ); } } -// All-or-error collection: a failed probe cannot publish a partial inventory. -async fn collect_dataset_probes( - probes: impl IntoIterator>>, -) -> Result, E> { - futures::stream::iter(probes) - .buffer_unordered(16) - .try_collect() - .await -} - #[cfg(test)] mod tests { use super::*; - use crate::storage::LocalFS; + use crate::storage::{LocalFS, ObjectStorage}; #[tokio::test] async fn dataset_probes_are_bounded_and_fail_atomically() { @@ -1683,22 +184,23 @@ mod tests { for fail in [false, true] { let active = AtomicUsize::new(0); let peak = AtomicUsize::new(0); - let result = collect_dataset_probes((0..64).map(|i| { - let active = &active; - let peak = &peak; - async move { - let n = active.fetch_add(1, Ordering::SeqCst) + 1; - peak.fetch_max(n, Ordering::SeqCst); - tokio::time::sleep(std::time::Duration::from_millis(5)).await; - active.fetch_sub(1, Ordering::SeqCst); - if fail && i == 25 { - Err("failed probe") - } else { - Ok(i) + let result = + super::super::document_metastore::collect_dataset_probes((0..64).map(|i| { + let active = &active; + let peak = &peak; + async move { + let n = active.fetch_add(1, Ordering::SeqCst) + 1; + peak.fetch_max(n, Ordering::SeqCst); + tokio::time::sleep(std::time::Duration::from_millis(5)).await; + active.fetch_sub(1, Ordering::SeqCst); + if fail && i == 25 { + Err("failed probe") + } else { + Ok(i) + } } - } - })) - .await; + })) + .await; assert_eq!(peak.load(Ordering::SeqCst), 16); if fail { assert_eq!(result, Err("failed probe")); @@ -2285,109 +787,3 @@ mod tests { )); } } - -#[cfg(test)] -mod boundary_compatibility_tests { - use super::*; - use crate::storage::{LocalFS, object_storage::tombstone_path}; - - #[tokio::test] - async fn partition_deletion_preserves_tenant_scope_and_adjacent_bytes() { - let dir = tempfile::tempdir().unwrap(); - let storage = Arc::new(LocalFS::new(dir.path().to_owned())); - let metastore = ObjectStoreMetastore { - storage: storage.clone(), - }; - let tenant = Some("tenant".to_owned()); - let keys = [ - "tenant/logs/date=2026-01-01/hour=00/data.parquet", - "tenant/logs/date=2026-01-01/ingestor.node.manifest.json", - "tenant/logs/date=2026-01-02/ingestor.node.manifest.json", - "logs/date=2026-01-01/ingestor.node.manifest.json", - ]; - for key in keys { - storage - .put_object( - RelativePath::new(key), - Bytes::from_static(b"original bytes"), - &tenant, - ) - .await - .unwrap(); - } - metastore - .delete_stream_partition("logs", "date=2026-01-01", &tenant) - .await - .unwrap(); - for key in &keys[..2] { - assert!( - storage - .get_object(RelativePath::new(key), &tenant) - .await - .is_err() - ); - } - for key in &keys[2..] { - assert_eq!( - storage - .get_object(RelativePath::new(key), &tenant) - .await - .unwrap(), - Bytes::from_static(b"original bytes") - ); - } - } - - #[tokio::test] - async fn lifecycle_seams_preserve_localfs_results_and_bytes() { - let dir = tempfile::tempdir().unwrap(); - let storage = Arc::new(LocalFS::new(dir.path().to_owned())); - let metastore = ObjectStoreMetastore { - storage: storage.clone(), - }; - assert!(!metastore.stream_exists("logs", &None).await.unwrap()); - assert!(!metastore.is_stream_deleting("logs", &None).await.unwrap()); - let key = RelativePath::new("logs/.stream/.stream.json"); - let bytes = Bytes::from_static(b"{\"version\":\"v7\",\"unknown\":true}"); - storage.put_object(key, bytes.clone(), &None).await.unwrap(); - assert_eq!( - metastore.stream_exists("logs", &None).await.unwrap(), - storage.stream_exists("logs", &None).await.unwrap() - ); - // Discovery/existence probes never normalize or rewrite stored JSON. - assert_eq!(storage.get_object(key, &None).await.unwrap(), bytes); - storage - .put_object( - &tombstone_path("logs", &None), - Bytes::from_static(b"null"), - &None, - ) - .await - .unwrap(); - assert!(metastore.is_stream_deleting("logs", &None).await.unwrap()); - let mut expected: Vec<_> = storage - .list_dirs_relative(RelativePath::new(""), &None) - .await - .unwrap() - .into_iter() - .filter(|d| !d.starts_with('.')) - .collect(); - let mut actual = metastore.list_tenant_candidates().await.unwrap(); - expected.sort(); - actual.sort(); - assert_eq!(actual, expected); - metastore.delete_stream("logs", &None).await.unwrap(); - assert!(!metastore.stream_exists("logs", &None).await.unwrap()); - // Historical prefix deletion leaves the separate marker untouched. - assert!(metastore.is_stream_deleting("logs", &None).await.unwrap()); - let raw_error = storage - .delete_stream("missing", &None) - .await - .unwrap_err() - .to_string(); - let trait_error = metastore.delete_stream("missing", &None).await.unwrap_err(); - assert!( - matches!(trait_error, MetastoreError::ObjectStorageError(error) if error.to_string() == raw_error) - ); - } -} diff --git a/src/metastore/metastores/postgres_client.rs b/src/metastore/metastores/postgres_client.rs new file mode 100644 index 000000000..c6395c459 --- /dev/null +++ b/src/metastore/metastores/postgres_client.rs @@ -0,0 +1,213 @@ +//! Deadline enforcement for every SQL operation, including transactions. +use crate::storage::ObjectStorageError; +use std::{ + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, + time::Duration, +}; +use tokio_postgres::{Row, types::ToSql}; + +pub(crate) struct Client { + inner: Option, + timeout: Duration, + poisoned: Arc, + cancel: tokio_postgres::CancelToken, + tls: tokio_postgres_rustls::MakeRustlsConnect, +} +impl Drop for Client { + fn drop(&mut self) { + if self.poisoned.load(Ordering::SeqCst) + && let Some(c) = self.inner.take() + { + drop(deadpool_postgres::Client::take(c)); + } + } +} +struct Pending<'a>(&'a AtomicBool, bool); +impl Drop for Pending<'_> { + fn drop(&mut self) { + if self.1 { + self.0.store(true, Ordering::SeqCst); + } + } +} +async fn bounded( + timeout: Duration, + poisoned: &AtomicBool, + cancel: &tokio_postgres::CancelToken, + tls: &tokio_postgres_rustls::MakeRustlsConnect, + call: impl std::future::Future>, +) -> Result { + let mut pending = Pending(poisoned, true); + let result = tokio::time::timeout(timeout, call).await; + pending.1 = false; + match result { + Ok(Ok(value)) => Ok(value), + Ok(Err(e)) => { + if e.code() == Some(&tokio_postgres::error::SqlState::READ_ONLY_SQL_TRANSACTION) + || e.is_closed() + { + poisoned.store(true, Ordering::SeqCst); + } + if e.as_db_error() + .is_some_and(|e| e.message().contains("stream deletion pending")) + { + Err(ObjectStorageError::StreamDeleting) + } else { + Err(ObjectStorageError::MetastoreUnavailable( + e.as_db_error() + .map(|d| d.to_string()) + .unwrap_or_else(|| e.to_string()), + )) + } + } + Err(_) => { + poisoned.store(true, Ordering::SeqCst); + // Cancellation has its own bounded network wait. A lost connection + // still leaves an ambiguous result; never automatically retry writes. + let _ = tokio::time::timeout(Duration::from_secs(1), cancel.cancel_query(tls.clone())) + .await; + Err(ObjectStorageError::MetastoreUnavailable( + "SQL client deadline exceeded".into(), + )) + } + } +} +macro_rules! methods { + () => { + pub async fn query( + &self, + sql: &str, + params: &[&(dyn ToSql + Sync)], + ) -> Result, ObjectStorageError> { + bounded( + self.timeout, + &self.poisoned, + &self.cancel, + &self.tls, + self.raw().query(sql, params), + ) + .await + } + pub async fn query_one( + &self, + sql: &str, + params: &[&(dyn ToSql + Sync)], + ) -> Result { + bounded( + self.timeout, + &self.poisoned, + &self.cancel, + &self.tls, + self.raw().query_one(sql, params), + ) + .await + } + pub async fn query_opt( + &self, + sql: &str, + params: &[&(dyn ToSql + Sync)], + ) -> Result, ObjectStorageError> { + bounded( + self.timeout, + &self.poisoned, + &self.cancel, + &self.tls, + self.raw().query_opt(sql, params), + ) + .await + } + pub async fn execute( + &self, + sql: &str, + params: &[&(dyn ToSql + Sync)], + ) -> Result { + bounded( + self.timeout, + &self.poisoned, + &self.cancel, + &self.tls, + self.raw().execute(sql, params), + ) + .await + } + pub async fn batch_execute(&self, sql: &str) -> Result<(), ObjectStorageError> { + bounded( + self.timeout, + &self.poisoned, + &self.cancel, + &self.tls, + self.raw().batch_execute(sql), + ) + .await + } + }; +} +impl Client { + pub fn new( + inner: deadpool_postgres::Client, + timeout: Duration, + tls: tokio_postgres_rustls::MakeRustlsConnect, + ) -> Self { + let cancel = inner.cancel_token(); + Self { + inner: Some(inner), + timeout, + cancel, + tls, + poisoned: Arc::new(AtomicBool::new(false)), + } + } + pub fn discard_on_drop(&self) { + self.poisoned.store(true, Ordering::SeqCst); + } + pub fn set_timeout(&mut self, timeout: Duration) { + self.timeout = timeout; + } + fn raw(&self) -> &tokio_postgres::Client { + self.inner.as_ref().unwrap() + } + methods!(); + pub async fn transaction(&mut self) -> Result, ObjectStorageError> { + let inner = bounded( + self.timeout, + &self.poisoned, + &self.cancel, + &self.tls, + self.inner.as_mut().unwrap().transaction(), + ) + .await?; + Ok(Transaction { + inner, + timeout: self.timeout, + poisoned: self.poisoned.clone(), + cancel: self.cancel.clone(), + tls: self.tls.clone(), + }) + } +} +pub(crate) struct Transaction<'a> { + inner: deadpool_postgres::Transaction<'a>, + timeout: Duration, + poisoned: Arc, + cancel: tokio_postgres::CancelToken, + tls: tokio_postgres_rustls::MakeRustlsConnect, +} +impl Transaction<'_> { + fn raw(&self) -> &tokio_postgres::Transaction<'_> { + &self.inner + } + methods!(); + pub async fn commit(self) -> Result<(), ObjectStorageError> { + bounded( + self.timeout, + &self.poisoned, + &self.cancel, + &self.tls, + self.inner.commit(), + ) + .await + } +} diff --git a/src/metastore/metastores/postgres_documents.rs b/src/metastore/metastores/postgres_documents.rs new file mode 100644 index 000000000..40042563a --- /dev/null +++ b/src/metastore/metastores/postgres_documents.rs @@ -0,0 +1,1847 @@ +//! SQL document primitives. No connection is held during object storage I/O. +use super::postgres_metastore::PostgresOptions; +use crate::storage::object_storage::{ + ConditionalMetadataError, MetadataVersion, MetadataWriteCondition, VersionedMetadata, +}; +use crate::storage::{ObjectStorage, ObjectStorageError}; +use bytes::Bytes; +use relative_path::RelativePath; +use std::{sync::Arc, time::Duration}; + +pub struct PostgresDocuments { + pool: deadpool_postgres::Pool, + lock_pool: deadpool_postgres::Pool, + tls: tokio_postgres_rustls::MakeRustlsConnect, + deadline: Duration, + bucket: String, + expected_id: Option, + storage: Arc, + initialized: tokio::sync::OnceCell<()>, + #[cfg(test)] + pub(super) deletion_fault: std::sync::atomic::AtomicU8, +} +impl std::fmt::Debug for PostgresDocuments { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("PostgresDocuments") + .field("bucket", &self.bucket) + .finish_non_exhaustive() + } +} +fn err(e: impl std::fmt::Display) -> ObjectStorageError { + if e.to_string().contains("stream deletion pending") { + ObjectStorageError::StreamDeleting + } else { + ObjectStorageError::MetastoreUnavailable(e.to_string()) + } +} +fn legacy_key<'a>(key: &'a str, base: &str) -> Option<&'a str> { + if base.is_empty() { + return None; + } + key.strip_prefix(base).and_then(|p| p.strip_prefix('/')) +} +fn read_keys(path: &str, base: &str) -> Vec { + let mut keys = vec![path.to_owned()]; + let legacy = legacy_key(path, base).unwrap_or(path); + if legacy != path { + keys.push(legacy.to_owned()); + } + let canonical = super::migration::canonical_key(legacy); + if canonical != legacy { + keys.push(canonical.to_owned()); + } + keys +} +fn tenant(t: &Option) -> &str { + t.as_deref() + .filter(|v| !v.is_empty() && *v != "DEFAULT_TENANT") + .unwrap_or("") +} +fn version(v: i64) -> MetadataVersion { + MetadataVersion { + e_tag: None, + version: Some(v.to_string()), + } +} +pub(super) fn configured_roots( + ca: Option<&std::path::Path>, +) -> anyhow::Result { + let mut roots = rustls::RootCertStore::empty(); + if let Some(path) = ca { + for certificate in + rustls_pemfile::certs(&mut std::io::BufReader::new(std::fs::File::open(path)?)) + { + roots.add(certificate?)?; + } + anyhow::ensure!( + !roots.is_empty(), + "configured CA bundle contains no certificates" + ); + } else { + roots.extend(webpki_roots::TLS_SERVER_ROOTS.iter().cloned()); + } + Ok(roots) +} +impl PostgresDocuments { + pub fn new( + options: &PostgresOptions, + bucket: String, + storage: Arc, + ) -> anyhow::Result { + let mut config: tokio_postgres::Config = + options.metastore_postgres_url.as_deref().unwrap().parse()?; + let timeout = Duration::from_millis(options.metastore_postgres_connect_timeout_ms); + config.connect_timeout(timeout); + config.tcp_user_timeout(Duration::from_millis( + options.metastore_postgres_statement_timeout_ms, + )); + config + .keepalives(true) + .keepalives_idle(Duration::from_secs(1)) + .keepalives_interval(Duration::from_secs(1)) + .keepalives_retries(3); + config.target_session_attrs(tokio_postgres::config::TargetSessionAttrs::ReadWrite); + let prior_options = config.get_options().unwrap_or("").to_owned(); + config.options(format!( + "{prior_options} -c statement_timeout={} -c lock_timeout={}", + options.metastore_postgres_statement_timeout_ms, + options.metastore_postgres_statement_timeout_ms + )); + // Prefer/allow permit plaintext downgrade; opt-in mode defaults to verified TLS. + if config.get_ssl_mode() != tokio_postgres::config::SslMode::Disable { + config.ssl_mode(tokio_postgres::config::SslMode::Require); + } + let roots = configured_roots(options.metastore_postgres_ca.as_deref())?; + if options.metastore_postgres_ca.is_some() + && config.get_ssl_mode() == tokio_postgres::config::SslMode::Disable + { + tracing::warn!("PostgreSQL CA configured with sslmode=disable; CA is unused"); + } + let tls = rustls::ClientConfig::builder_with_provider(Arc::new( + rustls::crypto::ring::default_provider(), + )) + .with_safe_default_protocol_versions()? + .with_root_certificates(roots) + .with_no_client_auth(); + let tls = tokio_postgres_rustls::MakeRustlsConnect::new(tls); + let lock_manager = deadpool_postgres::Manager::new(config.clone(), tls.clone()); + let manager = deadpool_postgres::Manager::new(config, tls.clone()); + let lock_pool = deadpool_postgres::Pool::builder(lock_manager) + .max_size(2) + .runtime(deadpool_postgres::Runtime::Tokio1) + .wait_timeout(Some(timeout)) + .create_timeout(Some(timeout)) + .build()?; + let pool = deadpool_postgres::Pool::builder(manager) + .max_size(options.metastore_postgres_pool_size as usize) + .runtime(deadpool_postgres::Runtime::Tokio1) + .wait_timeout(Some(timeout)) + .create_timeout(Some(timeout)) + .recycle_timeout(Some(timeout)) + .build()?; + Ok(Self { + pool, + lock_pool, + tls, + deadline: Duration::from_millis( + options + .metastore_postgres_statement_timeout_ms + .saturating_add(250), + ), + bucket, + expected_id: options.metastore_postgres_id.clone(), + storage, + initialized: tokio::sync::OnceCell::new(), + #[cfg(test)] + deletion_fault: std::sync::atomic::AtomicU8::new(0), + }) + } + pub(crate) async fn client( + &self, + ) -> Result { + Ok(super::postgres_client::Client::new( + tokio::time::timeout(self.deadline, self.pool.get()) + .await + .map_err(err)? + .map_err(err)?, + self.deadline, + self.tls.clone(), + )) + } + async fn migration_client(&self) -> Result { + let millis: u64 = std::env::var("P_METASTORE_POSTGRES_MIGRATION_TIMEOUT_MS") + .unwrap_or_else(|_| "1800000".into()) + .parse() + .map_err(err)?; + if millis == 0 { + return Err(err("migration timeout must be positive")); + } + let mut client = self.client().await?; + client.discard_on_drop(); + client.set_timeout(Duration::from_millis(millis.saturating_add(250))); + client + .batch_execute(&format!( + "SET statement_timeout={millis}; SET lock_timeout={millis}" + )) + .await?; + Ok(client) + } + pub(crate) async fn transfer_lock( + &self, + ) -> Result { + let client = super::postgres_client::Client::new( + self.lock_pool.get().await.map_err(err)?, + self.deadline, + self.tls.clone(), + ); + client.discard_on_drop(); // Close socket to release session lock on every exit. + client.batch_execute("SET idle_session_timeout=0").await?; + let acquired: bool = client + .query_one("SELECT pg_try_advisory_lock(8104050183641399917)", &[]) + .await? + .get(0); + if !acquired { + return Err(err("another offline metadata transfer is running")); + } + Ok(client) + } + pub async fn check_migration_guard(&self) -> Result<(), ObjectStorageError> { + super::migration::guard(self.storage.as_ref(), true) + .await + .map_err(err) + } + pub async fn check(&self, _: &Option) -> Result<(), ObjectStorageError> { + super::migration::guard(self.storage.as_ref(), true) + .await + .map_err(err)?; + self.initialized.get_or_try_init(|| async { + let mut client = self.migration_client().await?; + let tx = client.transaction().await.map_err(err)?; + tx.batch_execute("SELECT pg_advisory_xact_lock(8104050183641399916)").await.map_err(err)?; + let (actual, _, created) = self.bootstrap(&tx, &uuid::Uuid::new_v4().to_string()).await?; + if tx.query_opt("SELECT state FROM parseable_transfer WHERE singleton", &[]).await.map_err(err)?.is_some_and(|r| r.get::<_,String>(0) != "ready") { return Err(err("offline transfer incomplete or exported; use migration command")); } + + tx.commit().await.map_err(err)?; + drop(client); + // Pair outside SQL: never hold a pooled connection during bucket I/O. + let marker = RelativePath::new(".parseable/postgres-metastore-id"); + if !created { + // A concurrent first initializer may still be publishing pairing. + for _ in 0..20 { + if super::migration::optional(self.storage.as_ref(), ".parseable/postgres-metastore-id").await.map_err(err)?.is_some() { break; } + tokio::time::sleep(Duration::from_millis(10)).await; + } + } + if !created && super::migration::optional(self.storage.as_ref(), ".parseable/postgres-metastore-id").await.map_err(err)?.is_none() && !super::migration::recovery_override(std::env::var("P_METASTORE_POSTGRES_ALLOW_UNPAIRED").ok().as_deref(), actual.as_bytes()) { + return Err(err("nonempty database and missing bucket pairing marker; restore the marker or explicitly set P_METASTORE_POSTGRES_ALLOW_UNPAIRED= after verifying the bucket")); + } + match self.storage.put_metadata_conditional(marker, Bytes::from(actual.clone()), MetadataWriteCondition::Create, &None).await { + Ok(_) | Err(ConditionalMetadataError::Conflict) => {}, + Err(e) => return Err(err(e)), + } + let paired = self.storage.get_object(marker, &None).await?; + if paired.as_ref() != actual.as_bytes() { return Err(err("bucket is paired with a different PostgreSQL backend UUID")); } + tracing::info!(backend_id = actual, "PostgreSQL metastore initialized"); + Ok(()) + }).await.map(|_| ()) + } + async fn bootstrap( + &self, + tx: &super::postgres_client::Transaction<'_>, + id: &str, + ) -> Result<(String, bool, bool), ObjectStorageError> { + // Bootstrap is additive; version checks precede any future forward migration. + tx.batch_execute(include_str!("../migrations/000_identity.sql")) + .await + .map_err(err)?; + let created = tx.execute("INSERT INTO parseable_backend(id,bucket,schema_version,protocol_version) VALUES($1,$2,0,1) ON CONFLICT DO NOTHING", &[&id,&self.bucket]).await.map_err(err)? == 1; + let row = tx + .query_one( + "SELECT id,bucket,schema_version,protocol_version FROM parseable_backend", + &[], + ) + .await + .map_err(err)?; + let actual: String = row.get(0); + if row.get::<_, String>(1) != self.bucket + || self.expected_id.as_ref().is_some_and(|id| id != &actual) + { + return Err(err("backend/bucket identity mismatch")); + } + if !(0..=3).contains(&row.get::<_, i32>(2)) || row.get::<_, i32>(3) != 1 { + return Err(err( + "unsupported schema/protocol version; upgrade this server", + )); + } + if row.get::<_, i32>(2) == 0 { + tx.batch_execute(include_str!("../migrations/001_documents.sql")) + .await + .map_err(err)?; + tx.execute( + "UPDATE parseable_backend SET schema_version=1 WHERE singleton", + &[], + ) + .await + .map_err(err)?; + } + if row.get::<_, i32>(2) < 2 { + tx.batch_execute(include_str!("../migrations/002_review_fixes.sql")) + .await + .map_err(err)?; + } + if row.get::<_, i32>(2) < 3 { + tx.batch_execute(include_str!("../migrations/003_manifest_admission.sql")) + .await?; + } + tx.batch_execute("CREATE TABLE IF NOT EXISTS parseable_transfer (singleton boolean PRIMARY KEY DEFAULT true CHECK(singleton), state text NOT NULL)").await.map_err(err)?; + let has_documents: bool = tx + .query_one("SELECT EXISTS(SELECT 1 FROM parseable_documents)", &[]) + .await? + .get(0); + Ok((actual, has_documents, created)) + } + // Probe both dependencies without requiring a bucket metadata document. + pub async fn check_health(&self, t: &Option) -> Result<(), ObjectStorageError> { + let database = async { + self.client() + .await? + .batch_execute("SELECT 1") + .await + .map_err(err)?; + Ok::<_, ObjectStorageError>(()) + }; + let bucket = async { + self.storage + .get_object(RelativePath::new(".parseable/postgres-metastore-id"), t) + .await + .map(|_| ()) + }; + tokio::time::timeout(self.deadline, async { + tokio::try_join!(database, bucket)?; + Ok(()) + }) + .await + .map_err(err)? + } + pub async fn read_latency(&self, t: &Option) -> Duration { + let start = std::time::Instant::now(); + let result = self.check_health(t).await; + tracing::info!(?result, "PostgreSQL and data bucket combined probe"); + start.elapsed() + } + pub async fn stream_exists( + &self, + stream: &str, + t: &Option, + ) -> Result { + let path = crate::storage::object_storage::stream_data_path(stream, &[".stream"], t); + Ok(self.client().await?.query_one("SELECT EXISTS(SELECT 1 FROM parseable_documents WHERE tenant=$1 AND key >= $2 || '/' AND key < $2 || '0' AND key LIKE '%stream.json')", &[&tenant(t), &path.as_str()]).await?.get(0)) + } + + pub async fn is_stream_deleting( + &self, + stream: &str, + t: &Option, + ) -> Result { + let path = + crate::storage::object_storage::stream_data_path(stream, &[".stream", ".deleting"], t); + Ok(self + .get_metadata_versioned(&path, t) + .await + .map_err(err)? + .is_some()) + } + async fn remove_rows( + &self, + path: &RelativePath, + t: &Option, + ) -> Result<(), ObjectStorageError> { + self.client().await?.execute("DELETE FROM parseable_documents WHERE tenant=$1 AND ($2='' OR key=$2 OR (key >= $2 || '/' AND key < $2 || '0'))", &[&tenant(t), &path.as_str()]).await.map_err(err)?; + Ok(()) + } + async fn delete_data( + &self, + path: &RelativePath, + t: &Option, + ) -> Result<(), ObjectStorageError> { + match self.storage.delete_prefix(path, t).await { + Err(ObjectStorageError::NoSuchKey(_)) => Ok(()), + Err(ObjectStorageError::IoError(e)) if e.kind() == std::io::ErrorKind::NotFound => { + Ok(()) + } + result => result, + } + } + pub async fn delete_stream( + &self, + stream: &str, + t: &Option, + ) -> Result<(), ObjectStorageError> { + let marker = + crate::storage::object_storage::stream_data_path(stream, &[".stream", ".deleting"], t); + self.put_object(&marker, Bytes::new(), t).await?; + let path = crate::storage::object_storage::stream_data_path(stream, &[], t); + #[cfg(test)] + if self + .deletion_fault + .load(std::sync::atomic::Ordering::SeqCst) + == 1 + { + return Err(err("injected bucket failure")); + } + self.delete_data(&path, t).await?; + #[cfg(test)] + if self + .deletion_fault + .load(std::sync::atomic::Ordering::SeqCst) + == 2 + { + return Err(err("injected crash after data deletion")); + } + self.remove_rows(&path, t).await + } + pub async fn delete_partition( + &self, + path: &RelativePath, + t: &Option, + ) -> Result<(), ObjectStorageError> { + self.delete_data(path, t).await?; + #[cfg(test)] + if self + .deletion_fault + .load(std::sync::atomic::Ordering::SeqCst) + == 3 + { + return Err(err("injected partition cleanup failure")); + } + self.remove_rows(path, t).await + } + pub async fn list_tenant_candidates(&self) -> Result, ObjectStorageError> { + let mut candidates: std::collections::BTreeSet = self + .client() + .await? + .query( + "SELECT DISTINCT tenant FROM parseable_documents WHERE tenant <> ''", + &[], + ) + .await + .map_err(err)? + .into_iter() + .map(|r| r.get(0)) + .collect(); + let default_streams: std::collections::HashSet = self + .keys("", "") + .await? + .iter() + .filter_map(|k| { + k.split_once('/').and_then(|(s, rest)| { + (rest.starts_with(".stream/") || rest == ".stream.json" || rest == ".schema") + .then(|| s.to_owned()) + }) + }) + .collect(); + candidates.extend( + self.storage + .list_dirs_relative(RelativePath::new(""), &None) + .await? + .into_iter() + .filter(|p| !p.starts_with('.') && !default_streams.contains(p)), + ); + Ok(candidates.into_iter().collect()) + } + pub fn absolute_url(&self, path: &RelativePath) -> object_store::path::Path { + object_store::path::Path::parse(path.as_str()).expect("valid document key") + } + /// Yield at most 32 selected locators and 8 MiB of raw bodies at a time. + /// A larger single document is indivisible and is yielded alone. The caller + /// decodes each chunk before requesting another; connections end at yield. + pub fn manifest_chunks<'a>( + &'a self, + paths: &'a [String], + t: &'a Option, + ) -> impl futures::Stream, ObjectStorageError>> + Send + 'a + { + use std::collections::{HashMap, VecDeque}; + futures::stream::try_unfold( + (0usize, VecDeque::<(String, String, usize)>::new()), + move |(mut offset, mut pending)| async move { + loop { + if pending.is_empty() { + if offset == paths.len() { + return Ok(None); + } + let start = offset; + let mut key_bytes = 0; + while offset < paths.len() && offset - start < 32 { + let size = paths[offset].len(); + if offset > start && key_bytes + size > 64 * 1024 { + break; + } + key_bytes += size; + offset += 1; + } + let base = self.storage.absolute_url(RelativePath::new("")); + let candidates: Vec<_> = paths[start..offset] + .iter() + .flat_map(|path| read_keys(path, base.as_ref())) + .collect(); + let rows = self.client().await?.query( + "SELECT key,octet_length(body) FROM parseable_documents WHERE tenant=$1 AND key=ANY($2)", + &[&tenant(t), &candidates], + ).await?; + let sizes: HashMap = rows + .into_iter() + .map(|row| (row.get(0), row.get::<_, i32>(1) as usize)) + .collect(); + for original in &paths[start..offset] { + let keys = read_keys(original, base.as_ref()); + let matches: Vec<_> = keys + .iter() + .filter_map(|key| sizes.get(key).map(|size| (key, *size))) + .collect(); + if matches.len() > 1 { + tracing::warn!( + key = original, + "multiple metadata candidates exist; exact key takes precedence" + ); + } + if let Some((key, size)) = matches.first() { + pending.push_back((original.clone(), (*key).clone(), *size)); + } + } + if pending.is_empty() { + continue; + } + } + let mut selected = Vec::new(); + let mut body_bytes = 0usize; + while let Some((_, _, size)) = pending.front() { + if !selected.is_empty() && body_bytes + size > 8 * 1024 * 1024 { + break; + } + body_bytes += size; + selected.push(pending.pop_front().unwrap()); + } + let keys: Vec<_> = selected.iter().map(|(_, key, _)| key.clone()).collect(); + let rows = self.client().await?.query( + "WITH sized AS (SELECT key,body,octet_length(body) AS size,row_number() OVER (ORDER BY array_position($2,key)) AS n,sum(octet_length(body)) OVER (ORDER BY array_position($2,key)) AS total FROM parseable_documents WHERE tenant=$1 AND key=ANY($2)) SELECT key,CASE WHEN n=1 OR total<=8388608 THEN body END,size FROM sized", + &[&tenant(t), &keys], + ).await?; + // Recheck the byte bound in the body statement itself: a + // manifest can grow after the sizing query. Requeue only + // rows withheld by the byte budget, never deleted rows. + let bodies: HashMap, usize)> = rows + .into_iter() + .map(|row| { + ( + row.get(0), + ( + row.get::<_, Option>>(1).map(Bytes::from), + row.get::<_, i32>(2) as usize, + ), + ) + }) + .collect(); + for (original, key, _) in selected.iter().rev() { + if let Some((None, size)) = bodies.get(key) { + pending.push_front((original.clone(), key.clone(), *size)); + } + } + let batch = selected + .into_iter() + .filter_map(|(original, key, _)| { + bodies + .get(&key) + .and_then(|(body, _)| body.as_ref()) + .map(|body| (original, body.clone())) + }) + .collect(); + return Ok(Some((batch, (offset, pending)))); + } + }, + ) + } + + #[cfg(test)] + pub async fn manifest_batch( + &self, + paths: &[String], + t: &Option, + ) -> Result, ObjectStorageError> { + use futures::TryStreamExt; + let chunks = self.manifest_chunks(paths, t); + futures::pin_mut!(chunks); + let mut rows = Vec::new(); + while let Some(chunk) = chunks.try_next().await? { + rows.extend(chunk); + } + Ok(rows) + } + pub async fn get_object( + &self, + path: &RelativePath, + t: &Option, + ) -> Result { + self.get_metadata_versioned(path, t) + .await + .map_err(err)? + .map(|r| r.bytes) + .ok_or_else(|| ObjectStorageError::NoSuchKey(path.to_string())) + } + pub async fn get_metadata_versioned( + &self, + path: &RelativePath, + t: &Option, + ) -> Result, ConditionalMetadataError> { + let base = self.storage.absolute_url(RelativePath::new("")); + let c = self.client().await?; + let exact = path.as_str(); + let keys = read_keys(exact, base.as_ref()); + let rows = c.query( + "SELECT key,body,version FROM parseable_documents WHERE tenant=$1 AND key=ANY($2) ORDER BY array_position($2,key)", + &[&tenant(t), &keys], + ).await?; + if rows.len() > 1 { + tracing::warn!( + key = exact, + "exact and legacy metadata keys exist; using exact key" + ); + } + Ok(rows.first().map(|r| VersionedMetadata { + bytes: Bytes::from(r.get::<_, Vec>(1)), + version: version(r.get(2)), + })) + } + pub async fn put_object( + &self, + path: &RelativePath, + body: Bytes, + t: &Option, + ) -> Result<(), ObjectStorageError> { + let c = self.client().await?; + c.execute("INSERT INTO parseable_documents(tenant,kind,key,body) VALUES($1,$2,$3,$4) ON CONFLICT(tenant,key) DO UPDATE SET body=EXCLUDED.body, version=nextval('parseable_document_version'),updated_at=now()", &[&tenant(t), &kind(path), &path.as_str(), &&body[..]]).await.map_err(err)?; + Ok(()) + } + pub async fn put_metadata_conditional( + &self, + path: &RelativePath, + body: Bytes, + condition: MetadataWriteCondition, + t: &Option, + ) -> Result { + let c = self.client().await?; + let row = match condition { + MetadataWriteCondition::Create => c.query_opt("INSERT INTO parseable_documents(tenant,kind,key,body) VALUES($1,$2,$3,$4) ON CONFLICT DO NOTHING RETURNING version", &[&tenant(t),&kind(path),&path.as_str(),&&body[..]]).await.map_err(err)?, + MetadataWriteCondition::Update(v) => { let v = v.version.and_then(|v| v.parse::().ok()).ok_or(ConditionalMetadataError::Conflict)?; + c.query_opt("UPDATE parseable_documents SET body=$3,version=nextval('parseable_document_version'),updated_at=now() WHERE tenant=$1 AND key=$2 AND version=$4 RETURNING version", &[&tenant(t),&path.as_str(),&&body[..],&v]).await.map_err(err)? } + }; + row.map(|r| version(r.get(0))) + .ok_or(ConditionalMetadataError::Conflict) + } + pub async fn delete_metadata_conditional( + &self, + path: &RelativePath, + v: MetadataVersion, + t: &Option, + ) -> Result<(), ConditionalMetadataError> { + let v = v + .version + .and_then(|v| v.parse::().ok()) + .ok_or(ConditionalMetadataError::Conflict)?; + let count = self + .client() + .await? + .execute( + "DELETE FROM parseable_documents WHERE tenant=$1 AND key=$2 AND version=$3", + &[&tenant(t), &path.as_str(), &v], + ) + .await + .map_err(err)?; + if count == 0 { + Err(ConditionalMetadataError::Conflict) + } else { + Ok(()) + } + } + pub async fn delete_object( + &self, + path: &RelativePath, + t: &Option, + ) -> Result<(), ObjectStorageError> { + self.client() + .await? + .execute( + "DELETE FROM parseable_documents WHERE tenant=$1 AND key=$2", + &[&tenant(t), &path.as_str()], + ) + .await + .map_err(err)?; + Ok(()) + } + async fn rows( + &self, + prefix: &str, + t: Option<&str>, + ) -> Result, ObjectStorageError> { + let prefix = prefix.trim_matches('/'); + Ok(self.client().await?.query("SELECT key,body FROM parseable_documents WHERE tenant=$2 AND ($1='' OR key=$1 OR (key >= $1 || '/' AND key < $1 || '0')) ORDER BY key", &[&prefix,&t.unwrap_or("")]).await?.into_iter().map(|r| (r.get(0),Bytes::from(r.get::<_,Vec>(1)))).collect()) + } + async fn keys(&self, prefix: &str, t: &str) -> Result, ObjectStorageError> { + Ok(self.client().await?.query("SELECT key FROM parseable_documents WHERE tenant=$1 AND ($2='' OR key=$2 OR (key >= $2 || '/' AND key < $2 || '0')) ORDER BY key", &[&t,&prefix]).await?.into_iter().map(|r|r.get(0)).collect()) + } + pub async fn list_streams( + &self, + t: &Option, + ) -> Result, ObjectStorageError> { + let prefix = if tenant(t).is_empty() { + String::new() + } else { + format!("{}/", tenant(t)) + }; + let keys = self + .client() + .await? + .query( + "SELECT key FROM parseable_documents WHERE tenant=$1 AND key LIKE '%/.stream/%'", + &[&tenant(t)], + ) + .await?; + let keys: std::collections::HashSet = keys.into_iter().map(|r| r.get(0)).collect(); + Ok(keys + .iter() + .filter(|k| k.ends_with("stream.json")) + .filter_map(|k| { + k.strip_prefix(&prefix)? + .split_once('/') + .map(|(s, _)| s.to_owned()) + }) + .filter(|s| !keys.contains(&format!("{prefix}{s}/.stream/.deleting"))) + .collect()) + } + pub async fn get_objects( + &self, + path: Option<&RelativePath>, + filter: Box bool + Send>, + t: &Option, + ) -> Result, ObjectStorageError> { + // Node registry objects are deployment-global; tenant is metrics context + // for this existing trait API, and is not part of its path. + let root = path.map(|p| p.as_str()).unwrap_or(""); + let scope = if root == ".parseable" { "" } else { tenant(t) }; + Ok(self + .rows(root, Some(scope)) + .await? + .into_iter() + .filter(|(p, _)| filter(p.rsplit('/').next().unwrap().to_owned())) + .map(|(_, b)| b) + .collect()) + } + pub async fn delete_prefix( + &self, + path: &RelativePath, + _: &Option, + ) -> Result<(), ObjectStorageError> { + if path.as_str().is_empty() || path.as_str().contains('/') { + return Err(err("tenant deletion requires one nonempty path component")); + } + if self.client().await?.query_one( + "SELECT EXISTS(SELECT 1 FROM parseable_documents WHERE tenant='' AND (key >= $1 || '/' AND key < $1 || '0') AND (key LIKE '%/.stream/%' OR key LIKE '%/.stream.json' OR key LIKE '%/.schema'))", + &[&path.as_str()], + ).await?.get::<_, bool>(0) { + return Err(err(format!("tenant {} collides with a default-tenant stream; refusing data deletion", path))); + } + self.delete_data(path, &None).await?; + self.client() + .await? + .execute( + "DELETE FROM parseable_documents WHERE tenant=$1", + &[&path.as_str()], + ) + .await + .map_err(err)?; + Ok(()) + } + pub async fn list_dirs_relative( + &self, + path: &RelativePath, + t: &Option, + ) -> Result, ObjectStorageError> { + let prefix = format!("{}/", path.as_str().trim_end_matches('/')); + Ok(self + .keys(path.as_str(), tenant(t)) + .await? + .into_iter() + .filter_map(|p| { + p.strip_prefix(&prefix) + .and_then(|p| p.split_once('/')) + .map(|(p, _)| p.to_owned()) + }) + .collect::>() + .into_iter() + .collect()) + } + pub async fn list_with_delimiter( + &self, + path: Option, + t: &Option, + ) -> Result { + let base = path.as_ref().map(|p| p.as_ref()).unwrap_or(""); + let prefix = if base.is_empty() { + String::new() + } else { + format!("{base}/") + }; + let mut dirs = std::collections::BTreeSet::new(); + let mut objects = Vec::new(); + for key in self.keys(base, tenant(t)).await? { + let Some(suffix) = key.strip_prefix(&prefix) else { + continue; + }; + if let Some((dir, _)) = suffix.split_once('/') { + dirs.insert( + object_store::path::Path::parse(format!("{prefix}{dir}")).map_err(err)?, + ); + } else { + objects.push(object_store::ObjectMeta { + location: object_store::path::Path::parse(key).map_err(err)?, + last_modified: chrono::Utc::now(), + size: 0, + e_tag: None, + version: None, + }); + } + } + Ok(object_store::ListResult { + common_prefixes: dirs.into_iter().collect(), + objects, + }) + } + pub async fn schema_objects( + &self, + stream: &str, + t: &Option, + ) -> Result, ObjectStorageError> { + let prefix = format!( + "{}{stream}/.stream", + if tenant(t).is_empty() { + String::new() + } else { + format!("{}/", tenant(t)) + } + ); + Ok(self + .rows(&prefix, Some(tenant(t))) + .await? + .into_iter() + .filter_map(|(p, b)| { + let name = p.rsplit('/').next()?.to_owned(); + name.contains(".schema").then_some((name, b)) + }) + .collect()) + } +} +pub(crate) fn kind(path: &RelativePath) -> &'static str { + let name = path.file_name().unwrap_or(""); + if name.ends_with("manifest.json") { + "manifest" + } else if name.ends_with("stream.json") { + "stream" + } else if name.contains(".schema") { + "schema" + } else { + "document" + } +} + +#[cfg(test)] +impl PostgresDocuments { + pub async fn verify_partition_contract(&self) { + // The SQL trigger fails precisely after data deletion, before row deletion. + let tenants = [ + Some("partition-a".to_owned()), + Some("partition-b".to_owned()), + ]; + let dates = ["date=2026-01-01", "date=2026-01-02"]; + for t in &tenants { + for date in dates { + let prefix = format!("{}/logs/{date}", t.as_ref().unwrap()); + self.put_object( + RelativePath::new(&format!("{prefix}/manifest.json")), + Bytes::from(prefix.clone()), + t, + ) + .await + .unwrap(); + self.storage + .put_object( + RelativePath::new(&format!("{prefix}/data.parquet")), + Bytes::from(prefix), + t, + ) + .await + .unwrap(); + } + } + let client = self.client().await.unwrap(); + client.batch_execute("CREATE FUNCTION fail_partition_delete() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF OLD.key = 'partition-a/logs/date=2026-01-01/manifest.json' THEN RAISE EXCEPTION 'injected partition row failure'; END IF; RETURN OLD; END $$; CREATE TRIGGER partition_delete_fault BEFORE DELETE ON parseable_documents FOR EACH ROW EXECUTE FUNCTION fail_partition_delete();").await.unwrap(); + let path = RelativePath::new("partition-a/logs/date=2026-01-01"); + let error = self.delete_partition(path, &tenants[0]).await.unwrap_err(); + assert!(error.to_string().contains("PostgreSQL"), "{error}"); + assert!( + self.storage + .get_object( + RelativePath::new("partition-a/logs/date=2026-01-01/data.parquet"), + &tenants[0] + ) + .await + .is_err() + ); + assert!( + self.get_object( + RelativePath::new("partition-a/logs/date=2026-01-01/manifest.json"), + &tenants[0] + ) + .await + .is_ok() + ); + client.batch_execute("DROP TRIGGER partition_delete_fault ON parseable_documents; DROP FUNCTION fail_partition_delete();").await.unwrap(); + drop(client); + self.delete_partition(path, &tenants[0]).await.unwrap(); + self.delete_partition(path, &tenants[0]).await.unwrap(); + for (i, t) in tenants.iter().enumerate() { + for (j, date) in dates.iter().enumerate() { + let prefix = format!("{}/logs/{date}", t.as_ref().unwrap()); + let manifest = self + .get_object(RelativePath::new(&format!("{prefix}/manifest.json")), t) + .await; + let data = self + .storage + .get_object(RelativePath::new(&format!("{prefix}/data.parquet")), t) + .await; + if i == 0 && j == 0 { + assert!(manifest.is_err()); + assert!(data.is_err()); + } else { + assert_eq!(manifest.unwrap(), Bytes::from(prefix.clone())); + assert_eq!(data.unwrap(), Bytes::from(prefix)); + } + } + self.delete_prefix(RelativePath::new(t.as_ref().unwrap()), t) + .await + .unwrap(); + } + } + pub async fn verify_deleted_stream_rows(&self) { + let count: i64 = self.client().await.unwrap().query_one("SELECT count(*) FROM parseable_documents WHERE starts_with(key,'contractlogs/') OR starts_with(key,'tenant-contract/contractlogs/')", &[]).await.unwrap().get(0); + assert_eq!(count, 0); + } + pub async fn verify_database_contract(&self, options: &PostgresOptions) { + for fault in [1, 2] { + let t = Some("failure-tenant".to_owned()); + let key = RelativePath::new("failure-tenant/retry/.stream/.stream.json"); + self.put_object(key, Bytes::from_static(b"{}"), &t) + .await + .unwrap(); + self.put_object( + RelativePath::new("retry/.stream/.stream.json"), + Bytes::from_static(b"other tenant"), + &None, + ) + .await + .unwrap(); + let data = RelativePath::new("failure-tenant/retry/date=2026-01-01/data.parquet"); + self.storage + .put_object(data, Bytes::from_static(b"data"), &t) + .await + .unwrap(); + self.deletion_fault + .store(fault, std::sync::atomic::Ordering::SeqCst); + assert!(self.delete_stream("retry", &t).await.is_err()); + assert!(self.is_stream_deleting("retry", &t).await.unwrap()); + assert_eq!(self.storage.get_object(data, &t).await.is_ok(), fault == 1); + let restarted = + Arc::new(Self::new(options, self.bucket.clone(), self.storage.clone()).unwrap()); + restarted.check(&None).await.unwrap(); + let restarted_store = super::postgres_metastore::PostgresMetastore { + storage: restarted.clone(), + }; + assert!( + !crate::metastore::metastore_traits::Metastore::list_streams(&restarted_store, &t) + .await + .unwrap() + .contains("retry") + ); + assert!(restarted.is_stream_deleting("retry", &t).await.unwrap()); + restarted.delete_stream("retry", &t).await.unwrap(); + assert!( + restarted + .rows("failure-tenant/retry", Some("failure-tenant")) + .await + .unwrap() + .is_empty() + ); + restarted + .put_object(key, Bytes::from_static(b"recreated"), &t) + .await + .unwrap(); + assert_eq!( + restarted.get_object(key, &t).await.unwrap(), + Bytes::from_static(b"recreated") + ); + self.deletion_fault + .store(0, std::sync::atomic::Ordering::SeqCst); + self.delete_stream("retry", &t).await.unwrap(); + assert!(!self.stream_exists("retry", &t).await.unwrap()); + assert!(!self.is_stream_deleting("retry", &t).await.unwrap()); + assert!(self.stream_exists("retry", &None).await.unwrap()); + self.delete_stream("retry", &None).await.unwrap(); + } + // Distinct raw tenant keys coexist, including SQL-only and bucket-only + // candidates. Enumeration order is unspecified: compare sets. + let baseline: std::collections::BTreeSet<_> = self + .list_tenant_candidates() + .await + .unwrap() + .into_iter() + .collect(); + for name in ["discovery-sql", "discovery-duplicate", "discovery-invalid"] { + self.put_object( + RelativePath::new(&format!("{name}/.parseable/.parseable.json")), + Bytes::from_static(b"{}"), + &Some(name.into()), + ) + .await + .unwrap(); + } + for name in ["discovery-data", "discovery-duplicate", ".discovery-hidden"] { + self.storage + .put_object( + RelativePath::new(&format!("{name}/data.parquet")), + Bytes::from_static(b"data"), + &None, + ) + .await + .unwrap(); + } + let candidates: std::collections::BTreeSet<_> = self + .list_tenant_candidates() + .await + .unwrap() + .into_iter() + .collect(); + assert_eq!( + candidates + .difference(&baseline) + .cloned() + .collect::>(), + [ + "discovery-sql", + "discovery-data", + "discovery-duplicate", + "discovery-invalid" + ] + .into_iter() + .map(str::to_owned) + .collect() + ); + for name in [ + "discovery-sql", + "discovery-data", + "discovery-duplicate", + "discovery-invalid", + ".discovery-hidden", + ] { + self.delete_prefix(RelativePath::new(name), &None) + .await + .unwrap(); + } + let timed = Self::new( + &PostgresOptions { + metastore_postgres_connect_timeout_ms: 1000, + metastore_postgres_statement_timeout_ms: 3000, + ..options.clone() + }, + self.bucket.clone(), + self.storage.clone(), + ) + .unwrap(); + let client = timed.client().await.unwrap(); + let started = std::time::Instant::now(); + assert!(client.query("SELECT pg_sleep(10)", &[]).await.is_err()); + assert!(started.elapsed() < Duration::from_secs(5)); + assert!(client.query("SELECT 1", &[]).await.is_ok()); + drop(client); + // Pool saturation is bounded and returned as an error. + let held = futures::future::join_all((0..4).map(|_| timed.client())).await; + assert!(held.iter().all(Result::is_ok)); + let started = std::time::Instant::now(); + assert!(timed.client().await.is_err()); + assert!(started.elapsed() < Duration::from_secs(3)); + drop(held); + let marker = RelativePath::new(".parseable/postgres-metastore-id"); + let paired = self.storage.get_object(marker, &None).await.unwrap(); + self.storage.delete_object(marker, &None).await.unwrap(); + let started = std::time::Instant::now(); + assert!(timed.check_health(&None).await.is_err()); + assert!(started.elapsed() < Duration::from_secs(3)); + self.storage + .put_object(marker, Bytes::from_static(b"wrong-backend"), &None) + .await + .unwrap(); + let partial = Self::new(options, self.bucket.clone(), self.storage.clone()).unwrap(); + let error = partial.check(&None).await.unwrap_err(); + assert!( + error + .to_string() + .contains("different PostgreSQL backend UUID"), + "{error}" + ); + self.storage + .put_object(marker, paired, &None) + .await + .unwrap(); + partial.check(&None).await.unwrap(); + partial.check_health(&None).await.unwrap(); + let path = RelativePath::new("contract/concurrent.json"); + self.put_object(path, Bytes::from_static(b"initial"), &None) + .await + .unwrap(); + let original = self + .get_metadata_versioned(path, &None) + .await + .unwrap() + .unwrap(); + let writes = (0..12).map(|i| { + self.put_metadata_conditional( + path, + Bytes::from(i.to_string()), + MetadataWriteCondition::Update(original.version.clone()), + &None, + ) + }); + let results = futures::future::join_all(writes).await; + assert_eq!(results.iter().filter(|r| r.is_ok()).count(), 1); + let before_delete = self + .get_metadata_versioned(path, &None) + .await + .unwrap() + .unwrap() + .version; + self.delete_object(path, &None).await.unwrap(); + self.put_object(path, Bytes::new(), &None).await.unwrap(); + let recreated = self + .get_metadata_versioned(path, &None) + .await + .unwrap() + .unwrap() + .version; + assert!( + recreated.version.unwrap().parse::().unwrap() + > before_delete.version.unwrap().parse::().unwrap() + ); + self.delete_object(path, &None).await.unwrap(); + } +} + +impl PostgresDocuments { + pub(crate) async fn migrate( + &self, + store: Arc, + opts: &super::migration::MigrationOptions, + keys: &[super::migration::InventoryDocument], + transfer_lock: &mut super::postgres_client::Client, + ) -> anyhow::Result<()> { + use super::migration::{Action, BUSY, PAIR, digest, optional}; + use anyhow::ensure; + let action = opts.metadata_migration.unwrap(); + let mut client = self.migration_client().await?; + // All remote I/O and validation happen before the import transaction. + let tx = &client; + if action == Action::Verify { + let exists: bool = tx + .query_one("SELECT to_regclass('parseable_documents') IS NOT NULL", &[]) + .await? + .get(0); + let bucket_keys: std::collections::BTreeSet<_> = keys + .iter() + .map(|(k, (t, _), _)| (t.as_str(), super::migration::canonical_key(k))) + .collect(); + let mut differences = 0; + for (key, (tenant, kind), _) in keys { + let bucket = super::migration::import_body( + store.as_ref(), + key, + &store.get_object(RelativePath::new(key), &None).await?, + )?; + let sql = if exists { + tx.query_opt( + "SELECT body FROM parseable_documents WHERE tenant=$1 AND key=$2", + &[tenant, &super::migration::canonical_key(key)], + ) + .await? + .map(|r| r.get::<_, Vec>(0)) + } else { + None + }; + if sql.as_deref() != Some(bucket.as_ref()) { + println!( + "DIFFERENT tenant={tenant:?} kind={kind} key={key} bucket={} sql={}", + digest(&bucket), + sql.as_ref() + .map(|b| digest(b)) + .unwrap_or_else(|| "missing".into()) + ); + differences += 1; + } + } + if exists { + let mut last = (String::new(), String::new()); + loop { + let rows = tx.query("SELECT tenant,key,kind FROM parseable_documents WHERE (tenant COLLATE \"C\",key COLLATE \"C\") > ($1,$2) ORDER BY tenant COLLATE \"C\",key COLLATE \"C\" LIMIT 128", &[&last.0,&last.1]).await?; + if rows.is_empty() { + break; + } + for row in rows { + last = (row.get(0), row.get(1)); + if !bucket_keys.contains(&(last.0.as_str(), last.1.as_str())) { + println!( + "SQL ONLY tenant={:?} kind={} key={}", + last.0, + row.get::<_, String>(2), + last.1 + ); + differences += 1; + } + } + } + } + ensure!(differences == 0, "{differences} metadata differences"); + transfer_lock.batch_execute("SELECT 1").await?; + println!("representations match"); + return Ok(()); + } + let prior_busy = optional(store.as_ref(), BUSY).await?; + let initial_id = prior_busy + .as_ref() + .and_then(|b| std::str::from_utf8(b).ok()) + .and_then(|s| s.strip_prefix("Import:")) + .map(str::to_owned) + .unwrap_or_else(|| uuid::Uuid::new_v4().to_string()); + let bootstrap_tx = client.transaction().await?; + bootstrap_tx + .query("SELECT pg_advisory_xact_lock(8104050183641399916)", &[]) + .await?; + let (id, _, _) = self.bootstrap(&bootstrap_tx, &initial_id).await?; + bootstrap_tx.commit().await?; + let tx = &client; + let state = format!("{action:?}:{id}"); + let busy = optional(store.as_ref(), BUSY).await?; + ensure!( + busy.as_ref().is_none_or(|b| b.as_ref() == state.as_bytes()), + "another transfer owns the bucket migration marker" + ); + let pair = optional(store.as_ref(), PAIR).await?; + ensure!( + pair.as_ref().is_none_or(|b| b.as_ref() == id.as_bytes()), + "bucket paired with another database" + ); + let count: i64 = tx + .query_one("SELECT count(*) FROM parseable_documents", &[]) + .await? + .get(0); + if action == Action::Import { + ensure!( + count == 0 || busy.is_some(), + "refusing import into nonempty database" + ); + if count != 0 { + ensure!( + count as usize == keys.len(), + "resumed import inventory changed" + ); + } + } else { + ensure!( + pair.is_some() || busy.is_some(), + "export requires a paired bucket or interrupted export" + ); + self.export_preflight(&client, store.as_ref(), opts, keys) + .await?; + } + transfer_lock.batch_execute("SELECT 1").await?; + store + .put_metadata_conditional( + RelativePath::new(BUSY), + Bytes::from(state.clone()), + MetadataWriteCondition::Create, + &None, + ) + .await + .or_else(|e| { + if matches!(e, ConditionalMetadataError::Conflict) && busy.is_some() { + Ok(version(0)) + } else { + Err(e) + } + })?; + super::migration::failure_point(opts, "marker")?; + if action == Action::Import { + drop(client); + let spool_root = + std::env::current_dir()?.join(format!(".parseable-migration-spool-{id}")); + // The session transfer lock proves no other invocation uses this directory. + if spool_root.exists() { + std::fs::remove_dir_all(&spool_root)?; + } + std::fs::create_dir(&spool_root)?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&spool_root, std::fs::Permissions::from_mode(0o700))?; + } + let spool = tempfile::tempdir_in(&spool_root)?; + use futures::{StreamExt, TryStreamExt}; + futures::stream::iter(keys.iter().enumerate()) + .map(|(index, (key, _, _))| { + let store = &store; + let path = spool.path().join(index.to_string()); + async move { + let body = store.get_object(RelativePath::new(key), &None).await?; + std::fs::write(path, &body)?; + Ok::<_, anyhow::Error>(()) + } + }) + .buffer_unordered(16) + .try_collect::>() + .await?; + super::migration::failure_point(opts, "spool")?; + let current = store.metadata_inventory().await?; + let current: Vec<_> = current + .into_iter() + .filter(|(k, _)| super::migration::document_scope(k).is_some()) + .collect(); + ensure!( + current.len() == keys.len() + && current + .iter() + .zip(keys) + .all(|((k, m), (old, _, om))| k == old && m == om), + "bucket metadata changed during import" + ); + #[cfg(test)] + if opts.injected_failure == Some("lock-loss") { + let pid: i32 = transfer_lock + .query_one("SELECT pg_backend_pid()", &[]) + .await? + .get(0); + self.client() + .await? + .query_one("SELECT pg_terminate_backend($1,1000)", &[&pid]) + .await?; + } + // Commit on the lock-owning session: a lost lock connection must + // fail this transfer, rather than reconnecting and committing stale work. + let tx = transfer_lock.transaction().await?; + tx.query("SELECT pg_advisory_xact_lock(8104050183641399916)", &[]) + .await?; + tx.batch_execute("CREATE TABLE IF NOT EXISTS parseable_import_digests (key text PRIMARY KEY, digest text NOT NULL)").await?; + for (index, batch) in keys.chunks(32).enumerate() { + let mut tenants = Vec::new(); + let mut kinds = Vec::new(); + let mut paths = Vec::new(); + let mut bodies = Vec::new(); + let mut source_digests = Vec::new(); + for (key, (tenant, kind), _) in batch { + let body = Bytes::from(std::fs::read( + spool.path().join((index * 32 + bodies.len()).to_string()), + )?); + source_digests.push(digest(&body)); + let body = super::migration::import_body(store.as_ref(), key, &body)?; + tenants.push(tenant.clone()); + kinds.push(kind.to_string()); + paths.push(super::migration::canonical_key(key).to_owned()); + bodies.push(body.to_vec()); + } + if count == 0 { + tx.execute("INSERT INTO parseable_documents(tenant,kind,key,body) SELECT * FROM unnest($1::text[],$2::text[],$3::text[],$4::bytea[])", &[&tenants,&kinds,&paths,&bodies]).await?; + } + for (row_index, (((key, tenant), body), (source_key, _, _))) in paths + .iter() + .zip(&tenants) + .zip(&bodies) + .zip(batch) + .enumerate() + { + tx.execute("INSERT INTO parseable_import_digests(key,digest) VALUES($1,$2) ON CONFLICT DO NOTHING", &[source_key, &source_digests[row_index]]).await?; + let reread: Vec = tx + .query_one( + "SELECT body FROM parseable_documents WHERE tenant=$1 AND key=$2", + &[tenant, key], + ) + .await? + .get(0); + ensure!(digest(body) == digest(&reread), "SQL digest differs: {key}"); + } + eprintln!( + "verified import {}/{}", + ((index + 1) * 32).min(keys.len()), + keys.len() + ); + } + tx.execute("INSERT INTO parseable_transfer(state) VALUES('ready') ON CONFLICT(singleton) DO UPDATE SET state='ready'", &[]).await?; + super::migration::failure_point(opts, "import-before-commit")?; + tx.commit().await?; + super::migration::failure_point(opts, "import-commit")?; + transfer_lock.batch_execute("SELECT 1").await?; + store + .put_metadata_conditional( + RelativePath::new(PAIR), + Bytes::from(id.clone()), + MetadataWriteCondition::Create, + &None, + ) + .await + .or_else(|e| { + if matches!(e, ConditionalMetadataError::Conflict) { + Ok(version(0)) + } else { + Err(e) + } + })?; + ensure!( + optional(store.as_ref(), PAIR).await?.unwrap().as_ref() == id.as_bytes(), + "pair mismatch" + ); + super::migration::failure_point(opts, "paired")?; + } else { + drop(client); + let client = &*transfer_lock; + client.execute("INSERT INTO parseable_transfer(state) VALUES('exported') ON CONFLICT(singleton) DO UPDATE SET state='exported'", &[]).await?; + super::migration::failure_point(opts, "export-state")?; + let mut last = (String::new(), String::new()); + loop { + let rows = client.query("SELECT tenant,key,body FROM parseable_documents WHERE (tenant COLLATE \"C\",key COLLATE \"C\") > ($1,$2) ORDER BY tenant COLLATE \"C\",key COLLATE \"C\" LIMIT 32", &[&last.0,&last.1]).await?; + if rows.is_empty() { + break; + } + for row in rows { + last = (row.get(0), row.get(1)); + let body = Bytes::from(row.get::<_, Vec>(2)); + client.batch_execute("SELECT 1").await?; + store + .put_object(RelativePath::new(&last.1), body.clone(), &None) + .await?; + ensure!( + store.get_object(RelativePath::new(&last.1), &None).await? == body, + "export verification failed: {}", + last.1 + ); + super::migration::failure_point(opts, "object-write")?; + eprintln!("verified export {}", last.1); + } + } + for (key, (tenant, _), _) in keys { + if client + .query_opt( + "SELECT 1 FROM parseable_documents WHERE tenant=$1 AND key=$2", + &[tenant, key], + ) + .await? + .is_none() + { + store.delete_object(RelativePath::new(key), &None).await?; + super::migration::failure_point(opts, "bucket-delete")?; + } + } + // Validate stream references only after all destination documents exist. + let mut last_key = String::new(); + loop { + let rows = client.query("SELECT key,body FROM parseable_documents WHERE key COLLATE \"C\" > $1 ORDER BY key COLLATE \"C\" LIMIT 32", &[&last_key]).await?; + if rows.is_empty() { + break; + } + for row in rows { + last_key = row.get(0); + let result = super::migration::validate( + store.as_ref(), + &last_key, + &Bytes::from(row.get::<_, Vec>(1)), + ) + .await; + match result { + Err(e) + if opts.allow_dangling_snapshot_references + && e.to_string().contains("missing manifest") => + { + eprintln!("OVERRIDE {e:#}") + } + result => result?, + } + } + } + if optional(store.as_ref(), PAIR).await?.is_some() { + client.batch_execute("SELECT 1").await?; + store.delete_object(RelativePath::new(PAIR), &None).await?; + } + super::migration::failure_point(opts, "pair-removed")?; + } + transfer_lock.batch_execute("SELECT 1").await?; + store.delete_object(RelativePath::new(BUSY), &None).await?; + Ok(()) + } +} + +impl PostgresDocuments { + pub(crate) async fn export_preflight( + &self, + client: &super::postgres_client::Client, + store: &dyn ObjectStorage, + opts: &super::migration::MigrationOptions, + keys: &[super::migration::InventoryDocument], + ) -> anyhow::Result<()> { + use super::migration::{digest, optional}; + let mut conflicts = 0usize; + let resuming_export = optional(store, super::migration::BUSY) + .await? + .is_some_and(|marker| marker.starts_with(b"Export:")) + && client + .query_opt("SELECT state FROM parseable_transfer WHERE singleton", &[]) + .await? + .is_some_and(|row| row.get::<_, String>(0) == "exported"); + let has_digests: bool = client + .query_one( + "SELECT to_regclass('parseable_import_digests') IS NOT NULL", + &[], + ) + .await? + .get(0); + if has_digests && !resuming_export { + let mut last_key = String::new(); + loop { + let rows = client.query("SELECT key FROM parseable_import_digests WHERE key COLLATE \"C\" > $1 ORDER BY key COLLATE \"C\" LIMIT 128", &[&last_key]).await?; + if rows.is_empty() { + break; + } + for row in rows { + last_key = row.get(0); + let canonical = super::migration::canonical_key(&last_key); + if canonical != last_key + && optional(store, &last_key).await?.is_none() + && client + .query_opt( + "SELECT 1 FROM parseable_documents WHERE key=$1", + &[&canonical], + ) + .await? + .is_some() + { + conflicts += 1; + if conflicts <= 100 { + eprintln!("CONFLICT deleted import alias key={last_key}"); + } + } + } + } + } + // Preflight every destination before any overwrite. + let mut last = (String::new(), String::new()); + loop { + let rows = client.query("SELECT tenant,key,body FROM parseable_documents WHERE (tenant COLLATE \"C\",key COLLATE \"C\") > ($1,$2) ORDER BY tenant COLLATE \"C\",key COLLATE \"C\" LIMIT 32", &[&last.0,&last.1]).await?; + if rows.is_empty() { + break; + } + for row in rows { + last = (row.get(0), row.get(1)); + let body: Vec = row.get(2); + self.validate_export_row( + client, + &last.1, + &Bytes::from(body.clone()), + opts.allow_dangling_snapshot_references, + ) + .await?; + let old = optional(store, &last.1).await?; + let baseline = self.import_digest(client, &last.1).await?; + let unchanged = match old { + Some(old) => { + old.as_ref() == body || baseline.as_deref() == Some(digest(&old).as_str()) + } + None => baseline.is_none(), + }; + if !unchanged { + conflicts += 1; + if conflicts <= 100 { + eprintln!("CONFLICT key={}", last.1); + } + } + } + } + // Bucket-only metadata would resurrect deleted SQL objects on downgrade. + for (key, (tenant, _), _) in keys { + if client + .query_opt( + "SELECT 1 FROM parseable_documents WHERE tenant=$1 AND key=$2", + &[tenant, key], + ) + .await? + .is_none() + && self.import_digest(client, key).await?.as_deref() + != Some( + digest(&store.get_object(RelativePath::new(key), &None).await?).as_str(), + ) + { + conflicts += 1; + if conflicts <= 100 { + eprintln!("CONFLICT bucket-only key={key}"); + } + } + } + anyhow::ensure!( + conflicts == 0 || opts.overwrite_bucket_changes_since_import, + "{conflicts} bucket conflicts (first 100 shown); after backup use --overwrite-bucket-changes-since-import" + ); + Ok(()) + } + + pub(crate) async fn migration_preflight( + &self, + store: &dyn ObjectStorage, + opts: &super::migration::MigrationOptions, + ) -> anyhow::Result<()> { + use super::migration::{Action, BUSY, PAIR, optional}; + use anyhow::ensure; + let client = self.client().await?; + let busy = optional(store, BUSY).await?; + let pair = optional(store, PAIR).await?; + let exists: bool = client + .query_one("SELECT to_regclass('parseable_backend') IS NOT NULL", &[]) + .await? + .get(0); + if !exists { + ensure!( + self.expected_id.is_none(), + "pinned backend id requires an initialized database" + ); + ensure!( + busy.is_none(), + "migration marker requires its original initialized database" + ); + ensure!( + pair.is_none(), + "paired bucket requires its original database" + ); + ensure!( + opts.metadata_migration != Some(Action::Export), + "export requires an imported database" + ); + return Ok(()); + } + let row = client + .query_one( + "SELECT id,bucket,schema_version,protocol_version FROM parseable_backend", + &[], + ) + .await?; + let id: String = row.get(0); + ensure!( + row.get::<_, String>(1) == self.bucket + && (0..=3).contains(&row.get::<_, i32>(2)) + && row.get::<_, i32>(3) == 1, + "backend identity/version mismatch" + ); + ensure!( + self.expected_id.as_ref().is_none_or(|v| v == &id), + "pinned backend mismatch" + ); + ensure!( + pair.as_ref().is_none_or(|v| v.as_ref() == id.as_bytes()), + "bucket paired with another database" + ); + ensure!( + busy.as_ref() + .is_none_or(|v| v.as_ref() + == format!("{:?}:{id}", opts.metadata_migration.unwrap()).as_bytes()), + "another transfer owns migration marker" + ); + if opts.metadata_migration == Some(Action::Export) { + ensure!( + pair.is_some() || busy.is_some(), + "export requires paired bucket or interrupted export" + ); + } + let docs: bool = client + .query_one("SELECT to_regclass('parseable_documents') IS NOT NULL", &[]) + .await? + .get(0); + if docs && opts.metadata_migration == Some(Action::Import) && !opts.abort { + let count: i64 = client + .query_one("SELECT count(*) FROM parseable_documents", &[]) + .await? + .get(0); + ensure!( + count == 0 || busy.is_some(), + "refusing import into nonempty database" + ); + } + if opts.metadata_migration == Some(Action::Import) && !opts.migration_dry_run { + // Caller already holds the transfer lock and identity checks passed. + uuid::Uuid::parse_str(&id)?; + let stale = std::env::current_dir()?.join(format!(".parseable-migration-spool-{id}")); + if stale.exists() { + std::fs::remove_dir_all(stale)?; + } + } + Ok(()) + } + + async fn import_digest( + &self, + client: &super::postgres_client::Client, + key: &str, + ) -> anyhow::Result> { + let exists: bool = client + .query_one( + "SELECT to_regclass('parseable_import_digests') IS NOT NULL", + &[], + ) + .await? + .get(0); + if !exists { + return Ok(None); + } + Ok(client + .query_opt( + "SELECT digest FROM parseable_import_digests WHERE key=$1", + &[&key], + ) + .await? + .map(|r| r.get(0))) + } + + async fn validate_export_row( + &self, + client: &super::postgres_client::Client, + key: &str, + body: &Bytes, + allow_dangling: bool, + ) -> anyhow::Result<()> { + use anyhow::ensure; + ensure!( + !key.ends_with(".deleting") && !key.contains(".tombstones/"), + "unfinished deletion: {key}; start PostgreSQL mode and retry DELETE before export" + ); + ensure!( + super::migration::document_scope(key).is_some(), + "unknown SQL document: {key}" + ); + if key.ends_with("manifest.json") { + crate::catalog::manifest::decode_manifest(body)?; + } else if key.ends_with("stream.json") { + for manifest_path in super::migration::stream_references(body)? { + let base = self.storage.absolute_url(RelativePath::new("")); + let keys = read_keys(&manifest_path, base.as_ref()); + let row = client.query_opt( + "SELECT body FROM parseable_documents WHERE key=ANY($1) ORDER BY array_position($1,key) LIMIT 1", + &[&keys], + ).await?; + let Some(row) = row else { + if allow_dangling { + eprintln!("OVERRIDE {key}: missing SQL manifest {manifest_path}"); + continue; + } + anyhow::bail!("{key}: missing SQL manifest {manifest_path}"); + }; + crate::catalog::manifest::decode_manifest(&Bytes::from(row.get::<_, Vec>(0)))?; + } + } else if key.contains(".schema") { + serde_json::from_slice::(body)?; + } else { + serde_json::from_slice::(body)?; + } + Ok(()) + } + + pub(crate) async fn abort_migration( + &self, + store: &dyn ObjectStorage, + action: super::migration::Action, + client: &super::postgres_client::Client, + ) -> anyhow::Result<()> { + use super::migration::{Action, BUSY, optional}; + use anyhow::ensure; + let exists: bool = client + .query_one("SELECT to_regclass('parseable_transfer') IS NOT NULL", &[]) + .await? + .get(0); + let state = if exists { + client + .query_opt("SELECT state FROM parseable_transfer WHERE singleton", &[]) + .await? + .map(|r| r.get::<_, String>(0)) + } else { + None + }; + match action { + Action::Import => { + let docs: bool = client + .query_one("SELECT to_regclass('parseable_documents') IS NOT NULL", &[]) + .await? + .get(0); + let pair = optional(store, super::migration::PAIR).await?; + let busy = optional(store, BUSY).await?; + let id = if docs { + Some( + client + .query_one("SELECT id FROM parseable_backend", &[]) + .await? + .get::<_, String>(0), + ) + } else { + None + }; + let unpaired_commit = state.as_deref() == Some("ready") + && pair.is_none() + && id.as_ref().is_some_and(|id| { + busy.as_ref() + .is_some_and(|b| b.as_ref() == format!("Import:{id}").as_bytes()) + }); + if unpaired_commit { + client.batch_execute("BEGIN; DELETE FROM parseable_documents; DELETE FROM parseable_import_digests; DELETE FROM parseable_transfer; COMMIT;").await?; + } else { + let count: i64 = if docs { + client + .query_one("SELECT count(*) FROM parseable_documents", &[]) + .await? + .get(0) + } else { + 0 + }; + ensure!( + count == 0 && state.is_none() && pair.is_none(), + "import committed or paired; complete import then export instead of aborting" + ); + } + } + Action::Export => { + ensure!( + state.is_none() || state.as_deref() == Some("ready"), + "export has begun writing; rerun export to finish; abort is unsafe" + ); + } + Action::Verify => anyhow::bail!("verify cannot be aborted"), + } + if optional(store, BUSY).await?.is_some() { + client.batch_execute("SELECT 1").await?; + store.delete_object(RelativePath::new(BUSY), &None).await?; + } + println!( + "aborted safely; start {} mode", + if action == Action::Import { + "object-store" + } else { + "PostgreSQL" + } + ); + Ok(()) + } +} diff --git a/src/metastore/metastores/postgres_metastore.rs b/src/metastore/metastores/postgres_metastore.rs new file mode 100644 index 000000000..c85287530 --- /dev/null +++ b/src/metastore/metastores/postgres_metastore.rs @@ -0,0 +1,1773 @@ +super::document_metastore::document_metastore!( + PostgresMetastore, + super::postgres_documents::PostgresDocuments, + false, + { + async fn manifest_batch( + &self, + paths: &[String], + tenant: &Option, + ) -> Result>>, MetastoreError> { + let mut result: HashMap> = + paths.iter().map(|p| (p.clone(), None)).collect(); + let chunks = self.storage.manifest_chunks(paths, tenant); + futures::pin_mut!(chunks); + while let Some(rows) = chunks.try_next().await? { + for (key, body) in rows { + result.insert(key, Some(decode_manifest_blocking(body).await?)); + } + } + Ok(Some(result)) + } + async fn metadata_dates( + &self, + stream: &str, + tenant: &Option, + ) -> Result, MetastoreError> { + let path = crate::storage::object_storage::stream_data_path(stream, &[], tenant); + Ok(self.storage.list_dirs_relative(&path, tenant).await?) + } + async fn check_health(&self, tenant_id: &Option) -> Result<(), MetastoreError> { + Ok(self.storage.check_health(tenant_id).await?) + } + + async fn read_latency(&self, tenant_id: &Option) -> std::time::Duration { + self.storage.read_latency(tenant_id).await + } + + async fn stream_exists( + &self, + stream_name: &str, + tenant_id: &Option, + ) -> Result { + Ok(self.storage.stream_exists(stream_name, tenant_id).await?) + } + + async fn is_stream_deleting( + &self, + stream_name: &str, + tenant_id: &Option, + ) -> Result { + Ok(self + .storage + .is_stream_deleting(stream_name, tenant_id) + .await?) + } + + async fn delete_stream( + &self, + stream_name: &str, + tenant_id: &Option, + ) -> Result<(), MetastoreError> { + Ok(self.storage.delete_stream(stream_name, tenant_id).await?) + } + + async fn delete_stream_partition( + &self, + stream_name: &str, + date: &str, + tenant_id: &Option, + ) -> Result<(), MetastoreError> { + let path = + crate::storage::object_storage::stream_data_path(stream_name, &[date], tenant_id); + Ok(self.storage.delete_partition(&path, tenant_id).await?) + } + + async fn list_tenant_candidates(&self) -> Result, MetastoreError> { + Ok(self.storage.list_tenant_candidates().await?) + } + } +); + +// Parse tuning as text first: whether it is meaningful depends on the parsed URL, +// including CLI overrides of environment values. Do not inspect process argv. +#[derive(clap::Args)] +struct PostgresArguments { + /// Opt in to PostgreSQL metadata. Never falls back to bucket metadata. + #[arg(long, env = "P_METASTORE_POSTGRES_URL", hide_env_values = true)] + pub metastore_postgres_url: Option, + #[arg(long, env = "P_METASTORE_POSTGRES_POOL_SIZE", default_value = "16")] + pub metastore_postgres_pool_size: String, + #[arg( + long, + env = "P_METASTORE_POSTGRES_CONNECT_TIMEOUT_MS", + default_value = "5000" + )] + pub metastore_postgres_connect_timeout_ms: String, + #[arg( + long, + env = "P_METASTORE_POSTGRES_STATEMENT_TIMEOUT_MS", + default_value = "10000" + )] + pub metastore_postgres_statement_timeout_ms: String, + /// Optionally pin a previously initialized backend UUID. + #[arg(long, env = "P_METASTORE_POSTGRES_ID")] + pub metastore_postgres_id: Option, + /// Optional PEM CA bundle. TLS mode is controlled by sslmode in the URL. + #[arg(long, env = "P_METASTORE_POSTGRES_CA")] + pub metastore_postgres_ca: Option, +} + +#[derive(Clone)] +pub struct PostgresOptions { + pub metastore_postgres_url: Option, + pub metastore_postgres_pool_size: u32, + pub metastore_postgres_connect_timeout_ms: u64, + pub metastore_postgres_statement_timeout_ms: u64, + pub metastore_postgres_id: Option, + pub metastore_postgres_ca: Option, +} +impl clap::Args for PostgresOptions { + fn augment_args(cmd: clap::Command) -> clap::Command { + PostgresArguments::augment_args(cmd) + } + fn augment_args_for_update(cmd: clap::Command) -> clap::Command { + PostgresArguments::augment_args_for_update(cmd) + } +} +impl TryFrom for PostgresOptions { + type Error = clap::Error; + fn try_from(raw: PostgresArguments) -> Result { + let mut options = Self { + metastore_postgres_url: raw.metastore_postgres_url.filter(|v| !v.trim().is_empty()), + metastore_postgres_id: raw.metastore_postgres_id, + metastore_postgres_ca: raw.metastore_postgres_ca, + ..Default::default() + }; + fn positive( + value: &str, + name: &str, + ) -> Result { + value + .parse::() + .ok() + .filter(|v| *v != T::default()) + .ok_or_else(|| { + clap::Error::raw( + clap::error::ErrorKind::ValueValidation, + format!("{name} must be a positive integer"), + ) + }) + } + if options.enabled() { + options.metastore_postgres_pool_size = + positive(&raw.metastore_postgres_pool_size, "PostgreSQL pool size")?; + options.metastore_postgres_connect_timeout_ms = positive( + &raw.metastore_postgres_connect_timeout_ms, + "PostgreSQL connect timeout", + )?; + options.metastore_postgres_statement_timeout_ms = positive( + &raw.metastore_postgres_statement_timeout_ms, + "PostgreSQL statement timeout", + )?; + } + Ok(options) + } +} +impl clap::FromArgMatches for PostgresOptions { + fn from_arg_matches(matches: &clap::ArgMatches) -> Result { + PostgresArguments::from_arg_matches(matches)?.try_into() + } + fn update_from_arg_matches(&mut self, matches: &clap::ArgMatches) -> Result<(), clap::Error> { + let mut raw = PostgresArguments { + metastore_postgres_url: self.metastore_postgres_url.clone(), + metastore_postgres_pool_size: self.metastore_postgres_pool_size.to_string(), + metastore_postgres_connect_timeout_ms: self + .metastore_postgres_connect_timeout_ms + .to_string(), + metastore_postgres_statement_timeout_ms: self + .metastore_postgres_statement_timeout_ms + .to_string(), + metastore_postgres_id: self.metastore_postgres_id.clone(), + metastore_postgres_ca: self.metastore_postgres_ca.clone(), + }; + raw.update_from_arg_matches(matches)?; + *self = raw.try_into()?; + Ok(()) + } +} + +impl std::fmt::Debug for PostgresOptions { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("PostgresOptions") + .field("enabled", &self.enabled()) + .finish_non_exhaustive() + } +} +impl PostgresOptions { + pub fn enabled(&self) -> bool { + self.metastore_postgres_url + .as_deref() + .is_some_and(|v| !v.trim().is_empty()) + } +} +impl Default for PostgresOptions { + fn default() -> Self { + Self { + metastore_postgres_url: None, + metastore_postgres_pool_size: 16, + metastore_postgres_connect_timeout_ms: 5000, + metastore_postgres_statement_timeout_ms: 10000, + metastore_postgres_id: None, + metastore_postgres_ca: None, + } + } +} + +/// Construct offline transfer using the same provider and SQL options as the server. +pub async fn run_migration( + provider: &dyn crate::storage::ObjectStorageProvider, + options: &crate::cli::Options, +) -> anyhow::Result<()> { + anyhow::ensure!( + options.postgres.enabled(), + "P_METASTORE_POSTGRES_URL is required" + ); + anyhow::ensure!( + std::env::var_os("P_STAGING_DIR").is_some() + || std::env::args() + .any(|a| a == "--local-staging-path" || a.starts_with("--local-staging-path=")), + "migration requires explicit P_STAGING_DIR or --local-staging-path" + ); + // Staging metadata alone is normal; event buffers must have uploaded first. + let staging = options.local_staging_path.canonicalize().map_err(|e| { + anyhow::anyhow!( + "staging directory must exist ({}): {e}", + options.local_staging_path.display() + ) + })?; + anyhow::ensure!(staging.is_dir(), "staging path must be a directory"); + eprintln!("Resolved staging directory: {}", staging.display()); + eprintln!( + "Environment file: {}", + crate::interactive::loaded_env_path() + .map(|path| path.display().to_string()) + .unwrap_or_else(|| "none (explicit environment/CLI only)".into()) + ); + let mut dirs = vec![staging]; + while let Some(dir) = dirs.pop() { + if !dir.exists() { + continue; + } + for entry in std::fs::read_dir(dir)? { + let path = entry?.path(); + if path.is_dir() { + dirs.push(path); + } else if path + .extension() + .is_some_and(|e| e == "arrows" || e == "parquet" || e == "arrow" || e == "part") + { + anyhow::bail!("unsynced staging data: {}", path.display()); + } + } + } + let storage = provider.construct_client(); + let db = super::postgres_documents::PostgresDocuments::new( + &options.postgres, + provider.get_endpoint(), + storage.clone(), + )?; + super::migration::run(storage, &db, &options.migration).await +} + +#[cfg(test)] +mod contract_tests { + use super::*; + use crate::{ + metastore::metastore_traits::*, + storage::{LocalFS, ObjectStorage}, + }; + use bytes::Bytes; + use std::sync::Arc; + + fn test_dir() -> tempfile::TempDir { + let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join(".postgres-smoke/unit"); + std::fs::create_dir_all(&root).unwrap(); + tempfile::tempdir_in(root).unwrap() + } + + #[derive(serde::Serialize)] + struct Document { + id: String, + path: String, + value: String, + } + impl MetastoreObject for Document { + fn get_object_path(&self) -> String { + self.path.clone() + } + fn get_object_id(&self) -> String { + self.id.clone() + } + } + fn initialize_context() { + fn factory() -> crate::parseable::Parseable { + use crate::storage::object_storage::ObjectStorageProvider; + let startup_root = std::env::var("P_CONTRACT_STARTUP_ROOT").ok(); + let root = startup_root + .as_ref() + .map(std::path::PathBuf::from) + .or_else(|| { + std::env::var("P_CONTRACT_CONTEXT_ROOT") + .ok() + .map(std::path::PathBuf::from) + }) + .unwrap_or_else(|| test_dir().path().to_owned()); + let provider = Arc::new(crate::storage::FSConfig { root }); + let storage = provider.construct_client(); + let mut options = crate::cli::Options::default(); + let metastore: Arc = if startup_root.is_some() { + options.multi_tenancy = true; + options.postgres.metastore_postgres_url = + Some(std::env::var("P_CONTRACT_STARTUP_URL").unwrap()); + options.postgres.metastore_postgres_statement_timeout_ms = + if std::env::var("P_CONTRACT_STARTUP_FAILURE").as_deref() == Ok("none") { + 500 + } else { + 5000 + }; + Arc::new(PostgresMetastore { + storage: Arc::new( + super::super::postgres_documents::PostgresDocuments::new( + &options.postgres, + std::env::var("P_CONTRACT_STARTUP_BUCKET").unwrap(), + storage.clone(), + ) + .unwrap(), + ), + }) + } else { + Arc::new(super::super::object_store_metastore::ObjectStoreMetastore { + storage: storage.clone(), + }) + }; + let context = crate::parseable::Parseable::new( + options, + #[cfg(feature = "kafka")] + ::parse_from([ + "contract", + ]), + provider, + metastore, + storage, + ); + if startup_root.is_none() { + *context.tenants.write().unwrap() = vec![String::new(), "tenant-contract".into()]; + } + context + } + let _ = crate::parseable::CONTRACT_PARSEABLE_FACTORY.set(factory); + once_cell::sync::Lazy::force(&crate::parseable::PARSEABLE); + } + async fn contract(store: Arc, sql: bool) { + initialize_context(); + store.initiate_connection().await.unwrap(); + for tenant in [None, Some("tenant-contract".to_owned())] { + let root = tenant.as_ref().map(|t| format!("{t}/")).unwrap_or_default(); + let object = Document { + id: ulid::Ulid::new().to_string(), + path: format!("{root}.users/owner/dashboards/space %_é'.json"), + value: format!("faithful bytes-{root}"), + }; + store.put_dashboard(&object, &tenant).await.unwrap(); + let dashboards = store.get_tenant_dashboards(&tenant).await.unwrap(); + assert_eq!( + dashboards, + vec![crate::storage::object_storage::to_bytes(&object)] + ); + let read = store + .get_dashboard_versioned(&object.path, &tenant) + .await + .unwrap() + .unwrap(); + assert_eq!(read.bytes, dashboards[0]); + assert!(matches!( + store + .put_dashboard_conditional( + &object.path, + Bytes::new(), + MetadataWriteCondition::Create, + &tenant + ) + .await, + Err(ConditionalMetadataError::Conflict) + )); + let next = store + .put_dashboard_conditional( + &object.path, + Bytes::from_static(b"new"), + MetadataWriteCondition::Update(read.version.clone()), + &tenant, + ) + .await + .unwrap(); + assert!(matches!( + store + .put_dashboard_conditional( + &object.path, + Bytes::new(), + MetadataWriteCondition::Update(read.version.clone()), + &tenant + ) + .await, + Err(ConditionalMetadataError::Conflict) + )); + assert!(matches!( + store + .delete_dashboard_conditional(&object.path, read.version.clone(), &tenant) + .await, + Err(ConditionalMetadataError::Conflict) + )); + store + .delete_dashboard_conditional(&object.path, next, &tenant) + .await + .unwrap(); + assert!( + store + .get_dashboard_versioned(&object.path, &tenant) + .await + .unwrap() + .is_none() + ); + // Concurrent creates have exactly one winner. + let writes = (0..12).map(|_| { + store.put_dashboard_conditional( + &object.path, + Bytes::from_static(b"{}"), + MetadataWriteCondition::Create, + &tenant, + ) + }); + let results = futures::future::join_all(writes).await; + assert_eq!(results.iter().filter(|r| r.is_ok()).count(), 1); + assert_eq!( + results + .iter() + .filter(|r| matches!(r, Err(ConditionalMetadataError::Conflict))) + .count(), + 11 + ); + let recreated = store + .get_dashboard_versioned(&object.path, &tenant) + .await + .unwrap() + .unwrap(); + assert!(matches!( + store + .put_dashboard_conditional( + &object.path, + Bytes::from_static(b"stale"), + MetadataWriteCondition::Update(read.version.clone()), + &tenant + ) + .await, + Err(ConditionalMetadataError::Conflict) + )); + let updates = (0..12).map(|i| { + store.put_dashboard_conditional( + &object.path, + Bytes::from(i.to_string()), + MetadataWriteCondition::Update(recreated.version.clone()), + &tenant, + ) + }); + let updates = futures::future::join_all(updates).await; + assert_eq!(updates.iter().filter(|r| r.is_ok()).count(), 1); + assert_eq!( + updates + .iter() + .filter(|r| matches!(r, Err(ConditionalMetadataError::Conflict))) + .count(), + 11 + ); + let winner = updates.iter().position(Result::is_ok).unwrap(); + assert_eq!( + store + .get_dashboard_versioned(&object.path, &tenant) + .await + .unwrap() + .unwrap() + .bytes, + Bytes::from(winner.to_string()) + ); + store.delete_dashboard(&object, &tenant).await.unwrap(); + assert!( + store + .get_tenant_dashboards(&tenant) + .await + .unwrap() + .is_empty() + ); + store.put_keystone(&object, &tenant).await.unwrap(); + store.put_conversation(&object, &tenant).await.unwrap(); + let objects = store + .get_objects(&format!("{root}.keystone"), &tenant) + .await + .unwrap(); + assert_eq!( + objects, + vec![crate::storage::object_storage::to_bytes(&object); 2] + ); + store.delete_keystone(&object, &tenant).await.unwrap(); + store.delete_conversation(&object, &tenant).await.unwrap(); + store.put_llmconfig(&object, &tenant).await.unwrap(); + store.delete_llmconfig(&object, &tenant).await.unwrap(); + store.put_target(&object, &tenant).await.unwrap(); + store.delete_target(&object, &tenant).await.unwrap(); + store.put_chat(&object, &tenant).await.unwrap(); + store.delete_chat(&object, &tenant).await.unwrap(); + assert!( + store + .get_alert_state_entry(&ulid::Ulid::new(), &tenant) + .await + .unwrap() + .is_none() + ); + assert!(store.get_mttr_history(&tenant).await.unwrap().is_none()); + assert!( + store + .get_all_stream_jsons( + "absent", + Some(crate::option::Mode::Query), + &tenant, + false + ) + .await + .is_err() + ); + let directory_marker = Document { + id: object.id.clone(), + path: format!("{root}absent/.stream/placeholder"), + value: String::new(), + }; + store + .put_dashboard(&directory_marker, &tenant) + .await + .unwrap(); + store + .delete_dashboard(&directory_marker, &tenant) + .await + .unwrap(); + assert!( + store + .get_all_stream_jsons( + "absent", + Some(crate::option::Mode::Ingest), + &tenant, + false + ) + .await + .unwrap() + .is_empty() + ); + assert!( + store + .get_manifest( + "absent", + chrono::Utc::now(), + chrono::Utc::now(), + Some(format!("{root}absent/date=2026-01-01/manifest.json")), + &tenant + ) + .await + .unwrap() + .is_none() + ); + store.put_alert(&object, &tenant).await.unwrap(); + store.delete_alert(&object, &tenant).await.unwrap(); + store.delete_alert(&object, &tenant).await.unwrap(); + assert!( + store + .get_stream_json("absent", true, &tenant, false) + .await + .is_err() + ); + assert!( + store + .get_parseable_metadata(&tenant) + .await + .unwrap() + .is_none() + ); + store + .put_parseable_metadata(&object, &tenant) + .await + .unwrap(); + assert_eq!( + store + .get_parseable_metadata(&tenant) + .await + .unwrap() + .unwrap(), + crate::storage::object_storage::to_bytes(&object) + ); + } + for tenant in [None, Some("tenant-contract".to_owned())] { + let root = tenant.as_ref().map(|t| format!("{t}/")).unwrap_or_default(); + let schema = arrow_schema::Schema::new(vec![arrow_schema::Field::new( + if tenant.is_some() { + "named_value" + } else { + "default_value" + }, + arrow_schema::DataType::Int64, + false, + )]); + store + .put_schema(schema.clone(), "contractlogs", &tenant) + .await + .unwrap(); + assert_eq!( + serde_json::from_slice::( + &store.get_schema("contractlogs", &tenant).await.unwrap() + ) + .unwrap(), + schema + ); + assert_eq!( + store + .get_all_schemas("contractlogs", &tenant) + .await + .unwrap(), + vec![schema.clone()] + ); + assert_eq!( + store + .refresh_schema("contractlogs", &tenant, ".schema") + .await + .unwrap(), + schema + ); + if tenant.is_none() { + assert_eq!( + store + .get_all_schemas("contractlogs", &Some(String::new())) + .await + .unwrap(), + vec![schema.clone()] + ); + assert_eq!( + store + .refresh_schema("contractlogs", &Some(String::new()), ".schema") + .await + .unwrap(), + schema + ); + } + for field in ["ingestor_a", "ingestor_b"] { + let path = format!("{root}contractlogs/.stream/.{field}.schema"); + let fragment = arrow_schema::Schema::new(vec![arrow_schema::Field::new( + field, + arrow_schema::DataType::Utf8, + true, + )]); + store + .put_dashboard_conditional( + &path, + crate::storage::object_storage::to_bytes(&fragment), + MetadataWriteCondition::Create, + &tenant, + ) + .await + .unwrap(); + } + let merged = store + .refresh_schema("contractlogs", &tenant, ".schema") + .await + .unwrap(); + let names = |s: &arrow_schema::Schema| { + s.fields() + .iter() + .map(|f| f.name().clone()) + .collect::>() + }; + let expected_names = [ + schema.fields()[0].name().as_str(), + "ingestor_a", + "ingestor_b", + ] + .into_iter() + .map(str::to_owned) + .collect(); + assert_eq!(names(&merged), expected_names); + let persisted: arrow_schema::Schema = + serde_json::from_slice(&store.get_schema("contractlogs", &tenant).await.unwrap()) + .unwrap(); + assert_eq!(names(&persisted), names(&merged)); + let metadata = crate::storage::ObjectStoreFormat::default(); + store + .put_stream_json(&metadata, "contractlogs", &tenant) + .await + .unwrap(); + assert_eq!( + store + .get_stream_json("contractlogs", false, &tenant, false) + .await + .unwrap(), + crate::storage::object_storage::to_bytes(&metadata) + ); + assert_eq!( + store + .get_all_stream_jsons("contractlogs", None, &tenant, false) + .await + .unwrap(), + vec![crate::storage::object_storage::to_bytes(&metadata)] + ); + if sql { + assert!( + store + .list_streams(&tenant) + .await + .unwrap() + .contains("contractlogs") + ); + } + // LocalFS listing uses the process-global provider; covered by the + // separate regression so this defect cannot suppress all CRUD checks. + let lower = chrono::Utc::now(); + let upper = lower; + let manifest = crate::catalog::manifest::Manifest { + files: (0..10_000) + .map(|i| crate::catalog::manifest::File { + file_path: format!( + "{root}contractlogs/date=2026-01-01/space %_é'/{i}.parquet" + ), + num_rows: i, + file_size: 1234, + ..Default::default() + }) + .collect(), + ..Default::default() + }; + store + .put_manifest(&manifest, "contractlogs", lower, upper, &tenant) + .await + .unwrap(); + assert_eq!( + serde_json::to_value( + store + .get_manifest("contractlogs", lower, upper, None, &tenant) + .await + .unwrap() + .unwrap() + ) + .unwrap(), + serde_json::to_value(&manifest).unwrap() + ); + assert!( + store + .get_manifest_path("contractlogs", lower, upper, &tenant) + .await + .unwrap() + .ends_with("manifest.json") + ); + let listing = store + .get_manifest_files_for_dates( + "contractlogs", + &tenant, + &[format!("date={}", lower.format("%Y-%m-%d"))], + ) + .await; + if sql { + assert_eq!( + serde_json::to_value(listing.unwrap()).unwrap(), + serde_json::json!({ (format!("date={}", lower.format("%Y-%m-%d"))): [manifest] }) + ); + } else { + assert!(listing.is_err()); + } // LocalFS has no delimiter LIST. + assert!(store.stream_exists("contractlogs", &tenant).await.unwrap()); + assert!( + !store + .is_stream_deleting("contractlogs", &tenant) + .await + .unwrap() + ); + assert!(store.read_latency(&tenant).await < std::time::Duration::from_secs(5)); + store.check_health(&tenant).await.unwrap(); + let candidates = store.list_tenant_candidates().await.unwrap(); + if tenant.is_some() { + assert!(candidates.contains(&"tenant-contract".to_owned())); + } + + store + .delete_manifest("contractlogs", lower, upper, &tenant) + .await + .unwrap(); + assert!( + store + .get_manifest("contractlogs", lower, upper, None, &tenant) + .await + .unwrap() + .is_none() + ); + store + .delete_stream_partition( + "contractlogs", + &format!("date={}", lower.format("%Y-%m-%d")), + &tenant, + ) + .await + .unwrap(); + let state = crate::alerts::alert_structs::AlertStateEntry::new( + ulid::Ulid::new(), + crate::alerts::alert_enums::AlertState::default(), + tenant.clone(), + ); + store.put_alert_state(&state, &tenant).await.unwrap(); + assert!( + store + .get_alert_state_entry(&state.alert_id, &tenant) + .await + .unwrap() + .is_some() + ); + let states = store.get_alert_states(&tenant).await.unwrap(); + assert_eq!(states.len(), 1); + if tenant.is_none() { + assert_eq!( + store + .get_alert_states(&Some(crate::parseable::DEFAULT_TENANT.into())) + .await + .unwrap() + .len(), + 1 + ); + } + store.delete_alert_state(&state, &tenant).await.unwrap(); + assert!( + store + .get_alert_state_entry(&state.alert_id, &tenant) + .await + .unwrap() + .is_none() + ); + let history = crate::alerts::alert_structs::MTTRHistory { + daily_stats: vec![], + tenant_id: tenant.clone(), + }; + store.put_mttr_history(&history, &tenant).await.unwrap(); + assert_eq!( + serde_json::to_value(store.get_mttr_history(&tenant).await.unwrap().unwrap()) + .unwrap(), + serde_json::to_value(&history).unwrap() + ); + let policy = crate::alerts::outbound_http_policy::AlertTargetPolicyConfig { + allow_private: tenant.is_some(), + ..Default::default() + }; + store.put_outbound_policy(&tenant, &policy).await.unwrap(); + let filter:crate::users::filters::Filter=serde_json::from_value(serde_json::json!({ + "version":"v2","user_id":"owner","filter_id":"contract","stream_name":"orders","filter_name":"contract","tenant_id":tenant, + "query":{"filter_type":"sql","filter_query":"select * from orders","filter_builder":null},"time_filter":null + })).unwrap(); + store.put_filter(&filter, &tenant).await.unwrap(); + store.reconcile_filter(&filter, &tenant).await.unwrap(); + let config = Document { + id: "config".into(), + path: format!("{root}.settings/llmconfigs/contract.json"), + value: format!("config-{root}"), + }; + store.put_llmconfig(&config, &tenant).await.unwrap(); + let target: crate::alerts::target::Target = serde_json::from_value(serde_json::json!({ + "name": format!("target-{root}"), "type":"webhook", + "endpoint":"https://hook.example/x", "tenantId": tenant, + "id": "01ARZ3NDEKTSV4RRFFQ69G5FAV" + })) + .unwrap(); + store.put_target(&target, &tenant).await.unwrap(); + let chat = Document { + id: "chat".into(), + path: format!("{root}.users/owner/chats/contract.json"), + value: format!("chat-{root}"), + }; + store.put_chat(&chat, &tenant).await.unwrap(); + let key = Document { + id: "key".into(), + path: String::new(), + value: format!("key-{root}"), + }; + store.put_keystone(&key, &tenant).await.unwrap(); + store.put_conversation(&key, &tenant).await.unwrap(); + } + // The global getters must preserve both populated namespaces. Vec order + // is not part of Metastore's contract: compare byte sets per tenant. + let expected = |id: &str, + suffix: &str, + label: &str| + -> std::collections::HashMap> { + [None, Some("tenant-contract".to_owned())] + .into_iter() + .map(|tenant| { + let root = tenant.as_ref().map(|t| format!("{t}/")).unwrap_or_default(); + let document = Document { + id: id.into(), + path: if suffix.is_empty() { + String::new() + } else { + format!("{root}{suffix}") + }, + value: format!("{label}-{root}"), + }; + ( + tenant.unwrap_or_else(|| crate::parseable::DEFAULT_TENANT.into()), + vec![crate::storage::object_storage::to_bytes(&document)], + ) + }) + .collect() + }; + assert_eq!( + store.get_keystones().await.unwrap(), + expected("key", "", "key") + ); + assert_eq!( + store.get_conversations().await.unwrap(), + expected("key", "", "key") + ); + // MTTR history is incorrectly returned as an alert. The exact empty + // alert inventory assertion lives in the ignored regression below. + assert_eq!( + store.get_llmconfigs().await.unwrap(), + expected("config", ".settings/llmconfigs/contract.json", "config") + ); + let targets = store.get_targets().await.unwrap(); + assert_eq!(targets.len(), 2); + for (tenant, values) in targets { + assert_eq!(values.len(), 1, "typed target was silently discarded"); + assert_eq!( + values[0].name, + if tenant == crate::parseable::DEFAULT_TENANT { + "target-" + } else { + "target-tenant-contract/" + } + ); + } + let policies = store.get_outbound_policies().await.unwrap(); + assert_eq!(policies.len(), 2); + assert!(!policies[""].allow_private); + assert!(policies["tenant-contract"].allow_private); + assert_eq!( + store.get_dashboards().await.unwrap(), + expected("", "", "") + .into_keys() + .map(|t| (t, vec![])) + .collect() + ); + let chats = store.get_chats().await.unwrap(); + assert_eq!(chats.len(), 2); + for (tenant, bytes) in expected("chat", ".users/owner/chats/contract.json", "chat") { + let users = chats.get(&tenant).unwrap(); + assert_eq!(users.get("owner").unwrap().value(), &bytes); + } + let filters = store.get_filters().await.unwrap(); + assert_eq!(filters.len(), 2); + for (tenant, filters) in filters { + assert_eq!( + filters.len(), + 1, + "filter delete must exercise a populated tenant" + ); + for filter in filters { + store + .delete_filter( + &filter, + &Some(if tenant == crate::parseable::DEFAULT_TENANT { + String::new() + } else { + tenant.clone() + }), + ) + .await + .unwrap(); + } + } + let node = crate::handlers::http::modal::NodeMetadata { + domain_name: "contract.invalid".into(), + node_id: "contract".into(), + ..Default::default() + }; + store.put_node_metadata(&node).await.unwrap(); + assert_eq!( + store + .get_node_metadata(crate::handlers::http::modal::NodeType::Ingestor, &None) + .await + .unwrap() + .len(), + 1 + ); + assert_eq!( + store.get_ingestor_metadata().await.unwrap(), + vec![crate::storage::object_storage::to_bytes(&node)] + ); + assert!( + store + .delete_node_metadata( + "contract.invalid", + crate::handlers::http::modal::NodeType::Ingestor + ) + .await + .unwrap() + ); + assert!( + !store + .delete_node_metadata( + "missing.invalid", + crate::handlers::http::modal::NodeType::Ingestor + ) + .await + .unwrap() + ); + for tenant in [None, Some("tenant-contract".to_owned())] { + // LocalFS stream deletion historically ignores tenant scope; cloud/SQL do not. + if !sql && tenant.is_some() { + continue; + } + store.delete_stream("contractlogs", &tenant).await.unwrap(); + assert!(!store.stream_exists("contractlogs", &tenant).await.unwrap()); + store + .put_stream_json( + &crate::storage::ObjectStoreFormat::default(), + "contractlogs", + &tenant, + ) + .await + .unwrap(); + assert!(store.stream_exists("contractlogs", &tenant).await.unwrap()); + assert!( + !store + .is_stream_deleting("contractlogs", &tenant) + .await + .unwrap() + ); + store.delete_stream("contractlogs", &tenant).await.unwrap(); + } + let survivor = store.get_parseable_metadata(&None).await.unwrap().unwrap(); + assert!( + store + .get_parseable_metadata(&Some("tenant-contract".into())) + .await + .unwrap() + .is_some() + ); + store.delete_tenant("tenant-contract").await.unwrap(); + assert!( + store + .get_parseable_metadata(&Some("tenant-contract".into())) + .await + .unwrap() + .is_none() + ); + assert_eq!( + store.get_parseable_metadata(&None).await.unwrap().unwrap(), + survivor + ); + assert!( + store + .get_parseable_metadata(&Some("other-tenant".into())) + .await + .unwrap() + .is_none() + ); + } + // Explicit regression cases use disposable databases and always clean up, + // including when the assertion demonstrates a product defect. + async fn postgres_regression(case: u8) { + use futures::FutureExt; + initialize_context(); + let Ok(url) = std::env::var("P_TEST_POSTGRES_URL") else { + eprintln!("SKIP PostgreSQL regression {case}: P_TEST_POSTGRES_URL unset"); + return; + }; + if case == 3 && std::env::var_os("P_TEST_POSTGRES_CONTAINER").is_none() { + eprintln!("SKIP paused-server regression: P_TEST_POSTGRES_CONTAINER unset"); + return; + } + let (admin, connection) = tokio_postgres::connect(&url, tokio_postgres::NoTls) + .await + .unwrap(); + let task = tokio::spawn(async move { connection.await.unwrap() }); + let database = format!( + "regression_{}", + ulid::Ulid::new().to_string().to_lowercase() + ); + admin + .batch_execute(&format!("CREATE DATABASE {database}")) + .await + .unwrap(); + let mut url = url::Url::parse(&url).unwrap(); + url.set_path(&database); + let result = std::panic::AssertUnwindSafe(async { + let dir = test_dir(); + let storage: Arc = Arc::new(LocalFS::new(dir.path().to_owned())); + let options = PostgresOptions { + metastore_postgres_url: Some(url.to_string()), + metastore_postgres_statement_timeout_ms: if case == 7 { 5000 } else { 500 }, + metastore_postgres_connect_timeout_ms: 500, + ..Default::default() + }; + let documents = Arc::new( + super::super::postgres_documents::PostgresDocuments::new( + &options, + database.clone(), + storage, + ) + .unwrap(), + ); + documents.check(&None).await.unwrap(); + let store = PostgresMetastore { + storage: documents.clone(), + }; + if case == 7 { + let abort = super::super::migration::MigrationOptions { + metadata_migration: Some(super::super::migration::Action::Export), + acknowledge_server_stopped: true, abort: true, ..Default::default() + }; + super::super::migration::run(Arc::new(LocalFS::new(dir.path().to_owned())), &documents, &abort).await.unwrap(); + let name = dir.path().file_name().unwrap().to_str().unwrap(); + for tenant in [None, Some(name.to_owned())] { + let prefix = tenant.as_ref().map(|t| format!("{t}/")).unwrap_or_default(); + let exact = format!("{prefix}{name}/date=2026-01-01/manifest.json"); + documents.put_object(RelativePath::new(&exact), Bytes::from_static(b"exact"), &tenant).await.unwrap(); + assert_eq!(documents.get_object(RelativePath::new(&exact), &tenant).await.unwrap(), Bytes::from_static(b"exact")); + let absolute = format!("{}/{exact}", dir.path().to_string_lossy().trim_start_matches('/')); + assert_eq!(documents.get_object(RelativePath::new(&absolute), &tenant).await.unwrap(), Bytes::from_static(b"exact")); + let paths = vec![exact.clone(), absolute]; + assert_eq!(documents.manifest_batch(&paths, &tenant).await.unwrap().len(), 2); + documents.put_object(RelativePath::new(&format!("{prefix}{name}/.stream/.deleting")), Bytes::new(), &tenant).await.unwrap(); + assert!(matches!(documents.put_object(RelativePath::new(&exact), Bytes::new(), &tenant).await, Err(ObjectStorageError::StreamDeleting))); + } + let collision = super::super::postgres_documents::PostgresDocuments::new( + &options, database.clone(), Arc::new(LocalFS::new(std::path::PathBuf::from("logs")))).unwrap(); + for tenant in [None, Some("logs".to_owned())] { + let key = if tenant.is_none() { "logs/date=2026-01-01/exact.manifest.json" } else { "logs/logs/date=2026-01-01/exact.manifest.json" }; + let fallback = key.strip_prefix("logs/").unwrap(); + collision.put_object(RelativePath::new(key), Bytes::from_static(b"exact"), &tenant).await.unwrap(); + collision.put_object(RelativePath::new(fallback), Bytes::from_static(b"fallback"), &tenant).await.unwrap(); + assert_eq!(collision.get_object(RelativePath::new(key), &tenant).await.unwrap(), Bytes::from_static(b"exact")); + assert_eq!(collision.manifest_batch(&[key.to_owned()], &tenant).await.unwrap()[0].1, Bytes::from_static(b"exact")); + } + // SQL tenant candidates cannot bypass deletion collision protection. + store.put_stream_json(&crate::storage::ObjectStoreFormat::default(), "protected", &None).await.unwrap(); + documents.put_object(RelativePath::new("protected/other/.stream/.schema"), Bytes::new(), &Some("protected".into())).await.unwrap(); + let data = dir.path().join("protected/date=2026-01-01/data.parquet"); + std::fs::create_dir_all(data.parent().unwrap()).unwrap(); + std::fs::write(&data, b"sentinel").unwrap(); + assert!(documents.list_tenant_candidates().await.unwrap().contains(&"protected".to_owned())); + assert!(documents.delete_prefix(RelativePath::new("protected"), &None).await.unwrap_err().to_string().contains("collides")); + assert_eq!(std::fs::read(&data).unwrap(), b"sentinel"); + store.put_stream_json(&crate::storage::ObjectStoreFormat::default(), "alias", &Some(crate::parseable::DEFAULT_TENANT.into())).await.unwrap(); + assert!(store.stream_exists("alias", &None).await.unwrap()); + assert!(documents.get_object(RelativePath::new("alias/.stream/.stream.json"), &None).await.is_ok()); + let paths: Vec<_> = (0..70).map(|i| format!("batch/date=2026-01-01/{i}.manifest.json")).collect(); + for key in &paths { documents.put_object(RelativePath::new(key), Bytes::new(), &None).await.unwrap(); } + assert_eq!(documents.manifest_batch(&paths, &None).await.unwrap().len(), 70); + let large_paths: Vec<_> = (0..3).map(|i| format!("large/date=2026-01-01/{i}.manifest.json")).collect(); + for key in &large_paths { documents.put_object(RelativePath::new(key), Bytes::from(vec![b'x'; 5 * 1024 * 1024]), &None).await.unwrap(); } + // Model growth between the size lookup and the body read: + // only sizing queries see one byte; body queries see real data. + documents.client().await.unwrap().batch_execute("ALTER TABLE parseable_documents RENAME TO batch_documents; CREATE VIEW parseable_documents AS SELECT tenant,kind,key,CASE WHEN current_query() LIKE 'SELECT key,octet_length%' THEN substring(body from 1 for 1) ELSE body END AS body,version,created_at,updated_at FROM batch_documents;").await.unwrap(); + let chunks = documents.manifest_chunks(&large_paths, &None); + futures::pin_mut!(chunks); + use futures::TryStreamExt; + let mut chunks_seen = 0; + while let Some(rows) = chunks.try_next().await.unwrap() { + assert!(rows.iter().map(|(_, body)| body.len()).sum::() <= 8 * 1024 * 1024); + chunks_seen += 1; + } + assert_eq!(chunks_seen, 3); + documents.client().await.unwrap().batch_execute("DROP VIEW parseable_documents; ALTER TABLE batch_documents RENAME TO parseable_documents;").await.unwrap(); + documents.client().await.unwrap().batch_execute(include_str!("../migrations/002_review_fixes.sql")).await.unwrap(); + let restart_a = super::super::postgres_documents::PostgresDocuments::new(&options, database.clone(), Arc::new(LocalFS::new(dir.path().to_owned()))).unwrap(); + let restart_b = super::super::postgres_documents::PostgresDocuments::new(&options, database.clone(), Arc::new(LocalFS::new(dir.path().to_owned()))).unwrap(); + tokio::try_join!(restart_a.check(&None), restart_b.check(&None)).unwrap(); + assert_eq!(documents.get_object(RelativePath::new(&large_paths[0]), &None).await.unwrap().len(), 5 * 1024 * 1024); + assert_eq!(documents.client().await.unwrap().query_one("SELECT schema_version FROM parseable_backend", &[]).await.unwrap().get::<_, i32>(0), 3); + documents.put_object(RelativePath::new("upgrade/.stream/.deleting"), Bytes::new(), &None).await.unwrap(); + assert!(matches!(documents.put_object(RelativePath::new("upgrade/date=2026-01-01/manifest.json"), Bytes::new(), &None).await, Err(ObjectStorageError::StreamDeleting))); + + + } else if case == 0 { + let date = chrono::Utc::now(); + let manifest = crate::catalog::manifest::Manifest::default(); + store + .put_manifest(&manifest, "logs", date, date, &None) + .await + .unwrap(); + let path = store + .get_manifest_path("logs", date, date, &None) + .await + .unwrap(); + assert!( + store + .get_manifest("logs", date, date, Some(path), &None) + .await + .unwrap() + .is_some(), + "B1: returned manifest path must be readable" + ); + } else if case == 1 { + store + .put_stream_json(&crate::storage::ObjectStoreFormat::default(), "logs", &None) + .await + .unwrap(); + store + .delete_stream("logs", &Some(crate::parseable::DEFAULT_TENANT.into())) + .await + .unwrap(); + assert!( + !store.stream_exists("logs", &None).await.unwrap(), + "m5: default tenant alias must delete default rows" + ); + } else if case == 2 { + let fresh = test_dir(); + let empty: Arc = Arc::new(LocalFS::new(fresh.path().to_owned())); + let repaired = super::super::postgres_documents::PostgresDocuments::new( + &options, + database.clone(), + empty, + ) + .unwrap(); + assert!( + repaired.check(&None).await.is_err(), + "m9: a missing pairing marker must require explicit repair" + ); + } else if case == 3 { + let container = std::env::var("P_TEST_POSTGRES_CONTAINER") + .expect("requires disposable Docker container name"); + // Pausing an established server leaves TCP open and prevents + // server-side statement_timeout from sending a response. + assert!( + std::process::Command::new("docker") + .args(["pause", &container]) + .status() + .unwrap() + .success() + ); + let result = tokio::time::timeout( + std::time::Duration::from_secs(2), + documents.check_health(&None), + ) + .await; + assert!( + std::process::Command::new("docker") + .args(["unpause", &container]) + .status() + .unwrap() + .success() + ); + assert!( + result.is_ok(), + "M2: health exceeded its client deadline against a nonresponsive database" + ); + } else { + let output = std::process::Command::new(std::env::current_exe().unwrap()) + .args(["--exact", "metastore::metastores::postgres_metastore::contract_tests::startup_cleanup_child", "--ignored", "--nocapture"]) + .env("P_CONTRACT_STARTUP_ROOT", dir.path()) + .env("P_CONTRACT_STARTUP_URL", url.as_str()) + .env("P_CONTRACT_STARTUP_BUCKET", &database) + .env("P_CONTRACT_STARTUP_FAILURE", match case { 5 => "late-failure", 6 => "default-stream", _ => "none" }) + .output().unwrap(); + assert!(output.status.success(), "{}", String::from_utf8_lossy(&output.stdout).to_string() + &String::from_utf8_lossy(&output.stderr)); + } + }) + .catch_unwind() + .await; + admin + .batch_execute(&format!("DROP DATABASE {database} WITH (FORCE)")) + .await + .unwrap(); + drop(admin); + task.await.unwrap(); + if let Err(panic) = result { + std::panic::resume_unwind(panic); + } + } + #[tokio::test] + #[ignore = "requires PostgreSQL 17; starts an isolated process for the global context"] + async fn postgres_startup_cleanup() { + postgres_regression(4).await; + } + + #[tokio::test] + async fn postgres_startup_default_stream_regression() { + postgres_regression(6).await; + } + + #[tokio::test] + async fn postgres_startup_mid_scan_failure_regression() { + postgres_regression(5).await; + } + + #[tokio::test] + #[ignore = "internal child fixture; invoked by postgres_startup_cleanup"] + async fn startup_cleanup_child() { + initialize_context(); + let context = &crate::parseable::PARSEABLE; + let store = &context.metastore; + store.initiate_connection().await.unwrap(); + let url = std::env::var("P_CONTRACT_STARTUP_URL").unwrap(); + let (mut client, connection) = tokio_postgres::connect(&url, tokio_postgres::NoTls) + .await + .unwrap(); + let task = tokio::spawn(async move { connection.await.unwrap() }); + let storage = context.storage.get_object_store(); + if std::env::var("P_CONTRACT_STARTUP_FAILURE").as_deref() == Ok("default-stream") { + store + .put_stream_json( + &crate::storage::ObjectStoreFormat::default(), + "defaultlogs", + &None, + ) + .await + .unwrap(); + storage + .put_object( + RelativePath::new("defaultlogs/date=2026-01-01/data.parquet"), + Bytes::from_static(b"default tenant data"), + &None, + ) + .await + .unwrap(); + let invalid = Document { + id: "collision".into(), + path: "defaultlogs/other/.stream/.schema".into(), + value: "invalid".into(), + }; + store + .put_dashboard(&invalid, &Some("defaultlogs".into())) + .await + .unwrap(); + context.validate_multi_tenancy().await.unwrap(); + assert_eq!( + storage + .get_object( + RelativePath::new("defaultlogs/date=2026-01-01/data.parquet"), + &None + ) + .await + .expect("m6: startup must preserve default-tenant stream data"), + Bytes::from_static(b"default tenant data") + ); + assert!(store.stream_exists("defaultlogs", &None).await.unwrap()); + return; + } + if std::env::var("P_CONTRACT_STARTUP_FAILURE").as_deref() == Ok("late-failure") { + let invalid = Document { + id: "invalid".into(), + path: "a-invalid/.stream/.schema".into(), + value: "sentinel".into(), + }; + store + .put_dashboard(&invalid, &Some("a-invalid".into())) + .await + .unwrap(); + store + .put_parseable_metadata( + &crate::storage::store_metadata::StorageMetadata::default(), + &Some("z-valid".into()), + ) + .await + .unwrap(); + storage + .put_object( + RelativePath::new("a-invalid/data.parquet"), + Bytes::from_static(b"sentinel"), + &None, + ) + .await + .unwrap(); + // Restrict the injected SQL error to the per-tenant body read; + // volatile view expressions can also execute during discovery. + client.batch_execute("ALTER TABLE parseable_documents RENAME TO startup_documents; CREATE FUNCTION startup_read_failure() RETURNS bytea LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'injected transient tenant read failure' USING ERRCODE='08006'; END $$; CREATE VIEW parseable_documents AS SELECT tenant,kind,key,CASE WHEN tenant='z-valid' AND key='z-valid/.parseable.json' AND (current_query() LIKE 'SELECT body%FROM parseable_documents%' OR current_query() LIKE 'SELECT key,body%FROM parseable_documents%') THEN startup_read_failure() ELSE body END AS body,version,created_at,updated_at FROM startup_documents;").await.unwrap(); + assert!(context.validate_multi_tenancy().await.is_err()); + assert_eq!( + storage + .get_object(RelativePath::new("a-invalid/data.parquet"), &None) + .await + .unwrap(), + Bytes::from_static(b"sentinel"), + "a late database read failure must preserve earlier invalid tenant data" + ); + let remaining: i64 = client + .query_one( + "SELECT count(*) FROM startup_documents WHERE tenant='a-invalid'", + &[], + ) + .await + .unwrap() + .get(0); + assert_eq!( + remaining, 1, + "a late database read failure must preserve earlier invalid tenant rows" + ); + return; + } + for tenant in ["sqlonly", "duplicate"] { + let metadata = crate::storage::store_metadata::StorageMetadata::default(); + store + .put_parseable_metadata(&metadata, &Some(tenant.into())) + .await + .unwrap(); + } + let invalid = Document { + id: "invalid".into(), + path: "invalid/.stream/.schema".into(), + value: "invalid candidate".into(), + }; + store + .put_dashboard(&invalid, &Some("invalid".into())) + .await + .unwrap(); + for tenant in ["dataonly", "duplicate", "invalid", ".hidden"] { + storage + .put_object( + RelativePath::new(&format!("{tenant}/data.parquet")), + Bytes::from(tenant.to_owned()), + &None, + ) + .await + .unwrap(); + } + let before = client + .query( + "SELECT tenant,key,body FROM parseable_documents ORDER BY tenant,key", + &[], + ) + .await + .unwrap() + .into_iter() + .map(|r| { + ( + r.get::<_, String>(0), + r.get::<_, String>(1), + r.get::<_, Vec>(2), + ) + }) + .collect::>(); + // A transient database failure in discovery must admit no cleanup. + let tx = client.transaction().await.unwrap(); + tx.batch_execute("LOCK TABLE parseable_documents IN ACCESS EXCLUSIVE MODE") + .await + .unwrap(); + let started = std::time::Instant::now(); + let result = context.validate_multi_tenancy().await; + tx.rollback().await.unwrap(); + assert!(result.is_err()); + assert!(started.elapsed() < std::time::Duration::from_secs(3)); + // A temporarily unavailable bucket must also admit no cleanup. + let root = std::path::PathBuf::from(std::env::var("P_CONTRACT_STARTUP_ROOT").unwrap()); + let offline = root.with_extension("offline"); + std::fs::rename(&root, &offline).unwrap(); + std::fs::write(&root, b"temporarily unavailable bucket root").unwrap(); + let result = context.validate_multi_tenancy().await; + std::fs::remove_file(&root).unwrap(); + std::fs::rename(&offline, &root).unwrap(); + assert!(result.is_err()); + let after = client + .query( + "SELECT tenant,key,body FROM parseable_documents ORDER BY tenant,key", + &[], + ) + .await + .unwrap() + .into_iter() + .map(|r| { + ( + r.get::<_, String>(0), + r.get::<_, String>(1), + r.get::<_, Vec>(2), + ) + }) + .collect::>(); + assert_eq!(after, before); + for tenant in ["dataonly", "duplicate", "invalid", ".hidden"] { + assert_eq!( + storage + .get_object(RelativePath::new(&format!("{tenant}/data.parquet")), &None) + .await + .unwrap(), + Bytes::from(tenant.to_owned()) + ); + } + context.validate_multi_tenancy().await.unwrap(); + assert_eq!( + context + .list_tenants() + .unwrap() + .into_iter() + .collect::>(), + ["sqlonly".to_owned(), "duplicate".to_owned()] + .into_iter() + .collect() + ); + let rows = client + .query( + "SELECT tenant,key FROM parseable_documents ORDER BY tenant,key", + &[], + ) + .await + .unwrap() + .into_iter() + .map(|r| (r.get::<_, String>(0), r.get::<_, String>(1))) + .collect::>(); + assert_eq!( + rows, + vec![ + ("duplicate".into(), "duplicate/.parseable.json".into()), + ("sqlonly".into(), "sqlonly/.parseable.json".into()) + ] + ); + for tenant in ["dataonly", "invalid"] { + assert!( + storage + .get_object(RelativePath::new(&format!("{tenant}/data.parquet")), &None) + .await + .is_err() + ); + } + for tenant in ["duplicate", ".hidden"] { + assert_eq!( + storage + .get_object(RelativePath::new(&format!("{tenant}/data.parquet")), &None) + .await + .unwrap(), + Bytes::from(tenant.to_owned()) + ); + } + drop(client); + task.await.unwrap(); + } + #[tokio::test] + async fn alert_listing_excludes_mttr_history() { + initialize_context(); + let dir = test_dir(); + let storage: Arc = Arc::new(LocalFS::new(dir.path().to_owned())); + let store = super::super::object_store_metastore::ObjectStoreMetastore { storage }; + let history = crate::alerts::alert_structs::MTTRHistory { + daily_stats: vec![], + tenant_id: None, + }; + store.put_mttr_history(&history, &None).await.unwrap(); + assert!(store.get_alerts().await.unwrap()[crate::parseable::DEFAULT_TENANT].is_empty()); + } + #[tokio::test] + async fn postgres_health_deadline_regression() { + postgres_regression(3).await; + } + #[tokio::test] + async fn postgres_round_two_backend_regressions() { + postgres_regression(7).await; + } + #[tokio::test] + async fn postgres_manifest_url_regression() { + postgres_regression(0).await; + } + #[tokio::test] + async fn postgres_default_alias_regression() { + postgres_regression(1).await; + } + #[tokio::test] + async fn postgres_empty_bucket_pairing_regression() { + postgres_regression(2).await; + } + #[tokio::test] + async fn local_listing_uses_the_selected_backend() { + initialize_context(); + let dir = test_dir(); + let storage: Arc = Arc::new(LocalFS::new(dir.path().to_owned())); + let store = super::super::object_store_metastore::ObjectStoreMetastore { storage }; + store + .put_stream_json( + &crate::storage::ObjectStoreFormat::default(), + "isolated", + &None, + ) + .await + .unwrap(); + assert_eq!( + store.list_streams(&None).await.unwrap(), + ["isolated".to_owned()].into_iter().collect() + ); + } + #[test] + fn target_tenant_survives_persistence_roundtrip() { + let target: crate::alerts::target::Target = serde_json::from_value(serde_json::json!({ + "name":"named target", "type":"webhook", "endpoint":"https://hook.example/x", "tenantId":"tenant-contract" + })).unwrap(); + let mut legacy = serde_json::to_value(&target).unwrap(); + legacy.as_object_mut().unwrap().remove("tenantId"); + legacy["tenant"] = serde_json::json!("tenant-contract"); + assert_eq!( + serde_json::from_value::(legacy) + .unwrap() + .tenant, + target.tenant + ); + let persisted = crate::storage::object_storage::to_bytes(&target); + let loaded: crate::alerts::target::Target = serde_json::from_slice(&persisted).unwrap(); + assert_eq!( + loaded.tenant, target.tenant, + "stored Target must retain its tenant" + ); + } + // Global tenant/mode lookups belong to the product API. Give each shared + // contract its own process rather than depending on which test forced the + // global context first. The parent owns and removes the context directory. + fn contract_process(name: &str, sql: bool) -> bool { + if std::env::var("P_CONTRACT_ACTIVE_CHILD").as_deref() == Ok(name) { + return false; + } + let context = test_dir(); + let mut command = std::process::Command::new(std::env::current_exe().unwrap()); + command + .args([ + "--exact", + &format!("metastore::metastores::postgres_metastore::contract_tests::{name}"), + "--nocapture", + ]) + .env("P_CONTRACT_ACTIVE_CHILD", name) + .env("P_CONTRACT_CONTEXT_ROOT", context.path()); + if sql { + command.arg("--ignored"); + } + let output = command.output().unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stdout).to_string() + + &String::from_utf8_lossy(&output.stderr) + ); + true + } + #[tokio::test] + async fn object_store_contract() { + if contract_process("object_store_contract", false) { + return; + } + initialize_context(); + let dir = test_dir(); + let storage: Arc = Arc::new(LocalFS::new(dir.path().to_owned())); + contract( + Arc::new(super::super::object_store_metastore::ObjectStoreMetastore { storage }), + false, + ) + .await; + } + #[tokio::test] + #[ignore = "requires PostgreSQL 17; run explicitly with P_TEST_POSTGRES_URL"] + async fn postgres_contract() { + if contract_process("postgres_contract", true) { + return; + } + let url = std::env::var("P_TEST_POSTGRES_URL") + .expect("P_TEST_POSTGRES_URL required for explicitly requested SQL tests"); + let (admin, connection) = tokio_postgres::connect(&url, tokio_postgres::NoTls) + .await + .unwrap(); + let connection_task = tokio::spawn(async move { connection.await.unwrap() }); + let database = format!("contract_{}", ulid::Ulid::new().to_string().to_lowercase()); + admin + .batch_execute(&format!("CREATE DATABASE {database}")) + .await + .unwrap(); + let mut isolated = url::Url::parse(&url).unwrap(); + isolated.set_path(&database); + let url = isolated.to_string(); + use futures::FutureExt; + let outcome = std::panic::AssertUnwindSafe(async { + let dir = test_dir(); + let storage: Arc = Arc::new(LocalFS::new(dir.path().to_owned())); + let options = PostgresOptions { + metastore_postgres_url: Some(url), + metastore_postgres_pool_size: 4, + metastore_postgres_connect_timeout_ms: 5000, + metastore_postgres_statement_timeout_ms: 30000, + metastore_postgres_id: None, + metastore_postgres_ca: None, + }; + let documents = super::super::postgres_documents::PostgresDocuments::new( + &options, + database.clone(), + storage.clone(), + ) + .unwrap(); + let concurrent = super::super::postgres_documents::PostgresDocuments::new( + &options, + database.clone(), + storage.clone(), + ) + .unwrap(); + let (first, second) = tokio::join!(documents.check(&None), concurrent.check(&None)); + first.unwrap(); + second.unwrap(); + let mismatched = super::super::postgres_documents::PostgresDocuments::new( + &options, + "other-bucket".into(), + storage, + ) + .unwrap(); + assert!(mismatched.check(&None).await.is_err()); + documents.verify_database_contract(&options).await; + documents.verify_partition_contract().await; + let documents = Arc::new(documents); + let store = Arc::new(PostgresMetastore { + storage: documents.clone(), + }); + contract(store.clone(), true).await; + documents.verify_deleted_stream_rows().await; + let mut keys = Vec::new(); + fn inventory(root: &std::path::Path, path: &std::path::Path, keys: &mut Vec) { + for entry in std::fs::read_dir(path).unwrap() { + let path = entry.unwrap().path(); + if path.is_dir() { + inventory(root, &path, keys); + } else { + keys.push( + path.strip_prefix(root) + .unwrap() + .to_string_lossy() + .into_owned(), + ); + } + } + } + inventory(dir.path(), dir.path(), &mut keys); + // LocalFS conditional marker creation uses private process lock files; + // these are not object keys on cloud stores (verified by the MinIO smoke). + keys.retain(|key| !key.starts_with(".parseable-metadata-locks/")); + assert_eq!(keys, vec![".parseable/postgres-metastore-id"]); + documents + .put_object( + RelativePath::new("hidden/.stream/.stream.json"), + Bytes::from_static(b"{}"), + &None, + ) + .await + .unwrap(); + documents + .put_object( + RelativePath::new("hidden/.stream/.deleting"), + Bytes::new(), + &None, + ) + .await + .unwrap(); + assert!(!store.list_streams(&None).await.unwrap().contains("hidden")); + store.delete_stream("hidden", &None).await.unwrap(); + drop(store); + drop(documents); + }) + .catch_unwind() + .await; + admin + .batch_execute(&format!("DROP DATABASE {database} WITH (FORCE)")) + .await + .unwrap(); + drop(admin); + connection_task.await.unwrap(); + if let Err(panic) = outcome { + std::panic::resume_unwind(panic); + } + } +} diff --git a/src/metastore/metastores/review_fix_tests.rs b/src/metastore/metastores/review_fix_tests.rs new file mode 100644 index 000000000..aad4c475c --- /dev/null +++ b/src/metastore/metastores/review_fix_tests.rs @@ -0,0 +1,478 @@ +//! Isolated PostgreSQL review regressions. +#[cfg(test)] +mod tests { + //! Isolated regression tests; each database is disposable and unique. + use super::super::{ + postgres_documents::PostgresDocuments, + postgres_metastore::{PostgresMetastore, PostgresOptions}, + }; + use crate::{ + metastore::metastore_traits::Metastore, + storage::{LocalFS, ObjectStorage}, + }; + use bytes::Bytes; + use relative_path::RelativePath; + use std::sync::Arc; + + #[test] + fn empty_url_is_disabled() { + for url in [None, Some("".into()), Some(" ".into())] { + assert!( + !PostgresOptions { + metastore_postgres_url: url, + ..Default::default() + } + .enabled() + ); + } + } + #[test] + fn ca_replaces_public_roots() -> anyhow::Result<()> { + let dir = tempfile::tempdir_in(concat!(env!("CARGO_MANIFEST_DIR"), "/target"))?; + let ca = dir.path().join("ca.pem"); + let key = dir.path().join("key.pem"); + let output = std::process::Command::new("openssl") + .args([ + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-days", + "1", + "-subj", + "/CN=review", + ]) + .arg("-keyout") + .arg(key) + .arg("-out") + .arg(&ca) + .output()?; + assert!(output.status.success()); + assert_eq!( + super::super::postgres_documents::configured_roots(Some(&ca))?.len(), + 1 + ); + assert!(super::super::postgres_documents::configured_roots(None)?.len() > 1); + Ok(()) + } + #[test] + fn unavailable_and_deleting_http_status() { + use crate::storage::ObjectStorageError; + use actix_web::{ResponseError, http::StatusCode}; + let query = crate::handlers::http::query::QueryError::ObjectStorage( + ObjectStorageError::Custom("ordinary storage failure".into()), + ); + assert_eq!(query.status_code(), StatusCode::BAD_REQUEST); + for (error, status) in [ + (ObjectStorageError::StreamDeleting, StatusCode::CONFLICT), + ( + ObjectStorageError::MetastoreUnavailable("outage".into()), + StatusCode::SERVICE_UNAVAILABLE, + ), + ] { + let stream: crate::handlers::http::logstream::error::StreamError = error.into(); + assert_eq!(stream.status_code(), status); + let source = if status == StatusCode::CONFLICT { + ObjectStorageError::StreamDeleting + } else { + ObjectStorageError::MetastoreUnavailable("outage".into()) + }; + let detail = crate::metastore::MetastoreError::from(source).to_detail(); + let nested = crate::handlers::http::ingest::PostError::CreateStream( + crate::handlers::http::logstream::error::CreateStreamError::Storage { + stream_name: "review".into(), + err: ObjectStorageError::MetastoreError(Box::new(detail)), + }, + ); + assert_eq!(nested.status_code(), status); + } + } + #[test] + fn parsed_url_controls_tuning_validation() { + use clap::Parser; + #[derive(Parser)] + struct Args { + #[command(flatten)] + postgres: PostgresOptions, + } + for url in ["", " "] { + let opts = Args::try_parse_from([ + "review", + "--metastore-postgres-url", + url, + "--metastore-postgres-pool-size", + "bad", + "--metastore-postgres-statement-timeout-ms", + "bad", + ]) + .unwrap(); + assert!(!opts.postgres.enabled()); + assert_eq!(opts.postgres.metastore_postgres_statement_timeout_ms, 10000); + } + for (flag, value) in [ + ("--metastore-postgres-pool-size", "0"), + ("--metastore-postgres-connect-timeout-ms", "bad"), + ("--metastore-postgres-statement-timeout-ms", "0"), + ] { + assert!( + Args::try_parse_from([ + "review", + "--metastore-postgres-url", + "postgresql://example/test", + flag, + value + ]) + .is_err() + ); + } + let mut opts = Args::try_parse_from([ + "review", + "--metastore-postgres-url", + "postgresql://example/test", + "--metastore-postgres-pool-size", + "2", + "--metastore-postgres-statement-timeout-ms", + "321", + ]) + .unwrap(); + assert_eq!(opts.postgres.metastore_postgres_pool_size, 2); + assert_eq!(opts.postgres.metastore_postgres_statement_timeout_ms, 321); + opts.try_update_from(["review", "--metastore-postgres-pool-size", "3"]) + .unwrap(); + assert!(opts.postgres.enabled()); + assert_eq!(opts.postgres.metastore_postgres_pool_size, 3); + } + fn initialize_context() { + fn factory() -> crate::parseable::Parseable { + use crate::storage::object_storage::ObjectStorageProvider; + let root = tempfile::tempdir_in(concat!(env!("CARGO_MANIFEST_DIR"), "/target")) + .unwrap() + .keep(); + let provider = Arc::new(crate::storage::FSConfig { root }); + let storage = provider.construct_client(); + let metastore = Arc::new(super::super::object_store_metastore::ObjectStoreMetastore { + storage: storage.clone(), + }); + let context = crate::parseable::Parseable::new( + crate::cli::Options::default(), + #[cfg(feature = "kafka")] + ::parse_from([ + "contract", + ]), + provider, + metastore, + storage, + ); + *context.tenants.write().unwrap() = vec![String::new(), "tenant-contract".into()]; + context + } + let _ = crate::parseable::CONTRACT_PARSEABLE_FACTORY.set(factory); + once_cell::sync::Lazy::force(&crate::parseable::PARSEABLE); + } + + #[tokio::test] + async fn review_regressions() -> anyhow::Result<()> { + let Ok(url) = std::env::var("P_TEST_POSTGRES_URL") else { + eprintln!("SKIP review_regressions: P_TEST_POSTGRES_URL unset"); + return Ok(()); + }; + let dir = tempfile::tempdir_in(concat!(env!("CARGO_MANIFEST_DIR"), "/target"))?; + let storage: Arc = Arc::new(LocalFS::new(dir.path().to_owned())); + let opts = PostgresOptions { + metastore_postgres_url: Some(url.clone()), + metastore_postgres_pool_size: 2, + metastore_postgres_statement_timeout_ms: 2000, + ..Default::default() + }; + let admin_options = PostgresOptions { + metastore_postgres_statement_timeout_ms: 10000, + ..opts.clone() + }; + let admin = PostgresDocuments::new(&admin_options, "admin".into(), storage.clone())?; + let name = format!("review_{}", uuid::Uuid::new_v4().simple()); + admin + .client() + .await? + .batch_execute(&format!("CREATE DATABASE {name}")) + .await?; + initialize_context(); + let result = async { + let mut url = url::Url::parse(&url)?; + url.set_path(&format!("/{name}")); + url.query_pairs_mut().append_pair("options","-c application_name=review-preserved"); + let query = url.query().unwrap().replace('+',"%20"); + url.set_query(Some(&query)); + let opts = PostgresOptions { + metastore_postgres_url: Some(url.to_string()), + ..opts + }; + let db = Arc::new(PostgresDocuments::new( + &opts, + "review".into(), + storage.clone(), + )?); + db.check(&None).await?; + assert_eq!(db.client().await?.query_one("SHOW application_name",&[]).await?.get::<_,String>(0),"review-preserved"); + // With the server timeout disabled, a client timeout must send CancelRequest. + let cancel_db = PostgresDocuments::new(&PostgresOptions { + metastore_postgres_statement_timeout_ms: 300, ..opts.clone() + }, "review".into(), storage.clone())?; + let cancel_client = cancel_db.client().await?; + let pid: i32 = cancel_client.query_one("SELECT pg_backend_pid()", &[]).await?.get(0); + cancel_client.batch_execute("SET statement_timeout=0").await?; + assert!(cancel_client.query("SELECT pg_sleep(30)", &[]).await.is_err()); + let mut still_active = true; + for _ in 0..50 { + still_active = db.client().await?.query_one("SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE pid=$1 AND state='active')", &[&pid]).await?.get(0); + if !still_active { break; } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + assert!(!still_active, "client timeout left its statement running"); + drop(cancel_client); + let store = PostgresMetastore { + storage: db.clone(), + }; + for key in ["odd%stream/manifest%25.json", "odd%stream/dir%25/file"] { + db.put_object(RelativePath::new(key),Bytes::new(),&None).await?; + assert_eq!(db.absolute_url(RelativePath::new(key)).as_ref(),key); + } + let listed = db.list_with_delimiter(Some(object_store::path::Path::parse("odd%stream")?),&None).await?; + assert!(listed.objects.iter().any(|o|o.location.as_ref()=="odd%stream/manifest%25.json")); + assert!(listed.common_prefixes.iter().any(|p|p.as_ref()=="odd%stream/dir%25")); + let time = chrono::Utc::now(); + // Feed the returned locator back through SQL after two overwrites. + // Path helpers use the process context, initialized by the shared suite. + let locator = store.get_manifest_path("logs", time, time, &None).await?; + assert!(!locator.starts_with(dir.path().to_str().unwrap().trim_start_matches('/'))); + for n in [1, 2] { + let mut manifest = store + .get_manifest("logs", time, time, Some(locator.clone()), &None) + .await? + .unwrap_or_default(); + manifest.files.push(crate::catalog::manifest::File { + file_path: format!("{n}.parquet"), + ..Default::default() + }); + store + .put_manifest(&manifest, "logs", time, time, &None) + .await?; + assert_eq!( + store + .get_manifest("logs", time, time, Some(locator.clone()), &None) + .await? + .unwrap() + .files + .len(), + n + ); + } + if let Ok(endpoint) = std::env::var("P_TEST_REVIEW_S3_URL") { + use clap::Parser; + use crate::storage::ObjectStorageProvider; + #[derive(clap::Parser)] + struct S3Args { #[command(flatten)] storage: crate::storage::S3Config } + let provider = S3Args::parse_from(["review","--endpoint-url",&endpoint,"--region","us-east-1","--bucket-name","review","--access-key-id","minioadmin","--secret-key","minioadmin"]).storage; + let s3 = provider.construct_client(); + let cloud = PostgresMetastore {storage:Arc::new(PostgresDocuments::new(&opts,"review".into(),s3)?)}; + let locator = cloud.get_manifest_path("cloud",time,time,&None).await?; + for n in [1,2] { + let mut manifest = cloud.get_manifest("cloud",time,time,Some(locator.clone()),&None).await?.unwrap_or_default(); + manifest.files.push(crate::catalog::manifest::File {file_path:format!("{n}.parquet"),..Default::default()}); + cloud.put_manifest(&manifest,"cloud",time,time,&None).await?; + assert_eq!(cloud.get_manifest("cloud",time,time,Some(locator.clone()),&None).await?.unwrap().files.len(),n); + } + } else {eprintln!("SKIP S3 locator regression: P_TEST_REVIEW_S3_URL unset");} + let key = RelativePath::new("logs/.stream/.stream.json"); + db.put_object( + key, + Bytes::from_static(b"{}"), + &Some("DEFAULT_TENANT".into()), + ) + .await?; + assert!(db.stream_exists("logs", &None).await?); + let other = Some("logs-other".into()); + db.put_object( + RelativePath::new("logs-other/logs/.stream/.stream.json"), + Bytes::new(), + &other, + ) + .await?; + db.put_object( + RelativePath::new("logs-other/exclusive/.stream/.stream.json"), + Bytes::new(), + &other, + ) + .await?; + assert_eq!( + db.list_streams(&None).await?, + std::collections::HashSet::from(["logs".into()]) + ); + assert_eq!(db.list_streams(&other).await?.len(), 2); + // Concurrent marker/create admission; after the marker every write fails. + for _ in 0..10 { + db.delete_object(RelativePath::new("logs/.stream/.deleting"), &None) + .await?; + let (marker, _) = tokio::join!( + db.put_object( + RelativePath::new("logs/.stream/.deleting"), + Bytes::new(), + &None + ), + db.put_object(key, Bytes::new(), &None) + ); + marker?; + assert!(matches!( + db.put_object(key, Bytes::new(), &None).await, + Err(crate::storage::ObjectStorageError::StreamDeleting) + )); + } + db.delete_object(RelativePath::new("logs/.stream/.deleting"), &None).await?; + let mut held = db.client().await?; + let tx = held.transaction().await?; + tx.execute("INSERT INTO parseable_documents(tenant,kind,key,body) VALUES('','document','logs/.stream/.deleting',''::bytea)",&[]).await?; + let create = db.put_object(key,Bytes::new(),&None); + tokio::pin!(create); + assert!(tokio::time::timeout(std::time::Duration::from_millis(50),&mut create).await.is_err()); + tx.commit().await?; + assert!(matches!(create.await,Err(crate::storage::ObjectStorageError::StreamDeleting))); + drop(held); + assert!(db.list_streams(&None).await?.is_empty()); + db.delete_stream("logs", &None).await?; + db.put_object(key, Bytes::new(), &None).await?; + // SQL-only leftovers remain discoverable after physical partition deletion. + let partition = RelativePath::new("logs/date=2020-01-01"); + db.put_object( + RelativePath::new("logs/date=2020-01-01/node.manifest.json"), + Bytes::new(), + &None, + ) + .await?; + assert_eq!( + store.metadata_dates("logs", &None).await?, + vec![".stream", "date=2020-01-01"] + ); + storage.put_object(RelativePath::new("logs/date=2020-01-01/data.parquet"),Bytes::new(),&None).await?; + db.deletion_fault.store(3,std::sync::atomic::Ordering::SeqCst); + assert!(db.delete_partition(partition,&None).await.is_err()); + assert!(storage.get_object(RelativePath::new("logs/date=2020-01-01/data.parquet"),&None).await.is_err()); + assert!(store.metadata_dates("logs",&None).await?.contains(&"date=2020-01-01".into())); + db.deletion_fault.store(0,std::sync::atomic::Ordering::SeqCst); + db.delete_partition(partition, &None).await?; + assert_eq!(store.metadata_dates("logs", &None).await?, vec![".stream"]); + let paths: Vec<_> = (0..200) + .map(|n| format!("batch/{n}.manifest.json")) + .collect(); + for p in &paths { + let manifest = crate::catalog::manifest::Manifest { + files: vec![crate::catalog::manifest::File {file_path:"batch-data.parquet".into(),num_rows:1,..Default::default()}], + ..Default::default() + }; + db.put_object(RelativePath::new(p), crate::catalog::manifest::encode_manifest(&manifest)?, &None).await?; + } + let decoded = store.manifest_batch(&paths, &None).await?.expect("PostgreSQL batch override"); + assert_eq!(decoded.len(),200); + assert!(decoded.values().all(|m|m.as_ref().is_some_and(|m|m.files.len()==1 && m.files[0].num_rows==1))); + assert!( + db.delete_prefix(RelativePath::new("tenant/stream"), &None) + .await + .is_err() + ); + let client = db.client().await?; + client.batch_execute("SET default_transaction_read_only=on").await?; + assert!(client.execute("UPDATE parseable_documents SET kind=kind WHERE false",&[]).await.is_err()); + drop(client); + db.put_object(RelativePath::new("after-failover"),Bytes::new(),&None).await?; + // Server timeout and subsequent pool usability. + assert!( + db.client() + .await? + .query("SELECT pg_sleep(3)", &[]) + .await + .is_err() + ); + assert_eq!( + db.client() + .await? + .query_one("SELECT 1", &[]) + .await? + .get::<_, i32>(0), + 1 + ); + // Drop packets without resetting sockets, including on established sessions. + use std::sync::atomic::{AtomicBool, Ordering}; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let proxy_port = listener.local_addr()?.port(); + let destination = format!("127.0.0.1:{}", url.port().unwrap()); + let paused = Arc::new(AtomicBool::new(false)); + let gate = paused.clone(); + let proxy = tokio::spawn(async move { + while let Ok((incoming, _)) = listener.accept().await { + let Ok(outgoing) = tokio::net::TcpStream::connect(&destination).await else { + continue; + }; + let gate = gate.clone(); + tokio::spawn(async move { + let (mut ir, mut iw) = incoming.into_split(); + let (mut or, mut ow) = outgoing.into_split(); + let forward = async move { + let mut buf = [0; 8192]; + loop { + let n = ir.read(&mut buf).await?; + if n == 0 { + break; + } + while gate.load(Ordering::SeqCst) { + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + ow.write_all(&buf[..n]).await?; + } + Ok::<_, std::io::Error>(()) + }; + tokio::select! { _ = forward => {}, _ = tokio::io::copy(&mut or,&mut iw) => {} } + }); + } + }); + let mut proxy_url = url.clone(); + proxy_url.set_port(Some(proxy_port)).unwrap(); + let proxy_options = PostgresOptions { + metastore_postgres_url: Some(proxy_url.to_string()), + metastore_postgres_statement_timeout_ms: 300, + ..opts.clone() + }; + let proxied = PostgresDocuments::new(&proxy_options, "review".into(), storage.clone())?; + proxied.check(&None).await?; + paused.store(true, Ordering::SeqCst); + for _ in 0..4 { + let started = std::time::Instant::now(); + assert!(tokio::time::timeout(std::time::Duration::from_secs(2),proxied.check_health(&None)).await.expect("readiness exceeded client budget").is_err()); + assert!(started.elapsed() < std::time::Duration::from_secs(2)); + } + paused.store(false, Ordering::SeqCst); + proxied.check_health(&None).await?; + proxy.abort(); + // Nonempty SQL cannot establish a fresh bucket pairing on restart. + storage + .delete_object(RelativePath::new(super::super::migration::PAIR), &None) + .await?; + let restarted = PostgresDocuments::new(&opts, "review".into(), storage.clone())?; + assert!( + restarted + .check(&None) + .await + .unwrap_err() + .to_string() + .contains("nonempty database") + ); + Ok::<_, anyhow::Error>(()) + } + .await; + admin + .client() + .await? + .batch_execute(&format!("DROP DATABASE {name} WITH (FORCE)")) + .await?; + result + } +} diff --git a/src/metastore/migrations/000_identity.sql b/src/metastore/migrations/000_identity.sql new file mode 100644 index 000000000..ee07a7029 --- /dev/null +++ b/src/metastore/migrations/000_identity.sql @@ -0,0 +1,5 @@ +CREATE TABLE IF NOT EXISTS parseable_backend ( + singleton boolean PRIMARY KEY DEFAULT true CHECK (singleton), + id text NOT NULL, bucket text NOT NULL, schema_version integer NOT NULL, + protocol_version integer NOT NULL +); diff --git a/src/metastore/migrations/001_documents.sql b/src/metastore/migrations/001_documents.sql new file mode 100644 index 000000000..e68638e27 --- /dev/null +++ b/src/metastore/migrations/001_documents.sql @@ -0,0 +1,8 @@ +CREATE SEQUENCE IF NOT EXISTS parseable_document_version; +CREATE TABLE IF NOT EXISTS parseable_documents ( + tenant text NOT NULL, kind text NOT NULL, key text NOT NULL, + body bytea NOT NULL, + version bigint NOT NULL DEFAULT nextval('parseable_document_version'), + created_at timestamptz NOT NULL DEFAULT now(), updated_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (tenant, key) +); diff --git a/src/metastore/migrations/002_review_fixes.sql b/src/metastore/migrations/002_review_fixes.sql new file mode 100644 index 000000000..2e06daee5 --- /dev/null +++ b/src/metastore/migrations/002_review_fixes.sql @@ -0,0 +1,20 @@ +-- The caller holds the shared transaction advisory lock. ALTER rebuilds the +-- populated primary key transactionally; existing rows and versions survive. +ALTER TABLE parseable_documents ALTER COLUMN key TYPE text COLLATE "C"; +CREATE INDEX IF NOT EXISTS parseable_stream_inventory ON parseable_documents (tenant,key) WHERE key LIKE '%/.stream/%'; +CREATE OR REPLACE FUNCTION parseable_admit_document() RETURNS trigger LANGUAGE plpgsql AS $$ +DECLARE stream_prefix text; +BEGIN + IF position('/.stream/' in NEW.key) > 0 THEN + stream_prefix := split_part(NEW.key, '/.stream/', 1); + -- Serialize marker and stream writes, including writers on other nodes. + PERFORM pg_advisory_xact_lock(hashtextextended(NEW.tenant || '/' || stream_prefix, 0)); + IF NEW.key NOT LIKE '%/.deleting' AND EXISTS ( + SELECT 1 FROM parseable_documents WHERE tenant=NEW.tenant AND key=stream_prefix || '/.stream/.deleting' + ) THEN RAISE EXCEPTION 'stream deletion pending'; END IF; + END IF; + RETURN NEW; +END $$; +DROP TRIGGER IF EXISTS parseable_admit_document ON parseable_documents; +CREATE TRIGGER parseable_admit_document BEFORE INSERT OR UPDATE ON parseable_documents FOR EACH ROW EXECUTE FUNCTION parseable_admit_document(); +UPDATE parseable_backend SET schema_version=2 WHERE singleton; diff --git a/src/metastore/migrations/003_manifest_admission.sql b/src/metastore/migrations/003_manifest_admission.sql new file mode 100644 index 000000000..1bd6f70c9 --- /dev/null +++ b/src/metastore/migrations/003_manifest_admission.sql @@ -0,0 +1,18 @@ +-- Serialized by the startup advisory lock; replacement preserves populated rows. +CREATE OR REPLACE FUNCTION parseable_admit_document() RETURNS trigger LANGUAGE plpgsql AS $$ +DECLARE stream_prefix text; +BEGIN + IF position('/.stream/' in NEW.key) > 0 THEN + stream_prefix := split_part(NEW.key, '/.stream/', 1); + ELSIF NEW.kind = 'manifest' AND position('/date=' in NEW.key) > 0 THEN + stream_prefix := split_part(NEW.key, '/date=', 1); + END IF; + IF stream_prefix IS NOT NULL THEN + PERFORM pg_advisory_xact_lock(hashtextextended(NEW.tenant || '/' || stream_prefix, 0)); + IF NEW.key NOT LIKE '%/.deleting' AND EXISTS ( + SELECT 1 FROM parseable_documents WHERE tenant=NEW.tenant AND key=stream_prefix || '/.stream/.deleting' + ) THEN RAISE EXCEPTION 'stream deletion pending'; END IF; + END IF; + RETURN NEW; +END $$; +UPDATE parseable_backend SET schema_version=3 WHERE singleton; diff --git a/src/metastore/mod.rs b/src/metastore/mod.rs index aaf086657..62202ae96 100644 --- a/src/metastore/mod.rs +++ b/src/metastore/mod.rs @@ -97,7 +97,7 @@ impl MetastoreError { file_path: None, timestamp: Some(chrono::Utc::now()), metadata: std::collections::HashMap::new(), - status_code: 500, + status_code: self.status_code().as_u16(), }, MetastoreError::JsonParseError(e) => MetastoreErrorDetail { operation: "JsonParseError".to_string(), @@ -168,6 +168,12 @@ impl MetastoreError { pub fn status_code(&self) -> StatusCode { match self { + MetastoreError::ObjectStorageError(ObjectStorageError::MetastoreUnavailable(_)) => { + StatusCode::SERVICE_UNAVAILABLE + } + MetastoreError::ObjectStorageError(ObjectStorageError::StreamDeleting) => { + StatusCode::CONFLICT + } MetastoreError::ObjectStorageError(..) => StatusCode::INTERNAL_SERVER_ERROR, MetastoreError::JsonParseError(..) => StatusCode::INTERNAL_SERVER_ERROR, MetastoreError::ManifestCodecError(..) => StatusCode::INTERNAL_SERVER_ERROR, @@ -196,41 +202,173 @@ mod boundary_tests { | "storage/azure_blob.rs" | "storage/localfs.rs" | "metastore/metastores/object_store_metastore.rs" + | "metastore/metastores/document_metastore.rs" + | "metastore/metastores/migration.rs" + | "metastore/metastores/postgres_documents.rs" + | "metastore/metastores/postgres_metastore.rs" ) { return vec![]; } - let production = source.split("#[cfg(test)]").next().unwrap(); - let compact: String = production.chars().filter(|c| !c.is_whitespace()).collect(); + let tokens = production_tokens(source); let mut errors = Vec::new(); - for operation in [ - ".put_object(", - ".upload_file(", - ".put_metadata_conditional(", - ".delete_metadata_conditional(", - ] { - if compact.contains(operation) { - errors.push(format!( - "{path}: raw {operation} must enter through Metastore" - )); + let mut functions: Vec<(String, usize)> = Vec::new(); + let mut pending = None; + let mut depth = 0usize; + for (i, token) in tokens.iter().enumerate() { + if token == "fn" { + pending = tokens.get(i + 1).cloned(); + } + if token == "{" { + depth += 1; + if let Some(name) = pending.take() { + functions.push((name, depth)); + } + } else if token == "}" { + if functions.last().is_some_and(|(_, d)| *d == depth) { + functions.pop(); + } + depth = depth.saturating_sub(1); + } else if token == ";" { + pending = None; + } + let function = functions + .last() + .map(|(name, _)| name.as_str()) + .unwrap_or(""); + if i > 0 && tokens[i - 1] == "fn" { + continue; + } + if tokens.get(i + 1).map(String::as_str) != Some("(") { + continue; + } + let allowed = path == "storage/object_storage.rs" + && matches!( + (function, token.as_str()), + ("upload_single_parquet_file", "upload_multipart") + | ("upload_single_parquet_file", "delete_object") + | ("compensate_uploaded_parquet", "delete_object") + ); + if !allowed + && matches!( + token.as_str(), + "put_object" + | "upload_file" + | "put_metadata_conditional" + | "delete_metadata_conditional" + | "delete_object" + | "delete_prefix" + | "upload_multipart" + | "put" + | "put_opts" + | "put_multipart" + | "put_multipart_opts" + | "delete" + | "delete_stream" + ) + { + // Generic raw object_store names need an explicit receiver/type hint; + // otherwise ordinary HTTP delete handlers would be false positives. + let generic = matches!( + token.as_str(), + "put" + | "put_opts" + | "put_multipart" + | "put_multipart_opts" + | "delete" + | "delete_stream" + ); + let context = tokens[i.saturating_sub(5)..i].join(""); + if !generic || context.contains("object_store") || context.contains("ObjectStore::") + { + errors.push(format!( + "{path}::{function}: raw {token} must enter through Metastore" + )); + } } } - for (operation, allowed_path) in [ - (".delete_object(", "storage/object_storage.rs"), - (".delete_prefix(", ""), - ] { - let count = compact.matches(operation).count(); - if count != usize::from(path == allowed_path) { - errors.push(format!("{path}: unexpected raw {operation} ({count})")); + errors + } + + // Lexical policy, not Rust type/provenance analysis: aliases, wrappers, + // generated macro bodies and unrecognised mutation APIs can evade this. + // Allowed backend modules and the two named Parquet sites require review. + fn production_tokens(source: &str) -> Vec { + // Skip comments and literals before token matching, retaining punctuation. + let pattern = regex::Regex::new(r####"(?s)//[^\n]*|/\*|(?:br|cr|r)(\#*)"|"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])'|[A-Za-z_][A-Za-z_0-9]*|[^\s]"####).unwrap(); + let mut tokens = Vec::new(); + let mut offset = 0; + while let Some(m) = pattern.find(&source[offset..]) { + let text = m.as_str(); + let start = offset + m.start(); + offset += m.end(); + if text == "/*" { + let mut depth = 1; + while offset < source.len() && depth > 0 { + if source[offset..].starts_with("/*") { + depth += 1; + offset += 2; + } else if source[offset..].starts_with("*/") { + depth -= 1; + offset += 2; + } else { + offset += source[offset..].chars().next().unwrap().len_utf8(); + } + } + } else if (text.starts_with('r') || text.starts_with("br") || text.starts_with("cr")) + && text.ends_with('"') + { + let closing = format!( + "\"{}", + "#".repeat(text.bytes().filter(|b| *b == b'#').count()) + ); + offset = source[offset..] + .find(&closing) + .map(|n| offset + n + closing.len()) + .unwrap_or(source.len()); + } else if text.starts_with("//") || text.starts_with('"') || text.starts_with('\'') { + // Literal content cannot contain a production call. + } else { + tokens.push(source[start..offset].to_owned()); } } - let multipart = compact.matches(".upload_multipart(").count(); - let allowed = usize::from(path == "storage/object_storage.rs"); - if multipart != allowed { - errors.push(format!( - "{path}: unexpected multipart publisher ({multipart})" - )); + let mut production = Vec::new(); + let mut i = 0; + while i < tokens.len() { + let cfg = ["#", "[", "cfg", "(", "test", ")", "]", "mod"]; + if tokens[i..] + .iter() + .map(String::as_str) + .take(cfg.len()) + .eq(cfg) + { + i += cfg.len(); + while i < tokens.len() && tokens[i] != "{" && tokens[i] != ";" { + i += 1; + } + if tokens.get(i).map(String::as_str) == Some(";") { + i += 1; + continue; + } + let mut depth = 0; + while i < tokens.len() { + if tokens[i] == "{" { + depth += 1; + } + if tokens[i] == "}" { + depth -= 1; + if depth == 0 { + i += 1; + break; + } + } + i += 1; + } + } else { + production.push(tokens[i].clone()); + i += 1; + } } - errors + production } fn inspect(root: &Path, path: &Path, errors: &mut Vec) { @@ -260,12 +398,17 @@ mod boundary_tests { fn guard_rejects_literal_and_computed_metadata_writes() { for source in [ "store.put_object(stream_json_path(name, tenant), bytes, tenant).await", + "ObjectStorage::put_object(store, path, bytes, tenant)", + "#[cfg(test)] mod tests; fn production() { store.put_object(path); }", + "#[cfg(test)] mod tests {} fn production() { store.put_object(path); }", + "object_store::ObjectStore::put(store, path, bytes)", "store\n . put_object (path, bytes, tenant).await", "store.put_metadata_conditional(path, bytes, condition, tenant).await", "store.upload_file(key, local, tenant).await", ] { assert!(!violations("application.rs", source).is_empty()); } + assert!(violations("application.rs", r###"fn ok() { let s = r#"store.put_object(path)"#; /* outer /* nested */ store.put_object(path) */ }"###).is_empty()); assert!( violations( "application.rs", diff --git a/src/parseable/mod.rs b/src/parseable/mod.rs index 363e3360e..a757afc99 100644 --- a/src/parseable/mod.rs +++ b/src/parseable/mod.rs @@ -154,8 +154,15 @@ fn resolve_telemetry_type( } } +#[cfg(test)] +pub(crate) static CONTRACT_PARSEABLE_FACTORY: OnceCell Parseable> = OnceCell::new(); + /// Shared state of the Parseable server. pub static PARSEABLE: Lazy = Lazy::new(|| { + #[cfg(test)] + if let Some(factory) = CONTRACT_PARSEABLE_FACTORY.get() { + return factory(); + } let metastore = METASTORE.get_or_init(|| { // Prompt user for missing env vars before clap validates them. // Values are set in env but NOT saved to disk yet. @@ -179,9 +186,31 @@ pub static PARSEABLE: Lazy = Lazy::new(|| { StorageOptions::Blob(args) => args.storage.construct_client(), StorageOptions::Gcs(args) => args.storage.construct_client(), }; + let options = match &cli.storage { + StorageOptions::Local(a) => &a.options, + StorageOptions::S3(a) => &a.options, + StorageOptions::Blob(a) => &a.options, + StorageOptions::Gcs(a) => &a.options, + }; + if options.postgres.enabled() { + use crate::metastore::metastores::{ + postgres_documents::PostgresDocuments, postgres_metastore::PostgresMetastore, + }; + let endpoint = match &cli.storage { + StorageOptions::Local(a) => a.storage.get_endpoint(), + StorageOptions::S3(a) => a.storage.get_endpoint(), + StorageOptions::Blob(a) => a.storage.get_endpoint(), + StorageOptions::Gcs(a) => a.storage.get_endpoint(), + }; + let documents = PostgresDocuments::new(&options.postgres, endpoint, storage) + .unwrap_or_else(|e| panic!("PostgreSQL metastore configuration failed: {e}")); + return Arc::new(PostgresMetastore { + storage: Arc::new(documents), + }) as Arc; + } tracing::warn!("creating objectstoremetastore"); let metastore = ObjectStoreMetastore { storage }; - Arc::new(metastore) + Arc::new(metastore) as Arc }); // Prompt user for missing env vars before clap validates them. @@ -502,6 +531,9 @@ impl Parseable { .await .map_err(crate::metastore::MetastoreError::into_storage_error)? { + if self.options.postgres.enabled() { + return Err(crate::storage::ObjectStorageError::StreamDeleting.into()); + } return Ok(false); } if !self @@ -1250,6 +1282,7 @@ impl Parseable { .map_err(crate::metastore::MetastoreError::into_storage_error)?; let mut recognized_tenants = vec![]; + let mut invalid_tenants = vec![]; // validate the possible presence of tenant storage metadata for tenant_id in dirs.into_iter() { @@ -1267,24 +1300,26 @@ impl Parseable { recognized_tenants.push(tenant_id); } else if !is_multi_tenant { } else { - // found an invalid multi-tenant directory, delete - tracing::warn!( - "Found invalid tenant directory with multi-tenant mode- {tenant_id}.\nDeleting." - ); - - // delete from storage - if let Err(e) = self - .metastore - .delete_tenant(&tenant_id) - .await - .map_err(crate::metastore::MetastoreError::into_storage_error) - { - tracing::error!( - "Unable to delete invalid directory- {tenant_id} with error-\n{e}" - ); - }; + invalid_tenants.push(tenant_id); } } + // Discovery and validation must finish before destructive cleanup. + for tenant_id in invalid_tenants { + // found an invalid multi-tenant directory, delete + tracing::warn!( + "Found invalid tenant directory with multi-tenant mode- {tenant_id}.\nDeleting." + ); + + // delete from storage + if let Err(e) = self + .metastore + .delete_tenant(&tenant_id) + .await + .map_err(crate::metastore::MetastoreError::into_storage_error) + { + tracing::error!("Unable to delete invalid directory- {tenant_id} with error-\n{e}"); + }; + } if is_multi_tenant { let mut t = self diff --git a/src/query/stream_schema_provider.rs b/src/query/stream_schema_provider.rs index 125b5cf41..de4b6f01a 100644 --- a/src/query/stream_schema_provider.rs +++ b/src/query/stream_schema_provider.rs @@ -837,7 +837,26 @@ async fn collect_from_snapshot_cached( // Manifest fetches are one object store GET each and are pure round trip // latency, so run them concurrently instead of serially. `buffered` keeps // the snapshot ordering, which the `rev()` below depends on. - let manifest_files = futures_util::stream::iter(snapshot.manifests(time_filters)) + let items = snapshot.manifests(time_filters); + let mut uncached = Vec::new(); + for item in &items { + if let Some(cache) = cache + && cache + .manifests + .lock() + .await + .contains_key(&item.manifest_path) + { + continue; + } + uncached.push(item.manifest_path.clone()); + } + let batch = PARSEABLE + .metastore + .manifest_batch(&uncached, tenant_id) + .await + .map_err(|e| DataFusionError::Plan(e.to_string()))?; + let manifest_files = futures_util::stream::iter(items) .map(|manifest_item| { let ManifestItem { time_lower_bound, @@ -845,6 +864,7 @@ async fn collect_from_snapshot_cached( manifest_path, .. } = manifest_item; + let batch = batch.as_ref(); async move { if let Some(cache) = cache && let Some(manifest) = @@ -852,17 +872,21 @@ async fn collect_from_snapshot_cached( { return Ok(manifest); } - let manifest_opt = PARSEABLE - .metastore - .get_manifest( - stream_name, - time_lower_bound, - time_upper_bound, - Some(manifest_path.clone()), - tenant_id, - ) - .await - .map_err(|e| DataFusionError::Plan(e.to_string()))?; + let manifest_opt = if let Some(batch) = batch { + batch.get(&manifest_path).cloned().flatten() + } else { + PARSEABLE + .metastore + .get_manifest( + stream_name, + time_lower_bound, + time_upper_bound, + Some(manifest_path.clone()), + tenant_id, + ) + .await + .map_err(|e| DataFusionError::Plan(e.to_string()))? + }; if manifest_opt.is_none() { tracing::warn!( "Manifest missing for stream={} [{:?} - {:?}]", diff --git a/src/storage/azure_blob.rs b/src/storage/azure_blob.rs index b0a32b1a0..db02efaf7 100644 --- a/src/storage/azure_blob.rs +++ b/src/storage/azure_blob.rs @@ -1186,6 +1186,21 @@ impl ObjectStorage for BlobStore { .collect::>()) } + async fn metadata_hidden_prefix( + &self, + prefix: object_store::path::Path, + ) -> Result)>, ObjectStorageError> { + use futures::TryStreamExt; + let objects: Vec<_> = self.client.list(Some(&prefix)).try_collect().await?; + objects + .into_iter() + .map(|o| { + crate::storage::object_storage::decode_metadata_key(o.location.as_ref()) + .map(|key| (key, o.last_modified)) + }) + .collect() + } + async fn list_with_delimiter( &self, prefix: Option, diff --git a/src/storage/gcs.rs b/src/storage/gcs.rs index 7e105ab31..55458fe10 100644 --- a/src/storage/gcs.rs +++ b/src/storage/gcs.rs @@ -1165,6 +1165,21 @@ impl ObjectStorage for Gcs { .collect::>()) } + async fn metadata_hidden_prefix( + &self, + prefix: object_store::path::Path, + ) -> Result)>, ObjectStorageError> { + use futures::TryStreamExt; + let objects: Vec<_> = self.client.list(Some(&prefix)).try_collect().await?; + objects + .into_iter() + .map(|o| { + crate::storage::object_storage::decode_metadata_key(o.location.as_ref()) + .map(|key| (key, o.last_modified)) + }) + .collect() + } + async fn list_with_delimiter( &self, prefix: Option, diff --git a/src/storage/localfs.rs b/src/storage/localfs.rs index 0fe525e7f..ba8a38b9e 100644 --- a/src/storage/localfs.rs +++ b/src/storage/localfs.rs @@ -538,7 +538,10 @@ impl ObjectStorage for LocalFS { tenant_id: &Option, ) -> Result, ObjectStorageError> { let prefix = RelativePathBuf::from_iter([ - tenant_id.as_deref().unwrap_or(""), + tenant_id + .as_deref() + .filter(|t| !t.is_empty() && *t != crate::parseable::DEFAULT_TENANT) + .unwrap_or(""), stream_name, STREAM_ROOT_DIRECTORY, ]); @@ -1084,6 +1087,41 @@ impl ObjectStorage for LocalFS { url::Url::parse("file:///").unwrap() } + async fn metadata_inventory( + &self, + ) -> Result)>, ObjectStorageError> { + let mut dirs = vec![self.root.clone()]; + let mut keys = Vec::new(); + while let Some(dir) = dirs.pop() { + for entry in std::fs::read_dir(dir)? { + let entry = entry?; + let meta = entry.metadata()?; + if entry.file_type()?.is_symlink() { + continue; + } + if meta.is_dir() { + if !entry.file_name().to_string_lossy().starts_with("hour=") + && !entry.file_name().to_string_lossy().starts_with("minute=") + { + dirs.push(entry.path()); + } + } else if !entry.file_name().to_string_lossy().ends_with(".parquet") { + keys.push(( + entry + .path() + .strip_prefix(&self.root) + .unwrap() + .to_string_lossy() + .into_owned(), + meta.modified()?.into(), + )); + } + } + } + keys.sort_by(|a, b| a.0.cmp(&b.0)); + Ok(keys) + } + async fn list_with_delimiter( &self, _prefix: Option, diff --git a/src/storage/mod.rs b/src/storage/mod.rs index 647723762..9d098c87c 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -561,6 +561,10 @@ impl Default for ObjectStoreFormat { #[derive(Debug, thiserror::Error)] pub enum ObjectStorageError { + #[error("PostgreSQL metastore unavailable: {0}")] + MetastoreUnavailable(String), + #[error("stream deletion pending; retry DELETE before creating the stream")] + StreamDeleting, // no such key inside the object storage #[error("{0} not found")] NoSuchKey(String), @@ -607,6 +611,24 @@ pub enum ObjectStorageError { }, } +impl ObjectStorageError { + pub(crate) fn http_status(&self) -> actix_web::http::StatusCode { + use actix_web::http::StatusCode; + match self { + Self::MetastoreUnavailable(_) => StatusCode::SERVICE_UNAVAILABLE, + Self::StreamDeleting => StatusCode::CONFLICT, + Self::MetastoreError(detail) + if detail.operation == "ObjectStorageError" + && matches!(detail.status_code, 409 | 503) => + { + StatusCode::from_u16(detail.status_code) + .unwrap_or(StatusCode::INTERNAL_SERVER_ERROR) + } + _ => StatusCode::INTERNAL_SERVER_ERROR, + } + } +} + pub fn to_object_store_path(path: &RelativePath) -> Path { Path::from(path.as_str()) } diff --git a/src/storage/object_storage.rs b/src/storage/object_storage.rs index f1acfbcb9..1476a586d 100644 --- a/src/storage/object_storage.rs +++ b/src/storage/object_storage.rs @@ -662,6 +662,107 @@ pub trait ObjectStorage: Debug + Send + Sync + 'static { fn absolute_url(&self, prefix: &RelativePath) -> object_store::path::Path; fn store_url(&self) -> url::Url; + async fn check_migration_guard(&self) -> Result<(), ObjectStorageError> { + // ObjectStorage is object safe; use its primitives directly here. + for key in [ + crate::metastore::metastores::migration::BUSY, + crate::metastore::metastores::migration::PAIR, + ] { + match self.get_object(RelativePath::new(key), &None).await { + Ok(ref id) + if key.ends_with("postgres-metastore-id") + && std::env::var("P_ALLOW_PAIRED_BUCKET") + .is_ok_and(|v| v.as_bytes() == id.as_ref()) => {} + Ok(_) => { + return Err(ObjectStorageError::Custom(format!( + "bucket has {key}: PostgreSQL pairing or offline migration; complete export before object-store startup (P_ALLOW_PAIRED_BUCKET= overrides pairing only)" + ))); + } + Err(ObjectStorageError::NoSuchKey(_)) => {} + Err(ObjectStorageError::IoError(e)) if e.kind() == std::io::ErrorKind::NotFound => { + } + Err(e) => { + return Err(ObjectStorageError::Custom(format!( + "cannot probe migration marker {key}: {e}; retry after restoring bucket access" + ))); + } + } + } + Ok(()) + } + + /// Raw inventory for offline document migration; retains keys, never bodies. + async fn metadata_inventory( + &self, + ) -> Result)>, ObjectStorageError> { + let mut pending = vec![None]; + let mut keys = Vec::new(); + use futures::{StreamExt, TryStreamExt}; + while !pending.is_empty() { + let round = std::mem::take(&mut pending); + let listings: Vec<_> = futures::stream::iter(round) + .map(|prefix| async move { + let listing = self.list_with_delimiter(prefix).await?; + let mut children = Vec::new(); + let mut found = Vec::new(); + for prefix in listing.common_prefixes { + if prefix.parts().any(|c| { + c.as_ref().starts_with("hour=") || c.as_ref().starts_with("minute=") + }) { + continue; + } + if prefix.filename().is_some_and(|p| p.starts_with('.')) { + found.extend(self.metadata_hidden_prefix(prefix).await?); + } else { + children.push(Some(prefix)); + } + } + for object in listing + .objects + .into_iter() + .filter(|o| !o.location.as_ref().ends_with(".parquet")) + { + found.push(( + decode_metadata_key(object.location.as_ref())?, + object.last_modified, + )); + } + Ok::<_, ObjectStorageError>((children, found)) + }) + .buffer_unordered(16) + .try_collect() + .await?; + for (children, found) in listings { + pending.extend(children); + keys.extend(found); + } + } + keys.sort_by(|a, b| a.0.cmp(&b.0)); + Ok(keys) + } + + async fn metadata_hidden_prefix( + &self, + prefix: object_store::path::Path, + ) -> Result)>, ObjectStorageError> { + let mut pending = vec![prefix]; + let mut keys = Vec::new(); + while let Some(prefix) = pending.pop() { + let listing = self.list_with_delimiter(Some(prefix)).await?; + pending.extend(listing.common_prefixes); + keys.extend( + listing + .objects + .into_iter() + .map(|o| { + decode_metadata_key(o.location.as_ref()).map(|key| (key, o.last_modified)) + }) + .collect::, _>>()?, + ); + } + Ok(keys) + } + async fn list_with_delimiter( &self, prefix: Option, @@ -1264,13 +1365,70 @@ pub trait ObjectStorage: Debug + Send + Sync + 'static { let stream = PARSEABLE.get_or_create_stream(stream_name, &tenant_id); let upload_context = UploadContext::new(stream); + if PARSEABLE.options.postgres.enabled() + && PARSEABLE + .metastore + .is_stream_deleting(stream_name, &tenant_id) + .await + .map_err(|e| ObjectStorageError::Custom(e.to_string()))? + { + for path in upload_context.stream.parquet_files() { + let filename = path.file_name().unwrap().to_string_lossy(); + let key = stream_relative_path( + stream_name, + &filename, + &upload_context.custom_partition, + &tenant_id, + ); + compensate_uploaded_parquet(self, &key, &tenant_id).await?; + remove_uploaded_staged_file(path); + } + for path in upload_context.stream.schema_files() { + remove_uploaded_staged_file(path); + } + return Ok(()); + } // Process parquet files concurrently and collect results let uploaded_files = process_parquet_files(&upload_context, stream_name, tenant_id.clone()).await?; // Update snapshot with collected manifest files - update_snapshot_with_manifests(stream_name, &uploaded_files, &tenant_id).await?; + if let Err(e) = + update_snapshot_with_manifests(stream_name, &uploaded_files, &tenant_id).await + { + if PARSEABLE.options.postgres.enabled() + && e.http_status() == actix_web::http::StatusCode::CONFLICT + { + // The marker can land between upload and manifest publication. + // Compensate every upload in this batch, including partially + // published manifests, before retiring its local staging files. + for file in &uploaded_files { + let filename = file.file_path.file_name().unwrap().to_string_lossy(); + let key = stream_relative_path( + stream_name, + &filename, + &upload_context.custom_partition, + &tenant_id, + ); + match compensate_uploaded_parquet(self, &key, &tenant_id).await { + Ok(()) | Err(ObjectStorageError::NoSuchKey(_)) => {} + Err(cleanup) => { + for pending in &uploaded_files { + ACTIVE_OBJECT_STORE_SYNC_FILES.remove(&pending.file_path); + } + return Err(cleanup); + } + } + } + cleanup_uploaded_staged_files(uploaded_files).await; + for path in upload_context.stream.schema_files() { + remove_uploaded_staged_file(path); + } + return Ok(()); + } + return Err(e); + } // Calculate stats after snapshot update so uploads are visible before stats work starts. calculate_stats_for_uploaded_files( @@ -1290,6 +1448,24 @@ pub trait ObjectStorage: Debug + Send + Sync + 'static { } } +// Data-only compensation for an upload rejected by the metastore deletion guard. +async fn compensate_uploaded_parquet( + storage: &(impl ObjectStorage + ?Sized), + key: &str, + tenant: &Option, +) -> Result<(), ObjectStorageError> { + if !key.ends_with(".parquet") { + return Err(ObjectStorageError::Custom( + "upload compensation requires parquet key".into(), + )); + } + match storage.delete_object(RelativePath::new(key), tenant).await { + Ok(()) | Err(ObjectStorageError::NoSuchKey(_)) => Ok(()), + Err(ObjectStorageError::IoError(e)) if e.kind() == std::io::ErrorKind::NotFound => Ok(()), + result => result, + } +} + /// Processes parquet files concurrently and returns stats status and manifest files #[tracing::instrument( name = "object_store.process_parquet_files", @@ -1322,6 +1498,13 @@ async fn process_parquet_files( .filter_map(claim_staged_file_for_upload) .collect(); + let postgres = PARSEABLE.options.postgres.enabled(); + let retirement_paths = if postgres { + parquet_paths.clone() + } else { + Vec::new() + }; + let mut total_size: u64 = 0; let mut min_dt: Option> = None; let mut max_dt: Option> = None; @@ -1358,7 +1541,37 @@ async fn process_parquet_files( } // Collect results from all upload tasks - collect_upload_results(join_set).await + let result = collect_upload_results(join_set, postgres).await; + if result.is_err() && postgres { + // All tasks have settled, so no successful sibling upload can publish + // after compensation. Keep staging retryable if deletion is unconfirmed. + for path in &retirement_paths { + ACTIVE_OBJECT_STORE_SYNC_FILES.remove(path); + } + if PARSEABLE + .metastore + .is_stream_deleting(stream_name, &tenant_id) + .await + .is_ok_and(|pending| pending) + { + for path in retirement_paths { + let filename = path.file_name().unwrap().to_string_lossy(); + let key = stream_relative_path( + stream_name, + &filename, + &upload_context.custom_partition, + &tenant_id, + ); + compensate_uploaded_parquet(object_store.as_ref(), &key, &tenant_id).await?; + remove_uploaded_staged_file(path); + } + for path in upload_context.stream.schema_files() { + remove_uploaded_staged_file(path); + } + return Ok(Vec::new()); + } + } + result } /// Spawns an individual parquet file upload task @@ -1400,8 +1613,10 @@ async fn spawn_parquet_upload_task( /// Collects results from all upload tasks async fn collect_upload_results( mut join_set: JoinSet>, + drain_on_error: bool, ) -> Result, ObjectStorageError> { let mut uploaded_files = Vec::new(); + let mut failure = None; while let Some(result) = join_set.join_next().await { match result { @@ -1423,16 +1638,30 @@ async fn collect_upload_results( Ok(Err((path, e))) => { error!("Error uploading parquet file: {e}"); ACTIVE_OBJECT_STORE_SYNC_FILES.remove(&path); - return Err(e); + if !drain_on_error { + return Err(e); + } + if failure.is_none() { + failure = Some(e); + } } Err(e) => { error!("Task panicked: {e}"); - return Err(ObjectStorageError::UnhandledError(Box::new(e))); + let error = ObjectStorageError::UnhandledError(Box::new(e)); + if !drain_on_error { + return Err(error); + } + if failure.is_none() { + failure = Some(error); + } } } } - Ok(uploaded_files) + match failure { + Some(error) => Err(error), + None => Ok(uploaded_files), + } } /// Updates snapshot with collected manifest files @@ -1752,14 +1981,20 @@ fn stream_metadata_prefix( tenant_id: &Option, ) -> object_store::path::Path { object_store::path::Path::from_iter([ - tenant_id.as_deref().unwrap_or(""), + tenant_id + .as_deref() + .filter(|t| !t.is_empty() && *t != crate::parseable::DEFAULT_TENANT) + .unwrap_or(""), stream_name, STREAM_ROOT_DIRECTORY, ]) } pub fn schema_path(stream_name: &str, tenant_id: &Option) -> RelativePathBuf { - let tenant = tenant_id.as_deref().unwrap_or(""); + let tenant = tenant_id + .as_deref() + .filter(|t| !t.is_empty() && *t != crate::parseable::DEFAULT_TENANT) + .unwrap_or(""); if PARSEABLE.options.mode == Mode::Ingest { let id = INGESTOR_META .get() @@ -1775,7 +2010,10 @@ pub fn schema_path(stream_name: &str, tenant_id: &Option) -> RelativePat #[inline(always)] pub fn stream_json_path(stream_name: &str, tenant_id: &Option) -> RelativePathBuf { - let tenant = tenant_id.as_deref().unwrap_or(""); + let tenant = tenant_id + .as_deref() + .filter(|t| !t.is_empty() && *t != crate::parseable::DEFAULT_TENANT) + .unwrap_or(""); if PARSEABLE.options.mode == Mode::Ingest { let id = INGESTOR_META .get() @@ -1805,7 +2043,10 @@ pub fn stream_json_path(stream_name: &str, tenant_id: &Option) -> Relati /// scan to find it. #[inline(always)] pub fn tombstone_path(stream_name: &str, tenant_id: &Option) -> RelativePathBuf { - let tenant = tenant_id.as_deref().unwrap_or(""); + let tenant = tenant_id + .as_deref() + .filter(|t| !t.is_empty() && *t != crate::parseable::DEFAULT_TENANT) + .unwrap_or(""); RelativePathBuf::from_iter([ TOMBSTONE_ROOT_DIRECTORY, tenant, @@ -1855,7 +2096,10 @@ pub async fn list_tombstoned_streams( storage: &(impl ObjectStorage + ?Sized), tenant_id: &Option, ) -> Result, ObjectStorageError> { - let tenant = tenant_id.as_deref().unwrap_or(""); + let tenant = tenant_id + .as_deref() + .filter(|t| !t.is_empty() && *t != crate::parseable::DEFAULT_TENANT) + .unwrap_or(""); let root = RelativePathBuf::from_iter([TOMBSTONE_ROOT_DIRECTORY, tenant]); let candidates = storage.list_dirs_relative(&root, tenant_id).await?; @@ -1876,7 +2120,10 @@ pub fn filter_path( filter_file_name: &str, tenant_id: &Option, ) -> RelativePathBuf { - let root = tenant_id.as_deref().unwrap_or(""); + let root = tenant_id + .as_deref() + .filter(|t| !t.is_empty() && *t != crate::parseable::DEFAULT_TENANT) + .unwrap_or(""); RelativePathBuf::from_iter([ root, USERS_ROOT_DIR, @@ -1896,7 +2143,10 @@ pub fn parseable_json_path() -> RelativePathBuf { /// TODO: Needs to be updated for distributed mode #[inline(always)] pub fn alert_json_path(alert_id: Ulid, tenant_id: &Option) -> RelativePathBuf { - if let Some(tenant_id) = tenant_id.as_ref() { + if let Some(tenant_id) = tenant_id + .as_ref() + .filter(|t| !t.is_empty() && t.as_str() != crate::parseable::DEFAULT_TENANT) + { RelativePathBuf::from_iter([ tenant_id, ALERTS_ROOT_DIRECTORY, @@ -1910,7 +2160,10 @@ pub fn alert_json_path(alert_id: Ulid, tenant_id: &Option) -> RelativePa /// TODO: Needs to be updated for distributed mode #[inline(always)] pub fn target_json_path(target_id: &Ulid, tenant_id: &Option) -> RelativePathBuf { - let root = tenant_id.as_deref().unwrap_or(""); + let root = tenant_id + .as_deref() + .filter(|t| !t.is_empty() && *t != crate::parseable::DEFAULT_TENANT) + .unwrap_or(""); RelativePathBuf::from_iter([ root, SETTINGS_ROOT_DIRECTORY, @@ -1923,7 +2176,10 @@ pub fn target_json_path(target_id: &Ulid, tenant_id: &Option) -> Relativ /// Format: "/settings/outbound_http_policy.json" #[inline(always)] pub fn outbound_http_policy_json_path(tenant_id: &Option) -> RelativePathBuf { - let root = tenant_id.as_deref().unwrap_or(""); + let root = tenant_id + .as_deref() + .filter(|t| !t.is_empty() && *t != crate::parseable::DEFAULT_TENANT) + .unwrap_or(""); RelativePathBuf::from_iter([root, SETTINGS_ROOT_DIRECTORY, "outbound_http_policy.json"]) } @@ -1931,7 +2187,10 @@ pub fn outbound_http_policy_json_path(tenant_id: &Option) -> RelativePat /// Format: ".alerts/alert_state_{alert_id}.json" #[inline(always)] pub fn alert_state_json_path(alert_id: Ulid, tenant_id: &Option) -> RelativePathBuf { - let root = tenant_id.as_deref().unwrap_or(""); + let root = tenant_id + .as_deref() + .filter(|t| !t.is_empty() && *t != crate::parseable::DEFAULT_TENANT) + .unwrap_or(""); RelativePathBuf::from_iter([ root, ALERTS_ROOT_DIRECTORY, @@ -1943,7 +2202,10 @@ pub fn alert_state_json_path(alert_id: Ulid, tenant_id: &Option) -> Rela /// Format: ".alerts/mttr.json" #[inline(always)] pub fn mttr_json_path(tenant_id: &Option) -> RelativePathBuf { - if let Some(tenant) = tenant_id.as_ref() { + if let Some(tenant) = tenant_id + .as_ref() + .filter(|t| !t.is_empty() && t.as_str() != crate::parseable::DEFAULT_TENANT) + { RelativePathBuf::from_iter([tenant, ALERTS_ROOT_DIRECTORY, "mttr.json"]) } else { RelativePathBuf::from_iter([ALERTS_ROOT_DIRECTORY, "mttr.json"]) @@ -2479,3 +2741,91 @@ mod query_reload_tests { assert_eq!(stream.schema_files().len(), 1); } } + +/// Decode exactly one object_store path encoding layer. +pub(crate) fn decode_metadata_key(key: &str) -> Result { + let bytes = key.as_bytes(); + let mut out = Vec::with_capacity(bytes.len()); + let mut i = 0; + while i < bytes.len() { + if bytes[i] == b'%' { + let digits = bytes + .get(i + 1..i + 3) + .and_then(|d| std::str::from_utf8(d).ok()) + .and_then(|d| u8::from_str_radix(d, 16).ok()) + .ok_or_else(|| { + ObjectStorageError::Custom("invalid percent-encoded metadata key".into()) + })?; + out.push(digits); + i += 3; + } else { + out.push(bytes[i]); + i += 1; + } + } + String::from_utf8(out) + .map_err(|e| ObjectStorageError::Custom(format!("invalid UTF-8 metadata key: {e}"))) +} + +#[cfg(test)] +mod metadata_key_decode_tests { + use super::*; + #[tokio::test] + async fn postgres_batches_settle_sibling_uploads_before_error_cleanup() { + for drain in [false, true] { + let completed = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let flag = completed.clone(); + let mut uploads = JoinSet::new(); + uploads.spawn(async { + Err(( + PathBuf::from("failed.parquet"), + ObjectStorageError::Custom("upload failed".into()), + )) + }); + uploads.spawn(async move { + tokio::time::sleep(std::time::Duration::from_millis(30)).await; + flag.store(true, std::sync::atomic::Ordering::SeqCst); + Ok(UploadResult { + file_path: PathBuf::from("sibling.parquet"), + manifest_file: None, + }) + }); + assert!(collect_upload_results(uploads, drain).await.is_err()); + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + assert_eq!(completed.load(std::sync::atomic::Ordering::SeqCst), drain); + } + } + #[tokio::test] + async fn compensation_removes_only_parquet_and_tolerates_missing_objects() { + let dir = tempfile::tempdir().unwrap(); + let storage = crate::storage::LocalFS::new(dir.path().to_owned()); + let key = "logs/date=2026-01-01/file.parquet"; + storage + .put_object(RelativePath::new(key), Bytes::from_static(b"data"), &None) + .await + .unwrap(); + compensate_uploaded_parquet(&storage, key, &None) + .await + .unwrap(); + compensate_uploaded_parquet(&storage, key, &None) + .await + .unwrap(); + assert!( + storage + .get_object(RelativePath::new(key), &None) + .await + .is_err() + ); + assert!( + compensate_uploaded_parquet(&storage, "logs/.stream/.stream.json", &None) + .await + .is_err() + ); + } + #[test] + fn invalid_encoded_keys_never_panic() { + assert!(decode_metadata_key("%FF").is_err()); + assert!(decode_metadata_key("%é").is_err()); + assert_eq!(decode_metadata_key("caf%C3%A9").unwrap(), "café"); + } +} diff --git a/src/storage/retention.rs b/src/storage/retention.rs index 8f88f7044..cdd06b97b 100644 --- a/src/storage/retention.rs +++ b/src/storage/retention.rs @@ -242,9 +242,34 @@ mod action { let retain_until = get_retain_until(Utc::now().date_naive(), days as u64); - let Ok(mut dates) = store.list_dates(&stream_name, tenant_id).await else { - return; + let mut dates = match store.list_dates(&stream_name, tenant_id).await { + Ok(dates) => dates, + Err(crate::storage::ObjectStorageError::NoSuchKey(_)) + if PARSEABLE.options.postgres.enabled() => + { + Vec::new() + } + Err(crate::storage::ObjectStorageError::IoError(e)) + if PARSEABLE.options.postgres.enabled() + && e.kind() == std::io::ErrorKind::NotFound => + { + Vec::new() + } + Err(_) => return, }; + match PARSEABLE + .metastore + .metadata_dates(&stream_name, tenant_id) + .await + { + Ok(sql_dates) => dates.extend(sql_dates), + Err(e) => { + error!("retention SQL inventory failed: {e}"); + return; + } + } + dates.sort(); + dates.dedup(); dates.retain(|date| date.starts_with("date")); let dates_to_delete = dates .into_iter() diff --git a/src/storage/s3.rs b/src/storage/s3.rs index cd8f766ce..ceedb740c 100644 --- a/src/storage/s3.rs +++ b/src/storage/s3.rs @@ -1419,6 +1419,21 @@ impl ObjectStorage for S3 { self.bucket.clone() } + async fn metadata_hidden_prefix( + &self, + prefix: object_store::path::Path, + ) -> Result)>, ObjectStorageError> { + use futures::TryStreamExt; + let objects: Vec<_> = self.client.list(Some(&prefix)).try_collect().await?; + objects + .into_iter() + .map(|o| { + crate::storage::object_storage::decode_metadata_key(o.location.as_ref()) + .map(|key| (key, o.last_modified)) + }) + .collect() + } + async fn list_with_delimiter( &self, prefix: Option, diff --git a/src/users/dashboards.rs b/src/users/dashboards.rs index ed4c6c851..faedfd456 100644 --- a/src/users/dashboards.rs +++ b/src/users/dashboards.rs @@ -92,7 +92,11 @@ fn deserialize_dataset<'de, D: serde::Deserializer<'de>>( impl MetastoreObject for Dashboard { fn get_object_path(&self) -> String { - let root = self.tenant_id.as_deref().unwrap_or(""); + let root = self + .tenant_id + .as_deref() + .filter(|t| !t.is_empty() && *t != crate::parseable::DEFAULT_TENANT) + .unwrap_or(""); RelativePathBuf::from_iter([ root, USERS_ROOT_DIR, @@ -415,3 +419,21 @@ mod tests { } } } + +#[cfg(test)] +mod default_tenant_path_tests { + use super::*; + #[test] + fn dashboard_default_tenant_aliases_share_a_key() { + let mut dashboard = Dashboard { + author: Some("owner".into()), + dashboard_id: Some(Ulid::new()), + ..Default::default() + }; + let path = dashboard.get_object_path(); + for alias in ["", crate::parseable::DEFAULT_TENANT] { + dashboard.tenant_id = Some(alias.into()); + assert_eq!(dashboard.get_object_path(), path); + } + } +}