From b96259b05238bae99ca48d5bee3cee9943b399d5 Mon Sep 17 00:00:00 2001 From: VibeSWE <338779468+VibeSWE@users.noreply.github.com> Date: Wed, 11 Nov 2026 08:18:21 -0600 Subject: [PATCH 1/9] feat: opt-in PostgreSQL metastore with import, export and verify tooling Adds a PostgreSQL implementation of the Metastore trait, selected with P_METASTORE_POSTGRES_URL. Object storage remains the default metastore and needs no new configuration. In PostgreSQL mode metadata documents live in a parseable_documents table; parquet data stays in the bucket, paired to the database by a marker object. - Shared document layer for both backends; native row versions and atomic conditional writes; embedded migrations under an advisory lock. - Stream lifecycle in PostgreSQL mode: durable deleting marker, crash-safe delete ordering, creation refused while a deletion is pending. - Client-side deadlines on every SQL call, tenant-scoped key-only listings, C-collated keys so prefix reads use the index, batched manifest reads. - Offline --metadata-migration import|verify|export with abort, import-time digests, and startup guards for partially migrated state. - Contract suite run against both backends, fault and restart tests, a stronger metastore boundary guard, end-to-end scripts and a CI workflow. Also fixes four defects that exist in object-store mode: target tenant lost on reload, alert listing including MTTR history, local-filesystem listing ignoring the selected instance, and startup tenant cleanup deleting before discovery had fully succeeded. Squashed from local development branches. Machine-specific paths and host names in the generated evidence files were redacted. Co-Authored-By: Claude Fable 5.2 --- .github/workflows/postgres-metastore.yaml | 101 + .gitignore | 3 + Cargo.lock | 339 +- Cargo.toml | 4 + docs/metastore-boundary-audit.md | 14 + docs/postgres-metastore.md | 645 ++++ .../validation/postgres-hardening-checks.json | 213 ++ .../postgres-import-list-requests.html | 13 + .../postgres-metastore-requests.html | 1 + docs/validation/postgres-metastore-smoke.json | 3307 +++++++++++++++++ docs/validation/postgres-review-explain.txt | 133 + docs/validation/postgres-review-smoke.json | 205 + scripts/postgres-metastore-migration-smoke.py | 191 + scripts/postgres-metastore-review-explain.sql | 17 + scripts/postgres-metastore-smoke.py | 474 +++ src/alerts/target.rs | 2 + src/banner.rs | 8 +- src/cli.rs | 5 + src/handlers/http/ingest.rs | 19 +- src/handlers/http/logstream.rs | 20 +- .../http/modal/query/querier_logstream.rs | 10 +- src/handlers/http/oidc.rs | 2 +- src/handlers/http/query.rs | 1 + src/handlers/http/rbac.rs | 2 +- src/handlers/http/role.rs | 2 +- src/handlers/http/users/dashboards.rs | 3 +- src/handlers/http/users/filters.rs | 2 +- src/main.rs | 25 + src/metastore/metastore_traits.rs | 16 + .../metastores/document_metastore.rs | 1670 +++++++++ src/metastore/metastores/migration.rs | 685 ++++ src/metastore/metastores/mod.rs | 11 + .../metastores/object_store_metastore.rs | 1877 +--------- src/metastore/metastores/postgres_client.rs | 160 + .../metastores/postgres_documents.rs | 1504 ++++++++ .../metastores/postgres_metastore.rs | 1666 +++++++++ src/metastore/metastores/review_fix_tests.rs | 458 +++ src/metastore/migrations/000_identity.sql | 5 + src/metastore/migrations/001_documents.sql | 8 + src/metastore/migrations/002_review_fixes.sql | 20 + src/metastore/mod.rs | 196 +- src/parseable/mod.rs | 69 +- src/query/stream_schema_provider.rs | 18 +- src/storage/azure_blob.rs | 18 + src/storage/gcs.rs | 18 + src/storage/localfs.rs | 35 + src/storage/mod.rs | 22 + src/storage/object_storage.rs | 101 + src/storage/retention.rs | 29 +- src/storage/s3.rs | 18 + 50 files changed, 12541 insertions(+), 1824 deletions(-) create mode 100644 .github/workflows/postgres-metastore.yaml create mode 100644 docs/postgres-metastore.md create mode 100644 docs/validation/postgres-hardening-checks.json create mode 100644 docs/validation/postgres-import-list-requests.html create mode 100644 docs/validation/postgres-metastore-requests.html create mode 100644 docs/validation/postgres-metastore-smoke.json create mode 100644 docs/validation/postgres-review-explain.txt create mode 100644 docs/validation/postgres-review-smoke.json create mode 100755 scripts/postgres-metastore-migration-smoke.py create mode 100644 scripts/postgres-metastore-review-explain.sql create mode 100755 scripts/postgres-metastore-smoke.py create mode 100644 src/metastore/metastores/document_metastore.rs create mode 100644 src/metastore/metastores/migration.rs create mode 100644 src/metastore/metastores/postgres_client.rs create mode 100644 src/metastore/metastores/postgres_documents.rs create mode 100644 src/metastore/metastores/postgres_metastore.rs create mode 100644 src/metastore/metastores/review_fix_tests.rs create mode 100644 src/metastore/migrations/000_identity.sql create mode 100644 src/metastore/migrations/001_documents.sql create mode 100644 src/metastore/migrations/002_review_fixes.sql diff --git a/.github/workflows/postgres-metastore.yaml b/.github/workflows/postgres-metastore.yaml new file mode 100644 index 000000000..e9081d432 --- /dev/null +++ b/.github/workflows/postgres-metastore.yaml @@ -0,0 +1,101 @@ +name: PostgreSQL metastore +on: + pull_request: + push: + branches: [main] + workflow_dispatch: +permissions: + contents: read +concurrency: + group: postgres-metastore-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} +jobs: + contract-and-smoke: + runs-on: ubuntu-24.04 + timeout-minutes: 90 + services: + postgres: + image: postgres:17-alpine + env: + POSTGRES_PASSWORD: contract + POSTGRES_DB: contract + ports: + - 55439:5432 + options: >- + --health-cmd "pg_isready -U postgres" + --health-interval 5s --health-timeout 5s --health-retries 12 + env: + P_TEST_REVIEW_S3_URL: http://127.0.0.1:59087 + P_TEST_POSTGRES_URL: postgresql://postgres:contract@127.0.0.1:55439/contract?sslmode=disable + CARGO_BUILD_JOBS: 2 + P_BUILD_NO_UI: 1 + steps: + - uses: actions/checkout@v4 + - name: Prepare Linux build resources + run: | + sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc /opt/hostedtoolcache/CodeQL + if sudo fallocate -l 16G /swap.build && sudo chmod 600 /swap.build \ + && sudo mkswap /swap.build && sudo swapon /swap.build; then + echo "Added 16G swap" + else + sudo rm -f /swap.build + echo "::warning title=Swap::Could not add swap; continuing without it" + fi + free -h + df -h / + - uses: ./.github/actions/setup-rust-ci + with: + cache-key: postgres-metastore + tools: nextest + - name: Start isolated MinIO + run: | + docker run -d --name migration-minio -p 127.0.0.1:59087:9000 -e MINIO_ROOT_USER=minioadmin -e MINIO_ROOT_PASSWORD=minioadmin quay.io/parseablehq/minio:edge server /data + docker exec migration-minio mkdir -p /data/review /data/migration-unit + curl --fail --retry 30 --retry-connrefused --retry-delay 1 http://127.0.0.1:59087/minio/health/ready + - name: Migration and regressions on PostgreSQL 17 and MinIO + run: 'cargo test --locked --lib metastore:: -- --nocapture --test-threads=1' + - name: LocalFS migration and legacy locators + run: env -u P_TEST_REVIEW_S3_URL cargo test --locked --lib migration::tests::migration_round_trip_postgres -- --nocapture + - name: Deadline against paused PostgreSQL + env: + P_TEST_POSTGRES_CONTAINER: ${{ job.services.postgres.id }} + run: cargo test --locked --lib postgres_health_deadline_regression -- --nocapture --test-threads=1 + - name: Shared contract and SQL fault tests (must execute) + shell: bash + run: | + set -o pipefail + cargo test --locked --lib contract_tests::object_store_contract -- --nocapture 2>&1 | tee postgres-contract.log + cargo test --locked --lib contract_tests::postgres_contract -- --ignored --nocapture 2>&1 | tee -a postgres-contract.log + cargo test --locked --lib contract_tests::postgres_startup_cleanup -- --ignored --nocapture 2>&1 | tee -a postgres-contract.log + python3 - <<'PY' + from pathlib import Path + text = Path('postgres-contract.log').read_text() + assert 'postgres_contract ... ok' in text, text + assert 'postgres_startup_cleanup ... ok' in text, text + assert '0 failed; 0 ignored;' in text, text + assert 'skipping PostgreSQL' not in text, text + PY + - name: Build smoke binary + run: cargo build --locked + - name: Object store, SQL plaintext and verified TLS end to end + run: python3 scripts/postgres-metastore-smoke.py + - name: Offline migration end to end + run: python3 scripts/postgres-metastore-migration-smoke.py + - name: Offline migration with verified PostgreSQL TLS + env: + P_MIGRATION_TLS: 1 + run: python3 scripts/postgres-metastore-migration-smoke.py + - name: Remove isolated MinIO + if: always() + run: docker rm -f migration-minio + - uses: actions/upload-artifact@v4 + if: always() + with: + name: postgres-metastore-evidence + path: | + postgres-contract.log + .postgres-smoke/results.json + .postgres-smoke/runs/*.json + docs/validation/postgres-metastore-smoke.json + .postgres-smoke/*.log + if-no-files-found: error diff --git a/.gitignore b/.gitignore index 034f0d47e..9b84ba6e7 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,6 @@ cache # Local credentials and review notes *.pem pr-*.md + +# Isolated PostgreSQL smoke artifacts +.postgres-smoke/ diff --git a/Cargo.lock b/Cargo.lock index 002c99c52..677747710 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1592,7 +1592,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6139a8597ed92cf816dfb33f5dd6cf0bb93a6adc938f11039f371bc5bcd26c3" dependencies = [ "chrono", - "phf", + "phf 0.12.1", ] [[package]] @@ -1725,6 +1725,12 @@ dependencies = [ "serde_core", ] +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + [[package]] name = "const-oid" version = "0.10.2" @@ -2760,6 +2766,62 @@ dependencies = [ "stacker", ] +[[package]] +name = "deadpool" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e98a7e119cd347f4201e1159b19831029e203e2d8b790547708e8157b4acf1e" +dependencies = [ + "deadpool-runtime", + "tokio", +] + +[[package]] +name = "deadpool-postgres" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65a536565624b97fc19f758cd01b15d12908d3344425066efc8162236fbd3749" +dependencies = [ + "async-trait", + "deadpool", + "getrandom 0.4.2", + "tokio", + "tokio-postgres", + "tracing", +] + +[[package]] +name = "deadpool-runtime" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2657f61fb1dd8bf37a8d51093cc7cee4e77125b22f7753f49b289f831bec2bae" +dependencies = [ + "tokio", +] + +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid 0.9.6", + "der_derive", + "flagset", + "zeroize", +] + +[[package]] +name = "der_derive" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8034092389675178f570469e6c3b0465d3d30b4505c294a6550db47f3c17ad18" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "deranged" version = "0.5.5" @@ -2876,7 +2938,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4850db49bf08e663084f7fb5c87d202ef91a3907271aff24a94eb97ff039153c" dependencies = [ "block-buffer 0.12.0", - "const-oid", + "const-oid 0.10.2", "crypto-common 0.2.1", "ctutils", ] @@ -2986,6 +3048,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "fallible-iterator" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" + [[package]] name = "fastrand" version = "1.9.0" @@ -3023,6 +3091,12 @@ version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" +[[package]] +name = "flagset" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7ac824320a75a52197e8f2d787f6a38b6718bb6897a35142d749af3c0e8f4fe" + [[package]] name = "flatbuffers" version = "25.12.19" @@ -3875,11 +3949,12 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.83" +version = "0.3.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "464a3709c7f55f1f721e5389aa6ea4e3bc6aba669353300af094b29ffbdde1d8" +checksum = "8dc09d2154098bbf4ef54eab6b117a3c24cbbffe94046b16ee6d6d4510584d3a" dependencies = [ - "once_cell", + "cfg-if", + "futures-util", "wasm-bindgen", ] @@ -4394,6 +4469,15 @@ dependencies = [ "objc2-core-foundation", ] +[[package]] +name = "objc2-system-configuration" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7216bd11cbda54ccabcab84d523dc93b858ec75ecfb3a7d89513fa22464da396" +dependencies = [ + "objc2-core-foundation", +] + [[package]] name = "object" version = "0.32.2" @@ -4738,6 +4822,7 @@ dependencies = [ "dashmap", "datafusion", "datafusion-proto", + "deadpool-postgres", "derive_more 1.0.0", "erased-serde", "fs_extra", @@ -4798,6 +4883,8 @@ dependencies = [ "tikv-jemalloc-ctl", "tikv-jemallocator", "tokio", + "tokio-postgres", + "tokio-postgres-rustls", "tokio-stream", "tokio-util", "tonic", @@ -4813,6 +4900,7 @@ dependencies = [ "url", "uuid", "vergen-gitcl", + "webpki-roots", "xxhash-rust", "zstd", ] @@ -4885,7 +4973,17 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "913273894cec178f401a31ec4b656318d95473527be05c0752cc41cdc32be8b7" dependencies = [ - "phf_shared", + "phf_shared 0.12.1", +] + +[[package]] +name = "phf" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" +dependencies = [ + "phf_shared 0.13.1", + "serde", ] [[package]] @@ -4897,6 +4995,15 @@ dependencies = [ "siphasher", ] +[[package]] +name = "phf_shared" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project" version = "1.1.10" @@ -4941,6 +5048,35 @@ version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" +[[package]] +name = "postgres-protocol" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08808e3c483c46e999108051c78334f473d5adb59d78bb80a1268c7e6aa6c514" +dependencies = [ + "base64 0.22.1", + "byteorder", + "bytes", + "fallible-iterator", + "hmac", + "md-5 0.11.0", + "memchr", + "rand 0.10.1", + "sha2 0.11.0", + "stringprep", +] + +[[package]] +name = "postgres-types" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "851ca9db4932932d69f3ea811b1abe63087a0f740a47692619dd40d4899b68be" +dependencies = [ + "bytes", + "fallible-iterator", + "postgres-protocol", +] + [[package]] name = "potential_utf" version = "0.1.4" @@ -6151,6 +6287,16 @@ dependencies = [ "vob 3.0.6", ] +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + [[package]] name = "sqlparser" version = "0.62.0" @@ -6220,6 +6366,17 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + [[package]] name = "strsim" version = "0.11.1" @@ -6491,6 +6648,27 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" +[[package]] +name = "tls_codec" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de2e01245e2bb89d6f05801c564fa27624dbd7b1846859876c7dad82e90bf6b" +dependencies = [ + "tls_codec_derive", + "zeroize", +] + +[[package]] +name = "tls_codec_derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d2e76690929402faae40aebdda620a2c0e25dd6d3b9afe48867dfd95991f4bd" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "tokio" version = "1.52.3" @@ -6519,6 +6697,46 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "tokio-postgres" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a528f7d280f6d5b9cd149635c8705b0dd049754bc67d81d31fa25169a93809d3" +dependencies = [ + "async-trait", + "byteorder", + "bytes", + "fallible-iterator", + "futures-channel", + "futures-util", + "log", + "parking_lot", + "percent-encoding", + "phf 0.13.1", + "pin-project-lite", + "postgres-protocol", + "postgres-types", + "rand 0.10.1", + "socket2 0.6.4", + "tokio", + "tokio-util", + "whoami", +] + +[[package]] +name = "tokio-postgres-rustls" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c2ad44aa0ae96db89c4742212ed41645b2f597311ff6e1945542a4d9fadc2fb" +dependencies = [ + "rustls", + "sha2 0.11.0", + "tokio", + "tokio-postgres", + "tokio-rustls", + "x509-cert", +] + [[package]] name = "tokio-rustls" version = "0.26.4" @@ -6844,12 +7062,33 @@ version = "2.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539" +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + [[package]] name = "unicode-ident" version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" +[[package]] +name = "unicode-normalization" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75cd3fb03d5288b3ba9a1ee103ac7e9cbb59f0b03d64f8ddfcc04dd42758a85d" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "993ad287eb20e8ba755312801cfe1413e0f25d604075863acaddca10b04c1833" + [[package]] name = "unicode-segmentation" version = "1.12.0" @@ -7096,6 +7335,15 @@ version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" +[[package]] +name = "wasi" +version = "0.14.7+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "883478de20367e224c0090af9cf5f9fa85bed63a95c1abf3afc5c083ebc06e8c" +dependencies = [ + "wasip2", +] + [[package]] name = "wasip2" version = "1.0.1+wasi-0.2.4" @@ -7114,14 +7362,22 @@ dependencies = [ "wit-bindgen 0.51.0", ] +[[package]] +name = "wasite" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fe902b4a6b8028a753d5424909b764ccf79b7a209eac9bf97e59cda9f71a42" +dependencies = [ + "wasi 0.14.7+wasi-0.2.4", +] + [[package]] name = "wasm-bindgen" -version = "0.2.106" +version = "0.2.130" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d759f433fa64a2d763d1340820e46e111a7a5ab75f993d1852d70b03dbb80fd" +checksum = "a0181c14ddd6c7b8b0b0132a5f1ea2f32be0d893cd3ea97b6e3b28886b8a3d1c" dependencies = [ "cfg-if", - "once_cell", "rustversion", "wasm-bindgen-macro", "wasm-bindgen-shared", @@ -7129,22 +7385,20 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.56" +version = "0.4.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "836d9622d604feee9e5de25ac10e3ea5f2d65b41eac0d9ce72eb5deae707ce7c" +checksum = "5e14e0327cb5368ad60ccb22d04a9827a693c95a926d8d125e9fa39e0baec501" dependencies = [ - "cfg-if", "js-sys", - "once_cell", + "tokio", "wasm-bindgen", - "web-sys", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.106" +version = "0.2.130" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48cb0d2638f8baedbc542ed444afc0644a29166f1595371af4fecf8ce1e7eeb3" +checksum = "6e931a02a39abdbf66966fc35902b5ed58344750a70885bed4ebb3d8eaf28ed9" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -7152,22 +7406,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.106" +version = "0.2.130" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cefb59d5cd5f92d9dcf80e4683949f15ca4b511f4ac0a6e14d4e1ac60c6ecd40" +checksum = "2ca4b54280e455bd831e33a6b8e734baf432fe63fbc35c42ac14ce67a043a53a" dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.117", + "syn 3.0.4", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.106" +version = "0.2.130" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbc538057e648b67f72a982e708d485b2efa771e1ac05fec311f9f63e5800db4" +checksum = "3b1f9c3798725016c6ee5f2f57869ec0831e8bbc0ab0310a86e787e97f41389e" dependencies = [ "unicode-ident", ] @@ -7221,9 +7475,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.83" +version = "0.3.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b32828d774c412041098d182a8b38b16ea816958e07cf40eec2bc080ae137ac" +checksum = "d7d179596ff5ec0f6de4db64d54011bde6289cc49603d3a6e4ce3f329207ab88" dependencies = [ "js-sys", "wasm-bindgen", @@ -7248,6 +7502,19 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "whoami" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "626c4bac6755d76ffc12cb01b2eac751db1996b9e0041de9aa02c8c211ddc82c" +dependencies = [ + "libc", + "libredox", + "objc2-system-configuration", + "wasite", + "web-sys", +] + [[package]] name = "winapi" version = "0.3.9" @@ -7754,6 +8021,18 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" +[[package]] +name = "x509-cert" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1301e935010a701ae5f8655edc0ad17c44bad3ac5ce8c39185f75453b720ae94" +dependencies = [ + "const-oid 0.9.6", + "der", + "spki", + "tls_codec", +] + [[package]] name = "xmlparser" version = "0.13.6" @@ -7835,6 +8114,20 @@ name = "zeroize" version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] [[package]] name = "zerotrie" diff --git a/Cargo.toml b/Cargo.toml index 87d216f30..b62d95ff0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,10 @@ categories = ["logs", "observability", "metrics", "traces"] build = "build.rs" [dependencies] +tokio-postgres = "0.7" +deadpool-postgres = "0.14" +tokio-postgres-rustls = "0.14" +webpki-roots = "1" # Arrow and DataFusion ecosystem arrow = "59.2.0" arrow-array = "59.2.0" diff --git a/docs/metastore-boundary-audit.md b/docs/metastore-boundary-audit.md index 087ae05e2..1a0a3c850 100644 --- a/docs/metastore-boundary-audit.md +++ b/docs/metastore-boundary-audit.md @@ -144,3 +144,17 @@ Final validation on this branch: **1,379 passed, 0 failed, 4 ignored** (1,383 discovered), 79.65 seconds. - Both source-guard tests and both LocalFS compatibility fixtures passed. - `cargo fmt` and `git diff --check` passed. Cargo target remained in this worktree. + + +## Combined PostgreSQL lifecycle implementation + +The original inventory above records the routing branch. In the combined branch, +its seven missing methods are implemented by PostgreSQL as documented in +[postgres-metastore.md](postgres-metastore.md#lifecycle-and-metadata-boundary). +Backend primitives moved to `document_metastore.rs` and `postgres_documents.rs` +are allowed by the source guard. The real MinIO lifecycle smoke supplements +that source policy with bucket key assertions. All A operations now select the +configured backend. B/C/D entries remain data plane, local recovery state or +unused compatibility primitives. No additional active shared metadata bypass +was found. PostgreSQL stream listing suppresses durable deletion markers, and +HTTP DELETE permits explicit retry of such a stream after restart. diff --git a/docs/postgres-metastore.md b/docs/postgres-metastore.md new file mode 100644 index 000000000..41032b31f --- /dev/null +++ b/docs/postgres-metastore.md @@ -0,0 +1,645 @@ +# PostgreSQL document metastore + +Opt in using `--metastore-postgres-url` / `P_METASTORE_POSTGRES_URL` on any +storage subcommand. No setting is required for the default object-store backend. +For an isolated local PostgreSQL, for example: + +``` +P_METASTORE_POSTGRES_URL='postgresql://user:password@127.0.0.1:55432/parseable?sslmode=disable' \ +P_BUILD_NO_UI=1 target/debug/parseable s3-store +``` + +Keep the URL in a secret. Options Debug output redacts it. All participants +(standalone, ingest, query and prism) must select the same backend/database. +Neither startup nor runtime falls back to bucket metadata after selecting SQL. +Builds use runtime queries and need no `DATABASE_URL` or live database. + +| Option suffix (also `P_METASTORE_POSTGRES_`) | Default | +| --- | --- | +| URL | unset; object-store metadata | +| POOL_SIZE | 16 connections per process | +| CONNECT_TIMEOUT_MS | 5000; also bounds pool wait/create/recycle | +| STATEMENT_TIMEOUT_MS | 10000; also bounds SQL lock waits | +| ID | unset; optionally pin the backend UUID | +| CA | unset; optionally trust only a PEM CA bundle | + +CLI spellings use `--metastore-postgres-` plus lowercase hyphenated suffix. +TLS defaults to required with certificate and hostname verification against +webpki roots. URL `sslmode=disable` explicitly opts out. `prefer` and `allow` are +upgraded to required, so TLS cannot silently downgrade. Client certificates are +not implemented in this slice. + +## Pairing and migrations + +The first initialization creates a UUID and records the configured storage +endpoint/bucket identity. After committing SQL and releasing its connection, it +conditionally creates `.parseable/postgres-metastore-id` in that bucket. Existing +markers must contain the database UUID. Concurrent starts use a transactional +PostgreSQL advisory lock and atomic bucket creation. Partial initialization is +retryable, with no metadata service admitted until both checks succeed. +A preexisting marker paired to another database fails startup. A database bound +to another bucket/endpoint also fails. `ID` can pin an expected existing UUID. +Endpoint aliases are intentionally conservative: changing an endpoint spelling +requires an explicit operator pairing migration, even for the same bucket. +Never delete the marker to bypass a mismatch. Back up the database and marker +together; restoring a fresh empty database is not a metadata recovery procedure. +The marker is identity protection, not fencing against old binaries. + +`parseable_backend` records singleton UUID, bucket identity, schema and protocol +version. Embedded migrations `001_documents.sql` and `002_review_fixes.sql` are forward-only, transactional, +and serialized across starting nodes. Unknown versions fail closed. +`parseable_documents` stores `(tenant, key)` primary keys, an advisory `kind`, +faithful BYTEA bodies, BIGINT version, and creation/update timestamps. Keys retain +object-store path spelling including tenant prefixes. The default tenant uses +an empty tenant column, exactly as root bucket paths do. Versions come from a +backend-wide sequence, avoiding delete/recreate ABA; gaps are expected. Atomic +INSERT/UPDATE/DELETE preconditions implement dashboard versions. + +The documents table and broad `document` kind are interim. Manifests remain +compressed documents, with existing whole-document read/modify/write behavior. +This is not atomic catalog publication, retention, compaction, reader pinning or +multi-document transactions. Future normalized catalog tables can coexist with +the document keyspace without changing existing document versions. + +## Compatibility and data plane + +The implementations share the existing metastore path/encoding/behavior code. +SQL primitive operations preserve raw bytes and sort keys with C collation. +Not-found GETs use the existing NoSuchKey error; optional getters retain their +existing conversions. Default filesystem behavior is retained for object-store +metastore. SQL listing always uses its document inventory, including local-store +standalone. Local filesystem multitenancy is still unsupported product policy. + +Existing ambiguous behavior is retained: default alert-state listing uses the +literal `DEFAULT_TENANT` directory although state writes use root paths; global +outbound-policy maps use the empty-string default key unlike other maps; manifest +inventory completion order is unspecified. SQL directory enumeration returns +immediate child names; S3's current `list_dirs_relative` flattens ancestor path +components as well, an apparent implementation accident that does not define +useful directory semantics. SQL cannot see externally written bucket metadata. + +Parquet, local staging/WAL, hot-tier data and object-store discovery remain in +the data plane. Lifecycle routing is included in this branch. Existing buckets need a separate +import before opting in. Do not switch a populated deployment in place. + +An import tool needs the original tenant and exact path key, original bytes +(including compressed manifest bytes), a validated kind, and object-store mode +and owner naming context. It should allocate fresh native versions, validate +schemas/JSON and cross-document manifest references, quiesce old writers, and +pair/verify the bucket UUID before admission. Upstream ETags cannot be reused as +SQL versions. Offline import/export is described below; normalized file catalog conversion is +separate work. + +## Validation + +Use `PATH=$HOME/.cargo/bin:$PATH P_BUILD_NO_UI=1 CARGO_BUILD_JOBS=4` and keep +`target` in the checkout. Run one Cargo compilation at a time. + +``` +cargo test --lib contract_tests -- --nocapture +P_TEST_POSTGRES_URL='postgresql://postgres:password@127.0.0.1:55432/test?sslmode=disable' \ + cargo test --lib contract_tests -- --nocapture +cargo build +python3 scripts/postgres-metastore-smoke.py +cargo clippy --all-targets +``` + +Use an empty disposable database for the SQL contract. With no test URL the SQL +test reports a skip and succeeds. The smoke script creates PostgreSQL 17 Alpine (including a verified TLS run) +and MinIO containers on randomly selected spare ports, isolated buckets and +staging, and removes its containers in `finally`. It runs stream creation, +ingestion, upload, retention configuration/startup execution, dashboard creation, +restart, query/ingest cold loading, repeated queries, deletion/recreation and +dependency outages in both modes. A separate SQL database exercises a self-signed +CA through `P_METASTORE_POSTGRES_CA`; an unrelated CA must reject startup. `.postgres-smoke/results.json` records key inventory, SQL rows and +storage metric deltas (background storage activity can contribute to deltas). + +## Lifecycle and metadata boundary + +All active mutable shared metadata operations now enter `Metastore`, including +health probes, cold stream discovery, stream/partition deletion and tenant +startup discovery. The companion audit is in [metastore-boundary-audit.md](metastore-boundary-audit.md). +The shared document layer retains encodings and existing CRUD behavior; the +seven lifecycle methods are supplied by each backend. + +* `stream_exists`: SQL inventory of canonical/ingestor stream JSON under the + tenant's stream prefix; no bucket metadata lookup. +* `is_stream_deleting`: durable SQL `.stream/.deleting` document, visible across + processes and restarts, checked before lazy loading. +* `delete_stream`: commit the SQL marker, delete the bucket data prefix, then + delete all stream rows (including the marker) in one SQL statement. Before + data deletion a crash leaves hidden metadata and recoverable data; afterwards + it leaves hidden metadata with missing data. Retrying completes either case. + Once complete, recreation starts without old schemas/manifests/settings. +* `delete_stream_partition`: the caller first publishes snapshot removal, then + the backend deletes partition data, then SQL rows under that date prefix. + A crash can leave unreferenced data or unreferenced manifest rows; retry + removes them. No snapshot ever gets republished here. +* `list_tenant_candidates`: union of nonempty SQL tenant columns and nonhidden + root bucket prefixes. SQL-only tenants are retained when their deployment + metadata validates; data-only/invalid candidates follow existing startup + cleanup, deleting both data and rows. With multitenancy disabled these are + physical candidates only; cloud object stores remain the supported product + configuration for multitenancy. +* `check_health`: checks SQL `SELECT 1` and reads the bucket pairing marker on + every probe. Readiness returns 503 if either fails. Liveness retains its + process/shutdown semantics. Pool waits/connects and SQL statements are bounded + by the configured timeouts; there is no metadata fallback. +* `read_latency`: elapsed combined SQL and bucket probe, labeled as such in the + banner; probe errors are logged. It is not a standalone SQL latency measure. + +The bucket's only metadata object in PostgreSQL mode is the immutable pairing +id `.parseable/postgres-metastore-id`. Its read is an intentional health probe. +LocalFS conditional marker creation additionally uses a private filesystem lock +file; this is local coordination, not a second metadata authority. +Parquet reads/writes, date/hour/minute discovery for bounds and retention, +legacy file listing, hot-tier planning and physical occupancy remain bucket +operations. Dormant tombstone/legacy provider helpers remain unused by +application code. Staging/WAL and hot-tier local recovery files remain local. +No other active metadata bypass was found in the combined inventories. + +Deletion markers reject new stream metadata writes in PostgreSQL mode; deletion does not fully fence concurrent writers. Existing whole-document read/modify/ +write races, retention/publication races and check-then-load races remain. +Automatic resumption of interrupted deletion is not implemented: retry the +operation. Semantic catalog APIs, atomic publication, reader pins, compaction, +conditional writes beyond dashboards remain future work. + +## Driver cost + +The driver is `tokio-postgres` with `deadpool-postgres` and +`tokio-postgres-rustls`. The existing repository already uses Tokio and rustls +0.23; this avoids a second TLS stack and compile-time database introspection. +See the [client documentation](https://docs.rs/tokio-postgres/latest/tokio_postgres/) +and [pool documentation](https://docs.rs/deadpool-postgres/latest/deadpool_postgres/). +The lockfile adds 34 package/version entries, including target-specific entries +and resolution changes. The six core PostgreSQL/pool debug rlibs total about +16 MB. The first full debug build on this checkout took 4m17s after `cargo check` +had populated metadata artifacts; that is not an isolated driver compile-time +measurement. The incremental footprint does not justify a backend feature gate +relative to the existing Arrow/DataFusion build. + +The shared contract calls every existing trait method against filesystem-backed +object metadata and real PostgreSQL. It covers both tenant namespaces, missing +objects, global getters, schema union/refresh, manifest encoding/listing, +Ingest-only listing and invalid mode selection, filter reconciliation, alert +state/history, node registration, and conditional dashboard create/update/delete. +It also tests concurrent initialization, concurrent create/update winners, +delete/recreate versions, bounded pool exhaustion and statement timeouts. +Filesystem delimiter listing is unsupported and its stream listing rejects +unrelated root metadata directories; these preexisting differences are explicit +expectations. The SQL path uses bucket-style listing semantics. Empty-string and absent +default tenant schema lookups are equivalent. The smoke covers query/ingest cold loading; broader prism and multi-tenant cluster validation remains future work. + +## Recorded combined validation + +2026-11-11 UTC, PostgreSQL 17 Alpine and MinIO on isolated ephemeral host ports. +No live service or bucket was used. All temporary containers were removed. +Debug build passed. Real PostgreSQL and LocalFS contracts: 2 passed. Full library +suite without a database URL: 1,381 passed, 4 ignored. Integration: 1 passed. +Plain `cargo clippy --all-targets` passed with 21 existing library-test warnings +and no new warnings. Source guard and compatibility fixtures passed unchanged +in meaning. SQL contracts also check physical data before/after injected +deletion failures, retry, tenant isolation, partition deletion, hidden deletion +markers, recreation, and absence of deleted stream rows. + +The smoke passed in object-store, PostgreSQL plaintext and verified TLS modes: +create, ingest/upload, retention settings, dashboard, restart as query node +(running startup retention), ingestor publication of a new stream and cold query, +stream deletion/recreation, upload, and query of the new value (7 rather than 42). +Parquet visibility precedes manifest/snapshot publication; the harness permits +that existing pipeline to settle before cold queries. Normal requests have a +30-second client limit; cold/recreated queries allow 90 seconds. Database outage +requests retain the 30-second limit. + +Both SQL configurations observed readiness 503 and a failed metadata request +when PostgreSQL stopped (combined probes/requests completed in under 7 ms on +this local run), then successful readiness and queries after recovery. All +three configurations observed readiness 503 when MinIO stopped. TLS used a +self-signed CA through `P_METASTORE_POSTGRES_CA`; an unrelated CA rejected startup. +Future schema versions, wrong pinned UUIDs and unreachable databases also +rejected startup. + +Five repeated queries measured the query process's counters after cold loading, +with an ingestor also running: + +| Backend | Metadata GETs/query | LIST + LIST_DELIM/query | Parquet GET_RANGES/query | Total/query | +| --- | ---: | ---: | ---: | ---: | +| Object store | 6 | 2 | 1 | 9 | +| PostgreSQL plaintext | 0 | 0 | 1 | 1 | +| PostgreSQL verified TLS | 0 | 0 | 1 | 1 | + +These are node-local request counts, not latency or deployment-wide totals; +background activity can contribute. The earlier standalone measurement was 4 +versus 1 and used a different topology. Both SQL buckets contained only the +pairing marker and Parquet, including after recreation. Before recreation, the +deleted stream had zero SQL rows. Dashboard, node registry, schema, stream and +manifest documents remained exclusively SQL. Exact inventories, counter deltas, +outage outcomes and test summary are in [the evidence](validation/postgres-metastore-smoke.json). + +Remaining risks are unconditional document update races, publication/retention +races and lack of fencing against concurrent writers or resident readers during +deletion. Interrupted deletion requires explicit retry. Automatic recovery/GC, +reader pins, multi-tenant cluster/prism smoke and client-certificate +TLS remain future work. + +## Offline migration and rollback + +Use this build on **every** node before attempting a transfer. The offline command +shares each storage subcommand's provider configuration and PostgreSQL TLS/pool +options. It does not start HTTP, ingest, retention or background sync services: + +``` +parseable s3-store --metadata-migration verify --acknowledge-server-stopped +parseable s3-store --metadata-migration import --acknowledge-server-stopped --dry-run +parseable s3-store --metadata-migration import --acknowledge-server-stopped +parseable s3-store --metadata-migration export --acknowledge-server-stopped +``` + +`local-store`, `blob-store` and `gcs-store` accept the same flags. LocalFS is for +single-tenant use only. Configure `P_METASTORE_POSTGRES_URL` even for verify and +export, together with exactly the original storage endpoint/bucket. Keep the URL +out of shell history. An empty disposable database is required for a first import. +A successful import cannot be applied again to a populated database. An interrupted +import with its matching migration marker can resume, checking every body again. + +Single-node procedure: + +1. Back up the entire bucket (including hidden keys), the server configuration, + staging directory and any existing PostgreSQL database. Preserve object versions + if available. Record the deployed binary version. Test restoration separately. +2. Let uploads finish, then stop the server and disable automatic restarts. Stop + scripts or other writers too. Supply the original `P_STAGING_DIR`; the command + rejects `.arrow`, `.arrows`, `.part` and `.parquet` staging buffers. Resolve unsynced + buffers by restarting the old mode and allowing upload, then stopping again. +3. Set the destination PostgreSQL URL, leaving all storage settings unchanged. + Run verify; differences and a nonzero exit are expected before first import. + Save its per-key SHA-256 output. Run import with `--dry-run`, inspect + the counts per tenant/kind, bytes and recent-write warnings. This creates no + SQL tables, identity or bucket objects. +4. Run import without dry-run. It validates stream JSON, schemas, compressed and + plain manifests, and every snapshot manifest reference before publishing SQL. + It rereads SQL bodies and compares SHA-256 digests, checks the bucket inventory + for changes, commits the complete SQL inventory, then establishes pairing. +5. Run verify again: require zero differences. Start the server with the same + PostgreSQL URL. Confirm readiness, stream list, schemas, stats, first/latest + event bounds, RBAC, alerts, dashboards and filters. Query known time ranges and + compare row counts/values against saved results before admitting ingest. +6. Keep the original bucket metadata and backups until the rollback window ends. + PostgreSQL writes do not update those original objects. + +Rollback/downgrade: + +1. Back up PostgreSQL, the paired bucket, staging and configuration again. Drain + uploads and stop all writers and automatic restarts as above. +2. Run verify and retain the differences. SQL changes since import normally mean differing bucket objects. Export compares the current bucket against per-key import digests; ordinary SQL changes and deletions require no force flag. +3. Review the differences; export normally to replace imported bodies and remove imported bucket-only metadata. If a bucket writer changed metadata after import, export refuses; after comparing backups explicitly use `--overwrite-bucket-changes-since-import` to accept overwriting that writer’s changes. Parquet is unaffected. Export rereads + each written body, validates all stream references, then removes pairing. +4. Run verify and require zero differences. Remove the PostgreSQL URL and start + object-store mode. Repeat readiness, metadata and known-query checks, including + rows ingested while PostgreSQL was active. Keep the SQL backup. The exported + database is marked exported and refuses normal SQL startup; reimport into a + fresh empty database to switch back later. + +The shared document layer inventories exact stored keys rather than rebuilding +paths. All hidden document namespaces (deployment/RBAC, users, dashboard bodies, +locators/titles, filters, alerts/state/history, correlations, targets/settings, +node registry, stream JSON and schemas) and date-partition manifests, including +per-ingestor variants and tenant prefixes, are copied without reserialization. +SQL versions are freshly allocated. Unfinished deletion markers are rejected. +Parquet, staging/WAL, hot-tier/cache files and private LocalFS lock files are not +metadata documents and are excluded, as are the pairing and transfer controls. +Inventories retain keys/timestamps only; bodies are read sequentially and SQL +export pages contain at most 32 bodies. Progress/digests go to stdout/stderr; +protect these reports because keys may contain user/tenant identifiers. + +Crash recovery: + +* Before the durable `.parseable/metastore-migration` marker, failures leave the + source authoritative and no committed import rows. SQL bootstrap may already + have committed an empty identity; bucket mode remains available. +* During import, an open SQL transaction rolls back on process death. The marker + blocks both server modes. Rerun the same import/configuration to restart. +* After import commit but before pairing/cleanup, all SQL rows already exist and + were verified. Rerunning validates them against the bucket, completes pairing + and removes the transfer marker. A changed source fails closed. +* During export, the pairing and transfer marker block object-store startup; + the SQL transfer state also blocks SQL startup, even after pairing removal. + Rerun export with the same overwrite flag to finish writing and verification. +* After export pairing removal but before marker cleanup, rerun export. Never + manually remove a transfer marker to force startup. + +These controls are **not fencing**. The stopped-server acknowledgement is mandatory +also for verify/dry-run. Recent metadata writes (including registry updates) cause +loud warnings; staging buffers cause refusal. Neither check proves that a remote +writer or resident process is stopped, and a quiet writer can wake at any moment. +The staging check only examines the path supplied to the command. Disable writers +operationally and do not run two migration tools simultaneously. This build refuses +object-store startup on a paired bucket. `P_ALLOW_PAIRED_BUCKET=` explicitly +bypasses pairing for disaster recovery and can expose stale metadata; it never +bypasses an incomplete transfer. Older binaries cannot recognize either marker; +do not start them until a verified export completes. + +### Migration validation evidence + +The migration tests passed with PostgreSQL 17 Alpine on an ephemeral host port: +1,384 library tests passed, 4 ignored. The fixture uses a separate disposable +SQL database, checks both default and named tenant document scopes, plain and +compressed manifests, snapshot references, missing/invalid stream documents, +nonempty refusal, committed-import resumption, overwrite refusal, exported SQL +startup refusal and exact byte equality after the round trip. With no +`P_TEST_POSTGRES_URL`, the SQL fixture explicitly skips. Debug build and plain +`cargo clippy --all-targets` passed; Clippy retains the 21 preexisting library-test +warnings and the preexisting staging integration-test warning. + +Run `python3 scripts/postgres-metastore-migration-smoke.py` after building. This +sibling of the existing smoke creates its own PostgreSQL 17 and MinIO containers, +three datasets (including custom partitions), uploaded manifests, a user/role, +dashboard, saved filter and alert. It kills import and export after the durable +transfer marker becomes visible, reruns each, checks both server modes reject +an incomplete import, checks nonempty and paired-bucket refusal, compares SQL +queries and metadata, ingests a new SQL-mode row, exports, and checks rollback +queries and metadata. Response ordering and the generated stats response timestamp +are excluded from comparisons; persisted stats and event bounds are compared. +The MinIO scenario passed, including the final metadata comparisons after rollback. +All containers are removed in `finally`; the script uses only spare host ports. + +This does not establish production-scale throughput or multi-node fencing. +Legacy objects that the current provider cannot reread because of historical +path/encoding problems fail validation; the command does not guess replacement +keys. Resolve those anomalies with backups before migrating. + +## Review fixes and operating changes + +An empty or whitespace-only PostgreSQL URL selects object-store mode. Pool and timeout tuning is ignored when that backend is inactive. +When selected, numeric tuning must contain positive integers. The object-store metastore and +bulk storage client retain separate request budgets: sharing them would also +make metadata reads compete with Parquet traffic, altering default-mode request +scheduling. Lifecycle prefix deletes currently use the metadata client's budget. + +Migration `002_review_fixes.sql` changes the key column to `COLLATE "C"` and +rebuilds its primary key transactionally under the same startup advisory lock. +On a large database, budget startup time for that rebuild and its exclusive table +lock. Prefix reads and deletes use explicit component-bounded ranges; they do not +require PostgreSQL 15's `starts_with` optimization. Listings are tenant-scoped +and key-only; manifest query planning uses one SQL batch rather than one pooled +connection per manifest. SQL manifest locators are bucket-relative on every +provider; old absolute LocalFS locators are accepted on read. No other SQL +caller uses the provider-absolute locator form. Existing cloud +locators retain their spelling, including percent escapes; SQL delimiter listings +also preserve exact stored keys instead of escaping them a second time. Parquet paths remain provider-specific. + +Stream document writes and deletion markers serialize through a database trigger. +While a deletion marker exists, stream creation and schema/stream updates fail +with a distinct deletion-pending error (HTTP 409 on metastore error paths). +Retry DELETE to finish cleanup before recreating. Default object-store mode still +has its existing check/create behavior; interrupted-deletion fencing there is +not changed. This admission rule is not full fencing of Parquet publication or +resident writers. Retention unions bucket dates with SQL dates, so SQL-only +manifest leftovers after physical deletion are discoverable on the next run. +Tenant cleanup excludes root streams, including streams being deleted, and +requires a single tenant path component. Absent, empty and `DEFAULT_TENANT` +tenant identifiers use the same empty SQL tenant column. Stream lifecycle keys +follow the existing root-path normalization. + +Every SQL operation, including transaction start/commit, startup migration, +offline transfer and health, has a client deadline of statement timeout + 250 ms. +Canceled/timed-out connections are discarded instead of recycled. Readiness has +an overall deadline of that same duration, including pool acquisition and bucket +probe. TCP keepalive starts after one second with one-second probes and three +retries; TCP user timeout follows statement timeout. Connections target read-write +servers, and connections reporting read-only errors are discarded for failover. +Database failures use a dedicated unavailable error (503 through metastore HTTP +error mapping). URL startup `options` are preserved before timeout options are +appended; timeout settings take precedence. PgBouncer must accept startup +`options` (configure its ignore-startup-parameters policy as appropriate). +The database role needs schema/table/sequence creation and alteration privileges, +plus PL/pgSQL function and trigger creation for migration 002, and normal document +CRUD privileges afterward. + +A configured PEM CA bundle **replaces** public webpki trust roots. Supply managed +database private CAs with `P_METASTORE_POSTGRES_CA`; the operating system trust +store is not loaded. An empty CA bundle is rejected at configuration time. CA plus `sslmode=disable` emits a warning and sends plaintext. + +Normal startup refuses an existing database identity, including one with zero documents, when the bucket has no pairing marker. +Restore the matching marker from backup. For deliberate recovery only, after +verifying the bucket and database identity, set +`P_METASTORE_POSTGRES_ALLOW_UNPAIRED=true` to allow creation of the missing marker. +This override never permits mismatched identities or incomplete offline transfers. +The import tool's committed-import recovery continues to establish pairing itself +under its durable transfer marker, and migrates schema version 1 to 2. Exported +or incomplete databases still refuse server admission. + +Conditional-create lost responses remain ambiguous, as on object storage: a retry +returns conflict. Comparing bytes cannot distinguish another writer that wrote +the same body, so no success is synthesized from body equality. + +Reproduce the populated-table plans with +`scripts/postgres-metastore-review-explain.sql` in a disposable database after +applying migrations. Captured PostgreSQL 17 output (200,400 rows, four tenants) +is in [postgres-review-explain.txt](validation/postgres-review-explain.txt). + +Default-mode follow-up: bucket tombstones also make lazy loading report an absent +stream, while `ObjectStorage::create_stream` unconditionally writes schema and +stream JSON. Recreation while a tombstone remains is therefore the same admission +gap in object-store mode. It is reported here and intentionally unchanged in this +PostgreSQL production fix. The two new SQL HTTP errors also preserve their status +when stream creation wraps them in a serialized metastore error detail. + +### Review validation + +The review regression module is separate from the protected existing contracts. +It checks two manifest publications through the returned locator on LocalFS and +an S3-style provider; exact percent-key spelling; overlapping tenant listings; +`DEFAULT_TENANT`; marker/create races including a blocked SQL writer; a 200-key +batch with a two-connection pool; injected failure after physical partition +removal and subsequent SQL cleanup; missing pairing-marker refusal; URL startup +options; read-only connection discard; CA-only trust; and HTTP error propagation. +A TCP proxy stops forwarding without resetting sockets. Four consecutive +readiness calls each return within two seconds with a 300 ms statement timeout, +and a healthy probe succeeds after forwarding resumes. + +Full library runs with and without the PostgreSQL test URL each passed 1,389 +tests with four ignored. SQL skips are printed in the no-database run. Debug +build and plain `cargo clippy --all-targets` passed; Clippy retains the same 21 +library-test warnings and one existing integration-test warning, with no new +warnings. PostgreSQL was `postgres:17-alpine` on a spare port; no live database, +service or bucket was used. The populated collation migration preserved all +200,400 rows; its resulting key collation and query plans are recorded above. + +The offline smoke now waits for staging to remain drained before stopping a +writer, and polls published event bounds before capturing the export comparison. +This avoids stopping with an unsynced custom-partition file or comparing a cached +pre-export bound to a freshly loaded rollback bound. It passed interrupted import +and export, SQL-mode query/metadata equality, new ingestion, verified export and +object-store rollback. Existing Rust test modules and expectations are unchanged. +The protected smoke's only expectation change is future schema version 3, restoring +version 2 afterward, because migration 002 makes version 2 supported. + +The three-mode smoke passed object-store, verified TLS PostgreSQL and plaintext +PostgreSQL scenarios, including lifecycle, dependency outages, CA rejection and +future-schema/pinned-identity rejection. Raw results are in +[postgres-review-smoke.json](validation/postgres-review-smoke.json). +One complete run passed with the default SQL timeout. The final rebuilt-binary +confirmation used `P_METASTORE_POSTGRES_STATEMENT_TIMEOUT_MS=60000` because this +shared machine produced PostgreSQL filesystem syncs over 25 seconds; an earlier +confirmation correctly returned 503 at the default client deadline. The proxy +blackhole regression retained its 300 ms statement timeout. A first smoke attempt +also exceeded the existing auxiliary-process shutdown wait; a subsequent complete +run passed without changing that protected cleanup logic. All started containers +were removed. Large migrations or slow storage may require increasing the +statement timeout; the readiness/client deadline increases with that setting. + + +## Hardened offline procedure + +**Older binaries ignore pairing and transfer markers and can write stale bucket +metadata. Disable their restart policies and prevent redeployment for the entire +PostgreSQL interval.** Digests detect bucket changes during rollback; they do not +fence writers. Never run these commands against a running deployment. + +1. Save binary version, environment and `.parseable.env`, working directory, + staging, a PostgreSQL `pg_dump`, and all bucket metadata: hidden namespaces, + every manifest variant, and pairing/transfer markers. Preserve object versions. +2. Record stream list, schemas, RBAC, alerts, targets, dashboards, filters, known + query results and row counts, plus first/latest timestamps. +3. Drain staging, stop the single server and all other writers, and disable + restarts. Explicitly set `P_STAGING_DIR` to the server's actual directory. + The tool loads `.parseable.env` with the server's loader. If buffers can never + sync, retain a complete staging backup, restore normal mode and repair the + stream/storage problem. Alternatively move the stranded buffers to a separate + preserved recovery directory outside staging, accepting that those events + require later replay; never delete them to satisfy the check. +4. Configure the exact bucket and destination PostgreSQL URL. Run import + `--dry-run --acknowledge-server-stopped`. Database identity, pin, pairing and + transfer ownership are checked read-only before inventory. Review counts, + skipped keys, digests and recent-write warnings. +5. Run import with the same configuration and acknowledgement. Run verify and + require zero differences and SQL document count equal to the dry-run count. +6. Start PostgreSQL mode. Compare all observations from step 2 before ingest. + Preserve both backups until rollback closes. + +Rollback uses the same backups, drain and stopped-writer steps. Run verify and +save differences, then export with the acknowledgement and **no force flag**. +Imported bucket objects still matching their import digest can be overwritten or +deleted safely. Unknown or changed bucket objects require the narrow +`--overwrite-bucket-changes-since-import` flag after reviewing the risk of losing +bucket-writer metadata. `--migration-force` is rejected with an explanation of its replacement. +After successful export, verify, remove the PostgreSQL URL, start bucket mode +and compare all saved results, including events ingested in PostgreSQL mode. + +The command surface remains `--metadata-migration import|verify|export`. +`--abort` is accepted with import or export. Import abort is safe before its SQL +commit; export abort is safe before export state commits. Once irreversible +export writes can have begun, abort refuses and explains that export must finish. +`--allow-dangling-snapshot-references` reports an explicit override during import; +it preserves the existing dangling reference. Without it, missing references +fail validation. Migration modifiers require `--metadata-migration` and cannot +be silently accepted by a normal server start. + +| Failure boundary | Recovery command (same settings and stopped-writer acknowledgement) | +| --- | --- | +| Before marker | Retry import/export; original mode remains available | +| Import marker, no SQL commit | Retry import, or import --abort then start bucket mode | +| Import SQL committed | Retry import to establish pairing and clear marker | +| Import paired, marker remains | Retry import | +| Export validation failed before marker | Correct the SQL metadata in PostgreSQL mode, then retry export | +| Export marker, state still ready | Export --abort restores PostgreSQL startup | +| Export state committed / any destination write | Retry export; abort refuses | +| Export pairing removed, marker remains | Retry export to finish verification and marker cleanup | +| Complete export | Start bucket mode; SQL startup refuses exported state | + +Default startup performs two marker GETs. Not-found permits normal startup. +Other probe errors identify the marker and ask for retry after bucket access is +restored; they do not write a permanent refusal. A paired bucket requires export +or `P_ALLOW_PAIRED_BUCKET=`; incomplete transfers cannot be +bypassed. + +Inventory lists root/tenant/stream/date directories, skips hour and minute +prefixes, and uses flat paginated listings within hidden metadata prefixes. +Bodies are spooled in the command's working directory and imported transactionally; +the inventory recheck and all remote bucket I/O run outside SQL transactions. +For a single-tenant deployment with S streams, D dates per stream and one +`.stream` metadata page per stream, a walk needs approximately +1 + 2S + SD plus global hidden-metadata pages, rather than +1 + 2S + SD(1 + 24 + 1440). For S=20, D=30, that is about 641 versus +879,041 LIST requests per walk, or 1,282 versus 1,758,082 for two walks, +excluding global metadata pagination. GET costs scale with metadata documents, +not Parquet count. Large hidden namespaces add one request per object-store page. + +Four default-backend bugs also present on origin/main are corrected: target +serialization now writes tenantId and accepts legacy tenant on load; alert +listing excludes mttr.json; LocalFS stream listing uses the selected instance; +startup cleanup waits for the entire discovery/validation pass to succeed before +deleting any invalid tenant. + + +The merged CRUD contract and boundary guard kept their assertions. The smoke's +future-schema test now rejects 3 and restores 2 (migration 002 is supported). +The migration smoke no longer expects ordinary rollback to fail without force; +it interrupts and resumes an unforced export. The migration fixture still +rejects a bucket-writer edit, restores its import baseline, then proves that SQL +edits and SQL deletions export without force. + +### Import-tool review disposition + +| Finding | Disposition | +| --- | --- | +| B1 inventory cost | Fixed: metadata-only hidden-prefix pages and date-level listing; hour/minute subtrees excluded; source recheck outside SQL transaction. | +| B2 late export validation | Fixed: validate authoritative SQL rows and SQL snapshot references before marker/state/destination writes; abort available before irreversible boundaries. | +| M1 encoding | Fixed: decode one object-store path encoding layer; real MinIO inventory and SQL round trip cover café and auth0\|é. | +| M2 late preflight | Fixed: read-only database, identity, pin and marker checks precede inventory, including dry-run. | +| M3 old binaries | Mitigated as directed: prominent operational prohibition plus import digests; no mutation of metadata older binaries consume. | +| M4 broad force | Fixed: import digests distinguish SQL updates/deletions from bucket-writer changes; normal export requires no force; explicit narrow override handles the latter. | +| M5 staging/env | Fixed: server env-file loader, explicit staging argument/environment, .part refusal, documented repair/quarantine path. | +| m1 skipped/unknown keys | Fixed: skipped-key report/count; unknown hidden namespaces abort; dashboard locations/titles and LocalFS lock artifacts recognized. | +| m2 legacy tenant scope | Fixed: legacy root stream/schema keys use default tenant and are excluded from SQL tenant cleanup candidates. | +| m3 dangling/legacy snapshot | Fixed: explicit reported dangling-reference override; legacy v1–v3 stream references validated without rewriting source bytes. | +| m4 duplicated bootstrap | Fixed: offline tool and server call the same transactional bootstrap and forward migrations. | +| m5 missing abort | Fixed: safe precommit abort commands, with unsafe abort refusal and rerun instructions. | +| m6 startup probes | Documented: two marker GETs; missing permits startup, transient probe failure requests retry and writes no permanent refusal. | +| m7 broad pairing override | Fixed: P_ALLOW_PAIRED_BUCKET must equal the exact pairing id. | +| m8 duplicate validation | Fixed: import validates in pre-pass; second read spools source bytes and verifies SQL digests rather than validating again. | +| m9 writer fencing | Documented limitation: all writers/restarts must remain stopped; markers/digests are not writer fencing. | + +The old --migration-force flag is explicitly rejected. Modifier flags require +--metadata-migration; normal help hides them, while invoking migration help shows +them. Keys and digest reports can contain tenant/user identifiers and must be +protected like metadata backups. + +The review regression fixture now permits 2 seconds for SQL work and 10 seconds +for database creation, to avoid conflating filesystem setup latency with the +network deadline test. Its proxy blackhole still uses a 300 ms statement timeout, +four probes, and the original under-two-second assertions. + +### Hardening validation + +The combined implementation was validated locally with disposable PostgreSQL 17 +and MinIO, with all task containers removed afterward. Full library runs with +and without the database URL each passed 1,398 tests (seven ignored); SQL tests +print explicit skips without prerequisites. The PostgreSQL contract and startup +cleanup fixtures were then run explicitly and passed, as did the paused-server +deadline test. Focused metastore runs passed 34 tests against LocalFS/PostgreSQL +and MinIO/PostgreSQL; follow-up migration checks cover legacy absolute locators, +both legacy tenant scopes, safe/unsafe aborts, and early pin refusal. + +Both end-to-end scripts passed: the main smoke in object-store, PostgreSQL +plaintext and verified TLS modes; offline migration in plaintext and verified +TLS, including interrupted import/export, .part refusal, new SQL-mode ingestion +and unforced object-store rollback. The smoke generated its own +[raw evidence](validation/postgres-metastore-smoke.json). +[Validation commands and recovery results](validation/postgres-hardening-checks.json) +record the other checks. Plain cargo clippy --all-targets and the debug build +passed; Clippy retained 21 existing library-test warnings and one existing +integration-test warning, with no new warnings. The workflow YAML parses; its +hosted GitHub run has not been performed because this work is local and unpushed. + +Counts in the inventory chart are a model of request growth, not a throughput +benchmark. No concurrent-writer fencing, automatic resolution of an interrupted +stream deletion, or production-scale throughput claim is made. + +Review nits: migration modifiers are scoped in parsing/help and reports document +their identifier privacy risk. Deduplicating the object-safe storage guard and +SQL startup guard adapters is deferred: both enforce the same tested controls, +and consolidating their different error adapters adds no recovery capability. +The required real PostgreSQL/MinIO and injected-boundary coverage is implemented; +hosted CI remains deferred under the no-push constraint. diff --git a/docs/validation/postgres-hardening-checks.json b/docs/validation/postgres-hardening-checks.json new file mode 100644 index 000000000..df134384c --- /dev/null +++ b/docs/validation/postgres-hardening-checks.json @@ -0,0 +1,213 @@ +{ + "commands": [ + { + "name": "library-without-url", + "argv": [ + "cargo", + "test", + "--lib", + "--", + "--nocapture" + ], + "exit_code": 0, + "log": "/.postgres-smoke/library-without-url.log", + "summaries": [ + "test result: ok. 1398 passed; 0 failed; 7 ignored; 0 measured; 0 filtered out; finished in 75.00s" + ] + }, + { + "name": "library-with-url", + "argv": [ + "cargo", + "test", + "--lib", + "--", + "--test-threads=1", + "--nocapture" + ], + "exit_code": 0, + "log": "/.postgres-smoke/library-with-url.log", + "summaries": [ + "test result: ok. 1398 passed; 0 failed; 7 ignored; 0 measured; 0 filtered out; finished in 137.63s" + ] + }, + { + "name": "contracts", + "argv": [ + "cargo", + "test", + "--lib", + "contract_tests", + "--", + "--nocapture", + "--test-threads=1" + ], + "exit_code": 0, + "log": "/.postgres-smoke/contracts.log", + "summaries": [ + "test result: ok. 10 passed; 0 failed; 3 ignored; 0 measured; 1392 filtered out; finished in 1.77s" + ] + }, + { + "name": "postgres-contract", + "argv": [ + "cargo", + "test", + "--lib", + "contract_tests::postgres_contract", + "--", + "--ignored", + "--nocapture" + ], + "exit_code": 0, + "log": "/.postgres-smoke/postgres-contract.log", + "summaries": [ + "test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 1404 filtered out; finished in 5.51s" + ] + }, + { + "name": "startup-cleanup", + "argv": [ + "cargo", + "test", + "--lib", + "contract_tests::postgres_startup_cleanup", + "--", + "--ignored", + "--nocapture" + ], + "exit_code": 0, + "log": "/.postgres-smoke/startup-cleanup.log", + "summaries": [ + "test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 1404 filtered out; finished in 0.75s" + ] + }, + { + "name": "paused-health", + "argv": [ + "cargo", + "test", + "--lib", + "postgres_health_deadline_regression", + "--", + "--nocapture" + ], + "exit_code": 0, + "log": "/.postgres-smoke/paused-health.log", + "summaries": [ + "test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 1404 filtered out; finished in 0.94s" + ] + }, + { + "name": "clippy", + "argv": [ + "cargo", + "clippy", + "--all-targets" + ], + "exit_code": 0, + "log": "/.postgres-smoke/clippy.log", + "summaries": [] + }, + { + "name": "focused-localfs", + "exit_code": 0, + "summaries": [ + "test result: ok. 34 passed; 0 failed; 3 ignored; 0 measured; 1368 filtered out; finished in 9.08s" + ] + }, + { + "name": "focused-minio", + "argv": [ + "cargo", + "test", + "--lib", + "metastore::", + "--", + "--nocapture", + "--test-threads=1" + ], + "exit_code": 0, + "log": "/.postgres-smoke/focused-minio.log", + "summaries": [ + "test result: ok. 34 passed; 0 failed; 3 ignored; 0 measured; 1368 filtered out; finished in 10.77s" + ] + }, + { + "name": "final-build", + "argv": [ + "cargo", + "build" + ], + "exit_code": 0, + "log": "/.postgres-smoke/final-build.log", + "summaries": [] + }, + { + "name": "cli-scope-and-help", + "exit_code": 0, + "flags": 5 + }, + { + "name": "legacy-layouts-and-minio", + "exit_code": 0, + "summaries": [ + "test result: ok. 34 passed; 0 failed; 3 ignored; 0 measured; 1368 filtered out; finished in 10.64s" + ], + "log_sha256": "ff9958a1ad0394e52c04f10db1143cdb29bba5f57076c20da99b5f1090a72841" + }, + { + "name": "abort-and-pinned-preflight", + "exit_code": 0, + "summaries": [ + "test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 1404 filtered out; finished in 1.90s" + ], + "log_sha256": "7dbd844df0d6555df0b6c4393a0f3fdf68f680c3e396ec8de8abcadda8bfbdf8" + }, + { + "name": "final-clippy", + "exit_code": 0, + "log_sha256": "c36993dc1ec5f5e4a472a4a8c99619da26e0e30fbc46d259016bbd857db75179" + }, + { + "name": "verified-build", + "exit_code": 0, + "log_sha256": "b7a890fd5f3d3fbfff9c8b0a146f9880bf12076a1ebe4aef82460c7ffe8e22c3" + } + ], + "success": true, + "end_to_end": { + "main_modes": [ + "object-store", + "postgres-tls", + "postgres" + ], + "main_success": true, + "migration_plaintext_success": true, + "migration_verified_tls_success": true + }, + "migration_end_to_end": [ + { + "mode": "plaintext", + "success": true, + "log_sha256": "1dda51d6b4c53a068158afd8d7af090dff8145b58d278ac68f2a51ac552054a4" + }, + { + "mode": "verified-tls", + "success": true, + "log_sha256": "1dda51d6b4c53a068158afd8d7af090dff8145b58d278ac68f2a51ac552054a4" + } + ], + "failure_injection": { + "import-marker-spool-before-commit": "rerun import or import --abort; object-store guard admits startup after abort", + "import-commit-paired": "rerun import; abort refuses committed documents", + "export-marker-ready": "export --abort; restarted PostgreSQL backend admits startup", + "export-state-object-write-bucket-delete-pair-removed": "rerun export; abort refuses; exported bucket guard admits startup and PostgreSQL startup refuses" + }, + "clippy_warnings": { + "library_test": 21, + "integration_test": 1, + "new": 0 + }, + "workflow_yaml_parsed": true +} diff --git a/docs/validation/postgres-import-list-requests.html b/docs/validation/postgres-import-list-requests.html new file mode 100644 index 000000000..0c09912a9 --- /dev/null +++ b/docs/validation/postgres-import-list-requests.html @@ -0,0 +1,13 @@ + + +

Estimated LIST requests for one inventory: 20 streams Ă— 30 days

+ + + + + +PreviousHardened +879,041641 +110010,0001,000,000 + +Log scale. Excludes global hidden-metadata pages. Two import walks: 1,758,082 → 1,282 requests. GET costs depend on metadata count; hour/minute prefixes and Parquet files are excluded. diff --git a/docs/validation/postgres-metastore-requests.html b/docs/validation/postgres-metastore-requests.html new file mode 100644 index 000000000..9015b1657 --- /dev/null +++ b/docs/validation/postgres-metastore-requests.html @@ -0,0 +1 @@ +
Object store
9
PostgreSQL
1
PostgreSQL TLS
1

Bucket requests per query, five repeated queries. Query process counters with an ingestor running. PostgreSQL: one Parquet range request. Object store: six metadata GETs, two listings and one Parquet range request.

diff --git a/docs/validation/postgres-metastore-smoke.json b/docs/validation/postgres-metastore-smoke.json new file mode 100644 index 000000000..39e33aed1 --- /dev/null +++ b/docs/validation/postgres-metastore-smoke.json @@ -0,0 +1,3307 @@ +{ + "runs": [ + { + "requested_modes": [ + "object-store", + "postgres-tls", + "postgres" + ], + "source_commit": "1428d2cc45c4f60d164a89713ca93bcd896c76e5", + "binary_sha256": "8fb88e03aaf8417443f1e496cac9a5cf1a696661d3133adadcc252d75f2887ad", + "source_diff_sha256": "c6974df8d3d4d6c26e2fa5a22c93838e9fd8b7d21e14504d12b40e5813df0a31", + "image_digests": { + "postgres:17-alpine": [ + "postgres@sha256:b0f9560a2de083e2cc7382e75f808c7381a32852a7ec49117deedb300e552b24" + ], + "quay.io/parseablehq/minio:edge": [ + "quay.io/parseablehq/minio@sha256:88276d0a028d758ffaf80d65dd80071386e91b4114251a3dda101eadf000faeb" + ] + }, + "measurement": "Node-local storage counter delta during five queries, including background traffic; not isolated requests per query.", + "object-store": { + "lazy_reload": true, + "query": "[{\"count\":1}]", + "dashboard": { + "version": "v1", + "title": "Postgres smoke", + "author": "8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918", + "dashboardId": "01M74SV622YTQ2Y75HZE11AVJB", + "created": "2026-11-11T07:58:30.978106403+00:00", + "modified": "2026-11-11T07:58:30.978106403+00:00", + "tags": null, + "isFavorite": false, + "dashboardType": "Dashboard", + "tiles": [], + "tenantId": null, + "revision": "2d53005fca7a3b566744600d766140032fe8ad13cc9d01b1442619fc34bc8a24" + }, + "keys": [ + ".parseable/.parseable.json", + ".parseable/ingestor.d97d0c2e2ddd006.json", + ".parseable/querier.5bef5998cbd41f0.json", + ".users/.dashboard-locations/01M74SV622YTQ2Y75HZE11AVJB.json", + ".users/.dashboard-titles/571a2cefe2d0d1a9a26dab4810a5556875c7d7531d4f558a8f7b28ab486e2395.json", + ".users/8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918/dashboards/01M74SV622YTQ2Y75HZE11AVJB.json", + "lazy/.stream/.ingestor.d97d0c2e2ddd006.schema", + "lazy/.stream/.ingestor.d97d0c2e2ddd006.stream.json", + "lazy/.stream/.schema", + "lazy/.stream/.stream.json", + "lazy/date=2026-11-11/hour=07/minute=59/d97d0c2e2ddd006.data.01M74SXX475GMD681ENW5BAMAA.parquet", + "lazy/date=2026-11-11/ingestor..d97d0c2e2ddd006.manifest.json", + "pstats/.stream/.ingestor.d97d0c2e2ddd006.schema", + "pstats/.stream/.ingestor.d97d0c2e2ddd006.stream.json", + "pstats/.stream/.schema", + "pstats/.stream/.stream.json", + "pstats/date=2026-11-11/.manifest.json", + "pstats/date=2026-11-11/hour=07/minute=57/dataset_name=smoke/.data.01M74SW2MA5GZW7RNJRMWRNB3T.parquet", + "smoke/.stream/.ingestor.d97d0c2e2ddd006.schema", + "smoke/.stream/.ingestor.d97d0c2e2ddd006.stream.json", + "smoke/.stream/.schema", + "smoke/.stream/.stream.json", + "smoke/date=2026-11-11/.manifest.json", + "smoke/date=2026-11-11/hour=07/minute=57/.data.01M74ST81AX4XAJZZ14Z78ZGWZ.parquet" + ], + "non_parquet_keys": [ + ".parseable/.parseable.json", + ".parseable/ingestor.d97d0c2e2ddd006.json", + ".parseable/querier.5bef5998cbd41f0.json", + ".users/.dashboard-locations/01M74SV622YTQ2Y75HZE11AVJB.json", + ".users/.dashboard-titles/571a2cefe2d0d1a9a26dab4810a5556875c7d7531d4f558a8f7b28ab486e2395.json", + ".users/8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918/dashboards/01M74SV622YTQ2Y75HZE11AVJB.json", + "lazy/.stream/.ingestor.d97d0c2e2ddd006.schema", + "lazy/.stream/.ingestor.d97d0c2e2ddd006.stream.json", + "lazy/.stream/.schema", + "lazy/.stream/.stream.json", + "lazy/date=2026-11-11/ingestor..d97d0c2e2ddd006.manifest.json", + "pstats/.stream/.ingestor.d97d0c2e2ddd006.schema", + "pstats/.stream/.ingestor.d97d0c2e2ddd006.stream.json", + "pstats/.stream/.schema", + "pstats/.stream/.stream.json", + "pstats/date=2026-11-11/.manifest.json", + "smoke/.stream/.ingestor.d97d0c2e2ddd006.schema", + "smoke/.stream/.ingestor.d97d0c2e2ddd006.stream.json", + "smoke/.stream/.schema", + "smoke/.stream/.stream.json", + "smoke/date=2026-11-11/.manifest.json" + ], + "request_delta": { + "parseable_storage_request_response_time_count{method=\"GET\",provider=\"s3\",status=\"200\"}": 30.0, + "parseable_storage_request_response_time_count{method=\"GET_RANGES\",provider=\"s3\",status=\"200\"}": 5.0, + "parseable_storage_request_response_time_count{method=\"LIST\",provider=\"s3\",status=\"200\"}": 5.0, + "parseable_storage_request_response_time_count{method=\"LIST_DELIM\",provider=\"s3\",status=\"200\"}": 5.0 + }, + "observed_requests_during_window_divided_by_queries": 9.0, + "bucket_outage_readiness": 503, + "delete_recreate": true, + "final_keys": [ + ".parseable/.parseable.json", + ".parseable/ingestor.d97d0c2e2ddd006.json", + ".parseable/querier.5bef5998cbd41f0.json", + ".users/.dashboard-locations/01M74SV622YTQ2Y75HZE11AVJB.json", + ".users/.dashboard-titles/571a2cefe2d0d1a9a26dab4810a5556875c7d7531d4f558a8f7b28ab486e2395.json", + ".users/8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918/dashboards/01M74SV622YTQ2Y75HZE11AVJB.json", + "lazy/.stream/.ingestor.d97d0c2e2ddd006.schema", + "lazy/.stream/.ingestor.d97d0c2e2ddd006.stream.json", + "lazy/.stream/.schema", + "lazy/.stream/.stream.json", + "lazy/date=2026-11-11/hour=07/minute=59/d97d0c2e2ddd006.data.01M74SXX475GMD681ENW5BAMAA.parquet", + "lazy/date=2026-11-11/ingestor..d97d0c2e2ddd006.manifest.json", + "pstats/.stream/.ingestor.d97d0c2e2ddd006.schema", + "pstats/.stream/.ingestor.d97d0c2e2ddd006.stream.json", + "pstats/.stream/.schema", + "pstats/.stream/.stream.json", + "pstats/date=2026-11-11/.manifest.json", + "pstats/date=2026-11-11/hour=07/minute=57/dataset_name=smoke/.data.01M74SW2MA5GZW7RNJRMWRNB3T.parquet", + "pstats/date=2026-11-11/hour=07/minute=59/dataset_name=lazy/d97d0c2e2ddd006.data.01M74SZQQ6SAEGTFW58C90P919.parquet", + "pstats/date=2026-11-11/ingestor..d97d0c2e2ddd006.manifest.json", + "smoke/.stream/.ingestor.d97d0c2e2ddd006.schema", + "smoke/.stream/.ingestor.d97d0c2e2ddd006.stream.json", + "smoke/.stream/.schema", + "smoke/.stream/.stream.json", + "smoke/date=2026-11-11/hour=08/minute=00/d97d0c2e2ddd006.data.01M74SZQQ6PP5HSKS6P1JXW6TQ.parquet", + "smoke/date=2026-11-11/ingestor..d97d0c2e2ddd006.manifest.json" + ] + }, + "postgres-tls": { + "trusted_ca_restart": true, + "untrusted_ca_rejected": true, + "untrusted_ca_output": "Error: ObjectStorageError: PostgreSQL metastore unavailable: Error occurred while creating a new object: error performing TLS handshake\n\nCaused by:\n PostgreSQL metastore unavailable: Error occurred while creating a new object: error performing TLS handshake\n", + "untrusted_ca_server_alert": "2026-11-11 08:01:47.935 UTC [127] LOG: could not accept SSL connection: tlsv1 alert unknown ca\n", + "lazy_reload": true, + "query": "[{\"count\":1}]", + "dashboard": { + "version": "v1", + "title": "Postgres smoke", + "author": "8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918", + "dashboardId": "01M74T2GY6Y31B6B74N9XGF19Z", + "created": "2026-11-11T08:02:31.494659236+00:00", + "modified": "2026-11-11T08:02:31.494659236+00:00", + "tags": null, + "isFavorite": false, + "dashboardType": "Dashboard", + "tiles": [], + "tenantId": null, + "revision": "feb4fe847e41a4f08b8369a90a246a4a9e291aa10b9b85ad6b1463b6dab7b347" + }, + "keys": [ + ".parseable/postgres-metastore-id", + "lazy/date=2026-11-11/hour=08/minute=03/6f97a7d0983d3d9.data.01M74T57RBAGCEEZ73TRR5SGXR.parquet", + "pstats/date=2026-11-11/hour=08/minute=01/dataset_name=smoke/.data.01M74T3DK1EF85HBBCGG580X96.parquet", + "smoke/date=2026-11-11/hour=08/minute=01/.data.01M74T1K01S1D4BJKHY1AR1PYM.parquet" + ], + "non_parquet_keys": [ + ".parseable/postgres-metastore-id" + ], + "request_delta": { + "parseable_storage_request_response_time_count{method=\"GET_RANGES\",provider=\"s3\",status=\"200\"}": 5.0 + }, + "observed_requests_during_window_divided_by_queries": 1.0, + "database_outage": { + "readiness": 503, + "request_failed": true, + "seconds": 0.004449435044080019 + }, + "bucket_outage_readiness": 503, + "deleted_stream_rows": 0, + "delete_recreate": true, + "final_keys": [ + ".parseable/postgres-metastore-id", + "lazy/date=2026-11-11/hour=08/minute=03/6f97a7d0983d3d9.data.01M74T57RBAGCEEZ73TRR5SGXR.parquet", + "pstats/date=2026-11-11/hour=08/minute=01/dataset_name=smoke/.data.01M74T3DK1EF85HBBCGG580X96.parquet", + "pstats/date=2026-11-11/hour=08/minute=03/dataset_name=lazy/6f97a7d0983d3d9.data.01M74T72B8N681THKECS821NH4.parquet", + "smoke/date=2026-11-11/hour=08/minute=04/6f97a7d0983d3d9.data.01M74T72B8XG3BJ1XATNGYWSRA.parquet" + ], + "interrupted_deletion_restart": [ + { + "step": "before-data", + "hidden": true, + "cold_load_rejected": true, + "http_retry": true, + "recreation": true + }, + { + "step": "after-data", + "hidden": true, + "cold_load_rejected": true, + "http_retry": true, + "recreation": true + } + ] + }, + "postgres": { + "lazy_reload": true, + "query": "[{\"count\":1}]", + "dashboard": { + "version": "v1", + "title": "Postgres smoke", + "author": "8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918", + "dashboardId": "01M74TBP2FCCBXZH3H1K9PQW72", + "created": "2026-11-11T08:07:31.663110272+00:00", + "modified": "2026-11-11T08:07:31.663110272+00:00", + "tags": null, + "isFavorite": false, + "dashboardType": "Dashboard", + "tiles": [], + "tenantId": null, + "revision": "84b5f12dd1b98ed78f48a43b1b089615ee0da3e2db1df3e52e0f1e1dd2e83957" + }, + "keys": [ + ".parseable/postgres-metastore-id", + "lazy/date=2026-11-11/hour=08/minute=08/496c429a726b98c.data.01M74TEDA543YCHYXEA3F0QAEA.parquet", + "pstats/date=2026-11-11/hour=08/minute=06/dataset_name=smoke/.data.01M74TCJNQQ6KVX07JH6RJ4GD0.parquet", + "smoke/date=2026-11-11/hour=08/minute=06/.data.01M74TAR2QY8AMHTG8F6CHMSTZ.parquet" + ], + "non_parquet_keys": [ + ".parseable/postgres-metastore-id" + ], + "request_delta": { + "parseable_storage_request_response_time_count{method=\"GET_RANGES\",provider=\"s3\",status=\"200\"}": 5.0 + }, + "observed_requests_during_window_divided_by_queries": 1.0, + "database_outage": { + "readiness": 503, + "request_failed": true, + "seconds": 0.004181399941444397 + }, + "bucket_outage_readiness": 503, + "deleted_stream_rows": 0, + "delete_recreate": true, + "final_keys": [ + ".parseable/postgres-metastore-id", + "lazy/date=2026-11-11/hour=08/minute=08/496c429a726b98c.data.01M74TEDA543YCHYXEA3F0QAEA.parquet", + "pstats/date=2026-11-11/hour=08/minute=06/dataset_name=smoke/.data.01M74TCJNQQ6KVX07JH6RJ4GD0.parquet", + "pstats/date=2026-11-11/hour=08/minute=08/dataset_name=lazy/496c429a726b98c.data.01M74TG7X53KFBJAX3ESJVYHBF.parquet", + "smoke/date=2026-11-11/hour=08/minute=09/496c429a726b98c.data.01M74TG7X5ZB4F5KWD9PPBXNMP.parquet" + ], + "interrupted_deletion_restart": [ + { + "step": "before-data", + "hidden": true, + "cold_load_rejected": true, + "http_retry": true, + "recreation": true + }, + { + "step": "after-data", + "hidden": true, + "cold_load_rejected": true, + "http_retry": true, + "recreation": true + } + ] + }, + "startup_rejections": { + "future-schema": "Error: ObjectStorageError: PostgreSQL metastore unavailable: unsupported schema/protocol version; upgrade this server\n\nCaused by:\n PostgreSQL metastore unavailable: unsupported schema/protocol version; upgrade this server\n", + "wrong-id": "Error: ObjectStorageError: PostgreSQL metastore unavailable: backend/bucket identity mismatch\n\nCaused by:\n PostgreSQL metastore unavailable: backend/bucket identity mismatch\n", + "unreachable": "Error: ObjectStorageError: PostgreSQL metastore unavailable: Error occurred while creating a new object: error connecting to server\n\nCaused by:\n PostgreSQL metastore unavailable: Error occurred while creating a new object: error connecting to server\n" + }, + "postgres_rows": "tenant | kind | key | version | bytes \n--------+----------+--------------------------------------------------------------------------------------------------------------------+---------+-------\n | document | .parseable/.parseable.json | 121 | 384\n | document | .parseable/ingestor.496c429a726b98c.json | 39 | 241\n | document | .parseable/querier.36256ead0ab8108.json | 123 | 240\n | document | .users/.dashboard-locations/01M74TBP2FCCBXZH3H1K9PQW72.json | 16 | 119\n | document | .users/.dashboard-titles/571a2cefe2d0d1a9a26dab4810a5556875c7d7531d4f558a8f7b28ab486e2395.json | 18 | 170\n | document | .users/8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918/dashboards/01M74TBP2FCCBXZH3H1K9PQW72.json | 17 | 545\n | schema | lazy/.stream/.ingestor.496c429a726b98c.schema | 56 | 585\n | stream | lazy/.stream/.ingestor.496c429a726b98c.stream.json | 54 | 853\n | schema | lazy/.stream/.schema | 131 | 585\n | stream | lazy/.stream/.stream.json | 58 | 597\n | manifest | lazy/date=2026-11-11/ingestor..496c429a726b98c.manifest.json | 52 | 422\n | schema | pstats/.stream/.ingestor.496c429a726b98c.schema | 148 | 1117\n | stream | pstats/.stream/.ingestor.496c429a726b98c.stream.json | 146 | 898\n | schema | pstats/.stream/.schema | 127 | 1117\n | stream | pstats/.stream/.stream.json | 23 | 867\n | manifest | pstats/date=2026-11-11/.manifest.json | 21 | 533\n | manifest | pstats/date=2026-11-11/ingestor..496c429a726b98c.manifest.json | 144 | 804\n | schema | smoke/.stream/.ingestor.496c429a726b98c.schema | 141 | 27\n | stream | smoke/.stream/.ingestor.496c429a726b98c.stream.json | 142 | 597\n | schema | smoke/.stream/.schema | 139 | 27\n | stream | smoke/.stream/.stream.json | 140 | 597\n(21 rows)", + "commands": [ + { + "argv": [ + "docker", + "run", + "-d", + "--name", + "parseable-pg-smoke-3714167-0", + "-e", + "POSTGRES_PASSWORD=smoke", + "-e", + "POSTGRES_DB=parseable", + "-p", + "127.0.0.1:35731:5432", + "postgres:17-alpine" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "run", + "-d", + "--name", + "parseable-pg-smoke-3714167-1", + "-e", + "MINIO_ROOT_USER=minioadmin", + "-e", + "MINIO_ROOT_PASSWORD=minioadmin", + "-p", + "127.0.0.1:50141:9000", + "quay.io/parseablehq/minio:edge", + "server", + "/data" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "pg_isready", + "-h", + "127.0.0.1", + "-U", + "postgres" + ], + "exit_code": 2 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "pg_isready", + "-h", + "127.0.0.1", + "-U", + "postgres" + ], + "exit_code": 2 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "pg_isready", + "-h", + "127.0.0.1", + "-U", + "postgres" + ], + "exit_code": 0 + }, + { + "argv": [ + "openssl", + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-keyout", + "/.postgres-smoke/tls/ca.key", + "-out", + "/.postgres-smoke/tls/ca.crt", + "-days", + "2", + "-subj", + "/CN=Smoke CA" + ], + "exit_code": 0 + }, + { + "argv": [ + "openssl", + "req", + "-newkey", + "rsa:2048", + "-nodes", + "-keyout", + "/.postgres-smoke/tls/server.key", + "-out", + "/.postgres-smoke/tls/server.csr", + "-subj", + "/CN=localhost" + ], + "exit_code": 0 + }, + { + "argv": [ + "openssl", + "x509", + "-req", + "-in", + "/.postgres-smoke/tls/server.csr", + "-CA", + "/.postgres-smoke/tls/ca.crt", + "-CAkey", + "/.postgres-smoke/tls/ca.key", + "-CAcreateserial", + "-out", + "/.postgres-smoke/tls/server.crt", + "-days", + "2", + "-extfile", + "/.postgres-smoke/tls/extensions" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "cp", + "/.postgres-smoke/tls/server.key", + "parseable-pg-smoke-3714167-0:/var/lib/postgresql/data/server.key" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "cp", + "/.postgres-smoke/tls/server.crt", + "parseable-pg-smoke-3714167-0:/var/lib/postgresql/data/server.crt" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "sh", + "-c", + "chown postgres:postgres /var/lib/postgresql/data/server.* && chmod 600 /var/lib/postgresql/data/server.key" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-c", + "ALTER SYSTEM SET ssl='on';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-c", + "SELECT pg_reload_conf();" + ], + "exit_code": 0 + }, + { + "argv": [ + "git", + "rev-parse", + "HEAD" + ], + "exit_code": 0 + }, + { + "argv": [ + "git", + "diff", + "HEAD" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "image", + "inspect", + "postgres:17-alpine" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "image", + "inspect", + "quay.io/parseablehq/minio:edge" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-1", + "mkdir", + "-p", + "/data/smoke-object-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "stop", + "parseable-pg-smoke-3714167-1" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "start", + "parseable-pg-smoke-3714167-1" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-1", + "mkdir", + "-p", + "/data/smoke-postgres-tls" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-c", + "CREATE DATABASE tls;" + ], + "exit_code": 0 + }, + { + "argv": [ + "openssl", + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-keyout", + "/.postgres-smoke/tls/wrong.key", + "-out", + "/.postgres-smoke/tls/wrong.crt", + "-days", + "2", + "-subj", + "/CN=Wrong CA" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "logs", + "parseable-pg-smoke-3714167-0" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "logs", + "parseable-pg-smoke-3714167-0" + ], + "exit_code": 0 + }, + { + "argv": [ + "/target/debug/parseable", + "s3-store" + ], + "exit_code": 1 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "stop", + "parseable-pg-smoke-3714167-0" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "start", + "parseable-pg-smoke-3714167-0" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "stop", + "parseable-pg-smoke-3714167-1" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "start", + "parseable-pg-smoke-3714167-1" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT count(*) FROM parseable_documents WHERE starts_with(key,'smoke/');" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-v", + "ON_ERROR_STOP=1", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "INSERT INTO parseable_documents(tenant,key,kind,body) VALUES('','smoke/.stream/.deleting','document',''::bytea);" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-v", + "ON_ERROR_STOP=1", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT count(*) FROM parseable_documents WHERE starts_with(key,'smoke/');" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-v", + "ON_ERROR_STOP=1", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "CREATE FUNCTION smoke_delete_fault() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'injected row deletion failure'; END $$; CREATE TRIGGER smoke_delete_fault BEFORE DELETE ON parseable_documents FOR EACH ROW EXECUTE FUNCTION smoke_delete_fault();" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-v", + "ON_ERROR_STOP=1", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "DROP TRIGGER smoke_delete_fault ON parseable_documents; DROP FUNCTION smoke_delete_fault();" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-v", + "ON_ERROR_STOP=1", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT count(*) FROM parseable_documents WHERE starts_with(key,'smoke/');" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-1", + "mkdir", + "-p", + "/data/smoke-postgres" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "stop", + "parseable-pg-smoke-3714167-0" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "start", + "parseable-pg-smoke-3714167-0" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "stop", + "parseable-pg-smoke-3714167-1" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "start", + "parseable-pg-smoke-3714167-1" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT count(*) FROM parseable_documents WHERE starts_with(key,'smoke/');" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-v", + "ON_ERROR_STOP=1", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "INSERT INTO parseable_documents(tenant,key,kind,body) VALUES('','smoke/.stream/.deleting','document',''::bytea);" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-v", + "ON_ERROR_STOP=1", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT count(*) FROM parseable_documents WHERE starts_with(key,'smoke/');" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-v", + "ON_ERROR_STOP=1", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "CREATE FUNCTION smoke_delete_fault() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'injected row deletion failure'; END $$; CREATE TRIGGER smoke_delete_fault BEFORE DELETE ON parseable_documents FOR EACH ROW EXECUTE FUNCTION smoke_delete_fault();" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-v", + "ON_ERROR_STOP=1", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "DROP TRIGGER smoke_delete_fault ON parseable_documents; DROP FUNCTION smoke_delete_fault();" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-v", + "ON_ERROR_STOP=1", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT count(*) FROM parseable_documents WHERE starts_with(key,'smoke/');" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-c", + "UPDATE parseable_backend SET schema_version=3;" + ], + "exit_code": 0 + }, + { + "argv": [ + "/target/debug/parseable", + "s3-store" + ], + "exit_code": 1 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-c", + "UPDATE parseable_backend SET schema_version=2;" + ], + "exit_code": 0 + }, + { + "argv": [ + "/target/debug/parseable", + "s3-store" + ], + "exit_code": 1 + }, + { + "argv": [ + "/target/debug/parseable", + "s3-store" + ], + "exit_code": 1 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-3714167-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-c", + "SELECT tenant,kind,key,version,octet_length(body) AS bytes FROM parseable_documents ORDER BY tenant,key;" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "rm", + "-f", + "parseable-pg-smoke-3714167-1" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "rm", + "-f", + "parseable-pg-smoke-3714167-0" + ], + "exit_code": 0 + }, + { + "argv": [ + "/target/debug/parseable", + "s3-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "/target/debug/parseable", + "s3-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "/target/debug/parseable", + "s3-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "/target/debug/parseable", + "s3-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "/target/debug/parseable", + "s3-store" + ], + "exit_code": 1 + }, + { + "argv": [ + "/target/debug/parseable", + "s3-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "/target/debug/parseable", + "s3-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "/target/debug/parseable", + "s3-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "/target/debug/parseable", + "s3-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "/target/debug/parseable", + "s3-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "/target/debug/parseable", + "s3-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "/target/debug/parseable", + "s3-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "/target/debug/parseable", + "s3-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "/target/debug/parseable", + "s3-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "/target/debug/parseable", + "s3-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "/target/debug/parseable", + "s3-store" + ], + "exit_code": 1 + }, + { + "argv": [ + "/target/debug/parseable", + "s3-store" + ], + "exit_code": 1 + }, + { + "argv": [ + "/target/debug/parseable", + "s3-store" + ], + "exit_code": 1 + } + ], + "success": true, + "validation": { + "commands": [ + { + "name": "library-without-url", + "argv": [ + "cargo", + "test", + "--lib", + "--", + "--nocapture" + ], + "exit_code": 0, + "log": "/.postgres-smoke/library-without-url.log", + "summaries": [ + "test result: ok. 1398 passed; 0 failed; 7 ignored; 0 measured; 0 filtered out; finished in 75.00s" + ] + }, + { + "name": "library-with-url", + "argv": [ + "cargo", + "test", + "--lib", + "--", + "--test-threads=1", + "--nocapture" + ], + "exit_code": 0, + "log": "/.postgres-smoke/library-with-url.log", + "summaries": [ + "test result: ok. 1398 passed; 0 failed; 7 ignored; 0 measured; 0 filtered out; finished in 137.63s" + ] + }, + { + "name": "contracts", + "argv": [ + "cargo", + "test", + "--lib", + "contract_tests", + "--", + "--nocapture", + "--test-threads=1" + ], + "exit_code": 0, + "log": "/.postgres-smoke/contracts.log", + "summaries": [ + "test result: ok. 10 passed; 0 failed; 3 ignored; 0 measured; 1392 filtered out; finished in 1.77s" + ] + }, + { + "name": "postgres-contract", + "argv": [ + "cargo", + "test", + "--lib", + "contract_tests::postgres_contract", + "--", + "--ignored", + "--nocapture" + ], + "exit_code": 0, + "log": "/.postgres-smoke/postgres-contract.log", + "summaries": [ + "test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 1404 filtered out; finished in 5.51s" + ] + }, + { + "name": "startup-cleanup", + "argv": [ + "cargo", + "test", + "--lib", + "contract_tests::postgres_startup_cleanup", + "--", + "--ignored", + "--nocapture" + ], + "exit_code": 0, + "log": "/.postgres-smoke/startup-cleanup.log", + "summaries": [ + "test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 1404 filtered out; finished in 0.75s" + ] + }, + { + "name": "paused-health", + "argv": [ + "cargo", + "test", + "--lib", + "postgres_health_deadline_regression", + "--", + "--nocapture" + ], + "exit_code": 0, + "log": "/.postgres-smoke/paused-health.log", + "summaries": [ + "test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 1404 filtered out; finished in 0.94s" + ] + }, + { + "name": "clippy", + "argv": [ + "cargo", + "clippy", + "--all-targets" + ], + "exit_code": 0, + "log": "/.postgres-smoke/clippy.log", + "summaries": [] + } + ], + "success": true + }, + "http_requests": [ + { + "path": "/liveness", + "method": "GET", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "PUT", + "status": 200 + }, + { + "path": "/ingest", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/logstream/smoke/retention", + "method": "PUT", + "status": 200 + }, + { + "path": "/dashboards", + "method": "POST", + "status": 200 + }, + { + "path": "/liveness", + "method": "GET", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/dashboards", + "method": "GET", + "status": 200 + }, + { + "path": "/metrics", + "method": "GET", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/metrics", + "method": "GET", + "status": 200 + }, + { + "path": "/readiness", + "method": "GET", + "status": 503 + }, + { + "path": "/readiness", + "method": "GET", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "DELETE", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "PUT", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/liveness", + "method": "GET", + "status": 200 + }, + { + "path": "/liveness", + "method": "GET", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "PUT", + "status": 200 + }, + { + "path": "/ingest", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/logstream/smoke/retention", + "method": "PUT", + "status": 200 + }, + { + "path": "/dashboards", + "method": "POST", + "status": 200 + }, + { + "path": "/liveness", + "method": "GET", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/dashboards", + "method": "GET", + "status": 200 + }, + { + "path": "/metrics", + "method": "GET", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/metrics", + "method": "GET", + "status": 200 + }, + { + "path": "/readiness", + "method": "GET", + "status": 503 + }, + { + "path": "/logstream/outage", + "method": "PUT", + "status": 503 + }, + { + "path": "/readiness", + "method": "GET", + "status": 503 + }, + { + "path": "/readiness", + "method": "GET", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/readiness", + "method": "GET", + "status": 503 + }, + { + "path": "/readiness", + "method": "GET", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "DELETE", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "PUT", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/liveness", + "method": "GET", + "status": 200 + }, + { + "path": "/logstream", + "method": "GET", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 400 + }, + { + "path": "/logstream/smoke", + "method": "DELETE", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "PUT", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "DELETE", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "PUT", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "DELETE", + "status": 503 + }, + { + "path": "/liveness", + "method": "GET", + "status": 200 + }, + { + "path": "/logstream", + "method": "GET", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 400 + }, + { + "path": "/logstream/smoke", + "method": "DELETE", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "PUT", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "DELETE", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "PUT", + "status": 200 + }, + { + "path": "/liveness", + "method": "GET", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "PUT", + "status": 200 + }, + { + "path": "/ingest", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/logstream/smoke/retention", + "method": "PUT", + "status": 200 + }, + { + "path": "/dashboards", + "method": "POST", + "status": 200 + }, + { + "path": "/liveness", + "method": "GET", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/dashboards", + "method": "GET", + "status": 200 + }, + { + "path": "/metrics", + "method": "GET", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/metrics", + "method": "GET", + "status": 200 + }, + { + "path": "/readiness", + "method": "GET", + "status": 503 + }, + { + "path": "/logstream/outage", + "method": "PUT", + "status": 503 + }, + { + "path": "/readiness", + "method": "GET", + "status": 503 + }, + { + "path": "/readiness", + "method": "GET", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/readiness", + "method": "GET", + "status": 503 + }, + { + "path": "/readiness", + "method": "GET", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "DELETE", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "PUT", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/liveness", + "method": "GET", + "status": 200 + }, + { + "path": "/logstream", + "method": "GET", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 400 + }, + { + "path": "/logstream/smoke", + "method": "DELETE", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "PUT", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "DELETE", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "PUT", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "DELETE", + "status": 503 + }, + { + "path": "/liveness", + "method": "GET", + "status": 200 + }, + { + "path": "/logstream", + "method": "GET", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 400 + }, + { + "path": "/logstream/smoke", + "method": "DELETE", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "PUT", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "DELETE", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "PUT", + "status": 200 + } + ] + } + ] +} diff --git a/docs/validation/postgres-review-explain.txt b/docs/validation/postgres-review-explain.txt new file mode 100644 index 000000000..c7cdbc081 --- /dev/null +++ b/docs/validation/postgres-review-explain.txt @@ -0,0 +1,133 @@ +CREATE TABLE +CREATE SEQUENCE +CREATE TABLE +ALTER TABLE +CREATE INDEX +CREATE FUNCTION +DROP TRIGGER +CREATE TRIGGER +UPDATE 0 +INSERT 0 200000 +INSERT 0 400 +ANALYZE + count | count +--------+------- + 200400 | 4 +(1 row) + + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Sort (cost=16.88..16.88 rows=2 width=43) (actual time=8.587..8.656 rows=2000 loops=1) + Sort Key: key COLLATE "C" + Sort Method: quicksort Memory: 49kB + Buffers: shared hit=974 read=1064 written=961 + -> Bitmap Heap Scan on parseable_documents (cost=8.88..16.87 rows=2 width=43) (actual time=0.765..7.578 rows=2000 loops=1) + Recheck Cond: (((tenant = 'tenant-1'::text) AND (key = 'tenant-1/logs/date=001'::text)) OR ((tenant = 'tenant-1'::text) AND (key >= 'tenant-1/logs/date=001/'::text) AND (key < 'tenant-1/logs/date=0010'::text))) + Heap Blocks: exact=2000 + Buffers: shared hit=974 read=1064 written=961 + -> BitmapOr (cost=8.88..8.88 rows=2 width=0) (actual time=0.520..0.522 rows=0 loops=1) + Buffers: shared hit=25 read=13 + -> Bitmap Index Scan on parseable_documents_pkey (cost=0.00..4.43 rows=1 width=0) (actual time=0.009..0.009 rows=0 loops=1) + Index Cond: ((tenant = 'tenant-1'::text) AND (key = 'tenant-1/logs/date=001'::text)) + Buffers: shared hit=3 + -> Bitmap Index Scan on parseable_documents_pkey (cost=0.00..4.45 rows=2 width=0) (actual time=0.511..0.511 rows=2000 loops=1) + Index Cond: ((tenant = 'tenant-1'::text) AND (key >= 'tenant-1/logs/date=001/'::text) AND (key < 'tenant-1/logs/date=0010'::text)) + Buffers: shared hit=22 read=13 + Planning: + Buffers: shared hit=11 + Planning Time: 0.200 ms + Execution Time: 8.808 ms +(20 rows) + + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Sort (cost=16.88..16.88 rows=2 width=1068) (actual time=3.188..3.240 rows=2000 loops=1) + Sort Key: key COLLATE "C" + Sort Method: quicksort Memory: 2174kB + Buffers: shared hit=2038 + -> Bitmap Heap Scan on parseable_documents (cost=8.88..16.87 rows=2 width=1068) (actual time=0.645..1.645 rows=2000 loops=1) + Recheck Cond: (((tenant = 'tenant-1'::text) AND (key = 'tenant-1/logs/date=001'::text)) OR ((tenant = 'tenant-1'::text) AND (key >= 'tenant-1/logs/date=001/'::text) AND (key < 'tenant-1/logs/date=0010'::text))) + Heap Blocks: exact=2000 + Buffers: shared hit=2038 + -> BitmapOr (cost=8.88..8.88 rows=2 width=0) (actual time=0.410..0.411 rows=0 loops=1) + Buffers: shared hit=38 + -> Bitmap Index Scan on parseable_documents_pkey (cost=0.00..4.43 rows=1 width=0) (actual time=0.009..0.009 rows=0 loops=1) + Index Cond: ((tenant = 'tenant-1'::text) AND (key = 'tenant-1/logs/date=001'::text)) + Buffers: shared hit=3 + -> Bitmap Index Scan on parseable_documents_pkey (cost=0.00..4.45 rows=2 width=0) (actual time=0.400..0.400 rows=2000 loops=1) + Index Cond: ((tenant = 'tenant-1'::text) AND (key >= 'tenant-1/logs/date=001/'::text) AND (key < 'tenant-1/logs/date=0010'::text)) + Buffers: shared hit=35 + Planning: + Buffers: shared hit=3 + Planning Time: 0.157 ms + Execution Time: 3.436 ms +(20 rows) + + QUERY PLAN +----------------------------------------------------------------------------------------------------------------------------------------------------------- + Index Scan using parseable_documents_pkey on parseable_documents (cost=0.42..12.87 rows=1 width=1068) (actual time=0.050..0.053 rows=2 loops=1) + Index Cond: ((tenant = 'tenant-1'::text) AND (key = ANY ('{tenant-1/logs/date=001/1.manifest.json,tenant-1/logs/date=001/101.manifest.json}'::text[]))) + Buffers: shared hit=5 + Planning Time: 0.075 ms + Execution Time: 0.070 ms +(5 rows) + + QUERY PLAN +--------------------------------------------------------------------------------------------------------------------------------------------------------- + Index Only Scan using parseable_stream_inventory on parseable_documents (cost=0.27..23.76 rows=5 width=43) (actual time=0.044..0.076 rows=100 loops=1) + Index Cond: (tenant = 'tenant-1'::text) + Heap Fetches: 100 + Buffers: shared hit=41 + Planning: + Buffers: shared hit=2 + Planning Time: 0.194 ms + Execution Time: 0.093 ms +(8 rows) + +BEGIN + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Delete on parseable_documents (cost=8.88..16.87 rows=0 width=0) (actual time=2.070..2.071 rows=0 loops=1) + Buffers: shared hit=4038 dirtied=1054 + -> Bitmap Heap Scan on parseable_documents (cost=8.88..16.87 rows=2 width=6) (actual time=0.465..1.629 rows=2000 loops=1) + Recheck Cond: (((tenant = 'tenant-1'::text) AND (key = 'tenant-1/logs/date=001'::text)) OR ((tenant = 'tenant-1'::text) AND (key >= 'tenant-1/logs/date=001/'::text) AND (key < 'tenant-1/logs/date=0010'::text))) + Heap Blocks: exact=2000 + Buffers: shared hit=2038 + -> BitmapOr (cost=8.88..8.88 rows=2 width=0) (actual time=0.300..0.301 rows=0 loops=1) + Buffers: shared hit=38 + -> Bitmap Index Scan on parseable_documents_pkey (cost=0.00..4.43 rows=1 width=0) (actual time=0.006..0.006 rows=0 loops=1) + Index Cond: ((tenant = 'tenant-1'::text) AND (key = 'tenant-1/logs/date=001'::text)) + Buffers: shared hit=3 + -> Bitmap Index Scan on parseable_documents_pkey (cost=0.00..4.45 rows=2 width=0) (actual time=0.294..0.294 rows=2000 loops=1) + Index Cond: ((tenant = 'tenant-1'::text) AND (key >= 'tenant-1/logs/date=001/'::text) AND (key < 'tenant-1/logs/date=0010'::text)) + Buffers: shared hit=35 + Planning Time: 0.061 ms + Execution Time: 2.156 ms +(16 rows) + +ROLLBACK + rows_after_populated_collation_migration +------------------------------------------ + 200400 +(1 row) + + attcollation +-------------- + "C" +(1 row) + + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------ + Result (cost=18.65..18.66 rows=1 width=1) (actual time=0.047..0.048 rows=1 loops=1) + Buffers: shared hit=2 read=2 + InitPlan 1 + -> Index Only Scan using parseable_documents_pkey on parseable_documents (cost=0.42..18.65 rows=1 width=0) (actual time=0.046..0.046 rows=1 loops=1) + Index Cond: ((tenant = 'tenant-1'::text) AND (key >= 'tenant-1/stream-1/.stream/'::text) AND (key < 'tenant-1/stream-1/.stream0'::text)) + Filter: (key ~~ '%stream.json'::text) + Heap Fetches: 0 + Buffers: shared hit=2 read=2 + Planning: + Buffers: shared hit=102 read=2 + Planning Time: 0.772 ms + Execution Time: 0.101 ms +(12 rows) diff --git a/docs/validation/postgres-review-smoke.json b/docs/validation/postgres-review-smoke.json new file mode 100644 index 000000000..006f48ef9 --- /dev/null +++ b/docs/validation/postgres-review-smoke.json @@ -0,0 +1,205 @@ +{ + "object-store": { + "lazy_reload": true, + "query": "[{\"count\":1}]", + "dashboard": { + "version": "v1", + "title": "Postgres smoke", + "author": "8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918", + "dashboardId": "01M74QAKS40EBX60MNDX4ZJRJH", + "created": "2026-11-11T07:14:30.820382797+00:00", + "modified": "2026-11-11T07:14:30.820382797+00:00", + "tags": null, + "isFavorite": false, + "dashboardType": "Dashboard", + "tiles": [], + "tenantId": null, + "revision": "410760f4f6ce9417312ed35635650d0ed5d2121c45bd21069d7bea97a6214f64" + }, + "keys": [ + ".parseable/.parseable.json", + ".parseable/ingestor.991bf8aa66a3be0.json", + ".parseable/querier.ce5fd278a14f75c.json", + ".users/.dashboard-locations/01M74QAKS40EBX60MNDX4ZJRJH.json", + ".users/.dashboard-titles/571a2cefe2d0d1a9a26dab4810a5556875c7d7531d4f558a8f7b28ab486e2395.json", + ".users/8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918/dashboards/01M74QAKS40EBX60MNDX4ZJRJH.json", + "lazy/.stream/.ingestor.991bf8aa66a3be0.schema", + "lazy/.stream/.ingestor.991bf8aa66a3be0.stream.json", + "lazy/.stream/.schema", + "lazy/.stream/.stream.json", + "lazy/date=2026-11-11/hour=07/minute=14/991bf8aa66a3be0.data.01M74QBH6TVHY0D3F6775WMZDN.parquet", + "lazy/date=2026-11-11/ingestor..991bf8aa66a3be0.manifest.json", + "pstats/.stream/.ingestor.991bf8aa66a3be0.schema", + "pstats/.stream/.ingestor.991bf8aa66a3be0.stream.json", + "pstats/.stream/.schema", + "pstats/.stream/.stream.json", + "pstats/date=2026-11-11/.manifest.json", + "pstats/date=2026-11-11/hour=07/minute=13/dataset_name=smoke/.data.01M74QARNVN5BPQ7Z8CJJJYE04.parquet", + "smoke/.stream/.ingestor.991bf8aa66a3be0.schema", + "smoke/.stream/.ingestor.991bf8aa66a3be0.stream.json", + "smoke/.stream/.schema", + "smoke/.stream/.stream.json", + "smoke/date=2026-11-11/.manifest.json", + "smoke/date=2026-11-11/hour=07/minute=13/.data.01M74Q9NS8KCT5185X617N4M6Y.parquet" + ], + "non_parquet_keys": [ + ".parseable/.parseable.json", + ".parseable/ingestor.991bf8aa66a3be0.json", + ".parseable/querier.ce5fd278a14f75c.json", + ".users/.dashboard-locations/01M74QAKS40EBX60MNDX4ZJRJH.json", + ".users/.dashboard-titles/571a2cefe2d0d1a9a26dab4810a5556875c7d7531d4f558a8f7b28ab486e2395.json", + ".users/8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918/dashboards/01M74QAKS40EBX60MNDX4ZJRJH.json", + "lazy/.stream/.ingestor.991bf8aa66a3be0.schema", + "lazy/.stream/.ingestor.991bf8aa66a3be0.stream.json", + "lazy/.stream/.schema", + "lazy/.stream/.stream.json", + "lazy/date=2026-11-11/ingestor..991bf8aa66a3be0.manifest.json", + "pstats/.stream/.ingestor.991bf8aa66a3be0.schema", + "pstats/.stream/.ingestor.991bf8aa66a3be0.stream.json", + "pstats/.stream/.schema", + "pstats/.stream/.stream.json", + "pstats/date=2026-11-11/.manifest.json", + "smoke/.stream/.ingestor.991bf8aa66a3be0.schema", + "smoke/.stream/.ingestor.991bf8aa66a3be0.stream.json", + "smoke/.stream/.schema", + "smoke/.stream/.stream.json", + "smoke/date=2026-11-11/.manifest.json" + ], + "request_delta": { + "parseable_storage_request_response_time_count{method=\"GET\",provider=\"s3\",status=\"200\"}": 30.0, + "parseable_storage_request_response_time_count{method=\"GET_RANGES\",provider=\"s3\",status=\"200\"}": 5.0, + "parseable_storage_request_response_time_count{method=\"LIST\",provider=\"s3\",status=\"200\"}": 5.0, + "parseable_storage_request_response_time_count{method=\"LIST_DELIM\",provider=\"s3\",status=\"200\"}": 5.0 + }, + "requests_per_query": 9.0, + "bucket_outage_readiness": 503, + "delete_recreate": true, + "final_keys": [ + ".parseable/.parseable.json", + ".parseable/ingestor.991bf8aa66a3be0.json", + ".parseable/querier.ce5fd278a14f75c.json", + ".users/.dashboard-locations/01M74QAKS40EBX60MNDX4ZJRJH.json", + ".users/.dashboard-titles/571a2cefe2d0d1a9a26dab4810a5556875c7d7531d4f558a8f7b28ab486e2395.json", + ".users/8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918/dashboards/01M74QAKS40EBX60MNDX4ZJRJH.json", + "lazy/.stream/.ingestor.991bf8aa66a3be0.schema", + "lazy/.stream/.ingestor.991bf8aa66a3be0.stream.json", + "lazy/.stream/.schema", + "lazy/.stream/.stream.json", + "lazy/date=2026-11-11/hour=07/minute=14/991bf8aa66a3be0.data.01M74QBH6TVHY0D3F6775WMZDN.parquet", + "lazy/date=2026-11-11/ingestor..991bf8aa66a3be0.manifest.json", + "pstats/.stream/.ingestor.991bf8aa66a3be0.schema", + "pstats/.stream/.ingestor.991bf8aa66a3be0.stream.json", + "pstats/.stream/.schema", + "pstats/.stream/.stream.json", + "pstats/date=2026-11-11/.manifest.json", + "pstats/date=2026-11-11/hour=07/minute=13/dataset_name=smoke/.data.01M74QARNVN5BPQ7Z8CJJJYE04.parquet", + "pstats/date=2026-11-11/hour=07/minute=14/dataset_name=lazy/991bf8aa66a3be0.data.01M74QDBSTHC3RTVX7DSBDG4R6.parquet", + "pstats/date=2026-11-11/ingestor..991bf8aa66a3be0.manifest.json", + "smoke/.stream/.ingestor.991bf8aa66a3be0.schema", + "smoke/.stream/.ingestor.991bf8aa66a3be0.stream.json", + "smoke/.stream/.schema", + "smoke/.stream/.stream.json", + "smoke/date=2026-11-11/hour=07/minute=15/991bf8aa66a3be0.data.01M74QDBSTPQCEJ9QBFKANH753.parquet", + "smoke/date=2026-11-11/ingestor..991bf8aa66a3be0.manifest.json" + ] + }, + "postgres-tls": { + "lazy_reload": true, + "query": "[{\"count\":1}]", + "dashboard": { + "version": "v1", + "title": "Postgres smoke", + "author": "8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918", + "dashboardId": "01M74QG3P3MHGHPFMD9S98MPTQ", + "created": "2026-11-11T07:17:30.947201084+00:00", + "modified": "2026-11-11T07:17:30.947201084+00:00", + "tags": null, + "isFavorite": false, + "dashboardType": "Dashboard", + "tiles": [], + "tenantId": null, + "revision": "4d9eab070b7637613a929c14f1303c15ac4e4f9ad1e9c8066863366625de52d4" + }, + "keys": [ + ".parseable/postgres-metastore-id", + "lazy/date=2026-11-11/hour=07/minute=17/0c643310e0de82d.data.01M74QH0C4PR4JWSPKVY2FQYD9.parquet", + "pstats/date=2026-11-11/hour=07/minute=16/dataset_name=smoke/.data.01M74QG8JNMAZ9Z6MJWE1BE3SK.parquet", + "smoke/date=2026-11-11/hour=07/minute=16/.data.01M74QF5SMF7ND9BTQE3139FZ6.parquet" + ], + "non_parquet_keys": [ + ".parseable/postgres-metastore-id" + ], + "request_delta": { + "parseable_storage_request_response_time_count{method=\"GET_RANGES\",provider=\"s3\",status=\"200\"}": 5.0 + }, + "requests_per_query": 1.0, + "database_outage": { + "readiness": 503, + "request_failed": true, + "seconds": 0.004562790039926767 + }, + "bucket_outage_readiness": 503, + "deleted_stream_rows": 0, + "delete_recreate": true, + "final_keys": [ + ".parseable/postgres-metastore-id", + "lazy/date=2026-11-11/hour=07/minute=17/0c643310e0de82d.data.01M74QH0C4PR4JWSPKVY2FQYD9.parquet", + "pstats/date=2026-11-11/hour=07/minute=16/dataset_name=smoke/.data.01M74QG8JNMAZ9Z6MJWE1BE3SK.parquet", + "pstats/date=2026-11-11/hour=07/minute=17/dataset_name=lazy/0c643310e0de82d.data.01M74QJTZ4XZF0G5S9QQYCKVXT.parquet", + "smoke/date=2026-11-11/hour=07/minute=18/0c643310e0de82d.data.01M74QJTZ4TK4Y4VW9DNG33BXF.parquet" + ], + "untrusted_ca_rejected": true + }, + "postgres": { + "lazy_reload": true, + "query": "[{\"count\":1}]", + "dashboard": { + "version": "v1", + "title": "Postgres smoke", + "author": "8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918", + "dashboardId": "01M74QNKSP83NNEMXQKJET4Q55", + "created": "2026-11-11T07:20:31.286622700+00:00", + "modified": "2026-11-11T07:20:31.286622700+00:00", + "tags": null, + "isFavorite": false, + "dashboardType": "Dashboard", + "tiles": [], + "tenantId": null, + "revision": "0fff761b383dd11e360a71fdb08cf1df18bb871cab49903853a19b1aa608d99c" + }, + "keys": [ + ".parseable/postgres-metastore-id", + "lazy/date=2026-11-11/hour=07/minute=20/b98cdf90ef5f01d.data.01M74QPGD6ZHAMA3M61E0P12J1.parquet", + "pstats/date=2026-11-11/hour=07/minute=19/dataset_name=smoke/.data.01M74QNRP8KGH60YDSV2PMSBRQ.parquet", + "smoke/date=2026-11-11/hour=07/minute=19/.data.01M74QMNWYNPWNFQVWVVKRYPFH.parquet" + ], + "non_parquet_keys": [ + ".parseable/postgres-metastore-id" + ], + "request_delta": { + "parseable_storage_request_response_time_count{method=\"GET_RANGES\",provider=\"s3\",status=\"200\"}": 5.0 + }, + "requests_per_query": 1.0, + "database_outage": { + "readiness": 503, + "request_failed": true, + "seconds": 0.003669349942356348 + }, + "bucket_outage_readiness": 503, + "deleted_stream_rows": 0, + "delete_recreate": true, + "final_keys": [ + ".parseable/postgres-metastore-id", + "lazy/date=2026-11-11/hour=07/minute=20/b98cdf90ef5f01d.data.01M74QPGD6ZHAMA3M61E0P12J1.parquet", + "pstats/date=2026-11-11/hour=07/minute=19/dataset_name=smoke/.data.01M74QNRP8KGH60YDSV2PMSBRQ.parquet", + "pstats/date=2026-11-11/hour=07/minute=20/dataset_name=lazy/b98cdf90ef5f01d.data.01M74QRB07K2NWMWJCFB4WYB7G.parquet", + "smoke/date=2026-11-11/hour=07/minute=21/b98cdf90ef5f01d.data.01M74QRB07D45VQGFJ851YWQ8R.parquet" + ] + }, + "startup_rejections": { + "future-schema": "Error: ObjectStorageError: PostgreSQL metastore unavailable: unsupported schema/protocol version; upgrade this server\n\nCaused by:\n PostgreSQL metastore unavailable: unsupported schema/protocol version; upgrade this server\n", + "wrong-id": "Error: ObjectStorageError: PostgreSQL metastore unavailable: backend/bucket identity mismatch\n\nCaused by:\n PostgreSQL metastore unavailable: backend/bucket identity mismatch\n", + "unreachable": "Error: ObjectStorageError: PostgreSQL metastore unavailable: Error occurred while creating a new object: error connecting to server\n\nCaused by:\n PostgreSQL metastore unavailable: Error occurred while creating a new object: error connecting to server\n" + }, + "postgres_rows": "tenant | kind | key | version | bytes \n--------+----------+--------------------------------------------------------------------------------------------------------------------+---------+-------\n | document | .parseable/.parseable.json | 27 | 384\n | document | .parseable/ingestor.b98cdf90ef5f01d.json | 37 | 241\n | document | .parseable/querier.e89f2bd5be65301.json | 28 | 240\n | document | .users/.dashboard-locations/01M74QNKSP83NNEMXQKJET4Q55.json | 16 | 119\n | document | .users/.dashboard-titles/571a2cefe2d0d1a9a26dab4810a5556875c7d7531d4f558a8f7b28ab486e2395.json | 18 | 170\n | document | .users/8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918/dashboards/01M74QNKSP83NNEMXQKJET4Q55.json | 17 | 545\n | schema | lazy/.stream/.ingestor.b98cdf90ef5f01d.schema | 54 | 585\n | stream | lazy/.stream/.ingestor.b98cdf90ef5f01d.stream.json | 52 | 853\n | schema | lazy/.stream/.schema | 56 | 585\n | stream | lazy/.stream/.stream.json | 55 | 597\n | manifest | lazy/date=2026-11-11/ingestor..b98cdf90ef5f01d.manifest.json | 50 | 423\n | schema | pstats/.stream/.ingestor.b98cdf90ef5f01d.schema | 81 | 1117\n | stream | pstats/.stream/.ingestor.b98cdf90ef5f01d.stream.json | 79 | 895\n | schema | pstats/.stream/.schema | 32 | 1117\n | stream | pstats/.stream/.stream.json | 21 | 867\n | manifest | pstats/date=2026-11-11/.manifest.json | 19 | 559\n | manifest | pstats/date=2026-11-11/ingestor..b98cdf90ef5f01d.manifest.json | 77 | 697\n | schema | smoke/.stream/.ingestor.b98cdf90ef5f01d.schema | 73 | 585\n | stream | smoke/.stream/.ingestor.b98cdf90ef5f01d.stream.json | 71 | 854\n | schema | smoke/.stream/.schema | 75 | 585\n | stream | smoke/.stream/.stream.json | 59 | 597\n | manifest | smoke/date=2026-11-11/ingestor..b98cdf90ef5f01d.manifest.json | 69 | 412\n(22 rows)" +} \ No newline at end of file diff --git a/scripts/postgres-metastore-migration-smoke.py b/scripts/postgres-metastore-migration-smoke.py new file mode 100755 index 000000000..c94e45592 --- /dev/null +++ b/scripts/postgres-metastore-migration-smoke.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python3 +"""Isolated MinIO/PostgreSQL offline migration smoke. No live deployment access. +Uses the existing smoke fixture's helpers; every created container is removed. +""" +from pathlib import Path +import os +import subprocess as sp +import time +import json +import base64 +import datetime as dt +import urllib.request + +# Load only reusable definitions, never the existing smoke's executable driver. +namespace = {'__file__': str(Path(__file__).with_name('postgres-metastore-smoke.py'))} +source = Path(namespace['__file__']).read_text().split('\ntry:\n pgport, s3port')[0] +exec(compile(source, namespace['__file__'], 'exec'), namespace) +run, port, container, wait_for, list_keys = [namespace[k] for k in ('run','port','container','wait_for','list_keys')] +root, out = namespace['ROOT'], namespace['OUT'] +binary = str(Path(os.environ.get('CARGO_TARGET_DIR', root/'target'))/'debug/parseable') +server = None +log = open(out/'migration.log','w') +def stop(): + global server + if server: + server.terminate() + try: server.wait(timeout=30) + except sp.TimeoutExpired: server.kill(); server.wait() + server = None +try: + pgport, s3port, http = port(), port(), port() + pg, cmd = container('postgres:17-alpine',POSTGRES_PASSWORD='smoke',POSTGRES_DB='migration') + run(*cmd,'-p',f'127.0.0.1:{pgport}:5432','postgres:17-alpine') + mini, cmd = container('quay.io/parseablehq/minio:edge',MINIO_ROOT_USER='minioadmin',MINIO_ROOT_PASSWORD='minioadmin') + run(*cmd,'-p',f'127.0.0.1:{s3port}:9000','quay.io/parseablehq/minio:edge','server','/data') + run('docker','exec',mini,'mkdir','-p','/data/migration') + wait_for(lambda:'accepting connections' in run('docker','exec',pg,'pg_isready','-U','postgres')) + env = dict(os.environ,P_ADDR=f'127.0.0.1:{http}',P_GRPC_PORT=str(port()),P_FLIGHT_PORT=str(port()), + P_STAGING_DIR=str(out/f'migration-staging-{os.getpid()}'),P_USERNAME='admin',P_PASSWORD='smoke-pass', + P_S3_URL=f'http://127.0.0.1:{s3port}',P_S3_BUCKET='migration',P_S3_REGION='us-east-1', + P_S3_ACCESS_KEY='minioadmin',P_S3_SECRET_KEY='minioadmin',P_S3_PATH_STYLE='true',RUST_LOG='warn') + env.pop('P_METASTORE_POSTGRES_URL',None) + pgurl = f'postgresql://postgres:smoke@127.0.0.1:{pgport}/migration?sslmode=disable' + if os.environ.get('P_MIGRATION_TLS') == '1': + tls = out / 'migration-tls' + tls.mkdir(exist_ok=True) + run('openssl', 'req', '-x509', '-newkey', 'rsa:2048', '-nodes', '-keyout', str(tls/'ca.key'), '-out', str(tls/'ca.crt'), '-days', '2', '-subj', '/CN=Smoke CA') + run('openssl', 'req', '-newkey', 'rsa:2048', '-nodes', '-keyout', str(tls/'server.key'), '-out', str(tls/'server.csr'), '-subj', '/CN=localhost') + (tls/'extensions').write_text('subjectAltName=DNS:localhost,IP:127.0.0.1\nextendedKeyUsage=serverAuth\n') + run('openssl', 'x509', '-req', '-in', str(tls/'server.csr'), '-CA', str(tls/'ca.crt'), '-CAkey', str(tls/'ca.key'), '-CAcreateserial', '-out', str(tls/'server.crt'), '-days', '2', '-extfile', str(tls/'extensions')) + for name in ['server.key', 'server.crt']: + run('docker', 'cp', str(tls/name), pg+':/var/lib/postgresql/data/'+name) + run('docker', 'exec', pg, 'sh', '-c', 'chown postgres:postgres /var/lib/postgresql/data/server.* && chmod 600 /var/lib/postgresql/data/server.key') + run('docker', 'exec', pg, 'psql', '-U', 'postgres', '-d', 'migration', '-c', "ALTER SYSTEM SET ssl='on';") + run('docker', 'exec', pg, 'psql', '-U', 'postgres', '-d', 'migration', '-c', 'SELECT pg_reload_conf();') + + env['P_METASTORE_POSTGRES_CA'] = str(tls/'ca.crt') + pgurl = pgurl.replace('sslmode=disable', 'sslmode=require') + + def request(path,method='GET',body=None,headers=None): + req = urllib.request.Request(f'http://127.0.0.1:{http}/api/v1'+path,method=method, + data=None if body is None else json.dumps(body).encode(),headers={ + 'Authorization':'Basic '+base64.b64encode(b'admin:smoke-pass').decode(), + 'Content-Type':'application/json',**(headers or {})}) + with urllib.request.urlopen(req,timeout=90) as response: return response.read().decode() + def start(): + global server + server = sp.Popen([binary,'s3-store'],env=env,stdout=log,stderr=log) + def ready(): + if server.poll() is not None: raise RuntimeError('server exited: '+str(log.name)) + request('/liveness'); return True + wait_for(ready) + def wait_for_staging_drain(): + quiet_since = None + def drained(): + nonlocal quiet_since + pending = any(p.is_file() and p.suffix in {'.arrow', '.arrows', '.part', '.parquet'} + for p in Path(env['P_STAGING_DIR']).rglob('*')) + if pending: + quiet_since = None + return False + if quiet_since is None: + quiet_since = time.monotonic() + return time.monotonic() - quiet_since >= 2 + wait_for(drained, 300) + def migrate(action,*flags,expected=0): + result = sp.run([binary,'s3-store','--metadata-migration',action,'--acknowledge-server-stopped',*flags], + env=dict(env,P_METASTORE_POSTGRES_URL=pgurl),stdout=log,stderr=log,timeout=180) + assert (result.returncode == 0) == (expected == 0), (action,result.returncode) + start() + for stream in ['first','second','partitioned']: + request('/logstream/'+stream,'PUT',headers={'X-P-Custom-Partition':'region'} if stream=='partitioned' else None) + request('/ingest','POST',[{'message':'migration','value':42,'region':'west'}],{'X-P-Stream':stream}) + request('/role/migration-reader','PUT',{'roleType':'user','actions':[{'privilege':'reader','resource':{'stream':'first'}}]}) + request('/user/migration-user','POST',['migration-reader']) + request('/dashboards','POST',{'title':'Migration dashboard','tiles':[]}) + request('/filters','POST',{'stream_name':'first','filter_name':'Migration filter','query':{'filter_type':'sql','filter_query':'SELECT * FROM first','filter_builder':None},'time_filter':None}) + request('/alerts','POST',{'severity':'high','title':'Migration alert','alertType':'threshold', + 'queryType':'code','query':'select count(*) as count from first','thresholdConfig':{'operator':'>','value':1000}, + 'evalConfig':{'rollingWindow':{'evalStart':'10 minutes','evalEnd':'now','evalFrequency':10}}, + 'targets':[],'notificationConfig':{'interval':1}}) + wait_for(lambda:sum(k.endswith('manifest.json') for k in list_keys(env['P_S3_URL'],'migration'))>=3,240) + query={'query':'select sum(value) as value from first', + 'startTime':(dt.datetime.now(dt.timezone.utc)-dt.timedelta(hours=1)).isoformat(), + 'endTime':(dt.datetime.now(dt.timezone.utc)+dt.timedelta(hours=1)).isoformat()} + before=request('/query','POST',query) + def normalized(raw): + def ordered(value): + if isinstance(value,dict): return {k:ordered(v) for k,v in sorted(value.items())} + if isinstance(value,list): return sorted([ordered(v) for v in value],key=lambda v:json.dumps(v,sort_keys=True)) + return value + return ordered(json.loads(raw)) + def snapshot(path): + value=normalized(request(path)) + if path.endswith('/stats'): value.pop('time',None) # response generation time + return value + # Query statistics are created asynchronously; include the internal stream in both snapshots. + wait_for(lambda: any(item['name']=='pstats' for item in json.loads(request('/logstream'))),120) + wait_for_staging_drain() + saved={p:snapshot(p) for p in ['/logstream','/logstream/first/schema','/logstream/first/info','/logstream/first/stats','/dashboards','/filters','/user','/alerts']} + wait_for_staging_drain() + stop() + stuck = Path(env['P_STAGING_DIR'])/'stranded.part' + stuck.write_bytes(b'unsynced') + migrate('import','--dry-run',expected=1) + stuck.unlink() + + print('object-store fixture uploaded; starting offline import',flush=True) + migrate('verify',expected=1) + migrate('import','--dry-run') + # Upload actual S3 metadata objects so interruption is observable on MinIO's XL layout. + for i in range(500): + namespace['get_key'](env['P_S3_URL'], 'migration', f'.settings/migration-fixtures/{i}.json', body=b'{}') + interrupted=sp.Popen([binary,'s3-store','--metadata-migration','import','--acknowledge-server-stopped'], + env=dict(env,P_METASTORE_POSTGRES_URL=pgurl),stdout=log,stderr=log) + deadline=time.monotonic()+120 + while '.parseable/metastore-migration' not in list_keys(env['P_S3_URL'],'migration'): + assert interrupted.poll() is None, 'import completed before interruption' + assert time.monotonic()= expected_latest + wait_for(bounds_published) + latest={p:snapshot(p) for p in saved if p != '/alerts'} + stop() + print('new PostgreSQL rows ingested; starting offline export',flush=True) + export=sp.Popen([binary,'s3-store','--metadata-migration','export','--acknowledge-server-stopped'],env=env,stdout=log,stderr=log) + deadline=time.monotonic()+120 + while '.parseable/metastore-migration' not in list_keys(env['P_S3_URL'],'migration'): + assert export.poll() is None, 'export completed before interruption' + assert time.monotonic()= 'tenant-1/logs/date=001/' AND key < 'tenant-1/logs/date=0010')) ORDER BY key; +EXPLAIN (ANALYZE, BUFFERS) SELECT key,body FROM parseable_documents WHERE tenant='tenant-1' AND (key='tenant-1/logs/date=001' OR (key >= 'tenant-1/logs/date=001/' AND key < 'tenant-1/logs/date=0010')) ORDER BY key; +EXPLAIN (ANALYZE, BUFFERS) SELECT key,body FROM parseable_documents WHERE tenant='tenant-1' AND key=ANY(ARRAY['tenant-1/logs/date=001/1.manifest.json','tenant-1/logs/date=001/101.manifest.json']); +EXPLAIN (ANALYZE, BUFFERS) SELECT key FROM parseable_documents WHERE tenant='tenant-1' AND key LIKE '%/.stream/%'; +BEGIN; +EXPLAIN (ANALYZE, BUFFERS) DELETE FROM parseable_documents WHERE tenant='tenant-1' AND (key='tenant-1/logs/date=001' OR (key >= 'tenant-1/logs/date=001/' AND key < 'tenant-1/logs/date=0010')); +ROLLBACK; +EXPLAIN (ANALYZE, BUFFERS) SELECT EXISTS(SELECT 1 FROM parseable_documents WHERE tenant='tenant-1' AND key >= 'tenant-1/stream-1/.stream/' AND key < 'tenant-1/stream-1/.stream0' AND key LIKE '%stream.json'); diff --git a/scripts/postgres-metastore-smoke.py b/scripts/postgres-metastore-smoke.py new file mode 100755 index 000000000..1fe7164d1 --- /dev/null +++ b/scripts/postgres-metastore-smoke.py @@ -0,0 +1,474 @@ +#!/usr/bin/env python3 +"""Isolated MinIO/PostgreSQL smoke and request-count comparison (no live services). +Run after P_BUILD_NO_UI=1 CARGO_BUILD_JOBS=4 cargo build. +Artifacts stay in .postgres-smoke/; all started processes/containers are removed. +""" +import atexit +import signal +import sys +import traceback +import base64 +import datetime as dt +import json +import hashlib +import hmac +import xml.etree.ElementTree as ET +import os +from pathlib import Path +import socket +import subprocess as sp +import time +import urllib.request +import urllib.error + +ROOT = Path(__file__).resolve().parents[1] +OUT = ROOT / '.postgres-smoke' +OUT.mkdir(exist_ok=True) +ARCHIVE = OUT / 'runs' +ARCHIVE.mkdir(exist_ok=True) +if (OUT/'results.json').exists(): + previous = (OUT/'results.json').read_bytes() + (ARCHIVE/(hashlib.sha256(previous).hexdigest()+'.json')).write_bytes(previous) +selection = os.environ.get('P_SMOKE_MODE', 'all') +assert selection in ['all', 'postgres', 'postgres-tls'], selection +modes = ['object-store', 'postgres-tls', 'postgres'] if selection == 'all' else [selection] +containers = [] +server = None +auxiliary = None + +commands = [] +processes = [] +results = {"requested_modes": modes} +http_requests = [] +def run(*args): + attempt = sp.run(args, text=True, stdout=sp.PIPE, stderr=sp.PIPE) + commands.append(dict(argv=list(args), exit_code=attempt.returncode)) + attempt.check_returncode() + return attempt.stdout.strip() + +def postgres_logs(name): + args = ['docker', 'logs', name] + attempt = sp.run(args, text=True, stdout=sp.PIPE, stderr=sp.STDOUT) + commands.append(dict(argv=args, exit_code=attempt.returncode)) + attempt.check_returncode() + return attempt.stdout + +def scalar(response, column, expected): + value = json.loads(response) + assert value == [{column: expected}], value + return True + +def wait_for_staging_drain(staging): + quiet_since = None + def drained(): + nonlocal quiet_since + pending = any(p.is_file() and p.suffix in {'.arrow', '.arrows', '.part', '.parquet'} + for p in Path(staging).rglob('*')) + if pending: + quiet_since = None + return False + if quiet_since is None: + quiet_since = time.monotonic() + return time.monotonic() - quiet_since >= 2 + wait_for(drained, 300) + +def cleanup(): + global auxiliary + try: + if auxiliary is not None: + auxiliary.terminate() + try: + auxiliary.wait(timeout=15) + except sp.TimeoutExpired: + auxiliary.kill() + auxiliary.wait() + auxiliary = None + stop() + finally: + for name in reversed(containers): + attempt = sp.run(['docker', 'rm', '-f', name], stdout=sp.DEVNULL, stderr=sp.DEVNULL) + commands.append(dict(argv=['docker', 'rm', '-f', name], exit_code=attempt.returncode)) + containers.clear() + for process in processes: + commands.append(dict(argv=process.args, exit_code=process.poll())) + processes.clear() + results['commands'] = commands + results['success'] = sys.exc_info()[0] is None + if sys.exc_info()[0] is not None: + results['failure'] = traceback.format_exc() + if (OUT/'checks.json').exists(): + results['validation'] = json.loads((OUT/'checks.json').read_text()) + results['http_requests'] = http_requests + evidence = json.dumps(results, indent=2) + '\n' + (OUT/'results.json').write_text(evidence) + (ARCHIVE/(hashlib.sha256(evidence.encode()).hexdigest()+'.json')).write_text(evidence) + runs = [json.loads(path.read_text()) for path in sorted(ARCHIVE.glob('*.json'))] + (ROOT/'docs/validation/postgres-metastore-smoke.json').write_text(json.dumps({'runs': runs}, indent=2)+'\n') + +def interrupted(signum, frame): + raise SystemExit(128 + signum) +if globals().get('__name__') == '__main__': + atexit.register(cleanup) + signal.signal(signal.SIGTERM, interrupted) + signal.signal(signal.SIGINT, interrupted) + +def startup_attempt(environment, timeout): + process = sp.Popen([str(Path(os.environ.get('CARGO_TARGET_DIR', ROOT/'target'))/'debug/parseable'), 's3-store'], + env=environment, cwd=ROOT, stdout=sp.PIPE, stderr=sp.STDOUT, text=True) + processes.append(process) + try: + output, _ = process.communicate(timeout=timeout) + except sp.TimeoutExpired: + process.kill() + process.communicate() + raise + return sp.CompletedProcess(process.args, process.returncode, output) + +def port(): + with socket.socket() as s: + s.bind(('127.0.0.1', 0)) + return s.getsockname()[1] + +def container(image, *args, **env): + name = 'parseable-pg-smoke-' + str(os.getpid()) + '-' + str(len(containers)) + command = ['docker', 'run', '-d', '--name', name] + for key, value in env.items(): + command += ['-e', f'{key}={value}'] + containers.append(name) + return name, command + +def list_keys(endpoint, bucket): + """List real S3 keys with SigV4; no client package or image utilities needed.""" + now = dt.datetime.now(dt.timezone.utc) + stamp, day = now.strftime('%Y%m%dT%H%M%SZ'), now.strftime('%Y%m%d') + host = endpoint.removeprefix('http://') + payload = hashlib.sha256(b'').hexdigest() + headers = f'host:{host}\nx-amz-content-sha256:{payload}\nx-amz-date:{stamp}\n' + signed = 'host;x-amz-content-sha256;x-amz-date' + canonical = f'GET\n/{bucket}\nlist-type=2\n{headers}\n{signed}\n{payload}' + scope = f'{day}/us-east-1/s3/aws4_request' + message = f'AWS4-HMAC-SHA256\n{stamp}\n{scope}\n{hashlib.sha256(canonical.encode()).hexdigest()}' + key = b'AWS4minioadmin' + for value in [day, 'us-east-1', 's3', 'aws4_request']: + key = hmac.new(key, value.encode(), hashlib.sha256).digest() + signature = hmac.new(key, message.encode(), hashlib.sha256).hexdigest() + authorization = f'AWS4-HMAC-SHA256 Credential=minioadmin/{scope}, SignedHeaders={signed}, Signature={signature}' + req = urllib.request.Request(f'{endpoint}/{bucket}?list-type=2', headers={ + 'Authorization': authorization, 'X-Amz-Date': stamp, 'X-Amz-Content-Sha256': payload}) + with urllib.request.urlopen(req, timeout=10) as response: + root = ET.fromstring(response.read()) + assert root.findtext('{*}IsTruncated') != 'true', 'smoke fixture exceeded a listing page' + return sorted(node.text for node in root.findall('{*}Contents/{*}Key')) + +def get_key(endpoint, bucket, key, body=None): + """Read publication documents with SigV4 without an SDK dependency.""" + now = dt.datetime.now(dt.timezone.utc) + stamp, day = now.strftime('%Y%m%dT%H%M%SZ'), now.strftime('%Y%m%d') + host = endpoint.removeprefix('http://') + method = 'GET' if body is None else 'PUT' + payload = hashlib.sha256(body or b'').hexdigest() + headers = f'host:{host}\nx-amz-content-sha256:{payload}\nx-amz-date:{stamp}\n' + signed = 'host;x-amz-content-sha256;x-amz-date' + canonical = f'{method}\n/{bucket}/{key}\n\n{headers}\n{signed}\n{payload}' + scope = f'{day}/us-east-1/s3/aws4_request' + message = f'AWS4-HMAC-SHA256\n{stamp}\n{scope}\n{hashlib.sha256(canonical.encode()).hexdigest()}' + signing_key = b'AWS4minioadmin' + for value in [day, 'us-east-1', 's3', 'aws4_request']: + signing_key = hmac.new(signing_key, value.encode(), hashlib.sha256).digest() + signature = hmac.new(signing_key, message.encode(), hashlib.sha256).hexdigest() + authorization = f'AWS4-HMAC-SHA256 Credential=minioadmin/{scope}, SignedHeaders={signed}, Signature={signature}' + req = urllib.request.Request(f'{endpoint}/{bucket}/{key}', data=body, method=method, headers={ + 'Authorization': authorization, 'X-Amz-Date': stamp, 'X-Amz-Content-Sha256': payload}) + with urllib.request.urlopen(req, timeout=10) as response: + return response.read() + +def stop(): + global server + if server: + server.terminate() + try: + server.wait(timeout=15) + except sp.TimeoutExpired: + server.kill() + server.wait() + server = None + +def wait_for(check, seconds=180): + deadline = time.monotonic() + seconds + last_error = None + while time.monotonic() < deadline: + try: + result = check() + if result: + return result + except RuntimeError: + raise + except Exception as error: + last_error = error + time.sleep(1) + raise RuntimeError('timed out waiting for smoke fixture') from last_error + +try: + pgport, s3port = port(), port() + pg, cmd = container('postgres:17-alpine', POSTGRES_PASSWORD='smoke', POSTGRES_DB='parseable') + run(*cmd, '-p', f'127.0.0.1:{pgport}:5432', 'postgres:17-alpine') + mini, cmd = container('quay.io/parseablehq/minio:edge', MINIO_ROOT_USER='minioadmin', MINIO_ROOT_PASSWORD='minioadmin') + run(*cmd, '-p', f'127.0.0.1:{s3port}:9000', 'quay.io/parseablehq/minio:edge', 'server', '/data') + wait_for(lambda: 'accepting connections' in run('docker', 'exec', pg, 'pg_isready', '-h', '127.0.0.1', '-U', 'postgres')) + # An isolated CA and signed localhost server certificate; no host config changes. + tls = OUT / 'tls' + tls.mkdir(exist_ok=True) + run('openssl', 'req', '-x509', '-newkey', 'rsa:2048', '-nodes', '-keyout', str(tls/'ca.key'), '-out', str(tls/'ca.crt'), '-days', '2', '-subj', '/CN=Smoke CA') + run('openssl', 'req', '-newkey', 'rsa:2048', '-nodes', '-keyout', str(tls/'server.key'), '-out', str(tls/'server.csr'), '-subj', '/CN=localhost') + (tls/'extensions').write_text('subjectAltName=DNS:localhost,IP:127.0.0.1\nextendedKeyUsage=serverAuth\n') + run('openssl', 'x509', '-req', '-in', str(tls/'server.csr'), '-CA', str(tls/'ca.crt'), '-CAkey', str(tls/'ca.key'), '-CAcreateserial', '-out', str(tls/'server.crt'), '-days', '2', '-extfile', str(tls/'extensions')) + for name in ['server.key', 'server.crt']: + run('docker', 'cp', str(tls/name), pg+':/var/lib/postgresql/data/'+name) + run('docker', 'exec', pg, 'sh', '-c', 'chown postgres:postgres /var/lib/postgresql/data/server.* && chmod 600 /var/lib/postgresql/data/server.key') + run('docker', 'exec', pg, 'psql', '-U', 'postgres', '-d', 'parseable', '-c', "ALTER SYSTEM SET ssl='on';") + run('docker', 'exec', pg, 'psql', '-U', 'postgres', '-d', 'parseable', '-c', 'SELECT pg_reload_conf();') + results['source_commit'] = run('git', 'rev-parse', 'HEAD') + results['binary_sha256'] = hashlib.sha256((Path(os.environ.get('CARGO_TARGET_DIR', ROOT/'target'))/'debug/parseable').read_bytes()).hexdigest() + results['source_diff_sha256'] = hashlib.sha256(run('git', 'diff', 'HEAD').encode()).hexdigest() + results['image_digests'] = {image: json.loads(run('docker', 'image', 'inspect', image))[0]['RepoDigests'] + for image in ['postgres:17-alpine', 'quay.io/parseablehq/minio:edge']} + results['measurement'] = 'Node-local storage counter delta during five queries, including background traffic; not isolated requests per query.' + for mode in modes: + print('starting '+mode, flush=True) + bucket = 'smoke-' + mode + run('docker', 'exec', mini, 'mkdir', '-p', '/data/' + bucket) + http, grpc, flight = port(), port(), port() + base = f'http://127.0.0.1:{http}/api/v1' + env = dict(os.environ, P_ADDR=f'127.0.0.1:{http}', P_GRPC_PORT=str(grpc), P_FLIGHT_PORT=str(flight), + P_STAGING_DIR=str(OUT / (mode + '-staging-' + str(os.getpid()))), P_USERNAME='admin', P_PASSWORD='smoke-pass', + P_S3_URL=f'http://127.0.0.1:{s3port}', P_S3_BUCKET=bucket, P_S3_REGION='us-east-1', + P_S3_ACCESS_KEY='minioadmin', P_S3_SECRET_KEY='minioadmin', P_S3_PATH_STYLE='true', RUST_LOG='warn') + env.pop('P_METASTORE_POSTGRES_URL', None) + if mode.startswith('postgres'): + env['P_METASTORE_POSTGRES_URL'] = f'postgresql://postgres:smoke@127.0.0.1:{pgport}/parseable?sslmode=disable' + if mode == 'postgres-tls': + run('docker', 'exec', pg, 'psql', '-U', 'postgres', '-c', 'CREATE DATABASE tls;') + env['P_METASTORE_POSTGRES_URL'] = f'postgresql://postgres:smoke@localhost:{pgport}/tls?sslmode=require' + env['P_METASTORE_POSTGRES_CA'] = str(tls/'ca.crt') + log = open(OUT / (mode + '.log'), 'w') + def start(): + global server + server = sp.Popen([str(Path(os.environ.get('CARGO_TARGET_DIR', ROOT / 'target')) / 'debug/parseable'), 's3-store'], env=env, cwd=ROOT, stdout=log, stderr=log) + processes.append(server) + def request(path, method='GET', body=None, headers=None, timeout=30): + data = None if body is None else json.dumps(body).encode() + req = urllib.request.Request(base + path, data=data, method=method, + headers={'Authorization': 'Basic ' + base64.b64encode(b'admin:smoke-pass').decode(), + 'Content-Type': 'application/json', **(headers or {})}) + try: + with urllib.request.urlopen(req, timeout=timeout) as response: + http_requests.append(dict(path=path, method=method, status=response.status)) + return response.read().decode() + except urllib.error.HTTPError as error: + http_requests.append(dict(path=path, method=method, status=error.code)) + error.add_note(error.read().decode()) + raise + def status(path, method='GET', body=None): + try: + request(path, method, body) + return 200 + except urllib.error.HTTPError as error: + return error.code + def ready(): + if server.poll() is not None: + raise RuntimeError('server exited; inspect ' + str(log.name)) + request('/liveness') + return True + results[mode] = {} + start() + wait_for(ready) + if mode == 'postgres-tls': + stop() + rejected = env.copy() + run('openssl', 'req', '-x509', '-newkey', 'rsa:2048', '-nodes', '-keyout', str(tls/'wrong.key'), '-out', str(tls/'wrong.crt'), '-days', '2', '-subj', '/CN=Wrong CA') + rejected['P_METASTORE_POSTGRES_CA'] = str(tls/'wrong.crt') + prior_logs = postgres_logs(pg) + attempt = startup_attempt(rejected, 30) + rejection_logs = postgres_logs(pg)[len(prior_logs):] + commands.append(dict(argv=attempt.args, exit_code=attempt.returncode)) + assert attempt.returncode != 0 + assert 'error performing tls handshake' in attempt.stdout.lower(), attempt.stdout + assert 'unknown ca' in rejection_logs.lower(), (attempt.stdout, rejection_logs) + start() + wait_for(ready) + results[mode]['trusted_ca_restart'] = True + results[mode]['untrusted_ca_rejected'] = True + results[mode]['untrusted_ca_output'] = attempt.stdout + results[mode]['untrusted_ca_server_alert'] = rejection_logs + request('/logstream/smoke', 'PUT') + request('/ingest', 'POST', [{'message': 'postgres smoke', 'value': 42}], {'X-P-Stream': 'smoke'}) + database = 'tls' if mode == 'postgres-tls' else 'parseable' + def published(stream): + if mode.startswith('postgres'): + raw = run('docker', 'exec', pg, 'psql', '-U', 'postgres', '-d', database, '-Atc', + f"SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'{stream}/.stream/') AND right(key,11)='stream.json';") + documents = [json.loads(line) for line in raw.splitlines()] + else: + documents = [json.loads(get_key(env['P_S3_URL'], bucket, key)) + for key in list_keys(env['P_S3_URL'], bucket) + if key.startswith(stream+'/.stream/') and key.endswith('stream.json')] + return any(document.get('snapshot', {}).get('manifest_list') for document in documents) + def uploaded(): + return any(key.startswith('smoke/') and key.endswith('.parquet') + for key in list_keys(env['P_S3_URL'], bucket)) and published('smoke') + wait_for(uploaded, 240) + query = {'query': 'select count(*) as count from smoke', + 'startTime': (dt.datetime.now(dt.timezone.utc)-dt.timedelta(hours=1)).isoformat(), + 'endTime': (dt.datetime.now(dt.timezone.utc)+dt.timedelta(minutes=5)).isoformat()} + first = request('/query', 'POST', query) + scalar(first, 'count', 1) + request('/logstream/smoke/retention', 'PUT', [{'description': 'smoke retention', 'action': 'delete', 'duration': '30d'}]) + dashboard = request('/dashboards', 'POST', {'title': 'Postgres smoke', 'tiles': []}) + wait_for_staging_drain(env['P_STAGING_DIR']) + stop() + env['P_MODE'] = 'query' + env['P_QUERIER_ENDPOINT'] = f'127.0.0.1:{http}' + start() + wait_for(ready) + # Publish a stream from another process after this process restarted, + # so its first query must enter the cold-load path. + auxiliary_env = env.copy() + auxiliary_env['P_MODE'] = 'ingest' + auxiliary_port = port() + auxiliary_env.update(P_INGESTOR_ENDPOINT=f'127.0.0.1:{auxiliary_port}', P_ADDR=f'127.0.0.1:{auxiliary_port}', P_GRPC_PORT=str(port()), P_FLIGHT_PORT=str(port()), + P_STAGING_DIR=str(OUT/(mode+'-aux-'+str(os.getpid())))) + auxiliary = sp.Popen([str(Path(os.environ.get('CARGO_TARGET_DIR', ROOT/'target'))/'debug/parseable'), 's3-store'], env=auxiliary_env, stdout=log, stderr=log) + processes.append(auxiliary) + def aux_request(path, method='GET', body=None, headers=None): + original_base = base + # request() closes over base; temporarily use the other node. + return auxiliary_request(original_base.replace(str(http), str(auxiliary_port)), path, method, body, headers) + def auxiliary_request(node_base, path, method, body, headers): + req = urllib.request.Request(node_base+path, method=method, + data=None if body is None else json.dumps(body).encode(), headers={ + 'Authorization': 'Basic '+base64.b64encode(b'admin:smoke-pass').decode(), + 'Content-Type': 'application/json', **(headers or {})}) + with urllib.request.urlopen(req, timeout=30) as response: + return response.read().decode() + wait_for(lambda: aux_request('/liveness') == '') + aux_request('/ingest', 'POST', [{'message':'lazy reload', 'value':9}], {'X-P-Stream':'lazy'}) + wait_for(lambda: published('lazy'), 240) + # Snapshot publication is polled before the first cold-load request. + lazy_query = dict(query, query='select count(*) as count from lazy') + wait_for(lambda: scalar(request('/query', 'POST', lazy_query, timeout=90), 'count', 1), 240) + print(mode+': cold load passed', flush=True) + second = request('/query', 'POST', query) + assert json.loads(first) == json.loads(second), (first, second) + assert any(item.get('title') == 'Postgres smoke' for item in json.loads(request('/dashboards'))), request('/dashboards') + def counts(): + raw = request('/metrics') + return {line.split()[0]:float(line.split()[1]) for line in raw.splitlines() + if line.startswith('parseable_storage_request_response_time_count')} + before = counts() + for _ in range(5): + assert json.loads(request('/query', 'POST', query)) == json.loads(second) + after = counts() + delta = {key: value-before.get(key,0) for key,value in after.items() if value-before.get(key,0)} + keys = list_keys(env['P_S3_URL'], bucket) + results[mode].update(dict(lazy_reload=True, query=second,dashboard=json.loads(dashboard),keys=keys, + non_parquet_keys=[k for k in keys if not k.endswith('.parquet')], + request_delta=delta, observed_requests_during_window_divided_by_queries=sum(delta.values())/5)) + if mode.startswith('postgres'): + assert results[mode]['non_parquet_keys'] == ['.parseable/postgres-metastore-id'] + run('docker', 'stop', pg) + started = time.monotonic() + assert status('/readiness') == 503 + assert status('/logstream/outage', 'PUT') >= 400 + elapsed = time.monotonic()-started + assert elapsed < 10, elapsed + results[mode]['database_outage'] = dict(readiness=503, request_failed=True, seconds=elapsed) + run('docker', 'start', pg) + wait_for(lambda: status('/readiness') == 200) + assert json.loads(request('/query', 'POST', query)) == json.loads(second) + run('docker', 'stop', mini) + assert status('/readiness') == 503 + results[mode]['bucket_outage_readiness'] = 503 + run('docker', 'start', mini) + wait_for(lambda: status('/readiness') == 200) + request('/logstream/smoke', 'DELETE') + if mode.startswith('postgres'): + database = 'tls' if mode == 'postgres-tls' else 'parseable' + remaining = run('docker', 'exec', pg, 'psql', '-U', 'postgres', '-d', database, '-Atc', "SELECT count(*) FROM parseable_documents WHERE starts_with(key,'smoke/');") + assert remaining == '0', remaining + results[mode]['deleted_stream_rows'] = 0 + assert not any(k.startswith('smoke/') for k in list_keys(env['P_S3_URL'], bucket)) + request('/logstream/smoke', 'PUT') + aux_request('/ingest', 'POST', [{'message': 'recreated', 'value': 7}], {'X-P-Stream': 'smoke'}) + wait_for(uploaded, 240) + wait_for(lambda: scalar(request('/query', 'POST', query, timeout=90), 'count', 1), 240) + recreated_query = dict(query, query='select sum(value) as value from smoke') + wait_for(lambda: scalar(request('/query', 'POST', recreated_query, timeout=90), 'value', 7), 240) + results[mode]['delete_recreate'] = True + print(mode+': lifecycle and outages passed', flush=True) + final_keys = list_keys(env['P_S3_URL'], bucket) + results[mode]['final_keys'] = final_keys + if mode.startswith('postgres'): + assert [k for k in final_keys if not k.endswith('.parquet')] == ['.parseable/postgres-metastore-id'] + if mode.startswith('postgres'): + restart_faults = [] + for fault in ['before-data', 'after-data']: + aux_request('/ingest', 'POST', [{'value': 7}], {'X-P-Stream': 'smoke'}) + wait_for(uploaded, 240) + sql = lambda text: run('docker', 'exec', pg, 'psql', '-v', 'ON_ERROR_STOP=1', '-U', 'postgres', '-d', database, '-Atc', text) + if fault == 'before-data': + sql("INSERT INTO parseable_documents(tenant,key,kind,body) VALUES('','smoke/.stream/.deleting','document',''::bytea);") + else: + sql("CREATE FUNCTION smoke_delete_fault() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'injected row deletion failure'; END $$; CREATE TRIGGER smoke_delete_fault BEFORE DELETE ON parseable_documents FOR EACH ROW EXECUTE FUNCTION smoke_delete_fault();") + assert status('/logstream/smoke', 'DELETE') >= 400 + sql('DROP TRIGGER smoke_delete_fault ON parseable_documents; DROP FUNCTION smoke_delete_fault();') + stop() + start() + wait_for(ready) + listing = json.loads(request('/logstream')) + assert all(item.get('name') != 'smoke' for item in listing), listing + assert status('/query', 'POST', query) >= 400, 'cold loader admitted deleting stream' + request('/logstream/smoke', 'DELETE') + assert sql("SELECT count(*) FROM parseable_documents WHERE starts_with(key,'smoke/');") == '0' + assert not any(k.startswith('smoke/') for k in list_keys(env['P_S3_URL'], bucket)) + request('/logstream/smoke', 'PUT') + request('/logstream/smoke', 'DELETE') + restart_faults.append(dict(step=fault, hidden=True, cold_load_rejected=True, http_retry=True, recreation=True)) + request('/logstream/smoke', 'PUT') + results[mode]['interrupted_deletion_restart'] = restart_faults + stop() + auxiliary.terminate() + try: + auxiliary.wait(timeout=15) + except sp.TimeoutExpired: + auxiliary.kill() + auxiliary.wait() + auxiliary = None + log.close() + failures = {} + # The final loop environment is PostgreSQL mode. Assert startup rejects a + # future schema and a pinned UUID mismatch before any HTTP listener starts. + for case in ['future-schema', 'wrong-id', 'unreachable']: + rejected_env = env.copy() + if case == 'future-schema': + run('docker', 'exec', pg, 'psql', '-U', 'postgres', '-d', database, '-c', + 'UPDATE parseable_backend SET schema_version=3;') + elif case == 'wrong-id': + rejected_env['P_METASTORE_POSTGRES_ID'] = 'wrong-backend' + else: + rejected_env['P_METASTORE_POSTGRES_URL'] = f'postgresql://postgres:smoke@127.0.0.1:{port()}/parseable?sslmode=disable' + attempt = startup_attempt(rejected_env, 20) + commands.append(dict(argv=attempt.args, exit_code=attempt.returncode)) + assert attempt.returncode != 0, (case, attempt.stdout) + assert 'PostgreSQL' in attempt.stdout, (case, attempt.stdout) + failures[case] = attempt.stdout + if case == 'future-schema': + run('docker', 'exec', pg, 'psql', '-U', 'postgres', '-d', database, '-c', + 'UPDATE parseable_backend SET schema_version=2;') + results['startup_rejections'] = failures + results['postgres_rows'] = run('docker','exec',pg,'psql','-U','postgres','-d',database,'-c', + 'SELECT tenant,kind,key,version,octet_length(body) AS bytes FROM parseable_documents ORDER BY tenant,key;') + (OUT/'results.json').write_text(json.dumps(results,indent=2)) + print(json.dumps(results,indent=2)) +finally: + cleanup() + atexit.unregister(cleanup) diff --git a/src/alerts/target.rs b/src/alerts/target.rs index 3cc13286d..17b688744 100644 --- a/src/alerts/target.rs +++ b/src/alerts/target.rs @@ -208,6 +208,7 @@ pub struct Target { pub target: TargetType, #[serde(default = "Ulid::new")] pub id: Ulid, + #[serde(rename = "tenantId")] pub tenant: Option, } @@ -523,6 +524,7 @@ pub struct TargetVerifier { pub notification_config: Option, #[serde(default = "Ulid::new")] pub id: Ulid, + #[serde(alias = "tenant")] pub tenant_id: Option, } diff --git a/src/banner.rs b/src/banner.rs index 53cf9dfac..f022a0c77 100644 --- a/src/banner.rs +++ b/src/banner.rs @@ -109,11 +109,17 @@ async fn storage_info(config: &Parseable) { ); } + let probe = if config.options.postgres.enabled() { + "PostgreSQL + data bucket probe" + } else { + "latency" + }; eprintln!( "\ - {:8}Store: \"{}\", (latency: {:?})", + {:8}Store: \"{}\", ({}: {:?})", "", storage.get_endpoint(), + probe, latency ); } diff --git a/src/cli.rs b/src/cli.rs index 417a2a11a..ca6c939c7 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -137,6 +137,11 @@ pub struct GcsStoreArgs { #[derive(Parser, Debug, Default)] pub struct Options { + #[command(flatten)] + pub migration: crate::metastore::metastores::migration::MigrationOptions, + #[command(flatten)] + pub postgres: crate::metastore::metastores::postgres_metastore::PostgresOptions, + // Authentication #[arg(long, env = "P_USERNAME", help = "Admin username to be set for this Parseable server", default_value = DEFAULT_USERNAME)] pub username: String, diff --git a/src/handlers/http/ingest.rs b/src/handlers/http/ingest.rs index 31b75d819..cd7f297c2 100644 --- a/src/handlers/http/ingest.rs +++ b/src/handlers/http/ingest.rs @@ -582,15 +582,16 @@ impl actix_web::ResponseError for PostError { JsonFlattenError(e) if e.is_limit_exceeded() => StatusCode::BAD_REQUEST, - Event(_) - | CreateStream(_) - | CustomError(_) - | NetworkError(_) - | ObjectStorageError(_) - | DashboardError(_) - | FiltersError(_) - | StreamError(_) - | JsonFlattenError(_) => StatusCode::INTERNAL_SERVER_ERROR, + ObjectStorageError(e) => e.http_status(), + CreateStream(CreateStreamError::Storage { err, .. }) => err.http_status(), + StreamError(super::logstream::error::StreamError::Storage( + e @ (crate::storage::ObjectStorageError::MetastoreUnavailable(_) + | crate::storage::ObjectStorageError::StreamDeleting), + )) => e.http_status(), + Event(_) | CreateStream(_) | CustomError(_) | NetworkError(_) | DashboardError(_) + | FiltersError(_) | StreamError(_) | JsonFlattenError(_) => { + StatusCode::INTERNAL_SERVER_ERROR + } StreamNotFound(_) => StatusCode::NOT_FOUND, diff --git a/src/handlers/http/logstream.rs b/src/handlers/http/logstream.rs index 5b4e42987..7105f572f 100644 --- a/src/handlers/http/logstream.rs +++ b/src/handlers/http/logstream.rs @@ -59,9 +59,17 @@ pub async fn delete( let stream_name = logstream.into_inner(); // Error out if stream doesn't exist in memory, or in the case of query node, in storage as well let tenant_id = get_tenant_id_from_request(&req)?; - if !PARSEABLE - .check_or_load_stream(&stream_name, &tenant_id) - .await + // A crashed PostgreSQL deletion is deliberately absent from stream loading, + // but an explicit DELETE must still be able to finish it after restart. + let pending_delete = PARSEABLE.options.postgres.enabled() + && PARSEABLE + .metastore + .is_stream_deleting(&stream_name, &tenant_id) + .await?; + if !pending_delete + && !PARSEABLE + .check_or_load_stream(&stream_name, &tenant_id) + .await { return Err(StreamNotFound(stream_name).into()); } @@ -847,8 +855,8 @@ pub mod error { StreamError::CreateStream(CreateStreamError::StreamNameValidation(_)) => { StatusCode::BAD_REQUEST } - StreamError::CreateStream(CreateStreamError::Storage { .. }) => { - StatusCode::INTERNAL_SERVER_ERROR + StreamError::CreateStream(CreateStreamError::Storage { err, .. }) => { + err.http_status() } StreamError::CreateStream(CreateStreamError::Custom { .. }) => { StatusCode::INTERNAL_SERVER_ERROR @@ -860,7 +868,7 @@ pub mod error { StreamError::TenantNotFoundError(_) => StatusCode::NOT_FOUND, StreamError::Custom { status, .. } => *status, StreamError::UninitializedLogstream => StatusCode::METHOD_NOT_ALLOWED, - StreamError::Storage(_) => StatusCode::INTERNAL_SERVER_ERROR, + StreamError::Storage(e) => e.http_status(), StreamError::NoAlertsSet => StatusCode::NOT_FOUND, StreamError::BadAlertJson { .. } => StatusCode::BAD_REQUEST, StreamError::AlertValidation(_) => StatusCode::BAD_REQUEST, diff --git a/src/handlers/http/modal/query/querier_logstream.rs b/src/handlers/http/modal/query/querier_logstream.rs index 2655c9254..f51e8e671 100644 --- a/src/handlers/http/modal/query/querier_logstream.rs +++ b/src/handlers/http/modal/query/querier_logstream.rs @@ -69,7 +69,15 @@ pub async fn delete( // if the stream not found in memory map, //check if it exists in the storage //create stream and schema from storage - if !PARSEABLE.streams.contains(&stream_name, &tenant_id) + // A crashed PostgreSQL deletion is deliberately absent from stream loading, + // but an explicit DELETE must still be able to finish it after restart. + let pending_delete = PARSEABLE.options.postgres.enabled() + && PARSEABLE + .metastore + .is_stream_deleting(&stream_name, &tenant_id) + .await?; + if !pending_delete + && !PARSEABLE.streams.contains(&stream_name, &tenant_id) && !PARSEABLE .create_stream_and_schema_from_storage(&stream_name, &tenant_id) .await diff --git a/src/handlers/http/oidc.rs b/src/handlers/http/oidc.rs index 316e6b027..155168189 100644 --- a/src/handlers/http/oidc.rs +++ b/src/handlers/http/oidc.rs @@ -860,7 +860,7 @@ pub enum OIDCError { impl actix_web::ResponseError for OIDCError { fn status_code(&self) -> StatusCode { match self { - Self::ObjectStorageError(_) => StatusCode::INTERNAL_SERVER_ERROR, + Self::ObjectStorageError(e) => e.http_status(), Self::Serde(_) => StatusCode::INTERNAL_SERVER_ERROR, Self::BadRequest(_) => StatusCode::BAD_REQUEST, Self::Unauthorized => StatusCode::UNAUTHORIZED, diff --git a/src/handlers/http/query.rs b/src/handlers/http/query.rs index f23582fd1..2dd694c47 100644 --- a/src/handlers/http/query.rs +++ b/src/handlers/http/query.rs @@ -814,6 +814,7 @@ impl actix_web::ResponseError for QueryError { QueryError::JsonParse(_) => StatusCode::INTERNAL_SERVER_ERROR, QueryError::Execute(e) => e.status_code(), QueryError::MetastoreError(e) => e.status_code(), + QueryError::ObjectStorage(e) => e.http_status(), QueryError::Header(_) => StatusCode::BAD_REQUEST, QueryError::HeaderBuild(e) => actix_web::ResponseError::status_code(e), _ => StatusCode::BAD_REQUEST, diff --git a/src/handlers/http/rbac.rs b/src/handlers/http/rbac.rs index 1c107288a..d5b64be92 100644 --- a/src/handlers/http/rbac.rs +++ b/src/handlers/http/rbac.rs @@ -649,7 +649,7 @@ impl actix_web::ResponseError for RBACError { Self::UserDoesNotExist => StatusCode::NOT_FOUND, Self::SerdeError(_) => StatusCode::BAD_REQUEST, Self::ValidationError(_) => StatusCode::BAD_REQUEST, - Self::ObjectStorageError(_) => StatusCode::INTERNAL_SERVER_ERROR, + Self::ObjectStorageError(e) => e.http_status(), Self::Network(_) => StatusCode::BAD_GATEWAY, Self::Anyhow(_) => StatusCode::INTERNAL_SERVER_ERROR, Self::RoleValidationError => StatusCode::BAD_REQUEST, diff --git a/src/handlers/http/role.rs b/src/handlers/http/role.rs index 832b350d2..da5635f6b 100644 --- a/src/handlers/http/role.rs +++ b/src/handlers/http/role.rs @@ -338,7 +338,7 @@ impl actix_web::ResponseError for RoleError { fn status_code(&self) -> StatusCode { match self { Self::UnknownDefaultRole | Self::DefaultRoleInUse => StatusCode::BAD_REQUEST, - Self::ObjectStorageError(_) => StatusCode::INTERNAL_SERVER_ERROR, + Self::ObjectStorageError(e) => e.http_status(), Self::RoleInUse => StatusCode::BAD_REQUEST, Self::SuperAdminPrivilege => StatusCode::BAD_REQUEST, Self::ProtectedRole => StatusCode::BAD_REQUEST, diff --git a/src/handlers/http/users/dashboards.rs b/src/handlers/http/users/dashboards.rs index 225e01347..468cf5594 100644 --- a/src/handlers/http/users/dashboards.rs +++ b/src/handlers/http/users/dashboards.rs @@ -494,7 +494,8 @@ impl DashboardError { impl actix_web::ResponseError for DashboardError { fn status_code(&self) -> StatusCode { match self { - Self::ObjectStorage(_) | Self::Custom(_) => StatusCode::INTERNAL_SERVER_ERROR, + Self::ObjectStorage(e) => e.http_status(), + Self::Custom(_) => StatusCode::INTERNAL_SERVER_ERROR, Self::Serde(_) | Self::Metadata(_) | Self::InvalidQueryParameter | Self::Header(_) => { StatusCode::BAD_REQUEST } diff --git a/src/handlers/http/users/filters.rs b/src/handlers/http/users/filters.rs index a27438aba..af6b58a52 100644 --- a/src/handlers/http/users/filters.rs +++ b/src/handlers/http/users/filters.rs @@ -236,7 +236,7 @@ pub enum FiltersError { impl actix_web::ResponseError for FiltersError { fn status_code(&self) -> StatusCode { match self { - Self::ObjectStorage(_) => StatusCode::INTERNAL_SERVER_ERROR, + Self::ObjectStorage(e) => e.http_status(), Self::Serde(_) => StatusCode::BAD_REQUEST, Self::Metadata(_) => StatusCode::BAD_REQUEST, Self::UserDoesNotExist(_) => StatusCode::NOT_FOUND, diff --git a/src/main.rs b/src/main.rs index 9e42c4bb6..271f92357 100644 --- a/src/main.rs +++ b/src/main.rs @@ -58,6 +58,31 @@ async fn main() -> anyhow::Result<()> { warn!("Failed to install rustls crypto provider: {:?}", e); } + // Offline commands bypass server initialization and interactive configuration. + if std::env::args() + .any(|a| a == "--metadata-migration" || a.starts_with("--metadata-migration=")) + { + use clap::Parser; + use parseable::cli::{Cli, StorageOptions}; + use parseable::storage::ObjectStorageProvider; + parseable::interactive::load_env_file(); + let cli = Cli::parse(); + let (options, provider): (_, &dyn ObjectStorageProvider) = match &cli.storage { + StorageOptions::Local(a) => (&a.options, &a.storage), + StorageOptions::S3(a) => (&a.options, &a.storage), + StorageOptions::Blob(a) => (&a.options, &a.storage), + StorageOptions::Gcs(a) => (&a.options, &a.storage), + }; + if options.migration.metadata_migration.is_some() { + return parseable::metastore::metastores::postgres_metastore::run_migration( + provider, options, + ) + .await; + } + } + + PARSEABLE.metastore.initiate_connection().await?; + // these are empty ptrs so mem footprint should be minimal let server: Box = match &PARSEABLE.options.mode { Mode::Query => Box::new(QueryServer), diff --git a/src/metastore/metastore_traits.rs b/src/metastore/metastore_traits.rs index fa484a51b..128bbce26 100644 --- a/src/metastore/metastore_traits.rs +++ b/src/metastore/metastore_traits.rs @@ -49,6 +49,22 @@ pub use crate::storage::object_storage::{ /// need different methods for different kinds of metadata. #[async_trait] pub trait Metastore: std::fmt::Debug + Send + Sync { + /// Optional SQL batch; the default performs no requests. + async fn manifest_batch( + &self, + _paths: &[String], + _tenant: &Option, + ) -> Result>>, MetastoreError> { + Ok(None) + } + /// SQL-only partition inventory for retention recovery. + async fn metadata_dates( + &self, + _stream: &str, + _tenant: &Option, + ) -> Result, MetastoreError> { + Ok(Vec::new()) + } async fn initiate_connection(&self) -> Result<(), MetastoreError>; async fn get_objects( &self, diff --git a/src/metastore/metastores/document_metastore.rs b/src/metastore/metastores/document_metastore.rs new file mode 100644 index 000000000..92e092c77 --- /dev/null +++ b/src/metastore/metastores/document_metastore.rs @@ -0,0 +1,1670 @@ +macro_rules! scoped_delimiter { + (true, $s:ident, $p:expr, $t:expr) => { + $s.storage.list_with_delimiter($p).await + }; + (false, $s:ident, $p:expr, $t:expr) => { + $s.storage.list_with_delimiter($p, $t).await + }; +} +macro_rules! postgres_list_streams { + (true, $s:ident, $t:expr, $body:block) => { + $body + }; + (false, $s:ident, $t:expr, $body:block) => { + Ok($s.storage.list_streams($t).await?) + }; +} +// Shared path, encoding and behavioral contract for document metastores. +macro_rules! document_metastore { +($name:ident, $storage:ty, $local_listing:tt, {$($lifecycle:tt)*}) => { +/* + * Parseable Server (C) 2022 - 2025 Parseable, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +use std::{ + collections::{BTreeMap, HashMap, HashSet}, + sync::Arc, +}; + +use actix_web::http::StatusCode; +use arrow_schema::Schema; +use bytes::Bytes; +use chrono::{DateTime, Utc}; +use dashmap::DashMap; +use futures::{StreamExt, TryStreamExt}; +use relative_path::{RelativePath, RelativePathBuf}; +use tonic::async_trait; +use tracing::{info, warn}; +use ulid::Ulid; + +use crate::{ + alerts::{ + alert_structs::{AlertStateEntry, MTTRHistory}, + outbound_http_policy::AlertTargetPolicyConfig, + target::Target, + }, + catalog::{ + manifest::{Manifest, decode_manifest_blocking, encode_manifest}, + partition_path, + }, + handlers::http::{ + modal::{Metadata, NodeMetadata, NodeType}, + users::USERS_ROOT_DIR, + }, + metastore::{ + MetastoreError, + metastore_traits::{ + ConditionalMetadataError, MetadataVersion, MetadataWriteCondition, Metastore, + MetastoreObject, VersionedMetadata, + }, + }, + option::Mode, + parseable::{DEFAULT_TENANT, PARSEABLE}, + storage::{ + ALERTS_ROOT_DIRECTORY, ObjectStorageError, PARSEABLE_ROOT_DIRECTORY, + SETTINGS_ROOT_DIRECTORY, STREAM_METADATA_FILE_NAME, STREAM_ROOT_DIRECTORY, + TARGETS_ROOT_DIRECTORY, + object_storage::{ + alert_json_path, alert_state_json_path, filter_path, manifest_path, mttr_json_path, + outbound_http_policy_json_path, parseable_json_path, schema_path, stream_json_path, + to_bytes, + }, + }, + users::filters::{Filter, FilterType, SQL_FILTER_STREAM_NAME, migrate_v1_v2}, +}; + +/// Using PARSEABLE's storage as a metastore (default) +#[derive(Debug)] +pub struct $name { + pub storage: Arc<$storage>, +} + +impl $name { + async fn delete_filter_path( + &self, + path: &RelativePathBuf, + tenant_id: &Option, + ) -> Result<(), MetastoreError> { + match self.storage.delete_object(path, tenant_id).await { + Ok(()) => Ok(()), + Err(error) if is_missing_optional_dir(&error) => Ok(()), + Err(error) => Err(error.into()), + } + } + + /// Enumerate only this owner's ID, never a filter belonging to another user. + async fn filter_aliases( + &self, + filter: &Filter, + tenant_id: &Option, + ) -> Result, MetastoreError> { + let owner = filter + .user_id + .as_deref() + .expect("stored filters have an owner"); + let id = filter + .filter_id + .as_deref() + .expect("stored filters have an ID"); + let root = filter_path(owner, "", "", tenant_id); + let mut copies = Vec::new(); + for stream in self.storage.list_dirs_relative(&root, tenant_id).await? { + let path = root.join(&stream).join(format!("{id}.json")); + let bytes = match self.storage.get_object(&path, tenant_id).await { + Ok(bytes) => bytes, + Err(error) if is_missing_optional_dir(&error) => continue, + Err(error) => return Err(error.into()), + }; + let copy: Filter = serde_json::from_slice(&bytes)?; + if copy.filter_id == filter.filter_id && copy.user_id == filter.user_id { + copies.push((path, copy)); + } + } + copies.sort_by(|a, b| a.0.cmp(&b.0)); + Ok(copies) + } + + async fn load_tenant_filters( + &self, + tenant_id: &Option, + ) -> Result, MetastoreError> { + let users_dir = + RelativePathBuf::from_iter([tenant_id.as_deref().unwrap_or(""), USERS_ROOT_DIR]); + let mut grouped: BTreeMap<(String, String), Vec<(RelativePathBuf, Filter)>> = + BTreeMap::new(); + for user in self + .storage + .list_dirs_relative(&users_dir, tenant_id) + .await? + { + let root = users_dir.join(&user).join("filters"); + for stream in self.storage.list_dirs_relative(&root, tenant_id).await? { + let directory = root.join(&stream); + let objects = self + .storage + .get_objects( + Some(&directory), + Box::new(|name| name.ends_with(".json")), + tenant_id, + ) + .await?; + for bytes in objects { + let mut value: serde_json::Value = serde_json::from_slice(&bytes)?; + let is_v1 = value.get("version").and_then(|v| v.as_str()) == Some("v1"); + if is_v1 { + value = migrate_v1_v2(value, tenant_id); + } + let Ok(mut filter) = serde_json::from_value::(value) else { + continue; + }; + // The discovered tenant directory is authoritative, including + // default-tenant records whose payload predates tenant metadata. + filter.tenant_id = tenant_id.clone().filter(|tenant| !tenant.is_empty()); + let (Some(owner), Some(id)) = (&filter.user_id, &filter.filter_id) else { + continue; + }; + // Stored identity fields must remain single path components. + if !safe_filter_component(owner) + || !safe_filter_component(id) + || !safe_filter_component(&filter.stream_name) + || filter.stream_name != stream + || (!is_v1 && owner != &user) + { + warn!( + "Ignoring saved filter whose identity does not match its storage directory" + ); + continue; + } + let source = directory.join(format!("{id}.json")); + // get_objects returns bytes without filenames. Verify the inferred + // filename before any cleanup; never delete an unrelated object. + match self.storage.get_object(&source, tenant_id).await { + Ok(stored) if stored == bytes => {} + Ok(_) => continue, + Err(error) if is_missing_optional_dir(&error) => continue, + Err(error) => return Err(error.into()), + } + if is_v1 { + // Persist before removing the original so interrupted migration is safe. + self.put_filter(&filter, tenant_id).await?; + if source.as_str() != filter.get_object_path() { + self.delete_filter_path(&source, tenant_id).await?; + } + } + grouped + .entry((owner.clone(), id.clone())) + .or_default() + .push(( + if is_v1 { + RelativePathBuf::from(filter.get_object_path()) + } else { + source + }, + filter, + )); + } + } + } + let mut filters = Vec::new(); + for (_, mut copies) in grouped { + // A v1 migration may write a hashed-owner directory that is visited + // later in this same scan. It is still one physical object. + copies.sort_by(|a, b| a.0.cmp(&b.0)); + copies.dedup_by(|a, b| a.0 == b.0); + copies.sort_by(|a, b| { + let canonical = |filter: &Filter| { + filter.query.filter_type == FilterType::SQL + && filter.stream_name == SQL_FILTER_STREAM_NAME + }; + canonical(&b.1).cmp(&canonical(&a.1)).then(a.0.cmp(&b.0)) + }); + let (canonical_path, filter) = &copies[0]; + if filter.query.filter_type == FilterType::SQL + && filter.stream_name == SQL_FILTER_STREAM_NAME + { + for (path, alias) in copies.iter().skip(1) { + if path != canonical_path && alias.query.filter_type == FilterType::SQL { + // The canonical object is already durable. Retain it in memory + // even if cleanup is temporarily unavailable; retry on restart. + if let Err(error) = self.delete_filter_path(path, tenant_id).await { + warn!("Failed to clean saved SQL filter alias {path}: {error}"); + } + } else if alias.query.filter_type != FilterType::SQL { + warn!( + "Retaining non-SQL saved filter copy {path} with the same owner {:?} and ID {:?} as canonical SQL filter {canonical_path}", + filter.user_id, filter.filter_id + ); + } + } + } + if copies.len() > 1 + && !(filter.query.filter_type == FilterType::SQL + && filter.stream_name == SQL_FILTER_STREAM_NAME) + { + warn!( + "Multiple saved filter copies for owner {:?}, ID {:?}; using deterministic path {canonical_path}", + filter.user_id, filter.filter_id + ); + } + filters.push(filter.clone()); + } + Ok(filters) + } +} + +fn stored_filter_path(filter: &Filter, tenant_id: &Option) -> RelativePathBuf { + filter_path( + filter + .user_id + .as_deref() + .expect("stored filters have an owner"), + &filter.stream_name, + &format!( + "{}.json", + filter + .filter_id + .as_deref() + .expect("stored filters have an ID") + ), + tenant_id, + ) +} + +fn safe_filter_component(component: &str) -> bool { + !component.is_empty() + && component != "." + && component != ".." + && !component.contains(['/', '\\']) +} + +pub fn is_missing_optional_dir(err: &ObjectStorageError) -> bool { + match err { + ObjectStorageError::NoSuchKey(_) => true, + ObjectStorageError::IoError(err) => err.kind() == std::io::ErrorKind::NotFound, + _ => false, + } +} + +#[async_trait] +impl Metastore for $name { + /// Since Parseable already starts with a connection to an object store, no need to implement this + async fn initiate_connection(&self) -> Result<(), MetastoreError> { + if $local_listing { + self.storage.check_migration_guard().await?; + Ok(()) + } else { Ok(self.storage.check(&None).await?) } + } + + /// Fetch mutiple .json objects + async fn get_objects( + &self, + parent_path: &str, + tenant_id: &Option, + ) -> Result, MetastoreError> { + Ok(self + .storage + .get_objects( + Some(&RelativePathBuf::from(parent_path)), + Box::new(|file_name| file_name.ends_with(".json")), + tenant_id, + ) + .await?) + } + + /// This function fetches all the keystones from the underlying object store + async fn get_keystones(&self) -> Result>, MetastoreError> { + let base_paths = PARSEABLE.list_tenants().unwrap_or_else(|| vec!["".into()]); + let mut keystones = HashMap::new(); + for mut tenant in base_paths { + let keystone_path = RelativePathBuf::from_iter([&tenant, ".keystone"]); + let objs = self + .storage + .get_objects( + Some(&keystone_path), + Box::new(|file_name| { + file_name.ends_with(".json") && !file_name.starts_with("conv_") + }), + &Some(tenant.clone()), + ) + .await?; + if tenant.is_empty() { + tenant.clone_from(&DEFAULT_TENANT.to_string()); + } + keystones.insert(tenant, objs); + } + + Ok(keystones) + } + + /// This function puts a keystone in the object store at the given path + async fn put_keystone( + &self, + obj: &dyn MetastoreObject, + tenant_id: &Option, + ) -> Result<(), MetastoreError> { + let id = obj.get_object_id(); + let path = if let Some(tenant_id) = tenant_id.as_ref() { + RelativePathBuf::from_iter([tenant_id, ".keystone", &format!("{id}.json")]) + } else { + RelativePathBuf::from_iter([".keystone", &format!("{id}.json")]) + }; + Ok(self + .storage + .put_object(&path, to_bytes(obj), tenant_id) + .await?) + } + + /// Delete a keystone + async fn delete_keystone( + &self, + obj: &dyn MetastoreObject, + tenant_id: &Option, + ) -> Result<(), MetastoreError> { + let id = obj.get_object_id(); + let path = if let Some(tenant_id) = tenant_id.as_ref() { + RelativePathBuf::from_iter([tenant_id, ".keystone", &format!("{id}.json")]) + } else { + RelativePathBuf::from_iter([".keystone", &format!("{id}.json")]) + }; + Ok(self.storage.delete_object(&path, tenant_id).await?) + } + + /// This function fetches all the conversations from the underlying object store + async fn get_conversations(&self) -> Result>, MetastoreError> { + let base_paths = PARSEABLE.list_tenants().unwrap_or_else(|| vec!["".into()]); + let mut conversations = HashMap::new(); + for mut tenant in base_paths { + let conv_path = RelativePathBuf::from_iter([&tenant, ".keystone"]); + let objs = self + .storage + .get_objects( + Some(&conv_path), + Box::new(|file_name| { + file_name.ends_with(".json") && file_name.starts_with("conv_") + }), + &Some(tenant.clone()), + ) + .await?; + if tenant.is_empty() { + tenant.clone_from(&DEFAULT_TENANT.to_string()); + } + conversations.insert(tenant, objs); + } + + Ok(conversations) + } + + /// This function puts a conversation in the object store at the given path + async fn put_conversation( + &self, + obj: &dyn MetastoreObject, + tenant_id: &Option, + ) -> Result<(), MetastoreError> { + let id = obj.get_object_id(); + let path = if let Some(tenant_id) = tenant_id.as_ref() { + RelativePathBuf::from_iter([tenant_id, ".keystone", &format!("conv_{id}.json")]) + } else { + RelativePathBuf::from_iter([".keystone", &format!("conv_{id}.json")]) + }; + Ok(self + .storage + .put_object(&path, to_bytes(obj), tenant_id) + .await?) + } + + /// Delete a conversation + async fn delete_conversation( + &self, + obj: &dyn MetastoreObject, + tenant_id: &Option, + ) -> Result<(), MetastoreError> { + let id = obj.get_object_id(); + let path = if let Some(tenant_id) = tenant_id.as_ref() { + RelativePathBuf::from_iter([tenant_id, ".keystone", &format!("conv_{id}.json")]) + } else { + RelativePathBuf::from_iter([".keystone", &format!("conv_{id}.json")]) + }; + Ok(self.storage.delete_object(&path, tenant_id).await?) + } + + /// This function fetches all the alerts from the underlying object store + async fn get_alerts(&self) -> Result>, MetastoreError> { + let base_paths = PARSEABLE.list_tenants().unwrap_or_else(|| vec!["".into()]); + let mut all_alerts = HashMap::new(); + for mut tenant in base_paths { + let alerts_path = RelativePathBuf::from_iter([&tenant, ALERTS_ROOT_DIRECTORY]); + let alerts = match self + .storage + .get_objects( + Some(&alerts_path), + Box::new(|file_name| { + !file_name.starts_with("alert_state_") && file_name != "mttr.json" && file_name.ends_with(".json") + }), + &Some(tenant.clone()), + ) + .await + { + Ok(alerts) => alerts, + Err(err) if is_missing_optional_dir(&err) => Vec::new(), + Err(err) => return Err(MetastoreError::ObjectStorageError(err)), + }; + if tenant.is_empty() { + tenant.clone_from(&DEFAULT_TENANT.to_string()); + } + all_alerts.insert(tenant, alerts); + } + Ok(all_alerts) + } + + /// This function puts an alert in the object store at the given path + async fn put_alert( + &self, + obj: &dyn MetastoreObject, + tenant_id: &Option, + ) -> Result<(), MetastoreError> { + let id = Ulid::from_string(&obj.get_object_id()).map_err(|e| MetastoreError::Error { + status_code: StatusCode::BAD_REQUEST, + message: e.to_string(), + flow: "put_alert".into(), + })?; + let path = alert_json_path(id, tenant_id); + + Ok(self + .storage + .put_object(&path, to_bytes(obj), tenant_id) + .await?) + } + + /// Delete an alert + async fn delete_alert( + &self, + obj: &dyn MetastoreObject, + tenant_id: &Option, + ) -> Result<(), MetastoreError> { + let id = Ulid::from_string(&obj.get_object_id()).map_err(|e| MetastoreError::Error { + status_code: StatusCode::BAD_REQUEST, + message: e.to_string(), + flow: "delete_alert".into(), + })?; + let path = alert_json_path(id, tenant_id); + match self.storage.delete_object(&path, tenant_id).await { + Ok(()) => Ok(()), + Err(err) if is_missing_optional_dir(&err) => Ok(()), + Err(err) => Err(MetastoreError::ObjectStorageError(err)), + } + } + + /// alerts state + async fn get_alert_states( + &self, + tenant_id: &Option, + ) -> Result, MetastoreError> { + let tenant = tenant_id.as_deref().unwrap_or(DEFAULT_TENANT); + let base_path = RelativePathBuf::from_iter([tenant, ALERTS_ROOT_DIRECTORY]); + let alert_state_bytes = self + .storage + .get_objects( + Some(&base_path), + Box::new(|file_name| { + file_name.starts_with("alert_state_") && file_name.ends_with(".json") + }), + tenant_id, + ) + .await?; + + let mut alert_states = Vec::new(); + for bytes in alert_state_bytes { + if let Ok(entry) = serde_json::from_slice::(&bytes) { + alert_states.push(entry); + } + } + + Ok(alert_states) + } + + async fn get_alert_state_entry( + &self, + alert_id: &Ulid, + tenant_id: &Option, + ) -> Result, MetastoreError> { + let path = alert_state_json_path(*alert_id, tenant_id); + match self.storage.get_object(&path, tenant_id).await { + Ok(bytes) => { + if let Ok(entry) = serde_json::from_slice::(&bytes) { + Ok(Some(entry)) + } else { + Ok(None) + } + } + Err(ObjectStorageError::NoSuchKey(_)) => Ok(None), + Err(e) => Err(MetastoreError::ObjectStorageError(e)), + } + } + + async fn put_alert_state( + &self, + obj: &dyn MetastoreObject, + tenant_id: &Option, + ) -> Result<(), MetastoreError> { + let id = Ulid::from_string(&obj.get_object_id()).map_err(|e| MetastoreError::Error { + status_code: StatusCode::BAD_REQUEST, + message: e.to_string(), + flow: "put_alert_state".into(), + })?; + let path = alert_state_json_path(id, tenant_id); + + // Parse the new state entry from the MetastoreObject + let new_state_entry: AlertStateEntry = serde_json::from_slice(&to_bytes(obj))?; + let new_state = new_state_entry + .current_state() + .ok_or_else(|| MetastoreError::InvalidJsonStructure { + expected: "AlertStateEntry with at least one state".to_string(), + found: "AlertStateEntry with empty states".to_string(), + })? + .state; + + // Try to read and parse existing file + if let Ok(existing_bytes) = self.storage.get_object(&path, tenant_id).await { + // File exists - try to parse and update + if let Ok(mut existing_entry) = + serde_json::from_slice::(&existing_bytes) + { + // Update the state and only save if it actually changed + let state_changed = existing_entry.update_state(new_state); + + if state_changed { + let updated_bytes = serde_json::to_vec(&existing_entry) + .map_err(MetastoreError::JsonParseError)?; + + self.storage + .put_object(&path, updated_bytes.into(), tenant_id) + .await?; + } + return Ok(()); + } + } + + // Create and save new entry (either file didn't exist or parsing failed) + let new_entry = AlertStateEntry::new(id, new_state, tenant_id.clone()); + let new_bytes = serde_json::to_vec(&new_entry).map_err(MetastoreError::JsonParseError)?; + + self.storage + .put_object(&path, new_bytes.into(), tenant_id) + .await?; + + Ok(()) + } + + /// Delete an alert state file + async fn delete_alert_state( + &self, + obj: &dyn MetastoreObject, + tenant_id: &Option, + ) -> Result<(), MetastoreError> { + let id = Ulid::from_string(&obj.get_object_id()).map_err(|e| MetastoreError::Error { + status_code: StatusCode::BAD_REQUEST, + message: e.to_string(), + flow: "delete_alert_state".into(), + })?; + let path = alert_state_json_path(id, tenant_id); + match self.storage.delete_object(&path, tenant_id).await { + Ok(()) => Ok(()), + Err(err) if is_missing_optional_dir(&err) => Ok(()), + Err(err) => Err(MetastoreError::ObjectStorageError(err)), + } + } + + /// Get MTTR history from storage + async fn get_mttr_history( + &self, + tenant_id: &Option, + ) -> Result, MetastoreError> { + let path = mttr_json_path(tenant_id); + match self.storage.get_object(&path, tenant_id).await { + Ok(bytes) => { + if let Ok(mut history) = serde_json::from_slice::(&bytes) { + if let Some(tenant) = tenant_id.as_ref() { + history.tenant_id = Some(tenant.clone()); + } + Ok(Some(history)) + } else { + Ok(None) + } + } + Err(ObjectStorageError::NoSuchKey(_)) => Ok(None), + Err(e) => Err(MetastoreError::ObjectStorageError(e)), + } + } + + /// Put MTTR history to storage + async fn put_mttr_history( + &self, + obj: &dyn MetastoreObject, + tenant_id: &Option, + ) -> Result<(), MetastoreError> { + let path = RelativePathBuf::from(obj.get_object_path()); + Ok(self + .storage + .put_object(&path, to_bytes(obj), tenant_id) + .await?) + } + + /// This function fetches all the llmconfigs from the underlying object store + async fn get_llmconfigs(&self) -> Result>, MetastoreError> { + let base_paths = PARSEABLE.list_tenants().unwrap_or_else(|| vec!["".into()]); + let mut all_configs = HashMap::new(); + for mut tenant in base_paths { + let base_path = + RelativePathBuf::from_iter([&tenant, SETTINGS_ROOT_DIRECTORY, "llmconfigs"]); + let conf_bytes = self + .storage + .get_objects( + Some(&base_path), + Box::new(|file_name| file_name.ends_with(".json")), + &Some(tenant.clone()), + ) + .await?; + if tenant.is_empty() { + tenant.clone_from(&DEFAULT_TENANT.to_string()); + } + all_configs.insert(tenant, conf_bytes); + } + Ok(all_configs) + } + + /// This function puts an llmconfig in the object store at the given path + async fn put_llmconfig( + &self, + obj: &dyn MetastoreObject, + tenant_id: &Option, + ) -> Result<(), MetastoreError> { + let path = obj.get_object_path(); + + Ok(self + .storage + .put_object(&RelativePathBuf::from(path), to_bytes(obj), tenant_id) + .await?) + } + + /// Delete an llmconfig + async fn delete_llmconfig( + &self, + obj: &dyn MetastoreObject, + tenant_id: &Option, + ) -> Result<(), MetastoreError> { + let path = obj.get_object_path(); + Ok(self + .storage + .delete_object(&RelativePathBuf::from(path), tenant_id) + .await?) + } + + async fn get_dashboard_versioned( + &self, + path: &str, + tenant_id: &Option, + ) -> Result, ConditionalMetadataError> { + self.storage + .get_metadata_versioned(RelativePath::new(path), tenant_id) + .await + } + + async fn put_dashboard_conditional( + &self, + path: &str, + bytes: Bytes, + condition: MetadataWriteCondition, + tenant_id: &Option, + ) -> Result { + self.storage + .put_metadata_conditional(RelativePath::new(path), bytes, condition, tenant_id) + .await + } + + async fn delete_dashboard_conditional( + &self, + path: &str, + version: MetadataVersion, + tenant_id: &Option, + ) -> Result<(), ConditionalMetadataError> { + self.storage + .delete_metadata_conditional(RelativePath::new(path), version, tenant_id) + .await + } + + /// Fetch all dashboards; hidden service indexes are not dashboard owners. + async fn get_dashboards(&self) -> Result>, MetastoreError> { + let mut dashboards = HashMap::new(); + let tenants = PARSEABLE.list_tenants().unwrap_or_else(|| vec!["".into()]); + for tenant in tenants { + let tenant_id = if tenant.is_empty() { + None + } else { + Some(tenant.clone()) + }; + let bytes = self.get_tenant_dashboards(&tenant_id).await?; + dashboards.insert( + tenant_id.unwrap_or_else(|| DEFAULT_TENANT.to_string()), + bytes, + ); + } + Ok(dashboards) + } + + async fn get_tenant_dashboards( + &self, + tenant_id: &Option, + ) -> Result, MetastoreError> { + let users = + RelativePathBuf::from_iter([tenant_id.as_deref().unwrap_or(""), USERS_ROOT_DIR]); + let mut dashboards = Vec::new(); + for owner in self.storage.list_dirs_relative(&users, tenant_id).await? { + if owner.starts_with('.') { + continue; + } + let path = users.join(owner).join("dashboards"); + match self + .storage + .get_objects( + Some(&path), + Box::new(|name| name.ends_with(".json")), + tenant_id, + ) + .await + { + Ok(bytes) => dashboards.extend(bytes), + Err(error) if is_missing_optional_dir(&error) => {} + Err(error) => return Err(error.into()), + } + } + Ok(dashboards) + } + + /// Save a dashboard + async fn put_dashboard( + &self, + obj: &dyn MetastoreObject, + tenant_id: &Option, + ) -> Result<(), MetastoreError> { + // we need the path to store in obj store + let path = obj.get_object_path(); + + Ok(self + .storage + .put_object(&RelativePathBuf::from(path), to_bytes(obj), tenant_id) + .await?) + } + + /// Delete a dashboard + async fn delete_dashboard( + &self, + obj: &dyn MetastoreObject, + tenant_id: &Option, + ) -> Result<(), MetastoreError> { + let path = obj.get_object_path(); + Ok(self + .storage + .delete_object(&RelativePathBuf::from(path), tenant_id) + .await?) + } + + /// Fetch all chats + async fn get_chats( + &self, + ) -> Result>>, MetastoreError> { + let base_paths = PARSEABLE.list_tenants().unwrap_or_else(|| vec!["".into()]); + let all_chats = DashMap::new(); + for tenant in base_paths { + let all_user_chats = DashMap::new(); + let tenant_id = if tenant.is_empty() { + None + } else { + Some(tenant.clone()) + }; + let users_dir = RelativePathBuf::from_iter([&tenant, USERS_ROOT_DIR]); + for user in self + .storage + .list_dirs_relative(&users_dir, &tenant_id) + .await? + { + if user.starts_with(".") { + continue; + } + let mut chats = Vec::new(); + let chats_path = users_dir.join(&user).join("chats"); + let user_chats = self + .storage + .get_objects( + Some(&chats_path), + Box::new(|file_name| file_name.ends_with(".json")), + &tenant_id, + ) + .await?; + for chat in user_chats { + chats.push(chat); + } + + all_user_chats.insert(user, chats); + } + all_chats.insert( + tenant_id.as_deref().unwrap_or(DEFAULT_TENANT).into(), + all_user_chats, + ); + } + Ok(all_chats) + } + + /// Save a chat + async fn put_chat( + &self, + obj: &dyn MetastoreObject, + tenant_id: &Option, + ) -> Result<(), MetastoreError> { + // we need the path to store in obj store + let path = obj.get_object_path(); + + Ok(self + .storage + .put_object(&RelativePathBuf::from(path), to_bytes(obj), tenant_id) + .await?) + } + + /// Delete a chat + async fn delete_chat( + &self, + obj: &dyn MetastoreObject, + tenant_id: &Option, + ) -> Result<(), MetastoreError> { + let path = obj.get_object_path(); + Ok(self + .storage + .delete_object(&RelativePathBuf::from(path), tenant_id) + .await?) + } + + // Reconcile duplicate SQL paths before exposing filters to the cache. + async fn get_filters(&self) -> Result>, MetastoreError> { + let mut all = HashMap::new(); + let tenants = PARSEABLE.list_tenants().unwrap_or_else(|| vec!["".into()]); + for tenant in tenants { + let tenant_id = Some(tenant.clone()); + let filters = self.load_tenant_filters(&tenant_id).await?; + all.insert( + if tenant.is_empty() { + DEFAULT_TENANT.to_owned() + } else { + tenant + }, + filters, + ); + } + Ok(all) + } + + /// Save a filter + async fn put_filter( + &self, + obj: &dyn MetastoreObject, + tenant_id: &Option, + ) -> Result<(), MetastoreError> { + // we need the path to store in obj store + let path = obj.get_object_path(); + + Ok(self + .storage + .put_object(&RelativePathBuf::from(path), to_bytes(obj), tenant_id) + .await?) + } + + async fn reconcile_filter( + &self, + filter: &Filter, + tenant_id: &Option, + ) -> Result<(), MetastoreError> { + let canonical = stored_filter_path(filter, tenant_id); + for (path, _) in self.filter_aliases(filter, tenant_id).await? { + if path != canonical { + self.delete_filter_path(&path, tenant_id).await?; + } + } + Ok(()) + } + + /// Delete aliases first. If any cleanup fails, the canonical copy remains + /// available for a retry instead of allowing an old alias to reappear. + async fn delete_filter( + &self, + filter: &Filter, + tenant_id: &Option, + ) -> Result<(), MetastoreError> { + self.reconcile_filter(filter, tenant_id).await?; + self.delete_filter_path(&stored_filter_path(filter, tenant_id), tenant_id) + .await + } + + /// Fetch an `ObjectStoreFormat` file + /// + /// If `get_base` is true, get the one at the base of the stream directory else depends on Mode + async fn get_stream_json( + &self, + stream_name: &str, + get_base: bool, + tenant_id: &Option, + _is_migration: bool, + ) -> Result { + let tenant = tenant_id.as_deref().unwrap_or(""); + let path = if get_base { + RelativePathBuf::from_iter([ + tenant, + stream_name, + STREAM_ROOT_DIRECTORY, + STREAM_METADATA_FILE_NAME, + ]) + } else { + stream_json_path(stream_name, tenant_id) + }; + Ok(self.storage.get_object(&path, tenant_id).await?) + } + + /// Fetch all `ObjectStoreFormat` present in a stream folder + async fn get_all_stream_jsons( + &self, + stream_name: &str, + mode: Option, + tenant_id: &Option, + _is_migration: bool, + ) -> Result, MetastoreError> { + let root = tenant_id.as_deref().unwrap_or(""); + let path = RelativePathBuf::from_iter([root, stream_name, STREAM_ROOT_DIRECTORY]); + if let Some(mode) = mode { + if mode.eq(&Mode::Ingest) { + Ok(self + .storage + .get_objects( + Some(&path), + Box::new(|file_name| { + file_name.starts_with(".ingestor") && file_name.ends_with("stream.json") + }), + tenant_id, + ) + .await?) + } else { + return Err(MetastoreError::Error { + status_code: StatusCode::BAD_REQUEST, + message: "Incorrect server mode passed as input. Only `Ingest` is allowed." + .into(), + flow: "get_all_streams with mode".into(), + }); + } + } else { + Ok(self + .storage + .get_objects( + Some(&path), + Box::new(|file_name| file_name.ends_with("stream.json")), + tenant_id, + ) + .await?) + } + } + + /// Save an `ObjectStoreFormat` file + async fn put_stream_json( + &self, + obj: &dyn MetastoreObject, + stream_name: &str, + tenant_id: &Option, + ) -> Result<(), MetastoreError> { + let path = stream_json_path(stream_name, tenant_id); + + Ok(self + .storage + .put_object(&path, to_bytes(obj), tenant_id) + .await?) + } + + /// Fetch manifest files for the given date keys in parallel. + /// + /// Per-date work runs concurrently via `futures::future::try_join_all` + /// so a slow LIST on one date does not block the others. Each date's + /// timing is logged at info level so any pathologically slow dates + /// (S3 retries, oversized prefixes) are visible without needing extra + /// span instrumentation. + async fn get_manifest_files_for_dates( + &self, + stream_name: &str, + tenant_id: &Option, + dates: &[String], + ) -> Result>, MetastoreError> { + let inventory_limit = Arc::new(tokio::sync::Semaphore::new(16)); + let date_futures = dates.iter().map(|date| { + let stream = stream_name.to_string(); + let tenant = tenant_id.clone(); + let date = date.clone(); + let inventory_limit = inventory_limit.clone(); + async move { + let t_start = std::time::Instant::now(); + let date_path = if let Some(tenant) = tenant.as_ref() { + object_store::path::Path::from(format!("{}/{}/{}", tenant, stream, date)) + } else { + object_store::path::Path::from(format!("{}/{}", stream, date)) + }; + + let t_list = std::time::Instant::now(); + let list_permit = inventory_limit + .clone() + .acquire_owned() + .await + .expect("inventory semaphore remains open"); + let list_result = scoped_delimiter!($local_listing, self, Some(date_path), &tenant); + drop(list_permit); + let resp = match list_result { + Ok(r) => r, + Err(ObjectStorageError::NoSuchKey(_)) => { + info!( + stream = %stream, + tenant = ?tenant, + date = %date, + list_ms = t_list.elapsed().as_millis() as u64, + "manifest fetch: date prefix not found" + ); + return Ok::<(String, Vec), MetastoreError>((date, Vec::new())); + } + Err(e) => return Err(e.into()), + }; + let list_ms = t_list.elapsed().as_millis() as u64; + + let manifest_paths: Vec = resp + .objects + .iter() + .filter(|name| name.location.filename().unwrap().ends_with("manifest.json")) + .map(|name| name.location.to_string()) + .collect(); + let manifest_count = manifest_paths.len(); + + let t_gets = std::time::Instant::now(); + let manifests = futures::stream::iter(manifest_paths) + .map(|path| { + let tenant = tenant.clone(); + let inventory_limit = inventory_limit.clone(); + async move { + let _permit = inventory_limit + .acquire_owned() + .await + .expect("inventory semaphore remains open"); + let bytes = self + .storage + .get_object(&RelativePathBuf::from(path), &tenant) + .await?; + Ok::(decode_manifest_blocking(bytes).await?) + } + }) + .buffer_unordered(16) + .try_collect::>() + .await?; + let gets_ms = t_gets.elapsed().as_millis() as u64; + let total_ms = t_start.elapsed().as_millis() as u64; + + if total_ms > 100 { + info!( + stream = %stream, + tenant = ?tenant, + date = %date, + list_ms, + gets_ms, + total_ms, + manifest_count, + "manifest fetch per-date timing" + ); + } + Ok((date, manifests)) + } + }); + + let results: Vec<(String, Vec)> = + futures::future::try_join_all(date_futures).await?; + + let mut result_file_list: BTreeMap> = BTreeMap::new(); + for (date, manifests) in results { + if !manifests.is_empty() { + result_file_list.insert(date, manifests); + } + } + + Ok(result_file_list) + } + + /// Fetch a specific `Manifest` file + async fn get_manifest( + &self, + stream_name: &str, + lower_bound: DateTime, + upper_bound: DateTime, + manifest_url: Option, + tenant_id: &Option, + ) -> Result, MetastoreError> { + let path = match manifest_url { + Some(url) => RelativePathBuf::from(url), + None => { + let path = partition_path(stream_name, lower_bound, upper_bound, tenant_id); + manifest_path(path.as_str()) + } + }; + match self.storage.get_object(&path, tenant_id).await { + Ok(bytes) => { + let manifest = decode_manifest_blocking(bytes).await?; + Ok(Some(manifest)) + } + Err(ObjectStorageError::NoSuchKey(_)) => Ok(None), + Err(err) => Err(MetastoreError::ObjectStorageError(err)), + } + } + + /// Get the path for a specific `Manifest` file + async fn get_manifest_path( + &self, + stream_name: &str, + lower_bound: DateTime, + upper_bound: DateTime, + tenant_id: &Option, + ) -> Result { + let path = partition_path(stream_name, lower_bound, upper_bound, tenant_id); + Ok(self + .storage + .absolute_url(&manifest_path(path.as_str())) + .to_string()) + } + + async fn put_manifest( + &self, + manifest: &Manifest, + stream_name: &str, + lower_bound: DateTime, + upper_bound: DateTime, + tenant_id: &Option, + ) -> Result<(), MetastoreError> { + let manifest_file_name = manifest_path("").to_string(); + let path = partition_path(stream_name, lower_bound, upper_bound, tenant_id) + .join(&manifest_file_name); + + Ok(self + .storage + .put_object(&path, encode_manifest(manifest)?, tenant_id) + .await?) + } + + async fn delete_manifest( + &self, + stream_name: &str, + lower_bound: DateTime, + upper_bound: DateTime, + tenant_id: &Option, + ) -> Result<(), MetastoreError> { + let manifest_file_name = manifest_path("").to_string(); + let path = partition_path(stream_name, lower_bound, upper_bound, tenant_id) + .join(&manifest_file_name); + Ok(self.storage.delete_object(&path, tenant_id).await?) + } + + /// targets + async fn get_targets(&self) -> Result>, MetastoreError> { + let base_paths = PARSEABLE.list_tenants().unwrap_or_else(|| vec!["".into()]); + let mut all_targets = HashMap::new(); + for mut tenant in base_paths { + let targets_path = RelativePathBuf::from_iter([ + &tenant, + SETTINGS_ROOT_DIRECTORY, + TARGETS_ROOT_DIRECTORY, + ]); + let target_bytes = match self + .storage + .get_objects( + Some(&targets_path), + Box::new(|file_name| file_name.ends_with(".json")), + &Some(tenant.clone()), + ) + .await + { + Ok(targets) => targets, + Err(err) if is_missing_optional_dir(&err) => Vec::new(), + Err(err) => return Err(MetastoreError::ObjectStorageError(err)), + }; + let targets = target_bytes + .iter() + .filter_map(|bytes| { + serde_json::from_slice(bytes) + .inspect_err(|err| warn!("Expected compatible json, error = {err}")) + .ok() + }) + .collect(); + if tenant.is_empty() { + tenant.clone_from(&DEFAULT_TENANT.to_string()); + } + all_targets.insert(tenant, targets); + } + Ok(all_targets) + } + + async fn put_target( + &self, + obj: &dyn MetastoreObject, + tenant_id: &Option, + ) -> Result<(), MetastoreError> { + // we need the path to store in obj store + let path = obj.get_object_path(); + + Ok(self + .storage + .put_object(&RelativePathBuf::from(path), to_bytes(obj), tenant_id) + .await?) + } + + async fn delete_target( + &self, + obj: &dyn MetastoreObject, + tenant_id: &Option, + ) -> Result<(), MetastoreError> { + // we need the path to store in obj store + let path = obj.get_object_path(); + + Ok(self + .storage + .delete_object(&RelativePathBuf::from(path), tenant_id) + .await?) + } + + async fn get_outbound_policies( + &self, + ) -> Result, MetastoreError> { + let mut all_policies = HashMap::new(); + + let base_paths = PARSEABLE.list_tenants().unwrap_or_else(|| vec!["".into()]); + + for tenant in base_paths { + let tenant_id = if tenant.is_empty() { + None + } else { + Some(tenant.clone()) + }; + let path = outbound_http_policy_json_path(&tenant_id); + let policy = match self.storage.get_object(&path, &tenant_id).await { + Ok(bytes) => match serde_json::from_slice::(&bytes) { + Ok(policy) => policy, + Err(err) => { + warn!( + "Failed to deserialize outbound HTTP policy for tenant {tenant}: {err}" + ); + continue; + } + }, + Err(err) if is_missing_optional_dir(&err) => continue, + Err(err) => return Err(MetastoreError::ObjectStorageError(err)), + }; + + all_policies.insert(tenant, policy); + } + + Ok(all_policies) + } + + async fn put_outbound_policy( + &self, + tenant_id: &Option, + policy: &AlertTargetPolicyConfig, + ) -> Result<(), MetastoreError> { + let path = outbound_http_policy_json_path(tenant_id); + Ok(self + .storage + .put_object(&path, to_bytes(policy), tenant_id) + .await?) + } + + async fn get_all_schemas( + &self, + stream_name: &str, + tenant_id: &Option, + ) -> Result, MetastoreError> { + let path_prefix = if let Some(tenant) = tenant_id { + relative_path::RelativePathBuf::from(format!( + "{tenant}/{stream_name}/{STREAM_ROOT_DIRECTORY}" + )) + } else { + relative_path::RelativePathBuf::from(format!("{stream_name}/{STREAM_ROOT_DIRECTORY}")) + }; + Ok(self + .storage + .get_objects( + Some(&path_prefix), + Box::new(|file_name: String| file_name.contains(".schema")), + tenant_id, + ) + .await? + .iter() + // we should be able to unwrap as we know the data is valid schema + .map(|byte_obj| { + serde_json::from_slice(byte_obj) + .unwrap_or_else(|_| panic!("got an invalid schema for stream: {stream_name}")) + }) + .collect()) + } + + async fn refresh_schema( + &self, + stream_name: &str, + tenant_id: &Option, + schema_file: &str, + ) -> Result { + let objects = self.storage.schema_objects(stream_name, tenant_id).await?; + let mut schemas = Vec::with_capacity(objects.len()); + let mut canonical = None; + for (name, bytes) in objects { + let schema: Schema = serde_json::from_slice(&bytes) + .map_err(|e| ObjectStorageError::Custom(e.to_string()))?; + if name == schema_file { + canonical = Some(schema.clone()); + } + schemas.push(schema); + } + let merged = + Schema::try_merge(schemas).map_err(|e| ObjectStorageError::Custom(e.to_string()))?; + // Field order is immaterial, but nested types, nullability and field + // metadata all matter. Normalize only the top-level ordering. + let normalize = |schema: &Schema| { + let mut schema = schema.clone(); + let mut fields = schema.fields.to_vec(); + fields.sort_by(|a, b| a.name().cmp(b.name())); + schema.fields = fields.into(); + schema + }; + if canonical.as_ref().map(normalize) != Some(normalize(&merged)) { + let path = RelativePathBuf::from_iter([ + tenant_id.as_deref().unwrap_or(""), + stream_name, + STREAM_ROOT_DIRECTORY, + schema_file, + ]); + self.storage + .put_object(&path, to_bytes(&merged), tenant_id) + .await?; + } + Ok(merged) + } + + async fn get_schema( + &self, + stream_name: &str, + tenant_id: &Option, + ) -> Result { + Ok(self + .storage + .get_object(&schema_path(stream_name, tenant_id), tenant_id) + .await?) + } + + async fn put_schema( + &self, + obj: Schema, + stream_name: &str, + tenant_id: &Option, + ) -> Result<(), MetastoreError> { + let path = schema_path(stream_name, tenant_id); + Ok(self + .storage + .put_object(&path, to_bytes(&obj), tenant_id) + .await?) + } + + async fn get_parseable_metadata( + &self, + tenant_id: &Option, + ) -> Result, MetastoreError> { + let path = if let Some(tenant_id) = tenant_id.as_ref() { + RelativePathBuf::from_iter([tenant_id, ".parseable.json"]) + } else { + parseable_json_path() + }; + + let parseable_metadata: Option = + match self.storage.get_object(&path, tenant_id).await { + Ok(bytes) => Some(bytes), + Err(err) => { + if matches!(err, ObjectStorageError::NoSuchKey(_)) { + None + } else { + return Err(MetastoreError::ObjectStorageError(err)); + } + } + }; + + Ok(parseable_metadata) + } + + async fn delete_tenant(&self, tenant_id: &str) -> Result<(), MetastoreError> { + self.storage + .delete_prefix(&RelativePathBuf::from(tenant_id), &None) + .await + .map_err(MetastoreError::ObjectStorageError) + } + + async fn get_ingestor_metadata(&self) -> Result, MetastoreError> { + let base_path = RelativePathBuf::from(PARSEABLE_ROOT_DIRECTORY); + Ok(self + .storage + .get_objects( + Some(&base_path), + Box::new(|file_name| file_name.starts_with("ingestor")), + &None, + ) + .await?) + } + + async fn put_parseable_metadata( + &self, + obj: &dyn MetastoreObject, + tenant_id: &Option, + ) -> Result<(), MetastoreError> { + let path = if let Some(tenant_id) = tenant_id.as_ref() { + RelativePathBuf::from_iter([tenant_id, ".parseable.json"]) + } else { + parseable_json_path() + }; + + self.storage + .put_object(&path, to_bytes(obj), tenant_id) + .await + .map_err(MetastoreError::ObjectStorageError) + } + + async fn get_node_metadata( + &self, + node_type: NodeType, + tenant_id: &Option, + ) -> Result, MetastoreError> { + let root_path = RelativePathBuf::from(PARSEABLE_ROOT_DIRECTORY); + let prefix_owned = node_type.to_string(); + + let metadata = self + .storage + .get_objects( + Some(&root_path), + Box::new(move |file_name| file_name.starts_with(&prefix_owned)), // Use the owned copy + tenant_id, + ) + .await? + .into_iter() + .collect(); + + Ok(metadata) + } + + async fn put_node_metadata(&self, obj: &dyn MetastoreObject) -> Result<(), MetastoreError> { + let path = obj.get_object_path(); + self.storage + .put_object(&RelativePathBuf::from(path), to_bytes(obj), &None) + .await?; + Ok(()) + } + + async fn delete_node_metadata( + &self, + domain_name: &str, + node_type: NodeType, + ) -> Result { + let metadatas = self + .storage + .get_objects( + Some(&RelativePathBuf::from(PARSEABLE_ROOT_DIRECTORY)), + Box::new(move |file_name| file_name.starts_with(&node_type.to_string())), + &None, + ) + .await?; + + let node_metadatas = metadatas + .iter() + .filter_map(|elem| match serde_json::from_slice::(elem) { + Ok(meta) if meta.domain_name() == domain_name => Some(meta), + _ => None, + }) + .collect::>(); + + if node_metadatas.is_empty() { + return Ok(false); + } + + let node_meta_filename = node_metadatas[0].file_path().to_string(); + let file = RelativePathBuf::from(&node_meta_filename); + + match self.storage.delete_object(&file, &None).await { + Ok(_) => Ok(true), + Err(err) => { + if matches!(err, ObjectStorageError::IoError(_)) { + Ok(false) + } else { + Err(MetastoreError::ObjectStorageError(err)) + } + } + } + } + + $($lifecycle)* + + async fn list_streams( + &self, + tenant_id: &Option, + ) -> Result, MetastoreError> { + postgres_list_streams!($local_listing, self, tenant_id, { + // using LocalFS list_streams because it doesn't implement list_with_delimiter + if $local_listing && self.storage.store_url().scheme() == "file" { + self.storage.list_streams() + .await + .map_err(MetastoreError::ObjectStorageError) + } else { + // not local-disk, object storage + let mut result_file_list = HashSet::new(); + let root = tenant_id + .as_ref() + .map(|tenant| object_store::path::Path::from_iter([tenant.clone()])); + + let resp = scoped_delimiter!($local_listing, self, root.clone(), tenant_id)?; + + let streams = resp + .common_prefixes + .iter() + .flat_map(|path| path.parts()) + .map(|name| name.as_ref().to_string()) + .filter(|name| { + name != PARSEABLE_ROOT_DIRECTORY + && name != USERS_ROOT_DIR + && name != SETTINGS_ROOT_DIRECTORY + && name != ALERTS_ROOT_DIRECTORY + && name != crate::storage::TOMBSTONE_ROOT_DIRECTORY + }) + .collect::>(); + for stream in streams { + if !$local_listing && self.is_stream_deleting(&stream, tenant_id).await? { + continue; + } + let stream_path = if let Some(root) = root.as_ref() { + object_store::path::Path::from_iter([ + root.as_ref(), + &stream, + STREAM_ROOT_DIRECTORY, + ]) + } else { + object_store::path::Path::from(format!("{}/{}", stream, STREAM_ROOT_DIRECTORY)) + }; + let resp = scoped_delimiter!($local_listing, self, Some(stream_path), tenant_id)?; + if resp + .objects + .iter() + .any(|name| name.location.filename().unwrap().ends_with("stream.json")) + { + result_file_list.insert(stream); + } + } + Ok(result_file_list) + } + }) + } +} + + +}; +} +pub(crate) use document_metastore; + +/// Classification belongs to the shared document layer. Never reconstruct a key. +pub(crate) fn document_scope(key: &str) -> Option<(String, &'static str)> { + if key == super::migration::PAIR || key == super::migration::BUSY || key.ends_with(".lock") { + return None; + } + let parts: Vec<_> = key.split('/').collect(); + if key.ends_with("manifest.json") { + let date = parts.iter().position(|p| p.starts_with("date="))?; + return match date { + 1 => Some((String::new(), "manifest")), + 2 => Some((parts[0].to_owned(), "manifest")), + _ => None, + }; + } + let hidden = parts.iter().position(|p| p.starts_with('.'))?; + let tenant = match hidden { + 0 => "", + 1 if parts[1] == crate::storage::STREAM_ROOT_DIRECTORY + || parts[1].ends_with("stream.json") + || parts[1].contains(".schema") => + { + "" + } + 1 => parts[0], + 2 if parts[2] == crate::storage::STREAM_ROOT_DIRECTORY + || parts[2].ends_with("stream.json") + || parts[2].contains(".schema") => + { + parts[0] + } + _ => return None, + }; + Some(( + if tenant == crate::parseable::DEFAULT_TENANT { + String::new() + } else { + tenant.to_owned() + }, + super::postgres_documents::kind(relative_path::RelativePath::new(key)), + )) +} diff --git a/src/metastore/metastores/migration.rs b/src/metastore/metastores/migration.rs new file mode 100644 index 000000000..2cad412a5 --- /dev/null +++ b/src/metastore/metastores/migration.rs @@ -0,0 +1,685 @@ +//! Offline document transfer. Bodies are processed one at a time; inventories hold keys only. +use super::postgres_documents::PostgresDocuments; +use crate::storage::{ObjectStorage, ObjectStorageError}; +use anyhow::{Context, ensure}; +use bytes::Bytes; +use relative_path::RelativePath; +use sha2::{Digest, Sha256}; +use std::{collections::BTreeMap, sync::Arc}; + +pub(crate) type InventoryDocument = ( + String, + (String, &'static str), + chrono::DateTime, +); + +pub const PAIR: &str = ".parseable/postgres-metastore-id"; +pub const BUSY: &str = ".parseable/metastore-migration"; + +fn migration_help_requested() -> bool { + std::env::args().any(|a| a == "--metadata-migration" || a.starts_with("--metadata-migration=")) +} + +#[derive(clap::Args, Debug, Default)] +pub struct MigrationOptions { + /// Offline operation; requires the server and all other writers to be stopped. + #[arg(long, value_enum)] + pub metadata_migration: Option, + #[arg(long, hide = !migration_help_requested(), requires = "metadata_migration")] + pub acknowledge_server_stopped: bool, + #[arg( + long = "dry-run", + visible_alias = "migration-dry-run", + hide = !migration_help_requested(), requires = "metadata_migration" + )] + pub migration_dry_run: bool, + /// Retired broad override; use the explicit bucket-change override instead. + #[arg(long, hide = !migration_help_requested(), requires = "metadata_migration")] + pub migration_force: bool, + /// Explicitly authorize overwriting metadata changed by bucket writers since import. + #[arg(long, hide = !migration_help_requested(), requires = "metadata_migration")] + pub overwrite_bucket_changes_since_import: bool, + #[arg( + long, + hide = !migration_help_requested(), requires = "metadata_migration", + conflicts_with = "migration_dry_run" + )] + pub abort: bool, + /// Permit missing snapshot manifests during import, retaining the dangling reference. + #[arg(long, hide = !migration_help_requested(), requires = "metadata_migration")] + pub allow_dangling_snapshot_references: bool, + #[cfg(test)] + #[arg(skip)] + pub injected_failure: Option<&'static str>, +} +#[derive(clap::ValueEnum, Debug, Clone, Copy, PartialEq, Eq)] +pub enum Action { + Import, + Export, + Verify, +} + +pub async fn optional(store: &dyn ObjectStorage, key: &str) -> anyhow::Result> { + match store.get_object(RelativePath::new(key), &None).await { + Ok(b) => Ok(Some(b)), + Err(ObjectStorageError::NoSuchKey(_)) => Ok(None), + Err(ObjectStorageError::IoError(e)) if e.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(e) => Err(e.into()), + } +} +pub async fn guard(store: &dyn ObjectStorage, postgres: bool) -> anyhow::Result<()> { + ensure!( + optional(store, BUSY).await?.is_none(), + "offline metadata migration incomplete; stop all writers and rerun the migration command" + ); + if let Some(id) = optional(store, PAIR).await? + && !postgres + { + ensure!( + std::env::var("P_ALLOW_PAIRED_BUCKET").is_ok_and(|v| v.as_bytes() == id.as_ref()), + "bucket paired with PostgreSQL; object-store metadata is stale. Export first, or explicitly set P_ALLOW_PAIRED_BUCKET= for disaster recovery" + ); + } + Ok(()) +} + +pub fn digest(body: &[u8]) -> String { + format!("{:x}", Sha256::digest(body)) +} + +pub(crate) use super::document_metastore::document_scope; + +pub(crate) fn stream_references(body: &[u8]) -> anyhow::Result> { + let value: serde_json::Value = serde_json::from_slice(body)?; + if matches!(value["version"].as_str(), Some("v1" | "v2" | "v3")) { + let list = value + .pointer("/snapshot/manifest_list") + .and_then(|v| v.as_array()); + return list + .into_iter() + .flatten() + .map(|v| { + v["manifest_path"] + .as_str() + .map(str::to_owned) + .context("legacy snapshot has invalid manifest path") + }) + .collect(); + } + let config: crate::storage::ObjectStoreFormat = serde_json::from_value(value)?; + Ok(config + .snapshot + .manifest_list + .into_iter() + .map(|m| m.manifest_path) + .collect()) +} + +pub(crate) async fn validate( + store: &dyn ObjectStorage, + key: &str, + body: &Bytes, +) -> anyhow::Result<()> { + if key.ends_with("manifest.json") { + crate::catalog::manifest::decode_manifest(body)?; + } else if key.ends_with("stream.json") { + for manifest_path in stream_references(body).with_context(|| key.to_owned())? { + let bytes = store + .get_object(RelativePath::new(&manifest_path), &None) + .await + .with_context(|| format!("{key}: missing manifest {manifest_path}"))?; + crate::catalog::manifest::decode_manifest(&bytes)?; + } + } else if key.contains(".schema") { + serde_json::from_slice::(body).with_context(|| key.to_owned())?; + } + ensure!( + !key.ends_with(".deleting") && !key.contains(".tombstones/"), + "unfinished deletion: {key}" + ); + Ok(()) +} + +pub async fn run( + store: Arc, + db: &PostgresDocuments, + opts: &MigrationOptions, +) -> anyhow::Result<()> { + let action = opts.metadata_migration.context("missing operation")?; + ensure!( + !opts.migration_force, + "--migration-force is retired: normal rollback needs no flag; to overwrite metadata changed by bucket writers use --overwrite-bucket-changes-since-import after backup" + ); + ensure!( + opts.acknowledge_server_stopped, + "requires --acknowledge-server-stopped; stop ALL servers and writers first" + ); + eprintln!("OFFLINE migration: all servers must remain stopped. No writer fencing exists."); + db.migration_preflight(store.as_ref(), opts).await?; + if opts.abort { + return db.abort_migration(store.as_ref(), action).await; + } + let inventory = store.metadata_inventory().await?; + let mut skipped = 0; + for (key, _) in &inventory { + if document_scope(key).is_none() { + skipped += 1; + eprintln!("skipped key={key}"); + } + if key.starts_with(".parseable-metadata-locks/") && key.ends_with(".lock") { + continue; + } + for component in key.split('/').filter(|p| p.starts_with('.')) { + ensure!( + matches!( + component, + ".parseable" + | ".users" + | ".alerts" + | ".targets" + | ".settings" + | ".correlations" + | ".stream" + | ".dashboard-titles" + | ".dashboard-locators" + | ".dashboard-locations" + | ".parseable.json" + | ".schema" + | ".stream.json" + ) || component.starts_with(".ingestor.") + || component.starts_with(".querier."), + "unknown hidden metadata key: {key}" + ); + } + } + eprintln!("skipped keys: {skipped}"); + let keys: Vec<_> = inventory + .iter() + .filter_map(|(k, modified)| document_scope(k).map(|scope| (k.clone(), scope, *modified))) + .collect(); + let mut counts = BTreeMap::<(String, String), (usize, usize)>::new(); + for (key, (tenant, kind), modified) in &keys { + let body = store.get_object(RelativePath::new(key), &None).await?; + if action != Action::Export { + match validate(store.as_ref(), key, &body).await { + Err(e) + if opts.allow_dangling_snapshot_references + && e.to_string().contains("missing manifest") => + { + eprintln!("OVERRIDE dangling snapshot reference: {e:#}") + } + result => result?, + } + } + if chrono::Utc::now() + .signed_duration_since(*modified) + .num_seconds() + < 120 + { + eprintln!("WARNING recently modified metadata (possible active writer): {key}"); + } + let count = counts + .entry((tenant.clone(), kind.to_string())) + .or_default(); + count.0 += 1; + count.1 += body.len(); + println!( + "bucket tenant={tenant:?} kind={kind} key={key} bytes={} sha256={}", + body.len(), + digest(&body) + ); + } + eprintln!("counts by (tenant, kind): {counts:?}"); + // Dry-run never bootstraps SQL or writes a bucket marker. + if opts.migration_dry_run { + return Ok(()); + } + db.migrate(store, opts, &keys).await +} + +pub(crate) fn failure_point(opts: &MigrationOptions, step: &str) -> anyhow::Result<()> { + #[cfg(test)] + anyhow::ensure!( + opts.injected_failure != Some(step), + "injected failure: {step}" + ); + #[cfg(not(test))] + let _ = (opts, step); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{metastore::metastores::postgres_metastore::PostgresOptions, storage::LocalFS}; + + #[test] + fn inventory_scope_preserves_all_document_namespaces() { + for (key, tenant, kind) in [ + (".parseable.json", "", "document"), + ("logs/.stream.json", "", "stream"), + ("logs/.schema", "", "schema"), + ("tenant/logs/.stream.json", "tenant", "stream"), + ("tenant/logs/.schema", "tenant", "schema"), + ("tenant/.parseable.json", "tenant", "document"), + ("logs/.stream/.ingestor.node.stream.json", "", "stream"), + ( + "tenant/logs/.stream/.ingestor.node.schema", + "tenant", + "schema", + ), + ( + "tenant/logs/date=2026-01-01/ingestor.node.manifest.json", + "tenant", + "manifest", + ), + ( + "logs/date=2026-01-01/ingestor.node.manifest.json", + "", + "manifest", + ), + ( + "tenant/.users/owner/.dashboard-titles/key.json", + "tenant", + "document", + ), + (".parseable/ingestor.node.json", "", "document"), + ] { + assert_eq!( + document_scope(key), + Some((tenant.to_owned(), kind)), + "{key}" + ); + } + for key in [PAIR, BUSY, "logs/date=2026-01-01/data.parquet"] { + assert!(document_scope(key).is_none()); + } + } + + #[tokio::test] + async fn invalid_stream_and_missing_snapshot_manifest_fail_before_writes() { + let root = tempfile::tempdir().unwrap(); + let store = LocalFS::new(root.path().to_owned()); + assert!( + validate( + &store, + "logs/.stream/.stream.json", + &Bytes::from_static(b"{}") + ) + .await + .is_err() + ); + let mut config = crate::storage::ObjectStoreFormat::default(); + config + .snapshot + .manifest_list + .push(crate::catalog::snapshot::ManifestItem { + manifest_path: "logs/date=2026-01-01/ingestor.missing.manifest.json".into(), + time_lower_bound: chrono::Utc::now(), + time_upper_bound: chrono::Utc::now(), + events_ingested: 0, + ingestion_size: 0, + storage_size: 0, + }); + assert!( + validate( + &store, + "logs/.stream/.stream.json", + &Bytes::from(serde_json::to_vec(&config).unwrap()) + ) + .await + .is_err() + ); + assert!(optional(&store, BUSY).await.unwrap().is_none()); + } + + #[tokio::test] + async fn minio_metadata_inventory_encoding_and_scale() -> anyhow::Result<()> { + let Ok(endpoint) = std::env::var("P_TEST_REVIEW_S3_URL") else { + eprintln!("SKIP MinIO inventory: P_TEST_REVIEW_S3_URL unset"); + return Ok(()); + }; + if std::env::var_os("P_MINIO_INVENTORY_CHILD").is_none() { + let output=std::process::Command::new(std::env::current_exe()?).args(["--exact","metastore::metastores::migration::tests::minio_metadata_inventory_encoding_and_scale","--nocapture"]).env("P_MINIO_INVENTORY_CHILD","1").output()?; + anyhow::ensure!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stdout).to_string() + + &String::from_utf8_lossy(&output.stderr) + ); + return Ok(()); + } + use crate::storage::ObjectStorageProvider; + use clap::Parser; + #[derive(clap::Parser)] + struct Args { + #[command(flatten)] + storage: crate::storage::S3Config, + } + let provider = Args::parse_from([ + "test", + "--endpoint-url", + &endpoint, + "--region", + "us-east-1", + "--bucket-name", + "review", + "--access-key-id", + "minioadmin", + "--secret-key", + "minioadmin", + ]) + .storage; + let store = provider.construct_client(); + let date = format!("date=2026-01-01-{}", uuid::Uuid::new_v4().simple()); + let manifest_key = format!("café/{date}/manifest.json"); + for key in [ + "café/.stream/.schema", + ".users/auth0|é/filters/one.json", + manifest_key.as_str(), + ] { + store + .put_object(RelativePath::new(key), Bytes::from_static(b"{}"), &None) + .await?; + } + let counter = || { + crate::metrics::STORAGE_REQUEST_RESPONSE_TIME + .with_label_values(&["s3", "LIST_DELIM", "200"]) + .get_sample_count() + }; + let n = counter(); + let before = store.metadata_inventory().await?; + let requests = counter() - n; + for i in 0..100 { + store + .put_object( + RelativePath::new(&format!("café/{date}/hour=00/minute={i}/data.parquet")), + Bytes::new(), + &None, + ) + .await?; + } + let n = counter(); + let after = store.metadata_inventory().await?; + assert_eq!( + requests, + counter() - n, + "parquet files must not add inventory requests" + ); + assert_eq!(before, after); + for (key, _) in after { + store.get_object(RelativePath::new(&key), &None).await?; + } + Ok(()) + } + + #[tokio::test] + async fn migration_round_trip_postgres() -> anyhow::Result<()> { + let Ok(url) = std::env::var("P_TEST_POSTGRES_URL") else { + eprintln!("skip migration PostgreSQL test: P_TEST_POSTGRES_URL unset"); + return Ok(()); + }; + let root = tempfile::tempdir()?; + let store: Arc = + if let Ok(endpoint) = std::env::var("P_TEST_REVIEW_S3_URL") { + use crate::storage::ObjectStorageProvider; + use clap::Parser; + #[derive(clap::Parser)] + struct Args { + #[command(flatten)] + storage: crate::storage::S3Config, + } + Args::parse_from([ + "test", + "--endpoint-url", + &endpoint, + "--region", + "us-east-1", + "--bucket-name", + "migration-unit", + "--access-key-id", + "minioadmin", + "--secret-key", + "minioadmin", + ]) + .storage + .construct_client() + } else { + eprintln!("SKIP MinIO round trip: P_TEST_REVIEW_S3_URL unset; exercising LocalFS"); + Arc::new(LocalFS::new(root.path().to_owned())) + }; + let options = PostgresOptions { + metastore_postgres_url: Some(url.clone()), + ..Default::default() + }; + let admin = PostgresDocuments::new(&options, "test".into(), store.clone())?; + let schema = format!("migration_{}", uuid::Uuid::new_v4().simple()); + admin + .client() + .await? + .batch_execute(&format!("CREATE DATABASE {schema}")) + .await?; + let result = async { + let mut url = url::Url::parse(&url)?; + url.set_path(&format!("/{schema}")); + let opts = PostgresOptions { + metastore_postgres_url: Some(url.to_string()), + ..Default::default() + }; + let db = PostgresDocuments::new(&opts, "test".into(), store.clone())?; + let plain = Bytes::from(serde_json::to_vec( + &crate::catalog::manifest::Manifest::default(), + )?); + let compressed = crate::catalog::manifest::encode_manifest( + &crate::catalog::manifest::Manifest::default(), + )?; + let mut documents = vec![ + ( + ".parseable.json".to_owned(), + Bytes::from_static(b"{ \"users\":[], \"roles\":{} }\n"), + ), + ( + "café/date=2026-01-01/ingestor.one.manifest.json".into(), + plain, + ), + ( + "tenant/logs/date=2026-01-01/ingestor.two.manifest.json".into(), + compressed, + ), + ]; + let mut config = crate::storage::ObjectStoreFormat::default(); + config + .snapshot + .manifest_list + .push(crate::catalog::snapshot::ManifestItem { + manifest_path: if store.store_url().scheme() == "file" { + store + .absolute_url(RelativePath::new(&documents[1].0)) + .to_string() + } else { + documents[1].0.clone() + }, + time_lower_bound: chrono::Utc::now(), + time_upper_bound: chrono::Utc::now(), + events_ingested: 0, + ingestion_size: 0, + storage_size: 0, + }); + documents.push(( + "café/.stream/.ingestor.one.stream.json".into(), + Bytes::from(serde_json::to_vec(&config)?), + )); + for prefix in [ + ".users/auth0|é/dashboards", + ".users/owner/filters", + ".alerts", + ".targets", + ".settings", + ".parseable", + ".correlations", + ] { + for tenant in ["", "tenant/"] { + documents.push(( + format!("{tenant}{prefix}/fixture.json"), + Bytes::from_static(b"{\"unknown\":true} \n"), + )); + } + } + for (key, body) in &documents { + store + .put_object(RelativePath::new(key), body.clone(), &None) + .await?; + } + let mut command = MigrationOptions { + metadata_migration: Some(Action::Import), + acknowledge_server_stopped: true, + migration_dry_run: true, + migration_force: false, + ..Default::default() + }; + let pinned_options = PostgresOptions { + metastore_postgres_id: Some("wrong-id".into()), + ..opts.clone() + }; + let pinned = PostgresDocuments::new(&pinned_options, "test".into(), store.clone())?; + assert!( + run(store.clone(), &pinned, &command) + .await + .unwrap_err() + .to_string() + .contains("pinned backend") + ); + run(store.clone(), &db, &command).await?; + assert!(optional(store.as_ref(), PAIR).await?.is_none()); + command.migration_dry_run = false; + command.injected_failure = Some("marker"); + assert!(run(store.clone(), &db, &command).await.is_err()); + command.abort = true; + run(store.clone(), &db, &command).await?; + store.check_migration_guard().await?; + command.abort = false; + for step in [ + "marker", + "spool", + "import-before-commit", + "import-commit", + "paired", + ] { + command.injected_failure = Some(step); + assert!(run(store.clone(), &db, &command).await.is_err(), "{step}"); + } + command.injected_failure = None; + command.abort = true; + assert!( + run(store.clone(), &db, &command).await.is_err(), + "committed import cannot abort" + ); + command.abort = false; + run(store.clone(), &db, &command).await?; + assert!(store.check_migration_guard().await.is_err()); + assert!(run(store.clone(), &db, &command).await.is_err()); + command.metadata_migration = Some(Action::Verify); + run(store.clone(), &db, &command).await?; + // Simulate crash after committed import, before marker cleanup. + let id = optional(store.as_ref(), PAIR).await?.unwrap(); + store + .put_object( + RelativePath::new(BUSY), + Bytes::from(format!("Import:{}", std::str::from_utf8(&id)?)), + &None, + ) + .await?; + command.metadata_migration = Some(Action::Import); + run(store.clone(), &db, &command).await?; + // Invalid SQL documents must fail before a marker or destination write. + db.put_object( + RelativePath::new("broken/.stream/.deleting"), + Bytes::new(), + &None, + ) + .await?; + command.metadata_migration = Some(Action::Export); + assert!(run(store.clone(), &db, &command).await.is_err()); + assert!(optional(store.as_ref(), BUSY).await?.is_none()); + db.delete_object(RelativePath::new("broken/.stream/.deleting"), &None) + .await?; + // Bucket writers require explicit overwrite acknowledgement on export. + store + .put_object( + RelativePath::new(&documents[0].0), + Bytes::from_static(b"changed"), + &None, + ) + .await?; + command.metadata_migration = Some(Action::Export); + assert!(run(store.clone(), &db, &command).await.is_err()); + // Restore the imported bucket body; SQL edits must then export without force. + store + .put_object( + RelativePath::new(&documents[0].0), + documents[0].1.clone(), + &None, + ) + .await?; + documents[0].1 = Bytes::from_static(b"{\"users\":[],\"roles\":{},\"updated\":true}"); + db.put_object( + RelativePath::new(&documents[0].0), + documents[0].1.clone(), + &None, + ) + .await?; + let deleted = documents.pop().unwrap().0; + let deleted_tenant = document_scope(&deleted).unwrap().0; + db.delete_object(RelativePath::new(&deleted), &Some(deleted_tenant)) + .await?; + for step in [ + "marker", + "export-state", + "object-write", + "bucket-delete", + "pair-removed", + ] { + command.injected_failure = Some(step); + assert!(run(store.clone(), &db, &command).await.is_err(), "{step}"); + command.abort = true; + if step == "marker" { + run(store.clone(), &db, &command).await?; + let restarted = PostgresDocuments::new(&opts, "test".into(), store.clone())?; + restarted.check(&None).await?; + } else { + assert!( + run(store.clone(), &db, &command).await.is_err(), + "export after {step} cannot abort" + ); + } + command.abort = false; + } + command.injected_failure = None; + run(store.clone(), &db, &command).await?; + assert!(optional(store.as_ref(), PAIR).await?.is_none()); + store.check_migration_guard().await?; + assert!(optional(store.as_ref(), &deleted).await?.is_none()); + for (key, body) in documents { + assert_eq!( + store.get_object(RelativePath::new(&key), &None).await?, + body, + "byte-for-byte round trip: {key}" + ); + } + command.metadata_migration = Some(Action::Verify); + run(store.clone(), &db, &command).await?; + assert!( + db.check(&None).await.is_err(), + "exported database must not auto-pair" + ); + anyhow::Ok(()) + } + .await; + admin + .client() + .await? + .batch_execute(&format!("DROP DATABASE {schema} WITH (FORCE)")) + .await?; + result + } +} diff --git a/src/metastore/metastores/mod.rs b/src/metastore/metastores/mod.rs index 19663b9bb..ee711d152 100644 --- a/src/metastore/metastores/mod.rs +++ b/src/metastore/metastores/mod.rs @@ -16,4 +16,15 @@ * */ +#[macro_use] +mod document_metastore; pub mod object_store_metastore; +pub(crate) mod postgres_documents; +pub mod postgres_metastore; + +pub mod migration; + +mod postgres_client; + +#[cfg(test)] +mod review_fix_tests; diff --git a/src/metastore/metastores/object_store_metastore.rs b/src/metastore/metastores/object_store_metastore.rs index c9682de58..e4a1715c8 100644 --- a/src/metastore/metastores/object_store_metastore.rs +++ b/src/metastore/metastores/object_store_metastore.rs @@ -1,1663 +1,182 @@ -/* - * Parseable Server (C) 2022 - 2025 Parseable, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - */ - -use std::{ - collections::{BTreeMap, HashMap, HashSet}, - sync::Arc, -}; - -use actix_web::http::StatusCode; -use arrow_schema::Schema; -use bytes::Bytes; -use chrono::{DateTime, Utc}; -use dashmap::DashMap; -use futures::{StreamExt, TryStreamExt}; -use relative_path::{RelativePath, RelativePathBuf}; -use tonic::async_trait; -use tracing::{info, warn}; -use ulid::Ulid; - -use crate::{ - alerts::{ - alert_structs::{AlertStateEntry, MTTRHistory}, - outbound_http_policy::AlertTargetPolicyConfig, - target::Target, - }, - catalog::{ - manifest::{Manifest, decode_manifest_blocking, encode_manifest}, - partition_path, - }, - handlers::http::{ - modal::{Metadata, NodeMetadata, NodeType}, - users::USERS_ROOT_DIR, - }, - metastore::{ - MetastoreError, - metastore_traits::{ - ConditionalMetadataError, MetadataVersion, MetadataWriteCondition, Metastore, - MetastoreObject, VersionedMetadata, - }, - }, - option::Mode, - parseable::{DEFAULT_TENANT, PARSEABLE}, - storage::{ - ALERTS_ROOT_DIRECTORY, ObjectStorage, ObjectStorageError, PARSEABLE_ROOT_DIRECTORY, - SETTINGS_ROOT_DIRECTORY, STREAM_METADATA_FILE_NAME, STREAM_ROOT_DIRECTORY, - TARGETS_ROOT_DIRECTORY, TOMBSTONE_ROOT_DIRECTORY, - object_storage::{ - alert_json_path, alert_state_json_path, filter_path, manifest_path, mttr_json_path, - outbound_http_policy_json_path, parseable_json_path, schema_path, stream_json_path, - to_bytes, - }, - }, - users::filters::{Filter, FilterType, SQL_FILTER_STREAM_NAME, migrate_v1_v2}, -}; - -/// Using PARSEABLE's storage as a metastore (default) -#[derive(Debug)] -pub struct ObjectStoreMetastore { - pub storage: Arc, -} - -impl ObjectStoreMetastore { - async fn delete_filter_path( - &self, - path: &RelativePathBuf, - tenant_id: &Option, - ) -> Result<(), MetastoreError> { - match self.storage.delete_object(path, tenant_id).await { - Ok(()) => Ok(()), - Err(error) if is_missing_optional_dir(&error) => Ok(()), - Err(error) => Err(error.into()), - } - } - - /// Enumerate only this owner's ID, never a filter belonging to another user. - async fn filter_aliases( - &self, - filter: &Filter, - tenant_id: &Option, - ) -> Result, MetastoreError> { - let owner = filter - .user_id - .as_deref() - .expect("stored filters have an owner"); - let id = filter - .filter_id - .as_deref() - .expect("stored filters have an ID"); - let root = filter_path(owner, "", "", tenant_id); - let mut copies = Vec::new(); - for stream in self.storage.list_dirs_relative(&root, tenant_id).await? { - let path = root.join(&stream).join(format!("{id}.json")); - let bytes = match self.storage.get_object(&path, tenant_id).await { - Ok(bytes) => bytes, - Err(error) if is_missing_optional_dir(&error) => continue, - Err(error) => return Err(error.into()), - }; - let copy: Filter = serde_json::from_slice(&bytes)?; - if copy.filter_id == filter.filter_id && copy.user_id == filter.user_id { - copies.push((path, copy)); - } - } - copies.sort_by(|a, b| a.0.cmp(&b.0)); - Ok(copies) - } - - async fn load_tenant_filters( - &self, - tenant_id: &Option, - ) -> Result, MetastoreError> { - let users_dir = - RelativePathBuf::from_iter([tenant_id.as_deref().unwrap_or(""), USERS_ROOT_DIR]); - let mut grouped: BTreeMap<(String, String), Vec<(RelativePathBuf, Filter)>> = - BTreeMap::new(); - for user in self - .storage - .list_dirs_relative(&users_dir, tenant_id) - .await? - { - let root = users_dir.join(&user).join("filters"); - for stream in self.storage.list_dirs_relative(&root, tenant_id).await? { - let directory = root.join(&stream); - let objects = self - .storage - .get_objects( - Some(&directory), - Box::new(|name| name.ends_with(".json")), - tenant_id, - ) - .await?; - for bytes in objects { - let mut value: serde_json::Value = serde_json::from_slice(&bytes)?; - let is_v1 = value.get("version").and_then(|v| v.as_str()) == Some("v1"); - if is_v1 { - value = migrate_v1_v2(value, tenant_id); - } - let Ok(mut filter) = serde_json::from_value::(value) else { - continue; - }; - // The discovered tenant directory is authoritative, including - // default-tenant records whose payload predates tenant metadata. - filter.tenant_id = tenant_id.clone().filter(|tenant| !tenant.is_empty()); - let (Some(owner), Some(id)) = (&filter.user_id, &filter.filter_id) else { - continue; - }; - // Stored identity fields must remain single path components. - if !safe_filter_component(owner) - || !safe_filter_component(id) - || !safe_filter_component(&filter.stream_name) - || filter.stream_name != stream - || (!is_v1 && owner != &user) - { - warn!( - "Ignoring saved filter whose identity does not match its storage directory" - ); - continue; - } - let source = directory.join(format!("{id}.json")); - // get_objects returns bytes without filenames. Verify the inferred - // filename before any cleanup; never delete an unrelated object. - match self.storage.get_object(&source, tenant_id).await { - Ok(stored) if stored == bytes => {} - Ok(_) => continue, - Err(error) if is_missing_optional_dir(&error) => continue, - Err(error) => return Err(error.into()), - } - if is_v1 { - // Persist before removing the original so interrupted migration is safe. - self.put_filter(&filter, tenant_id).await?; - if source.as_str() != filter.get_object_path() { - self.delete_filter_path(&source, tenant_id).await?; - } - } - grouped - .entry((owner.clone(), id.clone())) - .or_default() - .push(( - if is_v1 { - RelativePathBuf::from(filter.get_object_path()) - } else { - source - }, - filter, - )); - } - } - } - let mut filters = Vec::new(); - for (_, mut copies) in grouped { - // A v1 migration may write a hashed-owner directory that is visited - // later in this same scan. It is still one physical object. - copies.sort_by(|a, b| a.0.cmp(&b.0)); - copies.dedup_by(|a, b| a.0 == b.0); - copies.sort_by(|a, b| { - let canonical = |filter: &Filter| { - filter.query.filter_type == FilterType::SQL - && filter.stream_name == SQL_FILTER_STREAM_NAME - }; - canonical(&b.1).cmp(&canonical(&a.1)).then(a.0.cmp(&b.0)) - }); - let (canonical_path, filter) = &copies[0]; - if filter.query.filter_type == FilterType::SQL - && filter.stream_name == SQL_FILTER_STREAM_NAME - { - for (path, alias) in copies.iter().skip(1) { - if path != canonical_path && alias.query.filter_type == FilterType::SQL { - // The canonical object is already durable. Retain it in memory - // even if cleanup is temporarily unavailable; retry on restart. - if let Err(error) = self.delete_filter_path(path, tenant_id).await { - warn!("Failed to clean saved SQL filter alias {path}: {error}"); - } - } else if alias.query.filter_type != FilterType::SQL { - warn!( - "Retaining non-SQL saved filter copy {path} with the same owner {:?} and ID {:?} as canonical SQL filter {canonical_path}", - filter.user_id, filter.filter_id - ); - } - } - } - if copies.len() > 1 - && !(filter.query.filter_type == FilterType::SQL - && filter.stream_name == SQL_FILTER_STREAM_NAME) - { - warn!( - "Multiple saved filter copies for owner {:?}, ID {:?}; using deterministic path {canonical_path}", - filter.user_id, filter.filter_id - ); - } - filters.push(filter.clone()); - } - Ok(filters) - } -} - -fn stored_filter_path(filter: &Filter, tenant_id: &Option) -> RelativePathBuf { - filter_path( - filter - .user_id - .as_deref() - .expect("stored filters have an owner"), - &filter.stream_name, - &format!( - "{}.json", - filter - .filter_id - .as_deref() - .expect("stored filters have an ID") - ), - tenant_id, - ) -} - -fn safe_filter_component(component: &str) -> bool { - !component.is_empty() - && component != "." - && component != ".." - && !component.contains(['/', '\\']) -} - -pub fn is_missing_optional_dir(err: &ObjectStorageError) -> bool { - match err { - ObjectStorageError::NoSuchKey(_) => true, - ObjectStorageError::IoError(err) => err.kind() == std::io::ErrorKind::NotFound, - _ => false, - } -} - -#[async_trait] -impl Metastore for ObjectStoreMetastore { - /// Since Parseable already starts with a connection to an object store, no need to implement this - async fn initiate_connection(&self) -> Result<(), MetastoreError> { - unimplemented!() - } - - /// Fetch mutiple .json objects - async fn get_objects( - &self, - parent_path: &str, - tenant_id: &Option, - ) -> Result, MetastoreError> { - Ok(self - .storage - .get_objects( - Some(&RelativePathBuf::from(parent_path)), - Box::new(|file_name| file_name.ends_with(".json")), - tenant_id, - ) - .await?) - } - - /// This function fetches all the keystones from the underlying object store - async fn get_keystones(&self) -> Result>, MetastoreError> { - let base_paths = PARSEABLE.list_tenants().unwrap_or_else(|| vec!["".into()]); - let mut keystones = HashMap::new(); - for mut tenant in base_paths { - let keystone_path = RelativePathBuf::from_iter([&tenant, ".keystone"]); - let objs = self - .storage - .get_objects( - Some(&keystone_path), - Box::new(|file_name| { - file_name.ends_with(".json") && !file_name.starts_with("conv_") - }), - &Some(tenant.clone()), - ) - .await?; - if tenant.is_empty() { - tenant.clone_from(&DEFAULT_TENANT.to_string()); - } - keystones.insert(tenant, objs); - } - - Ok(keystones) - } - - /// This function puts a keystone in the object store at the given path - async fn put_keystone( - &self, - obj: &dyn MetastoreObject, - tenant_id: &Option, - ) -> Result<(), MetastoreError> { - let id = obj.get_object_id(); - let path = if let Some(tenant_id) = tenant_id.as_ref() { - RelativePathBuf::from_iter([tenant_id, ".keystone", &format!("{id}.json")]) - } else { - RelativePathBuf::from_iter([".keystone", &format!("{id}.json")]) - }; - Ok(self - .storage - .put_object(&path, to_bytes(obj), tenant_id) - .await?) - } - - /// Delete a keystone - async fn delete_keystone( - &self, - obj: &dyn MetastoreObject, - tenant_id: &Option, - ) -> Result<(), MetastoreError> { - let id = obj.get_object_id(); - let path = if let Some(tenant_id) = tenant_id.as_ref() { - RelativePathBuf::from_iter([tenant_id, ".keystone", &format!("{id}.json")]) - } else { - RelativePathBuf::from_iter([".keystone", &format!("{id}.json")]) - }; - Ok(self.storage.delete_object(&path, tenant_id).await?) - } - - /// This function fetches all the conversations from the underlying object store - async fn get_conversations(&self) -> Result>, MetastoreError> { - let base_paths = PARSEABLE.list_tenants().unwrap_or_else(|| vec!["".into()]); - let mut conversations = HashMap::new(); - for mut tenant in base_paths { - let conv_path = RelativePathBuf::from_iter([&tenant, ".keystone"]); - let objs = self - .storage - .get_objects( - Some(&conv_path), - Box::new(|file_name| { - file_name.ends_with(".json") && file_name.starts_with("conv_") - }), - &Some(tenant.clone()), - ) - .await?; - if tenant.is_empty() { - tenant.clone_from(&DEFAULT_TENANT.to_string()); - } - conversations.insert(tenant, objs); - } - - Ok(conversations) - } - - /// This function puts a conversation in the object store at the given path - async fn put_conversation( - &self, - obj: &dyn MetastoreObject, - tenant_id: &Option, - ) -> Result<(), MetastoreError> { - let id = obj.get_object_id(); - let path = if let Some(tenant_id) = tenant_id.as_ref() { - RelativePathBuf::from_iter([tenant_id, ".keystone", &format!("conv_{id}.json")]) - } else { - RelativePathBuf::from_iter([".keystone", &format!("conv_{id}.json")]) - }; - Ok(self - .storage - .put_object(&path, to_bytes(obj), tenant_id) - .await?) - } - - /// Delete a conversation - async fn delete_conversation( - &self, - obj: &dyn MetastoreObject, - tenant_id: &Option, - ) -> Result<(), MetastoreError> { - let id = obj.get_object_id(); - let path = if let Some(tenant_id) = tenant_id.as_ref() { - RelativePathBuf::from_iter([tenant_id, ".keystone", &format!("conv_{id}.json")]) - } else { - RelativePathBuf::from_iter([".keystone", &format!("conv_{id}.json")]) - }; - Ok(self.storage.delete_object(&path, tenant_id).await?) - } - - /// This function fetches all the alerts from the underlying object store - async fn get_alerts(&self) -> Result>, MetastoreError> { - let base_paths = PARSEABLE.list_tenants().unwrap_or_else(|| vec!["".into()]); - let mut all_alerts = HashMap::new(); - for mut tenant in base_paths { - let alerts_path = RelativePathBuf::from_iter([&tenant, ALERTS_ROOT_DIRECTORY]); - let alerts = match self - .storage - .get_objects( - Some(&alerts_path), - Box::new(|file_name| { - !file_name.starts_with("alert_state_") && file_name.ends_with(".json") - }), - &Some(tenant.clone()), - ) - .await - { - Ok(alerts) => alerts, - Err(err) if is_missing_optional_dir(&err) => Vec::new(), - Err(err) => return Err(MetastoreError::ObjectStorageError(err)), - }; - if tenant.is_empty() { - tenant.clone_from(&DEFAULT_TENANT.to_string()); - } - all_alerts.insert(tenant, alerts); - } - Ok(all_alerts) - } - - /// This function puts an alert in the object store at the given path - async fn put_alert( - &self, - obj: &dyn MetastoreObject, - tenant_id: &Option, - ) -> Result<(), MetastoreError> { - let id = Ulid::from_string(&obj.get_object_id()).map_err(|e| MetastoreError::Error { - status_code: StatusCode::BAD_REQUEST, - message: e.to_string(), - flow: "put_alert".into(), - })?; - let path = alert_json_path(id, tenant_id); - - Ok(self - .storage - .put_object(&path, to_bytes(obj), tenant_id) - .await?) - } - - /// Delete an alert - async fn delete_alert( - &self, - obj: &dyn MetastoreObject, - tenant_id: &Option, - ) -> Result<(), MetastoreError> { - let id = Ulid::from_string(&obj.get_object_id()).map_err(|e| MetastoreError::Error { - status_code: StatusCode::BAD_REQUEST, - message: e.to_string(), - flow: "delete_alert".into(), - })?; - let path = alert_json_path(id, tenant_id); - match self.storage.delete_object(&path, tenant_id).await { - Ok(()) => Ok(()), - Err(err) if is_missing_optional_dir(&err) => Ok(()), - Err(err) => Err(MetastoreError::ObjectStorageError(err)), - } - } - - /// alerts state - async fn get_alert_states( - &self, - tenant_id: &Option, - ) -> Result, MetastoreError> { - let tenant = tenant_id.as_deref().unwrap_or(DEFAULT_TENANT); - let base_path = RelativePathBuf::from_iter([tenant, ALERTS_ROOT_DIRECTORY]); - let alert_state_bytes = self - .storage - .get_objects( - Some(&base_path), - Box::new(|file_name| { - file_name.starts_with("alert_state_") && file_name.ends_with(".json") - }), +super::document_metastore::document_metastore!( + ObjectStoreMetastore, + dyn crate::storage::ObjectStorage, + true, + { + async fn check_health(&self, tenant_id: &Option) -> Result<(), MetastoreError> { + Ok(self.storage.check(tenant_id).await?) + } + + async fn read_latency(&self, tenant_id: &Option) -> std::time::Duration { + self.storage.get_latency(tenant_id).await + } + + async fn stream_exists( + &self, + stream_name: &str, + tenant_id: &Option, + ) -> Result { + Ok(self.storage.stream_exists(stream_name, tenant_id).await?) + } + + async fn is_stream_deleting( + &self, + stream_name: &str, + tenant_id: &Option, + ) -> Result { + Ok(crate::storage::object_storage::is_tombstoned( + self.storage.as_ref(), + stream_name, tenant_id, ) - .await?; - - let mut alert_states = Vec::new(); - for bytes in alert_state_bytes { - if let Ok(entry) = serde_json::from_slice::(&bytes) { - alert_states.push(entry); - } - } - - Ok(alert_states) - } - - async fn get_alert_state_entry( - &self, - alert_id: &Ulid, - tenant_id: &Option, - ) -> Result, MetastoreError> { - let path = alert_state_json_path(*alert_id, tenant_id); - match self.storage.get_object(&path, tenant_id).await { - Ok(bytes) => { - if let Ok(entry) = serde_json::from_slice::(&bytes) { - Ok(Some(entry)) - } else { - Ok(None) - } - } - Err(ObjectStorageError::NoSuchKey(_)) => Ok(None), - Err(e) => Err(MetastoreError::ObjectStorageError(e)), - } - } - - async fn put_alert_state( - &self, - obj: &dyn MetastoreObject, - tenant_id: &Option, - ) -> Result<(), MetastoreError> { - let id = Ulid::from_string(&obj.get_object_id()).map_err(|e| MetastoreError::Error { - status_code: StatusCode::BAD_REQUEST, - message: e.to_string(), - flow: "put_alert_state".into(), - })?; - let path = alert_state_json_path(id, tenant_id); - - // Parse the new state entry from the MetastoreObject - let new_state_entry: AlertStateEntry = serde_json::from_slice(&to_bytes(obj))?; - let new_state = new_state_entry - .current_state() - .ok_or_else(|| MetastoreError::InvalidJsonStructure { - expected: "AlertStateEntry with at least one state".to_string(), - found: "AlertStateEntry with empty states".to_string(), - })? - .state; - - // Try to read and parse existing file - if let Ok(existing_bytes) = self.storage.get_object(&path, tenant_id).await { - // File exists - try to parse and update - if let Ok(mut existing_entry) = - serde_json::from_slice::(&existing_bytes) - { - // Update the state and only save if it actually changed - let state_changed = existing_entry.update_state(new_state); - - if state_changed { - let updated_bytes = serde_json::to_vec(&existing_entry) - .map_err(MetastoreError::JsonParseError)?; - - self.storage - .put_object(&path, updated_bytes.into(), tenant_id) - .await?; - } - return Ok(()); - } - } - - // Create and save new entry (either file didn't exist or parsing failed) - let new_entry = AlertStateEntry::new(id, new_state, tenant_id.clone()); - let new_bytes = serde_json::to_vec(&new_entry).map_err(MetastoreError::JsonParseError)?; - - self.storage - .put_object(&path, new_bytes.into(), tenant_id) - .await?; - - Ok(()) - } - - /// Delete an alert state file - async fn delete_alert_state( - &self, - obj: &dyn MetastoreObject, - tenant_id: &Option, - ) -> Result<(), MetastoreError> { - let id = Ulid::from_string(&obj.get_object_id()).map_err(|e| MetastoreError::Error { - status_code: StatusCode::BAD_REQUEST, - message: e.to_string(), - flow: "delete_alert_state".into(), - })?; - let path = alert_state_json_path(id, tenant_id); - match self.storage.delete_object(&path, tenant_id).await { - Ok(()) => Ok(()), - Err(err) if is_missing_optional_dir(&err) => Ok(()), - Err(err) => Err(MetastoreError::ObjectStorageError(err)), - } - } - - /// Get MTTR history from storage - async fn get_mttr_history( - &self, - tenant_id: &Option, - ) -> Result, MetastoreError> { - let path = mttr_json_path(tenant_id); - match self.storage.get_object(&path, tenant_id).await { - Ok(bytes) => { - if let Ok(mut history) = serde_json::from_slice::(&bytes) { - if let Some(tenant) = tenant_id.as_ref() { - history.tenant_id = Some(tenant.clone()); - } - Ok(Some(history)) - } else { - Ok(None) - } - } - Err(ObjectStorageError::NoSuchKey(_)) => Ok(None), - Err(e) => Err(MetastoreError::ObjectStorageError(e)), - } - } - - /// Put MTTR history to storage - async fn put_mttr_history( - &self, - obj: &dyn MetastoreObject, - tenant_id: &Option, - ) -> Result<(), MetastoreError> { - let path = RelativePathBuf::from(obj.get_object_path()); - Ok(self - .storage - .put_object(&path, to_bytes(obj), tenant_id) - .await?) - } - - /// This function fetches all the llmconfigs from the underlying object store - async fn get_llmconfigs(&self) -> Result>, MetastoreError> { - let base_paths = PARSEABLE.list_tenants().unwrap_or_else(|| vec!["".into()]); - let mut all_configs = HashMap::new(); - for mut tenant in base_paths { - let base_path = - RelativePathBuf::from_iter([&tenant, SETTINGS_ROOT_DIRECTORY, "llmconfigs"]); - let conf_bytes = self - .storage - .get_objects( - Some(&base_path), - Box::new(|file_name| file_name.ends_with(".json")), - &Some(tenant.clone()), - ) - .await?; - if tenant.is_empty() { - tenant.clone_from(&DEFAULT_TENANT.to_string()); - } - all_configs.insert(tenant, conf_bytes); - } - Ok(all_configs) - } - - /// This function puts an llmconfig in the object store at the given path - async fn put_llmconfig( - &self, - obj: &dyn MetastoreObject, - tenant_id: &Option, - ) -> Result<(), MetastoreError> { - let path = obj.get_object_path(); - - Ok(self - .storage - .put_object(&RelativePathBuf::from(path), to_bytes(obj), tenant_id) - .await?) - } - - /// Delete an llmconfig - async fn delete_llmconfig( - &self, - obj: &dyn MetastoreObject, - tenant_id: &Option, - ) -> Result<(), MetastoreError> { - let path = obj.get_object_path(); - Ok(self - .storage - .delete_object(&RelativePathBuf::from(path), tenant_id) - .await?) - } - - async fn get_dashboard_versioned( - &self, - path: &str, - tenant_id: &Option, - ) -> Result, ConditionalMetadataError> { - self.storage - .get_metadata_versioned(RelativePath::new(path), tenant_id) - .await - } - - async fn put_dashboard_conditional( - &self, - path: &str, - bytes: Bytes, - condition: MetadataWriteCondition, - tenant_id: &Option, - ) -> Result { - self.storage - .put_metadata_conditional(RelativePath::new(path), bytes, condition, tenant_id) - .await - } - - async fn delete_dashboard_conditional( - &self, - path: &str, - version: MetadataVersion, - tenant_id: &Option, - ) -> Result<(), ConditionalMetadataError> { - self.storage - .delete_metadata_conditional(RelativePath::new(path), version, tenant_id) - .await - } - - /// Fetch all dashboards; hidden service indexes are not dashboard owners. - async fn get_dashboards(&self) -> Result>, MetastoreError> { - let mut dashboards = HashMap::new(); - let tenants = PARSEABLE.list_tenants().unwrap_or_else(|| vec!["".into()]); - for tenant in tenants { - let tenant_id = if tenant.is_empty() { - None - } else { - Some(tenant.clone()) - }; - let bytes = self.get_tenant_dashboards(&tenant_id).await?; - dashboards.insert( - tenant_id.unwrap_or_else(|| DEFAULT_TENANT.to_string()), - bytes, - ); - } - Ok(dashboards) - } - - async fn get_tenant_dashboards( - &self, - tenant_id: &Option, - ) -> Result, MetastoreError> { - let users = - RelativePathBuf::from_iter([tenant_id.as_deref().unwrap_or(""), USERS_ROOT_DIR]); - let mut dashboards = Vec::new(); - for owner in self.storage.list_dirs_relative(&users, tenant_id).await? { - if owner.starts_with('.') { - continue; - } - let path = users.join(owner).join("dashboards"); - match self - .storage - .get_objects( - Some(&path), - Box::new(|name| name.ends_with(".json")), - tenant_id, - ) - .await - { - Ok(bytes) => dashboards.extend(bytes), - Err(error) if is_missing_optional_dir(&error) => {} - Err(error) => return Err(error.into()), - } - } - Ok(dashboards) - } - - /// Save a dashboard - async fn put_dashboard( - &self, - obj: &dyn MetastoreObject, - tenant_id: &Option, - ) -> Result<(), MetastoreError> { - // we need the path to store in obj store - let path = obj.get_object_path(); - - Ok(self - .storage - .put_object(&RelativePathBuf::from(path), to_bytes(obj), tenant_id) .await?) - } - - /// Delete a dashboard - async fn delete_dashboard( - &self, - obj: &dyn MetastoreObject, - tenant_id: &Option, - ) -> Result<(), MetastoreError> { - let path = obj.get_object_path(); - Ok(self - .storage - .delete_object(&RelativePathBuf::from(path), tenant_id) - .await?) - } - - /// Fetch all chats - async fn get_chats( - &self, - ) -> Result>>, MetastoreError> { - let base_paths = PARSEABLE.list_tenants().unwrap_or_else(|| vec!["".into()]); - let all_chats = DashMap::new(); - for tenant in base_paths { - let all_user_chats = DashMap::new(); - let tenant_id = if tenant.is_empty() { - None - } else { - Some(tenant.clone()) - }; - let users_dir = RelativePathBuf::from_iter([&tenant, USERS_ROOT_DIR]); - for user in self - .storage - .list_dirs_relative(&users_dir, &tenant_id) - .await? - { - if user.starts_with(".") { - continue; - } - let mut chats = Vec::new(); - let chats_path = users_dir.join(&user).join("chats"); - let user_chats = self - .storage - .get_objects( - Some(&chats_path), - Box::new(|file_name| file_name.ends_with(".json")), - &tenant_id, - ) - .await?; - for chat in user_chats { - chats.push(chat); - } - - all_user_chats.insert(user, chats); - } - all_chats.insert( - tenant_id.as_deref().unwrap_or(DEFAULT_TENANT).into(), - all_user_chats, - ); } - Ok(all_chats) - } - - /// Save a chat - async fn put_chat( - &self, - obj: &dyn MetastoreObject, - tenant_id: &Option, - ) -> Result<(), MetastoreError> { - // we need the path to store in obj store - let path = obj.get_object_path(); - - Ok(self - .storage - .put_object(&RelativePathBuf::from(path), to_bytes(obj), tenant_id) - .await?) - } - - /// Delete a chat - async fn delete_chat( - &self, - obj: &dyn MetastoreObject, - tenant_id: &Option, - ) -> Result<(), MetastoreError> { - let path = obj.get_object_path(); - Ok(self - .storage - .delete_object(&RelativePathBuf::from(path), tenant_id) - .await?) - } - // Reconcile duplicate SQL paths before exposing filters to the cache. - async fn get_filters(&self) -> Result>, MetastoreError> { - let mut all = HashMap::new(); - let tenants = PARSEABLE.list_tenants().unwrap_or_else(|| vec!["".into()]); - for tenant in tenants { - let tenant_id = Some(tenant.clone()); - let filters = self.load_tenant_filters(&tenant_id).await?; - all.insert( - if tenant.is_empty() { - DEFAULT_TENANT.to_owned() - } else { - tenant - }, - filters, - ); + async fn delete_stream( + &self, + stream_name: &str, + tenant_id: &Option, + ) -> Result<(), MetastoreError> { + Ok(self.storage.delete_stream(stream_name, tenant_id).await?) } - Ok(all) - } - - /// Save a filter - async fn put_filter( - &self, - obj: &dyn MetastoreObject, - tenant_id: &Option, - ) -> Result<(), MetastoreError> { - // we need the path to store in obj store - let path = obj.get_object_path(); - - Ok(self - .storage - .put_object(&RelativePathBuf::from(path), to_bytes(obj), tenant_id) - .await?) - } - async fn reconcile_filter( - &self, - filter: &Filter, - tenant_id: &Option, - ) -> Result<(), MetastoreError> { - let canonical = stored_filter_path(filter, tenant_id); - for (path, _) in self.filter_aliases(filter, tenant_id).await? { - if path != canonical { - self.delete_filter_path(&path, tenant_id).await?; - } + async fn delete_stream_partition( + &self, + stream_name: &str, + date: &str, + tenant_id: &Option, + ) -> Result<(), MetastoreError> { + let path = + crate::storage::object_storage::stream_data_path(stream_name, &[date], tenant_id); + Ok(self.storage.delete_prefix(&path, tenant_id).await?) } - Ok(()) - } - /// Delete aliases first. If any cleanup fails, the canonical copy remains - /// available for a retry instead of allowing an old alias to reappear. - async fn delete_filter( - &self, - filter: &Filter, - tenant_id: &Option, - ) -> Result<(), MetastoreError> { - self.reconcile_filter(filter, tenant_id).await?; - self.delete_filter_path(&stored_filter_path(filter, tenant_id), tenant_id) - .await - } - - /// Fetch an `ObjectStoreFormat` file - /// - /// If `get_base` is true, get the one at the base of the stream directory else depends on Mode - async fn get_stream_json( - &self, - stream_name: &str, - get_base: bool, - tenant_id: &Option, - _is_migration: bool, - ) -> Result { - let tenant = tenant_id.as_deref().unwrap_or(""); - let path = if get_base { - RelativePathBuf::from_iter([ - tenant, - stream_name, - STREAM_ROOT_DIRECTORY, - STREAM_METADATA_FILE_NAME, - ]) - } else { - stream_json_path(stream_name, tenant_id) - }; - Ok(self.storage.get_object(&path, tenant_id).await?) - } - - /// Fetch all `ObjectStoreFormat` present in a stream folder - async fn get_all_stream_jsons( - &self, - stream_name: &str, - mode: Option, - tenant_id: &Option, - _is_migration: bool, - ) -> Result, MetastoreError> { - let root = tenant_id.as_deref().unwrap_or(""); - let path = RelativePathBuf::from_iter([root, stream_name, STREAM_ROOT_DIRECTORY]); - if let Some(mode) = mode { - if mode.eq(&Mode::Ingest) { - Ok(self - .storage - .get_objects( - Some(&path), - Box::new(|file_name| { - file_name.starts_with(".ingestor") && file_name.ends_with("stream.json") - }), - tenant_id, - ) - .await?) - } else { - return Err(MetastoreError::Error { - status_code: StatusCode::BAD_REQUEST, - message: "Incorrect server mode passed as input. Only `Ingest` is allowed." - .into(), - flow: "get_all_streams with mode".into(), - }); - } - } else { + async fn list_tenant_candidates(&self) -> Result, MetastoreError> { Ok(self .storage - .get_objects( - Some(&path), - Box::new(|file_name| file_name.ends_with("stream.json")), - tenant_id, - ) - .await?) - } - } - - /// Save an `ObjectStoreFormat` file - async fn put_stream_json( - &self, - obj: &dyn MetastoreObject, - stream_name: &str, - tenant_id: &Option, - ) -> Result<(), MetastoreError> { - let path = stream_json_path(stream_name, tenant_id); - - Ok(self - .storage - .put_object(&path, to_bytes(obj), tenant_id) - .await?) - } - - /// Fetch manifest files for the given date keys in parallel. - /// - /// Per-date work runs concurrently via `futures::future::try_join_all` - /// so a slow LIST on one date does not block the others. Each date's - /// timing is logged at info level so any pathologically slow dates - /// (S3 retries, oversized prefixes) are visible without needing extra - /// span instrumentation. - async fn get_manifest_files_for_dates( - &self, - stream_name: &str, - tenant_id: &Option, - dates: &[String], - ) -> Result>, MetastoreError> { - let inventory_limit = Arc::new(tokio::sync::Semaphore::new(16)); - let date_futures = dates.iter().map(|date| { - let stream = stream_name.to_string(); - let tenant = tenant_id.clone(); - let date = date.clone(); - let inventory_limit = inventory_limit.clone(); - async move { - let t_start = std::time::Instant::now(); - let date_path = if let Some(tenant) = tenant.as_ref() { - object_store::path::Path::from(format!("{}/{}/{}", tenant, stream, date)) - } else { - object_store::path::Path::from(format!("{}/{}", stream, date)) - }; - - let t_list = std::time::Instant::now(); - let list_permit = inventory_limit - .clone() - .acquire_owned() - .await - .expect("inventory semaphore remains open"); - let list_result = self.storage.list_with_delimiter(Some(date_path)).await; - drop(list_permit); - let resp = match list_result { - Ok(r) => r, - Err(ObjectStorageError::NoSuchKey(_)) => { - info!( - stream = %stream, - tenant = ?tenant, - date = %date, - list_ms = t_list.elapsed().as_millis() as u64, - "manifest fetch: date prefix not found" - ); - return Ok::<(String, Vec), MetastoreError>((date, Vec::new())); - } - Err(e) => return Err(e.into()), - }; - let list_ms = t_list.elapsed().as_millis() as u64; - - let manifest_paths: Vec = resp - .objects - .iter() - .filter(|name| name.location.filename().unwrap().ends_with("manifest.json")) - .map(|name| name.location.to_string()) - .collect(); - let manifest_count = manifest_paths.len(); - - let t_gets = std::time::Instant::now(); - let manifests = futures::stream::iter(manifest_paths) - .map(|path| { - let tenant = tenant.clone(); - let inventory_limit = inventory_limit.clone(); - async move { - let _permit = inventory_limit - .acquire_owned() - .await - .expect("inventory semaphore remains open"); - let bytes = self - .storage - .get_object(&RelativePathBuf::from(path), &tenant) - .await?; - Ok::(decode_manifest_blocking(bytes).await?) - } - }) - .buffer_unordered(16) - .try_collect::>() - .await?; - let gets_ms = t_gets.elapsed().as_millis() as u64; - let total_ms = t_start.elapsed().as_millis() as u64; - - if total_ms > 100 { - info!( - stream = %stream, - tenant = ?tenant, - date = %date, - list_ms, - gets_ms, - total_ms, - manifest_count, - "manifest fetch per-date timing" - ); - } - Ok((date, manifests)) - } - }); - - let results: Vec<(String, Vec)> = - futures::future::try_join_all(date_futures).await?; - - let mut result_file_list: BTreeMap> = BTreeMap::new(); - for (date, manifests) in results { - if !manifests.is_empty() { - result_file_list.insert(date, manifests); - } + .list_dirs_relative(&RelativePathBuf::from_iter([""]), &None) + .await? + .into_iter() + .filter(|d| !d.starts_with('.')) + .collect()) } - - Ok(result_file_list) } +); +#[cfg(test)] +mod boundary_compatibility_tests { + use super::*; + use crate::storage::{LocalFS, ObjectStorage, object_storage::tombstone_path}; - /// Fetch a specific `Manifest` file - async fn get_manifest( - &self, - stream_name: &str, - lower_bound: DateTime, - upper_bound: DateTime, - manifest_url: Option, - tenant_id: &Option, - ) -> Result, MetastoreError> { - let path = match manifest_url { - Some(url) => RelativePathBuf::from(url), - None => { - let path = partition_path(stream_name, lower_bound, upper_bound, tenant_id); - manifest_path(path.as_str()) - } + #[tokio::test] + async fn partition_deletion_preserves_tenant_scope_and_adjacent_bytes() { + let dir = tempfile::tempdir().unwrap(); + let storage = Arc::new(LocalFS::new(dir.path().to_owned())); + let metastore = ObjectStoreMetastore { + storage: storage.clone(), }; - match self.storage.get_object(&path, tenant_id).await { - Ok(bytes) => { - let manifest = decode_manifest_blocking(bytes).await?; - Ok(Some(manifest)) - } - Err(ObjectStorageError::NoSuchKey(_)) => Ok(None), - Err(err) => Err(MetastoreError::ObjectStorageError(err)), - } - } - - /// Get the path for a specific `Manifest` file - async fn get_manifest_path( - &self, - stream_name: &str, - lower_bound: DateTime, - upper_bound: DateTime, - tenant_id: &Option, - ) -> Result { - let path = partition_path(stream_name, lower_bound, upper_bound, tenant_id); - Ok(self - .storage - .absolute_url(&manifest_path(path.as_str())) - .to_string()) - } - - async fn put_manifest( - &self, - manifest: &Manifest, - stream_name: &str, - lower_bound: DateTime, - upper_bound: DateTime, - tenant_id: &Option, - ) -> Result<(), MetastoreError> { - let manifest_file_name = manifest_path("").to_string(); - let path = partition_path(stream_name, lower_bound, upper_bound, tenant_id) - .join(&manifest_file_name); - - Ok(self - .storage - .put_object(&path, encode_manifest(manifest)?, tenant_id) - .await?) - } - - async fn delete_manifest( - &self, - stream_name: &str, - lower_bound: DateTime, - upper_bound: DateTime, - tenant_id: &Option, - ) -> Result<(), MetastoreError> { - let manifest_file_name = manifest_path("").to_string(); - let path = partition_path(stream_name, lower_bound, upper_bound, tenant_id) - .join(&manifest_file_name); - Ok(self.storage.delete_object(&path, tenant_id).await?) - } - - /// targets - async fn get_targets(&self) -> Result>, MetastoreError> { - let base_paths = PARSEABLE.list_tenants().unwrap_or_else(|| vec!["".into()]); - let mut all_targets = HashMap::new(); - for mut tenant in base_paths { - let targets_path = RelativePathBuf::from_iter([ - &tenant, - SETTINGS_ROOT_DIRECTORY, - TARGETS_ROOT_DIRECTORY, - ]); - let target_bytes = match self - .storage - .get_objects( - Some(&targets_path), - Box::new(|file_name| file_name.ends_with(".json")), - &Some(tenant.clone()), + let tenant = Some("tenant".to_owned()); + let keys = [ + "tenant/logs/date=2026-01-01/hour=00/data.parquet", + "tenant/logs/date=2026-01-01/ingestor.node.manifest.json", + "tenant/logs/date=2026-01-02/ingestor.node.manifest.json", + "logs/date=2026-01-01/ingestor.node.manifest.json", + ]; + for key in keys { + storage + .put_object( + RelativePath::new(key), + Bytes::from_static(b"original bytes"), + &tenant, ) .await - { - Ok(targets) => targets, - Err(err) if is_missing_optional_dir(&err) => Vec::new(), - Err(err) => return Err(MetastoreError::ObjectStorageError(err)), - }; - let targets = target_bytes - .iter() - .filter_map(|bytes| { - serde_json::from_slice(bytes) - .inspect_err(|err| warn!("Expected compatible json, error = {err}")) - .ok() - }) - .collect(); - if tenant.is_empty() { - tenant.clone_from(&DEFAULT_TENANT.to_string()); - } - all_targets.insert(tenant, targets); - } - Ok(all_targets) - } - - async fn put_target( - &self, - obj: &dyn MetastoreObject, - tenant_id: &Option, - ) -> Result<(), MetastoreError> { - // we need the path to store in obj store - let path = obj.get_object_path(); - - Ok(self - .storage - .put_object(&RelativePathBuf::from(path), to_bytes(obj), tenant_id) - .await?) - } - - async fn delete_target( - &self, - obj: &dyn MetastoreObject, - tenant_id: &Option, - ) -> Result<(), MetastoreError> { - // we need the path to store in obj store - let path = obj.get_object_path(); - - Ok(self - .storage - .delete_object(&RelativePathBuf::from(path), tenant_id) - .await?) - } - - async fn get_outbound_policies( - &self, - ) -> Result, MetastoreError> { - let mut all_policies = HashMap::new(); - - let base_paths = PARSEABLE.list_tenants().unwrap_or_else(|| vec!["".into()]); - - for tenant in base_paths { - let tenant_id = if tenant.is_empty() { - None - } else { - Some(tenant.clone()) - }; - let path = outbound_http_policy_json_path(&tenant_id); - let policy = match self.storage.get_object(&path, &tenant_id).await { - Ok(bytes) => match serde_json::from_slice::(&bytes) { - Ok(policy) => policy, - Err(err) => { - warn!( - "Failed to deserialize outbound HTTP policy for tenant {tenant}: {err}" - ); - continue; - } - }, - Err(err) if is_missing_optional_dir(&err) => continue, - Err(err) => return Err(MetastoreError::ObjectStorageError(err)), - }; - - all_policies.insert(tenant, policy); + .unwrap(); } - - Ok(all_policies) - } - - async fn put_outbound_policy( - &self, - tenant_id: &Option, - policy: &AlertTargetPolicyConfig, - ) -> Result<(), MetastoreError> { - let path = outbound_http_policy_json_path(tenant_id); - Ok(self - .storage - .put_object(&path, to_bytes(policy), tenant_id) - .await?) - } - - async fn get_all_schemas( - &self, - stream_name: &str, - tenant_id: &Option, - ) -> Result, MetastoreError> { - let path_prefix = if let Some(tenant) = tenant_id { - relative_path::RelativePathBuf::from(format!( - "{tenant}/{stream_name}/{STREAM_ROOT_DIRECTORY}" - )) - } else { - relative_path::RelativePathBuf::from(format!("{stream_name}/{STREAM_ROOT_DIRECTORY}")) - }; - Ok(self - .storage - .get_objects( - Some(&path_prefix), - Box::new(|file_name: String| file_name.contains(".schema")), - tenant_id, - ) - .await? - .iter() - // we should be able to unwrap as we know the data is valid schema - .map(|byte_obj| { - serde_json::from_slice(byte_obj) - .unwrap_or_else(|_| panic!("got an invalid schema for stream: {stream_name}")) - }) - .collect()) - } - - async fn refresh_schema( - &self, - stream_name: &str, - tenant_id: &Option, - schema_file: &str, - ) -> Result { - let objects = self.storage.schema_objects(stream_name, tenant_id).await?; - let mut schemas = Vec::with_capacity(objects.len()); - let mut canonical = None; - for (name, bytes) in objects { - let schema: Schema = serde_json::from_slice(&bytes) - .map_err(|e| ObjectStorageError::Custom(e.to_string()))?; - if name == schema_file { - canonical = Some(schema.clone()); - } - schemas.push(schema); + metastore + .delete_stream_partition("logs", "date=2026-01-01", &tenant) + .await + .unwrap(); + for key in &keys[..2] { + assert!( + storage + .get_object(RelativePath::new(key), &tenant) + .await + .is_err() + ); } - let merged = - Schema::try_merge(schemas).map_err(|e| ObjectStorageError::Custom(e.to_string()))?; - // Field order is immaterial, but nested types, nullability and field - // metadata all matter. Normalize only the top-level ordering. - let normalize = |schema: &Schema| { - let mut schema = schema.clone(); - let mut fields = schema.fields.to_vec(); - fields.sort_by(|a, b| a.name().cmp(b.name())); - schema.fields = fields.into(); - schema - }; - if canonical.as_ref().map(normalize) != Some(normalize(&merged)) { - let path = RelativePathBuf::from_iter([ - tenant_id.as_deref().unwrap_or(""), - stream_name, - STREAM_ROOT_DIRECTORY, - schema_file, - ]); - self.storage - .put_object(&path, to_bytes(&merged), tenant_id) - .await?; + for key in &keys[2..] { + assert_eq!( + storage + .get_object(RelativePath::new(key), &tenant) + .await + .unwrap(), + Bytes::from_static(b"original bytes") + ); } - Ok(merged) - } - - async fn get_schema( - &self, - stream_name: &str, - tenant_id: &Option, - ) -> Result { - Ok(self - .storage - .get_object(&schema_path(stream_name, tenant_id), tenant_id) - .await?) - } - - async fn put_schema( - &self, - obj: Schema, - stream_name: &str, - tenant_id: &Option, - ) -> Result<(), MetastoreError> { - let path = schema_path(stream_name, tenant_id); - Ok(self - .storage - .put_object(&path, to_bytes(&obj), tenant_id) - .await?) } - async fn get_parseable_metadata( - &self, - tenant_id: &Option, - ) -> Result, MetastoreError> { - let path = if let Some(tenant_id) = tenant_id.as_ref() { - RelativePathBuf::from_iter([tenant_id, ".parseable.json"]) - } else { - parseable_json_path() + #[tokio::test] + async fn lifecycle_seams_preserve_localfs_results_and_bytes() { + let dir = tempfile::tempdir().unwrap(); + let storage = Arc::new(LocalFS::new(dir.path().to_owned())); + let metastore = ObjectStoreMetastore { + storage: storage.clone(), }; - - let parseable_metadata: Option = - match self.storage.get_object(&path, tenant_id).await { - Ok(bytes) => Some(bytes), - Err(err) => { - if matches!(err, ObjectStorageError::NoSuchKey(_)) { - None - } else { - return Err(MetastoreError::ObjectStorageError(err)); - } - } - }; - - Ok(parseable_metadata) - } - - async fn delete_tenant(&self, tenant_id: &str) -> Result<(), MetastoreError> { - self.storage - .delete_prefix(&RelativePathBuf::from(tenant_id), &None) - .await - .map_err(MetastoreError::ObjectStorageError) - } - - async fn get_ingestor_metadata(&self) -> Result, MetastoreError> { - let base_path = RelativePathBuf::from(PARSEABLE_ROOT_DIRECTORY); - Ok(self - .storage - .get_objects( - Some(&base_path), - Box::new(|file_name| file_name.starts_with("ingestor")), + assert_eq!( + metastore.check_health(&None).await.is_ok(), + storage.check(&None).await.is_ok() + ); + metastore.read_latency(&None).await; + assert!(!metastore.stream_exists("logs", &None).await.unwrap()); + assert!(!metastore.is_stream_deleting("logs", &None).await.unwrap()); + let key = RelativePath::new("logs/.stream/.stream.json"); + let bytes = Bytes::from_static(b"{\"version\":\"v7\",\"unknown\":true}"); + storage.put_object(key, bytes.clone(), &None).await.unwrap(); + assert_eq!( + metastore.stream_exists("logs", &None).await.unwrap(), + storage.stream_exists("logs", &None).await.unwrap() + ); + // Discovery/existence probes never normalize or rewrite stored JSON. + assert_eq!(storage.get_object(key, &None).await.unwrap(), bytes); + storage + .put_object( + &tombstone_path("logs", &None), + Bytes::from_static(b"null"), &None, ) - .await?) - } - - async fn put_parseable_metadata( - &self, - obj: &dyn MetastoreObject, - tenant_id: &Option, - ) -> Result<(), MetastoreError> { - let path = if let Some(tenant_id) = tenant_id.as_ref() { - RelativePathBuf::from_iter([tenant_id, ".parseable.json"]) - } else { - parseable_json_path() - }; - - self.storage - .put_object(&path, to_bytes(obj), tenant_id) .await - .map_err(MetastoreError::ObjectStorageError) - } - - async fn get_node_metadata( - &self, - node_type: NodeType, - tenant_id: &Option, - ) -> Result, MetastoreError> { - let root_path = RelativePathBuf::from(PARSEABLE_ROOT_DIRECTORY); - let prefix_owned = node_type.to_string(); - - let metadata = self - .storage - .get_objects( - Some(&root_path), - Box::new(move |file_name| file_name.starts_with(&prefix_owned)), // Use the owned copy - tenant_id, - ) - .await? - .into_iter() - .collect(); - - Ok(metadata) - } - - async fn put_node_metadata(&self, obj: &dyn MetastoreObject) -> Result<(), MetastoreError> { - let path = obj.get_object_path(); - self.storage - .put_object(&RelativePathBuf::from(path), to_bytes(obj), &None) - .await?; - Ok(()) - } - - async fn delete_node_metadata( - &self, - domain_name: &str, - node_type: NodeType, - ) -> Result { - let metadatas = self - .storage - .get_objects( - Some(&RelativePathBuf::from(PARSEABLE_ROOT_DIRECTORY)), - Box::new(move |file_name| file_name.starts_with(&node_type.to_string())), - &None, - ) - .await?; - - let node_metadatas = metadatas - .iter() - .filter_map(|elem| match serde_json::from_slice::(elem) { - Ok(meta) if meta.domain_name() == domain_name => Some(meta), - _ => None, - }) - .collect::>(); - - if node_metadatas.is_empty() { - return Ok(false); - } - - let node_meta_filename = node_metadatas[0].file_path().to_string(); - let file = RelativePathBuf::from(&node_meta_filename); - - match self.storage.delete_object(&file, &None).await { - Ok(_) => Ok(true), - Err(err) => { - if matches!(err, ObjectStorageError::IoError(_)) { - Ok(false) - } else { - Err(MetastoreError::ObjectStorageError(err)) - } - } - } - } - - async fn check_health(&self, tenant_id: &Option) -> Result<(), MetastoreError> { - Ok(self.storage.check(tenant_id).await?) - } - - async fn read_latency(&self, tenant_id: &Option) -> std::time::Duration { - self.storage.get_latency(tenant_id).await - } - - async fn stream_exists( - &self, - stream_name: &str, - tenant_id: &Option, - ) -> Result { - Ok(self.storage.stream_exists(stream_name, tenant_id).await?) - } - - async fn is_stream_deleting( - &self, - stream_name: &str, - tenant_id: &Option, - ) -> Result { - Ok(crate::storage::object_storage::is_tombstoned( - self.storage.as_ref(), - stream_name, - tenant_id, - ) - .await?) - } - - async fn delete_stream( - &self, - stream_name: &str, - tenant_id: &Option, - ) -> Result<(), MetastoreError> { - Ok(self.storage.delete_stream(stream_name, tenant_id).await?) - } - - async fn delete_stream_partition( - &self, - stream_name: &str, - date: &str, - tenant_id: &Option, - ) -> Result<(), MetastoreError> { - let path = - crate::storage::object_storage::stream_data_path(stream_name, &[date], tenant_id); - Ok(self.storage.delete_prefix(&path, tenant_id).await?) - } - - async fn list_tenant_candidates(&self) -> Result, MetastoreError> { - Ok(self - .storage - .list_dirs_relative(&RelativePathBuf::from_iter([""]), &None) - .await? + .unwrap(); + assert!(metastore.is_stream_deleting("logs", &None).await.unwrap()); + let mut expected: Vec<_> = storage + .list_dirs_relative(RelativePath::new(""), &None) + .await + .unwrap() .into_iter() .filter(|d| !d.starts_with('.')) - .collect()) - } - - async fn list_streams( - &self, - tenant_id: &Option, - ) -> Result, MetastoreError> { - // using LocalFS list_streams because it doesn't implement list_with_delimiter - if PARSEABLE.storage.name() == "drive" { - PARSEABLE - .storage - .get_object_store() - .list_streams() - .await - .map_err(MetastoreError::ObjectStorageError) - } else { - // not local-disk, object storage - let mut result_file_list = HashSet::new(); - let root = tenant_id - .as_ref() - .map(|tenant| object_store::path::Path::from_iter([tenant.clone()])); - - let resp = self.storage.list_with_delimiter(root.clone()).await?; - - let streams = resp - .common_prefixes - .iter() - .flat_map(|path| path.parts()) - .map(|name| name.as_ref().to_string()) - .filter(|name| { - name != PARSEABLE_ROOT_DIRECTORY - && name != USERS_ROOT_DIR - && name != SETTINGS_ROOT_DIRECTORY - && name != ALERTS_ROOT_DIRECTORY - && name != TOMBSTONE_ROOT_DIRECTORY - }) - .collect::>(); - for stream in streams { - let stream_path = if let Some(root) = root.as_ref() { - object_store::path::Path::from_iter([ - root.as_ref(), - &stream, - STREAM_ROOT_DIRECTORY, - ]) - } else { - object_store::path::Path::from(format!("{}/{}", stream, STREAM_ROOT_DIRECTORY)) - }; - let resp = self.storage.list_with_delimiter(Some(stream_path)).await?; - if resp - .objects - .iter() - .any(|name| name.location.filename().unwrap().ends_with("stream.json")) - { - result_file_list.insert(stream); - } - } - Ok(result_file_list) - } + .collect(); + let mut actual = metastore.list_tenant_candidates().await.unwrap(); + expected.sort(); + actual.sort(); + assert_eq!(actual, expected); + metastore.delete_stream("logs", &None).await.unwrap(); + assert!(!metastore.stream_exists("logs", &None).await.unwrap()); + // Historical prefix deletion leaves the separate marker untouched. + assert!(metastore.is_stream_deleting("logs", &None).await.unwrap()); + let raw_error = storage + .delete_stream("missing", &None) + .await + .unwrap_err() + .to_string(); + let trait_error = metastore.delete_stream("missing", &None).await.unwrap_err(); + assert!( + matches!(trait_error, MetastoreError::ObjectStorageError(error) if error.to_string() == raw_error) + ); } } #[cfg(test)] mod tests { use super::*; - use crate::storage::LocalFS; + use crate::storage::{LocalFS, ObjectStorage}; #[tokio::test] async fn remote_schema_refresh_publishes_only_changes() { @@ -2157,109 +676,3 @@ mod tests { )); } } - -#[cfg(test)] -mod boundary_compatibility_tests { - use super::*; - use crate::storage::{LocalFS, object_storage::tombstone_path}; - - #[tokio::test] - async fn partition_deletion_preserves_tenant_scope_and_adjacent_bytes() { - let dir = tempfile::tempdir().unwrap(); - let storage = Arc::new(LocalFS::new(dir.path().to_owned())); - let metastore = ObjectStoreMetastore { - storage: storage.clone(), - }; - let tenant = Some("tenant".to_owned()); - let keys = [ - "tenant/logs/date=2026-01-01/hour=00/data.parquet", - "tenant/logs/date=2026-01-01/ingestor.node.manifest.json", - "tenant/logs/date=2026-01-02/ingestor.node.manifest.json", - "logs/date=2026-01-01/ingestor.node.manifest.json", - ]; - for key in keys { - storage - .put_object( - RelativePath::new(key), - Bytes::from_static(b"original bytes"), - &tenant, - ) - .await - .unwrap(); - } - metastore - .delete_stream_partition("logs", "date=2026-01-01", &tenant) - .await - .unwrap(); - for key in &keys[..2] { - assert!( - storage - .get_object(RelativePath::new(key), &tenant) - .await - .is_err() - ); - } - for key in &keys[2..] { - assert_eq!( - storage - .get_object(RelativePath::new(key), &tenant) - .await - .unwrap(), - Bytes::from_static(b"original bytes") - ); - } - } - - #[tokio::test] - async fn lifecycle_seams_preserve_localfs_results_and_bytes() { - let dir = tempfile::tempdir().unwrap(); - let storage = Arc::new(LocalFS::new(dir.path().to_owned())); - let metastore = ObjectStoreMetastore { - storage: storage.clone(), - }; - assert!(!metastore.stream_exists("logs", &None).await.unwrap()); - assert!(!metastore.is_stream_deleting("logs", &None).await.unwrap()); - let key = RelativePath::new("logs/.stream/.stream.json"); - let bytes = Bytes::from_static(b"{\"version\":\"v7\",\"unknown\":true}"); - storage.put_object(key, bytes.clone(), &None).await.unwrap(); - assert_eq!( - metastore.stream_exists("logs", &None).await.unwrap(), - storage.stream_exists("logs", &None).await.unwrap() - ); - // Discovery/existence probes never normalize or rewrite stored JSON. - assert_eq!(storage.get_object(key, &None).await.unwrap(), bytes); - storage - .put_object( - &tombstone_path("logs", &None), - Bytes::from_static(b"null"), - &None, - ) - .await - .unwrap(); - assert!(metastore.is_stream_deleting("logs", &None).await.unwrap()); - let mut expected: Vec<_> = storage - .list_dirs_relative(RelativePath::new(""), &None) - .await - .unwrap() - .into_iter() - .filter(|d| !d.starts_with('.')) - .collect(); - let mut actual = metastore.list_tenant_candidates().await.unwrap(); - expected.sort(); - actual.sort(); - assert_eq!(actual, expected); - metastore.delete_stream("logs", &None).await.unwrap(); - assert!(!metastore.stream_exists("logs", &None).await.unwrap()); - // Historical prefix deletion leaves the separate marker untouched. - assert!(metastore.is_stream_deleting("logs", &None).await.unwrap()); - let raw_error = storage - .delete_stream("missing", &None) - .await - .unwrap_err() - .to_string(); - let trait_error = metastore.delete_stream("missing", &None).await.unwrap_err(); - assert!( - matches!(trait_error, MetastoreError::ObjectStorageError(error) if error.to_string() == raw_error) - ); - } -} diff --git a/src/metastore/metastores/postgres_client.rs b/src/metastore/metastores/postgres_client.rs new file mode 100644 index 000000000..65ede61fd --- /dev/null +++ b/src/metastore/metastores/postgres_client.rs @@ -0,0 +1,160 @@ +//! Deadline enforcement for every SQL operation, including transactions. +use crate::storage::ObjectStorageError; +use std::{ + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, + time::Duration, +}; +use tokio_postgres::{Row, types::ToSql}; + +#[derive(Debug)] +pub(crate) struct Client { + inner: Option, + timeout: Duration, + poisoned: Arc, +} +impl Drop for Client { + fn drop(&mut self) { + if self.poisoned.load(Ordering::SeqCst) + && let Some(c) = self.inner.take() + { + drop(deadpool_postgres::Client::take(c)); + } + } +} +struct Pending<'a>(&'a AtomicBool, bool); +impl Drop for Pending<'_> { + fn drop(&mut self) { + if self.1 { + self.0.store(true, Ordering::SeqCst); + } + } +} +async fn bounded( + timeout: Duration, + poisoned: &AtomicBool, + call: impl std::future::Future>, +) -> Result { + let mut pending = Pending(poisoned, true); + let result = tokio::time::timeout(timeout, call).await; + pending.1 = false; + match result { + Ok(Ok(value)) => Ok(value), + Ok(Err(e)) => { + if e.code() == Some(&tokio_postgres::error::SqlState::READ_ONLY_SQL_TRANSACTION) + || e.is_closed() + { + poisoned.store(true, Ordering::SeqCst); + } + if e.as_db_error() + .is_some_and(|e| e.message().contains("stream deletion pending")) + { + Err(ObjectStorageError::StreamDeleting) + } else { + Err(ObjectStorageError::MetastoreUnavailable( + e.as_db_error() + .map(|d| d.to_string()) + .unwrap_or_else(|| e.to_string()), + )) + } + } + Err(_) => { + poisoned.store(true, Ordering::SeqCst); + Err(ObjectStorageError::MetastoreUnavailable( + "SQL client deadline exceeded".into(), + )) + } + } +} +macro_rules! methods { + () => { + pub async fn query( + &self, + sql: &str, + params: &[&(dyn ToSql + Sync)], + ) -> Result, ObjectStorageError> { + bounded(self.timeout, &self.poisoned, self.raw().query(sql, params)).await + } + pub async fn query_one( + &self, + sql: &str, + params: &[&(dyn ToSql + Sync)], + ) -> Result { + bounded( + self.timeout, + &self.poisoned, + self.raw().query_one(sql, params), + ) + .await + } + pub async fn query_opt( + &self, + sql: &str, + params: &[&(dyn ToSql + Sync)], + ) -> Result, ObjectStorageError> { + bounded( + self.timeout, + &self.poisoned, + self.raw().query_opt(sql, params), + ) + .await + } + pub async fn execute( + &self, + sql: &str, + params: &[&(dyn ToSql + Sync)], + ) -> Result { + bounded( + self.timeout, + &self.poisoned, + self.raw().execute(sql, params), + ) + .await + } + pub async fn batch_execute(&self, sql: &str) -> Result<(), ObjectStorageError> { + bounded(self.timeout, &self.poisoned, self.raw().batch_execute(sql)).await + } + }; +} +impl Client { + pub fn new(inner: deadpool_postgres::Client, timeout: Duration) -> Self { + Self { + inner: Some(inner), + timeout, + poisoned: Arc::new(AtomicBool::new(false)), + } + } + fn raw(&self) -> &tokio_postgres::Client { + self.inner.as_ref().unwrap() + } + methods!(); + pub async fn transaction(&mut self) -> Result, ObjectStorageError> { + let inner = bounded( + self.timeout, + &self.poisoned, + self.inner.as_mut().unwrap().transaction(), + ) + .await?; + Ok(Transaction { + inner, + timeout: self.timeout, + poisoned: self.poisoned.clone(), + }) + } +} +pub(crate) struct Transaction<'a> { + inner: deadpool_postgres::Transaction<'a>, + timeout: Duration, + poisoned: Arc, +} +impl Transaction<'_> { + fn raw(&self) -> &tokio_postgres::Transaction<'_> { + &self.inner + } + methods!(); + pub async fn commit(self) -> Result<(), ObjectStorageError> { + bounded(self.timeout, &self.poisoned, self.inner.commit()).await + } +} diff --git a/src/metastore/metastores/postgres_documents.rs b/src/metastore/metastores/postgres_documents.rs new file mode 100644 index 000000000..e79f9e423 --- /dev/null +++ b/src/metastore/metastores/postgres_documents.rs @@ -0,0 +1,1504 @@ +//! SQL document primitives. No connection is held during object storage I/O. +use super::postgres_metastore::PostgresOptions; +use crate::storage::object_storage::{ + ConditionalMetadataError, MetadataVersion, MetadataWriteCondition, VersionedMetadata, +}; +use crate::storage::{ObjectStorage, ObjectStorageError}; +use bytes::Bytes; +use relative_path::RelativePath; +use std::{sync::Arc, time::Duration}; + +#[derive(Debug)] +pub struct PostgresDocuments { + pool: deadpool_postgres::Pool, + deadline: Duration, + bucket: String, + expected_id: Option, + storage: Arc, + initialized: tokio::sync::OnceCell<()>, + #[cfg(test)] + pub(super) deletion_fault: std::sync::atomic::AtomicU8, +} +fn err(e: impl std::fmt::Display) -> ObjectStorageError { + if e.to_string().contains("stream deletion pending") { + ObjectStorageError::StreamDeleting + } else { + ObjectStorageError::MetastoreUnavailable(e.to_string()) + } +} +fn tenant(t: &Option) -> &str { + t.as_deref() + .filter(|v| !v.is_empty() && *v != "DEFAULT_TENANT") + .unwrap_or("") +} +fn version(v: i64) -> MetadataVersion { + MetadataVersion { + e_tag: None, + version: Some(v.to_string()), + } +} +pub(super) fn configured_roots( + ca: Option<&std::path::Path>, +) -> anyhow::Result { + let mut roots = rustls::RootCertStore::empty(); + if let Some(path) = ca { + for certificate in + rustls_pemfile::certs(&mut std::io::BufReader::new(std::fs::File::open(path)?)) + { + roots.add(certificate?)?; + } + anyhow::ensure!( + !roots.is_empty(), + "configured CA bundle contains no certificates" + ); + } else { + roots.extend(webpki_roots::TLS_SERVER_ROOTS.iter().cloned()); + } + Ok(roots) +} +impl PostgresDocuments { + pub fn new( + options: &PostgresOptions, + bucket: String, + storage: Arc, + ) -> anyhow::Result { + let mut config: tokio_postgres::Config = + options.metastore_postgres_url.as_deref().unwrap().parse()?; + let timeout = Duration::from_millis(options.metastore_postgres_connect_timeout_ms); + config.connect_timeout(timeout); + config.tcp_user_timeout(Duration::from_millis( + options.metastore_postgres_statement_timeout_ms, + )); + config + .keepalives(true) + .keepalives_idle(Duration::from_secs(1)) + .keepalives_interval(Duration::from_secs(1)) + .keepalives_retries(3); + config.target_session_attrs(tokio_postgres::config::TargetSessionAttrs::ReadWrite); + let prior_options = config.get_options().unwrap_or("").to_owned(); + config.options(format!( + "{prior_options} -c statement_timeout={} -c lock_timeout={}", + options.metastore_postgres_statement_timeout_ms, + options.metastore_postgres_statement_timeout_ms + )); + // Prefer/allow permit plaintext downgrade; opt-in mode defaults to verified TLS. + if config.get_ssl_mode() != tokio_postgres::config::SslMode::Disable { + config.ssl_mode(tokio_postgres::config::SslMode::Require); + } + let roots = configured_roots(options.metastore_postgres_ca.as_deref())?; + if options.metastore_postgres_ca.is_some() + && config.get_ssl_mode() == tokio_postgres::config::SslMode::Disable + { + tracing::warn!("PostgreSQL CA configured with sslmode=disable; CA is unused"); + } + let tls = rustls::ClientConfig::builder_with_provider(Arc::new( + rustls::crypto::ring::default_provider(), + )) + .with_safe_default_protocol_versions()? + .with_root_certificates(roots) + .with_no_client_auth(); + let manager = deadpool_postgres::Manager::new( + config, + tokio_postgres_rustls::MakeRustlsConnect::new(tls), + ); + let pool = deadpool_postgres::Pool::builder(manager) + .max_size(options.metastore_postgres_pool_size as usize) + .runtime(deadpool_postgres::Runtime::Tokio1) + .wait_timeout(Some(timeout)) + .create_timeout(Some(timeout)) + .recycle_timeout(Some(timeout)) + .build()?; + Ok(Self { + pool, + deadline: Duration::from_millis( + options + .metastore_postgres_statement_timeout_ms + .saturating_add(250), + ), + bucket, + expected_id: options.metastore_postgres_id.clone(), + storage, + initialized: tokio::sync::OnceCell::new(), + #[cfg(test)] + deletion_fault: std::sync::atomic::AtomicU8::new(0), + }) + } + pub(crate) async fn client( + &self, + ) -> Result { + Ok(super::postgres_client::Client::new( + tokio::time::timeout(self.deadline, self.pool.get()) + .await + .map_err(err)? + .map_err(err)?, + self.deadline, + )) + } + pub async fn check_migration_guard(&self) -> Result<(), ObjectStorageError> { + super::migration::guard(self.storage.as_ref(), true) + .await + .map_err(err) + } + pub async fn check(&self, _: &Option) -> Result<(), ObjectStorageError> { + super::migration::guard(self.storage.as_ref(), true) + .await + .map_err(err)?; + self.initialized.get_or_try_init(|| async { + let mut client = self.client().await?; + let tx = client.transaction().await.map_err(err)?; + tx.batch_execute("SELECT pg_advisory_xact_lock(8104050183641399916)").await.map_err(err)?; + let (actual, _, created) = self.bootstrap(&tx, &uuid::Uuid::new_v4().to_string()).await?; + if tx.query_opt("SELECT state FROM parseable_transfer WHERE singleton", &[]).await.map_err(err)?.is_some_and(|r| r.get::<_,String>(0) != "ready") { return Err(err("offline transfer incomplete or exported; use migration command")); } + + tx.commit().await.map_err(err)?; + drop(client); + // Pair outside SQL: never hold a pooled connection during bucket I/O. + let marker = RelativePath::new(".parseable/postgres-metastore-id"); + if !created { + // A concurrent first initializer may still be publishing pairing. + for _ in 0..20 { + if super::migration::optional(self.storage.as_ref(), ".parseable/postgres-metastore-id").await.map_err(err)?.is_some() { break; } + tokio::time::sleep(Duration::from_millis(10)).await; + } + } + if !created && super::migration::optional(self.storage.as_ref(), ".parseable/postgres-metastore-id").await.map_err(err)?.is_none() && std::env::var("P_METASTORE_POSTGRES_ALLOW_UNPAIRED").as_deref() != Ok("true") { + return Err(err("nonempty database and missing bucket pairing marker; restore the marker or explicitly set P_METASTORE_POSTGRES_ALLOW_UNPAIRED=true after verifying the bucket")); + } + match self.storage.put_metadata_conditional(marker, Bytes::from(actual.clone()), MetadataWriteCondition::Create, &None).await { + Ok(_) | Err(ConditionalMetadataError::Conflict) => {}, + Err(e) => return Err(err(e)), + } + let paired = self.storage.get_object(marker, &None).await?; + if paired.as_ref() != actual.as_bytes() { return Err(err("bucket is paired with a different PostgreSQL backend UUID")); } + tracing::info!(backend_id = actual, "PostgreSQL metastore initialized"); + Ok(()) + }).await.map(|_| ()) + } + async fn bootstrap( + &self, + tx: &super::postgres_client::Transaction<'_>, + id: &str, + ) -> Result<(String, bool, bool), ObjectStorageError> { + // Bootstrap is additive; version checks precede any future forward migration. + tx.batch_execute(include_str!("../migrations/000_identity.sql")) + .await + .map_err(err)?; + let created = tx.execute("INSERT INTO parseable_backend(id,bucket,schema_version,protocol_version) VALUES($1,$2,0,1) ON CONFLICT DO NOTHING", &[&id,&self.bucket]).await.map_err(err)? == 1; + let row = tx + .query_one( + "SELECT id,bucket,schema_version,protocol_version FROM parseable_backend", + &[], + ) + .await + .map_err(err)?; + let actual: String = row.get(0); + if row.get::<_, String>(1) != self.bucket + || self.expected_id.as_ref().is_some_and(|id| id != &actual) + { + return Err(err("backend/bucket identity mismatch")); + } + if !(0..=2).contains(&row.get::<_, i32>(2)) || row.get::<_, i32>(3) != 1 { + return Err(err( + "unsupported schema/protocol version; upgrade this server", + )); + } + if row.get::<_, i32>(2) == 0 { + tx.batch_execute(include_str!("../migrations/001_documents.sql")) + .await + .map_err(err)?; + tx.execute( + "UPDATE parseable_backend SET schema_version=1 WHERE singleton", + &[], + ) + .await + .map_err(err)?; + } + if row.get::<_, i32>(2) < 2 { + tx.batch_execute(include_str!("../migrations/002_review_fixes.sql")) + .await + .map_err(err)?; + } + tx.batch_execute("CREATE TABLE IF NOT EXISTS parseable_transfer (singleton boolean PRIMARY KEY DEFAULT true CHECK(singleton), state text NOT NULL)").await.map_err(err)?; + let has_documents: bool = tx + .query_one("SELECT EXISTS(SELECT 1 FROM parseable_documents)", &[]) + .await? + .get(0); + Ok((actual, has_documents, created)) + } + // Probe both dependencies without requiring a bucket metadata document. + pub async fn check_health(&self, t: &Option) -> Result<(), ObjectStorageError> { + let database = async { + self.client() + .await? + .batch_execute("SELECT 1") + .await + .map_err(err)?; + Ok::<_, ObjectStorageError>(()) + }; + let bucket = async { + self.storage + .get_object(RelativePath::new(".parseable/postgres-metastore-id"), t) + .await + .map(|_| ()) + }; + tokio::time::timeout(self.deadline, async { + tokio::try_join!(database, bucket)?; + Ok(()) + }) + .await + .map_err(err)? + } + pub async fn read_latency(&self, t: &Option) -> Duration { + let start = std::time::Instant::now(); + let result = self.check_health(t).await; + tracing::info!(?result, "PostgreSQL and data bucket combined probe"); + start.elapsed() + } + pub async fn stream_exists( + &self, + stream: &str, + t: &Option, + ) -> Result { + let path = crate::storage::object_storage::stream_data_path(stream, &[".stream"], t); + Ok(self.client().await?.query_one("SELECT EXISTS(SELECT 1 FROM parseable_documents WHERE tenant=$1 AND key >= $2 || '/' AND key < $2 || '0' AND key LIKE '%stream.json')", &[&tenant(t), &path.as_str()]).await?.get(0)) + } + + pub async fn is_stream_deleting( + &self, + stream: &str, + t: &Option, + ) -> Result { + let path = + crate::storage::object_storage::stream_data_path(stream, &[".stream", ".deleting"], t); + Ok(self + .get_metadata_versioned(&path, t) + .await + .map_err(err)? + .is_some()) + } + async fn remove_rows( + &self, + path: &RelativePath, + t: &Option, + ) -> Result<(), ObjectStorageError> { + self.client().await?.execute("DELETE FROM parseable_documents WHERE tenant=$1 AND ($2='' OR key=$2 OR (key >= $2 || '/' AND key < $2 || '0'))", &[&tenant(t), &path.as_str()]).await.map_err(err)?; + Ok(()) + } + async fn delete_data( + &self, + path: &RelativePath, + t: &Option, + ) -> Result<(), ObjectStorageError> { + match self.storage.delete_prefix(path, t).await { + Err(ObjectStorageError::NoSuchKey(_)) => Ok(()), + Err(ObjectStorageError::IoError(e)) if e.kind() == std::io::ErrorKind::NotFound => { + Ok(()) + } + result => result, + } + } + pub async fn delete_stream( + &self, + stream: &str, + t: &Option, + ) -> Result<(), ObjectStorageError> { + let marker = + crate::storage::object_storage::stream_data_path(stream, &[".stream", ".deleting"], t); + self.put_object(&marker, Bytes::new(), t).await?; + let path = crate::storage::object_storage::stream_data_path(stream, &[], t); + #[cfg(test)] + if self + .deletion_fault + .load(std::sync::atomic::Ordering::SeqCst) + == 1 + { + return Err(err("injected bucket failure")); + } + self.delete_data(&path, t).await?; + #[cfg(test)] + if self + .deletion_fault + .load(std::sync::atomic::Ordering::SeqCst) + == 2 + { + return Err(err("injected crash after data deletion")); + } + self.remove_rows(&path, t).await + } + pub async fn delete_partition( + &self, + path: &RelativePath, + t: &Option, + ) -> Result<(), ObjectStorageError> { + self.delete_data(path, t).await?; + #[cfg(test)] + if self + .deletion_fault + .load(std::sync::atomic::Ordering::SeqCst) + == 3 + { + return Err(err("injected partition cleanup failure")); + } + self.remove_rows(path, t).await + } + pub async fn list_tenant_candidates(&self) -> Result, ObjectStorageError> { + let mut candidates: std::collections::BTreeSet = self + .client() + .await? + .query( + "SELECT DISTINCT tenant FROM parseable_documents WHERE tenant <> ''", + &[], + ) + .await + .map_err(err)? + .into_iter() + .map(|r| r.get(0)) + .collect(); + let default_streams: std::collections::HashSet = self + .keys("", "") + .await? + .iter() + .filter_map(|k| { + k.split_once('/').and_then(|(s, rest)| { + (rest.starts_with(".stream/") || rest == ".stream.json" || rest == ".schema") + .then(|| s.to_owned()) + }) + }) + .collect(); + candidates.extend( + self.storage + .list_dirs_relative(RelativePath::new(""), &None) + .await? + .into_iter() + .filter(|p| !p.starts_with('.') && !default_streams.contains(p)), + ); + Ok(candidates.into_iter().collect()) + } + pub fn absolute_url(&self, path: &RelativePath) -> object_store::path::Path { + object_store::path::Path::parse(path.as_str()).expect("valid document key") + } + pub async fn manifest_batch( + &self, + paths: &[String], + t: &Option, + ) -> Result, ObjectStorageError> { + let base = self.storage.absolute_url(RelativePath::new("")); + let normalized: Vec = paths + .iter() + .map(|p| { + p.strip_prefix(base.as_ref()) + .and_then(|p| p.strip_prefix('/')) + .unwrap_or(p) + .to_owned() + }) + .collect(); + let rows = self + .client() + .await? + .query( + "SELECT key,body FROM parseable_documents WHERE tenant=$1 AND key=ANY($2)", + &[&tenant(t), &normalized], + ) + .await?; + let bodies: std::collections::HashMap = rows + .into_iter() + .map(|r| (r.get(0), Bytes::from(r.get::<_, Vec>(1)))) + .collect(); + Ok(paths + .iter() + .zip(normalized) + .filter_map(|(original, key)| bodies.get(&key).map(|b| (original.clone(), b.clone()))) + .collect()) + } + pub async fn get_object( + &self, + path: &RelativePath, + t: &Option, + ) -> Result { + self.get_metadata_versioned(path, t) + .await + .map_err(err)? + .map(|r| r.bytes) + .ok_or_else(|| ObjectStorageError::NoSuchKey(path.to_string())) + } + pub async fn get_metadata_versioned( + &self, + path: &RelativePath, + t: &Option, + ) -> Result, ConditionalMetadataError> { + let base = self.storage.absolute_url(RelativePath::new("")); + let normalized = path + .as_str() + .strip_prefix(base.as_ref()) + .and_then(|p| p.strip_prefix('/')) + .unwrap_or(path.as_str()); + let c = self.client().await?; + Ok(c.query_opt( + "SELECT body,version FROM parseable_documents WHERE tenant=$1 AND key=$2", + &[&tenant(t), &normalized], + ) + .await + .map_err(err)? + .map(|r| VersionedMetadata { + bytes: Bytes::from(r.get::<_, Vec>(0)), + version: version(r.get(1)), + })) + } + pub async fn put_object( + &self, + path: &RelativePath, + body: Bytes, + t: &Option, + ) -> Result<(), ObjectStorageError> { + let c = self.client().await?; + c.execute("INSERT INTO parseable_documents(tenant,kind,key,body) VALUES($1,$2,$3,$4) ON CONFLICT(tenant,key) DO UPDATE SET body=EXCLUDED.body, version=nextval('parseable_document_version'),updated_at=now()", &[&tenant(t), &kind(path), &path.as_str(), &&body[..]]).await.map_err(err)?; + Ok(()) + } + pub async fn put_metadata_conditional( + &self, + path: &RelativePath, + body: Bytes, + condition: MetadataWriteCondition, + t: &Option, + ) -> Result { + let c = self.client().await?; + let row = match condition { + MetadataWriteCondition::Create => c.query_opt("INSERT INTO parseable_documents(tenant,kind,key,body) VALUES($1,$2,$3,$4) ON CONFLICT DO NOTHING RETURNING version", &[&tenant(t),&kind(path),&path.as_str(),&&body[..]]).await.map_err(err)?, + MetadataWriteCondition::Update(v) => { let v = v.version.and_then(|v| v.parse::().ok()).ok_or(ConditionalMetadataError::Conflict)?; + c.query_opt("UPDATE parseable_documents SET body=$3,version=nextval('parseable_document_version'),updated_at=now() WHERE tenant=$1 AND key=$2 AND version=$4 RETURNING version", &[&tenant(t),&path.as_str(),&&body[..],&v]).await.map_err(err)? } + }; + row.map(|r| version(r.get(0))) + .ok_or(ConditionalMetadataError::Conflict) + } + pub async fn delete_metadata_conditional( + &self, + path: &RelativePath, + v: MetadataVersion, + t: &Option, + ) -> Result<(), ConditionalMetadataError> { + let v = v + .version + .and_then(|v| v.parse::().ok()) + .ok_or(ConditionalMetadataError::Conflict)?; + let count = self + .client() + .await? + .execute( + "DELETE FROM parseable_documents WHERE tenant=$1 AND key=$2 AND version=$3", + &[&tenant(t), &path.as_str(), &v], + ) + .await + .map_err(err)?; + if count == 0 { + Err(ConditionalMetadataError::Conflict) + } else { + Ok(()) + } + } + pub async fn delete_object( + &self, + path: &RelativePath, + t: &Option, + ) -> Result<(), ObjectStorageError> { + self.client() + .await? + .execute( + "DELETE FROM parseable_documents WHERE tenant=$1 AND key=$2", + &[&tenant(t), &path.as_str()], + ) + .await + .map_err(err)?; + Ok(()) + } + async fn rows( + &self, + prefix: &str, + t: Option<&str>, + ) -> Result, ObjectStorageError> { + let prefix = prefix.trim_matches('/'); + Ok(self.client().await?.query("SELECT key,body FROM parseable_documents WHERE tenant=$2 AND ($1='' OR key=$1 OR (key >= $1 || '/' AND key < $1 || '0')) ORDER BY key", &[&prefix,&t.unwrap_or("")]).await?.into_iter().map(|r| (r.get(0),Bytes::from(r.get::<_,Vec>(1)))).collect()) + } + async fn keys(&self, prefix: &str, t: &str) -> Result, ObjectStorageError> { + Ok(self.client().await?.query("SELECT key FROM parseable_documents WHERE tenant=$1 AND ($2='' OR key=$2 OR (key >= $2 || '/' AND key < $2 || '0')) ORDER BY key", &[&t,&prefix]).await?.into_iter().map(|r|r.get(0)).collect()) + } + pub async fn list_streams( + &self, + t: &Option, + ) -> Result, ObjectStorageError> { + let prefix = if tenant(t).is_empty() { + String::new() + } else { + format!("{}/", tenant(t)) + }; + let keys = self + .client() + .await? + .query( + "SELECT key FROM parseable_documents WHERE tenant=$1 AND key LIKE '%/.stream/%'", + &[&tenant(t)], + ) + .await?; + let keys: std::collections::HashSet = keys.into_iter().map(|r| r.get(0)).collect(); + Ok(keys + .iter() + .filter(|k| k.ends_with("stream.json")) + .filter_map(|k| { + k.strip_prefix(&prefix)? + .split_once('/') + .map(|(s, _)| s.to_owned()) + }) + .filter(|s| !keys.contains(&format!("{prefix}{s}/.stream/.deleting"))) + .collect()) + } + pub async fn get_objects( + &self, + path: Option<&RelativePath>, + filter: Box bool + Send>, + t: &Option, + ) -> Result, ObjectStorageError> { + // Node registry objects are deployment-global; tenant is metrics context + // for this existing trait API, and is not part of its path. + let root = path.map(|p| p.as_str()).unwrap_or(""); + let scope = if root == ".parseable" { "" } else { tenant(t) }; + Ok(self + .rows(root, Some(scope)) + .await? + .into_iter() + .filter(|(p, _)| filter(p.rsplit('/').next().unwrap().to_owned())) + .map(|(_, b)| b) + .collect()) + } + pub async fn delete_prefix( + &self, + path: &RelativePath, + _: &Option, + ) -> Result<(), ObjectStorageError> { + if path.as_str().is_empty() || path.as_str().contains('/') { + return Err(err("tenant deletion requires one nonempty path component")); + } + self.delete_data(path, &None).await?; + self.client() + .await? + .execute( + "DELETE FROM parseable_documents WHERE tenant=$1", + &[&path.as_str()], + ) + .await + .map_err(err)?; + Ok(()) + } + pub async fn list_dirs_relative( + &self, + path: &RelativePath, + t: &Option, + ) -> Result, ObjectStorageError> { + let prefix = format!("{}/", path.as_str().trim_end_matches('/')); + Ok(self + .keys(path.as_str(), tenant(t)) + .await? + .into_iter() + .filter_map(|p| { + p.strip_prefix(&prefix) + .and_then(|p| p.split_once('/')) + .map(|(p, _)| p.to_owned()) + }) + .collect::>() + .into_iter() + .collect()) + } + pub async fn list_with_delimiter( + &self, + path: Option, + t: &Option, + ) -> Result { + let base = path.as_ref().map(|p| p.as_ref()).unwrap_or(""); + let prefix = if base.is_empty() { + String::new() + } else { + format!("{base}/") + }; + let mut dirs = std::collections::BTreeSet::new(); + let mut objects = Vec::new(); + for key in self.keys(base, tenant(t)).await? { + let Some(suffix) = key.strip_prefix(&prefix) else { + continue; + }; + if let Some((dir, _)) = suffix.split_once('/') { + dirs.insert( + object_store::path::Path::parse(format!("{prefix}{dir}")).map_err(err)?, + ); + } else { + objects.push(object_store::ObjectMeta { + location: object_store::path::Path::parse(key).map_err(err)?, + last_modified: chrono::Utc::now(), + size: 0, + e_tag: None, + version: None, + }); + } + } + Ok(object_store::ListResult { + common_prefixes: dirs.into_iter().collect(), + objects, + }) + } + pub async fn schema_objects( + &self, + stream: &str, + t: &Option, + ) -> Result, ObjectStorageError> { + let prefix = format!( + "{}{stream}/.stream", + if tenant(t).is_empty() { + String::new() + } else { + format!("{}/", tenant(t)) + } + ); + Ok(self + .rows(&prefix, Some(tenant(t))) + .await? + .into_iter() + .filter_map(|(p, b)| { + let name = p.rsplit('/').next()?.to_owned(); + name.contains(".schema").then_some((name, b)) + }) + .collect()) + } +} +pub(crate) fn kind(path: &RelativePath) -> &'static str { + let name = path.file_name().unwrap_or(""); + if name.ends_with("manifest.json") { + "manifest" + } else if name.ends_with("stream.json") { + "stream" + } else if name.contains(".schema") { + "schema" + } else { + "document" + } +} + +#[cfg(test)] +impl PostgresDocuments { + pub async fn verify_partition_contract(&self) { + // The SQL trigger fails precisely after data deletion, before row deletion. + let tenants = [ + Some("partition-a".to_owned()), + Some("partition-b".to_owned()), + ]; + let dates = ["date=2026-01-01", "date=2026-01-02"]; + for t in &tenants { + for date in dates { + let prefix = format!("{}/logs/{date}", t.as_ref().unwrap()); + self.put_object( + RelativePath::new(&format!("{prefix}/manifest.json")), + Bytes::from(prefix.clone()), + t, + ) + .await + .unwrap(); + self.storage + .put_object( + RelativePath::new(&format!("{prefix}/data.parquet")), + Bytes::from(prefix), + t, + ) + .await + .unwrap(); + } + } + let client = self.client().await.unwrap(); + client.batch_execute("CREATE FUNCTION fail_partition_delete() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF OLD.key = 'partition-a/logs/date=2026-01-01/manifest.json' THEN RAISE EXCEPTION 'injected partition row failure'; END IF; RETURN OLD; END $$; CREATE TRIGGER partition_delete_fault BEFORE DELETE ON parseable_documents FOR EACH ROW EXECUTE FUNCTION fail_partition_delete();").await.unwrap(); + let path = RelativePath::new("partition-a/logs/date=2026-01-01"); + let error = self.delete_partition(path, &tenants[0]).await.unwrap_err(); + assert!(error.to_string().contains("PostgreSQL"), "{error}"); + assert!( + self.storage + .get_object( + RelativePath::new("partition-a/logs/date=2026-01-01/data.parquet"), + &tenants[0] + ) + .await + .is_err() + ); + assert!( + self.get_object( + RelativePath::new("partition-a/logs/date=2026-01-01/manifest.json"), + &tenants[0] + ) + .await + .is_ok() + ); + client.batch_execute("DROP TRIGGER partition_delete_fault ON parseable_documents; DROP FUNCTION fail_partition_delete();").await.unwrap(); + drop(client); + self.delete_partition(path, &tenants[0]).await.unwrap(); + self.delete_partition(path, &tenants[0]).await.unwrap(); + for (i, t) in tenants.iter().enumerate() { + for (j, date) in dates.iter().enumerate() { + let prefix = format!("{}/logs/{date}", t.as_ref().unwrap()); + let manifest = self + .get_object(RelativePath::new(&format!("{prefix}/manifest.json")), t) + .await; + let data = self + .storage + .get_object(RelativePath::new(&format!("{prefix}/data.parquet")), t) + .await; + if i == 0 && j == 0 { + assert!(manifest.is_err()); + assert!(data.is_err()); + } else { + assert_eq!(manifest.unwrap(), Bytes::from(prefix.clone())); + assert_eq!(data.unwrap(), Bytes::from(prefix)); + } + } + self.delete_prefix(RelativePath::new(t.as_ref().unwrap()), t) + .await + .unwrap(); + } + } + pub async fn verify_deleted_stream_rows(&self) { + let count: i64 = self.client().await.unwrap().query_one("SELECT count(*) FROM parseable_documents WHERE starts_with(key,'contractlogs/') OR starts_with(key,'tenant-contract/contractlogs/')", &[]).await.unwrap().get(0); + assert_eq!(count, 0); + } + pub async fn verify_database_contract(&self, options: &PostgresOptions) { + for fault in [1, 2] { + let t = Some("failure-tenant".to_owned()); + let key = RelativePath::new("failure-tenant/retry/.stream/.stream.json"); + self.put_object(key, Bytes::from_static(b"{}"), &t) + .await + .unwrap(); + self.put_object( + RelativePath::new("retry/.stream/.stream.json"), + Bytes::from_static(b"other tenant"), + &None, + ) + .await + .unwrap(); + let data = RelativePath::new("failure-tenant/retry/date=2026-01-01/data.parquet"); + self.storage + .put_object(data, Bytes::from_static(b"data"), &t) + .await + .unwrap(); + self.deletion_fault + .store(fault, std::sync::atomic::Ordering::SeqCst); + assert!(self.delete_stream("retry", &t).await.is_err()); + assert!(self.is_stream_deleting("retry", &t).await.unwrap()); + assert_eq!(self.storage.get_object(data, &t).await.is_ok(), fault == 1); + let restarted = + Arc::new(Self::new(options, self.bucket.clone(), self.storage.clone()).unwrap()); + restarted.check(&None).await.unwrap(); + let restarted_store = super::postgres_metastore::PostgresMetastore { + storage: restarted.clone(), + }; + assert!( + !crate::metastore::metastore_traits::Metastore::list_streams(&restarted_store, &t) + .await + .unwrap() + .contains("retry") + ); + assert!(restarted.is_stream_deleting("retry", &t).await.unwrap()); + restarted.delete_stream("retry", &t).await.unwrap(); + assert!( + restarted + .rows("failure-tenant/retry", Some("failure-tenant")) + .await + .unwrap() + .is_empty() + ); + restarted + .put_object(key, Bytes::from_static(b"recreated"), &t) + .await + .unwrap(); + assert_eq!( + restarted.get_object(key, &t).await.unwrap(), + Bytes::from_static(b"recreated") + ); + self.deletion_fault + .store(0, std::sync::atomic::Ordering::SeqCst); + self.delete_stream("retry", &t).await.unwrap(); + assert!(!self.stream_exists("retry", &t).await.unwrap()); + assert!(!self.is_stream_deleting("retry", &t).await.unwrap()); + assert!(self.stream_exists("retry", &None).await.unwrap()); + self.delete_stream("retry", &None).await.unwrap(); + } + // Distinct raw tenant keys coexist, including SQL-only and bucket-only + // candidates. Enumeration order is unspecified: compare sets. + let baseline: std::collections::BTreeSet<_> = self + .list_tenant_candidates() + .await + .unwrap() + .into_iter() + .collect(); + for name in ["discovery-sql", "discovery-duplicate", "discovery-invalid"] { + self.put_object( + RelativePath::new(&format!("{name}/.parseable/.parseable.json")), + Bytes::from_static(b"{}"), + &Some(name.into()), + ) + .await + .unwrap(); + } + for name in ["discovery-data", "discovery-duplicate", ".discovery-hidden"] { + self.storage + .put_object( + RelativePath::new(&format!("{name}/data.parquet")), + Bytes::from_static(b"data"), + &None, + ) + .await + .unwrap(); + } + let candidates: std::collections::BTreeSet<_> = self + .list_tenant_candidates() + .await + .unwrap() + .into_iter() + .collect(); + assert_eq!( + candidates + .difference(&baseline) + .cloned() + .collect::>(), + [ + "discovery-sql", + "discovery-data", + "discovery-duplicate", + "discovery-invalid" + ] + .into_iter() + .map(str::to_owned) + .collect() + ); + for name in [ + "discovery-sql", + "discovery-data", + "discovery-duplicate", + "discovery-invalid", + ".discovery-hidden", + ] { + self.delete_prefix(RelativePath::new(name), &None) + .await + .unwrap(); + } + let client = self.client().await.unwrap(); + let started = std::time::Instant::now(); + assert!(client.query("SELECT pg_sleep(10)", &[]).await.is_err()); + assert!(started.elapsed() < Duration::from_secs(5)); + assert!(client.query("SELECT 1", &[]).await.is_ok()); + drop(client); + // Pool saturation is bounded and returned as an error. + let held = futures::future::join_all((0..4).map(|_| self.client())).await; + assert!(held.iter().all(Result::is_ok)); + let started = std::time::Instant::now(); + assert!(self.client().await.is_err()); + assert!(started.elapsed() < Duration::from_secs(3)); + drop(held); + let marker = RelativePath::new(".parseable/postgres-metastore-id"); + let paired = self.storage.get_object(marker, &None).await.unwrap(); + self.storage.delete_object(marker, &None).await.unwrap(); + let started = std::time::Instant::now(); + assert!(self.check_health(&None).await.is_err()); + assert!(started.elapsed() < Duration::from_secs(3)); + self.storage + .put_object(marker, Bytes::from_static(b"wrong-backend"), &None) + .await + .unwrap(); + let partial = Self::new(options, self.bucket.clone(), self.storage.clone()).unwrap(); + let error = partial.check(&None).await.unwrap_err(); + assert!( + error + .to_string() + .contains("different PostgreSQL backend UUID"), + "{error}" + ); + self.storage + .put_object(marker, paired, &None) + .await + .unwrap(); + partial.check(&None).await.unwrap(); + partial.check_health(&None).await.unwrap(); + let path = RelativePath::new("contract/concurrent.json"); + self.put_object(path, Bytes::from_static(b"initial"), &None) + .await + .unwrap(); + let original = self + .get_metadata_versioned(path, &None) + .await + .unwrap() + .unwrap(); + let writes = (0..12).map(|i| { + self.put_metadata_conditional( + path, + Bytes::from(i.to_string()), + MetadataWriteCondition::Update(original.version.clone()), + &None, + ) + }); + let results = futures::future::join_all(writes).await; + assert_eq!(results.iter().filter(|r| r.is_ok()).count(), 1); + let before_delete = self + .get_metadata_versioned(path, &None) + .await + .unwrap() + .unwrap() + .version; + self.delete_object(path, &None).await.unwrap(); + self.put_object(path, Bytes::new(), &None).await.unwrap(); + let recreated = self + .get_metadata_versioned(path, &None) + .await + .unwrap() + .unwrap() + .version; + assert!( + recreated.version.unwrap().parse::().unwrap() + > before_delete.version.unwrap().parse::().unwrap() + ); + self.delete_object(path, &None).await.unwrap(); + } +} + +impl PostgresDocuments { + pub(crate) async fn migrate( + &self, + store: Arc, + opts: &super::migration::MigrationOptions, + keys: &[super::migration::InventoryDocument], + ) -> anyhow::Result<()> { + use super::migration::{Action, BUSY, PAIR, digest, optional}; + use anyhow::ensure; + let action = opts.metadata_migration.unwrap(); + let mut client = self.client().await?; + // All remote I/O and validation happen before the import transaction. + let tx = &client; + if action == Action::Verify { + let exists: bool = tx + .query_one("SELECT to_regclass('parseable_documents') IS NOT NULL", &[]) + .await? + .get(0); + let bucket_keys: std::collections::BTreeSet<_> = keys + .iter() + .map(|(k, (t, _), _)| (t.as_str(), k.as_str())) + .collect(); + let mut differences = 0; + for (key, (tenant, kind), _) in keys { + let bucket = store.get_object(RelativePath::new(key), &None).await?; + let sql = if exists { + tx.query_opt( + "SELECT body FROM parseable_documents WHERE tenant=$1 AND key=$2", + &[tenant, key], + ) + .await? + .map(|r| r.get::<_, Vec>(0)) + } else { + None + }; + if sql.as_deref() != Some(bucket.as_ref()) { + println!( + "DIFFERENT tenant={tenant:?} kind={kind} key={key} bucket={} sql={}", + digest(&bucket), + sql.as_ref() + .map(|b| digest(b)) + .unwrap_or_else(|| "missing".into()) + ); + differences += 1; + } + } + if exists { + let mut last = (String::new(), String::new()); + loop { + let rows = tx.query("SELECT tenant,key,kind FROM parseable_documents WHERE (tenant COLLATE \"C\",key COLLATE \"C\") > ($1,$2) ORDER BY tenant COLLATE \"C\",key COLLATE \"C\" LIMIT 128", &[&last.0,&last.1]).await?; + if rows.is_empty() { + break; + } + for row in rows { + last = (row.get(0), row.get(1)); + if !bucket_keys.contains(&(last.0.as_str(), last.1.as_str())) { + println!( + "SQL ONLY tenant={:?} kind={} key={}", + last.0, + row.get::<_, String>(2), + last.1 + ); + differences += 1; + } + } + } + } + ensure!(differences == 0, "{differences} metadata differences"); + println!("representations match"); + return Ok(()); + } + let prior_busy = optional(store.as_ref(), BUSY).await?; + let initial_id = prior_busy + .as_ref() + .and_then(|b| std::str::from_utf8(b).ok()) + .and_then(|s| s.strip_prefix("Import:")) + .map(str::to_owned) + .unwrap_or_else(|| uuid::Uuid::new_v4().to_string()); + let bootstrap_tx = client.transaction().await?; + bootstrap_tx + .query("SELECT pg_advisory_xact_lock(8104050183641399916)", &[]) + .await?; + let (id, _, _) = self.bootstrap(&bootstrap_tx, &initial_id).await?; + bootstrap_tx.commit().await?; + let tx = &client; + let state = format!("{action:?}:{id}"); + let busy = optional(store.as_ref(), BUSY).await?; + ensure!( + busy.as_ref().is_none_or(|b| b.as_ref() == state.as_bytes()), + "another transfer owns the bucket migration marker" + ); + let pair = optional(store.as_ref(), PAIR).await?; + ensure!( + pair.as_ref().is_none_or(|b| b.as_ref() == id.as_bytes()), + "bucket paired with another database" + ); + let count: i64 = tx + .query_one("SELECT count(*) FROM parseable_documents", &[]) + .await? + .get(0); + if action == Action::Import { + ensure!( + count == 0 || busy.is_some(), + "refusing import into nonempty database" + ); + if count != 0 { + ensure!( + count as usize == keys.len(), + "resumed import inventory changed" + ); + } + } else { + ensure!( + pair.is_some() || busy.is_some(), + "export requires a paired bucket or interrupted export" + ); + // Preflight every destination before any overwrite. + let mut last = (String::new(), String::new()); + loop { + let rows = tx.query("SELECT tenant,key,body FROM parseable_documents WHERE (tenant COLLATE \"C\",key COLLATE \"C\") > ($1,$2) ORDER BY tenant COLLATE \"C\",key COLLATE \"C\" LIMIT 32", &[&last.0,&last.1]).await?; + if rows.is_empty() { + break; + } + for row in rows { + last = (row.get(0), row.get(1)); + let body: Vec = row.get(2); + self.validate_export_row(&client, &last.1, &Bytes::from(body.clone())) + .await?; + if let Some(old) = optional(store.as_ref(), &last.1).await? { + let baseline = self.import_digest(&client, &last.1).await?; + ensure!( + old.as_ref() == body + || baseline.as_deref() == Some(digest(&old).as_str()) + || opts.overwrite_bucket_changes_since_import, + "bucket changed since import: {}; after backup use --overwrite-bucket-changes-since-import", + last.1 + ); + } + } + } + // Bucket-only metadata would resurrect deleted SQL objects on downgrade. + for (key, (tenant, _), _) in keys { + if tx + .query_opt( + "SELECT 1 FROM parseable_documents WHERE tenant=$1 AND key=$2", + &[tenant, key], + ) + .await? + .is_none() + { + ensure!( + self.import_digest(&client, key).await?.as_deref() + == Some( + digest(&store.get_object(RelativePath::new(key), &None).await?) + .as_str() + ) + || opts.overwrite_bucket_changes_since_import, + "bucket-only metadata changed since import: {key}; use --overwrite-bucket-changes-since-import after backup" + ); + } + } + } + store + .put_metadata_conditional( + RelativePath::new(BUSY), + Bytes::from(state.clone()), + MetadataWriteCondition::Create, + &None, + ) + .await + .or_else(|e| { + if matches!(e, ConditionalMetadataError::Conflict) && busy.is_some() { + Ok(version(0)) + } else { + Err(e) + } + })?; + super::migration::failure_point(opts, "marker")?; + if action == Action::Import { + let spool = tempfile::tempdir_in(std::env::current_dir()?)?; + for (index, (key, _, _)) in keys.iter().enumerate() { + let body = store.get_object(RelativePath::new(key), &None).await?; + std::fs::write(spool.path().join(index.to_string()), &body)?; + } + super::migration::failure_point(opts, "spool")?; + let current = store.metadata_inventory().await?; + let current: Vec<_> = current + .into_iter() + .filter(|(k, _)| super::migration::document_scope(k).is_some()) + .collect(); + ensure!( + current.len() == keys.len() + && current + .iter() + .zip(keys) + .all(|((k, m), (old, _, om))| k == old && m == om), + "bucket metadata changed during import" + ); + let tx = client.transaction().await?; + tx.query("SELECT pg_advisory_xact_lock(8104050183641399916)", &[]) + .await?; + tx.batch_execute("CREATE TABLE IF NOT EXISTS parseable_import_digests (key text PRIMARY KEY, digest text NOT NULL)").await?; + for (index, batch) in keys.chunks(32).enumerate() { + let mut tenants = Vec::new(); + let mut kinds = Vec::new(); + let mut paths = Vec::new(); + let mut bodies = Vec::new(); + for (key, (tenant, kind), _) in batch { + let body = Bytes::from(std::fs::read( + spool.path().join((index * 32 + bodies.len()).to_string()), + )?); + tenants.push(tenant.clone()); + kinds.push(kind.to_string()); + paths.push(key.clone()); + bodies.push(body.to_vec()); + } + if count == 0 { + tx.execute("INSERT INTO parseable_documents(tenant,kind,key,body) SELECT * FROM unnest($1::text[],$2::text[],$3::text[],$4::bytea[])", &[&tenants,&kinds,&paths,&bodies]).await?; + } + for ((key, tenant), body) in paths.iter().zip(&tenants).zip(&bodies) { + tx.execute("INSERT INTO parseable_import_digests(key,digest) VALUES($1,$2) ON CONFLICT DO NOTHING", &[key, &digest(body)]).await?; + let reread: Vec = tx + .query_one( + "SELECT body FROM parseable_documents WHERE tenant=$1 AND key=$2", + &[tenant, key], + ) + .await? + .get(0); + ensure!(digest(body) == digest(&reread), "SQL digest differs: {key}"); + } + eprintln!( + "verified import {}/{}", + ((index + 1) * 32).min(keys.len()), + keys.len() + ); + } + tx.execute("INSERT INTO parseable_transfer(state) VALUES('ready') ON CONFLICT(singleton) DO UPDATE SET state='ready'", &[]).await?; + super::migration::failure_point(opts, "import-before-commit")?; + tx.commit().await?; + super::migration::failure_point(opts, "import-commit")?; + store + .put_metadata_conditional( + RelativePath::new(PAIR), + Bytes::from(id.clone()), + MetadataWriteCondition::Create, + &None, + ) + .await + .or_else(|e| { + if matches!(e, ConditionalMetadataError::Conflict) { + Ok(version(0)) + } else { + Err(e) + } + })?; + ensure!( + optional(store.as_ref(), PAIR).await?.unwrap().as_ref() == id.as_bytes(), + "pair mismatch" + ); + super::migration::failure_point(opts, "paired")?; + } else { + tx.execute("INSERT INTO parseable_transfer(state) VALUES('exported') ON CONFLICT(singleton) DO UPDATE SET state='exported'", &[]).await?; + super::migration::failure_point(opts, "export-state")?; + let mut last = (String::new(), String::new()); + loop { + let rows = client.query("SELECT tenant,key,body FROM parseable_documents WHERE (tenant COLLATE \"C\",key COLLATE \"C\") > ($1,$2) ORDER BY tenant COLLATE \"C\",key COLLATE \"C\" LIMIT 32", &[&last.0,&last.1]).await?; + if rows.is_empty() { + break; + } + for row in rows { + last = (row.get(0), row.get(1)); + let body = Bytes::from(row.get::<_, Vec>(2)); + store + .put_object(RelativePath::new(&last.1), body.clone(), &None) + .await?; + ensure!( + store.get_object(RelativePath::new(&last.1), &None).await? == body, + "export verification failed: {}", + last.1 + ); + super::migration::failure_point(opts, "object-write")?; + eprintln!("verified export {}", last.1); + } + } + for (key, (tenant, _), _) in keys { + if client + .query_opt( + "SELECT 1 FROM parseable_documents WHERE tenant=$1 AND key=$2", + &[tenant, key], + ) + .await? + .is_none() + { + store.delete_object(RelativePath::new(key), &None).await?; + super::migration::failure_point(opts, "bucket-delete")?; + } + } + // Validate stream references only after all destination documents exist. + let mut last_key = String::new(); + loop { + let rows = client.query("SELECT key,body FROM parseable_documents WHERE key COLLATE \"C\" > $1 ORDER BY key COLLATE \"C\" LIMIT 32", &[&last_key]).await?; + if rows.is_empty() { + break; + } + for row in rows { + last_key = row.get(0); + super::migration::validate( + store.as_ref(), + &last_key, + &Bytes::from(row.get::<_, Vec>(1)), + ) + .await?; + } + } + if optional(store.as_ref(), PAIR).await?.is_some() { + store.delete_object(RelativePath::new(PAIR), &None).await?; + } + super::migration::failure_point(opts, "pair-removed")?; + } + store.delete_object(RelativePath::new(BUSY), &None).await?; + Ok(()) + } +} + +impl PostgresDocuments { + pub(crate) async fn migration_preflight( + &self, + store: &dyn ObjectStorage, + opts: &super::migration::MigrationOptions, + ) -> anyhow::Result<()> { + use super::migration::{Action, BUSY, PAIR, optional}; + use anyhow::ensure; + let client = self.client().await?; + let busy = optional(store, BUSY).await?; + let pair = optional(store, PAIR).await?; + let exists: bool = client + .query_one("SELECT to_regclass('parseable_backend') IS NOT NULL", &[]) + .await? + .get(0); + if !exists { + ensure!( + self.expected_id.is_none(), + "pinned backend id requires an initialized database" + ); + if let Some(marker) = &busy { + ensure!( + opts.metadata_migration == Some(Action::Import) + && std::str::from_utf8(marker).is_ok_and(|s| s + .strip_prefix("Import:") + .is_some_and(|id| uuid::Uuid::parse_str(id).is_ok())), + "another transfer owns migration marker" + ); + } + ensure!( + pair.is_none(), + "paired bucket requires its original database" + ); + ensure!( + opts.metadata_migration != Some(Action::Export), + "export requires an imported database" + ); + return Ok(()); + } + let row = client + .query_one( + "SELECT id,bucket,schema_version,protocol_version FROM parseable_backend", + &[], + ) + .await?; + let id: String = row.get(0); + ensure!( + row.get::<_, String>(1) == self.bucket + && (0..=2).contains(&row.get::<_, i32>(2)) + && row.get::<_, i32>(3) == 1, + "backend identity/version mismatch" + ); + ensure!( + self.expected_id.as_ref().is_none_or(|v| v == &id), + "pinned backend mismatch" + ); + ensure!( + pair.as_ref().is_none_or(|v| v.as_ref() == id.as_bytes()), + "bucket paired with another database" + ); + ensure!( + busy.as_ref() + .is_none_or(|v| v.as_ref() + == format!("{:?}:{id}", opts.metadata_migration.unwrap()).as_bytes()), + "another transfer owns migration marker" + ); + if opts.metadata_migration == Some(Action::Export) { + ensure!( + pair.is_some() || busy.is_some(), + "export requires paired bucket or interrupted export" + ); + } + let docs: bool = client + .query_one("SELECT to_regclass('parseable_documents') IS NOT NULL", &[]) + .await? + .get(0); + if docs && opts.metadata_migration == Some(Action::Import) && !opts.abort { + let count: i64 = client + .query_one("SELECT count(*) FROM parseable_documents", &[]) + .await? + .get(0); + ensure!( + count == 0 || busy.is_some(), + "refusing import into nonempty database" + ); + } + Ok(()) + } + + async fn import_digest( + &self, + client: &super::postgres_client::Client, + key: &str, + ) -> anyhow::Result> { + let exists: bool = client + .query_one( + "SELECT to_regclass('parseable_import_digests') IS NOT NULL", + &[], + ) + .await? + .get(0); + if !exists { + return Ok(None); + } + Ok(client + .query_opt( + "SELECT digest FROM parseable_import_digests WHERE key=$1", + &[&key], + ) + .await? + .map(|r| r.get(0))) + } + + async fn validate_export_row( + &self, + client: &super::postgres_client::Client, + key: &str, + body: &Bytes, + ) -> anyhow::Result<()> { + use anyhow::ensure; + ensure!( + !key.ends_with(".deleting") && !key.contains(".tombstones/"), + "unfinished deletion: {key}; start PostgreSQL mode and retry DELETE before export" + ); + ensure!( + super::migration::document_scope(key).is_some(), + "unknown SQL document: {key}" + ); + if key.ends_with("manifest.json") { + crate::catalog::manifest::decode_manifest(body)?; + } else if key.ends_with("stream.json") { + for manifest_path in super::migration::stream_references(body)? { + let base = self.storage.absolute_url(RelativePath::new("")); + let normalized = manifest_path + .strip_prefix(base.as_ref()) + .and_then(|p| p.strip_prefix('/')) + .unwrap_or(&manifest_path); + let row = client + .query_opt( + "SELECT body FROM parseable_documents WHERE key=$1", + &[&normalized], + ) + .await?; + let row = row.ok_or_else(|| { + anyhow::anyhow!("{key}: missing SQL manifest {}", manifest_path) + })?; + crate::catalog::manifest::decode_manifest(&Bytes::from(row.get::<_, Vec>(0)))?; + } + } else if key.contains(".schema") { + serde_json::from_slice::(body)?; + } else { + serde_json::from_slice::(body)?; + } + Ok(()) + } + + pub(crate) async fn abort_migration( + &self, + store: &dyn ObjectStorage, + action: super::migration::Action, + ) -> anyhow::Result<()> { + use super::migration::{Action, BUSY, optional}; + use anyhow::ensure; + let client = self.client().await?; + let exists: bool = client + .query_one("SELECT to_regclass('parseable_transfer') IS NOT NULL", &[]) + .await? + .get(0); + let state = if exists { + client + .query_opt("SELECT state FROM parseable_transfer WHERE singleton", &[]) + .await? + .map(|r| r.get::<_, String>(0)) + } else { + None + }; + match action { + Action::Import => { + let docs: bool = client + .query_one("SELECT to_regclass('parseable_documents') IS NOT NULL", &[]) + .await? + .get(0); + if docs { + let count: i64 = client + .query_one("SELECT count(*) FROM parseable_documents", &[]) + .await? + .get(0); + ensure!( + count == 0, + "SQL contains committed documents; complete import and export instead of aborting" + ); + } + ensure!( + state.is_none(), + "import committed; complete import, then export to return to bucket mode" + ); + } + Action::Export => { + ensure!( + state.as_deref() == Some("ready"), + "export has begun writing; rerun export to finish; abort is unsafe" + ); + } + Action::Verify => anyhow::bail!("verify cannot be aborted"), + } + if optional(store, BUSY).await?.is_some() { + store.delete_object(RelativePath::new(BUSY), &None).await?; + } + println!( + "aborted safely; start {} mode", + if action == Action::Import { + "object-store" + } else { + "PostgreSQL" + } + ); + Ok(()) + } +} diff --git a/src/metastore/metastores/postgres_metastore.rs b/src/metastore/metastores/postgres_metastore.rs new file mode 100644 index 000000000..072c9db6b --- /dev/null +++ b/src/metastore/metastores/postgres_metastore.rs @@ -0,0 +1,1666 @@ +super::document_metastore::document_metastore!( + PostgresMetastore, + super::postgres_documents::PostgresDocuments, + false, + { + async fn manifest_batch( + &self, + paths: &[String], + tenant: &Option, + ) -> Result>>, MetastoreError> { + let rows = self.storage.manifest_batch(paths, tenant).await?; + let mut result: HashMap> = + paths.iter().map(|p| (p.clone(), None)).collect(); + for (key, body) in rows { + result.insert(key, Some(decode_manifest_blocking(body).await?)); + } + Ok(Some(result)) + } + async fn metadata_dates( + &self, + stream: &str, + tenant: &Option, + ) -> Result, MetastoreError> { + let path = crate::storage::object_storage::stream_data_path(stream, &[], tenant); + Ok(self.storage.list_dirs_relative(&path, tenant).await?) + } + async fn check_health(&self, tenant_id: &Option) -> Result<(), MetastoreError> { + Ok(self.storage.check_health(tenant_id).await?) + } + + async fn read_latency(&self, tenant_id: &Option) -> std::time::Duration { + self.storage.read_latency(tenant_id).await + } + + async fn stream_exists( + &self, + stream_name: &str, + tenant_id: &Option, + ) -> Result { + Ok(self.storage.stream_exists(stream_name, tenant_id).await?) + } + + async fn is_stream_deleting( + &self, + stream_name: &str, + tenant_id: &Option, + ) -> Result { + Ok(self + .storage + .is_stream_deleting(stream_name, tenant_id) + .await?) + } + + async fn delete_stream( + &self, + stream_name: &str, + tenant_id: &Option, + ) -> Result<(), MetastoreError> { + Ok(self.storage.delete_stream(stream_name, tenant_id).await?) + } + + async fn delete_stream_partition( + &self, + stream_name: &str, + date: &str, + tenant_id: &Option, + ) -> Result<(), MetastoreError> { + let path = + crate::storage::object_storage::stream_data_path(stream_name, &[date], tenant_id); + Ok(self.storage.delete_partition(&path, tenant_id).await?) + } + + async fn list_tenant_candidates(&self) -> Result, MetastoreError> { + Ok(self.storage.list_tenant_candidates().await?) + } + } +); + +// Parse tuning as text first: whether it is meaningful depends on the parsed URL, +// including CLI overrides of environment values. Do not inspect process argv. +#[derive(clap::Args)] +struct PostgresArguments { + /// Opt in to PostgreSQL metadata. Never falls back to bucket metadata. + #[arg(long, env = "P_METASTORE_POSTGRES_URL", hide_env_values = true)] + pub metastore_postgres_url: Option, + #[arg(long, env = "P_METASTORE_POSTGRES_POOL_SIZE", default_value = "16")] + pub metastore_postgres_pool_size: String, + #[arg( + long, + env = "P_METASTORE_POSTGRES_CONNECT_TIMEOUT_MS", + default_value = "5000" + )] + pub metastore_postgres_connect_timeout_ms: String, + #[arg( + long, + env = "P_METASTORE_POSTGRES_STATEMENT_TIMEOUT_MS", + default_value = "10000" + )] + pub metastore_postgres_statement_timeout_ms: String, + /// Optionally pin a previously initialized backend UUID. + #[arg(long, env = "P_METASTORE_POSTGRES_ID")] + pub metastore_postgres_id: Option, + /// Optional PEM CA bundle. TLS mode is controlled by sslmode in the URL. + #[arg(long, env = "P_METASTORE_POSTGRES_CA")] + pub metastore_postgres_ca: Option, +} + +#[derive(Clone)] +pub struct PostgresOptions { + pub metastore_postgres_url: Option, + pub metastore_postgres_pool_size: u32, + pub metastore_postgres_connect_timeout_ms: u64, + pub metastore_postgres_statement_timeout_ms: u64, + pub metastore_postgres_id: Option, + pub metastore_postgres_ca: Option, +} +impl clap::Args for PostgresOptions { + fn augment_args(cmd: clap::Command) -> clap::Command { + PostgresArguments::augment_args(cmd) + } + fn augment_args_for_update(cmd: clap::Command) -> clap::Command { + PostgresArguments::augment_args_for_update(cmd) + } +} +impl TryFrom for PostgresOptions { + type Error = clap::Error; + fn try_from(raw: PostgresArguments) -> Result { + let mut options = Self { + metastore_postgres_url: raw.metastore_postgres_url.filter(|v| !v.trim().is_empty()), + metastore_postgres_id: raw.metastore_postgres_id, + metastore_postgres_ca: raw.metastore_postgres_ca, + ..Default::default() + }; + fn positive( + value: &str, + name: &str, + ) -> Result { + value + .parse::() + .ok() + .filter(|v| *v != T::default()) + .ok_or_else(|| { + clap::Error::raw( + clap::error::ErrorKind::ValueValidation, + format!("{name} must be a positive integer"), + ) + }) + } + if options.enabled() { + options.metastore_postgres_pool_size = + positive(&raw.metastore_postgres_pool_size, "PostgreSQL pool size")?; + options.metastore_postgres_connect_timeout_ms = positive( + &raw.metastore_postgres_connect_timeout_ms, + "PostgreSQL connect timeout", + )?; + options.metastore_postgres_statement_timeout_ms = positive( + &raw.metastore_postgres_statement_timeout_ms, + "PostgreSQL statement timeout", + )?; + } + Ok(options) + } +} +impl clap::FromArgMatches for PostgresOptions { + fn from_arg_matches(matches: &clap::ArgMatches) -> Result { + PostgresArguments::from_arg_matches(matches)?.try_into() + } + fn update_from_arg_matches(&mut self, matches: &clap::ArgMatches) -> Result<(), clap::Error> { + let mut raw = PostgresArguments { + metastore_postgres_url: self.metastore_postgres_url.clone(), + metastore_postgres_pool_size: self.metastore_postgres_pool_size.to_string(), + metastore_postgres_connect_timeout_ms: self + .metastore_postgres_connect_timeout_ms + .to_string(), + metastore_postgres_statement_timeout_ms: self + .metastore_postgres_statement_timeout_ms + .to_string(), + metastore_postgres_id: self.metastore_postgres_id.clone(), + metastore_postgres_ca: self.metastore_postgres_ca.clone(), + }; + raw.update_from_arg_matches(matches)?; + *self = raw.try_into()?; + Ok(()) + } +} + +impl std::fmt::Debug for PostgresOptions { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("PostgresOptions") + .field("enabled", &self.enabled()) + .finish_non_exhaustive() + } +} +impl PostgresOptions { + pub fn enabled(&self) -> bool { + self.metastore_postgres_url + .as_deref() + .is_some_and(|v| !v.trim().is_empty()) + } +} +impl Default for PostgresOptions { + fn default() -> Self { + Self { + metastore_postgres_url: None, + metastore_postgres_pool_size: 16, + metastore_postgres_connect_timeout_ms: 5000, + metastore_postgres_statement_timeout_ms: 10000, + metastore_postgres_id: None, + metastore_postgres_ca: None, + } + } +} + +/// Construct offline transfer using the same provider and SQL options as the server. +pub async fn run_migration( + provider: &dyn crate::storage::ObjectStorageProvider, + options: &crate::cli::Options, +) -> anyhow::Result<()> { + anyhow::ensure!( + options.postgres.enabled(), + "P_METASTORE_POSTGRES_URL is required" + ); + anyhow::ensure!( + std::env::var_os("P_STAGING_DIR").is_some() + || std::env::args() + .any(|a| a == "--local-staging-path" || a.starts_with("--local-staging-path=")), + "migration requires explicit P_STAGING_DIR or --local-staging-path" + ); + // Staging metadata alone is normal; event buffers must have uploaded first. + let mut dirs = vec![options.local_staging_path.clone()]; + while let Some(dir) = dirs.pop() { + if !dir.exists() { + continue; + } + for entry in std::fs::read_dir(dir)? { + let path = entry?.path(); + if path.is_dir() { + dirs.push(path); + } else if path + .extension() + .is_some_and(|e| e == "arrows" || e == "parquet" || e == "arrow" || e == "part") + { + anyhow::bail!("unsynced staging data: {}", path.display()); + } + } + } + let storage = provider.construct_client(); + let db = super::postgres_documents::PostgresDocuments::new( + &options.postgres, + provider.get_endpoint(), + storage.clone(), + )?; + super::migration::run(storage, &db, &options.migration).await +} + +#[cfg(test)] +mod contract_tests { + use super::*; + use crate::{ + metastore::metastore_traits::*, + storage::{LocalFS, ObjectStorage}, + }; + use bytes::Bytes; + use std::sync::Arc; + + fn test_dir() -> tempfile::TempDir { + let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join(".postgres-smoke/unit"); + std::fs::create_dir_all(&root).unwrap(); + tempfile::tempdir_in(root).unwrap() + } + + #[derive(serde::Serialize)] + struct Document { + id: String, + path: String, + value: String, + } + impl MetastoreObject for Document { + fn get_object_path(&self) -> String { + self.path.clone() + } + fn get_object_id(&self) -> String { + self.id.clone() + } + } + fn initialize_context() { + fn factory() -> crate::parseable::Parseable { + use crate::storage::object_storage::ObjectStorageProvider; + let startup_root = std::env::var("P_CONTRACT_STARTUP_ROOT").ok(); + let root = startup_root + .as_ref() + .map(std::path::PathBuf::from) + .or_else(|| { + std::env::var("P_CONTRACT_CONTEXT_ROOT") + .ok() + .map(std::path::PathBuf::from) + }) + .unwrap_or_else(|| test_dir().path().to_owned()); + let provider = Arc::new(crate::storage::FSConfig { root }); + let storage = provider.construct_client(); + let mut options = crate::cli::Options::default(); + let metastore: Arc = if startup_root.is_some() { + options.multi_tenancy = true; + options.postgres.metastore_postgres_url = + Some(std::env::var("P_CONTRACT_STARTUP_URL").unwrap()); + options.postgres.metastore_postgres_statement_timeout_ms = 500; + Arc::new(PostgresMetastore { + storage: Arc::new( + super::super::postgres_documents::PostgresDocuments::new( + &options.postgres, + std::env::var("P_CONTRACT_STARTUP_BUCKET").unwrap(), + storage.clone(), + ) + .unwrap(), + ), + }) + } else { + Arc::new(super::super::object_store_metastore::ObjectStoreMetastore { + storage: storage.clone(), + }) + }; + let context = crate::parseable::Parseable::new( + options, + #[cfg(feature = "kafka")] + ::parse_from([ + "contract", + ]), + provider, + metastore, + storage, + ); + if startup_root.is_none() { + *context.tenants.write().unwrap() = vec![String::new(), "tenant-contract".into()]; + } + context + } + let _ = crate::parseable::CONTRACT_PARSEABLE_FACTORY.set(factory); + once_cell::sync::Lazy::force(&crate::parseable::PARSEABLE); + } + async fn contract(store: Arc, sql: bool) { + initialize_context(); + store.initiate_connection().await.unwrap(); + for tenant in [None, Some("tenant-contract".to_owned())] { + let root = tenant.as_ref().map(|t| format!("{t}/")).unwrap_or_default(); + let object = Document { + id: ulid::Ulid::new().to_string(), + path: format!("{root}.users/owner/dashboards/space %_é'.json"), + value: format!("faithful bytes-{root}"), + }; + store.put_dashboard(&object, &tenant).await.unwrap(); + let dashboards = store.get_tenant_dashboards(&tenant).await.unwrap(); + assert_eq!( + dashboards, + vec![crate::storage::object_storage::to_bytes(&object)] + ); + let read = store + .get_dashboard_versioned(&object.path, &tenant) + .await + .unwrap() + .unwrap(); + assert_eq!(read.bytes, dashboards[0]); + assert!(matches!( + store + .put_dashboard_conditional( + &object.path, + Bytes::new(), + MetadataWriteCondition::Create, + &tenant + ) + .await, + Err(ConditionalMetadataError::Conflict) + )); + let next = store + .put_dashboard_conditional( + &object.path, + Bytes::from_static(b"new"), + MetadataWriteCondition::Update(read.version.clone()), + &tenant, + ) + .await + .unwrap(); + assert!(matches!( + store + .put_dashboard_conditional( + &object.path, + Bytes::new(), + MetadataWriteCondition::Update(read.version.clone()), + &tenant + ) + .await, + Err(ConditionalMetadataError::Conflict) + )); + assert!(matches!( + store + .delete_dashboard_conditional(&object.path, read.version.clone(), &tenant) + .await, + Err(ConditionalMetadataError::Conflict) + )); + store + .delete_dashboard_conditional(&object.path, next, &tenant) + .await + .unwrap(); + assert!( + store + .get_dashboard_versioned(&object.path, &tenant) + .await + .unwrap() + .is_none() + ); + // Concurrent creates have exactly one winner. + let writes = (0..12).map(|_| { + store.put_dashboard_conditional( + &object.path, + Bytes::from_static(b"{}"), + MetadataWriteCondition::Create, + &tenant, + ) + }); + let results = futures::future::join_all(writes).await; + assert_eq!(results.iter().filter(|r| r.is_ok()).count(), 1); + assert_eq!( + results + .iter() + .filter(|r| matches!(r, Err(ConditionalMetadataError::Conflict))) + .count(), + 11 + ); + let recreated = store + .get_dashboard_versioned(&object.path, &tenant) + .await + .unwrap() + .unwrap(); + assert!(matches!( + store + .put_dashboard_conditional( + &object.path, + Bytes::from_static(b"stale"), + MetadataWriteCondition::Update(read.version.clone()), + &tenant + ) + .await, + Err(ConditionalMetadataError::Conflict) + )); + let updates = (0..12).map(|i| { + store.put_dashboard_conditional( + &object.path, + Bytes::from(i.to_string()), + MetadataWriteCondition::Update(recreated.version.clone()), + &tenant, + ) + }); + let updates = futures::future::join_all(updates).await; + assert_eq!(updates.iter().filter(|r| r.is_ok()).count(), 1); + assert_eq!( + updates + .iter() + .filter(|r| matches!(r, Err(ConditionalMetadataError::Conflict))) + .count(), + 11 + ); + let winner = updates.iter().position(Result::is_ok).unwrap(); + assert_eq!( + store + .get_dashboard_versioned(&object.path, &tenant) + .await + .unwrap() + .unwrap() + .bytes, + Bytes::from(winner.to_string()) + ); + store.delete_dashboard(&object, &tenant).await.unwrap(); + assert!( + store + .get_tenant_dashboards(&tenant) + .await + .unwrap() + .is_empty() + ); + store.put_keystone(&object, &tenant).await.unwrap(); + store.put_conversation(&object, &tenant).await.unwrap(); + let objects = store + .get_objects(&format!("{root}.keystone"), &tenant) + .await + .unwrap(); + assert_eq!( + objects, + vec![crate::storage::object_storage::to_bytes(&object); 2] + ); + store.delete_keystone(&object, &tenant).await.unwrap(); + store.delete_conversation(&object, &tenant).await.unwrap(); + store.put_llmconfig(&object, &tenant).await.unwrap(); + store.delete_llmconfig(&object, &tenant).await.unwrap(); + store.put_target(&object, &tenant).await.unwrap(); + store.delete_target(&object, &tenant).await.unwrap(); + store.put_chat(&object, &tenant).await.unwrap(); + store.delete_chat(&object, &tenant).await.unwrap(); + assert!( + store + .get_alert_state_entry(&ulid::Ulid::new(), &tenant) + .await + .unwrap() + .is_none() + ); + assert!(store.get_mttr_history(&tenant).await.unwrap().is_none()); + assert!( + store + .get_all_stream_jsons( + "absent", + Some(crate::option::Mode::Query), + &tenant, + false + ) + .await + .is_err() + ); + let directory_marker = Document { + id: object.id.clone(), + path: format!("{root}absent/.stream/placeholder"), + value: String::new(), + }; + store + .put_dashboard(&directory_marker, &tenant) + .await + .unwrap(); + store + .delete_dashboard(&directory_marker, &tenant) + .await + .unwrap(); + assert!( + store + .get_all_stream_jsons( + "absent", + Some(crate::option::Mode::Ingest), + &tenant, + false + ) + .await + .unwrap() + .is_empty() + ); + assert!( + store + .get_manifest( + "absent", + chrono::Utc::now(), + chrono::Utc::now(), + Some(format!("{root}absent/date=2026-01-01/manifest.json")), + &tenant + ) + .await + .unwrap() + .is_none() + ); + store.put_alert(&object, &tenant).await.unwrap(); + store.delete_alert(&object, &tenant).await.unwrap(); + store.delete_alert(&object, &tenant).await.unwrap(); + assert!( + store + .get_stream_json("absent", true, &tenant, false) + .await + .is_err() + ); + assert!( + store + .get_parseable_metadata(&tenant) + .await + .unwrap() + .is_none() + ); + store + .put_parseable_metadata(&object, &tenant) + .await + .unwrap(); + assert_eq!( + store + .get_parseable_metadata(&tenant) + .await + .unwrap() + .unwrap(), + crate::storage::object_storage::to_bytes(&object) + ); + } + for tenant in [None, Some("tenant-contract".to_owned())] { + let root = tenant.as_ref().map(|t| format!("{t}/")).unwrap_or_default(); + let schema = arrow_schema::Schema::new(vec![arrow_schema::Field::new( + if tenant.is_some() { + "named_value" + } else { + "default_value" + }, + arrow_schema::DataType::Int64, + false, + )]); + store + .put_schema(schema.clone(), "contractlogs", &tenant) + .await + .unwrap(); + assert_eq!( + serde_json::from_slice::( + &store.get_schema("contractlogs", &tenant).await.unwrap() + ) + .unwrap(), + schema + ); + assert_eq!( + store + .get_all_schemas("contractlogs", &tenant) + .await + .unwrap(), + vec![schema.clone()] + ); + assert_eq!( + store + .refresh_schema("contractlogs", &tenant, ".schema") + .await + .unwrap(), + schema + ); + if tenant.is_none() { + assert_eq!( + store + .get_all_schemas("contractlogs", &Some(String::new())) + .await + .unwrap(), + vec![schema.clone()] + ); + assert_eq!( + store + .refresh_schema("contractlogs", &Some(String::new()), ".schema") + .await + .unwrap(), + schema + ); + } + for field in ["ingestor_a", "ingestor_b"] { + let path = format!("{root}contractlogs/.stream/.{field}.schema"); + let fragment = arrow_schema::Schema::new(vec![arrow_schema::Field::new( + field, + arrow_schema::DataType::Utf8, + true, + )]); + store + .put_dashboard_conditional( + &path, + crate::storage::object_storage::to_bytes(&fragment), + MetadataWriteCondition::Create, + &tenant, + ) + .await + .unwrap(); + } + let merged = store + .refresh_schema("contractlogs", &tenant, ".schema") + .await + .unwrap(); + let names = |s: &arrow_schema::Schema| { + s.fields() + .iter() + .map(|f| f.name().clone()) + .collect::>() + }; + let expected_names = [ + schema.fields()[0].name().as_str(), + "ingestor_a", + "ingestor_b", + ] + .into_iter() + .map(str::to_owned) + .collect(); + assert_eq!(names(&merged), expected_names); + let persisted: arrow_schema::Schema = + serde_json::from_slice(&store.get_schema("contractlogs", &tenant).await.unwrap()) + .unwrap(); + assert_eq!(names(&persisted), names(&merged)); + let metadata = crate::storage::ObjectStoreFormat::default(); + store + .put_stream_json(&metadata, "contractlogs", &tenant) + .await + .unwrap(); + assert_eq!( + store + .get_stream_json("contractlogs", false, &tenant, false) + .await + .unwrap(), + crate::storage::object_storage::to_bytes(&metadata) + ); + assert_eq!( + store + .get_all_stream_jsons("contractlogs", None, &tenant, false) + .await + .unwrap(), + vec![crate::storage::object_storage::to_bytes(&metadata)] + ); + if sql { + assert!( + store + .list_streams(&tenant) + .await + .unwrap() + .contains("contractlogs") + ); + } + // LocalFS listing uses the process-global provider; covered by the + // separate regression so this defect cannot suppress all CRUD checks. + let lower = chrono::Utc::now(); + let upper = lower; + let manifest = crate::catalog::manifest::Manifest { + files: (0..10_000) + .map(|i| crate::catalog::manifest::File { + file_path: format!( + "{root}contractlogs/date=2026-01-01/space %_é'/{i}.parquet" + ), + num_rows: i, + file_size: 1234, + ..Default::default() + }) + .collect(), + ..Default::default() + }; + store + .put_manifest(&manifest, "contractlogs", lower, upper, &tenant) + .await + .unwrap(); + assert_eq!( + serde_json::to_value( + store + .get_manifest("contractlogs", lower, upper, None, &tenant) + .await + .unwrap() + .unwrap() + ) + .unwrap(), + serde_json::to_value(&manifest).unwrap() + ); + assert!( + store + .get_manifest_path("contractlogs", lower, upper, &tenant) + .await + .unwrap() + .ends_with("manifest.json") + ); + let listing = store + .get_manifest_files_for_dates( + "contractlogs", + &tenant, + &[format!("date={}", lower.format("%Y-%m-%d"))], + ) + .await; + if sql { + assert_eq!( + serde_json::to_value(listing.unwrap()).unwrap(), + serde_json::json!({ (format!("date={}", lower.format("%Y-%m-%d"))): [manifest] }) + ); + } else { + assert!(listing.is_err()); + } // LocalFS has no delimiter LIST. + assert!(store.stream_exists("contractlogs", &tenant).await.unwrap()); + assert!( + !store + .is_stream_deleting("contractlogs", &tenant) + .await + .unwrap() + ); + assert!(store.read_latency(&tenant).await < std::time::Duration::from_secs(5)); + store.check_health(&tenant).await.unwrap(); + let candidates = store.list_tenant_candidates().await.unwrap(); + if tenant.is_some() { + assert!(candidates.contains(&"tenant-contract".to_owned())); + } + + store + .delete_manifest("contractlogs", lower, upper, &tenant) + .await + .unwrap(); + assert!( + store + .get_manifest("contractlogs", lower, upper, None, &tenant) + .await + .unwrap() + .is_none() + ); + store + .delete_stream_partition( + "contractlogs", + &format!("date={}", lower.format("%Y-%m-%d")), + &tenant, + ) + .await + .unwrap(); + let state = crate::alerts::alert_structs::AlertStateEntry::new( + ulid::Ulid::new(), + crate::alerts::alert_enums::AlertState::default(), + tenant.clone(), + ); + store.put_alert_state(&state, &tenant).await.unwrap(); + assert!( + store + .get_alert_state_entry(&state.alert_id, &tenant) + .await + .unwrap() + .is_some() + ); + // Preserve the existing default-state directory mismatch. + let placeholder = Document { + id: "placeholder".into(), + path: format!("{}/.alerts/placeholder", crate::parseable::DEFAULT_TENANT), + value: String::new(), + }; + store.put_dashboard(&placeholder, &None).await.unwrap(); + let states = store.get_alert_states(&tenant).await.unwrap(); + assert_eq!(states.len(), usize::from(tenant.is_some())); + store.delete_alert_state(&state, &tenant).await.unwrap(); + assert!( + store + .get_alert_state_entry(&state.alert_id, &tenant) + .await + .unwrap() + .is_none() + ); + let history = crate::alerts::alert_structs::MTTRHistory { + daily_stats: vec![], + tenant_id: tenant.clone(), + }; + store.put_mttr_history(&history, &tenant).await.unwrap(); + assert_eq!( + serde_json::to_value(store.get_mttr_history(&tenant).await.unwrap().unwrap()) + .unwrap(), + serde_json::to_value(&history).unwrap() + ); + let policy = crate::alerts::outbound_http_policy::AlertTargetPolicyConfig { + allow_private: tenant.is_some(), + ..Default::default() + }; + store.put_outbound_policy(&tenant, &policy).await.unwrap(); + let filter:crate::users::filters::Filter=serde_json::from_value(serde_json::json!({ + "version":"v2","user_id":"owner","filter_id":"contract","stream_name":"orders","filter_name":"contract","tenant_id":tenant, + "query":{"filter_type":"sql","filter_query":"select * from orders","filter_builder":null},"time_filter":null + })).unwrap(); + store.put_filter(&filter, &tenant).await.unwrap(); + store.reconcile_filter(&filter, &tenant).await.unwrap(); + let config = Document { + id: "config".into(), + path: format!("{root}.settings/llmconfigs/contract.json"), + value: format!("config-{root}"), + }; + store.put_llmconfig(&config, &tenant).await.unwrap(); + let target: crate::alerts::target::Target = serde_json::from_value(serde_json::json!({ + "name": format!("target-{root}"), "type":"webhook", + "endpoint":"https://hook.example/x", "tenantId": tenant, + "id": "01ARZ3NDEKTSV4RRFFQ69G5FAV" + })) + .unwrap(); + store.put_target(&target, &tenant).await.unwrap(); + let chat = Document { + id: "chat".into(), + path: format!("{root}.users/owner/chats/contract.json"), + value: format!("chat-{root}"), + }; + store.put_chat(&chat, &tenant).await.unwrap(); + let key = Document { + id: "key".into(), + path: String::new(), + value: format!("key-{root}"), + }; + store.put_keystone(&key, &tenant).await.unwrap(); + store.put_conversation(&key, &tenant).await.unwrap(); + } + // The global getters must preserve both populated namespaces. Vec order + // is not part of Metastore's contract: compare byte sets per tenant. + let expected = |id: &str, + suffix: &str, + label: &str| + -> std::collections::HashMap> { + [None, Some("tenant-contract".to_owned())] + .into_iter() + .map(|tenant| { + let root = tenant.as_ref().map(|t| format!("{t}/")).unwrap_or_default(); + let document = Document { + id: id.into(), + path: if suffix.is_empty() { + String::new() + } else { + format!("{root}{suffix}") + }, + value: format!("{label}-{root}"), + }; + ( + tenant.unwrap_or_else(|| crate::parseable::DEFAULT_TENANT.into()), + vec![crate::storage::object_storage::to_bytes(&document)], + ) + }) + .collect() + }; + assert_eq!( + store.get_keystones().await.unwrap(), + expected("key", "", "key") + ); + assert_eq!( + store.get_conversations().await.unwrap(), + expected("key", "", "key") + ); + // MTTR history is incorrectly returned as an alert. The exact empty + // alert inventory assertion lives in the ignored regression below. + assert_eq!( + store.get_llmconfigs().await.unwrap(), + expected("config", ".settings/llmconfigs/contract.json", "config") + ); + let targets = store.get_targets().await.unwrap(); + assert_eq!(targets.len(), 2); + for (tenant, values) in targets { + assert_eq!(values.len(), 1, "typed target was silently discarded"); + assert_eq!( + values[0].name, + if tenant == crate::parseable::DEFAULT_TENANT { + "target-" + } else { + "target-tenant-contract/" + } + ); + } + let policies = store.get_outbound_policies().await.unwrap(); + assert_eq!(policies.len(), 2); + assert!(!policies[""].allow_private); + assert!(policies["tenant-contract"].allow_private); + assert_eq!( + store.get_dashboards().await.unwrap(), + expected("", "", "") + .into_keys() + .map(|t| (t, vec![])) + .collect() + ); + let chats = store.get_chats().await.unwrap(); + assert_eq!(chats.len(), 2); + for (tenant, bytes) in expected("chat", ".users/owner/chats/contract.json", "chat") { + let users = chats.get(&tenant).unwrap(); + assert_eq!(users.get("owner").unwrap().value(), &bytes); + } + let filters = store.get_filters().await.unwrap(); + assert_eq!(filters.len(), 2); + for (tenant, filters) in filters { + assert_eq!( + filters.len(), + 1, + "filter delete must exercise a populated tenant" + ); + for filter in filters { + store + .delete_filter( + &filter, + &Some(if tenant == crate::parseable::DEFAULT_TENANT { + String::new() + } else { + tenant.clone() + }), + ) + .await + .unwrap(); + } + } + let node = crate::handlers::http::modal::NodeMetadata { + domain_name: "contract.invalid".into(), + node_id: "contract".into(), + ..Default::default() + }; + store.put_node_metadata(&node).await.unwrap(); + assert_eq!( + store + .get_node_metadata(crate::handlers::http::modal::NodeType::Ingestor, &None) + .await + .unwrap() + .len(), + 1 + ); + assert_eq!( + store.get_ingestor_metadata().await.unwrap(), + vec![crate::storage::object_storage::to_bytes(&node)] + ); + assert!( + store + .delete_node_metadata( + "contract.invalid", + crate::handlers::http::modal::NodeType::Ingestor + ) + .await + .unwrap() + ); + assert!( + !store + .delete_node_metadata( + "missing.invalid", + crate::handlers::http::modal::NodeType::Ingestor + ) + .await + .unwrap() + ); + for tenant in [None, Some("tenant-contract".to_owned())] { + // LocalFS stream deletion historically ignores tenant scope; cloud/SQL do not. + if !sql && tenant.is_some() { + continue; + } + store.delete_stream("contractlogs", &tenant).await.unwrap(); + assert!(!store.stream_exists("contractlogs", &tenant).await.unwrap()); + store + .put_stream_json( + &crate::storage::ObjectStoreFormat::default(), + "contractlogs", + &tenant, + ) + .await + .unwrap(); + assert!(store.stream_exists("contractlogs", &tenant).await.unwrap()); + assert!( + !store + .is_stream_deleting("contractlogs", &tenant) + .await + .unwrap() + ); + store.delete_stream("contractlogs", &tenant).await.unwrap(); + } + let survivor = store.get_parseable_metadata(&None).await.unwrap().unwrap(); + assert!( + store + .get_parseable_metadata(&Some("tenant-contract".into())) + .await + .unwrap() + .is_some() + ); + store.delete_tenant("tenant-contract").await.unwrap(); + assert!( + store + .get_parseable_metadata(&Some("tenant-contract".into())) + .await + .unwrap() + .is_none() + ); + assert_eq!( + store.get_parseable_metadata(&None).await.unwrap().unwrap(), + survivor + ); + assert!( + store + .get_parseable_metadata(&Some("other-tenant".into())) + .await + .unwrap() + .is_none() + ); + } + // Explicit regression cases use disposable databases and always clean up, + // including when the assertion demonstrates a product defect. + async fn postgres_regression(case: u8) { + use futures::FutureExt; + initialize_context(); + let Ok(url) = std::env::var("P_TEST_POSTGRES_URL") else { + eprintln!("SKIP PostgreSQL regression {case}: P_TEST_POSTGRES_URL unset"); + return; + }; + if case == 3 && std::env::var_os("P_TEST_POSTGRES_CONTAINER").is_none() { + eprintln!("SKIP paused-server regression: P_TEST_POSTGRES_CONTAINER unset"); + return; + } + let (admin, connection) = tokio_postgres::connect(&url, tokio_postgres::NoTls) + .await + .unwrap(); + let task = tokio::spawn(async move { connection.await.unwrap() }); + let database = format!( + "regression_{}", + ulid::Ulid::new().to_string().to_lowercase() + ); + admin + .batch_execute(&format!("CREATE DATABASE {database}")) + .await + .unwrap(); + let mut url = url::Url::parse(&url).unwrap(); + url.set_path(&database); + let result = std::panic::AssertUnwindSafe(async { + let dir = test_dir(); + let storage: Arc = Arc::new(LocalFS::new(dir.path().to_owned())); + let options = PostgresOptions { + metastore_postgres_url: Some(url.to_string()), + metastore_postgres_statement_timeout_ms: 500, + metastore_postgres_connect_timeout_ms: 500, + ..Default::default() + }; + let documents = Arc::new( + super::super::postgres_documents::PostgresDocuments::new( + &options, + database.clone(), + storage, + ) + .unwrap(), + ); + documents.check(&None).await.unwrap(); + let store = PostgresMetastore { + storage: documents.clone(), + }; + if case == 0 { + let date = chrono::Utc::now(); + let manifest = crate::catalog::manifest::Manifest::default(); + store + .put_manifest(&manifest, "logs", date, date, &None) + .await + .unwrap(); + let path = store + .get_manifest_path("logs", date, date, &None) + .await + .unwrap(); + assert!( + store + .get_manifest("logs", date, date, Some(path), &None) + .await + .unwrap() + .is_some(), + "B1: returned manifest path must be readable" + ); + } else if case == 1 { + store + .put_stream_json(&crate::storage::ObjectStoreFormat::default(), "logs", &None) + .await + .unwrap(); + store + .delete_stream("logs", &Some(crate::parseable::DEFAULT_TENANT.into())) + .await + .unwrap(); + assert!( + !store.stream_exists("logs", &None).await.unwrap(), + "m5: default tenant alias must delete default rows" + ); + } else if case == 2 { + let fresh = test_dir(); + let empty: Arc = Arc::new(LocalFS::new(fresh.path().to_owned())); + let repaired = super::super::postgres_documents::PostgresDocuments::new( + &options, + database.clone(), + empty, + ) + .unwrap(); + assert!( + repaired.check(&None).await.is_err(), + "m9: a missing pairing marker must require explicit repair" + ); + } else if case == 3 { + let container = std::env::var("P_TEST_POSTGRES_CONTAINER") + .expect("requires disposable Docker container name"); + // Pausing an established server leaves TCP open and prevents + // server-side statement_timeout from sending a response. + assert!( + std::process::Command::new("docker") + .args(["pause", &container]) + .status() + .unwrap() + .success() + ); + let result = tokio::time::timeout( + std::time::Duration::from_secs(2), + documents.check_health(&None), + ) + .await; + assert!( + std::process::Command::new("docker") + .args(["unpause", &container]) + .status() + .unwrap() + .success() + ); + assert!( + result.is_ok(), + "M2: health exceeded its client deadline against a nonresponsive database" + ); + } else { + let output = std::process::Command::new(std::env::current_exe().unwrap()) + .args(["--exact", "metastore::metastores::postgres_metastore::contract_tests::startup_cleanup_child", "--ignored", "--nocapture"]) + .env("P_CONTRACT_STARTUP_ROOT", dir.path()) + .env("P_CONTRACT_STARTUP_URL", url.as_str()) + .env("P_CONTRACT_STARTUP_BUCKET", &database) + .env("P_CONTRACT_STARTUP_FAILURE", match case { 5 => "late-failure", 6 => "default-stream", _ => "none" }) + .output().unwrap(); + assert!(output.status.success(), "{}", String::from_utf8_lossy(&output.stdout).to_string() + &String::from_utf8_lossy(&output.stderr)); + } + }) + .catch_unwind() + .await; + admin + .batch_execute(&format!("DROP DATABASE {database} WITH (FORCE)")) + .await + .unwrap(); + drop(admin); + task.await.unwrap(); + if let Err(panic) = result { + std::panic::resume_unwind(panic); + } + } + #[tokio::test] + #[ignore = "requires PostgreSQL 17; starts an isolated process for the global context"] + async fn postgres_startup_cleanup() { + postgres_regression(4).await; + } + + #[tokio::test] + async fn postgres_startup_default_stream_regression() { + postgres_regression(6).await; + } + + #[tokio::test] + async fn postgres_startup_mid_scan_failure_regression() { + postgres_regression(5).await; + } + + #[tokio::test] + #[ignore = "internal child fixture; invoked by postgres_startup_cleanup"] + async fn startup_cleanup_child() { + initialize_context(); + let context = &crate::parseable::PARSEABLE; + let store = &context.metastore; + store.initiate_connection().await.unwrap(); + let url = std::env::var("P_CONTRACT_STARTUP_URL").unwrap(); + let (mut client, connection) = tokio_postgres::connect(&url, tokio_postgres::NoTls) + .await + .unwrap(); + let task = tokio::spawn(async move { connection.await.unwrap() }); + let storage = context.storage.get_object_store(); + if std::env::var("P_CONTRACT_STARTUP_FAILURE").as_deref() == Ok("default-stream") { + store + .put_stream_json( + &crate::storage::ObjectStoreFormat::default(), + "defaultlogs", + &None, + ) + .await + .unwrap(); + storage + .put_object( + RelativePath::new("defaultlogs/date=2026-01-01/data.parquet"), + Bytes::from_static(b"default tenant data"), + &None, + ) + .await + .unwrap(); + context.validate_multi_tenancy().await.unwrap(); + assert_eq!( + storage + .get_object( + RelativePath::new("defaultlogs/date=2026-01-01/data.parquet"), + &None + ) + .await + .expect("m6: startup must preserve default-tenant stream data"), + Bytes::from_static(b"default tenant data") + ); + assert!(store.stream_exists("defaultlogs", &None).await.unwrap()); + return; + } + if std::env::var("P_CONTRACT_STARTUP_FAILURE").as_deref() == Ok("late-failure") { + let invalid = Document { + id: "invalid".into(), + path: "a-invalid/.stream/.schema".into(), + value: "sentinel".into(), + }; + store + .put_dashboard(&invalid, &Some("a-invalid".into())) + .await + .unwrap(); + store + .put_parseable_metadata( + &crate::storage::store_metadata::StorageMetadata::default(), + &Some("z-valid".into()), + ) + .await + .unwrap(); + storage + .put_object( + RelativePath::new("a-invalid/data.parquet"), + Bytes::from_static(b"sentinel"), + &None, + ) + .await + .unwrap(); + // Restrict the injected SQL error to the per-tenant body read; + // volatile view expressions can also execute during discovery. + client.batch_execute("ALTER TABLE parseable_documents RENAME TO startup_documents; CREATE FUNCTION startup_read_failure() RETURNS bytea LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'injected transient tenant read failure' USING ERRCODE='08006'; END $$; CREATE VIEW parseable_documents AS SELECT tenant,kind,key,CASE WHEN tenant='z-valid' AND key='z-valid/.parseable.json' AND current_query() LIKE 'SELECT body%FROM parseable_documents%' THEN startup_read_failure() ELSE body END AS body,version,created_at,updated_at FROM startup_documents;").await.unwrap(); + assert!(context.validate_multi_tenancy().await.is_err()); + assert_eq!( + storage + .get_object(RelativePath::new("a-invalid/data.parquet"), &None) + .await + .unwrap(), + Bytes::from_static(b"sentinel"), + "a late database read failure must preserve earlier invalid tenant data" + ); + let remaining: i64 = client + .query_one( + "SELECT count(*) FROM startup_documents WHERE tenant='a-invalid'", + &[], + ) + .await + .unwrap() + .get(0); + assert_eq!( + remaining, 1, + "a late database read failure must preserve earlier invalid tenant rows" + ); + return; + } + for tenant in ["sqlonly", "duplicate"] { + let metadata = crate::storage::store_metadata::StorageMetadata::default(); + store + .put_parseable_metadata(&metadata, &Some(tenant.into())) + .await + .unwrap(); + } + let invalid = Document { + id: "invalid".into(), + path: "invalid/.stream/.schema".into(), + value: "invalid candidate".into(), + }; + store + .put_dashboard(&invalid, &Some("invalid".into())) + .await + .unwrap(); + for tenant in ["dataonly", "duplicate", "invalid", ".hidden"] { + storage + .put_object( + RelativePath::new(&format!("{tenant}/data.parquet")), + Bytes::from(tenant.to_owned()), + &None, + ) + .await + .unwrap(); + } + let before = client + .query( + "SELECT tenant,key,body FROM parseable_documents ORDER BY tenant,key", + &[], + ) + .await + .unwrap() + .into_iter() + .map(|r| { + ( + r.get::<_, String>(0), + r.get::<_, String>(1), + r.get::<_, Vec>(2), + ) + }) + .collect::>(); + // A transient database failure in discovery must admit no cleanup. + let tx = client.transaction().await.unwrap(); + tx.batch_execute("LOCK TABLE parseable_documents IN ACCESS EXCLUSIVE MODE") + .await + .unwrap(); + let started = std::time::Instant::now(); + let result = context.validate_multi_tenancy().await; + tx.rollback().await.unwrap(); + assert!(result.is_err()); + assert!(started.elapsed() < std::time::Duration::from_secs(3)); + // A temporarily unavailable bucket must also admit no cleanup. + let root = std::path::PathBuf::from(std::env::var("P_CONTRACT_STARTUP_ROOT").unwrap()); + let offline = root.with_extension("offline"); + std::fs::rename(&root, &offline).unwrap(); + std::fs::write(&root, b"temporarily unavailable bucket root").unwrap(); + let result = context.validate_multi_tenancy().await; + std::fs::remove_file(&root).unwrap(); + std::fs::rename(&offline, &root).unwrap(); + assert!(result.is_err()); + let after = client + .query( + "SELECT tenant,key,body FROM parseable_documents ORDER BY tenant,key", + &[], + ) + .await + .unwrap() + .into_iter() + .map(|r| { + ( + r.get::<_, String>(0), + r.get::<_, String>(1), + r.get::<_, Vec>(2), + ) + }) + .collect::>(); + assert_eq!(after, before); + for tenant in ["dataonly", "duplicate", "invalid", ".hidden"] { + assert_eq!( + storage + .get_object(RelativePath::new(&format!("{tenant}/data.parquet")), &None) + .await + .unwrap(), + Bytes::from(tenant.to_owned()) + ); + } + context.validate_multi_tenancy().await.unwrap(); + assert_eq!( + context + .list_tenants() + .unwrap() + .into_iter() + .collect::>(), + ["sqlonly".to_owned(), "duplicate".to_owned()] + .into_iter() + .collect() + ); + let rows = client + .query( + "SELECT tenant,key FROM parseable_documents ORDER BY tenant,key", + &[], + ) + .await + .unwrap() + .into_iter() + .map(|r| (r.get::<_, String>(0), r.get::<_, String>(1))) + .collect::>(); + assert_eq!( + rows, + vec![ + ("duplicate".into(), "duplicate/.parseable.json".into()), + ("sqlonly".into(), "sqlonly/.parseable.json".into()) + ] + ); + for tenant in ["dataonly", "invalid"] { + assert!( + storage + .get_object(RelativePath::new(&format!("{tenant}/data.parquet")), &None) + .await + .is_err() + ); + } + for tenant in ["duplicate", ".hidden"] { + assert_eq!( + storage + .get_object(RelativePath::new(&format!("{tenant}/data.parquet")), &None) + .await + .unwrap(), + Bytes::from(tenant.to_owned()) + ); + } + drop(client); + task.await.unwrap(); + } + #[tokio::test] + async fn alert_listing_excludes_mttr_history() { + initialize_context(); + let dir = test_dir(); + let storage: Arc = Arc::new(LocalFS::new(dir.path().to_owned())); + let store = super::super::object_store_metastore::ObjectStoreMetastore { storage }; + let history = crate::alerts::alert_structs::MTTRHistory { + daily_stats: vec![], + tenant_id: None, + }; + store.put_mttr_history(&history, &None).await.unwrap(); + assert!(store.get_alerts().await.unwrap()[crate::parseable::DEFAULT_TENANT].is_empty()); + } + #[tokio::test] + async fn postgres_health_deadline_regression() { + postgres_regression(3).await; + } + #[tokio::test] + async fn postgres_manifest_url_regression() { + postgres_regression(0).await; + } + #[tokio::test] + async fn postgres_default_alias_regression() { + postgres_regression(1).await; + } + #[tokio::test] + async fn postgres_empty_bucket_pairing_regression() { + postgres_regression(2).await; + } + #[tokio::test] + async fn local_listing_uses_the_selected_backend() { + initialize_context(); + let dir = test_dir(); + let storage: Arc = Arc::new(LocalFS::new(dir.path().to_owned())); + let store = super::super::object_store_metastore::ObjectStoreMetastore { storage }; + store + .put_stream_json( + &crate::storage::ObjectStoreFormat::default(), + "isolated", + &None, + ) + .await + .unwrap(); + assert_eq!( + store.list_streams(&None).await.unwrap(), + ["isolated".to_owned()].into_iter().collect() + ); + } + #[test] + fn target_tenant_survives_persistence_roundtrip() { + let target: crate::alerts::target::Target = serde_json::from_value(serde_json::json!({ + "name":"named target", "type":"webhook", "endpoint":"https://hook.example/x", "tenantId":"tenant-contract" + })).unwrap(); + let mut legacy = serde_json::to_value(&target).unwrap(); + legacy.as_object_mut().unwrap().remove("tenantId"); + legacy["tenant"] = serde_json::json!("tenant-contract"); + assert_eq!( + serde_json::from_value::(legacy) + .unwrap() + .tenant, + target.tenant + ); + let persisted = crate::storage::object_storage::to_bytes(&target); + let loaded: crate::alerts::target::Target = serde_json::from_slice(&persisted).unwrap(); + assert_eq!( + loaded.tenant, target.tenant, + "stored Target must retain its tenant" + ); + } + // Global tenant/mode lookups belong to the product API. Give each shared + // contract its own process rather than depending on which test forced the + // global context first. The parent owns and removes the context directory. + fn contract_process(name: &str, sql: bool) -> bool { + if std::env::var("P_CONTRACT_ACTIVE_CHILD").as_deref() == Ok(name) { + return false; + } + let context = test_dir(); + let mut command = std::process::Command::new(std::env::current_exe().unwrap()); + command + .args([ + "--exact", + &format!("metastore::metastores::postgres_metastore::contract_tests::{name}"), + "--nocapture", + ]) + .env("P_CONTRACT_ACTIVE_CHILD", name) + .env("P_CONTRACT_CONTEXT_ROOT", context.path()); + if sql { + command.arg("--ignored"); + } + let output = command.output().unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stdout).to_string() + + &String::from_utf8_lossy(&output.stderr) + ); + true + } + #[tokio::test] + async fn object_store_contract() { + if contract_process("object_store_contract", false) { + return; + } + initialize_context(); + let dir = test_dir(); + let storage: Arc = Arc::new(LocalFS::new(dir.path().to_owned())); + contract( + Arc::new(super::super::object_store_metastore::ObjectStoreMetastore { storage }), + false, + ) + .await; + } + #[tokio::test] + #[ignore = "requires PostgreSQL 17; run explicitly with P_TEST_POSTGRES_URL"] + async fn postgres_contract() { + if contract_process("postgres_contract", true) { + return; + } + let url = std::env::var("P_TEST_POSTGRES_URL") + .expect("P_TEST_POSTGRES_URL required for explicitly requested SQL tests"); + let (admin, connection) = tokio_postgres::connect(&url, tokio_postgres::NoTls) + .await + .unwrap(); + let connection_task = tokio::spawn(async move { connection.await.unwrap() }); + let database = format!("contract_{}", ulid::Ulid::new().to_string().to_lowercase()); + admin + .batch_execute(&format!("CREATE DATABASE {database}")) + .await + .unwrap(); + let mut isolated = url::Url::parse(&url).unwrap(); + isolated.set_path(&database); + let url = isolated.to_string(); + use futures::FutureExt; + let outcome = std::panic::AssertUnwindSafe(async { + let dir = test_dir(); + let storage: Arc = Arc::new(LocalFS::new(dir.path().to_owned())); + let options = PostgresOptions { + metastore_postgres_url: Some(url), + metastore_postgres_pool_size: 4, + metastore_postgres_connect_timeout_ms: 1000, + metastore_postgres_statement_timeout_ms: 3000, + metastore_postgres_id: None, + metastore_postgres_ca: None, + }; + let documents = super::super::postgres_documents::PostgresDocuments::new( + &options, + database.clone(), + storage.clone(), + ) + .unwrap(); + let concurrent = super::super::postgres_documents::PostgresDocuments::new( + &options, + database.clone(), + storage.clone(), + ) + .unwrap(); + let (first, second) = tokio::join!(documents.check(&None), concurrent.check(&None)); + first.unwrap(); + second.unwrap(); + let mismatched = super::super::postgres_documents::PostgresDocuments::new( + &options, + "other-bucket".into(), + storage, + ) + .unwrap(); + assert!(mismatched.check(&None).await.is_err()); + documents.verify_database_contract(&options).await; + documents.verify_partition_contract().await; + let documents = Arc::new(documents); + let store = Arc::new(PostgresMetastore { + storage: documents.clone(), + }); + contract(store.clone(), true).await; + documents.verify_deleted_stream_rows().await; + let mut keys = Vec::new(); + fn inventory(root: &std::path::Path, path: &std::path::Path, keys: &mut Vec) { + for entry in std::fs::read_dir(path).unwrap() { + let path = entry.unwrap().path(); + if path.is_dir() { + inventory(root, &path, keys); + } else { + keys.push( + path.strip_prefix(root) + .unwrap() + .to_string_lossy() + .into_owned(), + ); + } + } + } + inventory(dir.path(), dir.path(), &mut keys); + // LocalFS conditional marker creation uses private process lock files; + // these are not object keys on cloud stores (verified by the MinIO smoke). + keys.retain(|key| !key.starts_with(".parseable-metadata-locks/")); + assert_eq!(keys, vec![".parseable/postgres-metastore-id"]); + documents + .put_object( + RelativePath::new("hidden/.stream/.stream.json"), + Bytes::from_static(b"{}"), + &None, + ) + .await + .unwrap(); + documents + .put_object( + RelativePath::new("hidden/.stream/.deleting"), + Bytes::new(), + &None, + ) + .await + .unwrap(); + assert!(!store.list_streams(&None).await.unwrap().contains("hidden")); + store.delete_stream("hidden", &None).await.unwrap(); + drop(store); + drop(documents); + }) + .catch_unwind() + .await; + admin + .batch_execute(&format!("DROP DATABASE {database} WITH (FORCE)")) + .await + .unwrap(); + drop(admin); + connection_task.await.unwrap(); + if let Err(panic) = outcome { + std::panic::resume_unwind(panic); + } + } +} diff --git a/src/metastore/metastores/review_fix_tests.rs b/src/metastore/metastores/review_fix_tests.rs new file mode 100644 index 000000000..72f6d528f --- /dev/null +++ b/src/metastore/metastores/review_fix_tests.rs @@ -0,0 +1,458 @@ +//! Isolated PostgreSQL review regressions. +#[cfg(test)] +mod tests { + //! Isolated regression tests; each database is disposable and unique. + use super::super::{ + postgres_documents::PostgresDocuments, + postgres_metastore::{PostgresMetastore, PostgresOptions}, + }; + use crate::{ + metastore::metastore_traits::Metastore, + storage::{LocalFS, ObjectStorage}, + }; + use bytes::Bytes; + use relative_path::RelativePath; + use std::sync::Arc; + + #[test] + fn empty_url_is_disabled() { + for url in [None, Some("".into()), Some(" ".into())] { + assert!( + !PostgresOptions { + metastore_postgres_url: url, + ..Default::default() + } + .enabled() + ); + } + } + #[test] + fn ca_replaces_public_roots() -> anyhow::Result<()> { + let dir = tempfile::tempdir_in(concat!(env!("CARGO_MANIFEST_DIR"), "/target"))?; + let ca = dir.path().join("ca.pem"); + let key = dir.path().join("key.pem"); + let output = std::process::Command::new("openssl") + .args([ + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-days", + "1", + "-subj", + "/CN=review", + ]) + .arg("-keyout") + .arg(key) + .arg("-out") + .arg(&ca) + .output()?; + assert!(output.status.success()); + assert_eq!( + super::super::postgres_documents::configured_roots(Some(&ca))?.len(), + 1 + ); + assert!(super::super::postgres_documents::configured_roots(None)?.len() > 1); + Ok(()) + } + #[test] + fn unavailable_and_deleting_http_status() { + use crate::storage::ObjectStorageError; + use actix_web::{ResponseError, http::StatusCode}; + for (error, status) in [ + (ObjectStorageError::StreamDeleting, StatusCode::CONFLICT), + ( + ObjectStorageError::MetastoreUnavailable("outage".into()), + StatusCode::SERVICE_UNAVAILABLE, + ), + ] { + let stream: crate::handlers::http::logstream::error::StreamError = error.into(); + assert_eq!(stream.status_code(), status); + let source = if status == StatusCode::CONFLICT { + ObjectStorageError::StreamDeleting + } else { + ObjectStorageError::MetastoreUnavailable("outage".into()) + }; + let detail = crate::metastore::MetastoreError::from(source).to_detail(); + let nested = crate::handlers::http::ingest::PostError::CreateStream( + crate::handlers::http::logstream::error::CreateStreamError::Storage { + stream_name: "review".into(), + err: ObjectStorageError::MetastoreError(Box::new(detail)), + }, + ); + assert_eq!(nested.status_code(), status); + } + } + #[test] + fn parsed_url_controls_tuning_validation() { + use clap::Parser; + #[derive(Parser)] + struct Args { + #[command(flatten)] + postgres: PostgresOptions, + } + for url in ["", " "] { + let opts = Args::try_parse_from([ + "review", + "--metastore-postgres-url", + url, + "--metastore-postgres-pool-size", + "bad", + "--metastore-postgres-statement-timeout-ms", + "bad", + ]) + .unwrap(); + assert!(!opts.postgres.enabled()); + assert_eq!(opts.postgres.metastore_postgres_statement_timeout_ms, 10000); + } + for (flag, value) in [ + ("--metastore-postgres-pool-size", "0"), + ("--metastore-postgres-connect-timeout-ms", "bad"), + ("--metastore-postgres-statement-timeout-ms", "0"), + ] { + assert!( + Args::try_parse_from([ + "review", + "--metastore-postgres-url", + "postgresql://example/test", + flag, + value + ]) + .is_err() + ); + } + let mut opts = Args::try_parse_from([ + "review", + "--metastore-postgres-url", + "postgresql://example/test", + "--metastore-postgres-pool-size", + "2", + "--metastore-postgres-statement-timeout-ms", + "321", + ]) + .unwrap(); + assert_eq!(opts.postgres.metastore_postgres_pool_size, 2); + assert_eq!(opts.postgres.metastore_postgres_statement_timeout_ms, 321); + opts.try_update_from(["review", "--metastore-postgres-pool-size", "3"]) + .unwrap(); + assert!(opts.postgres.enabled()); + assert_eq!(opts.postgres.metastore_postgres_pool_size, 3); + } + fn initialize_context() { + fn factory() -> crate::parseable::Parseable { + use crate::storage::object_storage::ObjectStorageProvider; + let root = tempfile::tempdir_in(concat!(env!("CARGO_MANIFEST_DIR"), "/target")) + .unwrap() + .keep(); + let provider = Arc::new(crate::storage::FSConfig { root }); + let storage = provider.construct_client(); + let metastore = Arc::new(super::super::object_store_metastore::ObjectStoreMetastore { + storage: storage.clone(), + }); + let context = crate::parseable::Parseable::new( + crate::cli::Options::default(), + #[cfg(feature = "kafka")] + ::parse_from([ + "contract", + ]), + provider, + metastore, + storage, + ); + *context.tenants.write().unwrap() = vec![String::new(), "tenant-contract".into()]; + context + } + let _ = crate::parseable::CONTRACT_PARSEABLE_FACTORY.set(factory); + once_cell::sync::Lazy::force(&crate::parseable::PARSEABLE); + } + + #[tokio::test] + async fn review_regressions() -> anyhow::Result<()> { + let Ok(url) = std::env::var("P_TEST_POSTGRES_URL") else { + eprintln!("SKIP review_regressions: P_TEST_POSTGRES_URL unset"); + return Ok(()); + }; + let dir = tempfile::tempdir_in(concat!(env!("CARGO_MANIFEST_DIR"), "/target"))?; + let storage: Arc = Arc::new(LocalFS::new(dir.path().to_owned())); + let opts = PostgresOptions { + metastore_postgres_url: Some(url.clone()), + metastore_postgres_pool_size: 2, + metastore_postgres_statement_timeout_ms: 2000, + ..Default::default() + }; + let admin_options = PostgresOptions { + metastore_postgres_statement_timeout_ms: 10000, + ..opts.clone() + }; + let admin = PostgresDocuments::new(&admin_options, "admin".into(), storage.clone())?; + let name = format!("review_{}", uuid::Uuid::new_v4().simple()); + admin + .client() + .await? + .batch_execute(&format!("CREATE DATABASE {name}")) + .await?; + initialize_context(); + let result = async { + let mut url = url::Url::parse(&url)?; + url.set_path(&format!("/{name}")); + url.query_pairs_mut().append_pair("options","-c application_name=review-preserved"); + let query = url.query().unwrap().replace('+',"%20"); + url.set_query(Some(&query)); + let opts = PostgresOptions { + metastore_postgres_url: Some(url.to_string()), + ..opts + }; + let db = Arc::new(PostgresDocuments::new( + &opts, + "review".into(), + storage.clone(), + )?); + db.check(&None).await?; + assert_eq!(db.client().await?.query_one("SHOW application_name",&[]).await?.get::<_,String>(0),"review-preserved"); + let store = PostgresMetastore { + storage: db.clone(), + }; + for key in ["odd%stream/manifest%25.json", "odd%stream/dir%25/file"] { + db.put_object(RelativePath::new(key),Bytes::new(),&None).await?; + assert_eq!(db.absolute_url(RelativePath::new(key)).as_ref(),key); + } + let listed = db.list_with_delimiter(Some(object_store::path::Path::parse("odd%stream")?),&None).await?; + assert!(listed.objects.iter().any(|o|o.location.as_ref()=="odd%stream/manifest%25.json")); + assert!(listed.common_prefixes.iter().any(|p|p.as_ref()=="odd%stream/dir%25")); + let time = chrono::Utc::now(); + // Feed the returned locator back through SQL after two overwrites. + // Path helpers use the process context, initialized by the shared suite. + let locator = store.get_manifest_path("logs", time, time, &None).await?; + assert!(!locator.starts_with(dir.path().to_str().unwrap().trim_start_matches('/'))); + for n in [1, 2] { + let mut manifest = store + .get_manifest("logs", time, time, Some(locator.clone()), &None) + .await? + .unwrap_or_default(); + manifest.files.push(crate::catalog::manifest::File { + file_path: format!("{n}.parquet"), + ..Default::default() + }); + store + .put_manifest(&manifest, "logs", time, time, &None) + .await?; + assert_eq!( + store + .get_manifest("logs", time, time, Some(locator.clone()), &None) + .await? + .unwrap() + .files + .len(), + n + ); + } + if let Ok(endpoint) = std::env::var("P_TEST_REVIEW_S3_URL") { + use clap::Parser; + use crate::storage::ObjectStorageProvider; + #[derive(clap::Parser)] + struct S3Args { #[command(flatten)] storage: crate::storage::S3Config } + let provider = S3Args::parse_from(["review","--endpoint-url",&endpoint,"--region","us-east-1","--bucket-name","review","--access-key-id","minioadmin","--secret-key","minioadmin"]).storage; + let s3 = provider.construct_client(); + let cloud = PostgresMetastore {storage:Arc::new(PostgresDocuments::new(&opts,"review".into(),s3)?)}; + let locator = cloud.get_manifest_path("cloud",time,time,&None).await?; + for n in [1,2] { + let mut manifest = cloud.get_manifest("cloud",time,time,Some(locator.clone()),&None).await?.unwrap_or_default(); + manifest.files.push(crate::catalog::manifest::File {file_path:format!("{n}.parquet"),..Default::default()}); + cloud.put_manifest(&manifest,"cloud",time,time,&None).await?; + assert_eq!(cloud.get_manifest("cloud",time,time,Some(locator.clone()),&None).await?.unwrap().files.len(),n); + } + } else {eprintln!("SKIP S3 locator regression: P_TEST_REVIEW_S3_URL unset");} + let key = RelativePath::new("logs/.stream/.stream.json"); + db.put_object( + key, + Bytes::from_static(b"{}"), + &Some("DEFAULT_TENANT".into()), + ) + .await?; + assert!(db.stream_exists("logs", &None).await?); + let other = Some("logs-other".into()); + db.put_object( + RelativePath::new("logs-other/logs/.stream/.stream.json"), + Bytes::new(), + &other, + ) + .await?; + db.put_object( + RelativePath::new("logs-other/exclusive/.stream/.stream.json"), + Bytes::new(), + &other, + ) + .await?; + assert_eq!( + db.list_streams(&None).await?, + std::collections::HashSet::from(["logs".into()]) + ); + assert_eq!(db.list_streams(&other).await?.len(), 2); + // Concurrent marker/create admission; after the marker every write fails. + for _ in 0..10 { + db.delete_object(RelativePath::new("logs/.stream/.deleting"), &None) + .await?; + let (marker, _) = tokio::join!( + db.put_object( + RelativePath::new("logs/.stream/.deleting"), + Bytes::new(), + &None + ), + db.put_object(key, Bytes::new(), &None) + ); + marker?; + assert!(matches!( + db.put_object(key, Bytes::new(), &None).await, + Err(crate::storage::ObjectStorageError::StreamDeleting) + )); + } + db.delete_object(RelativePath::new("logs/.stream/.deleting"), &None).await?; + let mut held = db.client().await?; + let tx = held.transaction().await?; + tx.execute("INSERT INTO parseable_documents(tenant,kind,key,body) VALUES('','document','logs/.stream/.deleting',''::bytea)",&[]).await?; + let create = db.put_object(key,Bytes::new(),&None); + tokio::pin!(create); + assert!(tokio::time::timeout(std::time::Duration::from_millis(50),&mut create).await.is_err()); + tx.commit().await?; + assert!(matches!(create.await,Err(crate::storage::ObjectStorageError::StreamDeleting))); + drop(held); + assert!(db.list_streams(&None).await?.is_empty()); + db.delete_stream("logs", &None).await?; + db.put_object(key, Bytes::new(), &None).await?; + // SQL-only leftovers remain discoverable after physical partition deletion. + let partition = RelativePath::new("logs/date=2020-01-01"); + db.put_object( + RelativePath::new("logs/date=2020-01-01/node.manifest.json"), + Bytes::new(), + &None, + ) + .await?; + assert_eq!( + store.metadata_dates("logs", &None).await?, + vec![".stream", "date=2020-01-01"] + ); + storage.put_object(RelativePath::new("logs/date=2020-01-01/data.parquet"),Bytes::new(),&None).await?; + db.deletion_fault.store(3,std::sync::atomic::Ordering::SeqCst); + assert!(db.delete_partition(partition,&None).await.is_err()); + assert!(storage.get_object(RelativePath::new("logs/date=2020-01-01/data.parquet"),&None).await.is_err()); + assert!(store.metadata_dates("logs",&None).await?.contains(&"date=2020-01-01".into())); + db.deletion_fault.store(0,std::sync::atomic::Ordering::SeqCst); + db.delete_partition(partition, &None).await?; + assert_eq!(store.metadata_dates("logs", &None).await?, vec![".stream"]); + let paths: Vec<_> = (0..200) + .map(|n| format!("batch/{n}.manifest.json")) + .collect(); + for p in &paths { + let manifest = crate::catalog::manifest::Manifest { + files: vec![crate::catalog::manifest::File {file_path:"batch-data.parquet".into(),num_rows:1,..Default::default()}], + ..Default::default() + }; + db.put_object(RelativePath::new(p), crate::catalog::manifest::encode_manifest(&manifest)?, &None).await?; + } + let decoded = store.manifest_batch(&paths, &None).await?.expect("PostgreSQL batch override"); + assert_eq!(decoded.len(),200); + assert!(decoded.values().all(|m|m.as_ref().is_some_and(|m|m.files.len()==1 && m.files[0].num_rows==1))); + assert!( + db.delete_prefix(RelativePath::new("tenant/stream"), &None) + .await + .is_err() + ); + let client = db.client().await?; + client.batch_execute("SET default_transaction_read_only=on").await?; + assert!(client.execute("UPDATE parseable_documents SET kind=kind WHERE false",&[]).await.is_err()); + drop(client); + db.put_object(RelativePath::new("after-failover"),Bytes::new(),&None).await?; + // Server timeout and subsequent pool usability. + assert!( + db.client() + .await? + .query("SELECT pg_sleep(3)", &[]) + .await + .is_err() + ); + assert_eq!( + db.client() + .await? + .query_one("SELECT 1", &[]) + .await? + .get::<_, i32>(0), + 1 + ); + // Drop packets without resetting sockets, including on established sessions. + use std::sync::atomic::{AtomicBool, Ordering}; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let proxy_port = listener.local_addr()?.port(); + let destination = format!("127.0.0.1:{}", url.port().unwrap()); + let paused = Arc::new(AtomicBool::new(false)); + let gate = paused.clone(); + let proxy = tokio::spawn(async move { + while let Ok((incoming, _)) = listener.accept().await { + let Ok(outgoing) = tokio::net::TcpStream::connect(&destination).await else { + continue; + }; + let gate = gate.clone(); + tokio::spawn(async move { + let (mut ir, mut iw) = incoming.into_split(); + let (mut or, mut ow) = outgoing.into_split(); + let forward = async move { + let mut buf = [0; 8192]; + loop { + let n = ir.read(&mut buf).await?; + if n == 0 { + break; + } + while gate.load(Ordering::SeqCst) { + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + ow.write_all(&buf[..n]).await?; + } + Ok::<_, std::io::Error>(()) + }; + tokio::select! { _ = forward => {}, _ = tokio::io::copy(&mut or,&mut iw) => {} } + }); + } + }); + let mut proxy_url = url.clone(); + proxy_url.set_port(Some(proxy_port)).unwrap(); + let proxy_options = PostgresOptions { + metastore_postgres_url: Some(proxy_url.to_string()), + metastore_postgres_statement_timeout_ms: 300, + ..opts.clone() + }; + let proxied = PostgresDocuments::new(&proxy_options, "review".into(), storage.clone())?; + proxied.check(&None).await?; + paused.store(true, Ordering::SeqCst); + for _ in 0..4 { + let started = std::time::Instant::now(); + assert!(tokio::time::timeout(std::time::Duration::from_secs(2),proxied.check_health(&None)).await.expect("readiness exceeded client budget").is_err()); + assert!(started.elapsed() < std::time::Duration::from_secs(2)); + } + paused.store(false, Ordering::SeqCst); + proxied.check_health(&None).await?; + proxy.abort(); + // Nonempty SQL cannot establish a fresh bucket pairing on restart. + storage + .delete_object(RelativePath::new(super::super::migration::PAIR), &None) + .await?; + let restarted = PostgresDocuments::new(&opts, "review".into(), storage.clone())?; + assert!( + restarted + .check(&None) + .await + .unwrap_err() + .to_string() + .contains("nonempty database") + ); + Ok::<_, anyhow::Error>(()) + } + .await; + admin + .client() + .await? + .batch_execute(&format!("DROP DATABASE {name} WITH (FORCE)")) + .await?; + result + } +} diff --git a/src/metastore/migrations/000_identity.sql b/src/metastore/migrations/000_identity.sql new file mode 100644 index 000000000..ee07a7029 --- /dev/null +++ b/src/metastore/migrations/000_identity.sql @@ -0,0 +1,5 @@ +CREATE TABLE IF NOT EXISTS parseable_backend ( + singleton boolean PRIMARY KEY DEFAULT true CHECK (singleton), + id text NOT NULL, bucket text NOT NULL, schema_version integer NOT NULL, + protocol_version integer NOT NULL +); diff --git a/src/metastore/migrations/001_documents.sql b/src/metastore/migrations/001_documents.sql new file mode 100644 index 000000000..e68638e27 --- /dev/null +++ b/src/metastore/migrations/001_documents.sql @@ -0,0 +1,8 @@ +CREATE SEQUENCE IF NOT EXISTS parseable_document_version; +CREATE TABLE IF NOT EXISTS parseable_documents ( + tenant text NOT NULL, kind text NOT NULL, key text NOT NULL, + body bytea NOT NULL, + version bigint NOT NULL DEFAULT nextval('parseable_document_version'), + created_at timestamptz NOT NULL DEFAULT now(), updated_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (tenant, key) +); diff --git a/src/metastore/migrations/002_review_fixes.sql b/src/metastore/migrations/002_review_fixes.sql new file mode 100644 index 000000000..2e06daee5 --- /dev/null +++ b/src/metastore/migrations/002_review_fixes.sql @@ -0,0 +1,20 @@ +-- The caller holds the shared transaction advisory lock. ALTER rebuilds the +-- populated primary key transactionally; existing rows and versions survive. +ALTER TABLE parseable_documents ALTER COLUMN key TYPE text COLLATE "C"; +CREATE INDEX IF NOT EXISTS parseable_stream_inventory ON parseable_documents (tenant,key) WHERE key LIKE '%/.stream/%'; +CREATE OR REPLACE FUNCTION parseable_admit_document() RETURNS trigger LANGUAGE plpgsql AS $$ +DECLARE stream_prefix text; +BEGIN + IF position('/.stream/' in NEW.key) > 0 THEN + stream_prefix := split_part(NEW.key, '/.stream/', 1); + -- Serialize marker and stream writes, including writers on other nodes. + PERFORM pg_advisory_xact_lock(hashtextextended(NEW.tenant || '/' || stream_prefix, 0)); + IF NEW.key NOT LIKE '%/.deleting' AND EXISTS ( + SELECT 1 FROM parseable_documents WHERE tenant=NEW.tenant AND key=stream_prefix || '/.stream/.deleting' + ) THEN RAISE EXCEPTION 'stream deletion pending'; END IF; + END IF; + RETURN NEW; +END $$; +DROP TRIGGER IF EXISTS parseable_admit_document ON parseable_documents; +CREATE TRIGGER parseable_admit_document BEFORE INSERT OR UPDATE ON parseable_documents FOR EACH ROW EXECUTE FUNCTION parseable_admit_document(); +UPDATE parseable_backend SET schema_version=2 WHERE singleton; diff --git a/src/metastore/mod.rs b/src/metastore/mod.rs index aaf086657..1b397fc8c 100644 --- a/src/metastore/mod.rs +++ b/src/metastore/mod.rs @@ -97,7 +97,7 @@ impl MetastoreError { file_path: None, timestamp: Some(chrono::Utc::now()), metadata: std::collections::HashMap::new(), - status_code: 500, + status_code: self.status_code().as_u16(), }, MetastoreError::JsonParseError(e) => MetastoreErrorDetail { operation: "JsonParseError".to_string(), @@ -168,6 +168,12 @@ impl MetastoreError { pub fn status_code(&self) -> StatusCode { match self { + MetastoreError::ObjectStorageError(ObjectStorageError::MetastoreUnavailable(_)) => { + StatusCode::SERVICE_UNAVAILABLE + } + MetastoreError::ObjectStorageError(ObjectStorageError::StreamDeleting) => { + StatusCode::CONFLICT + } MetastoreError::ObjectStorageError(..) => StatusCode::INTERNAL_SERVER_ERROR, MetastoreError::JsonParseError(..) => StatusCode::INTERNAL_SERVER_ERROR, MetastoreError::ManifestCodecError(..) => StatusCode::INTERNAL_SERVER_ERROR, @@ -196,41 +202,172 @@ mod boundary_tests { | "storage/azure_blob.rs" | "storage/localfs.rs" | "metastore/metastores/object_store_metastore.rs" + | "metastore/metastores/document_metastore.rs" + | "metastore/metastores/migration.rs" + | "metastore/metastores/postgres_documents.rs" + | "metastore/metastores/postgres_metastore.rs" ) { return vec![]; } - let production = source.split("#[cfg(test)]").next().unwrap(); - let compact: String = production.chars().filter(|c| !c.is_whitespace()).collect(); + let tokens = production_tokens(source); let mut errors = Vec::new(); - for operation in [ - ".put_object(", - ".upload_file(", - ".put_metadata_conditional(", - ".delete_metadata_conditional(", - ] { - if compact.contains(operation) { - errors.push(format!( - "{path}: raw {operation} must enter through Metastore" - )); + let mut functions: Vec<(String, usize)> = Vec::new(); + let mut pending = None; + let mut depth = 0usize; + for (i, token) in tokens.iter().enumerate() { + if token == "fn" { + pending = tokens.get(i + 1).cloned(); + } + if token == "{" { + depth += 1; + if let Some(name) = pending.take() { + functions.push((name, depth)); + } + } else if token == "}" { + if functions.last().is_some_and(|(_, d)| *d == depth) { + functions.pop(); + } + depth = depth.saturating_sub(1); + } else if token == ";" { + pending = None; + } + let function = functions + .last() + .map(|(name, _)| name.as_str()) + .unwrap_or(""); + if i > 0 && tokens[i - 1] == "fn" { + continue; + } + if tokens.get(i + 1).map(String::as_str) != Some("(") { + continue; + } + let allowed = path == "storage/object_storage.rs" + && matches!( + (function, token.as_str()), + ("upload_single_parquet_file", "upload_multipart") + | ("upload_single_parquet_file", "delete_object") + ); + if !allowed + && matches!( + token.as_str(), + "put_object" + | "upload_file" + | "put_metadata_conditional" + | "delete_metadata_conditional" + | "delete_object" + | "delete_prefix" + | "upload_multipart" + | "put" + | "put_opts" + | "put_multipart" + | "put_multipart_opts" + | "delete" + | "delete_stream" + ) + { + // Generic raw object_store names need an explicit receiver/type hint; + // otherwise ordinary HTTP delete handlers would be false positives. + let generic = matches!( + token.as_str(), + "put" + | "put_opts" + | "put_multipart" + | "put_multipart_opts" + | "delete" + | "delete_stream" + ); + let context = tokens[i.saturating_sub(5)..i].join(""); + if !generic || context.contains("object_store") || context.contains("ObjectStore::") + { + errors.push(format!( + "{path}::{function}: raw {token} must enter through Metastore" + )); + } } } - for (operation, allowed_path) in [ - (".delete_object(", "storage/object_storage.rs"), - (".delete_prefix(", ""), - ] { - let count = compact.matches(operation).count(); - if count != usize::from(path == allowed_path) { - errors.push(format!("{path}: unexpected raw {operation} ({count})")); + errors + } + + // Lexical policy, not Rust type/provenance analysis: aliases, wrappers, + // generated macro bodies and unrecognised mutation APIs can evade this. + // Allowed backend modules and the two named Parquet sites require review. + fn production_tokens(source: &str) -> Vec { + // Skip comments and literals before token matching, retaining punctuation. + let pattern = regex::Regex::new(r####"(?s)//[^\n]*|/\*|(?:br|cr|r)(\#*)"|"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])'|[A-Za-z_][A-Za-z_0-9]*|[^\s]"####).unwrap(); + let mut tokens = Vec::new(); + let mut offset = 0; + while let Some(m) = pattern.find(&source[offset..]) { + let text = m.as_str(); + let start = offset + m.start(); + offset += m.end(); + if text == "/*" { + let mut depth = 1; + while offset < source.len() && depth > 0 { + if source[offset..].starts_with("/*") { + depth += 1; + offset += 2; + } else if source[offset..].starts_with("*/") { + depth -= 1; + offset += 2; + } else { + offset += source[offset..].chars().next().unwrap().len_utf8(); + } + } + } else if (text.starts_with('r') || text.starts_with("br") || text.starts_with("cr")) + && text.ends_with('"') + { + let closing = format!( + "\"{}", + "#".repeat(text.bytes().filter(|b| *b == b'#').count()) + ); + offset = source[offset..] + .find(&closing) + .map(|n| offset + n + closing.len()) + .unwrap_or(source.len()); + } else if text.starts_with("//") || text.starts_with('"') || text.starts_with('\'') { + // Literal content cannot contain a production call. + } else { + tokens.push(source[start..offset].to_owned()); } } - let multipart = compact.matches(".upload_multipart(").count(); - let allowed = usize::from(path == "storage/object_storage.rs"); - if multipart != allowed { - errors.push(format!( - "{path}: unexpected multipart publisher ({multipart})" - )); + let mut production = Vec::new(); + let mut i = 0; + while i < tokens.len() { + let cfg = ["#", "[", "cfg", "(", "test", ")", "]", "mod"]; + if tokens[i..] + .iter() + .map(String::as_str) + .take(cfg.len()) + .eq(cfg) + { + i += cfg.len(); + while i < tokens.len() && tokens[i] != "{" && tokens[i] != ";" { + i += 1; + } + if tokens.get(i).map(String::as_str) == Some(";") { + i += 1; + continue; + } + let mut depth = 0; + while i < tokens.len() { + if tokens[i] == "{" { + depth += 1; + } + if tokens[i] == "}" { + depth -= 1; + if depth == 0 { + i += 1; + break; + } + } + i += 1; + } + } else { + production.push(tokens[i].clone()); + i += 1; + } } - errors + production } fn inspect(root: &Path, path: &Path, errors: &mut Vec) { @@ -260,12 +397,17 @@ mod boundary_tests { fn guard_rejects_literal_and_computed_metadata_writes() { for source in [ "store.put_object(stream_json_path(name, tenant), bytes, tenant).await", + "ObjectStorage::put_object(store, path, bytes, tenant)", + "#[cfg(test)] mod tests; fn production() { store.put_object(path); }", + "#[cfg(test)] mod tests {} fn production() { store.put_object(path); }", + "object_store::ObjectStore::put(store, path, bytes)", "store\n . put_object (path, bytes, tenant).await", "store.put_metadata_conditional(path, bytes, condition, tenant).await", "store.upload_file(key, local, tenant).await", ] { assert!(!violations("application.rs", source).is_empty()); } + assert!(violations("application.rs", r###"fn ok() { let s = r#"store.put_object(path)"#; /* outer /* nested */ store.put_object(path) */ }"###).is_empty()); assert!( violations( "application.rs", diff --git a/src/parseable/mod.rs b/src/parseable/mod.rs index 363e3360e..a757afc99 100644 --- a/src/parseable/mod.rs +++ b/src/parseable/mod.rs @@ -154,8 +154,15 @@ fn resolve_telemetry_type( } } +#[cfg(test)] +pub(crate) static CONTRACT_PARSEABLE_FACTORY: OnceCell Parseable> = OnceCell::new(); + /// Shared state of the Parseable server. pub static PARSEABLE: Lazy = Lazy::new(|| { + #[cfg(test)] + if let Some(factory) = CONTRACT_PARSEABLE_FACTORY.get() { + return factory(); + } let metastore = METASTORE.get_or_init(|| { // Prompt user for missing env vars before clap validates them. // Values are set in env but NOT saved to disk yet. @@ -179,9 +186,31 @@ pub static PARSEABLE: Lazy = Lazy::new(|| { StorageOptions::Blob(args) => args.storage.construct_client(), StorageOptions::Gcs(args) => args.storage.construct_client(), }; + let options = match &cli.storage { + StorageOptions::Local(a) => &a.options, + StorageOptions::S3(a) => &a.options, + StorageOptions::Blob(a) => &a.options, + StorageOptions::Gcs(a) => &a.options, + }; + if options.postgres.enabled() { + use crate::metastore::metastores::{ + postgres_documents::PostgresDocuments, postgres_metastore::PostgresMetastore, + }; + let endpoint = match &cli.storage { + StorageOptions::Local(a) => a.storage.get_endpoint(), + StorageOptions::S3(a) => a.storage.get_endpoint(), + StorageOptions::Blob(a) => a.storage.get_endpoint(), + StorageOptions::Gcs(a) => a.storage.get_endpoint(), + }; + let documents = PostgresDocuments::new(&options.postgres, endpoint, storage) + .unwrap_or_else(|e| panic!("PostgreSQL metastore configuration failed: {e}")); + return Arc::new(PostgresMetastore { + storage: Arc::new(documents), + }) as Arc; + } tracing::warn!("creating objectstoremetastore"); let metastore = ObjectStoreMetastore { storage }; - Arc::new(metastore) + Arc::new(metastore) as Arc }); // Prompt user for missing env vars before clap validates them. @@ -502,6 +531,9 @@ impl Parseable { .await .map_err(crate::metastore::MetastoreError::into_storage_error)? { + if self.options.postgres.enabled() { + return Err(crate::storage::ObjectStorageError::StreamDeleting.into()); + } return Ok(false); } if !self @@ -1250,6 +1282,7 @@ impl Parseable { .map_err(crate::metastore::MetastoreError::into_storage_error)?; let mut recognized_tenants = vec![]; + let mut invalid_tenants = vec![]; // validate the possible presence of tenant storage metadata for tenant_id in dirs.into_iter() { @@ -1267,24 +1300,26 @@ impl Parseable { recognized_tenants.push(tenant_id); } else if !is_multi_tenant { } else { - // found an invalid multi-tenant directory, delete - tracing::warn!( - "Found invalid tenant directory with multi-tenant mode- {tenant_id}.\nDeleting." - ); - - // delete from storage - if let Err(e) = self - .metastore - .delete_tenant(&tenant_id) - .await - .map_err(crate::metastore::MetastoreError::into_storage_error) - { - tracing::error!( - "Unable to delete invalid directory- {tenant_id} with error-\n{e}" - ); - }; + invalid_tenants.push(tenant_id); } } + // Discovery and validation must finish before destructive cleanup. + for tenant_id in invalid_tenants { + // found an invalid multi-tenant directory, delete + tracing::warn!( + "Found invalid tenant directory with multi-tenant mode- {tenant_id}.\nDeleting." + ); + + // delete from storage + if let Err(e) = self + .metastore + .delete_tenant(&tenant_id) + .await + .map_err(crate::metastore::MetastoreError::into_storage_error) + { + tracing::error!("Unable to delete invalid directory- {tenant_id} with error-\n{e}"); + }; + } if is_multi_tenant { let mut t = self diff --git a/src/query/stream_schema_provider.rs b/src/query/stream_schema_provider.rs index c4baf009d..8d6a3713a 100644 --- a/src/query/stream_schema_provider.rs +++ b/src/query/stream_schema_provider.rs @@ -751,7 +751,19 @@ async fn collect_from_snapshot_cached( // Manifest fetches are one object store GET each and are pure round trip // latency, so run them concurrently instead of serially. `buffered` keeps // the snapshot ordering, which the `rev()` below depends on. - let manifest_files = futures_util::stream::iter(snapshot.manifests(time_filters)) + let items = snapshot.manifests(time_filters); + let batch = PARSEABLE + .metastore + .manifest_batch( + &items + .iter() + .map(|m| m.manifest_path.clone()) + .collect::>(), + tenant_id, + ) + .await + .map_err(|e| DataFusionError::Plan(e.to_string()))?; + let manifest_files = futures_util::stream::iter(items) .map(|manifest_item| { let ManifestItem { time_lower_bound, @@ -759,7 +771,11 @@ async fn collect_from_snapshot_cached( manifest_path, .. } = manifest_item; + let batch = batch.as_ref(); async move { + if let Some(batch) = batch { + return Ok(batch.get(&manifest_path).cloned().flatten()); + } if let Some(cache) = cache && let Some(manifest) = cache.manifests.lock().await.get(&manifest_path).cloned() diff --git a/src/storage/azure_blob.rs b/src/storage/azure_blob.rs index b0a32b1a0..8e23722dc 100644 --- a/src/storage/azure_blob.rs +++ b/src/storage/azure_blob.rs @@ -1186,6 +1186,24 @@ impl ObjectStorage for BlobStore { .collect::>()) } + async fn metadata_hidden_prefix( + &self, + prefix: object_store::path::Path, + ) -> Result)>, ObjectStorageError> { + use futures::TryStreamExt; + self.client + .list(Some(&prefix)) + .map_ok(|o| { + ( + crate::storage::object_storage::decode_metadata_key(o.location.as_ref()), + o.last_modified, + ) + }) + .try_collect() + .await + .map_err(Into::into) + } + async fn list_with_delimiter( &self, prefix: Option, diff --git a/src/storage/gcs.rs b/src/storage/gcs.rs index 7e105ab31..dc984f840 100644 --- a/src/storage/gcs.rs +++ b/src/storage/gcs.rs @@ -1165,6 +1165,24 @@ impl ObjectStorage for Gcs { .collect::>()) } + async fn metadata_hidden_prefix( + &self, + prefix: object_store::path::Path, + ) -> Result)>, ObjectStorageError> { + use futures::TryStreamExt; + self.client + .list(Some(&prefix)) + .map_ok(|o| { + ( + crate::storage::object_storage::decode_metadata_key(o.location.as_ref()), + o.last_modified, + ) + }) + .try_collect() + .await + .map_err(Into::into) + } + async fn list_with_delimiter( &self, prefix: Option, diff --git a/src/storage/localfs.rs b/src/storage/localfs.rs index 0fe525e7f..17c448c91 100644 --- a/src/storage/localfs.rs +++ b/src/storage/localfs.rs @@ -1084,6 +1084,41 @@ impl ObjectStorage for LocalFS { url::Url::parse("file:///").unwrap() } + async fn metadata_inventory( + &self, + ) -> Result)>, ObjectStorageError> { + let mut dirs = vec![self.root.clone()]; + let mut keys = Vec::new(); + while let Some(dir) = dirs.pop() { + for entry in std::fs::read_dir(dir)? { + let entry = entry?; + let meta = entry.metadata()?; + if entry.file_type()?.is_symlink() { + continue; + } + if meta.is_dir() { + if !entry.file_name().to_string_lossy().starts_with("hour=") + && !entry.file_name().to_string_lossy().starts_with("minute=") + { + dirs.push(entry.path()); + } + } else if !entry.file_name().to_string_lossy().ends_with(".parquet") { + keys.push(( + entry + .path() + .strip_prefix(&self.root) + .unwrap() + .to_string_lossy() + .into_owned(), + meta.modified()?.into(), + )); + } + } + } + keys.sort_by(|a, b| a.0.cmp(&b.0)); + Ok(keys) + } + async fn list_with_delimiter( &self, _prefix: Option, diff --git a/src/storage/mod.rs b/src/storage/mod.rs index c376a487f..046715922 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -561,6 +561,10 @@ impl Default for ObjectStoreFormat { #[derive(Debug, thiserror::Error)] pub enum ObjectStorageError { + #[error("PostgreSQL metastore unavailable: {0}")] + MetastoreUnavailable(String), + #[error("stream deletion pending; retry DELETE before creating the stream")] + StreamDeleting, // no such key inside the object storage #[error("{0} not found")] NoSuchKey(String), @@ -607,6 +611,24 @@ pub enum ObjectStorageError { }, } +impl ObjectStorageError { + pub(crate) fn http_status(&self) -> actix_web::http::StatusCode { + use actix_web::http::StatusCode; + match self { + Self::MetastoreUnavailable(_) => StatusCode::SERVICE_UNAVAILABLE, + Self::StreamDeleting => StatusCode::CONFLICT, + Self::MetastoreError(detail) + if detail.operation == "ObjectStorageError" + && matches!(detail.status_code, 409 | 503) => + { + StatusCode::from_u16(detail.status_code) + .unwrap_or(StatusCode::INTERNAL_SERVER_ERROR) + } + _ => StatusCode::INTERNAL_SERVER_ERROR, + } + } +} + pub fn to_object_store_path(path: &RelativePath) -> Path { Path::from(path.as_str()) } diff --git a/src/storage/object_storage.rs b/src/storage/object_storage.rs index f1acfbcb9..70d6cad18 100644 --- a/src/storage/object_storage.rs +++ b/src/storage/object_storage.rs @@ -662,6 +662,87 @@ pub trait ObjectStorage: Debug + Send + Sync + 'static { fn absolute_url(&self, prefix: &RelativePath) -> object_store::path::Path; fn store_url(&self) -> url::Url; + async fn check_migration_guard(&self) -> Result<(), ObjectStorageError> { + // ObjectStorage is object safe; use its primitives directly here. + for key in [ + crate::metastore::metastores::migration::BUSY, + crate::metastore::metastores::migration::PAIR, + ] { + match self.get_object(RelativePath::new(key), &None).await { + Ok(ref id) + if key.ends_with("postgres-metastore-id") + && std::env::var("P_ALLOW_PAIRED_BUCKET") + .is_ok_and(|v| v.as_bytes() == id.as_ref()) => {} + Ok(_) => { + return Err(ObjectStorageError::Custom(format!( + "bucket has {key}: PostgreSQL pairing or offline migration; complete export before object-store startup (P_ALLOW_PAIRED_BUCKET= overrides pairing only)" + ))); + } + Err(ObjectStorageError::NoSuchKey(_)) => {} + Err(ObjectStorageError::IoError(e)) if e.kind() == std::io::ErrorKind::NotFound => { + } + Err(e) => { + return Err(ObjectStorageError::Custom(format!( + "cannot probe migration marker {key}: {e}; retry after restoring bucket access" + ))); + } + } + } + Ok(()) + } + + /// Raw inventory for offline document migration; retains keys, never bodies. + async fn metadata_inventory( + &self, + ) -> Result)>, ObjectStorageError> { + let mut pending = vec![None]; + let mut keys = Vec::new(); + while let Some(prefix) = pending.pop() { + let listing = self.list_with_delimiter(prefix).await?; + for prefix in listing.common_prefixes { + if prefix + .parts() + .any(|c| c.as_ref().starts_with("hour=") || c.as_ref().starts_with("minute=")) + { + continue; + } + if prefix.filename().is_some_and(|p| p.starts_with('.')) { + keys.extend(self.metadata_hidden_prefix(prefix).await?); + } else { + pending.push(Some(prefix)); + } + } + keys.extend( + listing + .objects + .into_iter() + .filter(|o| !o.location.as_ref().ends_with(".parquet")) + .map(|o| (decode_metadata_key(o.location.as_ref()), o.last_modified)), + ); + } + keys.sort_by(|a, b| a.0.cmp(&b.0)); + Ok(keys) + } + + async fn metadata_hidden_prefix( + &self, + prefix: object_store::path::Path, + ) -> Result)>, ObjectStorageError> { + let mut pending = vec![prefix]; + let mut keys = Vec::new(); + while let Some(prefix) = pending.pop() { + let listing = self.list_with_delimiter(Some(prefix)).await?; + pending.extend(listing.common_prefixes); + keys.extend( + listing + .objects + .into_iter() + .map(|o| (decode_metadata_key(o.location.as_ref()), o.last_modified)), + ); + } + Ok(keys) + } + async fn list_with_delimiter( &self, prefix: Option, @@ -2479,3 +2560,23 @@ mod query_reload_tests { assert_eq!(stream.schema_files().len(), 1); } } + +/// Decode exactly one object_store path encoding layer. +pub(crate) fn decode_metadata_key(key: &str) -> String { + let bytes = key.as_bytes(); + let mut out = Vec::with_capacity(bytes.len()); + let mut i = 0; + while i < bytes.len() { + if bytes[i] == b'%' + && i + 2 < bytes.len() + && let Ok(n) = u8::from_str_radix(&key[i + 1..i + 3], 16) + { + out.push(n); + i += 3; + } else { + out.push(bytes[i]); + i += 1; + } + } + String::from_utf8(out).expect("object_store paths encode UTF-8 keys") +} diff --git a/src/storage/retention.rs b/src/storage/retention.rs index 8f88f7044..cdd06b97b 100644 --- a/src/storage/retention.rs +++ b/src/storage/retention.rs @@ -242,9 +242,34 @@ mod action { let retain_until = get_retain_until(Utc::now().date_naive(), days as u64); - let Ok(mut dates) = store.list_dates(&stream_name, tenant_id).await else { - return; + let mut dates = match store.list_dates(&stream_name, tenant_id).await { + Ok(dates) => dates, + Err(crate::storage::ObjectStorageError::NoSuchKey(_)) + if PARSEABLE.options.postgres.enabled() => + { + Vec::new() + } + Err(crate::storage::ObjectStorageError::IoError(e)) + if PARSEABLE.options.postgres.enabled() + && e.kind() == std::io::ErrorKind::NotFound => + { + Vec::new() + } + Err(_) => return, }; + match PARSEABLE + .metastore + .metadata_dates(&stream_name, tenant_id) + .await + { + Ok(sql_dates) => dates.extend(sql_dates), + Err(e) => { + error!("retention SQL inventory failed: {e}"); + return; + } + } + dates.sort(); + dates.dedup(); dates.retain(|date| date.starts_with("date")); let dates_to_delete = dates .into_iter() diff --git a/src/storage/s3.rs b/src/storage/s3.rs index cd8f766ce..c431b8d4c 100644 --- a/src/storage/s3.rs +++ b/src/storage/s3.rs @@ -1419,6 +1419,24 @@ impl ObjectStorage for S3 { self.bucket.clone() } + async fn metadata_hidden_prefix( + &self, + prefix: object_store::path::Path, + ) -> Result)>, ObjectStorageError> { + use futures::TryStreamExt; + self.client + .list(Some(&prefix)) + .map_ok(|o| { + ( + crate::storage::object_storage::decode_metadata_key(o.location.as_ref()), + o.last_modified, + ) + }) + .try_collect() + .await + .map_err(Into::into) + } + async fn list_with_delimiter( &self, prefix: Option, From b8ce87d02c824f3d1d6d294e513b366a7007e487 Mon Sep 17 00:00:00 2001 From: VibeSWE <338779468+VibeSWE@users.noreply.github.com> Date: Wed, 11 Nov 2026 09:11:59 -0600 Subject: [PATCH 2/9] fix: harden PostgreSQL document admission and offline metadata rollback --- src/catalog/mod.rs | 5 +- src/handlers/http/ingest.rs | 3 +- src/handlers/http/query.rs | 6 +- src/interactive.rs | 5 + .../metastores/document_metastore.rs | 37 +- src/metastore/metastores/migration.rs | 326 +++++++++-- src/metastore/metastores/postgres_client.rs | 63 ++- .../metastores/postgres_documents.rs | 510 +++++++++++++----- .../metastores/postgres_metastore.rs | 93 +++- src/metastore/metastores/review_fix_tests.rs | 20 + .../migrations/003_manifest_admission.sql | 18 + src/metastore/mod.rs | 1 + src/query/stream_schema_provider.rs | 50 +- src/storage/azure_blob.rs | 17 +- src/storage/gcs.rs | 17 +- src/storage/localfs.rs | 5 +- src/storage/object_storage.rs | 251 +++++++-- src/storage/s3.rs | 17 +- src/users/dashboards.rs | 24 +- 19 files changed, 1171 insertions(+), 297 deletions(-) create mode 100644 src/metastore/migrations/003_manifest_admission.sql diff --git a/src/catalog/mod.rs b/src/catalog/mod.rs index 5aac2ef03..8bab4d2d7 100644 --- a/src/catalog/mod.rs +++ b/src/catalog/mod.rs @@ -585,7 +585,10 @@ pub fn partition_path( upper_bound: DateTime, tenant_id: &Option, ) -> RelativePathBuf { - let root = tenant_id.as_deref().unwrap_or(""); + let root = tenant_id + .as_deref() + .filter(|t| !t.is_empty() && *t != crate::parseable::DEFAULT_TENANT) + .unwrap_or(""); let lower = lower_bound.date_naive().format("%Y-%m-%d").to_string(); let upper = upper_bound.date_naive().format("%Y-%m-%d").to_string(); if lower == upper { diff --git a/src/handlers/http/ingest.rs b/src/handlers/http/ingest.rs index cd7f297c2..ea8242371 100644 --- a/src/handlers/http/ingest.rs +++ b/src/handlers/http/ingest.rs @@ -400,7 +400,8 @@ pub async fn post_event( .await { Ok(true) => {} - Ok(false) | Err(_) => return Err(StreamNotFound(stream_name.clone()).into()), + Ok(false) => return Err(StreamNotFound(stream_name.clone()).into()), + Err(e) => return Err(e.into()), } } else { return Err(StreamNotFound(stream_name.clone()).into()); diff --git a/src/handlers/http/query.rs b/src/handlers/http/query.rs index 2dd694c47..246d14ba3 100644 --- a/src/handlers/http/query.rs +++ b/src/handlers/http/query.rs @@ -814,7 +814,11 @@ impl actix_web::ResponseError for QueryError { QueryError::JsonParse(_) => StatusCode::INTERNAL_SERVER_ERROR, QueryError::Execute(e) => e.status_code(), QueryError::MetastoreError(e) => e.status_code(), - QueryError::ObjectStorage(e) => e.http_status(), + QueryError::ObjectStorage(e) => match e { + ObjectStorageError::MetastoreUnavailable(_) + | ObjectStorageError::StreamDeleting => e.http_status(), + _ => StatusCode::BAD_REQUEST, + }, QueryError::Header(_) => StatusCode::BAD_REQUEST, QueryError::HeaderBuild(e) => actix_web::ResponseError::status_code(e), _ => StatusCode::BAD_REQUEST, diff --git a/src/interactive.rs b/src/interactive.rs index 0bac1e882..1a2ba98e5 100644 --- a/src/interactive.rs +++ b/src/interactive.rs @@ -26,6 +26,10 @@ const ENV_FILE_NAME: &str = ".parseable.env"; /// Guard to avoid parsing `.parseable.env` more than once per process. static ENV_FILE_LOADED: AtomicBool = AtomicBool::new(false); +static LOADED_ENV_PATH: std::sync::OnceLock = std::sync::OnceLock::new(); +pub(crate) fn loaded_env_path() -> Option<&'static std::path::Path> { + LOADED_ENV_PATH.get().map(PathBuf::as_path) +} /// A required or optional env var that the user may need to provide. struct EnvPrompt { @@ -127,6 +131,7 @@ pub fn load_env_file() { Err(_) => return, }; + let _ = LOADED_ENV_PATH.set(path); for line in io::BufReader::new(file).lines() { let line = match line { Ok(l) => l, diff --git a/src/metastore/metastores/document_metastore.rs b/src/metastore/metastores/document_metastore.rs index 92e092c77..8ffccc729 100644 --- a/src/metastore/metastores/document_metastore.rs +++ b/src/metastore/metastores/document_metastore.rs @@ -143,7 +143,7 @@ impl $name { tenant_id: &Option, ) -> Result, MetastoreError> { let users_dir = - RelativePathBuf::from_iter([tenant_id.as_deref().unwrap_or(""), USERS_ROOT_DIR]); + RelativePathBuf::from_iter([tenant_id.as_deref().filter(|t| !t.is_empty() && *t != crate::parseable::DEFAULT_TENANT).unwrap_or(""), USERS_ROOT_DIR]); let mut grouped: BTreeMap<(String, String), Vec<(RelativePathBuf, Filter)>> = BTreeMap::new(); for user in self @@ -357,7 +357,7 @@ impl Metastore for $name { tenant_id: &Option, ) -> Result<(), MetastoreError> { let id = obj.get_object_id(); - let path = if let Some(tenant_id) = tenant_id.as_ref() { + let path = if let Some(tenant_id) = tenant_id.as_ref().filter(|t| !t.is_empty() && t.as_str() != crate::parseable::DEFAULT_TENANT) { RelativePathBuf::from_iter([tenant_id, ".keystone", &format!("{id}.json")]) } else { RelativePathBuf::from_iter([".keystone", &format!("{id}.json")]) @@ -375,7 +375,7 @@ impl Metastore for $name { tenant_id: &Option, ) -> Result<(), MetastoreError> { let id = obj.get_object_id(); - let path = if let Some(tenant_id) = tenant_id.as_ref() { + let path = if let Some(tenant_id) = tenant_id.as_ref().filter(|t| !t.is_empty() && t.as_str() != crate::parseable::DEFAULT_TENANT) { RelativePathBuf::from_iter([tenant_id, ".keystone", &format!("{id}.json")]) } else { RelativePathBuf::from_iter([".keystone", &format!("{id}.json")]) @@ -415,7 +415,7 @@ impl Metastore for $name { tenant_id: &Option, ) -> Result<(), MetastoreError> { let id = obj.get_object_id(); - let path = if let Some(tenant_id) = tenant_id.as_ref() { + let path = if let Some(tenant_id) = tenant_id.as_ref().filter(|t| !t.is_empty() && t.as_str() != crate::parseable::DEFAULT_TENANT) { RelativePathBuf::from_iter([tenant_id, ".keystone", &format!("conv_{id}.json")]) } else { RelativePathBuf::from_iter([".keystone", &format!("conv_{id}.json")]) @@ -433,7 +433,7 @@ impl Metastore for $name { tenant_id: &Option, ) -> Result<(), MetastoreError> { let id = obj.get_object_id(); - let path = if let Some(tenant_id) = tenant_id.as_ref() { + let path = if let Some(tenant_id) = tenant_id.as_ref().filter(|t| !t.is_empty() && t.as_str() != crate::parseable::DEFAULT_TENANT) { RelativePathBuf::from_iter([tenant_id, ".keystone", &format!("conv_{id}.json")]) } else { RelativePathBuf::from_iter([".keystone", &format!("conv_{id}.json")]) @@ -513,7 +513,7 @@ impl Metastore for $name { &self, tenant_id: &Option, ) -> Result, MetastoreError> { - let tenant = tenant_id.as_deref().unwrap_or(DEFAULT_TENANT); + let tenant = tenant_id.as_deref().filter(|t| !t.is_empty() && *t != DEFAULT_TENANT).unwrap_or(""); let base_path = RelativePathBuf::from_iter([tenant, ALERTS_ROOT_DIRECTORY]); let alert_state_bytes = self .storage @@ -770,7 +770,7 @@ impl Metastore for $name { tenant_id: &Option, ) -> Result, MetastoreError> { let users = - RelativePathBuf::from_iter([tenant_id.as_deref().unwrap_or(""), USERS_ROOT_DIR]); + RelativePathBuf::from_iter([tenant_id.as_deref().filter(|t| !t.is_empty() && *t != crate::parseable::DEFAULT_TENANT).unwrap_or(""), USERS_ROOT_DIR]); let mut dashboards = Vec::new(); for owner in self.storage.list_dirs_relative(&users, tenant_id).await? { if owner.starts_with('.') { @@ -966,7 +966,7 @@ impl Metastore for $name { tenant_id: &Option, _is_migration: bool, ) -> Result { - let tenant = tenant_id.as_deref().unwrap_or(""); + let tenant = tenant_id.as_deref().filter(|t| !t.is_empty() && *t != crate::parseable::DEFAULT_TENANT).unwrap_or(""); let path = if get_base { RelativePathBuf::from_iter([ tenant, @@ -988,7 +988,7 @@ impl Metastore for $name { tenant_id: &Option, _is_migration: bool, ) -> Result, MetastoreError> { - let root = tenant_id.as_deref().unwrap_or(""); + let root = tenant_id.as_deref().filter(|t| !t.is_empty() && *t != crate::parseable::DEFAULT_TENANT).unwrap_or(""); let path = RelativePathBuf::from_iter([root, stream_name, STREAM_ROOT_DIRECTORY]); if let Some(mode) = mode { if mode.eq(&Mode::Ingest) { @@ -1058,7 +1058,7 @@ impl Metastore for $name { let inventory_limit = inventory_limit.clone(); async move { let t_start = std::time::Instant::now(); - let date_path = if let Some(tenant) = tenant.as_ref() { + let date_path = if let Some(tenant) = tenant.as_ref().filter(|t| !t.is_empty() && t.as_str() != DEFAULT_TENANT) { object_store::path::Path::from(format!("{}/{}/{}", tenant, stream, date)) } else { object_store::path::Path::from(format!("{}/{}", stream, date)) @@ -1338,7 +1338,7 @@ impl Metastore for $name { stream_name: &str, tenant_id: &Option, ) -> Result, MetastoreError> { - let path_prefix = if let Some(tenant) = tenant_id { + let path_prefix = if let Some(tenant) = tenant_id.as_ref().filter(|t| !t.is_empty() && t.as_str() != crate::parseable::DEFAULT_TENANT) { relative_path::RelativePathBuf::from(format!( "{tenant}/{stream_name}/{STREAM_ROOT_DIRECTORY}" )) @@ -1392,7 +1392,7 @@ impl Metastore for $name { }; if canonical.as_ref().map(normalize) != Some(normalize(&merged)) { let path = RelativePathBuf::from_iter([ - tenant_id.as_deref().unwrap_or(""), + tenant_id.as_deref().filter(|t| !t.is_empty() && *t != crate::parseable::DEFAULT_TENANT).unwrap_or(""), stream_name, STREAM_ROOT_DIRECTORY, schema_file, @@ -1432,7 +1432,7 @@ impl Metastore for $name { &self, tenant_id: &Option, ) -> Result, MetastoreError> { - let path = if let Some(tenant_id) = tenant_id.as_ref() { + let path = if let Some(tenant_id) = tenant_id.as_ref().filter(|t| !t.is_empty() && t.as_str() != crate::parseable::DEFAULT_TENANT) { RelativePathBuf::from_iter([tenant_id, ".parseable.json"]) } else { parseable_json_path() @@ -1477,7 +1477,7 @@ impl Metastore for $name { obj: &dyn MetastoreObject, tenant_id: &Option, ) -> Result<(), MetastoreError> { - let path = if let Some(tenant_id) = tenant_id.as_ref() { + let path = if let Some(tenant_id) = tenant_id.as_ref().filter(|t| !t.is_empty() && t.as_str() != crate::parseable::DEFAULT_TENANT) { RelativePathBuf::from_iter([tenant_id, ".parseable.json"]) } else { parseable_json_path() @@ -1637,7 +1637,14 @@ pub(crate) fn document_scope(key: &str) -> Option<(String, &'static str)> { let date = parts.iter().position(|p| p.starts_with("date="))?; return match date { 1 => Some((String::new(), "manifest")), - 2 => Some((parts[0].to_owned(), "manifest")), + 2 => Some(( + if parts[0] == crate::parseable::DEFAULT_TENANT { + String::new() + } else { + parts[0].to_owned() + }, + "manifest", + )), _ => None, }; } diff --git a/src/metastore/metastores/migration.rs b/src/metastore/metastores/migration.rs index 2cad412a5..a713bf744 100644 --- a/src/metastore/metastores/migration.rs +++ b/src/metastore/metastores/migration.rs @@ -67,6 +67,9 @@ pub async fn optional(store: &dyn ObjectStorage, key: &str) -> anyhow::Result Err(e.into()), } } +pub(crate) fn recovery_override(value: Option<&str>, id: &[u8]) -> bool { + value.is_some_and(|v| v.as_bytes() == id) +} pub async fn guard(store: &dyn ObjectStorage, postgres: bool) -> anyhow::Result<()> { ensure!( optional(store, BUSY).await?.is_none(), @@ -76,7 +79,10 @@ pub async fn guard(store: &dyn ObjectStorage, postgres: bool) -> anyhow::Result< && !postgres { ensure!( - std::env::var("P_ALLOW_PAIRED_BUCKET").is_ok_and(|v| v.as_bytes() == id.as_ref()), + recovery_override( + std::env::var("P_ALLOW_PAIRED_BUCKET").ok().as_deref(), + id.as_ref() + ), "bucket paired with PostgreSQL; object-store metadata is stale. Export first, or explicitly set P_ALLOW_PAIRED_BUCKET= for disaster recovery" ); } @@ -115,6 +121,27 @@ pub(crate) fn stream_references(body: &[u8]) -> anyhow::Result> { .collect()) } +/// An explicit default-tenant prefix is an alias only for tenant-shaped +/// metadata (not for a default stream actually named DEFAULT_TENANT). +pub(crate) fn canonical_key(key: &str) -> &str { + let Some(rest) = key.strip_prefix("DEFAULT_TENANT/") else { + return key; + }; + let parts: Vec<_> = rest.split('/').collect(); + if parts.first().is_some_and(|p| { + p.starts_with('.') && *p != ".stream" && *p != ".schema" && *p != ".stream.json" + }) || parts.get(1).is_some_and(|p| { + *p == ".stream" + || p.contains(".schema") + || p.ends_with("stream.json") + || p.starts_with("date=") + }) { + rest + } else { + key + } +} + pub(crate) async fn validate( store: &dyn ObjectStorage, key: &str, @@ -132,6 +159,8 @@ pub(crate) async fn validate( } } else if key.contains(".schema") { serde_json::from_slice::(body).with_context(|| key.to_owned())?; + } else { + serde_json::from_slice::(body).with_context(|| key.to_owned())?; } ensure!( !key.ends_with(".deleting") && !key.contains(".tombstones/"), @@ -155,6 +184,7 @@ pub async fn run( "requires --acknowledge-server-stopped; stop ALL servers and writers first" ); eprintln!("OFFLINE migration: all servers must remain stopped. No writer fencing exists."); + let _transfer_lock = db.transfer_lock().await?; db.migration_preflight(store.as_ref(), opts).await?; if opts.abort { return db.abort_migration(store.as_ref(), action).await; @@ -174,6 +204,7 @@ pub async fn run( matches!( component, ".parseable" + | ".keystone" | ".users" | ".alerts" | ".targets" @@ -197,41 +228,61 @@ pub async fn run( .iter() .filter_map(|(k, modified)| document_scope(k).map(|scope| (k.clone(), scope, *modified))) .collect(); + let canonical: std::collections::HashSet<_> = + keys.iter().map(|(key, _, _)| canonical_key(key)).collect(); + ensure!( + canonical.len() == keys.len(), + "duplicate metadata keys after default-tenant normalization" + ); let mut counts = BTreeMap::<(String, String), (usize, usize)>::new(); - for (key, (tenant, kind), modified) in &keys { - let body = store.get_object(RelativePath::new(key), &None).await?; - if action != Action::Export { - match validate(store.as_ref(), key, &body).await { - Err(e) - if opts.allow_dangling_snapshot_references - && e.to_string().contains("missing manifest") => + use futures::{StreamExt, TryStreamExt}; + let statistics: Vec<_> = futures::stream::iter(&keys) + .map(|(key, (tenant, kind), modified)| { + let store = &store; + async move { + let body = store.get_object(RelativePath::new(key), &None).await?; + if action != Action::Export { + match validate(store.as_ref(), key, &body).await { + Err(e) + if opts.allow_dangling_snapshot_references + && e.to_string().contains("missing manifest") => + { + eprintln!("OVERRIDE dangling snapshot reference: {e:#}") + } + result => result?, + } + } + if chrono::Utc::now() + .signed_duration_since(*modified) + .num_seconds() + < 120 { - eprintln!("OVERRIDE dangling snapshot reference: {e:#}") + eprintln!("WARNING recently modified metadata (possible active writer): {key}"); } - result => result?, + println!( + "bucket tenant={tenant:?} kind={kind} key={key} bytes={} sha256={}", + body.len(), + digest(&body) + ); + Ok::<_, anyhow::Error>(((tenant.clone(), kind.to_string()), body.len())) } - } - if chrono::Utc::now() - .signed_duration_since(*modified) - .num_seconds() - < 120 - { - eprintln!("WARNING recently modified metadata (possible active writer): {key}"); - } - let count = counts - .entry((tenant.clone(), kind.to_string())) - .or_default(); + }) + .buffer_unordered(16) + .try_collect() + .await?; + for (scope, bytes) in statistics { + let count = counts.entry(scope).or_default(); count.0 += 1; - count.1 += body.len(); - println!( - "bucket tenant={tenant:?} kind={kind} key={key} bytes={} sha256={}", - body.len(), - digest(&body) - ); + count.1 += bytes; } eprintln!("counts by (tenant, kind): {counts:?}"); // Dry-run never bootstraps SQL or writes a bucket marker. if opts.migration_dry_run { + if action == Action::Export { + let client = db.client().await?; + db.export_preflight(&client, store.as_ref(), opts, &keys) + .await?; + } return Ok(()); } db.migrate(store, opts, &keys).await @@ -253,8 +304,29 @@ mod tests { use super::*; use crate::{metastore::metastores::postgres_metastore::PostgresOptions, storage::LocalFS}; + #[test] + fn recovery_overrides_require_exact_backend_id() { + for value in [ + None, + Some("true"), + Some("false"), + Some(""), + Some("another-backend"), + ] { + assert!(!recovery_override(value, b"backend")); + } + assert!(recovery_override(Some("backend"), b"backend")); + } #[test] fn inventory_scope_preserves_all_document_namespaces() { + assert_eq!( + canonical_key("DEFAULT_TENANT/logs/.ingestor.node.schema"), + "logs/.ingestor.node.schema" + ); + assert_eq!( + canonical_key("DEFAULT_TENANT/logs/.ingestor.node.stream.json"), + "logs/.ingestor.node.stream.json" + ); for (key, tenant, kind) in [ (".parseable.json", "", "document"), ("logs/.stream.json", "", "stream"), @@ -407,12 +479,74 @@ mod tests { "parquet files must not add inventory requests" ); assert_eq!(before, after); + for expected in [ + "café/.stream/.schema", + ".users/auth0|é/filters/one.json", + manifest_key.as_str(), + ] { + assert!( + after.iter().any(|(key, _)| key == expected), + "inventory missing {expected}" + ); + } for (key, _) in after { store.get_object(RelativePath::new(&key), &None).await?; } Ok(()) } + #[tokio::test] + async fn dangling_import_can_be_exported_with_the_same_override() -> anyhow::Result<()> { + let Ok(url) = std::env::var("P_TEST_POSTGRES_URL") else { + eprintln!("SKIP dangling import/export: P_TEST_POSTGRES_URL unset"); + return Ok(()); + }; + let root = tempfile::tempdir()?; + let storage: Arc = Arc::new(LocalFS::new(root.path().to_owned())); + let admin = PostgresDocuments::new( + &PostgresOptions { + metastore_postgres_url: Some(url.clone()), + ..Default::default() + }, + "dangling".into(), + storage.clone(), + )?; + let database = format!("dangling_{}", uuid::Uuid::new_v4().simple()); + admin + .client() + .await? + .batch_execute(&format!("CREATE DATABASE {database}")) + .await?; + let result = async { + let mut url = url::Url::parse(&url)?; + url.set_path(&database); + let db = PostgresDocuments::new(&PostgresOptions { metastore_postgres_url: Some(url.to_string()), ..Default::default() }, "dangling".into(), storage.clone())?; + let body = Bytes::from_static(br#"{"version":"v3","snapshot":{"manifest_list":[{"manifest_path":"logs/date=2026-01-01/missing.manifest.json"}]}}"#); + storage.put_object(RelativePath::new("logs/.stream/.stream.json"), body.clone(), &None).await?; + let mut command = MigrationOptions { metadata_migration: Some(Action::Import), acknowledge_server_stopped: true, ..Default::default() }; + assert!(run(storage.clone(), &db, &command).await.is_err()); + command.allow_dangling_snapshot_references = true; + run(storage.clone(), &db, &command).await?; + command.metadata_migration = Some(Action::Export); + command.migration_dry_run = true; + command.allow_dangling_snapshot_references = false; + assert!(run(storage.clone(), &db, &command).await.is_err()); + command.allow_dangling_snapshot_references = true; + run(storage.clone(), &db, &command).await?; + command.migration_dry_run = false; + run(storage.clone(), &db, &command).await?; + assert_eq!(storage.get_object(RelativePath::new("logs/.stream/.stream.json"), &None).await?, body); + storage.check_migration_guard().await?; + anyhow::Ok(()) + }.await; + admin + .client() + .await? + .batch_execute(&format!("DROP DATABASE {database} WITH (FORCE)")) + .await?; + result + } + #[tokio::test] async fn migration_round_trip_postgres() -> anyhow::Result<()> { let Ok(url) = std::env::var("P_TEST_POSTGRES_URL") else { @@ -525,6 +659,18 @@ mod tests { )); } } + documents.push(( + "DEFAULT_TENANT/alias/.stream/.stream.json".into(), + Bytes::from(serde_json::to_vec( + &crate::storage::ObjectStoreFormat::default(), + )?), + )); + documents.push(( + "DEFAULT_TENANT/alias/date=2026-01-01/manifest.json".into(), + crate::catalog::manifest::encode_manifest( + &crate::catalog::manifest::Manifest::default(), + )?, + )); for (key, body) in &documents { store .put_object(RelativePath::new(key), body.clone(), &None) @@ -549,6 +695,15 @@ mod tests { .to_string() .contains("pinned backend") ); + let lock = db.transfer_lock().await?; + assert!( + run(store.clone(), &db, &command) + .await + .unwrap_err() + .to_string() + .contains("another offline") + ); + drop(lock); run(store.clone(), &db, &command).await?; assert!(optional(store.as_ref(), PAIR).await?.is_none()); command.migration_dry_run = false; @@ -567,6 +722,19 @@ mod tests { ] { command.injected_failure = Some(step); assert!(run(store.clone(), &db, &command).await.is_err(), "{step}"); + if step == "import-commit" { + command.abort = true; + run(store.clone(), &db, &command).await?; + assert_eq!( + db.client() + .await? + .query_one("SELECT count(*) FROM parseable_documents", &[]) + .await? + .get::<_, i64>(0), + 0 + ); + command.abort = false; + } } command.injected_failure = None; command.abort = true; @@ -580,6 +748,99 @@ mod tests { assert!(run(store.clone(), &db, &command).await.is_err()); command.metadata_migration = Some(Action::Verify); run(store.clone(), &db, &command).await?; + command.metadata_migration = Some(Action::Export); + command.migration_dry_run = true; + run(store.clone(), &db, &command).await?; + let original = store + .get_object(RelativePath::new(&documents[0].0), &None) + .await?; + store + .delete_object(RelativePath::new(&documents[0].0), &None) + .await?; + assert!( + run(store.clone(), &db, &command).await.is_err(), + "deleted imported key must conflict even in dry run" + ); + command.overwrite_bucket_changes_since_import = true; + run(store.clone(), &db, &command).await?; + assert!( + optional(store.as_ref(), &documents[0].0).await?.is_none(), + "dry run must not restore key" + ); + store + .put_object(RelativePath::new(&documents[0].0), original, &None) + .await?; + let extra = ".users/owner/filters/after-import.json"; + store + .put_object(RelativePath::new(extra), Bytes::from_static(b"{}"), &None) + .await?; + command.overwrite_bucket_changes_since_import = false; + assert!(run(store.clone(), &db, &command).await.is_err()); + command.overwrite_bucket_changes_since_import = true; + run(store.clone(), &db, &command).await?; + assert!(optional(store.as_ref(), extra).await?.is_some()); + store.delete_object(RelativePath::new(extra), &None).await?; + command.overwrite_bucket_changes_since_import = false; + db.client() + .await? + .batch_execute( + "ALTER TABLE parseable_import_digests RENAME TO saved_import_digests", + ) + .await?; + // Equal bodies need no baseline, but any changed body does. + db.put_object( + RelativePath::new(&documents[0].0), + Bytes::from_static(b"{}"), + &None, + ) + .await?; + assert!(run(store.clone(), &db, &command).await.is_err()); + command.overwrite_bucket_changes_since_import = true; + run(store.clone(), &db, &command).await?; + db.put_object( + RelativePath::new(&documents[0].0), + documents[0].1.clone(), + &None, + ) + .await?; + db.client() + .await? + .batch_execute( + "ALTER TABLE saved_import_digests RENAME TO parseable_import_digests", + ) + .await?; + command.overwrite_bucket_changes_since_import = false; + let stream_key = documents + .iter() + .find(|(key, _)| key.ends_with("stream.json")) + .unwrap() + .0 + .clone(); + let mut dangling = config.clone(); + dangling.snapshot.manifest_list[0].manifest_path = + "café/date=2026-01-01/missing.manifest.json".into(); + db.put_object( + RelativePath::new(&stream_key), + Bytes::from(serde_json::to_vec(&dangling)?), + &None, + ) + .await?; + assert!(run(store.clone(), &db, &command).await.is_err()); + command.allow_dangling_snapshot_references = true; + run(store.clone(), &db, &command).await?; + db.put_object( + RelativePath::new(&stream_key), + documents + .iter() + .find(|(k, _)| k == &stream_key) + .unwrap() + .1 + .clone(), + &None, + ) + .await?; + command.allow_dangling_snapshot_references = false; + command.migration_dry_run = false; // Simulate crash after committed import, before marker cleanup. let id = optional(store.as_ref(), PAIR).await?.unwrap(); store @@ -630,8 +891,11 @@ mod tests { .await?; let deleted = documents.pop().unwrap().0; let deleted_tenant = document_scope(&deleted).unwrap().0; - db.delete_object(RelativePath::new(&deleted), &Some(deleted_tenant)) - .await?; + db.delete_object( + RelativePath::new(canonical_key(&deleted)), + &Some(deleted_tenant), + ) + .await?; for step in [ "marker", "export-state", @@ -661,7 +925,9 @@ mod tests { assert!(optional(store.as_ref(), &deleted).await?.is_none()); for (key, body) in documents { assert_eq!( - store.get_object(RelativePath::new(&key), &None).await?, + store + .get_object(RelativePath::new(canonical_key(&key)), &None) + .await?, body, "byte-for-byte round trip: {key}" ); diff --git a/src/metastore/metastores/postgres_client.rs b/src/metastore/metastores/postgres_client.rs index 65ede61fd..c6395c459 100644 --- a/src/metastore/metastores/postgres_client.rs +++ b/src/metastore/metastores/postgres_client.rs @@ -9,11 +9,12 @@ use std::{ }; use tokio_postgres::{Row, types::ToSql}; -#[derive(Debug)] pub(crate) struct Client { inner: Option, timeout: Duration, poisoned: Arc, + cancel: tokio_postgres::CancelToken, + tls: tokio_postgres_rustls::MakeRustlsConnect, } impl Drop for Client { fn drop(&mut self) { @@ -35,6 +36,8 @@ impl Drop for Pending<'_> { async fn bounded( timeout: Duration, poisoned: &AtomicBool, + cancel: &tokio_postgres::CancelToken, + tls: &tokio_postgres_rustls::MakeRustlsConnect, call: impl std::future::Future>, ) -> Result { let mut pending = Pending(poisoned, true); @@ -62,6 +65,10 @@ async fn bounded( } Err(_) => { poisoned.store(true, Ordering::SeqCst); + // Cancellation has its own bounded network wait. A lost connection + // still leaves an ambiguous result; never automatically retry writes. + let _ = tokio::time::timeout(Duration::from_secs(1), cancel.cancel_query(tls.clone())) + .await; Err(ObjectStorageError::MetastoreUnavailable( "SQL client deadline exceeded".into(), )) @@ -75,7 +82,14 @@ macro_rules! methods { sql: &str, params: &[&(dyn ToSql + Sync)], ) -> Result, ObjectStorageError> { - bounded(self.timeout, &self.poisoned, self.raw().query(sql, params)).await + bounded( + self.timeout, + &self.poisoned, + &self.cancel, + &self.tls, + self.raw().query(sql, params), + ) + .await } pub async fn query_one( &self, @@ -85,6 +99,8 @@ macro_rules! methods { bounded( self.timeout, &self.poisoned, + &self.cancel, + &self.tls, self.raw().query_one(sql, params), ) .await @@ -97,6 +113,8 @@ macro_rules! methods { bounded( self.timeout, &self.poisoned, + &self.cancel, + &self.tls, self.raw().query_opt(sql, params), ) .await @@ -109,23 +127,45 @@ macro_rules! methods { bounded( self.timeout, &self.poisoned, + &self.cancel, + &self.tls, self.raw().execute(sql, params), ) .await } pub async fn batch_execute(&self, sql: &str) -> Result<(), ObjectStorageError> { - bounded(self.timeout, &self.poisoned, self.raw().batch_execute(sql)).await + bounded( + self.timeout, + &self.poisoned, + &self.cancel, + &self.tls, + self.raw().batch_execute(sql), + ) + .await } }; } impl Client { - pub fn new(inner: deadpool_postgres::Client, timeout: Duration) -> Self { + pub fn new( + inner: deadpool_postgres::Client, + timeout: Duration, + tls: tokio_postgres_rustls::MakeRustlsConnect, + ) -> Self { + let cancel = inner.cancel_token(); Self { inner: Some(inner), timeout, + cancel, + tls, poisoned: Arc::new(AtomicBool::new(false)), } } + pub fn discard_on_drop(&self) { + self.poisoned.store(true, Ordering::SeqCst); + } + pub fn set_timeout(&mut self, timeout: Duration) { + self.timeout = timeout; + } fn raw(&self) -> &tokio_postgres::Client { self.inner.as_ref().unwrap() } @@ -134,6 +174,8 @@ impl Client { let inner = bounded( self.timeout, &self.poisoned, + &self.cancel, + &self.tls, self.inner.as_mut().unwrap().transaction(), ) .await?; @@ -141,6 +183,8 @@ impl Client { inner, timeout: self.timeout, poisoned: self.poisoned.clone(), + cancel: self.cancel.clone(), + tls: self.tls.clone(), }) } } @@ -148,6 +192,8 @@ pub(crate) struct Transaction<'a> { inner: deadpool_postgres::Transaction<'a>, timeout: Duration, poisoned: Arc, + cancel: tokio_postgres::CancelToken, + tls: tokio_postgres_rustls::MakeRustlsConnect, } impl Transaction<'_> { fn raw(&self) -> &tokio_postgres::Transaction<'_> { @@ -155,6 +201,13 @@ impl Transaction<'_> { } methods!(); pub async fn commit(self) -> Result<(), ObjectStorageError> { - bounded(self.timeout, &self.poisoned, self.inner.commit()).await + bounded( + self.timeout, + &self.poisoned, + &self.cancel, + &self.tls, + self.inner.commit(), + ) + .await } } diff --git a/src/metastore/metastores/postgres_documents.rs b/src/metastore/metastores/postgres_documents.rs index e79f9e423..178bc6563 100644 --- a/src/metastore/metastores/postgres_documents.rs +++ b/src/metastore/metastores/postgres_documents.rs @@ -8,9 +8,10 @@ use bytes::Bytes; use relative_path::RelativePath; use std::{sync::Arc, time::Duration}; -#[derive(Debug)] pub struct PostgresDocuments { pool: deadpool_postgres::Pool, + lock_pool: deadpool_postgres::Pool, + tls: tokio_postgres_rustls::MakeRustlsConnect, deadline: Duration, bucket: String, expected_id: Option, @@ -19,6 +20,13 @@ pub struct PostgresDocuments { #[cfg(test)] pub(super) deletion_fault: std::sync::atomic::AtomicU8, } +impl std::fmt::Debug for PostgresDocuments { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("PostgresDocuments") + .field("bucket", &self.bucket) + .finish_non_exhaustive() + } +} fn err(e: impl std::fmt::Display) -> ObjectStorageError { if e.to_string().contains("stream deletion pending") { ObjectStorageError::StreamDeleting @@ -26,6 +34,12 @@ fn err(e: impl std::fmt::Display) -> ObjectStorageError { ObjectStorageError::MetastoreUnavailable(e.to_string()) } } +fn legacy_key<'a>(key: &'a str, base: &str) -> Option<&'a str> { + if base.is_empty() { + return None; + } + key.strip_prefix(base).and_then(|p| p.strip_prefix('/')) +} fn tenant(t: &Option) -> &str { t.as_deref() .filter(|v| !v.is_empty() && *v != "DEFAULT_TENANT") @@ -97,10 +111,15 @@ impl PostgresDocuments { .with_safe_default_protocol_versions()? .with_root_certificates(roots) .with_no_client_auth(); - let manager = deadpool_postgres::Manager::new( - config, - tokio_postgres_rustls::MakeRustlsConnect::new(tls), - ); + let tls = tokio_postgres_rustls::MakeRustlsConnect::new(tls); + let lock_manager = deadpool_postgres::Manager::new(config.clone(), tls.clone()); + let manager = deadpool_postgres::Manager::new(config, tls.clone()); + let lock_pool = deadpool_postgres::Pool::builder(lock_manager) + .max_size(2) + .runtime(deadpool_postgres::Runtime::Tokio1) + .wait_timeout(Some(timeout)) + .create_timeout(Some(timeout)) + .build()?; let pool = deadpool_postgres::Pool::builder(manager) .max_size(options.metastore_postgres_pool_size as usize) .runtime(deadpool_postgres::Runtime::Tokio1) @@ -110,6 +129,8 @@ impl PostgresDocuments { .build()?; Ok(Self { pool, + lock_pool, + tls, deadline: Duration::from_millis( options .metastore_postgres_statement_timeout_ms @@ -132,8 +153,46 @@ impl PostgresDocuments { .map_err(err)? .map_err(err)?, self.deadline, + self.tls.clone(), )) } + async fn migration_client(&self) -> Result { + let millis: u64 = std::env::var("P_METASTORE_POSTGRES_MIGRATION_TIMEOUT_MS") + .unwrap_or_else(|_| "1800000".into()) + .parse() + .map_err(err)?; + if millis == 0 { + return Err(err("migration timeout must be positive")); + } + let mut client = self.client().await?; + client.discard_on_drop(); + client.set_timeout(Duration::from_millis(millis.saturating_add(250))); + client + .batch_execute(&format!( + "SET statement_timeout={millis}; SET lock_timeout={millis}" + )) + .await?; + Ok(client) + } + pub(crate) async fn transfer_lock( + &self, + ) -> Result { + let client = super::postgres_client::Client::new( + self.lock_pool.get().await.map_err(err)?, + self.deadline, + self.tls.clone(), + ); + client.discard_on_drop(); // Close socket to release session lock on every exit. + client.batch_execute("SET idle_session_timeout=0").await?; + let acquired: bool = client + .query_one("SELECT pg_try_advisory_lock(8104050183641399917)", &[]) + .await? + .get(0); + if !acquired { + return Err(err("another offline metadata transfer is running")); + } + Ok(client) + } pub async fn check_migration_guard(&self) -> Result<(), ObjectStorageError> { super::migration::guard(self.storage.as_ref(), true) .await @@ -144,7 +203,7 @@ impl PostgresDocuments { .await .map_err(err)?; self.initialized.get_or_try_init(|| async { - let mut client = self.client().await?; + let mut client = self.migration_client().await?; let tx = client.transaction().await.map_err(err)?; tx.batch_execute("SELECT pg_advisory_xact_lock(8104050183641399916)").await.map_err(err)?; let (actual, _, created) = self.bootstrap(&tx, &uuid::Uuid::new_v4().to_string()).await?; @@ -161,8 +220,8 @@ impl PostgresDocuments { tokio::time::sleep(Duration::from_millis(10)).await; } } - if !created && super::migration::optional(self.storage.as_ref(), ".parseable/postgres-metastore-id").await.map_err(err)?.is_none() && std::env::var("P_METASTORE_POSTGRES_ALLOW_UNPAIRED").as_deref() != Ok("true") { - return Err(err("nonempty database and missing bucket pairing marker; restore the marker or explicitly set P_METASTORE_POSTGRES_ALLOW_UNPAIRED=true after verifying the bucket")); + if !created && super::migration::optional(self.storage.as_ref(), ".parseable/postgres-metastore-id").await.map_err(err)?.is_none() && !super::migration::recovery_override(std::env::var("P_METASTORE_POSTGRES_ALLOW_UNPAIRED").ok().as_deref(), actual.as_bytes()) { + return Err(err("nonempty database and missing bucket pairing marker; restore the marker or explicitly set P_METASTORE_POSTGRES_ALLOW_UNPAIRED= after verifying the bucket")); } match self.storage.put_metadata_conditional(marker, Bytes::from(actual.clone()), MetadataWriteCondition::Create, &None).await { Ok(_) | Err(ConditionalMetadataError::Conflict) => {}, @@ -197,7 +256,7 @@ impl PostgresDocuments { { return Err(err("backend/bucket identity mismatch")); } - if !(0..=2).contains(&row.get::<_, i32>(2)) || row.get::<_, i32>(3) != 1 { + if !(0..=3).contains(&row.get::<_, i32>(2)) || row.get::<_, i32>(3) != 1 { return Err(err( "unsupported schema/protocol version; upgrade this server", )); @@ -218,6 +277,10 @@ impl PostgresDocuments { .await .map_err(err)?; } + if row.get::<_, i32>(2) < 3 { + tx.batch_execute(include_str!("../migrations/003_manifest_admission.sql")) + .await?; + } tx.batch_execute("CREATE TABLE IF NOT EXISTS parseable_transfer (singleton boolean PRIMARY KEY DEFAULT true CHECK(singleton), state text NOT NULL)").await.map_err(err)?; let has_documents: bool = tx .query_one("SELECT EXISTS(SELECT 1 FROM parseable_documents)", &[]) @@ -382,33 +445,72 @@ impl PostgresDocuments { paths: &[String], t: &Option, ) -> Result, ObjectStorageError> { - let base = self.storage.absolute_url(RelativePath::new("")); - let normalized: Vec = paths - .iter() - .map(|p| { - p.strip_prefix(base.as_ref()) - .and_then(|p| p.strip_prefix('/')) - .unwrap_or(p) - .to_owned() - }) - .collect(); - let rows = self - .client() - .await? - .query( - "SELECT key,body FROM parseable_documents WHERE tenant=$1 AND key=ANY($2)", - &[&tenant(t), &normalized], - ) - .await?; - let bodies: std::collections::HashMap = rows - .into_iter() - .map(|r| (r.get(0), Bytes::from(r.get::<_, Vec>(1)))) - .collect(); - Ok(paths - .iter() - .zip(normalized) - .filter_map(|(original, key)| bodies.get(&key).map(|b| (original.clone(), b.clone()))) - .collect()) + // Bound both parameter count and encoded key bytes. Each query returns at + // most 32 bodies; a single large manifest remains an indivisible document. + let mut result = Vec::new(); + let mut offset = 0; + while offset < paths.len() { + let mut end = offset; + let mut bytes = 0; + while end < paths.len() && end - offset < 32 { + let size = paths[end].len(); + if end > offset && bytes + size > 64 * 1024 { + break; + } + bytes += size; + end += 1; + } + let base = self.storage.absolute_url(RelativePath::new("")); + let mut keys = Vec::new(); + for path in &paths[offset..end] { + keys.push(path.clone()); + if let Some(legacy) = legacy_key(path, base.as_ref()) { + keys.push(legacy.to_owned()); + } + } + let client = self.client().await?; + let sizes = client.query("SELECT key,octet_length(body) FROM parseable_documents WHERE tenant=$1 AND key=ANY($2)", &[&tenant(t), &keys]).await?; + let mut bodies = std::collections::HashMap::::new(); + let mut batch = Vec::::new(); + let mut batch_bytes = 0usize; + for row in sizes { + let size = row.get::<_, i32>(1) as usize; + if !batch.is_empty() && batch_bytes + size > 8 * 1024 * 1024 { + for r in client.query("SELECT key,body FROM parseable_documents WHERE tenant=$1 AND key=ANY($2)", &[&tenant(t), &batch]).await? { + bodies.insert(r.get(0), Bytes::from(r.get::<_, Vec>(1))); + } + batch.clear(); + batch_bytes = 0; + } + batch.push(row.get(0)); + batch_bytes += size; + } + if !batch.is_empty() { + for r in client + .query( + "SELECT key,body FROM parseable_documents WHERE tenant=$1 AND key=ANY($2)", + &[&tenant(t), &batch], + ) + .await? + { + bodies.insert(r.get(0), Bytes::from(r.get::<_, Vec>(1))); + } + } + for path in &paths[offset..end] { + let fallback = legacy_key(path, base.as_ref()).and_then(|k| bodies.get(k)); + if bodies.contains_key(path) && fallback.is_some() { + tracing::warn!( + key = path, + "exact and legacy metadata keys exist; using exact key" + ); + } + if let Some(body) = bodies.get(path).or(fallback) { + result.push((path.clone(), body.clone())); + } + } + offset = end; + } + Ok(result) } pub async fn get_object( &self, @@ -427,21 +529,22 @@ impl PostgresDocuments { t: &Option, ) -> Result, ConditionalMetadataError> { let base = self.storage.absolute_url(RelativePath::new("")); - let normalized = path - .as_str() - .strip_prefix(base.as_ref()) - .and_then(|p| p.strip_prefix('/')) - .unwrap_or(path.as_str()); let c = self.client().await?; - Ok(c.query_opt( - "SELECT body,version FROM parseable_documents WHERE tenant=$1 AND key=$2", - &[&tenant(t), &normalized], - ) - .await - .map_err(err)? - .map(|r| VersionedMetadata { - bytes: Bytes::from(r.get::<_, Vec>(0)), - version: version(r.get(1)), + let exact = path.as_str(); + let fallback = legacy_key(exact, base.as_ref()).unwrap_or(exact); + let rows = c.query( + "SELECT key,body,version FROM parseable_documents WHERE tenant=$1 AND (key=$2 OR key=$3) ORDER BY (key=$2) DESC", + &[&tenant(t), &exact, &fallback], + ).await?; + if rows.len() > 1 { + tracing::warn!( + key = exact, + "exact and legacy metadata keys exist; using exact key" + ); + } + Ok(rows.first().map(|r| VersionedMetadata { + bytes: Bytes::from(r.get::<_, Vec>(1)), + version: version(r.get(2)), })) } pub async fn put_object( @@ -576,6 +679,12 @@ impl PostgresDocuments { if path.as_str().is_empty() || path.as_str().contains('/') { return Err(err("tenant deletion requires one nonempty path component")); } + if self.client().await?.query_one( + "SELECT EXISTS(SELECT 1 FROM parseable_documents WHERE tenant='' AND (key >= $1 || '/' AND key < $1 || '0') AND (key LIKE '%/.stream/%' OR key LIKE '%/.stream.json' OR key LIKE '%/.schema'))", + &[&path.as_str()], + ).await?.get::<_, bool>(0) { + return Err(err(format!("tenant {} collides with a default-tenant stream; refusing data deletion", path))); + } self.delete_data(path, &None).await?; self.client() .await? @@ -969,7 +1078,7 @@ impl PostgresDocuments { use super::migration::{Action, BUSY, PAIR, digest, optional}; use anyhow::ensure; let action = opts.metadata_migration.unwrap(); - let mut client = self.client().await?; + let mut client = self.migration_client().await?; // All remote I/O and validation happen before the import transaction. let tx = &client; if action == Action::Verify { @@ -979,7 +1088,7 @@ impl PostgresDocuments { .get(0); let bucket_keys: std::collections::BTreeSet<_> = keys .iter() - .map(|(k, (t, _), _)| (t.as_str(), k.as_str())) + .map(|(k, (t, _), _)| (t.as_str(), super::migration::canonical_key(k))) .collect(); let mut differences = 0; for (key, (tenant, kind), _) in keys { @@ -987,7 +1096,7 @@ impl PostgresDocuments { let sql = if exists { tx.query_opt( "SELECT body FROM parseable_documents WHERE tenant=$1 AND key=$2", - &[tenant, key], + &[tenant, &super::migration::canonical_key(key)], ) .await? .map(|r| r.get::<_, Vec>(0)) @@ -1075,51 +1184,8 @@ impl PostgresDocuments { pair.is_some() || busy.is_some(), "export requires a paired bucket or interrupted export" ); - // Preflight every destination before any overwrite. - let mut last = (String::new(), String::new()); - loop { - let rows = tx.query("SELECT tenant,key,body FROM parseable_documents WHERE (tenant COLLATE \"C\",key COLLATE \"C\") > ($1,$2) ORDER BY tenant COLLATE \"C\",key COLLATE \"C\" LIMIT 32", &[&last.0,&last.1]).await?; - if rows.is_empty() { - break; - } - for row in rows { - last = (row.get(0), row.get(1)); - let body: Vec = row.get(2); - self.validate_export_row(&client, &last.1, &Bytes::from(body.clone())) - .await?; - if let Some(old) = optional(store.as_ref(), &last.1).await? { - let baseline = self.import_digest(&client, &last.1).await?; - ensure!( - old.as_ref() == body - || baseline.as_deref() == Some(digest(&old).as_str()) - || opts.overwrite_bucket_changes_since_import, - "bucket changed since import: {}; after backup use --overwrite-bucket-changes-since-import", - last.1 - ); - } - } - } - // Bucket-only metadata would resurrect deleted SQL objects on downgrade. - for (key, (tenant, _), _) in keys { - if tx - .query_opt( - "SELECT 1 FROM parseable_documents WHERE tenant=$1 AND key=$2", - &[tenant, key], - ) - .await? - .is_none() - { - ensure!( - self.import_digest(&client, key).await?.as_deref() - == Some( - digest(&store.get_object(RelativePath::new(key), &None).await?) - .as_str() - ) - || opts.overwrite_bucket_changes_since_import, - "bucket-only metadata changed since import: {key}; use --overwrite-bucket-changes-since-import after backup" - ); - } - } + self.export_preflight(&client, store.as_ref(), opts, keys) + .await?; } store .put_metadata_conditional( @@ -1138,11 +1204,34 @@ impl PostgresDocuments { })?; super::migration::failure_point(opts, "marker")?; if action == Action::Import { - let spool = tempfile::tempdir_in(std::env::current_dir()?)?; - for (index, (key, _, _)) in keys.iter().enumerate() { - let body = store.get_object(RelativePath::new(key), &None).await?; - std::fs::write(spool.path().join(index.to_string()), &body)?; + drop(client); + let spool_root = + std::env::current_dir()?.join(format!(".parseable-migration-spool-{id}")); + // The session transfer lock proves no other invocation uses this directory. + if spool_root.exists() { + std::fs::remove_dir_all(&spool_root)?; + } + std::fs::create_dir(&spool_root)?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&spool_root, std::fs::Permissions::from_mode(0o700))?; } + let spool = tempfile::tempdir_in(&spool_root)?; + use futures::{StreamExt, TryStreamExt}; + futures::stream::iter(keys.iter().enumerate()) + .map(|(index, (key, _, _))| { + let store = &store; + let path = spool.path().join(index.to_string()); + async move { + let body = store.get_object(RelativePath::new(key), &None).await?; + std::fs::write(path, &body)?; + Ok::<_, anyhow::Error>(()) + } + }) + .buffer_unordered(16) + .try_collect::>() + .await?; super::migration::failure_point(opts, "spool")?; let current = store.metadata_inventory().await?; let current: Vec<_> = current @@ -1157,6 +1246,7 @@ impl PostgresDocuments { .all(|((k, m), (old, _, om))| k == old && m == om), "bucket metadata changed during import" ); + let mut client = self.client().await?; let tx = client.transaction().await?; tx.query("SELECT pg_advisory_xact_lock(8104050183641399916)", &[]) .await?; @@ -1172,14 +1262,16 @@ impl PostgresDocuments { )?); tenants.push(tenant.clone()); kinds.push(kind.to_string()); - paths.push(key.clone()); + paths.push(super::migration::canonical_key(key).to_owned()); bodies.push(body.to_vec()); } if count == 0 { tx.execute("INSERT INTO parseable_documents(tenant,kind,key,body) SELECT * FROM unnest($1::text[],$2::text[],$3::text[],$4::bytea[])", &[&tenants,&kinds,&paths,&bodies]).await?; } - for ((key, tenant), body) in paths.iter().zip(&tenants).zip(&bodies) { - tx.execute("INSERT INTO parseable_import_digests(key,digest) VALUES($1,$2) ON CONFLICT DO NOTHING", &[key, &digest(body)]).await?; + for (((key, tenant), body), (source_key, _, _)) in + paths.iter().zip(&tenants).zip(&bodies).zip(batch) + { + tx.execute("INSERT INTO parseable_import_digests(key,digest) VALUES($1,$2) ON CONFLICT DO NOTHING", &[source_key, &digest(body)]).await?; let reread: Vec = tx .query_one( "SELECT body FROM parseable_documents WHERE tenant=$1 AND key=$2", @@ -1265,12 +1357,21 @@ impl PostgresDocuments { } for row in rows { last_key = row.get(0); - super::migration::validate( + let result = super::migration::validate( store.as_ref(), &last_key, &Bytes::from(row.get::<_, Vec>(1)), ) - .await?; + .await; + match result { + Err(e) + if opts.allow_dangling_snapshot_references + && e.to_string().contains("missing manifest") => + { + eprintln!("OVERRIDE {e:#}") + } + result => result?, + } } } if optional(store.as_ref(), PAIR).await?.is_some() { @@ -1284,6 +1385,119 @@ impl PostgresDocuments { } impl PostgresDocuments { + pub(crate) async fn export_preflight( + &self, + client: &super::postgres_client::Client, + store: &dyn ObjectStorage, + opts: &super::migration::MigrationOptions, + keys: &[super::migration::InventoryDocument], + ) -> anyhow::Result<()> { + use super::migration::{digest, optional}; + let mut conflicts = 0usize; + let resuming_export = optional(store, super::migration::BUSY) + .await? + .is_some_and(|marker| marker.starts_with(b"Export:")) + && client + .query_opt("SELECT state FROM parseable_transfer WHERE singleton", &[]) + .await? + .is_some_and(|row| row.get::<_, String>(0) == "exported"); + let has_digests: bool = client + .query_one( + "SELECT to_regclass('parseable_import_digests') IS NOT NULL", + &[], + ) + .await? + .get(0); + if has_digests && !resuming_export { + let mut last_key = String::new(); + loop { + let rows = client.query("SELECT key FROM parseable_import_digests WHERE key COLLATE \"C\" > $1 ORDER BY key COLLATE \"C\" LIMIT 128", &[&last_key]).await?; + if rows.is_empty() { + break; + } + for row in rows { + last_key = row.get(0); + let canonical = super::migration::canonical_key(&last_key); + if canonical != last_key + && optional(store, &last_key).await?.is_none() + && client + .query_opt( + "SELECT 1 FROM parseable_documents WHERE key=$1", + &[&canonical], + ) + .await? + .is_some() + { + conflicts += 1; + if conflicts <= 100 { + eprintln!("CONFLICT deleted import alias key={last_key}"); + } + } + } + } + } + // Preflight every destination before any overwrite. + let mut last = (String::new(), String::new()); + loop { + let rows = client.query("SELECT tenant,key,body FROM parseable_documents WHERE (tenant COLLATE \"C\",key COLLATE \"C\") > ($1,$2) ORDER BY tenant COLLATE \"C\",key COLLATE \"C\" LIMIT 32", &[&last.0,&last.1]).await?; + if rows.is_empty() { + break; + } + for row in rows { + last = (row.get(0), row.get(1)); + let body: Vec = row.get(2); + self.validate_export_row( + client, + &last.1, + &Bytes::from(body.clone()), + opts.allow_dangling_snapshot_references, + ) + .await?; + let old = optional(store, &last.1).await?; + let baseline = self.import_digest(client, &last.1).await?; + let unchanged = match old { + Some(old) => { + old.as_ref() == body || baseline.as_deref() == Some(digest(&old).as_str()) + } + None => baseline.is_none(), + }; + if !unchanged { + conflicts += 1; + if conflicts <= 100 { + eprintln!("CONFLICT key={}", last.1); + } + } + } + } + // Bucket-only metadata would resurrect deleted SQL objects on downgrade. + for (key, (tenant, _), _) in keys { + if client + .query_opt( + "SELECT 1 FROM parseable_documents WHERE tenant=$1 AND key=$2", + &[tenant, key], + ) + .await? + .is_none() + { + if self.import_digest(client, key).await?.as_deref() + != Some( + digest(&store.get_object(RelativePath::new(key), &None).await?).as_str(), + ) + { + conflicts += 1; + if conflicts <= 100 { + eprintln!("CONFLICT bucket-only key={key}"); + } + } + } + } + anyhow::ensure!( + conflicts == 0 || opts.overwrite_bucket_changes_since_import, + "{conflicts} bucket conflicts (first 100 shown); after backup use --overwrite-bucket-changes-since-import" + ); + Ok(()) + } + pub(crate) async fn migration_preflight( &self, store: &dyn ObjectStorage, @@ -1303,15 +1517,10 @@ impl PostgresDocuments { self.expected_id.is_none(), "pinned backend id requires an initialized database" ); - if let Some(marker) = &busy { - ensure!( - opts.metadata_migration == Some(Action::Import) - && std::str::from_utf8(marker).is_ok_and(|s| s - .strip_prefix("Import:") - .is_some_and(|id| uuid::Uuid::parse_str(id).is_ok())), - "another transfer owns migration marker" - ); - } + ensure!( + busy.is_none(), + "migration marker requires its original initialized database" + ); ensure!( pair.is_none(), "paired bucket requires its original database" @@ -1331,7 +1540,7 @@ impl PostgresDocuments { let id: String = row.get(0); ensure!( row.get::<_, String>(1) == self.bucket - && (0..=2).contains(&row.get::<_, i32>(2)) + && (0..=3).contains(&row.get::<_, i32>(2)) && row.get::<_, i32>(3) == 1, "backend identity/version mismatch" ); @@ -1369,6 +1578,14 @@ impl PostgresDocuments { "refusing import into nonempty database" ); } + if opts.metadata_migration == Some(Action::Import) && !opts.migration_dry_run { + // Caller already holds the transfer lock and identity checks passed. + uuid::Uuid::parse_str(&id)?; + let stale = std::env::current_dir()?.join(format!(".parseable-migration-spool-{id}")); + if stale.exists() { + std::fs::remove_dir_all(stale)?; + } + } Ok(()) } @@ -1401,6 +1618,7 @@ impl PostgresDocuments { client: &super::postgres_client::Client, key: &str, body: &Bytes, + allow_dangling: bool, ) -> anyhow::Result<()> { use anyhow::ensure; ensure!( @@ -1422,13 +1640,17 @@ impl PostgresDocuments { .unwrap_or(&manifest_path); let row = client .query_opt( - "SELECT body FROM parseable_documents WHERE key=$1", - &[&normalized], + "SELECT body FROM parseable_documents WHERE key=$1 OR key=$2 ORDER BY (key=$1) DESC LIMIT 1", + &[&manifest_path, &normalized], ) .await?; - let row = row.ok_or_else(|| { - anyhow::anyhow!("{key}: missing SQL manifest {}", manifest_path) - })?; + let Some(row) = row else { + if allow_dangling { + eprintln!("OVERRIDE {key}: missing SQL manifest {manifest_path}"); + continue; + } + anyhow::bail!("{key}: missing SQL manifest {manifest_path}"); + }; crate::catalog::manifest::decode_manifest(&Bytes::from(row.get::<_, Vec>(0)))?; } } else if key.contains(".schema") { @@ -1465,24 +1687,44 @@ impl PostgresDocuments { .query_one("SELECT to_regclass('parseable_documents') IS NOT NULL", &[]) .await? .get(0); - if docs { - let count: i64 = client - .query_one("SELECT count(*) FROM parseable_documents", &[]) - .await? - .get(0); + let pair = optional(store, super::migration::PAIR).await?; + let busy = optional(store, BUSY).await?; + let id = if docs { + Some( + client + .query_one("SELECT id FROM parseable_backend", &[]) + .await? + .get::<_, String>(0), + ) + } else { + None + }; + let unpaired_commit = state.as_deref() == Some("ready") + && pair.is_none() + && id.as_ref().is_some_and(|id| { + busy.as_ref() + .is_some_and(|b| b.as_ref() == format!("Import:{id}").as_bytes()) + }); + if unpaired_commit { + client.batch_execute("BEGIN; DELETE FROM parseable_documents; DELETE FROM parseable_import_digests; DELETE FROM parseable_transfer; COMMIT;").await?; + } else { + let count: i64 = if docs { + client + .query_one("SELECT count(*) FROM parseable_documents", &[]) + .await? + .get(0) + } else { + 0 + }; ensure!( - count == 0, - "SQL contains committed documents; complete import and export instead of aborting" + count == 0 && state.is_none() && pair.is_none(), + "import committed or paired; complete import then export instead of aborting" ); } - ensure!( - state.is_none(), - "import committed; complete import, then export to return to bucket mode" - ); } Action::Export => { ensure!( - state.as_deref() == Some("ready"), + state.is_none() || state.as_deref() == Some("ready"), "export has begun writing; rerun export to finish; abort is unsafe" ); } diff --git a/src/metastore/metastores/postgres_metastore.rs b/src/metastore/metastores/postgres_metastore.rs index 072c9db6b..7e93ecace 100644 --- a/src/metastore/metastores/postgres_metastore.rs +++ b/src/metastore/metastores/postgres_metastore.rs @@ -8,11 +8,12 @@ super::document_metastore::document_metastore!( paths: &[String], tenant: &Option, ) -> Result>>, MetastoreError> { - let rows = self.storage.manifest_batch(paths, tenant).await?; let mut result: HashMap> = paths.iter().map(|p| (p.clone(), None)).collect(); - for (key, body) in rows { - result.insert(key, Some(decode_manifest_blocking(body).await?)); + for chunk in paths.chunks(32) { + for (key, body) in self.storage.manifest_batch(chunk, tenant).await? { + result.insert(key, Some(decode_manifest_blocking(body).await?)); + } } Ok(Some(result)) } @@ -227,7 +228,21 @@ pub async fn run_migration( "migration requires explicit P_STAGING_DIR or --local-staging-path" ); // Staging metadata alone is normal; event buffers must have uploaded first. - let mut dirs = vec![options.local_staging_path.clone()]; + let staging = options.local_staging_path.canonicalize().map_err(|e| { + anyhow::anyhow!( + "staging directory must exist ({}): {e}", + options.local_staging_path.display() + ) + })?; + anyhow::ensure!(staging.is_dir(), "staging path must be a directory"); + eprintln!("Resolved staging directory: {}", staging.display()); + eprintln!( + "Environment file: {}", + crate::interactive::loaded_env_path() + .map(|path| path.display().to_string()) + .unwrap_or_else(|| "none (explicit environment/CLI only)".into()) + ); + let mut dirs = vec![staging]; while let Some(dir) = dirs.pop() { if !dir.exists() { continue; @@ -303,7 +318,12 @@ mod contract_tests { options.multi_tenancy = true; options.postgres.metastore_postgres_url = Some(std::env::var("P_CONTRACT_STARTUP_URL").unwrap()); - options.postgres.metastore_postgres_statement_timeout_ms = 500; + options.postgres.metastore_postgres_statement_timeout_ms = + if std::env::var("P_CONTRACT_STARTUP_FAILURE").as_deref() == Ok("none") { + 500 + } else { + 5000 + }; Arc::new(PostgresMetastore { storage: Arc::new( super::super::postgres_documents::PostgresDocuments::new( @@ -800,15 +820,18 @@ mod contract_tests { .unwrap() .is_some() ); - // Preserve the existing default-state directory mismatch. - let placeholder = Document { - id: "placeholder".into(), - path: format!("{}/.alerts/placeholder", crate::parseable::DEFAULT_TENANT), - value: String::new(), - }; - store.put_dashboard(&placeholder, &None).await.unwrap(); let states = store.get_alert_states(&tenant).await.unwrap(); - assert_eq!(states.len(), usize::from(tenant.is_some())); + assert_eq!(states.len(), 1); + if tenant.is_none() { + assert_eq!( + store + .get_alert_states(&Some(crate::parseable::DEFAULT_TENANT.into())) + .await + .unwrap() + .len(), + 1 + ); + } store.delete_alert_state(&state, &tenant).await.unwrap(); assert!( store @@ -1077,7 +1100,7 @@ mod contract_tests { let storage: Arc = Arc::new(LocalFS::new(dir.path().to_owned())); let options = PostgresOptions { metastore_postgres_url: Some(url.to_string()), - metastore_postgres_statement_timeout_ms: 500, + metastore_postgres_statement_timeout_ms: if case == 7 { 5000 } else { 500 }, metastore_postgres_connect_timeout_ms: 500, ..Default::default() }; @@ -1093,7 +1116,41 @@ mod contract_tests { let store = PostgresMetastore { storage: documents.clone(), }; - if case == 0 { + if case == 7 { + let name = dir.path().file_name().unwrap().to_str().unwrap(); + for tenant in [None, Some(name.to_owned())] { + let prefix = tenant.as_ref().map(|t| format!("{t}/")).unwrap_or_default(); + let exact = format!("{prefix}{name}/date=2026-01-01/manifest.json"); + documents.put_object(RelativePath::new(&exact), Bytes::from_static(b"exact"), &tenant).await.unwrap(); + assert_eq!(documents.get_object(RelativePath::new(&exact), &tenant).await.unwrap(), Bytes::from_static(b"exact")); + let absolute = format!("{}/{exact}", dir.path().to_string_lossy().trim_start_matches('/')); + assert_eq!(documents.get_object(RelativePath::new(&absolute), &tenant).await.unwrap(), Bytes::from_static(b"exact")); + let paths = vec![exact.clone(), absolute]; + assert_eq!(documents.manifest_batch(&paths, &tenant).await.unwrap().len(), 2); + documents.put_object(RelativePath::new(&format!("{prefix}{name}/.stream/.deleting")), Bytes::new(), &tenant).await.unwrap(); + assert!(matches!(documents.put_object(RelativePath::new(&exact), Bytes::new(), &tenant).await, Err(ObjectStorageError::StreamDeleting))); + } + let collision = super::super::postgres_documents::PostgresDocuments::new( + &options, database.clone(), Arc::new(LocalFS::new(std::path::PathBuf::from("logs")))).unwrap(); + for tenant in [None, Some("logs".to_owned())] { + let key = if tenant.is_none() { "logs/date=2026-01-01/exact.manifest.json" } else { "logs/logs/date=2026-01-01/exact.manifest.json" }; + let fallback = key.strip_prefix("logs/").unwrap(); + collision.put_object(RelativePath::new(key), Bytes::from_static(b"exact"), &tenant).await.unwrap(); + collision.put_object(RelativePath::new(fallback), Bytes::from_static(b"fallback"), &tenant).await.unwrap(); + assert_eq!(collision.get_object(RelativePath::new(key), &tenant).await.unwrap(), Bytes::from_static(b"exact")); + assert_eq!(collision.manifest_batch(&[key.to_owned()], &tenant).await.unwrap()[0].1, Bytes::from_static(b"exact")); + } + // SQL tenant candidates cannot bypass deletion collision protection. + store.put_stream_json(&crate::storage::ObjectStoreFormat::default(), "protected", &None).await.unwrap(); + documents.put_object(RelativePath::new("protected/other/.stream/.schema"), Bytes::new(), &Some("protected".into())).await.unwrap(); + assert!(documents.delete_prefix(RelativePath::new("protected"), &None).await.unwrap_err().to_string().contains("collides")); + store.put_stream_json(&crate::storage::ObjectStoreFormat::default(), "alias", &Some(crate::parseable::DEFAULT_TENANT.into())).await.unwrap(); + assert!(store.stream_exists("alias", &None).await.unwrap()); + assert!(documents.get_object(RelativePath::new("alias/.stream/.stream.json"), &None).await.is_ok()); + let paths: Vec<_> = (0..70).map(|i| format!("batch/date=2026-01-01/{i}.manifest.json")).collect(); + for key in &paths { documents.put_object(RelativePath::new(key), Bytes::new(), &None).await.unwrap(); } + assert_eq!(documents.manifest_batch(&paths, &None).await.unwrap().len(), 70); + } else if case == 0 { let date = chrono::Utc::now(); let manifest = crate::catalog::manifest::Manifest::default(); store @@ -1276,7 +1333,7 @@ mod contract_tests { .unwrap(); // Restrict the injected SQL error to the per-tenant body read; // volatile view expressions can also execute during discovery. - client.batch_execute("ALTER TABLE parseable_documents RENAME TO startup_documents; CREATE FUNCTION startup_read_failure() RETURNS bytea LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'injected transient tenant read failure' USING ERRCODE='08006'; END $$; CREATE VIEW parseable_documents AS SELECT tenant,kind,key,CASE WHEN tenant='z-valid' AND key='z-valid/.parseable.json' AND current_query() LIKE 'SELECT body%FROM parseable_documents%' THEN startup_read_failure() ELSE body END AS body,version,created_at,updated_at FROM startup_documents;").await.unwrap(); + client.batch_execute("ALTER TABLE parseable_documents RENAME TO startup_documents; CREATE FUNCTION startup_read_failure() RETURNS bytea LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'injected transient tenant read failure' USING ERRCODE='08006'; END $$; CREATE VIEW parseable_documents AS SELECT tenant,kind,key,CASE WHEN tenant='z-valid' AND key='z-valid/.parseable.json' AND (current_query() LIKE 'SELECT body%FROM parseable_documents%' OR current_query() LIKE 'SELECT key,body%FROM parseable_documents%') THEN startup_read_failure() ELSE body END AS body,version,created_at,updated_at FROM startup_documents;").await.unwrap(); assert!(context.validate_multi_tenancy().await.is_err()); assert_eq!( storage @@ -1453,6 +1510,10 @@ mod contract_tests { postgres_regression(3).await; } #[tokio::test] + async fn postgres_round_two_backend_regressions() { + postgres_regression(7).await; + } + #[tokio::test] async fn postgres_manifest_url_regression() { postgres_regression(0).await; } diff --git a/src/metastore/metastores/review_fix_tests.rs b/src/metastore/metastores/review_fix_tests.rs index 72f6d528f..aad4c475c 100644 --- a/src/metastore/metastores/review_fix_tests.rs +++ b/src/metastore/metastores/review_fix_tests.rs @@ -60,6 +60,10 @@ mod tests { fn unavailable_and_deleting_http_status() { use crate::storage::ObjectStorageError; use actix_web::{ResponseError, http::StatusCode}; + let query = crate::handlers::http::query::QueryError::ObjectStorage( + ObjectStorageError::Custom("ordinary storage failure".into()), + ); + assert_eq!(query.status_code(), StatusCode::BAD_REQUEST); for (error, status) in [ (ObjectStorageError::StreamDeleting, StatusCode::CONFLICT), ( @@ -210,6 +214,22 @@ mod tests { )?); db.check(&None).await?; assert_eq!(db.client().await?.query_one("SHOW application_name",&[]).await?.get::<_,String>(0),"review-preserved"); + // With the server timeout disabled, a client timeout must send CancelRequest. + let cancel_db = PostgresDocuments::new(&PostgresOptions { + metastore_postgres_statement_timeout_ms: 300, ..opts.clone() + }, "review".into(), storage.clone())?; + let cancel_client = cancel_db.client().await?; + let pid: i32 = cancel_client.query_one("SELECT pg_backend_pid()", &[]).await?.get(0); + cancel_client.batch_execute("SET statement_timeout=0").await?; + assert!(cancel_client.query("SELECT pg_sleep(30)", &[]).await.is_err()); + let mut still_active = true; + for _ in 0..50 { + still_active = db.client().await?.query_one("SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE pid=$1 AND state='active')", &[&pid]).await?.get(0); + if !still_active { break; } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + assert!(!still_active, "client timeout left its statement running"); + drop(cancel_client); let store = PostgresMetastore { storage: db.clone(), }; diff --git a/src/metastore/migrations/003_manifest_admission.sql b/src/metastore/migrations/003_manifest_admission.sql new file mode 100644 index 000000000..1bd6f70c9 --- /dev/null +++ b/src/metastore/migrations/003_manifest_admission.sql @@ -0,0 +1,18 @@ +-- Serialized by the startup advisory lock; replacement preserves populated rows. +CREATE OR REPLACE FUNCTION parseable_admit_document() RETURNS trigger LANGUAGE plpgsql AS $$ +DECLARE stream_prefix text; +BEGIN + IF position('/.stream/' in NEW.key) > 0 THEN + stream_prefix := split_part(NEW.key, '/.stream/', 1); + ELSIF NEW.kind = 'manifest' AND position('/date=' in NEW.key) > 0 THEN + stream_prefix := split_part(NEW.key, '/date=', 1); + END IF; + IF stream_prefix IS NOT NULL THEN + PERFORM pg_advisory_xact_lock(hashtextextended(NEW.tenant || '/' || stream_prefix, 0)); + IF NEW.key NOT LIKE '%/.deleting' AND EXISTS ( + SELECT 1 FROM parseable_documents WHERE tenant=NEW.tenant AND key=stream_prefix || '/.stream/.deleting' + ) THEN RAISE EXCEPTION 'stream deletion pending'; END IF; + END IF; + RETURN NEW; +END $$; +UPDATE parseable_backend SET schema_version=3 WHERE singleton; diff --git a/src/metastore/mod.rs b/src/metastore/mod.rs index 1b397fc8c..62202ae96 100644 --- a/src/metastore/mod.rs +++ b/src/metastore/mod.rs @@ -246,6 +246,7 @@ mod boundary_tests { (function, token.as_str()), ("upload_single_parquet_file", "upload_multipart") | ("upload_single_parquet_file", "delete_object") + | ("compensate_uploaded_parquet", "delete_object") ); if !allowed && matches!( diff --git a/src/query/stream_schema_provider.rs b/src/query/stream_schema_provider.rs index 8d6a3713a..8d021c7ec 100644 --- a/src/query/stream_schema_provider.rs +++ b/src/query/stream_schema_provider.rs @@ -752,15 +752,22 @@ async fn collect_from_snapshot_cached( // latency, so run them concurrently instead of serially. `buffered` keeps // the snapshot ordering, which the `rev()` below depends on. let items = snapshot.manifests(time_filters); + let mut uncached = Vec::new(); + for item in &items { + if let Some(cache) = cache + && cache + .manifests + .lock() + .await + .contains_key(&item.manifest_path) + { + continue; + } + uncached.push(item.manifest_path.clone()); + } let batch = PARSEABLE .metastore - .manifest_batch( - &items - .iter() - .map(|m| m.manifest_path.clone()) - .collect::>(), - tenant_id, - ) + .manifest_batch(&uncached, tenant_id) .await .map_err(|e| DataFusionError::Plan(e.to_string()))?; let manifest_files = futures_util::stream::iter(items) @@ -773,26 +780,27 @@ async fn collect_from_snapshot_cached( } = manifest_item; let batch = batch.as_ref(); async move { - if let Some(batch) = batch { - return Ok(batch.get(&manifest_path).cloned().flatten()); - } if let Some(cache) = cache && let Some(manifest) = cache.manifests.lock().await.get(&manifest_path).cloned() { return Ok(manifest); } - let manifest_opt = PARSEABLE - .metastore - .get_manifest( - stream_name, - time_lower_bound, - time_upper_bound, - Some(manifest_path.clone()), - tenant_id, - ) - .await - .map_err(|e| DataFusionError::Plan(e.to_string()))?; + let manifest_opt = if let Some(batch) = batch { + batch.get(&manifest_path).cloned().flatten() + } else { + PARSEABLE + .metastore + .get_manifest( + stream_name, + time_lower_bound, + time_upper_bound, + Some(manifest_path.clone()), + tenant_id, + ) + .await + .map_err(|e| DataFusionError::Plan(e.to_string()))? + }; if manifest_opt.is_none() { tracing::warn!( "Manifest missing for stream={} [{:?} - {:?}]", diff --git a/src/storage/azure_blob.rs b/src/storage/azure_blob.rs index 8e23722dc..db02efaf7 100644 --- a/src/storage/azure_blob.rs +++ b/src/storage/azure_blob.rs @@ -1191,17 +1191,14 @@ impl ObjectStorage for BlobStore { prefix: object_store::path::Path, ) -> Result)>, ObjectStorageError> { use futures::TryStreamExt; - self.client - .list(Some(&prefix)) - .map_ok(|o| { - ( - crate::storage::object_storage::decode_metadata_key(o.location.as_ref()), - o.last_modified, - ) + let objects: Vec<_> = self.client.list(Some(&prefix)).try_collect().await?; + objects + .into_iter() + .map(|o| { + crate::storage::object_storage::decode_metadata_key(o.location.as_ref()) + .map(|key| (key, o.last_modified)) }) - .try_collect() - .await - .map_err(Into::into) + .collect() } async fn list_with_delimiter( diff --git a/src/storage/gcs.rs b/src/storage/gcs.rs index dc984f840..55458fe10 100644 --- a/src/storage/gcs.rs +++ b/src/storage/gcs.rs @@ -1170,17 +1170,14 @@ impl ObjectStorage for Gcs { prefix: object_store::path::Path, ) -> Result)>, ObjectStorageError> { use futures::TryStreamExt; - self.client - .list(Some(&prefix)) - .map_ok(|o| { - ( - crate::storage::object_storage::decode_metadata_key(o.location.as_ref()), - o.last_modified, - ) + let objects: Vec<_> = self.client.list(Some(&prefix)).try_collect().await?; + objects + .into_iter() + .map(|o| { + crate::storage::object_storage::decode_metadata_key(o.location.as_ref()) + .map(|key| (key, o.last_modified)) }) - .try_collect() - .await - .map_err(Into::into) + .collect() } async fn list_with_delimiter( diff --git a/src/storage/localfs.rs b/src/storage/localfs.rs index 17c448c91..ba8a38b9e 100644 --- a/src/storage/localfs.rs +++ b/src/storage/localfs.rs @@ -538,7 +538,10 @@ impl ObjectStorage for LocalFS { tenant_id: &Option, ) -> Result, ObjectStorageError> { let prefix = RelativePathBuf::from_iter([ - tenant_id.as_deref().unwrap_or(""), + tenant_id + .as_deref() + .filter(|t| !t.is_empty() && *t != crate::parseable::DEFAULT_TENANT) + .unwrap_or(""), stream_name, STREAM_ROOT_DIRECTORY, ]); diff --git a/src/storage/object_storage.rs b/src/storage/object_storage.rs index 70d6cad18..0785a2d5a 100644 --- a/src/storage/object_storage.rs +++ b/src/storage/object_storage.rs @@ -697,28 +697,45 @@ pub trait ObjectStorage: Debug + Send + Sync + 'static { ) -> Result)>, ObjectStorageError> { let mut pending = vec![None]; let mut keys = Vec::new(); - while let Some(prefix) = pending.pop() { - let listing = self.list_with_delimiter(prefix).await?; - for prefix in listing.common_prefixes { - if prefix - .parts() - .any(|c| c.as_ref().starts_with("hour=") || c.as_ref().starts_with("minute=")) - { - continue; - } - if prefix.filename().is_some_and(|p| p.starts_with('.')) { - keys.extend(self.metadata_hidden_prefix(prefix).await?); - } else { - pending.push(Some(prefix)); - } + use futures::{StreamExt, TryStreamExt}; + while !pending.is_empty() { + let round = std::mem::take(&mut pending); + let listings: Vec<_> = futures::stream::iter(round) + .map(|prefix| async move { + let listing = self.list_with_delimiter(prefix).await?; + let mut children = Vec::new(); + let mut found = Vec::new(); + for prefix in listing.common_prefixes { + if prefix.parts().any(|c| { + c.as_ref().starts_with("hour=") || c.as_ref().starts_with("minute=") + }) { + continue; + } + if prefix.filename().is_some_and(|p| p.starts_with('.')) { + found.extend(self.metadata_hidden_prefix(prefix).await?); + } else { + children.push(Some(prefix)); + } + } + for object in listing + .objects + .into_iter() + .filter(|o| !o.location.as_ref().ends_with(".parquet")) + { + found.push(( + decode_metadata_key(object.location.as_ref())?, + object.last_modified, + )); + } + Ok::<_, ObjectStorageError>((children, found)) + }) + .buffer_unordered(16) + .try_collect() + .await?; + for (children, found) in listings { + pending.extend(children); + keys.extend(found); } - keys.extend( - listing - .objects - .into_iter() - .filter(|o| !o.location.as_ref().ends_with(".parquet")) - .map(|o| (decode_metadata_key(o.location.as_ref()), o.last_modified)), - ); } keys.sort_by(|a, b| a.0.cmp(&b.0)); Ok(keys) @@ -737,7 +754,10 @@ pub trait ObjectStorage: Debug + Send + Sync + 'static { listing .objects .into_iter() - .map(|o| (decode_metadata_key(o.location.as_ref()), o.last_modified)), + .map(|o| { + decode_metadata_key(o.location.as_ref()).map(|key| (key, o.last_modified)) + }) + .collect::, _>>()?, ); } Ok(keys) @@ -1345,13 +1365,68 @@ pub trait ObjectStorage: Debug + Send + Sync + 'static { let stream = PARSEABLE.get_or_create_stream(stream_name, &tenant_id); let upload_context = UploadContext::new(stream); + if PARSEABLE.options.postgres.enabled() + && PARSEABLE + .metastore + .is_stream_deleting(stream_name, &tenant_id) + .await + .map_err(|e| ObjectStorageError::Custom(e.to_string()))? + { + for path in upload_context.stream.parquet_files() { + let filename = path.file_name().unwrap().to_string_lossy(); + let key = stream_relative_path( + stream_name, + &filename, + &upload_context.custom_partition, + &tenant_id, + ); + compensate_uploaded_parquet(self, &key, &tenant_id).await?; + remove_uploaded_staged_file(path); + } + for path in upload_context.stream.schema_files() { + remove_uploaded_staged_file(path); + } + return Ok(()); + } // Process parquet files concurrently and collect results let uploaded_files = process_parquet_files(&upload_context, stream_name, tenant_id.clone()).await?; // Update snapshot with collected manifest files - update_snapshot_with_manifests(stream_name, &uploaded_files, &tenant_id).await?; + if let Err(e) = + update_snapshot_with_manifests(stream_name, &uploaded_files, &tenant_id).await + { + if e.http_status() == actix_web::http::StatusCode::CONFLICT { + // The marker can land between upload and manifest publication. + // Compensate every upload in this batch, including partially + // published manifests, before retiring its local staging files. + for file in &uploaded_files { + let filename = file.file_path.file_name().unwrap().to_string_lossy(); + let key = stream_relative_path( + stream_name, + &filename, + &upload_context.stream.get_custom_partition(), + &tenant_id, + ); + match compensate_uploaded_parquet(self, &key, &tenant_id).await { + Ok(()) | Err(ObjectStorageError::NoSuchKey(_)) => {} + Err(cleanup) => { + for pending in &uploaded_files { + ACTIVE_OBJECT_STORE_SYNC_FILES.remove(&pending.file_path); + } + return Err(cleanup); + } + } + } + cleanup_uploaded_staged_files(uploaded_files).await; + for path in upload_context.stream.schema_files() { + remove_uploaded_staged_file(path); + } + return Ok(()); + } + return Err(e); + } // Calculate stats after snapshot update so uploads are visible before stats work starts. calculate_stats_for_uploaded_files( @@ -1371,6 +1446,24 @@ pub trait ObjectStorage: Debug + Send + Sync + 'static { } } +// Data-only compensation for an upload rejected by the metastore deletion guard. +async fn compensate_uploaded_parquet( + storage: &(impl ObjectStorage + ?Sized), + key: &str, + tenant: &Option, +) -> Result<(), ObjectStorageError> { + if !key.ends_with(".parquet") { + return Err(ObjectStorageError::Custom( + "upload compensation requires parquet key".into(), + )); + } + match storage.delete_object(RelativePath::new(key), tenant).await { + Ok(()) | Err(ObjectStorageError::NoSuchKey(_)) => Ok(()), + Err(ObjectStorageError::IoError(e)) if e.kind() == std::io::ErrorKind::NotFound => Ok(()), + result => result, + } +} + /// Processes parquet files concurrently and returns stats status and manifest files #[tracing::instrument( name = "object_store.process_parquet_files", @@ -1833,14 +1926,20 @@ fn stream_metadata_prefix( tenant_id: &Option, ) -> object_store::path::Path { object_store::path::Path::from_iter([ - tenant_id.as_deref().unwrap_or(""), + tenant_id + .as_deref() + .filter(|t| !t.is_empty() && *t != crate::parseable::DEFAULT_TENANT) + .unwrap_or(""), stream_name, STREAM_ROOT_DIRECTORY, ]) } pub fn schema_path(stream_name: &str, tenant_id: &Option) -> RelativePathBuf { - let tenant = tenant_id.as_deref().unwrap_or(""); + let tenant = tenant_id + .as_deref() + .filter(|t| !t.is_empty() && *t != crate::parseable::DEFAULT_TENANT) + .unwrap_or(""); if PARSEABLE.options.mode == Mode::Ingest { let id = INGESTOR_META .get() @@ -1856,7 +1955,10 @@ pub fn schema_path(stream_name: &str, tenant_id: &Option) -> RelativePat #[inline(always)] pub fn stream_json_path(stream_name: &str, tenant_id: &Option) -> RelativePathBuf { - let tenant = tenant_id.as_deref().unwrap_or(""); + let tenant = tenant_id + .as_deref() + .filter(|t| !t.is_empty() && *t != crate::parseable::DEFAULT_TENANT) + .unwrap_or(""); if PARSEABLE.options.mode == Mode::Ingest { let id = INGESTOR_META .get() @@ -1886,7 +1988,10 @@ pub fn stream_json_path(stream_name: &str, tenant_id: &Option) -> Relati /// scan to find it. #[inline(always)] pub fn tombstone_path(stream_name: &str, tenant_id: &Option) -> RelativePathBuf { - let tenant = tenant_id.as_deref().unwrap_or(""); + let tenant = tenant_id + .as_deref() + .filter(|t| !t.is_empty() && *t != crate::parseable::DEFAULT_TENANT) + .unwrap_or(""); RelativePathBuf::from_iter([ TOMBSTONE_ROOT_DIRECTORY, tenant, @@ -1936,7 +2041,10 @@ pub async fn list_tombstoned_streams( storage: &(impl ObjectStorage + ?Sized), tenant_id: &Option, ) -> Result, ObjectStorageError> { - let tenant = tenant_id.as_deref().unwrap_or(""); + let tenant = tenant_id + .as_deref() + .filter(|t| !t.is_empty() && *t != crate::parseable::DEFAULT_TENANT) + .unwrap_or(""); let root = RelativePathBuf::from_iter([TOMBSTONE_ROOT_DIRECTORY, tenant]); let candidates = storage.list_dirs_relative(&root, tenant_id).await?; @@ -1957,7 +2065,10 @@ pub fn filter_path( filter_file_name: &str, tenant_id: &Option, ) -> RelativePathBuf { - let root = tenant_id.as_deref().unwrap_or(""); + let root = tenant_id + .as_deref() + .filter(|t| !t.is_empty() && *t != crate::parseable::DEFAULT_TENANT) + .unwrap_or(""); RelativePathBuf::from_iter([ root, USERS_ROOT_DIR, @@ -1977,7 +2088,10 @@ pub fn parseable_json_path() -> RelativePathBuf { /// TODO: Needs to be updated for distributed mode #[inline(always)] pub fn alert_json_path(alert_id: Ulid, tenant_id: &Option) -> RelativePathBuf { - if let Some(tenant_id) = tenant_id.as_ref() { + if let Some(tenant_id) = tenant_id + .as_ref() + .filter(|t| !t.is_empty() && t.as_str() != crate::parseable::DEFAULT_TENANT) + { RelativePathBuf::from_iter([ tenant_id, ALERTS_ROOT_DIRECTORY, @@ -1991,7 +2105,10 @@ pub fn alert_json_path(alert_id: Ulid, tenant_id: &Option) -> RelativePa /// TODO: Needs to be updated for distributed mode #[inline(always)] pub fn target_json_path(target_id: &Ulid, tenant_id: &Option) -> RelativePathBuf { - let root = tenant_id.as_deref().unwrap_or(""); + let root = tenant_id + .as_deref() + .filter(|t| !t.is_empty() && *t != crate::parseable::DEFAULT_TENANT) + .unwrap_or(""); RelativePathBuf::from_iter([ root, SETTINGS_ROOT_DIRECTORY, @@ -2004,7 +2121,10 @@ pub fn target_json_path(target_id: &Ulid, tenant_id: &Option) -> Relativ /// Format: "/settings/outbound_http_policy.json" #[inline(always)] pub fn outbound_http_policy_json_path(tenant_id: &Option) -> RelativePathBuf { - let root = tenant_id.as_deref().unwrap_or(""); + let root = tenant_id + .as_deref() + .filter(|t| !t.is_empty() && *t != crate::parseable::DEFAULT_TENANT) + .unwrap_or(""); RelativePathBuf::from_iter([root, SETTINGS_ROOT_DIRECTORY, "outbound_http_policy.json"]) } @@ -2012,7 +2132,10 @@ pub fn outbound_http_policy_json_path(tenant_id: &Option) -> RelativePat /// Format: ".alerts/alert_state_{alert_id}.json" #[inline(always)] pub fn alert_state_json_path(alert_id: Ulid, tenant_id: &Option) -> RelativePathBuf { - let root = tenant_id.as_deref().unwrap_or(""); + let root = tenant_id + .as_deref() + .filter(|t| !t.is_empty() && *t != crate::parseable::DEFAULT_TENANT) + .unwrap_or(""); RelativePathBuf::from_iter([ root, ALERTS_ROOT_DIRECTORY, @@ -2024,7 +2147,10 @@ pub fn alert_state_json_path(alert_id: Ulid, tenant_id: &Option) -> Rela /// Format: ".alerts/mttr.json" #[inline(always)] pub fn mttr_json_path(tenant_id: &Option) -> RelativePathBuf { - if let Some(tenant) = tenant_id.as_ref() { + if let Some(tenant) = tenant_id + .as_ref() + .filter(|t| !t.is_empty() && t.as_str() != crate::parseable::DEFAULT_TENANT) + { RelativePathBuf::from_iter([tenant, ALERTS_ROOT_DIRECTORY, "mttr.json"]) } else { RelativePathBuf::from_iter([ALERTS_ROOT_DIRECTORY, "mttr.json"]) @@ -2562,21 +2688,64 @@ mod query_reload_tests { } /// Decode exactly one object_store path encoding layer. -pub(crate) fn decode_metadata_key(key: &str) -> String { +pub(crate) fn decode_metadata_key(key: &str) -> Result { let bytes = key.as_bytes(); let mut out = Vec::with_capacity(bytes.len()); let mut i = 0; while i < bytes.len() { - if bytes[i] == b'%' - && i + 2 < bytes.len() - && let Ok(n) = u8::from_str_radix(&key[i + 1..i + 3], 16) - { - out.push(n); + if bytes[i] == b'%' { + let digits = bytes + .get(i + 1..i + 3) + .and_then(|d| std::str::from_utf8(d).ok()) + .and_then(|d| u8::from_str_radix(d, 16).ok()) + .ok_or_else(|| { + ObjectStorageError::Custom("invalid percent-encoded metadata key".into()) + })?; + out.push(digits); i += 3; } else { out.push(bytes[i]); i += 1; } } - String::from_utf8(out).expect("object_store paths encode UTF-8 keys") + String::from_utf8(out) + .map_err(|e| ObjectStorageError::Custom(format!("invalid UTF-8 metadata key: {e}"))) +} + +#[cfg(test)] +mod metadata_key_decode_tests { + use super::*; + #[tokio::test] + async fn compensation_removes_only_parquet_and_tolerates_missing_objects() { + let dir = tempfile::tempdir().unwrap(); + let storage = crate::storage::LocalFS::new(dir.path().to_owned()); + let key = "logs/date=2026-01-01/file.parquet"; + storage + .put_object(RelativePath::new(key), Bytes::from_static(b"data"), &None) + .await + .unwrap(); + compensate_uploaded_parquet(&storage, key, &None) + .await + .unwrap(); + compensate_uploaded_parquet(&storage, key, &None) + .await + .unwrap(); + assert!( + storage + .get_object(RelativePath::new(key), &None) + .await + .is_err() + ); + assert!( + compensate_uploaded_parquet(&storage, "logs/.stream/.stream.json", &None) + .await + .is_err() + ); + } + #[test] + fn invalid_encoded_keys_never_panic() { + assert!(decode_metadata_key("%FF").is_err()); + assert!(decode_metadata_key("%é").is_err()); + assert_eq!(decode_metadata_key("caf%C3%A9").unwrap(), "café"); + } } diff --git a/src/storage/s3.rs b/src/storage/s3.rs index c431b8d4c..ceedb740c 100644 --- a/src/storage/s3.rs +++ b/src/storage/s3.rs @@ -1424,17 +1424,14 @@ impl ObjectStorage for S3 { prefix: object_store::path::Path, ) -> Result)>, ObjectStorageError> { use futures::TryStreamExt; - self.client - .list(Some(&prefix)) - .map_ok(|o| { - ( - crate::storage::object_storage::decode_metadata_key(o.location.as_ref()), - o.last_modified, - ) + let objects: Vec<_> = self.client.list(Some(&prefix)).try_collect().await?; + objects + .into_iter() + .map(|o| { + crate::storage::object_storage::decode_metadata_key(o.location.as_ref()) + .map(|key| (key, o.last_modified)) }) - .try_collect() - .await - .map_err(Into::into) + .collect() } async fn list_with_delimiter( diff --git a/src/users/dashboards.rs b/src/users/dashboards.rs index ed4c6c851..faedfd456 100644 --- a/src/users/dashboards.rs +++ b/src/users/dashboards.rs @@ -92,7 +92,11 @@ fn deserialize_dataset<'de, D: serde::Deserializer<'de>>( impl MetastoreObject for Dashboard { fn get_object_path(&self) -> String { - let root = self.tenant_id.as_deref().unwrap_or(""); + let root = self + .tenant_id + .as_deref() + .filter(|t| !t.is_empty() && *t != crate::parseable::DEFAULT_TENANT) + .unwrap_or(""); RelativePathBuf::from_iter([ root, USERS_ROOT_DIR, @@ -415,3 +419,21 @@ mod tests { } } } + +#[cfg(test)] +mod default_tenant_path_tests { + use super::*; + #[test] + fn dashboard_default_tenant_aliases_share_a_key() { + let mut dashboard = Dashboard { + author: Some("owner".into()), + dashboard_id: Some(Ulid::new()), + ..Default::default() + }; + let path = dashboard.get_object_path(); + for alias in ["", crate::parseable::DEFAULT_TENANT] { + dashboard.tenant_id = Some(alias.into()); + assert_eq!(dashboard.get_object_path(), path); + } + } +} From 43e1d600deb7fda8ab2eebde8ca5d86bf89a944f Mon Sep 17 00:00:00 2001 From: VibeSWE <338779468+VibeSWE@users.noreply.github.com> Date: Wed, 11 Nov 2026 09:43:23 -0600 Subject: [PATCH 3/9] fix: bound manifest decoding and preserve canonical migration locators --- src/metastore/metastores/migration.rs | 95 ++++++- .../metastores/postgres_documents.rs | 247 +++++++++++------- .../metastores/postgres_metastore.rs | 50 +++- 3 files changed, 294 insertions(+), 98 deletions(-) diff --git a/src/metastore/metastores/migration.rs b/src/metastore/metastores/migration.rs index a713bf744..0f14f05c9 100644 --- a/src/metastore/metastores/migration.rs +++ b/src/metastore/metastores/migration.rs @@ -142,6 +142,49 @@ pub(crate) fn canonical_key(key: &str) -> &str { } } +// Keep bytes unchanged unless a legacy explicit default-tenant locator must +// follow its document to the canonical namespace. Unknown JSON fields survive. +pub(crate) fn import_body( + store: &dyn ObjectStorage, + key: &str, + body: &Bytes, +) -> anyhow::Result { + if !key.ends_with("stream.json") { + return Ok(body.clone()); + } + let mut value: serde_json::Value = serde_json::from_slice(body)?; + let base = store.absolute_url(RelativePath::new("")); + let mut changed = false; + if let Some(items) = value + .pointer_mut("/snapshot/manifest_list") + .and_then(|v| v.as_array_mut()) + { + for item in items { + if let Some(path) = item.get("manifest_path").and_then(|v| v.as_str()) { + let relative = path + .strip_prefix(base.as_ref()) + .and_then(|s| s.strip_prefix('/')) + .unwrap_or(path); + let canonical = canonical_key(relative); + if canonical != relative { + let replacement = if relative != path { + format!("{base}/{canonical}") + } else { + canonical.to_owned() + }; + item["manifest_path"] = replacement.into(); + changed = true; + } + } + } + } + Ok(if changed { + Bytes::from(serde_json::to_vec(&value)?) + } else { + body.clone() + }) +} + pub(crate) async fn validate( store: &dyn ObjectStorage, key: &str, @@ -231,7 +274,7 @@ pub async fn run( let canonical: std::collections::HashSet<_> = keys.iter().map(|(key, _, _)| canonical_key(key)).collect(); ensure!( - canonical.len() == keys.len(), + action != Action::Import || canonical.len() == keys.len(), "duplicate metadata keys after default-tenant normalization" ); let mut counts = BTreeMap::<(String, String), (usize, usize)>::new(); @@ -402,6 +445,15 @@ mod tests { .await .is_err() ); + assert!( + validate( + &store, + ".users/owner/filters/broken.json", + &Bytes::from_static(b"not-json") + ) + .await + .is_err() + ); assert!(optional(&store, BUSY).await.unwrap().is_none()); } @@ -598,6 +650,7 @@ mod tests { url.set_path(&format!("/{schema}")); let opts = PostgresOptions { metastore_postgres_url: Some(url.to_string()), + metastore_postgres_pool_size: 1, ..Default::default() }; let db = PostgresDocuments::new(&opts, "test".into(), store.clone())?; @@ -651,6 +704,7 @@ mod tests { ".settings", ".parseable", ".correlations", + ".keystone", ] { for tenant in ["", "tenant/"] { documents.push(( @@ -661,9 +715,7 @@ mod tests { } documents.push(( "DEFAULT_TENANT/alias/.stream/.stream.json".into(), - Bytes::from(serde_json::to_vec( - &crate::storage::ObjectStoreFormat::default(), - )?), + Bytes::from_static(br#"{"version":"v3","snapshot":{"manifest_list":[{"manifest_path":"DEFAULT_TENANT/alias/date=2026-01-01/manifest.json"}]}}"#), )); documents.push(( "DEFAULT_TENANT/alias/date=2026-01-01/manifest.json".into(), @@ -703,6 +755,9 @@ mod tests { .to_string() .contains("another offline") ); + command.abort = true; + assert!(run(store.clone(), &db, &command).await.unwrap_err().to_string().contains("another offline")); + command.abort = false; drop(lock); run(store.clone(), &db, &command).await?; assert!(optional(store.as_ref(), PAIR).await?.is_none()); @@ -722,6 +777,19 @@ mod tests { ] { command.injected_failure = Some(step); assert!(run(store.clone(), &db, &command).await.is_err(), "{step}"); + if step == "spool" { + let id: String = db.client().await?.query_one("SELECT id FROM parseable_backend", &[]).await?.get(0); + let spool = std::env::current_dir()?.join(format!(".parseable-migration-spool-{id}")); + #[cfg(unix)] { + use std::os::unix::fs::PermissionsExt; + assert_eq!(std::fs::metadata(&spool)?.permissions().mode() & 0o777, 0o700); + } + std::fs::write(spool.join("stale"), b"private RBAC metadata")?; + command.abort = true; + run(store.clone(), &db, &command).await?; + assert!(!spool.exists(), "abort must clean stale private spools under the transfer lock"); + command.abort = false; + } if step == "import-commit" { command.abort = true; run(store.clone(), &db, &command).await?; @@ -748,6 +816,9 @@ mod tests { assert!(run(store.clone(), &db, &command).await.is_err()); command.metadata_migration = Some(Action::Verify); run(store.clone(), &db, &command).await?; + let alias = "DEFAULT_TENANT/alias/date=2026-01-01/manifest.json"; + assert!(db.get_object(RelativePath::new(alias), &None).await.is_ok()); + assert_eq!(db.manifest_batch(&[alias.to_owned()], &None).await?.len(), 1); command.metadata_migration = Some(Action::Export); command.migration_dry_run = true; run(store.clone(), &db, &command).await?; @@ -889,13 +960,22 @@ mod tests { &None, ) .await?; - let deleted = documents.pop().unwrap().0; + let deleted_index = documents.iter().position(|(key, _)| key == "tenant/.correlations/fixture.json").unwrap(); + let deleted = documents.remove(deleted_index).0; let deleted_tenant = document_scope(&deleted).unwrap().0; db.delete_object( RelativePath::new(canonical_key(&deleted)), &Some(deleted_tenant), ) .await?; + // Exercise the actual destructive override, with multiple conflicts. + store.put_object(RelativePath::new(extra), Bytes::from_static(b"{}"), &None).await?; + store.put_object(RelativePath::new(&documents[0].0), Bytes::from_static(b"{\"external\":true}"), &None).await?; + command.migration_dry_run = true; + let conflicts = run(store.clone(), &db, &command).await.unwrap_err(); + assert!(conflicts.to_string().contains("2 bucket conflicts"), "{conflicts:#}"); + command.migration_dry_run = false; + command.overwrite_bucket_changes_since_import = true; for step in [ "marker", "export-state", @@ -923,13 +1003,14 @@ mod tests { assert!(optional(store.as_ref(), PAIR).await?.is_none()); store.check_migration_guard().await?; assert!(optional(store.as_ref(), &deleted).await?.is_none()); + assert!(optional(store.as_ref(), extra).await?.is_none(), "override must remove bucket-only keys"); for (key, body) in documents { assert_eq!( store .get_object(RelativePath::new(canonical_key(&key)), &None) .await?, - body, - "byte-for-byte round trip: {key}" + import_body(store.as_ref(), &key, &body)?, + "byte-for-byte round trip (canonical default locator): {key}" ); } command.metadata_migration = Some(Action::Verify); diff --git a/src/metastore/metastores/postgres_documents.rs b/src/metastore/metastores/postgres_documents.rs index 178bc6563..e143df62e 100644 --- a/src/metastore/metastores/postgres_documents.rs +++ b/src/metastore/metastores/postgres_documents.rs @@ -40,6 +40,18 @@ fn legacy_key<'a>(key: &'a str, base: &str) -> Option<&'a str> { } key.strip_prefix(base).and_then(|p| p.strip_prefix('/')) } +fn read_keys(path: &str, base: &str) -> Vec { + let mut keys = vec![path.to_owned()]; + let legacy = legacy_key(path, base).unwrap_or(path); + if legacy != path { + keys.push(legacy.to_owned()); + } + let canonical = super::migration::canonical_key(legacy); + if canonical != legacy { + keys.push(canonical.to_owned()); + } + keys +} fn tenant(t: &Option) -> &str { t.as_deref() .filter(|v| !v.is_empty() && *v != "DEFAULT_TENANT") @@ -440,77 +452,130 @@ impl PostgresDocuments { pub fn absolute_url(&self, path: &RelativePath) -> object_store::path::Path { object_store::path::Path::parse(path.as_str()).expect("valid document key") } + /// Yield at most 32 selected locators and 8 MiB of raw bodies at a time. + /// A larger single document is indivisible and is yielded alone. The caller + /// decodes each chunk before requesting another; connections end at yield. + pub fn manifest_chunks<'a>( + &'a self, + paths: &'a [String], + t: &'a Option, + ) -> impl futures::Stream, ObjectStorageError>> + Send + 'a + { + use std::collections::{HashMap, VecDeque}; + futures::stream::try_unfold( + (0usize, VecDeque::<(String, String, usize)>::new()), + move |(mut offset, mut pending)| async move { + loop { + if pending.is_empty() { + if offset == paths.len() { + return Ok(None); + } + let start = offset; + let mut key_bytes = 0; + while offset < paths.len() && offset - start < 32 { + let size = paths[offset].len(); + if offset > start && key_bytes + size > 64 * 1024 { + break; + } + key_bytes += size; + offset += 1; + } + let base = self.storage.absolute_url(RelativePath::new("")); + let candidates: Vec<_> = paths[start..offset] + .iter() + .flat_map(|path| read_keys(path, base.as_ref())) + .collect(); + let rows = self.client().await?.query( + "SELECT key,octet_length(body) FROM parseable_documents WHERE tenant=$1 AND key=ANY($2)", + &[&tenant(t), &candidates], + ).await?; + let sizes: HashMap = rows + .into_iter() + .map(|row| (row.get(0), row.get::<_, i32>(1) as usize)) + .collect(); + for original in &paths[start..offset] { + let keys = read_keys(original, base.as_ref()); + let matches: Vec<_> = keys + .iter() + .filter_map(|key| sizes.get(key).map(|size| (key, *size))) + .collect(); + if matches.len() > 1 { + tracing::warn!( + key = original, + "multiple metadata candidates exist; exact key takes precedence" + ); + } + if let Some((key, size)) = matches.first() { + pending.push_back((original.clone(), (*key).clone(), *size)); + } + } + if pending.is_empty() { + continue; + } + } + let mut selected = Vec::new(); + let mut body_bytes = 0usize; + while let Some((_, _, size)) = pending.front() { + if !selected.is_empty() && body_bytes + size > 8 * 1024 * 1024 { + break; + } + body_bytes += size; + selected.push(pending.pop_front().unwrap()); + } + let keys: Vec<_> = selected.iter().map(|(_, key, _)| key.clone()).collect(); + let rows = self.client().await?.query( + "WITH sized AS (SELECT key,body,octet_length(body) AS size,row_number() OVER (ORDER BY array_position($2,key)) AS n,sum(octet_length(body)) OVER (ORDER BY array_position($2,key)) AS total FROM parseable_documents WHERE tenant=$1 AND key=ANY($2)) SELECT key,CASE WHEN n=1 OR total<=8388608 THEN body END,size FROM sized", + &[&tenant(t), &keys], + ).await?; + // Recheck the byte bound in the body statement itself: a + // manifest can grow after the sizing query. Requeue only + // rows withheld by the byte budget, never deleted rows. + let bodies: HashMap, usize)> = rows + .into_iter() + .map(|row| { + ( + row.get(0), + ( + row.get::<_, Option>>(1).map(Bytes::from), + row.get::<_, i32>(2) as usize, + ), + ) + }) + .collect(); + for (original, key, _) in selected.iter().rev() { + if let Some((None, size)) = bodies.get(key) { + pending.push_front((original.clone(), key.clone(), *size)); + } + } + let batch = selected + .into_iter() + .filter_map(|(original, key, _)| { + bodies + .get(&key) + .and_then(|(body, _)| body.as_ref()) + .map(|body| (original, body.clone())) + }) + .collect(); + return Ok(Some((batch, (offset, pending)))); + } + }, + ) + } + + #[cfg(test)] pub async fn manifest_batch( &self, paths: &[String], t: &Option, ) -> Result, ObjectStorageError> { - // Bound both parameter count and encoded key bytes. Each query returns at - // most 32 bodies; a single large manifest remains an indivisible document. - let mut result = Vec::new(); - let mut offset = 0; - while offset < paths.len() { - let mut end = offset; - let mut bytes = 0; - while end < paths.len() && end - offset < 32 { - let size = paths[end].len(); - if end > offset && bytes + size > 64 * 1024 { - break; - } - bytes += size; - end += 1; - } - let base = self.storage.absolute_url(RelativePath::new("")); - let mut keys = Vec::new(); - for path in &paths[offset..end] { - keys.push(path.clone()); - if let Some(legacy) = legacy_key(path, base.as_ref()) { - keys.push(legacy.to_owned()); - } - } - let client = self.client().await?; - let sizes = client.query("SELECT key,octet_length(body) FROM parseable_documents WHERE tenant=$1 AND key=ANY($2)", &[&tenant(t), &keys]).await?; - let mut bodies = std::collections::HashMap::::new(); - let mut batch = Vec::::new(); - let mut batch_bytes = 0usize; - for row in sizes { - let size = row.get::<_, i32>(1) as usize; - if !batch.is_empty() && batch_bytes + size > 8 * 1024 * 1024 { - for r in client.query("SELECT key,body FROM parseable_documents WHERE tenant=$1 AND key=ANY($2)", &[&tenant(t), &batch]).await? { - bodies.insert(r.get(0), Bytes::from(r.get::<_, Vec>(1))); - } - batch.clear(); - batch_bytes = 0; - } - batch.push(row.get(0)); - batch_bytes += size; - } - if !batch.is_empty() { - for r in client - .query( - "SELECT key,body FROM parseable_documents WHERE tenant=$1 AND key=ANY($2)", - &[&tenant(t), &batch], - ) - .await? - { - bodies.insert(r.get(0), Bytes::from(r.get::<_, Vec>(1))); - } - } - for path in &paths[offset..end] { - let fallback = legacy_key(path, base.as_ref()).and_then(|k| bodies.get(k)); - if bodies.contains_key(path) && fallback.is_some() { - tracing::warn!( - key = path, - "exact and legacy metadata keys exist; using exact key" - ); - } - if let Some(body) = bodies.get(path).or(fallback) { - result.push((path.clone(), body.clone())); - } - } - offset = end; + use futures::TryStreamExt; + let chunks = self.manifest_chunks(paths, t); + futures::pin_mut!(chunks); + let mut rows = Vec::new(); + while let Some(chunk) = chunks.try_next().await? { + rows.extend(chunk); } - Ok(result) + Ok(rows) } pub async fn get_object( &self, @@ -531,10 +596,10 @@ impl PostgresDocuments { let base = self.storage.absolute_url(RelativePath::new("")); let c = self.client().await?; let exact = path.as_str(); - let fallback = legacy_key(exact, base.as_ref()).unwrap_or(exact); + let keys = read_keys(exact, base.as_ref()); let rows = c.query( - "SELECT key,body,version FROM parseable_documents WHERE tenant=$1 AND (key=$2 OR key=$3) ORDER BY (key=$2) DESC", - &[&tenant(t), &exact, &fallback], + "SELECT key,body,version FROM parseable_documents WHERE tenant=$1 AND key=ANY($2) ORDER BY array_position($2,key)", + &[&tenant(t), &keys], ).await?; if rows.len() > 1 { tracing::warn!( @@ -1092,7 +1157,11 @@ impl PostgresDocuments { .collect(); let mut differences = 0; for (key, (tenant, kind), _) in keys { - let bucket = store.get_object(RelativePath::new(key), &None).await?; + let bucket = super::migration::import_body( + store.as_ref(), + key, + &store.get_object(RelativePath::new(key), &None).await?, + )?; let sql = if exists { tx.query_opt( "SELECT body FROM parseable_documents WHERE tenant=$1 AND key=$2", @@ -1256,10 +1325,13 @@ impl PostgresDocuments { let mut kinds = Vec::new(); let mut paths = Vec::new(); let mut bodies = Vec::new(); + let mut source_digests = Vec::new(); for (key, (tenant, kind), _) in batch { let body = Bytes::from(std::fs::read( spool.path().join((index * 32 + bodies.len()).to_string()), )?); + source_digests.push(digest(&body)); + let body = super::migration::import_body(store.as_ref(), key, &body)?; tenants.push(tenant.clone()); kinds.push(kind.to_string()); paths.push(super::migration::canonical_key(key).to_owned()); @@ -1268,10 +1340,14 @@ impl PostgresDocuments { if count == 0 { tx.execute("INSERT INTO parseable_documents(tenant,kind,key,body) SELECT * FROM unnest($1::text[],$2::text[],$3::text[],$4::bytea[])", &[&tenants,&kinds,&paths,&bodies]).await?; } - for (((key, tenant), body), (source_key, _, _)) in - paths.iter().zip(&tenants).zip(&bodies).zip(batch) + for (row_index, (((key, tenant), body), (source_key, _, _))) in paths + .iter() + .zip(&tenants) + .zip(&bodies) + .zip(batch) + .enumerate() { - tx.execute("INSERT INTO parseable_import_digests(key,digest) VALUES($1,$2) ON CONFLICT DO NOTHING", &[source_key, &digest(body)]).await?; + tx.execute("INSERT INTO parseable_import_digests(key,digest) VALUES($1,$2) ON CONFLICT DO NOTHING", &[source_key, &source_digests[row_index]]).await?; let reread: Vec = tx .query_one( "SELECT body FROM parseable_documents WHERE tenant=$1 AND key=$2", @@ -1478,16 +1554,14 @@ impl PostgresDocuments { ) .await? .is_none() - { - if self.import_digest(client, key).await?.as_deref() + && self.import_digest(client, key).await?.as_deref() != Some( digest(&store.get_object(RelativePath::new(key), &None).await?).as_str(), ) - { - conflicts += 1; - if conflicts <= 100 { - eprintln!("CONFLICT bucket-only key={key}"); - } + { + conflicts += 1; + if conflicts <= 100 { + eprintln!("CONFLICT bucket-only key={key}"); } } } @@ -1634,16 +1708,11 @@ impl PostgresDocuments { } else if key.ends_with("stream.json") { for manifest_path in super::migration::stream_references(body)? { let base = self.storage.absolute_url(RelativePath::new("")); - let normalized = manifest_path - .strip_prefix(base.as_ref()) - .and_then(|p| p.strip_prefix('/')) - .unwrap_or(&manifest_path); - let row = client - .query_opt( - "SELECT body FROM parseable_documents WHERE key=$1 OR key=$2 ORDER BY (key=$1) DESC LIMIT 1", - &[&manifest_path, &normalized], - ) - .await?; + let keys = read_keys(&manifest_path, base.as_ref()); + let row = client.query_opt( + "SELECT body FROM parseable_documents WHERE key=ANY($1) ORDER BY array_position($1,key) LIMIT 1", + &[&keys], + ).await?; let Some(row) = row else { if allow_dangling { eprintln!("OVERRIDE {key}: missing SQL manifest {manifest_path}"); diff --git a/src/metastore/metastores/postgres_metastore.rs b/src/metastore/metastores/postgres_metastore.rs index 7e93ecace..6854f73b8 100644 --- a/src/metastore/metastores/postgres_metastore.rs +++ b/src/metastore/metastores/postgres_metastore.rs @@ -10,8 +10,10 @@ super::document_metastore::document_metastore!( ) -> Result>>, MetastoreError> { let mut result: HashMap> = paths.iter().map(|p| (p.clone(), None)).collect(); - for chunk in paths.chunks(32) { - for (key, body) in self.storage.manifest_batch(chunk, tenant).await? { + let chunks = self.storage.manifest_chunks(paths, tenant); + futures::pin_mut!(chunks); + while let Some(rows) = chunks.try_next().await? { + for (key, body) in rows { result.insert(key, Some(decode_manifest_blocking(body).await?)); } } @@ -1117,6 +1119,11 @@ mod contract_tests { storage: documents.clone(), }; if case == 7 { + let abort = super::super::migration::MigrationOptions { + metadata_migration: Some(super::super::migration::Action::Export), + acknowledge_server_stopped: true, abort: true, ..Default::default() + }; + super::super::migration::run(Arc::new(LocalFS::new(dir.path().to_owned())), &documents, &abort).await.unwrap(); let name = dir.path().file_name().unwrap().to_str().unwrap(); for tenant in [None, Some(name.to_owned())] { let prefix = tenant.as_ref().map(|t| format!("{t}/")).unwrap_or_default(); @@ -1143,13 +1150,43 @@ mod contract_tests { // SQL tenant candidates cannot bypass deletion collision protection. store.put_stream_json(&crate::storage::ObjectStoreFormat::default(), "protected", &None).await.unwrap(); documents.put_object(RelativePath::new("protected/other/.stream/.schema"), Bytes::new(), &Some("protected".into())).await.unwrap(); + let data = dir.path().join("protected/date=2026-01-01/data.parquet"); + std::fs::create_dir_all(data.parent().unwrap()).unwrap(); + std::fs::write(&data, b"sentinel").unwrap(); + assert!(documents.list_tenant_candidates().await.unwrap().contains(&"protected".to_owned())); assert!(documents.delete_prefix(RelativePath::new("protected"), &None).await.unwrap_err().to_string().contains("collides")); + assert_eq!(std::fs::read(&data).unwrap(), b"sentinel"); store.put_stream_json(&crate::storage::ObjectStoreFormat::default(), "alias", &Some(crate::parseable::DEFAULT_TENANT.into())).await.unwrap(); assert!(store.stream_exists("alias", &None).await.unwrap()); assert!(documents.get_object(RelativePath::new("alias/.stream/.stream.json"), &None).await.is_ok()); let paths: Vec<_> = (0..70).map(|i| format!("batch/date=2026-01-01/{i}.manifest.json")).collect(); for key in &paths { documents.put_object(RelativePath::new(key), Bytes::new(), &None).await.unwrap(); } assert_eq!(documents.manifest_batch(&paths, &None).await.unwrap().len(), 70); + let large_paths: Vec<_> = (0..3).map(|i| format!("large/date=2026-01-01/{i}.manifest.json")).collect(); + for key in &large_paths { documents.put_object(RelativePath::new(key), Bytes::from(vec![b'x'; 5 * 1024 * 1024]), &None).await.unwrap(); } + // Model growth between the size lookup and the body read: + // only sizing queries see one byte; body queries see real data. + documents.client().await.unwrap().batch_execute("ALTER TABLE parseable_documents RENAME TO batch_documents; CREATE VIEW parseable_documents AS SELECT tenant,kind,key,CASE WHEN current_query() LIKE 'SELECT key,octet_length%' THEN substring(body from 1 for 1) ELSE body END AS body,version,created_at,updated_at FROM batch_documents;").await.unwrap(); + let chunks = documents.manifest_chunks(&large_paths, &None); + futures::pin_mut!(chunks); + use futures::TryStreamExt; + let mut chunks_seen = 0; + while let Some(rows) = chunks.try_next().await.unwrap() { + assert!(rows.iter().map(|(_, body)| body.len()).sum::() <= 8 * 1024 * 1024); + chunks_seen += 1; + } + assert_eq!(chunks_seen, 3); + documents.client().await.unwrap().batch_execute("DROP VIEW parseable_documents; ALTER TABLE batch_documents RENAME TO parseable_documents;").await.unwrap(); + documents.client().await.unwrap().batch_execute(include_str!("../migrations/002_review_fixes.sql")).await.unwrap(); + let restart_a = super::super::postgres_documents::PostgresDocuments::new(&options, database.clone(), Arc::new(LocalFS::new(dir.path().to_owned()))).unwrap(); + let restart_b = super::super::postgres_documents::PostgresDocuments::new(&options, database.clone(), Arc::new(LocalFS::new(dir.path().to_owned()))).unwrap(); + tokio::try_join!(restart_a.check(&None), restart_b.check(&None)).unwrap(); + assert_eq!(documents.get_object(RelativePath::new(&large_paths[0]), &None).await.unwrap().len(), 5 * 1024 * 1024); + assert_eq!(documents.client().await.unwrap().query_one("SELECT schema_version FROM parseable_backend", &[]).await.unwrap().get::<_, i32>(0), 3); + documents.put_object(RelativePath::new("upgrade/.stream/.deleting"), Bytes::new(), &None).await.unwrap(); + assert!(matches!(documents.put_object(RelativePath::new("upgrade/date=2026-01-01/manifest.json"), Bytes::new(), &None).await, Err(ObjectStorageError::StreamDeleting))); + + } else if case == 0 { let date = chrono::Utc::now(); let manifest = crate::catalog::manifest::Manifest::default(); @@ -1292,6 +1329,15 @@ mod contract_tests { ) .await .unwrap(); + let invalid = Document { + id: "collision".into(), + path: "defaultlogs/other/.stream/.schema".into(), + value: "invalid".into(), + }; + store + .put_dashboard(&invalid, &Some("defaultlogs".into())) + .await + .unwrap(); context.validate_multi_tenancy().await.unwrap(); assert_eq!( storage From 49924e2ed81ff4e1ce9bccbe6cdf69d585c3cd31 Mon Sep 17 00:00:00 2001 From: VibeSWE <338779468+VibeSWE@users.noreply.github.com> Date: Wed, 11 Nov 2026 09:50:58 -0600 Subject: [PATCH 4/9] fix: scope upload retirement to PostgreSQL deletion conflicts --- src/storage/object_storage.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/storage/object_storage.rs b/src/storage/object_storage.rs index 0785a2d5a..5cccabbf8 100644 --- a/src/storage/object_storage.rs +++ b/src/storage/object_storage.rs @@ -1397,7 +1397,9 @@ pub trait ObjectStorage: Debug + Send + Sync + 'static { if let Err(e) = update_snapshot_with_manifests(stream_name, &uploaded_files, &tenant_id).await { - if e.http_status() == actix_web::http::StatusCode::CONFLICT { + if PARSEABLE.options.postgres.enabled() + && e.http_status() == actix_web::http::StatusCode::CONFLICT + { // The marker can land between upload and manifest publication. // Compensate every upload in this batch, including partially // published manifests, before retiring its local staging files. From 30bd6ce316fb6689ffc6371f8afb58dd361fcc66 Mon Sep 17 00:00:00 2001 From: VibeSWE <338779468+VibeSWE@users.noreply.github.com> Date: Wed, 11 Nov 2026 10:13:18 -0600 Subject: [PATCH 5/9] fix: retain transfer lock ownership through migration commits --- src/metastore/metastores/migration.rs | 36 +++++++++++++++---- .../metastores/postgres_documents.rs | 30 +++++++++++++--- 2 files changed, 56 insertions(+), 10 deletions(-) diff --git a/src/metastore/metastores/migration.rs b/src/metastore/metastores/migration.rs index 0f14f05c9..1f8238243 100644 --- a/src/metastore/metastores/migration.rs +++ b/src/metastore/metastores/migration.rs @@ -227,10 +227,12 @@ pub async fn run( "requires --acknowledge-server-stopped; stop ALL servers and writers first" ); eprintln!("OFFLINE migration: all servers must remain stopped. No writer fencing exists."); - let _transfer_lock = db.transfer_lock().await?; + let mut transfer_lock = db.transfer_lock().await?; db.migration_preflight(store.as_ref(), opts).await?; if opts.abort { - return db.abort_migration(store.as_ref(), action).await; + return db + .abort_migration(store.as_ref(), action, &transfer_lock) + .await; } let inventory = store.metadata_inventory().await?; let mut skipped = 0; @@ -326,9 +328,11 @@ pub async fn run( db.export_preflight(&client, store.as_ref(), opts, &keys) .await?; } + transfer_lock.batch_execute("SELECT 1").await?; return Ok(()); } - db.migrate(store, opts, &keys).await + transfer_lock.batch_execute("SELECT 1").await?; + db.migrate(store, opts, &keys, &mut transfer_lock).await } pub(crate) fn failure_point(opts: &MigrationOptions, step: &str) -> anyhow::Result<()> { @@ -771,6 +775,7 @@ mod tests { for step in [ "marker", "spool", + "lock-loss", "import-before-commit", "import-commit", "paired", @@ -790,6 +795,13 @@ mod tests { assert!(!spool.exists(), "abort must clean stale private spools under the transfer lock"); command.abort = false; } + if step == "lock-loss" { + assert_eq!(db.client().await?.query_one("SELECT count(*) FROM parseable_documents", &[]).await?.get::<_, i64>(0), 0); + assert!(optional(store.as_ref(), PAIR).await?.is_none()); + command.abort = true; + run(store.clone(), &db, &command).await?; + command.abort = false; + } if step == "import-commit" { command.abort = true; run(store.clone(), &db, &command).await?; @@ -1022,11 +1034,23 @@ mod tests { anyhow::Ok(()) } .await; - admin + // The CI workflow runs this fixture repeatedly against one isolated + // bucket. Remove both source aliases and canonical export keys. + let cleanup = async { + for (key, _) in store.metadata_inventory().await? { + store.delete_object(RelativePath::new(&key), &None).await?; + } + anyhow::Ok(()) + } + .await; + let dropped = admin .client() .await? .batch_execute(&format!("DROP DATABASE {schema} WITH (FORCE)")) - .await?; - result + .await; + result?; + cleanup?; + dropped?; + Ok(()) } } diff --git a/src/metastore/metastores/postgres_documents.rs b/src/metastore/metastores/postgres_documents.rs index e143df62e..0326b7ee3 100644 --- a/src/metastore/metastores/postgres_documents.rs +++ b/src/metastore/metastores/postgres_documents.rs @@ -1139,6 +1139,7 @@ impl PostgresDocuments { store: Arc, opts: &super::migration::MigrationOptions, keys: &[super::migration::InventoryDocument], + transfer_lock: &mut super::postgres_client::Client, ) -> anyhow::Result<()> { use super::migration::{Action, BUSY, PAIR, digest, optional}; use anyhow::ensure; @@ -1205,6 +1206,7 @@ impl PostgresDocuments { } } ensure!(differences == 0, "{differences} metadata differences"); + transfer_lock.batch_execute("SELECT 1").await?; println!("representations match"); return Ok(()); } @@ -1256,6 +1258,7 @@ impl PostgresDocuments { self.export_preflight(&client, store.as_ref(), opts, keys) .await?; } + transfer_lock.batch_execute("SELECT 1").await?; store .put_metadata_conditional( RelativePath::new(BUSY), @@ -1315,8 +1318,20 @@ impl PostgresDocuments { .all(|((k, m), (old, _, om))| k == old && m == om), "bucket metadata changed during import" ); - let mut client = self.client().await?; - let tx = client.transaction().await?; + #[cfg(test)] + if opts.injected_failure == Some("lock-loss") { + let pid: i32 = transfer_lock + .query_one("SELECT pg_backend_pid()", &[]) + .await? + .get(0); + self.client() + .await? + .query_one("SELECT pg_terminate_backend($1,1000)", &[&pid]) + .await?; + } + // Commit on the lock-owning session: a lost lock connection must + // fail this transfer, rather than reconnecting and committing stale work. + let tx = transfer_lock.transaction().await?; tx.query("SELECT pg_advisory_xact_lock(8104050183641399916)", &[]) .await?; tx.batch_execute("CREATE TABLE IF NOT EXISTS parseable_import_digests (key text PRIMARY KEY, digest text NOT NULL)").await?; @@ -1367,6 +1382,7 @@ impl PostgresDocuments { super::migration::failure_point(opts, "import-before-commit")?; tx.commit().await?; super::migration::failure_point(opts, "import-commit")?; + transfer_lock.batch_execute("SELECT 1").await?; store .put_metadata_conditional( RelativePath::new(PAIR), @@ -1388,7 +1404,9 @@ impl PostgresDocuments { ); super::migration::failure_point(opts, "paired")?; } else { - tx.execute("INSERT INTO parseable_transfer(state) VALUES('exported') ON CONFLICT(singleton) DO UPDATE SET state='exported'", &[]).await?; + drop(client); + let client = &*transfer_lock; + client.execute("INSERT INTO parseable_transfer(state) VALUES('exported') ON CONFLICT(singleton) DO UPDATE SET state='exported'", &[]).await?; super::migration::failure_point(opts, "export-state")?; let mut last = (String::new(), String::new()); loop { @@ -1399,6 +1417,7 @@ impl PostgresDocuments { for row in rows { last = (row.get(0), row.get(1)); let body = Bytes::from(row.get::<_, Vec>(2)); + client.batch_execute("SELECT 1").await?; store .put_object(RelativePath::new(&last.1), body.clone(), &None) .await?; @@ -1451,10 +1470,12 @@ impl PostgresDocuments { } } if optional(store.as_ref(), PAIR).await?.is_some() { + client.batch_execute("SELECT 1").await?; store.delete_object(RelativePath::new(PAIR), &None).await?; } super::migration::failure_point(opts, "pair-removed")?; } + transfer_lock.batch_execute("SELECT 1").await?; store.delete_object(RelativePath::new(BUSY), &None).await?; Ok(()) } @@ -1734,10 +1755,10 @@ impl PostgresDocuments { &self, store: &dyn ObjectStorage, action: super::migration::Action, + client: &super::postgres_client::Client, ) -> anyhow::Result<()> { use super::migration::{Action, BUSY, optional}; use anyhow::ensure; - let client = self.client().await?; let exists: bool = client .query_one("SELECT to_regclass('parseable_transfer') IS NOT NULL", &[]) .await? @@ -1800,6 +1821,7 @@ impl PostgresDocuments { Action::Verify => anyhow::bail!("verify cannot be aborted"), } if optional(store, BUSY).await?.is_some() { + client.batch_execute("SELECT 1").await?; store.delete_object(RelativePath::new(BUSY), &None).await?; } println!( From 483a6fee583e3c0f23f174ac9ed4c39f50da13d1 Mon Sep 17 00:00:00 2001 From: VibeSWE <338779468+VibeSWE@users.noreply.github.com> Date: Wed, 11 Nov 2026 10:30:39 -0600 Subject: [PATCH 6/9] fix: settle PostgreSQL upload batches before deletion cleanup --- src/storage/object_storage.rs | 88 +++++++++++++++++++++++++++++++++-- 1 file changed, 83 insertions(+), 5 deletions(-) diff --git a/src/storage/object_storage.rs b/src/storage/object_storage.rs index 5cccabbf8..1476a586d 100644 --- a/src/storage/object_storage.rs +++ b/src/storage/object_storage.rs @@ -1408,7 +1408,7 @@ pub trait ObjectStorage: Debug + Send + Sync + 'static { let key = stream_relative_path( stream_name, &filename, - &upload_context.stream.get_custom_partition(), + &upload_context.custom_partition, &tenant_id, ); match compensate_uploaded_parquet(self, &key, &tenant_id).await { @@ -1498,6 +1498,13 @@ async fn process_parquet_files( .filter_map(claim_staged_file_for_upload) .collect(); + let postgres = PARSEABLE.options.postgres.enabled(); + let retirement_paths = if postgres { + parquet_paths.clone() + } else { + Vec::new() + }; + let mut total_size: u64 = 0; let mut min_dt: Option> = None; let mut max_dt: Option> = None; @@ -1534,7 +1541,37 @@ async fn process_parquet_files( } // Collect results from all upload tasks - collect_upload_results(join_set).await + let result = collect_upload_results(join_set, postgres).await; + if result.is_err() && postgres { + // All tasks have settled, so no successful sibling upload can publish + // after compensation. Keep staging retryable if deletion is unconfirmed. + for path in &retirement_paths { + ACTIVE_OBJECT_STORE_SYNC_FILES.remove(path); + } + if PARSEABLE + .metastore + .is_stream_deleting(stream_name, &tenant_id) + .await + .is_ok_and(|pending| pending) + { + for path in retirement_paths { + let filename = path.file_name().unwrap().to_string_lossy(); + let key = stream_relative_path( + stream_name, + &filename, + &upload_context.custom_partition, + &tenant_id, + ); + compensate_uploaded_parquet(object_store.as_ref(), &key, &tenant_id).await?; + remove_uploaded_staged_file(path); + } + for path in upload_context.stream.schema_files() { + remove_uploaded_staged_file(path); + } + return Ok(Vec::new()); + } + } + result } /// Spawns an individual parquet file upload task @@ -1576,8 +1613,10 @@ async fn spawn_parquet_upload_task( /// Collects results from all upload tasks async fn collect_upload_results( mut join_set: JoinSet>, + drain_on_error: bool, ) -> Result, ObjectStorageError> { let mut uploaded_files = Vec::new(); + let mut failure = None; while let Some(result) = join_set.join_next().await { match result { @@ -1599,16 +1638,30 @@ async fn collect_upload_results( Ok(Err((path, e))) => { error!("Error uploading parquet file: {e}"); ACTIVE_OBJECT_STORE_SYNC_FILES.remove(&path); - return Err(e); + if !drain_on_error { + return Err(e); + } + if failure.is_none() { + failure = Some(e); + } } Err(e) => { error!("Task panicked: {e}"); - return Err(ObjectStorageError::UnhandledError(Box::new(e))); + let error = ObjectStorageError::UnhandledError(Box::new(e)); + if !drain_on_error { + return Err(error); + } + if failure.is_none() { + failure = Some(error); + } } } } - Ok(uploaded_files) + match failure { + Some(error) => Err(error), + None => Ok(uploaded_files), + } } /// Updates snapshot with collected manifest files @@ -2718,6 +2771,31 @@ pub(crate) fn decode_metadata_key(key: &str) -> Result Date: Wed, 11 Nov 2026 11:42:00 -0600 Subject: [PATCH 7/9] test: validate and document PostgreSQL migration hardening --- .github/workflows/postgres-metastore.yaml | 5 + docs/postgres-metastore.md | 157 +- docs/validation/postgres-metastore-smoke.json | 5725 +++++++++++++++-- docs/validation/postgres-migration-tls.json | 1062 +++ docs/validation/postgres-migration.json | 960 +++ docs/validation/postgres-round2-checks.json | 155 + scripts/postgres-metastore-migration-smoke.py | 90 +- scripts/postgres-metastore-smoke.py | 66 +- src/metastore/metastores/migration.rs | 8 + .../metastores/postgres_documents.rs | 18 +- .../metastores/postgres_metastore.rs | 4 +- 11 files changed, 7640 insertions(+), 610 deletions(-) create mode 100644 docs/validation/postgres-migration-tls.json create mode 100644 docs/validation/postgres-migration.json create mode 100644 docs/validation/postgres-round2-checks.json diff --git a/.github/workflows/postgres-metastore.yaml b/.github/workflows/postgres-metastore.yaml index e9081d432..5e17fe2e3 100644 --- a/.github/workflows/postgres-metastore.yaml +++ b/.github/workflows/postgres-metastore.yaml @@ -54,6 +54,10 @@ jobs: curl --fail --retry 30 --retry-connrefused --retry-delay 1 http://127.0.0.1:59087/minio/health/ready - name: Migration and regressions on PostgreSQL 17 and MinIO run: 'cargo test --locked --lib metastore:: -- --nocapture --test-threads=1' + - name: Full library suite with PostgreSQL + run: cargo test --locked --lib -- --test-threads=1 + - name: Full library suite without PostgreSQL + run: env -u P_TEST_POSTGRES_URL -u P_TEST_REVIEW_S3_URL cargo test --locked --lib -- --nocapture --test-threads=1 - name: LocalFS migration and legacy locators run: env -u P_TEST_REVIEW_S3_URL cargo test --locked --lib migration::tests::migration_round_trip_postgres -- --nocapture - name: Deadline against paused PostgreSQL @@ -97,5 +101,6 @@ jobs: .postgres-smoke/results.json .postgres-smoke/runs/*.json docs/validation/postgres-metastore-smoke.json + docs/validation/postgres-migration*.json .postgres-smoke/*.log if-no-files-found: error diff --git a/docs/postgres-metastore.md b/docs/postgres-metastore.md index 41032b31f..7b0837b36 100644 --- a/docs/postgres-metastore.md +++ b/docs/postgres-metastore.md @@ -304,7 +304,7 @@ The shared document layer inventories exact stored keys rather than rebuilding paths. All hidden document namespaces (deployment/RBAC, users, dashboard bodies, locators/titles, filters, alerts/state/history, correlations, targets/settings, node registry, stream JSON and schemas) and date-partition manifests, including -per-ingestor variants and tenant prefixes, are copied without reserialization. +per-ingestor variants and tenant prefixes, are copied without reserialization, except explicit default-tenant snapshot locators that require canonicalization (described below). SQL versions are freshly allocated. Unfinished deletion markers are rejected. Parquet, staging/WAL, hot-tier/cache files and private LocalFS lock files are not metadata documents and are excluded, as are the pairing and transfer controls. @@ -381,7 +381,7 @@ rebuilds its primary key transactionally under the same startup advisory lock. On a large database, budget startup time for that rebuild and its exclusive table lock. Prefix reads and deletes use explicit component-bounded ranges; they do not require PostgreSQL 15's `starts_with` optimization. Listings are tenant-scoped -and key-only; manifest query planning uses one SQL batch rather than one pooled +and key-only; manifest query planning uses bounded SQL batches rather than one pooled connection per manifest. SQL manifest locators are bucket-relative on every provider; old absolute LocalFS locators are accepted on read. No other SQL caller uses the provider-absolute locator form. Existing cloud @@ -401,8 +401,11 @@ requires a single tenant path component. Absent, empty and `DEFAULT_TENANT` tenant identifiers use the same empty SQL tenant column. Stream lifecycle keys follow the existing root-path normalization. -Every SQL operation, including transaction start/commit, startup migration, -offline transfer and health, has a client deadline of statement timeout + 250 ms. +Server SQL operations, including transaction start/commit and health, have a +client deadline of statement timeout + 250 ms. Startup/schema migrations use +the separate migration budget described below. Offline SQL validation can also +use that budget; import commits and aborts use the lock session's normal +statement deadline. Canceled/timed-out connections are discarded instead of recycled. Readiness has an overall deadline of that same duration, including pool acquisition and bucket probe. TCP keepalive starts after one second with one-second probes and three @@ -423,10 +426,10 @@ store is not loaded. An empty CA bundle is rejected at configuration time. CA pl Normal startup refuses an existing database identity, including one with zero documents, when the bucket has no pairing marker. Restore the matching marker from backup. For deliberate recovery only, after verifying the bucket and database identity, set -`P_METASTORE_POSTGRES_ALLOW_UNPAIRED=true` to allow creation of the missing marker. +`P_METASTORE_POSTGRES_ALLOW_UNPAIRED=` to allow creation of the missing marker. This override never permits mismatched identities or incomplete offline transfers. The import tool's committed-import recovery continues to establish pairing itself -under its durable transfer marker, and migrates schema version 1 to 2. Exported +under its durable transfer marker, and migrates schema versions 1 and 2 to 3. Exported or incomplete databases still refuse server admission. Conditional-create lost responses remain ambiguous, as on object storage: a retry @@ -472,8 +475,9 @@ This avoids stopping with an unsynced custom-partition file or comparing a cache pre-export bound to a freshly loaded rollback bound. It passed interrupted import and export, SQL-mode query/metadata equality, new ingestion, verified export and object-store rollback. Existing Rust test modules and expectations are unchanged. -The protected smoke's only expectation change is future schema version 3, restoring -version 2 afterward, because migration 002 makes version 2 supported. +That historical review smoke rejected schema version 3 and restored version 2, +which was current with migration 002. Current validation rejects version 999 +and restores version 3. The three-mode smoke passed object-store, verified TLS PostgreSQL and plaintext PostgreSQL scenarios, including lifecycle, dependency outages, CA rejection and @@ -489,7 +493,6 @@ run passed without changing that protected cleanup logic. All started containers were removed. Large migrations or slow storage may require increasing the statement timeout; the readiness/client deadline increases with that setting. - ## Hardened offline procedure **Older binaries ignore pairing and transfer markers and can write stale bucket @@ -572,7 +575,6 @@ listing excludes mttr.json; LocalFS stream listing uses the selected instance; startup cleanup waits for the entire discovery/validation pass to succeed before deleting any invalid tenant. - The merged CRUD contract and boundary guard kept their assertions. The smoke's future-schema test now rejects 3 and restores 2 (migration 002 is supported). The migration smoke no longer expects ordinary rollback to fail without force; @@ -615,10 +617,10 @@ four probes, and the original under-two-second assertions. The combined implementation was validated locally with disposable PostgreSQL 17 and MinIO, with all task containers removed afterward. Full library runs with -and without the database URL each passed 1,398 tests (seven ignored); SQL tests +and without the database URL each passed 1,405 tests (seven ignored); SQL tests print explicit skips without prerequisites. The PostgreSQL contract and startup cleanup fixtures were then run explicitly and passed, as did the paused-server -deadline test. Focused metastore runs passed 34 tests against LocalFS/PostgreSQL +deadline test. Focused metastore runs passed 37 tests against LocalFS/PostgreSQL and MinIO/PostgreSQL; follow-up migration checks cover legacy absolute locators, both legacy tenant scopes, safe/unsafe aborts, and early pin refusal. @@ -627,11 +629,14 @@ plaintext and verified TLS modes; offline migration in plaintext and verified TLS, including interrupted import/export, .part refusal, new SQL-mode ingestion and unforced object-store rollback. The smoke generated its own [raw evidence](validation/postgres-metastore-smoke.json). -[Validation commands and recovery results](validation/postgres-hardening-checks.json) -record the other checks. Plain cargo clippy --all-targets and the debug build +[Round-two validation commands and results](validation/postgres-round2-checks.json) +record the latest checks; [earlier results](validation/postgres-hardening-checks.json) +remain as historical evidence. Plain cargo clippy --all-targets and the debug build passed; Clippy retained 21 existing library-test warnings and one existing -integration-test warning, with no new warnings. The workflow YAML parses; its -hosted GitHub run has not been performed because this work is local and unpushed. +integration-test warning, with no new warnings. The first hosted workflow run +[failed during runner setup](https://github.com/VibeEngineeringAI/parseable/actions/runs/43037438052) +because the Ubuntu package mirror could not be resolved; compilation and tests +never started. The expanded workflow has not been pushed or run remotely. Counts in the inventory chart are a model of request growth, not a throughput benchmark. No concurrent-writer fencing, automatic resolution of an interrupted @@ -642,4 +647,122 @@ their identifier privacy risk. Deduplicating the object-safe storage guard and SQL startup guard adapters is deferred: both enforce the same tested controls, and consolidating their different error adapters adds no recovery capability. The required real PostgreSQL/MinIO and injected-boundary coverage is implemented; -hosted CI remains deferred under the no-push constraint. +the expanded hosted checks remain unrun under the no-push constraint. + +### Review hardening and operational limits + +LocalFS lookups try the exact document key first, then the old root-prefixed +locator compatibility form. If both rows exist, the exact row wins and a warning +is logged. Cloud locators retain their existing interpretation. Empty tenants and +`DEFAULT_TENANT` use the same metadata paths as an absent tenant. + +Schema version 3 admits manifests under the same transaction advisory lock as +stream documents and deletion markers. Startup migrations have a separate +`P_METASTORE_POSTGRES_MIGRATION_TIMEOUT_MS` budget (default 1800000, 30 minutes). +Migration 002 takes an exclusive table lock while changing collation and rebuilding +indexes; stop old writers before upgrading a large populated database. Concurrent +new initializers serialize on the startup advisory lock. Increase the migration +budget for a large table; ordinary queries retain the statement deadline. + +When deletion is confirmed pending, the uploader drops staged parquet and schema +files for that stream. If deletion wins after upload began, it first removes all +parquet objects uploaded by that batch, then retires those staged files. A bucket +cleanup failure preserves staging for retry. Publication delayed beyond the final +SQL row deletion remains possible: an incarnation token is required to fence that +window (issue 142). Do not recreate a stream while old uploaders are still active. +Tenant deletion refuses a tenant name that collides with a known default-tenant +stream, including SQL-only cleanup candidates. Rename or resolve that collision +before retrying cleanup. + +A client SQL timeout sends a separately bounded PostgreSQL cancel request and +discards the connection. Cancellation or its response can be lost, and a write or +COMMIT may already have completed. Such failures have an ambiguous outcome: inspect +state before retrying; cancellation is not a rollback guarantee. + +Offline transfers, including abort, hold a session advisory lock for the entire +operation; a second invocation fails immediately. Import releases its ordinary +database connection during spooling and the second inventory walk. The dedicated +lock session remains connected. Bucket reads and inventory walks use concurrency +16. No bucket I/O occurs inside the SQL import transaction. + +The tool requires an existing explicit staging directory, resolves it to an +absolute path, and prints it together with the current directory's environment +file (or that none was loaded). Run from the deployment's configuration directory; +the tool does not search other directories for an environment file. + +Import validates all JSON documents before committing. The dangling snapshot +reference override applies to both import and export, including export's final +validation. Export dry-run reads SQL and bucket state, checks references and +reports bucket conflicts without bootstrapping SQL or writing bucket markers. +An imported key deleted from the bucket is a conflict. New SQL keys absent from +the bucket are normal. Conflicts are counted, with the first 100 keys printed, +before requiring `--overwrite-bucket-changes-since-import`. This flag also permits +removing bucket-only metadata; back up the bucket first. Databases imported by an +older tool without digests can export equal documents without an override; changed +documents require it. Import abort may clear committed SQL only when the matching +Import marker still exists and no pairing marker has ever been published. + +The disaster recovery setting `P_METASTORE_POSTGRES_ALLOW_UNPAIRED` must equal the +backend UUID, rather than a boolean. Remove it after recovery. + +Spools contain sensitive metadata, including RBAC. They live below a private +(mode 0700) `.parseable-migration-spool-` directory in the working +directory. Normal exit removes the temporary body directory; SIGKILL can leave +it behind. The next import with the same backend id and exclusive transfer lock +removes that stale spool before reading metadata again. + +For 20 streams × 30 days, assuming 650 LIST requests per walk, one manifest per +stream/day, and two stream documents, import makes approximately 3180 bucket +requests: two inventories (1300 LIST), two reads of 640 documents, and 600 +snapshot-reference reads. For 1000 streams the corresponding estimate is 159000 +requests. At 150 ms/request and concurrency 16 the latency-only lower bounds are +about 30 seconds and 25 minutes respectively; shallow discovery rounds, pagination, +SQL verification, body sizes and network throughput add time. These are estimates, +not measured throughput. The committed literal EXPLAIN output does not demonstrate +prepared generic-plan behavior. + +Default object-store mode retains query storage-error HTTP 400 responses. +Explicit-stream ingest in distributed mode propagates storage lookup failures +instead of reporting a missing stream (404); actual missing streams remain 404. +Shared +metadata path helpers now normalize empty and explicit default tenants; malformed +percent-encoded inventory keys return errors instead of panicking, `.keystone/` +is accepted by the offline metadata inventory, and inventory +discovery uses bounded concurrency. The uploader retirement policy applies to +confirmed PostgreSQL deletion conflicts. + +Default-tenant alert-state enumeration now uses the canonical `.alerts/` +directory, consistent with writes and single-state reads. It therefore includes +states previously missed by the default-tenant listing. Per-date manifest +inventory also normalizes explicit default-tenant aliases. + +Import canonicalizes explicit `DEFAULT_TENANT/` metadata keys and snapshot +manifest locators. It retains the original source digests for rollback conflict +checks, and verify compares the corresponding canonical representation. Other +metadata bodies, including legacy absolute locators without that alias, retain +their bytes. This avoids exporting a snapshot that points to an alias key +removed during rollback; unknown JSON fields are retained when locators change. + +Offline commands require a direct PostgreSQL connection or session pooling: the +whole-transfer advisory lock must stay on one database session. Transaction +pooling cannot provide that lock lifetime. + +Manifest planning consults the cache for both documents and known misses. Cold +reads select at most 32 locators (64 KiB of locator text) per sizing batch and +transfer at most 8 MiB of bodies per chunk, decoding before fetching the next +chunk. The body statement rechecks the limit to handle concurrent growth. One +larger manifest is returned alone. Missing manifests still produce individual +warnings. + +`export --abort` before export starts is a safe no-op, including databases +initialized directly in PostgreSQL mode without a transfer row. + +Import commits and abort SQL execute on the session owning the transfer lock. +If that session is lost during spooling, the command fails without reconnecting +and committing its old work; rerun or abort under a new lock. Functional smoke +fixtures use a 30-second SQL budget on shared build machines; separate timeout +regressions retain their short deadlines. + +Failed PostgreSQL upload batches wait for sibling uploads to settle before +compensation. If deletion cannot be confirmed, staging remains available for +retry. Default object-store batches retain their existing error handling. diff --git a/docs/validation/postgres-metastore-smoke.json b/docs/validation/postgres-metastore-smoke.json index 39e33aed1..fb90c93cf 100644 --- a/docs/validation/postgres-metastore-smoke.json +++ b/docs/validation/postgres-metastore-smoke.json @@ -6,9 +6,9 @@ "postgres-tls", "postgres" ], - "source_commit": "1428d2cc45c4f60d164a89713ca93bcd896c76e5", - "binary_sha256": "8fb88e03aaf8417443f1e496cac9a5cf1a696661d3133adadcc252d75f2887ad", - "source_diff_sha256": "c6974df8d3d4d6c26e2fa5a22c93838e9fd8b7d21e14504d12b40e5813df0a31", + "source_commit": "49924e2ed81ff4e1ce9bccbe6cdf69d585c3cd31", + "binary_sha256": "bb0af87e40744d6798700aa269afa3083570bf89ed5613a67ec550df68353653", + "source_diff_sha256": "e810e0ac8997bc3c815ddf95d21838593d8cb1d97520b113e05cd154ed04acbb", "image_digests": { "postgres:17-alpine": [ "postgres@sha256:b0f9560a2de083e2cc7382e75f808c7381a32852a7ec49117deedb300e552b24" @@ -25,61 +25,61 @@ "version": "v1", "title": "Postgres smoke", "author": "8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918", - "dashboardId": "01M74SV622YTQ2Y75HZE11AVJB", - "created": "2026-11-11T07:58:30.978106403+00:00", - "modified": "2026-11-11T07:58:30.978106403+00:00", + "dashboardId": "01M75NFJS7A10VN0ZFSBBY6ZAT", + "created": "2026-11-11T16:01:30.919827850+00:00", + "modified": "2026-11-11T16:01:30.919827850+00:00", "tags": null, "isFavorite": false, "dashboardType": "Dashboard", "tiles": [], "tenantId": null, - "revision": "2d53005fca7a3b566744600d766140032fe8ad13cc9d01b1442619fc34bc8a24" + "revision": "0d4c1de651e8d0cb27704f511e0becd54950ddaf09051115b487084b2d3e7b80" }, "keys": [ ".parseable/.parseable.json", - ".parseable/ingestor.d97d0c2e2ddd006.json", - ".parseable/querier.5bef5998cbd41f0.json", - ".users/.dashboard-locations/01M74SV622YTQ2Y75HZE11AVJB.json", + ".parseable/ingestor.ca453944eff07df.json", + ".parseable/querier.6e37ab554650b70.json", + ".users/.dashboard-locations/01M75NFJS7A10VN0ZFSBBY6ZAT.json", ".users/.dashboard-titles/571a2cefe2d0d1a9a26dab4810a5556875c7d7531d4f558a8f7b28ab486e2395.json", - ".users/8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918/dashboards/01M74SV622YTQ2Y75HZE11AVJB.json", - "lazy/.stream/.ingestor.d97d0c2e2ddd006.schema", - "lazy/.stream/.ingestor.d97d0c2e2ddd006.stream.json", + ".users/8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918/dashboards/01M75NFJS7A10VN0ZFSBBY6ZAT.json", + "lazy/.stream/.ingestor.ca453944eff07df.schema", + "lazy/.stream/.ingestor.ca453944eff07df.stream.json", "lazy/.stream/.schema", "lazy/.stream/.stream.json", - "lazy/date=2026-11-11/hour=07/minute=59/d97d0c2e2ddd006.data.01M74SXX475GMD681ENW5BAMAA.parquet", - "lazy/date=2026-11-11/ingestor..d97d0c2e2ddd006.manifest.json", - "pstats/.stream/.ingestor.d97d0c2e2ddd006.schema", - "pstats/.stream/.ingestor.d97d0c2e2ddd006.stream.json", + "lazy/date=2026-11-11/hour=16/minute=02/ca453944eff07df.data.01M75NJAP4C59XEDHX8HVK5QBE.parquet", + "lazy/date=2026-11-11/ingestor..ca453944eff07df.manifest.json", + "pstats/.stream/.ingestor.ca453944eff07df.schema", + "pstats/.stream/.ingestor.ca453944eff07df.stream.json", "pstats/.stream/.schema", "pstats/.stream/.stream.json", "pstats/date=2026-11-11/.manifest.json", - "pstats/date=2026-11-11/hour=07/minute=57/dataset_name=smoke/.data.01M74SW2MA5GZW7RNJRMWRNB3T.parquet", - "smoke/.stream/.ingestor.d97d0c2e2ddd006.schema", - "smoke/.stream/.ingestor.d97d0c2e2ddd006.stream.json", + "pstats/date=2026-11-11/hour=16/minute=00/dataset_name=smoke/.data.01M75NGFXMDMRDEPRWCR78SP2S.parquet", + "smoke/.stream/.ingestor.ca453944eff07df.schema", + "smoke/.stream/.ingestor.ca453944eff07df.stream.json", "smoke/.stream/.schema", "smoke/.stream/.stream.json", "smoke/date=2026-11-11/.manifest.json", - "smoke/date=2026-11-11/hour=07/minute=57/.data.01M74ST81AX4XAJZZ14Z78ZGWZ.parquet" + "smoke/date=2026-11-11/hour=16/minute=00/.data.01M75NENAKWMP0MHNV6WWF9ZCD.parquet" ], "non_parquet_keys": [ ".parseable/.parseable.json", - ".parseable/ingestor.d97d0c2e2ddd006.json", - ".parseable/querier.5bef5998cbd41f0.json", - ".users/.dashboard-locations/01M74SV622YTQ2Y75HZE11AVJB.json", + ".parseable/ingestor.ca453944eff07df.json", + ".parseable/querier.6e37ab554650b70.json", + ".users/.dashboard-locations/01M75NFJS7A10VN0ZFSBBY6ZAT.json", ".users/.dashboard-titles/571a2cefe2d0d1a9a26dab4810a5556875c7d7531d4f558a8f7b28ab486e2395.json", - ".users/8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918/dashboards/01M74SV622YTQ2Y75HZE11AVJB.json", - "lazy/.stream/.ingestor.d97d0c2e2ddd006.schema", - "lazy/.stream/.ingestor.d97d0c2e2ddd006.stream.json", + ".users/8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918/dashboards/01M75NFJS7A10VN0ZFSBBY6ZAT.json", + "lazy/.stream/.ingestor.ca453944eff07df.schema", + "lazy/.stream/.ingestor.ca453944eff07df.stream.json", "lazy/.stream/.schema", "lazy/.stream/.stream.json", - "lazy/date=2026-11-11/ingestor..d97d0c2e2ddd006.manifest.json", - "pstats/.stream/.ingestor.d97d0c2e2ddd006.schema", - "pstats/.stream/.ingestor.d97d0c2e2ddd006.stream.json", + "lazy/date=2026-11-11/ingestor..ca453944eff07df.manifest.json", + "pstats/.stream/.ingestor.ca453944eff07df.schema", + "pstats/.stream/.ingestor.ca453944eff07df.stream.json", "pstats/.stream/.schema", "pstats/.stream/.stream.json", "pstats/date=2026-11-11/.manifest.json", - "smoke/.stream/.ingestor.d97d0c2e2ddd006.schema", - "smoke/.stream/.ingestor.d97d0c2e2ddd006.stream.json", + "smoke/.stream/.ingestor.ca453944eff07df.schema", + "smoke/.stream/.ingestor.ca453944eff07df.stream.json", "smoke/.stream/.schema", "smoke/.stream/.stream.json", "smoke/date=2026-11-11/.manifest.json" @@ -95,167 +95,39 @@ "delete_recreate": true, "final_keys": [ ".parseable/.parseable.json", - ".parseable/ingestor.d97d0c2e2ddd006.json", - ".parseable/querier.5bef5998cbd41f0.json", - ".users/.dashboard-locations/01M74SV622YTQ2Y75HZE11AVJB.json", + ".parseable/ingestor.ca453944eff07df.json", + ".parseable/querier.6e37ab554650b70.json", + ".users/.dashboard-locations/01M75NFJS7A10VN0ZFSBBY6ZAT.json", ".users/.dashboard-titles/571a2cefe2d0d1a9a26dab4810a5556875c7d7531d4f558a8f7b28ab486e2395.json", - ".users/8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918/dashboards/01M74SV622YTQ2Y75HZE11AVJB.json", - "lazy/.stream/.ingestor.d97d0c2e2ddd006.schema", - "lazy/.stream/.ingestor.d97d0c2e2ddd006.stream.json", + ".users/8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918/dashboards/01M75NFJS7A10VN0ZFSBBY6ZAT.json", + "lazy/.stream/.ingestor.ca453944eff07df.schema", + "lazy/.stream/.ingestor.ca453944eff07df.stream.json", "lazy/.stream/.schema", "lazy/.stream/.stream.json", - "lazy/date=2026-11-11/hour=07/minute=59/d97d0c2e2ddd006.data.01M74SXX475GMD681ENW5BAMAA.parquet", - "lazy/date=2026-11-11/ingestor..d97d0c2e2ddd006.manifest.json", - "pstats/.stream/.ingestor.d97d0c2e2ddd006.schema", - "pstats/.stream/.ingestor.d97d0c2e2ddd006.stream.json", + "lazy/date=2026-11-11/hour=16/minute=02/ca453944eff07df.data.01M75NJAP4C59XEDHX8HVK5QBE.parquet", + "lazy/date=2026-11-11/ingestor..ca453944eff07df.manifest.json", + "pstats/.stream/.ingestor.ca453944eff07df.schema", + "pstats/.stream/.ingestor.ca453944eff07df.stream.json", "pstats/.stream/.schema", "pstats/.stream/.stream.json", "pstats/date=2026-11-11/.manifest.json", - "pstats/date=2026-11-11/hour=07/minute=57/dataset_name=smoke/.data.01M74SW2MA5GZW7RNJRMWRNB3T.parquet", - "pstats/date=2026-11-11/hour=07/minute=59/dataset_name=lazy/d97d0c2e2ddd006.data.01M74SZQQ6SAEGTFW58C90P919.parquet", - "pstats/date=2026-11-11/ingestor..d97d0c2e2ddd006.manifest.json", - "smoke/.stream/.ingestor.d97d0c2e2ddd006.schema", - "smoke/.stream/.ingestor.d97d0c2e2ddd006.stream.json", + "pstats/date=2026-11-11/hour=16/minute=00/dataset_name=smoke/.data.01M75NGFXMDMRDEPRWCR78SP2S.parquet", + "pstats/date=2026-11-11/hour=16/minute=02/dataset_name=lazy/ca453944eff07df.data.01M75NM594TPMV98RWN5XAZY56.parquet", + "pstats/date=2026-11-11/ingestor..ca453944eff07df.manifest.json", + "smoke/.stream/.ingestor.ca453944eff07df.schema", + "smoke/.stream/.ingestor.ca453944eff07df.stream.json", "smoke/.stream/.schema", "smoke/.stream/.stream.json", - "smoke/date=2026-11-11/hour=08/minute=00/d97d0c2e2ddd006.data.01M74SZQQ6PP5HSKS6P1JXW6TQ.parquet", - "smoke/date=2026-11-11/ingestor..d97d0c2e2ddd006.manifest.json" + "smoke/date=2026-11-11/hour=16/minute=03/ca453944eff07df.data.01M75NM5946JWKF95XGEZEP8Z2.parquet", + "smoke/date=2026-11-11/ingestor..ca453944eff07df.manifest.json" ] }, "postgres-tls": { "trusted_ca_restart": true, "untrusted_ca_rejected": true, "untrusted_ca_output": "Error: ObjectStorageError: PostgreSQL metastore unavailable: Error occurred while creating a new object: error performing TLS handshake\n\nCaused by:\n PostgreSQL metastore unavailable: Error occurred while creating a new object: error performing TLS handshake\n", - "untrusted_ca_server_alert": "2026-11-11 08:01:47.935 UTC [127] LOG: could not accept SSL connection: tlsv1 alert unknown ca\n", - "lazy_reload": true, - "query": "[{\"count\":1}]", - "dashboard": { - "version": "v1", - "title": "Postgres smoke", - "author": "8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918", - "dashboardId": "01M74T2GY6Y31B6B74N9XGF19Z", - "created": "2026-11-11T08:02:31.494659236+00:00", - "modified": "2026-11-11T08:02:31.494659236+00:00", - "tags": null, - "isFavorite": false, - "dashboardType": "Dashboard", - "tiles": [], - "tenantId": null, - "revision": "feb4fe847e41a4f08b8369a90a246a4a9e291aa10b9b85ad6b1463b6dab7b347" - }, - "keys": [ - ".parseable/postgres-metastore-id", - "lazy/date=2026-11-11/hour=08/minute=03/6f97a7d0983d3d9.data.01M74T57RBAGCEEZ73TRR5SGXR.parquet", - "pstats/date=2026-11-11/hour=08/minute=01/dataset_name=smoke/.data.01M74T3DK1EF85HBBCGG580X96.parquet", - "smoke/date=2026-11-11/hour=08/minute=01/.data.01M74T1K01S1D4BJKHY1AR1PYM.parquet" - ], - "non_parquet_keys": [ - ".parseable/postgres-metastore-id" - ], - "request_delta": { - "parseable_storage_request_response_time_count{method=\"GET_RANGES\",provider=\"s3\",status=\"200\"}": 5.0 - }, - "observed_requests_during_window_divided_by_queries": 1.0, - "database_outage": { - "readiness": 503, - "request_failed": true, - "seconds": 0.004449435044080019 - }, - "bucket_outage_readiness": 503, - "deleted_stream_rows": 0, - "delete_recreate": true, - "final_keys": [ - ".parseable/postgres-metastore-id", - "lazy/date=2026-11-11/hour=08/minute=03/6f97a7d0983d3d9.data.01M74T57RBAGCEEZ73TRR5SGXR.parquet", - "pstats/date=2026-11-11/hour=08/minute=01/dataset_name=smoke/.data.01M74T3DK1EF85HBBCGG580X96.parquet", - "pstats/date=2026-11-11/hour=08/minute=03/dataset_name=lazy/6f97a7d0983d3d9.data.01M74T72B8N681THKECS821NH4.parquet", - "smoke/date=2026-11-11/hour=08/minute=04/6f97a7d0983d3d9.data.01M74T72B8XG3BJ1XATNGYWSRA.parquet" - ], - "interrupted_deletion_restart": [ - { - "step": "before-data", - "hidden": true, - "cold_load_rejected": true, - "http_retry": true, - "recreation": true - }, - { - "step": "after-data", - "hidden": true, - "cold_load_rejected": true, - "http_retry": true, - "recreation": true - } - ] - }, - "postgres": { - "lazy_reload": true, - "query": "[{\"count\":1}]", - "dashboard": { - "version": "v1", - "title": "Postgres smoke", - "author": "8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918", - "dashboardId": "01M74TBP2FCCBXZH3H1K9PQW72", - "created": "2026-11-11T08:07:31.663110272+00:00", - "modified": "2026-11-11T08:07:31.663110272+00:00", - "tags": null, - "isFavorite": false, - "dashboardType": "Dashboard", - "tiles": [], - "tenantId": null, - "revision": "84b5f12dd1b98ed78f48a43b1b089615ee0da3e2db1df3e52e0f1e1dd2e83957" - }, - "keys": [ - ".parseable/postgres-metastore-id", - "lazy/date=2026-11-11/hour=08/minute=08/496c429a726b98c.data.01M74TEDA543YCHYXEA3F0QAEA.parquet", - "pstats/date=2026-11-11/hour=08/minute=06/dataset_name=smoke/.data.01M74TCJNQQ6KVX07JH6RJ4GD0.parquet", - "smoke/date=2026-11-11/hour=08/minute=06/.data.01M74TAR2QY8AMHTG8F6CHMSTZ.parquet" - ], - "non_parquet_keys": [ - ".parseable/postgres-metastore-id" - ], - "request_delta": { - "parseable_storage_request_response_time_count{method=\"GET_RANGES\",provider=\"s3\",status=\"200\"}": 5.0 - }, - "observed_requests_during_window_divided_by_queries": 1.0, - "database_outage": { - "readiness": 503, - "request_failed": true, - "seconds": 0.004181399941444397 - }, - "bucket_outage_readiness": 503, - "deleted_stream_rows": 0, - "delete_recreate": true, - "final_keys": [ - ".parseable/postgres-metastore-id", - "lazy/date=2026-11-11/hour=08/minute=08/496c429a726b98c.data.01M74TEDA543YCHYXEA3F0QAEA.parquet", - "pstats/date=2026-11-11/hour=08/minute=06/dataset_name=smoke/.data.01M74TCJNQQ6KVX07JH6RJ4GD0.parquet", - "pstats/date=2026-11-11/hour=08/minute=08/dataset_name=lazy/496c429a726b98c.data.01M74TG7X53KFBJAX3ESJVYHBF.parquet", - "smoke/date=2026-11-11/hour=08/minute=09/496c429a726b98c.data.01M74TG7X5ZB4F5KWD9PPBXNMP.parquet" - ], - "interrupted_deletion_restart": [ - { - "step": "before-data", - "hidden": true, - "cold_load_rejected": true, - "http_retry": true, - "recreation": true - }, - { - "step": "after-data", - "hidden": true, - "cold_load_rejected": true, - "http_retry": true, - "recreation": true - } - ] - }, - "startup_rejections": { - "future-schema": "Error: ObjectStorageError: PostgreSQL metastore unavailable: unsupported schema/protocol version; upgrade this server\n\nCaused by:\n PostgreSQL metastore unavailable: unsupported schema/protocol version; upgrade this server\n", - "wrong-id": "Error: ObjectStorageError: PostgreSQL metastore unavailable: backend/bucket identity mismatch\n\nCaused by:\n PostgreSQL metastore unavailable: backend/bucket identity mismatch\n", - "unreachable": "Error: ObjectStorageError: PostgreSQL metastore unavailable: Error occurred while creating a new object: error connecting to server\n\nCaused by:\n PostgreSQL metastore unavailable: Error occurred while creating a new object: error connecting to server\n" + "untrusted_ca_server_alert": "2026-11-11 16:04:49.430 UTC [125] LOG: could not accept SSL connection: tlsv1 alert unknown ca\n" }, - "postgres_rows": "tenant | kind | key | version | bytes \n--------+----------+--------------------------------------------------------------------------------------------------------------------+---------+-------\n | document | .parseable/.parseable.json | 121 | 384\n | document | .parseable/ingestor.496c429a726b98c.json | 39 | 241\n | document | .parseable/querier.36256ead0ab8108.json | 123 | 240\n | document | .users/.dashboard-locations/01M74TBP2FCCBXZH3H1K9PQW72.json | 16 | 119\n | document | .users/.dashboard-titles/571a2cefe2d0d1a9a26dab4810a5556875c7d7531d4f558a8f7b28ab486e2395.json | 18 | 170\n | document | .users/8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918/dashboards/01M74TBP2FCCBXZH3H1K9PQW72.json | 17 | 545\n | schema | lazy/.stream/.ingestor.496c429a726b98c.schema | 56 | 585\n | stream | lazy/.stream/.ingestor.496c429a726b98c.stream.json | 54 | 853\n | schema | lazy/.stream/.schema | 131 | 585\n | stream | lazy/.stream/.stream.json | 58 | 597\n | manifest | lazy/date=2026-11-11/ingestor..496c429a726b98c.manifest.json | 52 | 422\n | schema | pstats/.stream/.ingestor.496c429a726b98c.schema | 148 | 1117\n | stream | pstats/.stream/.ingestor.496c429a726b98c.stream.json | 146 | 898\n | schema | pstats/.stream/.schema | 127 | 1117\n | stream | pstats/.stream/.stream.json | 23 | 867\n | manifest | pstats/date=2026-11-11/.manifest.json | 21 | 533\n | manifest | pstats/date=2026-11-11/ingestor..496c429a726b98c.manifest.json | 144 | 804\n | schema | smoke/.stream/.ingestor.496c429a726b98c.schema | 141 | 27\n | stream | smoke/.stream/.ingestor.496c429a726b98c.stream.json | 142 | 597\n | schema | smoke/.stream/.schema | 139 | 27\n | stream | smoke/.stream/.stream.json | 140 | 597\n(21 rows)", "commands": [ { "argv": [ @@ -263,13 +135,13 @@ "run", "-d", "--name", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-1181462-0", "-e", "POSTGRES_PASSWORD=smoke", "-e", "POSTGRES_DB=parseable", "-p", - "127.0.0.1:35731:5432", + "127.0.0.1:44333:5432", "postgres:17-alpine" ], "exit_code": 0 @@ -280,13 +152,13 @@ "run", "-d", "--name", - "parseable-pg-smoke-3714167-1", + "parseable-pg-smoke-1181462-1", "-e", "MINIO_ROOT_USER=minioadmin", "-e", "MINIO_ROOT_PASSWORD=minioadmin", "-p", - "127.0.0.1:50141:9000", + "127.0.0.1:37943:9000", "quay.io/parseablehq/minio:edge", "server", "/data" @@ -297,7 +169,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-1181462-0", "pg_isready", "-h", "127.0.0.1", @@ -310,7 +182,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-1181462-0", "pg_isready", "-h", "127.0.0.1", @@ -323,7 +195,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-1181462-0", "pg_isready", "-h", "127.0.0.1", @@ -341,9 +213,9 @@ "rsa:2048", "-nodes", "-keyout", - "/.postgres-smoke/tls/ca.key", + ".postgres-smoke/tls/ca.key", "-out", - "/.postgres-smoke/tls/ca.crt", + ".postgres-smoke/tls/ca.crt", "-days", "2", "-subj", @@ -359,9 +231,9 @@ "rsa:2048", "-nodes", "-keyout", - "/.postgres-smoke/tls/server.key", + ".postgres-smoke/tls/server.key", "-out", - "/.postgres-smoke/tls/server.csr", + ".postgres-smoke/tls/server.csr", "-subj", "/CN=localhost" ], @@ -369,40 +241,4824 @@ }, { "argv": [ - "openssl", - "x509", - "-req", - "-in", - "/.postgres-smoke/tls/server.csr", - "-CA", - "/.postgres-smoke/tls/ca.crt", - "-CAkey", - "/.postgres-smoke/tls/ca.key", - "-CAcreateserial", - "-out", - "/.postgres-smoke/tls/server.crt", - "-days", - "2", - "-extfile", - "/.postgres-smoke/tls/extensions" + "openssl", + "x509", + "-req", + "-in", + ".postgres-smoke/tls/server.csr", + "-CA", + ".postgres-smoke/tls/ca.crt", + "-CAkey", + ".postgres-smoke/tls/ca.key", + "-CAcreateserial", + "-out", + ".postgres-smoke/tls/server.crt", + "-days", + "2", + "-extfile", + ".postgres-smoke/tls/extensions" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "cp", + ".postgres-smoke/tls/server.key", + "parseable-pg-smoke-1181462-0:/var/lib/postgresql/data/server.key" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "cp", + ".postgres-smoke/tls/server.crt", + "parseable-pg-smoke-1181462-0:/var/lib/postgresql/data/server.crt" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1181462-0", + "sh", + "-c", + "chown postgres:postgres /var/lib/postgresql/data/server.* && chmod 600 /var/lib/postgresql/data/server.key" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1181462-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-c", + "ALTER SYSTEM SET ssl='on';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1181462-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-c", + "SELECT pg_reload_conf();" + ], + "exit_code": 0 + }, + { + "argv": [ + "git", + "rev-parse", + "HEAD" + ], + "exit_code": 0 + }, + { + "argv": [ + "git", + "diff", + "HEAD" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "image", + "inspect", + "postgres:17-alpine" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "image", + "inspect", + "quay.io/parseablehq/minio:edge" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1181462-1", + "mkdir", + "-p", + "/data/smoke-object-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "stop", + "parseable-pg-smoke-1181462-1" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "start", + "parseable-pg-smoke-1181462-1" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1181462-1", + "mkdir", + "-p", + "/data/smoke-postgres-tls" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1181462-0", + "psql", + "-U", + "postgres", + "-c", + "CREATE DATABASE tls;" + ], + "exit_code": 0 + }, + { + "argv": [ + "openssl", + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-keyout", + ".postgres-smoke/tls/wrong.key", + "-out", + ".postgres-smoke/tls/wrong.crt", + "-days", + "2", + "-subj", + "/CN=Wrong CA" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "logs", + "parseable-pg-smoke-1181462-0" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "logs", + "parseable-pg-smoke-1181462-0" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1181462-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1181462-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1181462-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1181462-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1181462-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1181462-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1181462-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1181462-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1181462-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1181462-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1181462-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1181462-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1181462-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1181462-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1181462-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1181462-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "rm", + "-f", + "parseable-pg-smoke-1181462-1" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "rm", + "-f", + "parseable-pg-smoke-1181462-0" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 0 + } + ], + "success": false, + "failure": "Traceback (most recent call last):\n File \"scripts/postgres-metastore-smoke.py\", line 344, in \n request('/logstream/smoke/retention', 'PUT', [{'description': 'smoke retention', 'action': 'delete', 'duration': '30d'}])\n ~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"scripts/postgres-metastore-smoke.py\", line 283, in request\n with urllib.request.urlopen(req, timeout=timeout) as response:\n ~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/lib/python3.14/urllib/request.py\", line 187, in urlopen\n return opener.open(url, data, timeout)\n ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^\n File \"/usr/lib/python3.14/urllib/request.py\", line 493, in open\n response = meth(req, response)\n File \"/usr/lib/python3.14/urllib/request.py\", line 602, in http_response\n response = self.parent.error(\n 'http', request, response, code, msg, hdrs)\n File \"/usr/lib/python3.14/urllib/request.py\", line 531, in error\n return self._call_chain(*args)\n ~~~~~~~~~~~~~~~~^^^^^^^\n File \"/usr/lib/python3.14/urllib/request.py\", line 464, in _call_chain\n result = func(*args)\n File \"/usr/lib/python3.14/urllib/request.py\", line 611, in http_error_default\n raise HTTPError(req.full_url, code, msg, hdrs, fp)\nurllib.error.HTTPError: HTTP Error 503: Service Unavailable\nStorage Error MetastoreError: MetastoreErrorDetail { operation: \"ObjectStorageError\", message: \"PostgreSQL metastore unavailable: PostgreSQL metastore unavailable: SQL client deadline exceeded\", stream_name: None, file_path: None, timestamp: Some(2026-11-11T16:06:02.469332470Z), metadata: {}, status_code: 503 }\n", + "http_requests": [ + { + "path": "/liveness", + "method": "GET", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "PUT", + "status": 200 + }, + { + "path": "/ingest", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/logstream/smoke/retention", + "method": "PUT", + "status": 200 + }, + { + "path": "/dashboards", + "method": "POST", + "status": 200 + }, + { + "path": "/liveness", + "method": "GET", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/dashboards", + "method": "GET", + "status": 200 + }, + { + "path": "/metrics", + "method": "GET", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/metrics", + "method": "GET", + "status": 200 + }, + { + "path": "/readiness", + "method": "GET", + "status": 503 + }, + { + "path": "/readiness", + "method": "GET", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "DELETE", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "PUT", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/liveness", + "method": "GET", + "status": 200 + }, + { + "path": "/liveness", + "method": "GET", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "PUT", + "status": 200 + }, + { + "path": "/ingest", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/logstream/smoke/retention", + "method": "PUT", + "status": 503 + } + ] + }, + { + "requested_modes": [ + "object-store", + "postgres-tls", + "postgres" + ], + "source_commit": "30bd6ce316fb6689ffc6371f8afb58dd361fcc66", + "binary_sha256": "76cb03a095ab04cb1645eb95032c9da94aeab4486009cb5589c66463b5e329e9", + "source_diff_sha256": "2977ab72da8198c1e8ff5f455b2ad9052e26adfbb080738fed2a130407a3b28d", + "image_digests": { + "postgres:17-alpine": [ + "postgres@sha256:b0f9560a2de083e2cc7382e75f808c7381a32852a7ec49117deedb300e552b24" + ], + "quay.io/parseablehq/minio:edge": [ + "quay.io/parseablehq/minio@sha256:88276d0a028d758ffaf80d65dd80071386e91b4114251a3dda101eadf000faeb" + ] + }, + "measurement": "Node-local storage counter delta during five queries, including background traffic; not isolated requests per query.", + "object-store": { + "lazy_reload": true, + "query": "[{\"count\":1}]", + "dashboard": { + "version": "v1", + "title": "Postgres smoke", + "author": "8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918", + "dashboardId": "01M75Q46A1TPT0FMVP1V702SN4", + "created": "2026-11-11T16:30:14.849272674+00:00", + "modified": "2026-11-11T16:30:14.849272674+00:00", + "tags": null, + "isFavorite": false, + "dashboardType": "Dashboard", + "tiles": [], + "tenantId": null, + "revision": "4712ade4f49952d0bdeb55a6356bec2aff0c0535ffb9ddcb59ecdc1c4013e52b" + }, + "keys": [ + ".parseable/.parseable.json", + ".parseable/ingestor.5f166cb2f1e1418.json", + ".parseable/querier.04822b872795bce.json", + ".users/.dashboard-locations/01M75Q46A1TPT0FMVP1V702SN4.json", + ".users/.dashboard-titles/571a2cefe2d0d1a9a26dab4810a5556875c7d7531d4f558a8f7b28ab486e2395.json", + ".users/8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918/dashboards/01M75Q46A1TPT0FMVP1V702SN4.json", + "lazy/.stream/.ingestor.5f166cb2f1e1418.schema", + "lazy/.stream/.ingestor.5f166cb2f1e1418.stream.json", + "lazy/.stream/.schema", + "lazy/.stream/.stream.json", + "lazy/date=2026-11-11/hour=16/minute=31/5f166cb2f1e1418.data.01M75Q7DTBPJJWJCBH8AE4YN8S.parquet", + "lazy/date=2026-11-11/ingestor..5f166cb2f1e1418.manifest.json", + "pstats/.stream/.ingestor.5f166cb2f1e1418.schema", + "pstats/.stream/.ingestor.5f166cb2f1e1418.stream.json", + "pstats/.stream/.schema", + "pstats/.stream/.stream.json", + "pstats/date=2026-11-11/.manifest.json", + "pstats/date=2026-11-11/hour=16/minute=28/dataset_name=smoke/.data.01M75Q5K2P76MWJ3C5H3MPQJGR.parquet", + "smoke/.stream/.ingestor.5f166cb2f1e1418.schema", + "smoke/.stream/.ingestor.5f166cb2f1e1418.stream.json", + "smoke/.stream/.schema", + "smoke/.stream/.stream.json", + "smoke/date=2026-11-11/.manifest.json", + "smoke/date=2026-11-11/hour=16/minute=28/.data.01M75Q1XWPVJ100X36PKXWZ2BR.parquet" + ], + "non_parquet_keys": [ + ".parseable/.parseable.json", + ".parseable/ingestor.5f166cb2f1e1418.json", + ".parseable/querier.04822b872795bce.json", + ".users/.dashboard-locations/01M75Q46A1TPT0FMVP1V702SN4.json", + ".users/.dashboard-titles/571a2cefe2d0d1a9a26dab4810a5556875c7d7531d4f558a8f7b28ab486e2395.json", + ".users/8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918/dashboards/01M75Q46A1TPT0FMVP1V702SN4.json", + "lazy/.stream/.ingestor.5f166cb2f1e1418.schema", + "lazy/.stream/.ingestor.5f166cb2f1e1418.stream.json", + "lazy/.stream/.schema", + "lazy/.stream/.stream.json", + "lazy/date=2026-11-11/ingestor..5f166cb2f1e1418.manifest.json", + "pstats/.stream/.ingestor.5f166cb2f1e1418.schema", + "pstats/.stream/.ingestor.5f166cb2f1e1418.stream.json", + "pstats/.stream/.schema", + "pstats/.stream/.stream.json", + "pstats/date=2026-11-11/.manifest.json", + "smoke/.stream/.ingestor.5f166cb2f1e1418.schema", + "smoke/.stream/.ingestor.5f166cb2f1e1418.stream.json", + "smoke/.stream/.schema", + "smoke/.stream/.stream.json", + "smoke/date=2026-11-11/.manifest.json" + ], + "request_delta": { + "parseable_storage_request_response_time_count{method=\"GET\",provider=\"s3\",status=\"200\"}": 30.0, + "parseable_storage_request_response_time_count{method=\"GET_RANGES\",provider=\"s3\",status=\"200\"}": 5.0, + "parseable_storage_request_response_time_count{method=\"LIST\",provider=\"s3\",status=\"200\"}": 5.0, + "parseable_storage_request_response_time_count{method=\"LIST_DELIM\",provider=\"s3\",status=\"200\"}": 5.0 + }, + "observed_requests_during_window_divided_by_queries": 9.0, + "bucket_outage_readiness": 503, + "delete_recreate": true, + "final_keys": [ + ".parseable/.parseable.json", + ".parseable/ingestor.5f166cb2f1e1418.json", + ".parseable/querier.04822b872795bce.json", + ".users/.dashboard-locations/01M75Q46A1TPT0FMVP1V702SN4.json", + ".users/.dashboard-titles/571a2cefe2d0d1a9a26dab4810a5556875c7d7531d4f558a8f7b28ab486e2395.json", + ".users/8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918/dashboards/01M75Q46A1TPT0FMVP1V702SN4.json", + "lazy/.stream/.ingestor.5f166cb2f1e1418.schema", + "lazy/.stream/.ingestor.5f166cb2f1e1418.stream.json", + "lazy/.stream/.schema", + "lazy/.stream/.stream.json", + "lazy/date=2026-11-11/hour=16/minute=31/5f166cb2f1e1418.data.01M75Q7DTBPJJWJCBH8AE4YN8S.parquet", + "lazy/date=2026-11-11/ingestor..5f166cb2f1e1418.manifest.json", + "pstats/.stream/.ingestor.5f166cb2f1e1418.schema", + "pstats/.stream/.ingestor.5f166cb2f1e1418.stream.json", + "pstats/.stream/.schema", + "pstats/.stream/.stream.json", + "pstats/date=2026-11-11/.manifest.json", + "pstats/date=2026-11-11/hour=16/minute=28/dataset_name=smoke/.data.01M75Q5K2P76MWJ3C5H3MPQJGR.parquet", + "pstats/date=2026-11-11/hour=16/minute=31/dataset_name=lazy/5f166cb2f1e1418.data.01M75Q98DCXTRQG16AV1VMJQQ0.parquet", + "pstats/date=2026-11-11/ingestor..5f166cb2f1e1418.manifest.json", + "smoke/.stream/.ingestor.5f166cb2f1e1418.schema", + "smoke/.stream/.ingestor.5f166cb2f1e1418.stream.json", + "smoke/.stream/.schema", + "smoke/.stream/.stream.json", + "smoke/date=2026-11-11/hour=16/minute=32/5f166cb2f1e1418.data.01M75Q98DCX5GHSZ2DT9V938W5.parquet", + "smoke/date=2026-11-11/ingestor..5f166cb2f1e1418.manifest.json" + ] + }, + "postgres-tls": { + "trusted_ca_restart": true, + "untrusted_ca_rejected": true, + "untrusted_ca_output": "Error: ObjectStorageError: PostgreSQL metastore unavailable: Error occurred while creating a new object: error performing TLS handshake\n\nCaused by:\n PostgreSQL metastore unavailable: Error occurred while creating a new object: error performing TLS handshake\n", + "untrusted_ca_server_alert": "2026-11-11 16:33:48.704 UTC [134] LOG: could not accept SSL connection: tlsv1 alert unknown ca\n", + "lazy_reload": true, + "query": "[{\"count\":1}]", + "dashboard": { + "version": "v1", + "title": "Postgres smoke", + "author": "8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918", + "dashboardId": "01M75QC0CEB0PTBM5878FAVP92", + "created": "2026-11-11T16:34:30.926914794+00:00", + "modified": "2026-11-11T16:34:30.926914794+00:00", + "tags": null, + "isFavorite": false, + "dashboardType": "Dashboard", + "tiles": [], + "tenantId": null, + "revision": "6ec933632984267e6241812ea5543f0673ce168cff1fd35e5c9a1f850ace5cf4" + }, + "keys": [ + ".parseable/postgres-metastore-id", + "lazy/date=2026-11-11/hour=16/minute=35/6b0aca88cac3b75.data.01M75QEQZRYCNSWYRX3T9X8J74.parquet", + "pstats/date=2026-11-11/hour=16/minute=33/dataset_name=smoke/.data.01M75QCXHFRZYMYAYTQMD83EC3.parquet", + "smoke/date=2026-11-11/hour=16/minute=33/.data.01M75QB2YFJG5RWDGDQ99WAZK6.parquet" + ], + "non_parquet_keys": [ + ".parseable/postgres-metastore-id" + ], + "request_delta": { + "parseable_storage_request_response_time_count{method=\"GET_RANGES\",provider=\"s3\",status=\"200\"}": 5.0 + }, + "observed_requests_during_window_divided_by_queries": 1.0, + "database_outage": { + "readiness": 503, + "request_failed": true, + "seconds": 0.005427954951301217 + }, + "explicit_stream_errors": { + "pending_deletion": 409, + "database_outage": 503 + }, + "bucket_outage_readiness": 503, + "deleted_stream_rows": 0, + "delete_recreate": true, + "final_keys": [ + ".parseable/postgres-metastore-id", + "lazy/date=2026-11-11/hour=16/minute=35/6b0aca88cac3b75.data.01M75QEQZRYCNSWYRX3T9X8J74.parquet", + "pstats/date=2026-11-11/hour=16/minute=33/dataset_name=smoke/.data.01M75QCXHFRZYMYAYTQMD83EC3.parquet", + "pstats/date=2026-11-11/hour=16/minute=35/dataset_name=lazy/6b0aca88cac3b75.data.01M75QGJJR3RGNQQ96RGNFQK0N.parquet", + "smoke/date=2026-11-11/hour=16/minute=36/6b0aca88cac3b75.data.01M75QGJJRRTFFK2GHBC5J9WXQ.parquet" + ], + "interrupted_deletion_restart": [ + { + "step": "before-data", + "hidden": true, + "cold_load_rejected": true, + "http_retry": true, + "recreation": true + }, + { + "step": "after-data", + "hidden": true, + "cold_load_rejected": true, + "http_retry": true, + "recreation": true + } + ] + }, + "postgres": { + "lazy_reload": true, + "query": "[{\"count\":1}]", + "dashboard": { + "version": "v1", + "title": "Postgres smoke", + "author": "8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918", + "dashboardId": "01M75QN9GBN0VPHXE5CC4GK4ZY", + "created": "2026-11-11T16:39:35.179279671+00:00", + "modified": "2026-11-11T16:39:35.179279671+00:00", + "tags": null, + "isFavorite": false, + "dashboardType": "Dashboard", + "tiles": [], + "tenantId": null, + "revision": "3b457f579bb60bc8eecc0709e7b33bc818ffc2e96df084430487d7f090f5101a" + }, + "keys": [ + ".parseable/postgres-metastore-id", + "lazy/date=2026-11-11/hour=16/minute=40/028b583f6797003.data.01M75QQX8G7JYT2K06T248BANP.parquet", + "pstats/date=2026-11-11/hour=16/minute=38/dataset_name=smoke/.data.01M75QP21YY11Q5KTHGP8XN94H.parquet", + "smoke/date=2026-11-11/hour=16/minute=38/.data.01M75QM7EZ784610Z48E8M9VXF.parquet" + ], + "non_parquet_keys": [ + ".parseable/postgres-metastore-id" + ], + "request_delta": { + "parseable_storage_request_response_time_count{method=\"GET_RANGES\",provider=\"s3\",status=\"200\"}": 5.0 + }, + "observed_requests_during_window_divided_by_queries": 1.0, + "database_outage": { + "readiness": 503, + "request_failed": true, + "seconds": 0.004398432793095708 + }, + "explicit_stream_errors": { + "pending_deletion": 409, + "database_outage": 503 + }, + "bucket_outage_readiness": 503, + "deleted_stream_rows": 0, + "delete_recreate": true, + "final_keys": [ + ".parseable/postgres-metastore-id", + "lazy/date=2026-11-11/hour=16/minute=40/028b583f6797003.data.01M75QQX8G7JYT2K06T248BANP.parquet", + "pstats/date=2026-11-11/hour=16/minute=38/dataset_name=smoke/.data.01M75QP21YY11Q5KTHGP8XN94H.parquet", + "pstats/date=2026-11-11/hour=16/minute=40/dataset_name=lazy/028b583f6797003.data.01M75QSQVGB4N665G100P7TE8F.parquet", + "smoke/date=2026-11-11/hour=16/minute=43/028b583f6797003.data.01M75QXD1G8Z00Q5V5QP6NQXBH.parquet" + ], + "interrupted_deletion_restart": [ + { + "step": "before-data", + "hidden": true, + "cold_load_rejected": true, + "http_retry": true, + "recreation": true + }, + { + "step": "after-data", + "hidden": true, + "cold_load_rejected": true, + "http_retry": true, + "recreation": true + } + ] + }, + "startup_rejections": { + "future-schema": "Error: ObjectStorageError: PostgreSQL metastore unavailable: unsupported schema/protocol version; upgrade this server\n\nCaused by:\n PostgreSQL metastore unavailable: unsupported schema/protocol version; upgrade this server\n", + "wrong-id": "Error: ObjectStorageError: PostgreSQL metastore unavailable: backend/bucket identity mismatch\n\nCaused by:\n PostgreSQL metastore unavailable: backend/bucket identity mismatch\n", + "unreachable": "Error: ObjectStorageError: PostgreSQL metastore unavailable: Error occurred while creating a new object: error connecting to server\n\nCaused by:\n PostgreSQL metastore unavailable: Error occurred while creating a new object: error connecting to server\n" + }, + "postgres_rows": "tenant | kind | key | version | bytes \n--------+----------+--------------------------------------------------------------------------------------------------------------------+---------+-------\n | document | .parseable/.parseable.json | 150 | 384\n | document | .parseable/ingestor.028b583f6797003.json | 39 | 241\n | document | .parseable/querier.ec99af452c30213.json | 152 | 240\n | document | .users/.dashboard-locations/01M75QN9GBN0VPHXE5CC4GK4ZY.json | 16 | 119\n | document | .users/.dashboard-titles/571a2cefe2d0d1a9a26dab4810a5556875c7d7531d4f558a8f7b28ab486e2395.json | 18 | 170\n | document | .users/8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918/dashboards/01M75QN9GBN0VPHXE5CC4GK4ZY.json | 17 | 545\n | schema | lazy/.stream/.ingestor.028b583f6797003.schema | 56 | 585\n | stream | lazy/.stream/.ingestor.028b583f6797003.stream.json | 54 | 853\n | schema | lazy/.stream/.schema | 157 | 585\n | stream | lazy/.stream/.stream.json | 57 | 597\n | manifest | lazy/date=2026-11-11/ingestor..028b583f6797003.manifest.json | 52 | 420\n | schema | pstats/.stream/.ingestor.028b583f6797003.schema | 113 | 1117\n | stream | pstats/.stream/.ingestor.028b583f6797003.stream.json | 140 | 895\n | schema | pstats/.stream/.schema | 160 | 1117\n | stream | pstats/.stream/.stream.json | 23 | 867\n | manifest | pstats/date=2026-11-11/.manifest.json | 21 | 551\n | manifest | pstats/date=2026-11-11/ingestor..028b583f6797003.manifest.json | 138 | 671\n | schema | smoke/.stream/.ingestor.028b583f6797003.schema | 170 | 27\n | stream | smoke/.stream/.ingestor.028b583f6797003.stream.json | 171 | 597\n | schema | smoke/.stream/.schema | 168 | 27\n | stream | smoke/.stream/.stream.json | 169 | 597\n(21 rows)", + "commands": [ + { + "argv": [ + "docker", + "run", + "-d", + "--name", + "parseable-pg-smoke-1309689-0", + "-e", + "POSTGRES_PASSWORD=smoke", + "-e", + "POSTGRES_DB=parseable", + "-p", + "127.0.0.1:50913:5432", + "postgres:17-alpine" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "run", + "-d", + "--name", + "parseable-pg-smoke-1309689-1", + "-e", + "MINIO_ROOT_USER=minioadmin", + "-e", + "MINIO_ROOT_PASSWORD=minioadmin", + "-p", + "127.0.0.1:56891:9000", + "quay.io/parseablehq/minio:edge", + "server", + "/data" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "pg_isready", + "-h", + "127.0.0.1", + "-U", + "postgres" + ], + "exit_code": 2 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "pg_isready", + "-h", + "127.0.0.1", + "-U", + "postgres" + ], + "exit_code": 2 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "pg_isready", + "-h", + "127.0.0.1", + "-U", + "postgres" + ], + "exit_code": 2 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "pg_isready", + "-h", + "127.0.0.1", + "-U", + "postgres" + ], + "exit_code": 0 + }, + { + "argv": [ + "openssl", + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-keyout", + ".postgres-smoke/tls/ca.key", + "-out", + ".postgres-smoke/tls/ca.crt", + "-days", + "2", + "-subj", + "/CN=Smoke CA" + ], + "exit_code": 0 + }, + { + "argv": [ + "openssl", + "req", + "-newkey", + "rsa:2048", + "-nodes", + "-keyout", + ".postgres-smoke/tls/server.key", + "-out", + ".postgres-smoke/tls/server.csr", + "-subj", + "/CN=localhost" + ], + "exit_code": 0 + }, + { + "argv": [ + "openssl", + "x509", + "-req", + "-in", + ".postgres-smoke/tls/server.csr", + "-CA", + ".postgres-smoke/tls/ca.crt", + "-CAkey", + ".postgres-smoke/tls/ca.key", + "-CAcreateserial", + "-out", + ".postgres-smoke/tls/server.crt", + "-days", + "2", + "-extfile", + ".postgres-smoke/tls/extensions" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "cp", + ".postgres-smoke/tls/server.key", + "parseable-pg-smoke-1309689-0:/var/lib/postgresql/data/server.key" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "cp", + ".postgres-smoke/tls/server.crt", + "parseable-pg-smoke-1309689-0:/var/lib/postgresql/data/server.crt" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "sh", + "-c", + "chown postgres:postgres /var/lib/postgresql/data/server.* && chmod 600 /var/lib/postgresql/data/server.key" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-c", + "ALTER SYSTEM SET ssl='on';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-c", + "SELECT pg_reload_conf();" + ], + "exit_code": 0 + }, + { + "argv": [ + "git", + "rev-parse", + "HEAD" + ], + "exit_code": 0 + }, + { + "argv": [ + "git", + "diff", + "HEAD" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "image", + "inspect", + "postgres:17-alpine" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "image", + "inspect", + "quay.io/parseablehq/minio:edge" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-1", + "mkdir", + "-p", + "/data/smoke-object-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "stop", + "parseable-pg-smoke-1309689-1" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "start", + "parseable-pg-smoke-1309689-1" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-1", + "mkdir", + "-p", + "/data/smoke-postgres-tls" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-c", + "CREATE DATABASE tls;" + ], + "exit_code": 0 + }, + { + "argv": [ + "openssl", + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-keyout", + ".postgres-smoke/tls/wrong.key", + "-out", + ".postgres-smoke/tls/wrong.crt", + "-days", + "2", + "-subj", + "/CN=Wrong CA" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "logs", + "parseable-pg-smoke-1309689-0" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "logs", + "parseable-pg-smoke-1309689-0" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-v", + "ON_ERROR_STOP=1", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "INSERT INTO parseable_documents(tenant,key,kind,body) VALUES('','deletion-explicit/.stream/.deleting','document',''::bytea);" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-v", + "ON_ERROR_STOP=1", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "DELETE FROM parseable_documents WHERE tenant='' AND key='deletion-explicit/.stream/.deleting';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "stop", + "parseable-pg-smoke-1309689-0" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "start", + "parseable-pg-smoke-1309689-0" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "stop", + "parseable-pg-smoke-1309689-1" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "start", + "parseable-pg-smoke-1309689-1" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT count(*) FROM parseable_documents WHERE starts_with(key,'smoke/');" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-v", + "ON_ERROR_STOP=1", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "INSERT INTO parseable_documents(tenant,key,kind,body) VALUES('','smoke/.stream/.deleting','document',''::bytea);" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-v", + "ON_ERROR_STOP=1", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT count(*) FROM parseable_documents WHERE starts_with(key,'smoke/');" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-v", + "ON_ERROR_STOP=1", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "CREATE FUNCTION smoke_delete_fault() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'injected row deletion failure'; END $$; CREATE TRIGGER smoke_delete_fault BEFORE DELETE ON parseable_documents FOR EACH ROW EXECUTE FUNCTION smoke_delete_fault();" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-v", + "ON_ERROR_STOP=1", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "DROP TRIGGER smoke_delete_fault ON parseable_documents; DROP FUNCTION smoke_delete_fault();" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-v", + "ON_ERROR_STOP=1", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT count(*) FROM parseable_documents WHERE starts_with(key,'smoke/');" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-1", + "mkdir", + "-p", + "/data/smoke-postgres" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-v", + "ON_ERROR_STOP=1", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "INSERT INTO parseable_documents(tenant,key,kind,body) VALUES('','deletion-explicit/.stream/.deleting','document',''::bytea);" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-v", + "ON_ERROR_STOP=1", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "DELETE FROM parseable_documents WHERE tenant='' AND key='deletion-explicit/.stream/.deleting';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "stop", + "parseable-pg-smoke-1309689-0" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "start", + "parseable-pg-smoke-1309689-0" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "stop", + "parseable-pg-smoke-1309689-1" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "start", + "parseable-pg-smoke-1309689-1" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT count(*) FROM parseable_documents WHERE starts_with(key,'smoke/');" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-v", + "ON_ERROR_STOP=1", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "INSERT INTO parseable_documents(tenant,key,kind,body) VALUES('','smoke/.stream/.deleting','document',''::bytea);" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-v", + "ON_ERROR_STOP=1", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT count(*) FROM parseable_documents WHERE starts_with(key,'smoke/');" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-v", + "ON_ERROR_STOP=1", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "CREATE FUNCTION smoke_delete_fault() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'injected row deletion failure'; END $$; CREATE TRIGGER smoke_delete_fault BEFORE DELETE ON parseable_documents FOR EACH ROW EXECUTE FUNCTION smoke_delete_fault();" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-v", + "ON_ERROR_STOP=1", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "DROP TRIGGER smoke_delete_fault ON parseable_documents; DROP FUNCTION smoke_delete_fault();" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-v", + "ON_ERROR_STOP=1", + "-U", + "postgres", + "-d", + "parseable", + "-Atc", + "SELECT count(*) FROM parseable_documents WHERE starts_with(key,'smoke/');" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-c", + "UPDATE parseable_backend SET schema_version=999;" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-c", + "UPDATE parseable_backend SET schema_version=3;" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1309689-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-c", + "SELECT tenant,kind,key,version,octet_length(body) AS bytes FROM parseable_documents ORDER BY tenant,key;" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "rm", + "-f", + "parseable-pg-smoke-1309689-1" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "rm", + "-f", + "parseable-pg-smoke-1309689-0" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": -9 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1 + } + ], + "success": true, + "http_requests": [ + { + "path": "/liveness", + "method": "GET", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "PUT", + "status": 200 + }, + { + "path": "/ingest", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/logstream/smoke/retention", + "method": "PUT", + "status": 200 + }, + { + "path": "/dashboards", + "method": "POST", + "status": 200 + }, + { + "path": "/liveness", + "method": "GET", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/dashboards", + "method": "GET", + "status": 200 + }, + { + "path": "/metrics", + "method": "GET", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/metrics", + "method": "GET", + "status": 200 + }, + { + "path": "/readiness", + "method": "GET", + "status": 503 + }, + { + "path": "/readiness", + "method": "GET", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "DELETE", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "PUT", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/liveness", + "method": "GET", + "status": 200 + }, + { + "path": "/liveness", + "method": "GET", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "PUT", + "status": 200 + }, + { + "path": "/ingest", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/logstream/smoke/retention", + "method": "PUT", + "status": 200 + }, + { + "path": "/dashboards", + "method": "POST", + "status": 200 + }, + { + "path": "/liveness", + "method": "GET", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/dashboards", + "method": "GET", + "status": 200 + }, + { + "path": "/metrics", + "method": "GET", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/metrics", + "method": "GET", + "status": 200 + }, + { + "path": "/readiness", + "method": "GET", + "status": 503 + }, + { + "path": "/logstream/outage", + "method": "PUT", + "status": 503 + }, + { + "path": "/readiness", + "method": "GET", + "status": 503 + }, + { + "path": "/readiness", + "method": "GET", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/readiness", + "method": "GET", + "status": 503 + }, + { + "path": "/readiness", + "method": "GET", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "DELETE", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "PUT", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/liveness", + "method": "GET", + "status": 200 + }, + { + "path": "/logstream", + "method": "GET", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 400 + }, + { + "path": "/logstream/smoke", + "method": "DELETE", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "PUT", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "DELETE", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "PUT", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "DELETE", + "status": 503 + }, + { + "path": "/liveness", + "method": "GET", + "status": 200 + }, + { + "path": "/logstream", + "method": "GET", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 400 + }, + { + "path": "/logstream/smoke", + "method": "DELETE", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "PUT", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "DELETE", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "PUT", + "status": 200 + }, + { + "path": "/liveness", + "method": "GET", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "PUT", + "status": 200 + }, + { + "path": "/ingest", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/logstream/smoke/retention", + "method": "PUT", + "status": 200 + }, + { + "path": "/dashboards", + "method": "POST", + "status": 200 + }, + { + "path": "/liveness", + "method": "GET", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/dashboards", + "method": "GET", + "status": 200 + }, + { + "path": "/metrics", + "method": "GET", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/metrics", + "method": "GET", + "status": 200 + }, + { + "path": "/readiness", + "method": "GET", + "status": 503 + }, + { + "path": "/logstream/outage", + "method": "PUT", + "status": 503 + }, + { + "path": "/readiness", + "method": "GET", + "status": 503 + }, + { + "path": "/readiness", + "method": "GET", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/readiness", + "method": "GET", + "status": 503 + }, + { + "path": "/readiness", + "method": "GET", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "DELETE", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "PUT", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 200 + }, + { + "path": "/liveness", + "method": "GET", + "status": 200 + }, + { + "path": "/logstream", + "method": "GET", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 400 + }, + { + "path": "/logstream/smoke", + "method": "DELETE", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "PUT", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "DELETE", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "PUT", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "DELETE", + "status": 503 + }, + { + "path": "/liveness", + "method": "GET", + "status": 200 + }, + { + "path": "/logstream", + "method": "GET", + "status": 200 + }, + { + "path": "/query", + "method": "POST", + "status": 400 + }, + { + "path": "/logstream/smoke", + "method": "DELETE", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "PUT", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "DELETE", + "status": 200 + }, + { + "path": "/logstream/smoke", + "method": "PUT", + "status": 200 + } + ] + }, + { + "requested_modes": [ + "object-store", + "postgres-tls", + "postgres" + ], + "source_commit": "b96259b05238bae99ca48d5bee3cee9943b399d5", + "binary_sha256": "b30aa28e2dac81b076dbe41a11bc8d686c3b9ea3c6490d3471682a507def0d39", + "source_diff_sha256": "03b1e883545353e632d87207d72bb644a88731f4761ca4494fa9c40e47277cec", + "image_digests": { + "postgres:17-alpine": [ + "postgres@sha256:b0f9560a2de083e2cc7382e75f808c7381a32852a7ec49117deedb300e552b24" + ], + "quay.io/parseablehq/minio:edge": [ + "quay.io/parseablehq/minio@sha256:88276d0a028d758ffaf80d65dd80071386e91b4114251a3dda101eadf000faeb" + ] + }, + "measurement": "Node-local storage counter delta during five queries, including background traffic; not isolated requests per query.", + "object-store": { + "lazy_reload": true, + "query": "[{\"count\":1}]", + "dashboard": { + "version": "v1", + "title": "Postgres smoke", + "author": "8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918", + "dashboardId": "01M75JNVK6AYE1WXADG6JA3DQZ", + "created": "2026-11-11T15:12:30.822910300+00:00", + "modified": "2026-11-11T15:12:30.822910300+00:00", + "tags": null, + "isFavorite": false, + "dashboardType": "Dashboard", + "tiles": [], + "tenantId": null, + "revision": "2e2d1ea944667dfde6dab5d44c0d29b0768305f9059cbed7133c0406f4bea6c4" + }, + "keys": [ + ".parseable/.parseable.json", + ".parseable/ingestor.a878d4a37bf1656.json", + ".parseable/querier.c1530d7252e7658.json", + ".users/.dashboard-locations/01M75JNVK6AYE1WXADG6JA3DQZ.json", + ".users/.dashboard-titles/571a2cefe2d0d1a9a26dab4810a5556875c7d7531d4f558a8f7b28ab486e2395.json", + ".users/8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918/dashboards/01M75JNVK6AYE1WXADG6JA3DQZ.json", + "lazy/.stream/.ingestor.a878d4a37bf1656.schema", + "lazy/.stream/.ingestor.a878d4a37bf1656.stream.json", + "lazy/.stream/.schema", + "lazy/.stream/.stream.json", + "lazy/date=2026-11-11/hour=15/minute=13/a878d4a37bf1656.data.01M75JRJYKT089B5QZ24C9WCDG.parquet", + "lazy/date=2026-11-11/ingestor..a878d4a37bf1656.manifest.json", + "pstats/.stream/.ingestor.a878d4a37bf1656.schema", + "pstats/.stream/.ingestor.a878d4a37bf1656.stream.json", + "pstats/.stream/.schema", + "pstats/.stream/.stream.json", + "pstats/date=2026-11-11/.manifest.json", + "pstats/date=2026-11-11/hour=15/minute=11/dataset_name=smoke/.data.01M75JPRNSZ6BDJG6PFGB2QWT2.parquet", + "smoke/.stream/.ingestor.a878d4a37bf1656.schema", + "smoke/.stream/.ingestor.a878d4a37bf1656.stream.json", + "smoke/.stream/.schema", + "smoke/.stream/.stream.json", + "smoke/date=2026-11-11/.manifest.json", + "smoke/date=2026-11-11/hour=15/minute=11/.data.01M75JMY2SDDNVZR0S50MDPK9Y.parquet" + ], + "non_parquet_keys": [ + ".parseable/.parseable.json", + ".parseable/ingestor.a878d4a37bf1656.json", + ".parseable/querier.c1530d7252e7658.json", + ".users/.dashboard-locations/01M75JNVK6AYE1WXADG6JA3DQZ.json", + ".users/.dashboard-titles/571a2cefe2d0d1a9a26dab4810a5556875c7d7531d4f558a8f7b28ab486e2395.json", + ".users/8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918/dashboards/01M75JNVK6AYE1WXADG6JA3DQZ.json", + "lazy/.stream/.ingestor.a878d4a37bf1656.schema", + "lazy/.stream/.ingestor.a878d4a37bf1656.stream.json", + "lazy/.stream/.schema", + "lazy/.stream/.stream.json", + "lazy/date=2026-11-11/ingestor..a878d4a37bf1656.manifest.json", + "pstats/.stream/.ingestor.a878d4a37bf1656.schema", + "pstats/.stream/.ingestor.a878d4a37bf1656.stream.json", + "pstats/.stream/.schema", + "pstats/.stream/.stream.json", + "pstats/date=2026-11-11/.manifest.json", + "smoke/.stream/.ingestor.a878d4a37bf1656.schema", + "smoke/.stream/.ingestor.a878d4a37bf1656.stream.json", + "smoke/.stream/.schema", + "smoke/.stream/.stream.json", + "smoke/date=2026-11-11/.manifest.json" + ], + "request_delta": { + "parseable_storage_request_response_time_count{method=\"GET\",provider=\"s3\",status=\"200\"}": 30.0, + "parseable_storage_request_response_time_count{method=\"GET_RANGES\",provider=\"s3\",status=\"200\"}": 5.0, + "parseable_storage_request_response_time_count{method=\"LIST\",provider=\"s3\",status=\"200\"}": 5.0, + "parseable_storage_request_response_time_count{method=\"LIST_DELIM\",provider=\"s3\",status=\"200\"}": 5.0 + }, + "observed_requests_during_window_divided_by_queries": 9.0, + "bucket_outage_readiness": 503, + "delete_recreate": true, + "final_keys": [ + ".parseable/.parseable.json", + ".parseable/ingestor.a878d4a37bf1656.json", + ".parseable/querier.c1530d7252e7658.json", + ".users/.dashboard-locations/01M75JNVK6AYE1WXADG6JA3DQZ.json", + ".users/.dashboard-titles/571a2cefe2d0d1a9a26dab4810a5556875c7d7531d4f558a8f7b28ab486e2395.json", + ".users/8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918/dashboards/01M75JNVK6AYE1WXADG6JA3DQZ.json", + "lazy/.stream/.ingestor.a878d4a37bf1656.schema", + "lazy/.stream/.ingestor.a878d4a37bf1656.stream.json", + "lazy/.stream/.schema", + "lazy/.stream/.stream.json", + "lazy/date=2026-11-11/hour=15/minute=13/a878d4a37bf1656.data.01M75JRJYKT089B5QZ24C9WCDG.parquet", + "lazy/date=2026-11-11/ingestor..a878d4a37bf1656.manifest.json", + "pstats/.stream/.ingestor.a878d4a37bf1656.schema", + "pstats/.stream/.ingestor.a878d4a37bf1656.stream.json", + "pstats/.stream/.schema", + "pstats/.stream/.stream.json", + "pstats/date=2026-11-11/.manifest.json", + "pstats/date=2026-11-11/hour=15/minute=11/dataset_name=smoke/.data.01M75JPRNSZ6BDJG6PFGB2QWT2.parquet", + "pstats/date=2026-11-11/hour=15/minute=13/dataset_name=lazy/a878d4a37bf1656.data.01M75JTDHJA1V5QTMHR68NFYFX.parquet", + "pstats/date=2026-11-11/ingestor..a878d4a37bf1656.manifest.json", + "smoke/.stream/.ingestor.a878d4a37bf1656.schema", + "smoke/.stream/.ingestor.a878d4a37bf1656.stream.json", + "smoke/.stream/.schema", + "smoke/.stream/.stream.json", + "smoke/date=2026-11-11/hour=15/minute=14/a878d4a37bf1656.data.01M75JTDHJCXR3RTGPN9ARG161.parquet", + "smoke/date=2026-11-11/ingestor..a878d4a37bf1656.manifest.json" + ] + }, + "postgres-tls": { + "trusted_ca_restart": true, + "untrusted_ca_rejected": true, + "untrusted_ca_output": "Error: ObjectStorageError: PostgreSQL metastore unavailable: Error occurred while creating a new object: error performing TLS handshake\n\nCaused by:\n PostgreSQL metastore unavailable: Error occurred while creating a new object: error performing TLS handshake\n", + "untrusted_ca_server_alert": "2026-11-11 15:15:48.818 UTC [131] LOG: could not accept SSL connection: tlsv1 alert unknown ca\n", + "lazy_reload": true, + "query": "[{\"count\":1}]", + "dashboard": { + "version": "v1", + "title": "Postgres smoke", + "author": "8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918", + "dashboardId": "01M75JX776RMF67VDZC03QBV80", + "created": "2026-11-11T15:16:32.102431645+00:00", + "modified": "2026-11-11T15:16:32.102431645+00:00", + "tags": null, + "isFavorite": false, + "dashboardType": "Dashboard", + "tiles": [], + "tenantId": null, + "revision": "f4a0456e421a611f22c150d81cf0b2350c36772618d218293ffe1d12c8c6f5ec" + }, + "keys": [ + ".parseable/postgres-metastore-id", + "lazy/date=2026-11-11/hour=15/minute=18/8ed27ce105d0786.data.01M75K1RM89ZBQZG7NXGCMKY59.parquet", + "pstats/date=2026-11-11/hour=15/minute=15/dataset_name=smoke/.data.01M75JY3CJR1NXRYQPC06KFWWT.parquet", + "smoke/date=2026-11-11/hour=15/minute=15/.data.01M75JW8SK0YDV8R9GHVTGQPAZ.parquet" + ], + "non_parquet_keys": [ + ".parseable/postgres-metastore-id" + ], + "request_delta": { + "parseable_storage_request_response_time_count{method=\"GET_RANGES\",provider=\"s3\",status=\"200\"}": 5.0 + }, + "observed_requests_during_window_divided_by_queries": 1.0, + "database_outage": { + "readiness": 503, + "request_failed": true, + "seconds": 0.006427645916119218 + }, + "bucket_outage_readiness": 503, + "deleted_stream_rows": 0, + "delete_recreate": true, + "final_keys": [ + ".parseable/postgres-metastore-id", + "lazy/date=2026-11-11/hour=15/minute=18/8ed27ce105d0786.data.01M75K1RM89ZBQZG7NXGCMKY59.parquet", + "pstats/date=2026-11-11/hour=15/minute=15/dataset_name=smoke/.data.01M75JY3CJR1NXRYQPC06KFWWT.parquet", + "pstats/date=2026-11-11/hour=15/minute=18/dataset_name=lazy/8ed27ce105d0786.data.01M75K3K788KV2CVEGMCSFSZF5.parquet", + "smoke/date=2026-11-11/hour=15/minute=19/8ed27ce105d0786.data.01M75K3K78967A00SJ62H7ZNT4.parquet" + ], + "interrupted_deletion_restart": [ + { + "step": "before-data", + "hidden": true, + "cold_load_rejected": true, + "http_retry": true, + "recreation": true + }, + { + "step": "after-data", + "hidden": true, + "cold_load_rejected": true, + "http_retry": true, + "recreation": true + } + ] + }, + "postgres": { + "lazy_reload": true, + "query": "[{\"count\":1}]", + "dashboard": { + "version": "v1", + "title": "Postgres smoke", + "author": "8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918", + "dashboardId": "01M75KA0K4HTDY1R3F6TS6F4XM", + "created": "2026-11-11T15:23:31.300938938+00:00", + "modified": "2026-11-11T15:23:31.300938938+00:00", + "tags": null, + "isFavorite": false, + "dashboardType": "Dashboard", + "tiles": [], + "tenantId": null, + "revision": "d3245c653cbb43fc182dcb0fcd1f52857cc95f9557bd80ee994c5504fdfb5bb9" + }, + "keys": [ + ".parseable/postgres-metastore-id", + "lazy/date=2026-11-11/hour=15/minute=24/da8518511ee84c0.data.01M75KCQV2HKAMSQCGK79BRZTR.parquet", + "pstats/date=2026-11-11/hour=15/minute=22/dataset_name=smoke/.data.01M75KAX7FTN48CQVS8XXY0M31.parquet", + "smoke/date=2026-11-11/hour=15/minute=22/.data.01M75K92MFMGVRDR8V49HY99V6.parquet" + ], + "non_parquet_keys": [ + ".parseable/postgres-metastore-id" + ], + "request_delta": { + "parseable_storage_request_response_time_count{method=\"GET_RANGES\",provider=\"s3\",status=\"200\"}": 5.0 + }, + "observed_requests_during_window_divided_by_queries": 1.0, + "database_outage": { + "readiness": 503, + "request_failed": true, + "seconds": 0.006476623937487602 + }, + "bucket_outage_readiness": 503, + "deleted_stream_rows": 0, + "delete_recreate": true, + "final_keys": [ + ".parseable/postgres-metastore-id", + "lazy/date=2026-11-11/hour=15/minute=24/da8518511ee84c0.data.01M75KCQV2HKAMSQCGK79BRZTR.parquet", + "pstats/date=2026-11-11/hour=15/minute=22/dataset_name=smoke/.data.01M75KAX7FTN48CQVS8XXY0M31.parquet", + "pstats/date=2026-11-11/hour=15/minute=24/dataset_name=lazy/da8518511ee84c0.data.01M75KEMZ0DJ8K16GNDD13EVJ1.parquet", + "smoke/date=2026-11-11/hour=15/minute=27/da8518511ee84c0.data.01M75KJ7M27Q4HM890XEA3QRMX.parquet" + ], + "interrupted_deletion_restart": [ + { + "step": "before-data", + "hidden": true, + "cold_load_rejected": true, + "http_retry": true, + "recreation": true + }, + { + "step": "after-data", + "hidden": true, + "cold_load_rejected": true, + "http_retry": true, + "recreation": true + } + ] + }, + "startup_rejections": { + "future-schema": "Error: ObjectStorageError: PostgreSQL metastore unavailable: unsupported schema/protocol version; upgrade this server\n\nCaused by:\n PostgreSQL metastore unavailable: unsupported schema/protocol version; upgrade this server\n", + "wrong-id": "Error: ObjectStorageError: PostgreSQL metastore unavailable: backend/bucket identity mismatch\n\nCaused by:\n PostgreSQL metastore unavailable: backend/bucket identity mismatch\n", + "unreachable": "Error: ObjectStorageError: PostgreSQL metastore unavailable: Error occurred while creating a new object: error connecting to server\n\nCaused by:\n PostgreSQL metastore unavailable: Error occurred while creating a new object: error connecting to server\n" + }, + "postgres_rows": "tenant | kind | key | version | bytes \n--------+----------+--------------------------------------------------------------------------------------------------------------------+---------+-------\n | document | .parseable/.parseable.json | 121 | 383\n | document | .parseable/ingestor.da8518511ee84c0.json | 39 | 241\n | document | .parseable/querier.d960519648dca85.json | 123 | 240\n | document | .users/.dashboard-locations/01M75KA0K4HTDY1R3F6TS6F4XM.json | 16 | 119\n | document | .users/.dashboard-titles/571a2cefe2d0d1a9a26dab4810a5556875c7d7531d4f558a8f7b28ab486e2395.json | 18 | 170\n | document | .users/8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918/dashboards/01M75KA0K4HTDY1R3F6TS6F4XM.json | 17 | 545\n | schema | lazy/.stream/.ingestor.da8518511ee84c0.schema | 56 | 585\n | stream | lazy/.stream/.ingestor.da8518511ee84c0.stream.json | 54 | 853\n | schema | lazy/.stream/.schema | 131 | 585\n | stream | lazy/.stream/.stream.json | 57 | 597\n | manifest | lazy/date=2026-11-11/ingestor..da8518511ee84c0.manifest.json | 52 | 418\n | schema | pstats/.stream/.ingestor.da8518511ee84c0.schema | 148 | 1117\n | stream | pstats/.stream/.ingestor.da8518511ee84c0.stream.json | 146 | 898\n | schema | pstats/.stream/.schema | 127 | 1117\n | stream | pstats/.stream/.stream.json | 23 | 867\n | manifest | pstats/date=2026-11-11/.manifest.json | 21 | 563\n | manifest | pstats/date=2026-11-11/ingestor..da8518511ee84c0.manifest.json | 144 | 790\n | schema | smoke/.stream/.ingestor.da8518511ee84c0.schema | 141 | 27\n | stream | smoke/.stream/.ingestor.da8518511ee84c0.stream.json | 142 | 597\n | schema | smoke/.stream/.schema | 139 | 27\n | stream | smoke/.stream/.stream.json | 140 | 597\n(21 rows)", + "commands": [ + { + "argv": [ + "docker", + "run", + "-d", + "--name", + "parseable-pg-smoke-804785-0", + "-e", + "POSTGRES_PASSWORD=smoke", + "-e", + "POSTGRES_DB=parseable", + "-p", + "127.0.0.1:49281:5432", + "postgres:17-alpine" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "run", + "-d", + "--name", + "parseable-pg-smoke-804785-1", + "-e", + "MINIO_ROOT_USER=minioadmin", + "-e", + "MINIO_ROOT_PASSWORD=minioadmin", + "-p", + "127.0.0.1:38415:9000", + "quay.io/parseablehq/minio:edge", + "server", + "/data" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "pg_isready", + "-h", + "127.0.0.1", + "-U", + "postgres" + ], + "exit_code": 2 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "pg_isready", + "-h", + "127.0.0.1", + "-U", + "postgres" + ], + "exit_code": 2 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "pg_isready", + "-h", + "127.0.0.1", + "-U", + "postgres" + ], + "exit_code": 2 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "pg_isready", + "-h", + "127.0.0.1", + "-U", + "postgres" + ], + "exit_code": 0 + }, + { + "argv": [ + "openssl", + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-keyout", + ".postgres-smoke/tls/ca.key", + "-out", + ".postgres-smoke/tls/ca.crt", + "-days", + "2", + "-subj", + "/CN=Smoke CA" + ], + "exit_code": 0 + }, + { + "argv": [ + "openssl", + "req", + "-newkey", + "rsa:2048", + "-nodes", + "-keyout", + ".postgres-smoke/tls/server.key", + "-out", + ".postgres-smoke/tls/server.csr", + "-subj", + "/CN=localhost" + ], + "exit_code": 0 + }, + { + "argv": [ + "openssl", + "x509", + "-req", + "-in", + ".postgres-smoke/tls/server.csr", + "-CA", + ".postgres-smoke/tls/ca.crt", + "-CAkey", + ".postgres-smoke/tls/ca.key", + "-CAcreateserial", + "-out", + ".postgres-smoke/tls/server.crt", + "-days", + "2", + "-extfile", + ".postgres-smoke/tls/extensions" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "cp", + ".postgres-smoke/tls/server.key", + "parseable-pg-smoke-804785-0:/var/lib/postgresql/data/server.key" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "cp", + ".postgres-smoke/tls/server.crt", + "parseable-pg-smoke-804785-0:/var/lib/postgresql/data/server.crt" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "sh", + "-c", + "chown postgres:postgres /var/lib/postgresql/data/server.* && chmod 600 /var/lib/postgresql/data/server.key" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-c", + "ALTER SYSTEM SET ssl='on';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "parseable", + "-c", + "SELECT pg_reload_conf();" + ], + "exit_code": 0 + }, + { + "argv": [ + "git", + "rev-parse", + "HEAD" + ], + "exit_code": 0 + }, + { + "argv": [ + "git", + "diff", + "HEAD" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "image", + "inspect", + "postgres:17-alpine" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "image", + "inspect", + "quay.io/parseablehq/minio:edge" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-1", + "mkdir", + "-p", + "/data/smoke-object-store" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "stop", + "parseable-pg-smoke-804785-1" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "start", + "parseable-pg-smoke-804785-1" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-1", + "mkdir", + "-p", + "/data/smoke-postgres-tls" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-c", + "CREATE DATABASE tls;" + ], + "exit_code": 0 + }, + { + "argv": [ + "openssl", + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-keyout", + ".postgres-smoke/tls/wrong.key", + "-out", + ".postgres-smoke/tls/wrong.crt", + "-days", + "2", + "-subj", + "/CN=Wrong CA" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "logs", + "parseable-pg-smoke-804785-0" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "logs", + "parseable-pg-smoke-804785-0" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" ], "exit_code": 0 }, { "argv": [ "docker", - "cp", - "/.postgres-smoke/tls/server.key", - "parseable-pg-smoke-3714167-0:/var/lib/postgresql/data/server.key" + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" ], "exit_code": 0 }, { "argv": [ "docker", - "cp", - "/.postgres-smoke/tls/server.crt", - "parseable-pg-smoke-3714167-0:/var/lib/postgresql/data/server.crt" + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" ], "exit_code": 0 }, @@ -410,10 +5066,14 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", - "sh", - "-c", - "chown postgres:postgres /var/lib/postgresql/data/server.* && chmod 600 /var/lib/postgresql/data/server.key" + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" ], "exit_code": 0 }, @@ -421,14 +5081,14 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", "-d", - "parseable", - "-c", - "ALTER SYSTEM SET ssl='on';" + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" ], "exit_code": 0 }, @@ -436,48 +5096,74 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", "-d", - "parseable", - "-c", - "SELECT pg_reload_conf();" + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" ], "exit_code": 0 }, { "argv": [ - "git", - "rev-parse", - "HEAD" + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" ], "exit_code": 0 }, { "argv": [ - "git", - "diff", - "HEAD" + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" ], "exit_code": 0 }, { "argv": [ "docker", - "image", - "inspect", - "postgres:17-alpine" + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" ], "exit_code": 0 }, { "argv": [ "docker", - "image", - "inspect", - "quay.io/parseablehq/minio:edge" + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" ], "exit_code": 0 }, @@ -485,26 +5171,44 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-1", - "mkdir", - "-p", - "/data/smoke-object-store" + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" ], "exit_code": 0 }, { "argv": [ "docker", - "stop", - "parseable-pg-smoke-3714167-1" + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" ], "exit_code": 0 }, { "argv": [ "docker", - "start", - "parseable-pg-smoke-3714167-1" + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" ], "exit_code": 0 }, @@ -512,10 +5216,14 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-1", - "mkdir", - "-p", - "/data/smoke-postgres-tls" + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" ], "exit_code": 0 }, @@ -523,69 +5231,89 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", - "-c", - "CREATE DATABASE tls;" + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" ], "exit_code": 0 }, { "argv": [ - "openssl", - "req", - "-x509", - "-newkey", - "rsa:2048", - "-nodes", - "-keyout", - "/.postgres-smoke/tls/wrong.key", - "-out", - "/.postgres-smoke/tls/wrong.crt", - "-days", - "2", - "-subj", - "/CN=Wrong CA" + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" ], "exit_code": 0 }, { "argv": [ "docker", - "logs", - "parseable-pg-smoke-3714167-0" + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" ], "exit_code": 0 }, { "argv": [ "docker", - "logs", - "parseable-pg-smoke-3714167-0" + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" ], "exit_code": 0 }, { "argv": [ - "/target/debug/parseable", - "s3-store" + "docker", + "exec", + "parseable-pg-smoke-804785-0", + "psql", + "-U", + "postgres", + "-d", + "tls", + "-Atc", + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" ], - "exit_code": 1 + "exit_code": 0 }, { "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", "-d", "tls", "-Atc", - "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'smoke/.stream/') AND right(key,11)='stream.json';" + "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" ], "exit_code": 0 }, @@ -593,7 +5321,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -608,7 +5336,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -623,7 +5351,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -638,7 +5366,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -653,7 +5381,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -668,7 +5396,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -683,7 +5411,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -698,7 +5426,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -713,7 +5441,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -728,7 +5456,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -743,7 +5471,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -758,7 +5486,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -773,7 +5501,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -788,7 +5516,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -803,7 +5531,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -818,7 +5546,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -833,7 +5561,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -848,7 +5576,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -863,7 +5591,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -878,7 +5606,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -893,7 +5621,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -908,7 +5636,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -923,7 +5651,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -938,7 +5666,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -953,7 +5681,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -968,7 +5696,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -983,7 +5711,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -998,7 +5726,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -1013,7 +5741,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -1028,7 +5756,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -1043,7 +5771,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -1058,7 +5786,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -1073,7 +5801,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -1088,7 +5816,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -1103,7 +5831,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -1118,7 +5846,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -1133,7 +5861,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -1148,7 +5876,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -1163,7 +5891,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -1178,7 +5906,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -1193,7 +5921,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -1208,7 +5936,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -1223,7 +5951,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -1238,7 +5966,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -1253,7 +5981,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -1268,7 +5996,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -1283,7 +6011,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -1298,7 +6026,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -1313,7 +6041,7 @@ "argv": [ "docker", "stop", - "parseable-pg-smoke-3714167-0" + "parseable-pg-smoke-804785-0" ], "exit_code": 0 }, @@ -1321,7 +6049,7 @@ "argv": [ "docker", "start", - "parseable-pg-smoke-3714167-0" + "parseable-pg-smoke-804785-0" ], "exit_code": 0 }, @@ -1329,7 +6057,7 @@ "argv": [ "docker", "stop", - "parseable-pg-smoke-3714167-1" + "parseable-pg-smoke-804785-1" ], "exit_code": 0 }, @@ -1337,7 +6065,7 @@ "argv": [ "docker", "start", - "parseable-pg-smoke-3714167-1" + "parseable-pg-smoke-804785-1" ], "exit_code": 0 }, @@ -1345,7 +6073,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -1360,7 +6088,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -1375,7 +6103,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -1390,7 +6118,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-v", "ON_ERROR_STOP=1", @@ -1407,7 +6135,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-v", "ON_ERROR_STOP=1", @@ -1424,7 +6152,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -1439,7 +6167,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-v", "ON_ERROR_STOP=1", @@ -1456,7 +6184,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-v", "ON_ERROR_STOP=1", @@ -1473,7 +6201,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-v", "ON_ERROR_STOP=1", @@ -1490,7 +6218,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-1", + "parseable-pg-smoke-804785-1", "mkdir", "-p", "/data/smoke-postgres" @@ -1501,7 +6229,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -1516,37 +6244,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", - "psql", - "-U", - "postgres", - "-d", - "parseable", - "-Atc", - "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" - ], - "exit_code": 0 - }, - { - "argv": [ - "docker", - "exec", - "parseable-pg-smoke-3714167-0", - "psql", - "-U", - "postgres", - "-d", - "parseable", - "-Atc", - "SELECT convert_from(body,'UTF8') FROM parseable_documents WHERE tenant='' AND starts_with(key,'lazy/.stream/') AND right(key,11)='stream.json';" - ], - "exit_code": 0 - }, - { - "argv": [ - "docker", - "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -1561,7 +6259,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -1576,7 +6274,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -1591,7 +6289,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -1606,7 +6304,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -1621,7 +6319,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -1636,7 +6334,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -1651,7 +6349,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -1666,7 +6364,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -1681,7 +6379,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -1696,7 +6394,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -1711,7 +6409,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -1726,7 +6424,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -1741,7 +6439,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -1756,7 +6454,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -1771,7 +6469,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -1786,7 +6484,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -1801,7 +6499,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -1816,7 +6514,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -1831,7 +6529,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -1846,7 +6544,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -1861,7 +6559,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -1876,7 +6574,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -1891,7 +6589,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -1906,7 +6604,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -1921,7 +6619,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -1936,7 +6634,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -1951,7 +6649,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -1966,7 +6664,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -1981,7 +6679,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -1996,7 +6694,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -2011,7 +6709,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -2026,7 +6724,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -2041,7 +6739,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -2056,7 +6754,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -2071,7 +6769,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -2086,7 +6784,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -2101,7 +6799,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -2116,7 +6814,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -2131,7 +6829,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -2146,7 +6844,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -2161,7 +6859,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -2176,7 +6874,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -2191,7 +6889,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -2206,7 +6904,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -2221,7 +6919,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -2236,7 +6934,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -2251,7 +6949,7 @@ "argv": [ "docker", "stop", - "parseable-pg-smoke-3714167-0" + "parseable-pg-smoke-804785-0" ], "exit_code": 0 }, @@ -2259,7 +6957,7 @@ "argv": [ "docker", "start", - "parseable-pg-smoke-3714167-0" + "parseable-pg-smoke-804785-0" ], "exit_code": 0 }, @@ -2267,7 +6965,7 @@ "argv": [ "docker", "stop", - "parseable-pg-smoke-3714167-1" + "parseable-pg-smoke-804785-1" ], "exit_code": 0 }, @@ -2275,7 +6973,7 @@ "argv": [ "docker", "start", - "parseable-pg-smoke-3714167-1" + "parseable-pg-smoke-804785-1" ], "exit_code": 0 }, @@ -2283,7 +6981,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -2298,7 +6996,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -2313,7 +7011,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -2328,7 +7026,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-v", "ON_ERROR_STOP=1", @@ -2345,7 +7043,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-v", "ON_ERROR_STOP=1", @@ -2362,7 +7060,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -2377,7 +7075,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-v", "ON_ERROR_STOP=1", @@ -2394,7 +7092,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-v", "ON_ERROR_STOP=1", @@ -2411,7 +7109,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-v", "ON_ERROR_STOP=1", @@ -2428,20 +7126,20 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", "-d", "parseable", "-c", - "UPDATE parseable_backend SET schema_version=3;" + "UPDATE parseable_backend SET schema_version=999;" ], "exit_code": 0 }, { "argv": [ - "/target/debug/parseable", + "target/debug/parseable", "s3-store" ], "exit_code": 1 @@ -2450,27 +7148,27 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", "-d", "parseable", "-c", - "UPDATE parseable_backend SET schema_version=2;" + "UPDATE parseable_backend SET schema_version=3;" ], "exit_code": 0 }, { "argv": [ - "/target/debug/parseable", + "target/debug/parseable", "s3-store" ], "exit_code": 1 }, { "argv": [ - "/target/debug/parseable", + "target/debug/parseable", "s3-store" ], "exit_code": 1 @@ -2479,7 +7177,7 @@ "argv": [ "docker", "exec", - "parseable-pg-smoke-3714167-0", + "parseable-pg-smoke-804785-0", "psql", "-U", "postgres", @@ -2495,7 +7193,7 @@ "docker", "rm", "-f", - "parseable-pg-smoke-3714167-1" + "parseable-pg-smoke-804785-1" ], "exit_code": 0 }, @@ -2504,252 +7202,138 @@ "docker", "rm", "-f", - "parseable-pg-smoke-3714167-0" + "parseable-pg-smoke-804785-0" ], "exit_code": 0 }, { "argv": [ - "/target/debug/parseable", + "target/debug/parseable", "s3-store" ], "exit_code": 0 }, { "argv": [ - "/target/debug/parseable", + "target/debug/parseable", "s3-store" ], "exit_code": 0 }, { "argv": [ - "/target/debug/parseable", + "target/debug/parseable", "s3-store" ], "exit_code": 0 }, { "argv": [ - "/target/debug/parseable", + "target/debug/parseable", "s3-store" ], "exit_code": 0 }, { "argv": [ - "/target/debug/parseable", + "target/debug/parseable", "s3-store" ], "exit_code": 1 }, { "argv": [ - "/target/debug/parseable", + "target/debug/parseable", "s3-store" ], "exit_code": 0 }, { "argv": [ - "/target/debug/parseable", + "target/debug/parseable", "s3-store" ], "exit_code": 0 }, { "argv": [ - "/target/debug/parseable", + "target/debug/parseable", "s3-store" ], "exit_code": 0 }, { "argv": [ - "/target/debug/parseable", + "target/debug/parseable", "s3-store" ], "exit_code": 0 }, { "argv": [ - "/target/debug/parseable", + "target/debug/parseable", "s3-store" ], "exit_code": 0 }, { "argv": [ - "/target/debug/parseable", + "target/debug/parseable", "s3-store" ], "exit_code": 0 }, { "argv": [ - "/target/debug/parseable", + "target/debug/parseable", "s3-store" ], "exit_code": 0 }, { "argv": [ - "/target/debug/parseable", + "target/debug/parseable", "s3-store" ], "exit_code": 0 }, { "argv": [ - "/target/debug/parseable", + "target/debug/parseable", "s3-store" ], "exit_code": 0 }, { "argv": [ - "/target/debug/parseable", + "target/debug/parseable", "s3-store" ], "exit_code": 0 }, { "argv": [ - "/target/debug/parseable", + "target/debug/parseable", "s3-store" ], "exit_code": 1 }, { "argv": [ - "/target/debug/parseable", + "target/debug/parseable", "s3-store" ], "exit_code": 1 }, { "argv": [ - "/target/debug/parseable", + "target/debug/parseable", "s3-store" ], "exit_code": 1 } ], "success": true, - "validation": { - "commands": [ - { - "name": "library-without-url", - "argv": [ - "cargo", - "test", - "--lib", - "--", - "--nocapture" - ], - "exit_code": 0, - "log": "/.postgres-smoke/library-without-url.log", - "summaries": [ - "test result: ok. 1398 passed; 0 failed; 7 ignored; 0 measured; 0 filtered out; finished in 75.00s" - ] - }, - { - "name": "library-with-url", - "argv": [ - "cargo", - "test", - "--lib", - "--", - "--test-threads=1", - "--nocapture" - ], - "exit_code": 0, - "log": "/.postgres-smoke/library-with-url.log", - "summaries": [ - "test result: ok. 1398 passed; 0 failed; 7 ignored; 0 measured; 0 filtered out; finished in 137.63s" - ] - }, - { - "name": "contracts", - "argv": [ - "cargo", - "test", - "--lib", - "contract_tests", - "--", - "--nocapture", - "--test-threads=1" - ], - "exit_code": 0, - "log": "/.postgres-smoke/contracts.log", - "summaries": [ - "test result: ok. 10 passed; 0 failed; 3 ignored; 0 measured; 1392 filtered out; finished in 1.77s" - ] - }, - { - "name": "postgres-contract", - "argv": [ - "cargo", - "test", - "--lib", - "contract_tests::postgres_contract", - "--", - "--ignored", - "--nocapture" - ], - "exit_code": 0, - "log": "/.postgres-smoke/postgres-contract.log", - "summaries": [ - "test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 1404 filtered out; finished in 5.51s" - ] - }, - { - "name": "startup-cleanup", - "argv": [ - "cargo", - "test", - "--lib", - "contract_tests::postgres_startup_cleanup", - "--", - "--ignored", - "--nocapture" - ], - "exit_code": 0, - "log": "/.postgres-smoke/startup-cleanup.log", - "summaries": [ - "test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 1404 filtered out; finished in 0.75s" - ] - }, - { - "name": "paused-health", - "argv": [ - "cargo", - "test", - "--lib", - "postgres_health_deadline_regression", - "--", - "--nocapture" - ], - "exit_code": 0, - "log": "/.postgres-smoke/paused-health.log", - "summaries": [ - "test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 1404 filtered out; finished in 0.94s" - ] - }, - { - "name": "clippy", - "argv": [ - "cargo", - "clippy", - "--all-targets" - ], - "exit_code": 0, - "log": "/.postgres-smoke/clippy.log", - "summaries": [] - } - ], - "success": true - }, "http_requests": [ { "path": "/liveness", @@ -3186,6 +7770,11 @@ "method": "GET", "status": 503 }, + { + "path": "/readiness", + "method": "GET", + "status": 503 + }, { "path": "/readiness", "method": "GET", diff --git a/docs/validation/postgres-migration-tls.json b/docs/validation/postgres-migration-tls.json new file mode 100644 index 000000000..f203d7a1e --- /dev/null +++ b/docs/validation/postgres-migration-tls.json @@ -0,0 +1,1062 @@ +{ + "success": true, + "tls": true, + "commands": [ + { + "argv": [ + "docker", + "run", + "-d", + "--name", + "parseable-pg-smoke-1476557-0", + "-e", + "POSTGRES_PASSWORD=smoke", + "-e", + "POSTGRES_DB=migration", + "-p", + "127.0.0.1:34747:5432", + "postgres:17-alpine" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "run", + "-d", + "--name", + "parseable-pg-smoke-1476557-1", + "-e", + "MINIO_ROOT_USER=minioadmin", + "-e", + "MINIO_ROOT_PASSWORD=minioadmin", + "-p", + "127.0.0.1:40655:9000", + "quay.io/parseablehq/minio:edge", + "server", + "/data" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1476557-1", + "mkdir", + "-p", + "/data/migration" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1476557-0", + "pg_isready", + "-h", + "127.0.0.1", + "-U", + "postgres" + ], + "exit_code": 2 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1476557-0", + "pg_isready", + "-h", + "127.0.0.1", + "-U", + "postgres" + ], + "exit_code": 0 + }, + { + "argv": [ + "openssl", + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-keyout", + ".postgres-smoke/migration-tls/ca.key", + "-out", + ".postgres-smoke/migration-tls/ca.crt", + "-days", + "2", + "-subj", + "/CN=Smoke CA" + ], + "exit_code": 0 + }, + { + "argv": [ + "openssl", + "req", + "-newkey", + "rsa:2048", + "-nodes", + "-keyout", + ".postgres-smoke/migration-tls/server.key", + "-out", + ".postgres-smoke/migration-tls/server.csr", + "-subj", + "/CN=localhost" + ], + "exit_code": 0 + }, + { + "argv": [ + "openssl", + "x509", + "-req", + "-in", + ".postgres-smoke/migration-tls/server.csr", + "-CA", + ".postgres-smoke/migration-tls/ca.crt", + "-CAkey", + ".postgres-smoke/migration-tls/ca.key", + "-CAcreateserial", + "-out", + ".postgres-smoke/migration-tls/server.crt", + "-days", + "2", + "-extfile", + ".postgres-smoke/migration-tls/extensions" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "cp", + ".postgres-smoke/migration-tls/server.key", + "parseable-pg-smoke-1476557-0:/var/lib/postgresql/data/server.key" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "cp", + ".postgres-smoke/migration-tls/server.crt", + "parseable-pg-smoke-1476557-0:/var/lib/postgresql/data/server.crt" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1476557-0", + "sh", + "-c", + "chown postgres:postgres /var/lib/postgresql/data/server.* && chmod 600 /var/lib/postgresql/data/server.key" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1476557-0", + "psql", + "-U", + "postgres", + "-d", + "migration", + "-c", + "ALTER SYSTEM SET ssl='on';" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1476557-0", + "psql", + "-U", + "postgres", + "-d", + "migration", + "-c", + "SELECT pg_reload_conf();" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store", + "--metadata-migration", + "import", + "--acknowledge-server-stopped", + "--dry-run" + ], + "exit_code": 1, + "expected_success": false + }, + { + "argv": [ + "target/debug/parseable", + "s3-store", + "--metadata-migration", + "verify", + "--acknowledge-server-stopped" + ], + "exit_code": 1, + "expected_success": false + }, + { + "argv": [ + "target/debug/parseable", + "s3-store", + "--metadata-migration", + "import", + "--dry-run", + "--acknowledge-server-stopped" + ], + "cwd": ".postgres-smoke/migration-env-fixture", + "exit_code": 0, + "env_file_checked": true + }, + { + "argv": [ + "target/debug/parseable", + "s3-store", + "--metadata-migration", + "import", + "--acknowledge-server-stopped", + "--dry-run" + ], + "exit_code": 0, + "expected_success": true + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1, + "refusal_state": "import-before-commit", + "postgres": false + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1, + "refusal_state": "import-before-commit", + "postgres": true + }, + { + "argv": [ + "target/debug/parseable", + "s3-store", + "--metadata-migration", + "import", + "--acknowledge-server-stopped" + ], + "exit_code": 0, + "expected_success": true + }, + { + "argv": [ + "target/debug/parseable", + "s3-store", + "--metadata-migration", + "verify", + "--acknowledge-server-stopped" + ], + "exit_code": 0, + "expected_success": true + }, + { + "argv": [ + "target/debug/parseable", + "s3-store", + "--metadata-migration", + "export", + "--acknowledge-server-stopped", + "--dry-run" + ], + "exit_code": 0, + "expected_success": true + }, + { + "argv": [ + "target/debug/parseable", + "s3-store", + "--metadata-migration", + "import", + "--acknowledge-server-stopped" + ], + "exit_code": 1, + "expected_success": false + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1476557-0", + "psql", + "-U", + "postgres", + "-d", + "migration", + "-qAt", + "-c", + "SELECT id FROM parseable_backend" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1, + "refusal_state": "import-committed-unpaired", + "postgres": false + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1, + "refusal_state": "import-committed-unpaired", + "postgres": true + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1, + "refusal_state": "import-paired-before-marker-cleanup", + "postgres": false + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1, + "refusal_state": "import-paired-before-marker-cleanup", + "postgres": true + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1, + "refusal_state": "paired-boolean-override", + "postgres": false + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1, + "refusal_state": "unpaired-boolean-override", + "postgres": true + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1, + "refusal_state": "export-marker-before-state", + "postgres": false + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1, + "refusal_state": "export-marker-before-state", + "postgres": true + }, + { + "argv": [ + "target/debug/parseable", + "s3-store", + "--metadata-migration", + "export", + "--acknowledge-server-stopped", + "--abort" + ], + "exit_code": 0, + "expected_success": true + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1476557-0", + "psql", + "-U", + "postgres", + "-d", + "migration", + "-qAt", + "-c", + "UPDATE parseable_transfer SET state='exported'" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1, + "refusal_state": "export-state-before-object-writes", + "postgres": false + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1, + "refusal_state": "export-state-before-object-writes", + "postgres": true + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1476557-0", + "psql", + "-U", + "postgres", + "-d", + "migration", + "-qAt", + "-c", + "UPDATE parseable_transfer SET state='ready'" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1, + "refusal_state": "interrupted-export", + "postgres": false + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1, + "refusal_state": "interrupted-export", + "postgres": true + }, + { + "argv": [ + "target/debug/parseable", + "s3-store", + "--metadata-migration", + "export", + "--acknowledge-server-stopped" + ], + "exit_code": 0, + "expected_success": true + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1, + "refusal_state": "export-pair-removed-before-marker-cleanup", + "postgres": false + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1, + "refusal_state": "export-pair-removed-before-marker-cleanup", + "postgres": true + }, + { + "argv": [ + "target/debug/parseable", + "s3-store", + "--metadata-migration", + "export", + "--acknowledge-server-stopped" + ], + "exit_code": 0, + "expected_success": true + }, + { + "argv": [ + "target/debug/parseable", + "s3-store", + "--metadata-migration", + "verify", + "--acknowledge-server-stopped" + ], + "exit_code": 0, + "expected_success": true + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1, + "refusal_state": "exported-database-without-marker", + "postgres": true + } + ], + "bucket_keys": [ + ".alerts/01M75RG3APZZ2T0DVYD48GB62H.json", + ".alerts/alert_state_01M75RG3APZZ2T0DVYD48GB62H.json", + ".parseable/.parseable.json", + ".settings/migration-fixtures/0.json", + ".settings/migration-fixtures/1.json", + ".settings/migration-fixtures/10.json", + ".settings/migration-fixtures/100.json", + ".settings/migration-fixtures/101.json", + ".settings/migration-fixtures/102.json", + ".settings/migration-fixtures/103.json", + ".settings/migration-fixtures/104.json", + ".settings/migration-fixtures/105.json", + ".settings/migration-fixtures/106.json", + ".settings/migration-fixtures/107.json", + ".settings/migration-fixtures/108.json", + ".settings/migration-fixtures/109.json", + ".settings/migration-fixtures/11.json", + ".settings/migration-fixtures/110.json", + ".settings/migration-fixtures/111.json", + ".settings/migration-fixtures/112.json", + ".settings/migration-fixtures/113.json", + ".settings/migration-fixtures/114.json", + ".settings/migration-fixtures/115.json", + ".settings/migration-fixtures/116.json", + ".settings/migration-fixtures/117.json", + ".settings/migration-fixtures/118.json", + ".settings/migration-fixtures/119.json", + ".settings/migration-fixtures/12.json", + ".settings/migration-fixtures/120.json", + ".settings/migration-fixtures/121.json", + ".settings/migration-fixtures/122.json", + ".settings/migration-fixtures/123.json", + ".settings/migration-fixtures/124.json", + ".settings/migration-fixtures/125.json", + ".settings/migration-fixtures/126.json", + ".settings/migration-fixtures/127.json", + ".settings/migration-fixtures/128.json", + ".settings/migration-fixtures/129.json", + ".settings/migration-fixtures/13.json", + ".settings/migration-fixtures/130.json", + ".settings/migration-fixtures/131.json", + ".settings/migration-fixtures/132.json", + ".settings/migration-fixtures/133.json", + ".settings/migration-fixtures/134.json", + ".settings/migration-fixtures/135.json", + ".settings/migration-fixtures/136.json", + ".settings/migration-fixtures/137.json", + ".settings/migration-fixtures/138.json", + ".settings/migration-fixtures/139.json", + ".settings/migration-fixtures/14.json", + ".settings/migration-fixtures/140.json", + ".settings/migration-fixtures/141.json", + ".settings/migration-fixtures/142.json", + ".settings/migration-fixtures/143.json", + ".settings/migration-fixtures/144.json", + ".settings/migration-fixtures/145.json", + ".settings/migration-fixtures/146.json", + ".settings/migration-fixtures/147.json", + ".settings/migration-fixtures/148.json", + ".settings/migration-fixtures/149.json", + ".settings/migration-fixtures/15.json", + ".settings/migration-fixtures/150.json", + ".settings/migration-fixtures/151.json", + ".settings/migration-fixtures/152.json", + ".settings/migration-fixtures/153.json", + ".settings/migration-fixtures/154.json", + ".settings/migration-fixtures/155.json", + ".settings/migration-fixtures/156.json", + ".settings/migration-fixtures/157.json", + ".settings/migration-fixtures/158.json", + ".settings/migration-fixtures/159.json", + ".settings/migration-fixtures/16.json", + ".settings/migration-fixtures/160.json", + ".settings/migration-fixtures/161.json", + ".settings/migration-fixtures/162.json", + ".settings/migration-fixtures/163.json", + ".settings/migration-fixtures/164.json", + ".settings/migration-fixtures/165.json", + ".settings/migration-fixtures/166.json", + ".settings/migration-fixtures/167.json", + ".settings/migration-fixtures/168.json", + ".settings/migration-fixtures/169.json", + ".settings/migration-fixtures/17.json", + ".settings/migration-fixtures/170.json", + ".settings/migration-fixtures/171.json", + ".settings/migration-fixtures/172.json", + ".settings/migration-fixtures/173.json", + ".settings/migration-fixtures/174.json", + ".settings/migration-fixtures/175.json", + ".settings/migration-fixtures/176.json", + ".settings/migration-fixtures/177.json", + ".settings/migration-fixtures/178.json", + ".settings/migration-fixtures/179.json", + ".settings/migration-fixtures/18.json", + ".settings/migration-fixtures/180.json", + ".settings/migration-fixtures/181.json", + ".settings/migration-fixtures/182.json", + ".settings/migration-fixtures/183.json", + ".settings/migration-fixtures/184.json", + ".settings/migration-fixtures/185.json", + ".settings/migration-fixtures/186.json", + ".settings/migration-fixtures/187.json", + ".settings/migration-fixtures/188.json", + ".settings/migration-fixtures/189.json", + ".settings/migration-fixtures/19.json", + ".settings/migration-fixtures/190.json", + ".settings/migration-fixtures/191.json", + ".settings/migration-fixtures/192.json", + ".settings/migration-fixtures/193.json", + ".settings/migration-fixtures/194.json", + ".settings/migration-fixtures/195.json", + ".settings/migration-fixtures/196.json", + ".settings/migration-fixtures/197.json", + ".settings/migration-fixtures/198.json", + ".settings/migration-fixtures/199.json", + ".settings/migration-fixtures/2.json", + ".settings/migration-fixtures/20.json", + ".settings/migration-fixtures/200.json", + ".settings/migration-fixtures/201.json", + ".settings/migration-fixtures/202.json", + ".settings/migration-fixtures/203.json", + ".settings/migration-fixtures/204.json", + ".settings/migration-fixtures/205.json", + ".settings/migration-fixtures/206.json", + ".settings/migration-fixtures/207.json", + ".settings/migration-fixtures/208.json", + ".settings/migration-fixtures/209.json", + ".settings/migration-fixtures/21.json", + ".settings/migration-fixtures/210.json", + ".settings/migration-fixtures/211.json", + ".settings/migration-fixtures/212.json", + ".settings/migration-fixtures/213.json", + ".settings/migration-fixtures/214.json", + ".settings/migration-fixtures/215.json", + ".settings/migration-fixtures/216.json", + ".settings/migration-fixtures/217.json", + ".settings/migration-fixtures/218.json", + ".settings/migration-fixtures/219.json", + ".settings/migration-fixtures/22.json", + ".settings/migration-fixtures/220.json", + ".settings/migration-fixtures/221.json", + ".settings/migration-fixtures/222.json", + ".settings/migration-fixtures/223.json", + ".settings/migration-fixtures/224.json", + ".settings/migration-fixtures/225.json", + ".settings/migration-fixtures/226.json", + ".settings/migration-fixtures/227.json", + ".settings/migration-fixtures/228.json", + ".settings/migration-fixtures/229.json", + ".settings/migration-fixtures/23.json", + ".settings/migration-fixtures/230.json", + ".settings/migration-fixtures/231.json", + ".settings/migration-fixtures/232.json", + ".settings/migration-fixtures/233.json", + ".settings/migration-fixtures/234.json", + ".settings/migration-fixtures/235.json", + ".settings/migration-fixtures/236.json", + ".settings/migration-fixtures/237.json", + ".settings/migration-fixtures/238.json", + ".settings/migration-fixtures/239.json", + ".settings/migration-fixtures/24.json", + ".settings/migration-fixtures/240.json", + ".settings/migration-fixtures/241.json", + ".settings/migration-fixtures/242.json", + ".settings/migration-fixtures/243.json", + ".settings/migration-fixtures/244.json", + ".settings/migration-fixtures/245.json", + ".settings/migration-fixtures/246.json", + ".settings/migration-fixtures/247.json", + ".settings/migration-fixtures/248.json", + ".settings/migration-fixtures/249.json", + ".settings/migration-fixtures/25.json", + ".settings/migration-fixtures/250.json", + ".settings/migration-fixtures/251.json", + ".settings/migration-fixtures/252.json", + ".settings/migration-fixtures/253.json", + ".settings/migration-fixtures/254.json", + ".settings/migration-fixtures/255.json", + ".settings/migration-fixtures/256.json", + ".settings/migration-fixtures/257.json", + ".settings/migration-fixtures/258.json", + ".settings/migration-fixtures/259.json", + ".settings/migration-fixtures/26.json", + ".settings/migration-fixtures/260.json", + ".settings/migration-fixtures/261.json", + ".settings/migration-fixtures/262.json", + ".settings/migration-fixtures/263.json", + ".settings/migration-fixtures/264.json", + ".settings/migration-fixtures/265.json", + ".settings/migration-fixtures/266.json", + ".settings/migration-fixtures/267.json", + ".settings/migration-fixtures/268.json", + ".settings/migration-fixtures/269.json", + ".settings/migration-fixtures/27.json", + ".settings/migration-fixtures/270.json", + ".settings/migration-fixtures/271.json", + ".settings/migration-fixtures/272.json", + ".settings/migration-fixtures/273.json", + ".settings/migration-fixtures/274.json", + ".settings/migration-fixtures/275.json", + ".settings/migration-fixtures/276.json", + ".settings/migration-fixtures/277.json", + ".settings/migration-fixtures/278.json", + ".settings/migration-fixtures/279.json", + ".settings/migration-fixtures/28.json", + ".settings/migration-fixtures/280.json", + ".settings/migration-fixtures/281.json", + ".settings/migration-fixtures/282.json", + ".settings/migration-fixtures/283.json", + ".settings/migration-fixtures/284.json", + ".settings/migration-fixtures/285.json", + ".settings/migration-fixtures/286.json", + ".settings/migration-fixtures/287.json", + ".settings/migration-fixtures/288.json", + ".settings/migration-fixtures/289.json", + ".settings/migration-fixtures/29.json", + ".settings/migration-fixtures/290.json", + ".settings/migration-fixtures/291.json", + ".settings/migration-fixtures/292.json", + ".settings/migration-fixtures/293.json", + ".settings/migration-fixtures/294.json", + ".settings/migration-fixtures/295.json", + ".settings/migration-fixtures/296.json", + ".settings/migration-fixtures/297.json", + ".settings/migration-fixtures/298.json", + ".settings/migration-fixtures/299.json", + ".settings/migration-fixtures/3.json", + ".settings/migration-fixtures/30.json", + ".settings/migration-fixtures/300.json", + ".settings/migration-fixtures/301.json", + ".settings/migration-fixtures/302.json", + ".settings/migration-fixtures/303.json", + ".settings/migration-fixtures/304.json", + ".settings/migration-fixtures/305.json", + ".settings/migration-fixtures/306.json", + ".settings/migration-fixtures/307.json", + ".settings/migration-fixtures/308.json", + ".settings/migration-fixtures/309.json", + ".settings/migration-fixtures/31.json", + ".settings/migration-fixtures/310.json", + ".settings/migration-fixtures/311.json", + ".settings/migration-fixtures/312.json", + ".settings/migration-fixtures/313.json", + ".settings/migration-fixtures/314.json", + ".settings/migration-fixtures/315.json", + ".settings/migration-fixtures/316.json", + ".settings/migration-fixtures/317.json", + ".settings/migration-fixtures/318.json", + ".settings/migration-fixtures/319.json", + ".settings/migration-fixtures/32.json", + ".settings/migration-fixtures/320.json", + ".settings/migration-fixtures/321.json", + ".settings/migration-fixtures/322.json", + ".settings/migration-fixtures/323.json", + ".settings/migration-fixtures/324.json", + ".settings/migration-fixtures/325.json", + ".settings/migration-fixtures/326.json", + ".settings/migration-fixtures/327.json", + ".settings/migration-fixtures/328.json", + ".settings/migration-fixtures/329.json", + ".settings/migration-fixtures/33.json", + ".settings/migration-fixtures/330.json", + ".settings/migration-fixtures/331.json", + ".settings/migration-fixtures/332.json", + ".settings/migration-fixtures/333.json", + ".settings/migration-fixtures/334.json", + ".settings/migration-fixtures/335.json", + ".settings/migration-fixtures/336.json", + ".settings/migration-fixtures/337.json", + ".settings/migration-fixtures/338.json", + ".settings/migration-fixtures/339.json", + ".settings/migration-fixtures/34.json", + ".settings/migration-fixtures/340.json", + ".settings/migration-fixtures/341.json", + ".settings/migration-fixtures/342.json", + ".settings/migration-fixtures/343.json", + ".settings/migration-fixtures/344.json", + ".settings/migration-fixtures/345.json", + ".settings/migration-fixtures/346.json", + ".settings/migration-fixtures/347.json", + ".settings/migration-fixtures/348.json", + ".settings/migration-fixtures/349.json", + ".settings/migration-fixtures/35.json", + ".settings/migration-fixtures/350.json", + ".settings/migration-fixtures/351.json", + ".settings/migration-fixtures/352.json", + ".settings/migration-fixtures/353.json", + ".settings/migration-fixtures/354.json", + ".settings/migration-fixtures/355.json", + ".settings/migration-fixtures/356.json", + ".settings/migration-fixtures/357.json", + ".settings/migration-fixtures/358.json", + ".settings/migration-fixtures/359.json", + ".settings/migration-fixtures/36.json", + ".settings/migration-fixtures/360.json", + ".settings/migration-fixtures/361.json", + ".settings/migration-fixtures/362.json", + ".settings/migration-fixtures/363.json", + ".settings/migration-fixtures/364.json", + ".settings/migration-fixtures/365.json", + ".settings/migration-fixtures/366.json", + ".settings/migration-fixtures/367.json", + ".settings/migration-fixtures/368.json", + ".settings/migration-fixtures/369.json", + ".settings/migration-fixtures/37.json", + ".settings/migration-fixtures/370.json", + ".settings/migration-fixtures/371.json", + ".settings/migration-fixtures/372.json", + ".settings/migration-fixtures/373.json", + ".settings/migration-fixtures/374.json", + ".settings/migration-fixtures/375.json", + ".settings/migration-fixtures/376.json", + ".settings/migration-fixtures/377.json", + ".settings/migration-fixtures/378.json", + ".settings/migration-fixtures/379.json", + ".settings/migration-fixtures/38.json", + ".settings/migration-fixtures/380.json", + ".settings/migration-fixtures/381.json", + ".settings/migration-fixtures/382.json", + ".settings/migration-fixtures/383.json", + ".settings/migration-fixtures/384.json", + ".settings/migration-fixtures/385.json", + ".settings/migration-fixtures/386.json", + ".settings/migration-fixtures/387.json", + ".settings/migration-fixtures/388.json", + ".settings/migration-fixtures/389.json", + ".settings/migration-fixtures/39.json", + ".settings/migration-fixtures/390.json", + ".settings/migration-fixtures/391.json", + ".settings/migration-fixtures/392.json", + ".settings/migration-fixtures/393.json", + ".settings/migration-fixtures/394.json", + ".settings/migration-fixtures/395.json", + ".settings/migration-fixtures/396.json", + ".settings/migration-fixtures/397.json", + ".settings/migration-fixtures/398.json", + ".settings/migration-fixtures/399.json", + ".settings/migration-fixtures/4.json", + ".settings/migration-fixtures/40.json", + ".settings/migration-fixtures/400.json", + ".settings/migration-fixtures/401.json", + ".settings/migration-fixtures/402.json", + ".settings/migration-fixtures/403.json", + ".settings/migration-fixtures/404.json", + ".settings/migration-fixtures/405.json", + ".settings/migration-fixtures/406.json", + ".settings/migration-fixtures/407.json", + ".settings/migration-fixtures/408.json", + ".settings/migration-fixtures/409.json", + ".settings/migration-fixtures/41.json", + ".settings/migration-fixtures/410.json", + ".settings/migration-fixtures/411.json", + ".settings/migration-fixtures/412.json", + ".settings/migration-fixtures/413.json", + ".settings/migration-fixtures/414.json", + ".settings/migration-fixtures/415.json", + ".settings/migration-fixtures/416.json", + ".settings/migration-fixtures/417.json", + ".settings/migration-fixtures/418.json", + ".settings/migration-fixtures/419.json", + ".settings/migration-fixtures/42.json", + ".settings/migration-fixtures/420.json", + ".settings/migration-fixtures/421.json", + ".settings/migration-fixtures/422.json", + ".settings/migration-fixtures/423.json", + ".settings/migration-fixtures/424.json", + ".settings/migration-fixtures/425.json", + ".settings/migration-fixtures/426.json", + ".settings/migration-fixtures/427.json", + ".settings/migration-fixtures/428.json", + ".settings/migration-fixtures/429.json", + ".settings/migration-fixtures/43.json", + ".settings/migration-fixtures/430.json", + ".settings/migration-fixtures/431.json", + ".settings/migration-fixtures/432.json", + ".settings/migration-fixtures/433.json", + ".settings/migration-fixtures/434.json", + ".settings/migration-fixtures/435.json", + ".settings/migration-fixtures/436.json", + ".settings/migration-fixtures/437.json", + ".settings/migration-fixtures/438.json", + ".settings/migration-fixtures/439.json", + ".settings/migration-fixtures/44.json", + ".settings/migration-fixtures/440.json", + ".settings/migration-fixtures/441.json", + ".settings/migration-fixtures/442.json", + ".settings/migration-fixtures/443.json", + ".settings/migration-fixtures/444.json", + ".settings/migration-fixtures/445.json", + ".settings/migration-fixtures/446.json", + ".settings/migration-fixtures/447.json", + ".settings/migration-fixtures/448.json", + ".settings/migration-fixtures/449.json", + ".settings/migration-fixtures/45.json", + ".settings/migration-fixtures/450.json", + ".settings/migration-fixtures/451.json", + ".settings/migration-fixtures/452.json", + ".settings/migration-fixtures/453.json", + ".settings/migration-fixtures/454.json", + ".settings/migration-fixtures/455.json", + ".settings/migration-fixtures/456.json", + ".settings/migration-fixtures/457.json", + ".settings/migration-fixtures/458.json", + ".settings/migration-fixtures/459.json", + ".settings/migration-fixtures/46.json", + ".settings/migration-fixtures/460.json", + ".settings/migration-fixtures/461.json", + ".settings/migration-fixtures/462.json", + ".settings/migration-fixtures/463.json", + ".settings/migration-fixtures/464.json", + ".settings/migration-fixtures/465.json", + ".settings/migration-fixtures/466.json", + ".settings/migration-fixtures/467.json", + ".settings/migration-fixtures/468.json", + ".settings/migration-fixtures/469.json", + ".settings/migration-fixtures/47.json", + ".settings/migration-fixtures/470.json", + ".settings/migration-fixtures/471.json", + ".settings/migration-fixtures/472.json", + ".settings/migration-fixtures/473.json", + ".settings/migration-fixtures/474.json", + ".settings/migration-fixtures/475.json", + ".settings/migration-fixtures/476.json", + ".settings/migration-fixtures/477.json", + ".settings/migration-fixtures/478.json", + ".settings/migration-fixtures/479.json", + ".settings/migration-fixtures/48.json", + ".settings/migration-fixtures/480.json", + ".settings/migration-fixtures/481.json", + ".settings/migration-fixtures/482.json", + ".settings/migration-fixtures/483.json", + ".settings/migration-fixtures/484.json", + ".settings/migration-fixtures/485.json", + ".settings/migration-fixtures/486.json", + ".settings/migration-fixtures/487.json", + ".settings/migration-fixtures/488.json", + ".settings/migration-fixtures/489.json", + ".settings/migration-fixtures/49.json", + ".settings/migration-fixtures/490.json", + ".settings/migration-fixtures/491.json", + ".settings/migration-fixtures/492.json", + ".settings/migration-fixtures/493.json", + ".settings/migration-fixtures/494.json", + ".settings/migration-fixtures/495.json", + ".settings/migration-fixtures/496.json", + ".settings/migration-fixtures/497.json", + ".settings/migration-fixtures/498.json", + ".settings/migration-fixtures/499.json", + ".settings/migration-fixtures/5.json", + ".settings/migration-fixtures/50.json", + ".settings/migration-fixtures/51.json", + ".settings/migration-fixtures/52.json", + ".settings/migration-fixtures/53.json", + ".settings/migration-fixtures/54.json", + ".settings/migration-fixtures/55.json", + ".settings/migration-fixtures/56.json", + ".settings/migration-fixtures/57.json", + ".settings/migration-fixtures/58.json", + ".settings/migration-fixtures/59.json", + ".settings/migration-fixtures/6.json", + ".settings/migration-fixtures/60.json", + ".settings/migration-fixtures/61.json", + ".settings/migration-fixtures/62.json", + ".settings/migration-fixtures/63.json", + ".settings/migration-fixtures/64.json", + ".settings/migration-fixtures/65.json", + ".settings/migration-fixtures/66.json", + ".settings/migration-fixtures/67.json", + ".settings/migration-fixtures/68.json", + ".settings/migration-fixtures/69.json", + ".settings/migration-fixtures/7.json", + ".settings/migration-fixtures/70.json", + ".settings/migration-fixtures/71.json", + ".settings/migration-fixtures/72.json", + ".settings/migration-fixtures/73.json", + ".settings/migration-fixtures/74.json", + ".settings/migration-fixtures/75.json", + ".settings/migration-fixtures/76.json", + ".settings/migration-fixtures/77.json", + ".settings/migration-fixtures/78.json", + ".settings/migration-fixtures/79.json", + ".settings/migration-fixtures/8.json", + ".settings/migration-fixtures/80.json", + ".settings/migration-fixtures/81.json", + ".settings/migration-fixtures/82.json", + ".settings/migration-fixtures/83.json", + ".settings/migration-fixtures/84.json", + ".settings/migration-fixtures/85.json", + ".settings/migration-fixtures/86.json", + ".settings/migration-fixtures/87.json", + ".settings/migration-fixtures/88.json", + ".settings/migration-fixtures/89.json", + ".settings/migration-fixtures/9.json", + ".settings/migration-fixtures/90.json", + ".settings/migration-fixtures/91.json", + ".settings/migration-fixtures/92.json", + ".settings/migration-fixtures/93.json", + ".settings/migration-fixtures/94.json", + ".settings/migration-fixtures/95.json", + ".settings/migration-fixtures/96.json", + ".settings/migration-fixtures/97.json", + ".settings/migration-fixtures/98.json", + ".settings/migration-fixtures/99.json", + ".users/.dashboard-locations/01M75RG39Z4NKWZK8SA8NFW8Q6.json", + ".users/.dashboard-titles/2b9aa3aaf18aa9aa60fcc45d30833be66dc5907be190dfff57197c19883297e4.json", + ".users/8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918/dashboards/01M75RG39Z4NKWZK8SA8NFW8Q6.json", + ".users/8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918/filters/first/01M75RG3AG7WMBR9T8RWFJMBRQ.json", + "first/.stream/.schema", + "first/.stream/.stream.json", + "first/date=2026-11-11/.manifest.json", + "first/date=2026-11-11/hour=16/minute=54/.data.01M75RHH4MMRY8P8Z2VV37G1GJ.parquet", + "first/date=2026-11-11/hour=16/minute=56/.data.01M75RN67QPP2693W1AFKVZ62Z.parquet", + "partitioned/.stream/.schema", + "partitioned/.stream/.stream.json", + "partitioned/date=2026-11-11/.manifest.json", + "partitioned/date=2026-11-11/hour=16/minute=54/region=west/.data.01M75RHH4MPPMHAPHBNY12A63B.parquet", + "pstats/.stream/.schema", + "pstats/.stream/.stream.json", + "pstats/date=2026-11-11/.manifest.json", + "pstats/date=2026-11-11/hour=16/minute=54/dataset_name=first/.data.01M75RKBQMSRE7XKV9SQSKSVDA.parquet", + "pstats/date=2026-11-11/hour=16/minute=54/dataset_name=partitioned/.data.01M75RKBQMSRE7XKV9SQSKSVDA.parquet", + "pstats/date=2026-11-11/hour=16/minute=54/dataset_name=second/.data.01M75RKBQMSRE7XKV9SQSKSVDA.parquet", + "pstats/date=2026-11-11/hour=16/minute=56/dataset_name=first/.data.01M75RQ0TPPX471FAN826ARJ2V.parquet", + "second/.stream/.schema", + "second/.stream/.stream.json", + "second/date=2026-11-11/.manifest.json", + "second/date=2026-11-11/hour=16/minute=54/.data.01M75RHH4MF5M9DZ6MTG796EK5.parquet" + ] +} diff --git a/docs/validation/postgres-migration.json b/docs/validation/postgres-migration.json new file mode 100644 index 000000000..025e9ca67 --- /dev/null +++ b/docs/validation/postgres-migration.json @@ -0,0 +1,960 @@ +{ + "success": true, + "tls": false, + "commands": [ + { + "argv": [ + "docker", + "run", + "-d", + "--name", + "parseable-pg-smoke-1458428-0", + "-e", + "POSTGRES_PASSWORD=smoke", + "-e", + "POSTGRES_DB=migration", + "-p", + "127.0.0.1:42245:5432", + "postgres:17-alpine" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "run", + "-d", + "--name", + "parseable-pg-smoke-1458428-1", + "-e", + "MINIO_ROOT_USER=minioadmin", + "-e", + "MINIO_ROOT_PASSWORD=minioadmin", + "-p", + "127.0.0.1:59295:9000", + "quay.io/parseablehq/minio:edge", + "server", + "/data" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1458428-1", + "mkdir", + "-p", + "/data/migration" + ], + "exit_code": 0 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1458428-0", + "pg_isready", + "-h", + "127.0.0.1", + "-U", + "postgres" + ], + "exit_code": 2 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1458428-0", + "pg_isready", + "-h", + "127.0.0.1", + "-U", + "postgres" + ], + "exit_code": 2 + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1458428-0", + "pg_isready", + "-h", + "127.0.0.1", + "-U", + "postgres" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store", + "--metadata-migration", + "import", + "--acknowledge-server-stopped", + "--dry-run" + ], + "exit_code": 1, + "expected_success": false + }, + { + "argv": [ + "target/debug/parseable", + "s3-store", + "--metadata-migration", + "verify", + "--acknowledge-server-stopped" + ], + "exit_code": 1, + "expected_success": false + }, + { + "argv": [ + "target/debug/parseable", + "s3-store", + "--metadata-migration", + "import", + "--dry-run", + "--acknowledge-server-stopped" + ], + "cwd": ".postgres-smoke/migration-env-fixture", + "exit_code": 0, + "env_file_checked": true + }, + { + "argv": [ + "target/debug/parseable", + "s3-store", + "--metadata-migration", + "import", + "--acknowledge-server-stopped", + "--dry-run" + ], + "exit_code": 0, + "expected_success": true + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1, + "refusal_state": "import-before-commit", + "postgres": false + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1, + "refusal_state": "import-before-commit", + "postgres": true + }, + { + "argv": [ + "target/debug/parseable", + "s3-store", + "--metadata-migration", + "import", + "--acknowledge-server-stopped" + ], + "exit_code": 0, + "expected_success": true + }, + { + "argv": [ + "target/debug/parseable", + "s3-store", + "--metadata-migration", + "verify", + "--acknowledge-server-stopped" + ], + "exit_code": 0, + "expected_success": true + }, + { + "argv": [ + "target/debug/parseable", + "s3-store", + "--metadata-migration", + "export", + "--acknowledge-server-stopped", + "--dry-run" + ], + "exit_code": 0, + "expected_success": true + }, + { + "argv": [ + "target/debug/parseable", + "s3-store", + "--metadata-migration", + "import", + "--acknowledge-server-stopped" + ], + "exit_code": 1, + "expected_success": false + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1458428-0", + "psql", + "-U", + "postgres", + "-d", + "migration", + "-qAt", + "-c", + "SELECT id FROM parseable_backend" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1, + "refusal_state": "import-committed-unpaired", + "postgres": false + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1, + "refusal_state": "import-committed-unpaired", + "postgres": true + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1, + "refusal_state": "import-paired-before-marker-cleanup", + "postgres": false + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1, + "refusal_state": "import-paired-before-marker-cleanup", + "postgres": true + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1, + "refusal_state": "paired-boolean-override", + "postgres": false + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1, + "refusal_state": "unpaired-boolean-override", + "postgres": true + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1, + "refusal_state": "export-marker-before-state", + "postgres": false + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1, + "refusal_state": "export-marker-before-state", + "postgres": true + }, + { + "argv": [ + "target/debug/parseable", + "s3-store", + "--metadata-migration", + "export", + "--acknowledge-server-stopped", + "--abort" + ], + "exit_code": 0, + "expected_success": true + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1458428-0", + "psql", + "-U", + "postgres", + "-d", + "migration", + "-qAt", + "-c", + "UPDATE parseable_transfer SET state='exported'" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1, + "refusal_state": "export-state-before-object-writes", + "postgres": false + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1, + "refusal_state": "export-state-before-object-writes", + "postgres": true + }, + { + "argv": [ + "docker", + "exec", + "parseable-pg-smoke-1458428-0", + "psql", + "-U", + "postgres", + "-d", + "migration", + "-qAt", + "-c", + "UPDATE parseable_transfer SET state='ready'" + ], + "exit_code": 0 + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1, + "refusal_state": "interrupted-export", + "postgres": false + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1, + "refusal_state": "interrupted-export", + "postgres": true + }, + { + "argv": [ + "target/debug/parseable", + "s3-store", + "--metadata-migration", + "export", + "--acknowledge-server-stopped" + ], + "exit_code": 0, + "expected_success": true + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1, + "refusal_state": "export-pair-removed-before-marker-cleanup", + "postgres": false + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1, + "refusal_state": "export-pair-removed-before-marker-cleanup", + "postgres": true + }, + { + "argv": [ + "target/debug/parseable", + "s3-store", + "--metadata-migration", + "export", + "--acknowledge-server-stopped" + ], + "exit_code": 0, + "expected_success": true + }, + { + "argv": [ + "target/debug/parseable", + "s3-store", + "--metadata-migration", + "verify", + "--acknowledge-server-stopped" + ], + "exit_code": 0, + "expected_success": true + }, + { + "argv": [ + "target/debug/parseable", + "s3-store" + ], + "exit_code": 1, + "refusal_state": "exported-database-without-marker", + "postgres": true + } + ], + "bucket_keys": [ + ".alerts/01M75R7GG5CH5F1F34AJWY19TV.json", + ".alerts/alert_state_01M75R7GG5CH5F1F34AJWY19TV.json", + ".parseable/.parseable.json", + ".settings/migration-fixtures/0.json", + ".settings/migration-fixtures/1.json", + ".settings/migration-fixtures/10.json", + ".settings/migration-fixtures/100.json", + ".settings/migration-fixtures/101.json", + ".settings/migration-fixtures/102.json", + ".settings/migration-fixtures/103.json", + ".settings/migration-fixtures/104.json", + ".settings/migration-fixtures/105.json", + ".settings/migration-fixtures/106.json", + ".settings/migration-fixtures/107.json", + ".settings/migration-fixtures/108.json", + ".settings/migration-fixtures/109.json", + ".settings/migration-fixtures/11.json", + ".settings/migration-fixtures/110.json", + ".settings/migration-fixtures/111.json", + ".settings/migration-fixtures/112.json", + ".settings/migration-fixtures/113.json", + ".settings/migration-fixtures/114.json", + ".settings/migration-fixtures/115.json", + ".settings/migration-fixtures/116.json", + ".settings/migration-fixtures/117.json", + ".settings/migration-fixtures/118.json", + ".settings/migration-fixtures/119.json", + ".settings/migration-fixtures/12.json", + ".settings/migration-fixtures/120.json", + ".settings/migration-fixtures/121.json", + ".settings/migration-fixtures/122.json", + ".settings/migration-fixtures/123.json", + ".settings/migration-fixtures/124.json", + ".settings/migration-fixtures/125.json", + ".settings/migration-fixtures/126.json", + ".settings/migration-fixtures/127.json", + ".settings/migration-fixtures/128.json", + ".settings/migration-fixtures/129.json", + ".settings/migration-fixtures/13.json", + ".settings/migration-fixtures/130.json", + ".settings/migration-fixtures/131.json", + ".settings/migration-fixtures/132.json", + ".settings/migration-fixtures/133.json", + ".settings/migration-fixtures/134.json", + ".settings/migration-fixtures/135.json", + ".settings/migration-fixtures/136.json", + ".settings/migration-fixtures/137.json", + ".settings/migration-fixtures/138.json", + ".settings/migration-fixtures/139.json", + ".settings/migration-fixtures/14.json", + ".settings/migration-fixtures/140.json", + ".settings/migration-fixtures/141.json", + ".settings/migration-fixtures/142.json", + ".settings/migration-fixtures/143.json", + ".settings/migration-fixtures/144.json", + ".settings/migration-fixtures/145.json", + ".settings/migration-fixtures/146.json", + ".settings/migration-fixtures/147.json", + ".settings/migration-fixtures/148.json", + ".settings/migration-fixtures/149.json", + ".settings/migration-fixtures/15.json", + ".settings/migration-fixtures/150.json", + ".settings/migration-fixtures/151.json", + ".settings/migration-fixtures/152.json", + ".settings/migration-fixtures/153.json", + ".settings/migration-fixtures/154.json", + ".settings/migration-fixtures/155.json", + ".settings/migration-fixtures/156.json", + ".settings/migration-fixtures/157.json", + ".settings/migration-fixtures/158.json", + ".settings/migration-fixtures/159.json", + ".settings/migration-fixtures/16.json", + ".settings/migration-fixtures/160.json", + ".settings/migration-fixtures/161.json", + ".settings/migration-fixtures/162.json", + ".settings/migration-fixtures/163.json", + ".settings/migration-fixtures/164.json", + ".settings/migration-fixtures/165.json", + ".settings/migration-fixtures/166.json", + ".settings/migration-fixtures/167.json", + ".settings/migration-fixtures/168.json", + ".settings/migration-fixtures/169.json", + ".settings/migration-fixtures/17.json", + ".settings/migration-fixtures/170.json", + ".settings/migration-fixtures/171.json", + ".settings/migration-fixtures/172.json", + ".settings/migration-fixtures/173.json", + ".settings/migration-fixtures/174.json", + ".settings/migration-fixtures/175.json", + ".settings/migration-fixtures/176.json", + ".settings/migration-fixtures/177.json", + ".settings/migration-fixtures/178.json", + ".settings/migration-fixtures/179.json", + ".settings/migration-fixtures/18.json", + ".settings/migration-fixtures/180.json", + ".settings/migration-fixtures/181.json", + ".settings/migration-fixtures/182.json", + ".settings/migration-fixtures/183.json", + ".settings/migration-fixtures/184.json", + ".settings/migration-fixtures/185.json", + ".settings/migration-fixtures/186.json", + ".settings/migration-fixtures/187.json", + ".settings/migration-fixtures/188.json", + ".settings/migration-fixtures/189.json", + ".settings/migration-fixtures/19.json", + ".settings/migration-fixtures/190.json", + ".settings/migration-fixtures/191.json", + ".settings/migration-fixtures/192.json", + ".settings/migration-fixtures/193.json", + ".settings/migration-fixtures/194.json", + ".settings/migration-fixtures/195.json", + ".settings/migration-fixtures/196.json", + ".settings/migration-fixtures/197.json", + ".settings/migration-fixtures/198.json", + ".settings/migration-fixtures/199.json", + ".settings/migration-fixtures/2.json", + ".settings/migration-fixtures/20.json", + ".settings/migration-fixtures/200.json", + ".settings/migration-fixtures/201.json", + ".settings/migration-fixtures/202.json", + ".settings/migration-fixtures/203.json", + ".settings/migration-fixtures/204.json", + ".settings/migration-fixtures/205.json", + ".settings/migration-fixtures/206.json", + ".settings/migration-fixtures/207.json", + ".settings/migration-fixtures/208.json", + ".settings/migration-fixtures/209.json", + ".settings/migration-fixtures/21.json", + ".settings/migration-fixtures/210.json", + ".settings/migration-fixtures/211.json", + ".settings/migration-fixtures/212.json", + ".settings/migration-fixtures/213.json", + ".settings/migration-fixtures/214.json", + ".settings/migration-fixtures/215.json", + ".settings/migration-fixtures/216.json", + ".settings/migration-fixtures/217.json", + ".settings/migration-fixtures/218.json", + ".settings/migration-fixtures/219.json", + ".settings/migration-fixtures/22.json", + ".settings/migration-fixtures/220.json", + ".settings/migration-fixtures/221.json", + ".settings/migration-fixtures/222.json", + ".settings/migration-fixtures/223.json", + ".settings/migration-fixtures/224.json", + ".settings/migration-fixtures/225.json", + ".settings/migration-fixtures/226.json", + ".settings/migration-fixtures/227.json", + ".settings/migration-fixtures/228.json", + ".settings/migration-fixtures/229.json", + ".settings/migration-fixtures/23.json", + ".settings/migration-fixtures/230.json", + ".settings/migration-fixtures/231.json", + ".settings/migration-fixtures/232.json", + ".settings/migration-fixtures/233.json", + ".settings/migration-fixtures/234.json", + ".settings/migration-fixtures/235.json", + ".settings/migration-fixtures/236.json", + ".settings/migration-fixtures/237.json", + ".settings/migration-fixtures/238.json", + ".settings/migration-fixtures/239.json", + ".settings/migration-fixtures/24.json", + ".settings/migration-fixtures/240.json", + ".settings/migration-fixtures/241.json", + ".settings/migration-fixtures/242.json", + ".settings/migration-fixtures/243.json", + ".settings/migration-fixtures/244.json", + ".settings/migration-fixtures/245.json", + ".settings/migration-fixtures/246.json", + ".settings/migration-fixtures/247.json", + ".settings/migration-fixtures/248.json", + ".settings/migration-fixtures/249.json", + ".settings/migration-fixtures/25.json", + ".settings/migration-fixtures/250.json", + ".settings/migration-fixtures/251.json", + ".settings/migration-fixtures/252.json", + ".settings/migration-fixtures/253.json", + ".settings/migration-fixtures/254.json", + ".settings/migration-fixtures/255.json", + ".settings/migration-fixtures/256.json", + ".settings/migration-fixtures/257.json", + ".settings/migration-fixtures/258.json", + ".settings/migration-fixtures/259.json", + ".settings/migration-fixtures/26.json", + ".settings/migration-fixtures/260.json", + ".settings/migration-fixtures/261.json", + ".settings/migration-fixtures/262.json", + ".settings/migration-fixtures/263.json", + ".settings/migration-fixtures/264.json", + ".settings/migration-fixtures/265.json", + ".settings/migration-fixtures/266.json", + ".settings/migration-fixtures/267.json", + ".settings/migration-fixtures/268.json", + ".settings/migration-fixtures/269.json", + ".settings/migration-fixtures/27.json", + ".settings/migration-fixtures/270.json", + ".settings/migration-fixtures/271.json", + ".settings/migration-fixtures/272.json", + ".settings/migration-fixtures/273.json", + ".settings/migration-fixtures/274.json", + ".settings/migration-fixtures/275.json", + ".settings/migration-fixtures/276.json", + ".settings/migration-fixtures/277.json", + ".settings/migration-fixtures/278.json", + ".settings/migration-fixtures/279.json", + ".settings/migration-fixtures/28.json", + ".settings/migration-fixtures/280.json", + ".settings/migration-fixtures/281.json", + ".settings/migration-fixtures/282.json", + ".settings/migration-fixtures/283.json", + ".settings/migration-fixtures/284.json", + ".settings/migration-fixtures/285.json", + ".settings/migration-fixtures/286.json", + ".settings/migration-fixtures/287.json", + ".settings/migration-fixtures/288.json", + ".settings/migration-fixtures/289.json", + ".settings/migration-fixtures/29.json", + ".settings/migration-fixtures/290.json", + ".settings/migration-fixtures/291.json", + ".settings/migration-fixtures/292.json", + ".settings/migration-fixtures/293.json", + ".settings/migration-fixtures/294.json", + ".settings/migration-fixtures/295.json", + ".settings/migration-fixtures/296.json", + ".settings/migration-fixtures/297.json", + ".settings/migration-fixtures/298.json", + ".settings/migration-fixtures/299.json", + ".settings/migration-fixtures/3.json", + ".settings/migration-fixtures/30.json", + ".settings/migration-fixtures/300.json", + ".settings/migration-fixtures/301.json", + ".settings/migration-fixtures/302.json", + ".settings/migration-fixtures/303.json", + ".settings/migration-fixtures/304.json", + ".settings/migration-fixtures/305.json", + ".settings/migration-fixtures/306.json", + ".settings/migration-fixtures/307.json", + ".settings/migration-fixtures/308.json", + ".settings/migration-fixtures/309.json", + ".settings/migration-fixtures/31.json", + ".settings/migration-fixtures/310.json", + ".settings/migration-fixtures/311.json", + ".settings/migration-fixtures/312.json", + ".settings/migration-fixtures/313.json", + ".settings/migration-fixtures/314.json", + ".settings/migration-fixtures/315.json", + ".settings/migration-fixtures/316.json", + ".settings/migration-fixtures/317.json", + ".settings/migration-fixtures/318.json", + ".settings/migration-fixtures/319.json", + ".settings/migration-fixtures/32.json", + ".settings/migration-fixtures/320.json", + ".settings/migration-fixtures/321.json", + ".settings/migration-fixtures/322.json", + ".settings/migration-fixtures/323.json", + ".settings/migration-fixtures/324.json", + ".settings/migration-fixtures/325.json", + ".settings/migration-fixtures/326.json", + ".settings/migration-fixtures/327.json", + ".settings/migration-fixtures/328.json", + ".settings/migration-fixtures/329.json", + ".settings/migration-fixtures/33.json", + ".settings/migration-fixtures/330.json", + ".settings/migration-fixtures/331.json", + ".settings/migration-fixtures/332.json", + ".settings/migration-fixtures/333.json", + ".settings/migration-fixtures/334.json", + ".settings/migration-fixtures/335.json", + ".settings/migration-fixtures/336.json", + ".settings/migration-fixtures/337.json", + ".settings/migration-fixtures/338.json", + ".settings/migration-fixtures/339.json", + ".settings/migration-fixtures/34.json", + ".settings/migration-fixtures/340.json", + ".settings/migration-fixtures/341.json", + ".settings/migration-fixtures/342.json", + ".settings/migration-fixtures/343.json", + ".settings/migration-fixtures/344.json", + ".settings/migration-fixtures/345.json", + ".settings/migration-fixtures/346.json", + ".settings/migration-fixtures/347.json", + ".settings/migration-fixtures/348.json", + ".settings/migration-fixtures/349.json", + ".settings/migration-fixtures/35.json", + ".settings/migration-fixtures/350.json", + ".settings/migration-fixtures/351.json", + ".settings/migration-fixtures/352.json", + ".settings/migration-fixtures/353.json", + ".settings/migration-fixtures/354.json", + ".settings/migration-fixtures/355.json", + ".settings/migration-fixtures/356.json", + ".settings/migration-fixtures/357.json", + ".settings/migration-fixtures/358.json", + ".settings/migration-fixtures/359.json", + ".settings/migration-fixtures/36.json", + ".settings/migration-fixtures/360.json", + ".settings/migration-fixtures/361.json", + ".settings/migration-fixtures/362.json", + ".settings/migration-fixtures/363.json", + ".settings/migration-fixtures/364.json", + ".settings/migration-fixtures/365.json", + ".settings/migration-fixtures/366.json", + ".settings/migration-fixtures/367.json", + ".settings/migration-fixtures/368.json", + ".settings/migration-fixtures/369.json", + ".settings/migration-fixtures/37.json", + ".settings/migration-fixtures/370.json", + ".settings/migration-fixtures/371.json", + ".settings/migration-fixtures/372.json", + ".settings/migration-fixtures/373.json", + ".settings/migration-fixtures/374.json", + ".settings/migration-fixtures/375.json", + ".settings/migration-fixtures/376.json", + ".settings/migration-fixtures/377.json", + ".settings/migration-fixtures/378.json", + ".settings/migration-fixtures/379.json", + ".settings/migration-fixtures/38.json", + ".settings/migration-fixtures/380.json", + ".settings/migration-fixtures/381.json", + ".settings/migration-fixtures/382.json", + ".settings/migration-fixtures/383.json", + ".settings/migration-fixtures/384.json", + ".settings/migration-fixtures/385.json", + ".settings/migration-fixtures/386.json", + ".settings/migration-fixtures/387.json", + ".settings/migration-fixtures/388.json", + ".settings/migration-fixtures/389.json", + ".settings/migration-fixtures/39.json", + ".settings/migration-fixtures/390.json", + ".settings/migration-fixtures/391.json", + ".settings/migration-fixtures/392.json", + ".settings/migration-fixtures/393.json", + ".settings/migration-fixtures/394.json", + ".settings/migration-fixtures/395.json", + ".settings/migration-fixtures/396.json", + ".settings/migration-fixtures/397.json", + ".settings/migration-fixtures/398.json", + ".settings/migration-fixtures/399.json", + ".settings/migration-fixtures/4.json", + ".settings/migration-fixtures/40.json", + ".settings/migration-fixtures/400.json", + ".settings/migration-fixtures/401.json", + ".settings/migration-fixtures/402.json", + ".settings/migration-fixtures/403.json", + ".settings/migration-fixtures/404.json", + ".settings/migration-fixtures/405.json", + ".settings/migration-fixtures/406.json", + ".settings/migration-fixtures/407.json", + ".settings/migration-fixtures/408.json", + ".settings/migration-fixtures/409.json", + ".settings/migration-fixtures/41.json", + ".settings/migration-fixtures/410.json", + ".settings/migration-fixtures/411.json", + ".settings/migration-fixtures/412.json", + ".settings/migration-fixtures/413.json", + ".settings/migration-fixtures/414.json", + ".settings/migration-fixtures/415.json", + ".settings/migration-fixtures/416.json", + ".settings/migration-fixtures/417.json", + ".settings/migration-fixtures/418.json", + ".settings/migration-fixtures/419.json", + ".settings/migration-fixtures/42.json", + ".settings/migration-fixtures/420.json", + ".settings/migration-fixtures/421.json", + ".settings/migration-fixtures/422.json", + ".settings/migration-fixtures/423.json", + ".settings/migration-fixtures/424.json", + ".settings/migration-fixtures/425.json", + ".settings/migration-fixtures/426.json", + ".settings/migration-fixtures/427.json", + ".settings/migration-fixtures/428.json", + ".settings/migration-fixtures/429.json", + ".settings/migration-fixtures/43.json", + ".settings/migration-fixtures/430.json", + ".settings/migration-fixtures/431.json", + ".settings/migration-fixtures/432.json", + ".settings/migration-fixtures/433.json", + ".settings/migration-fixtures/434.json", + ".settings/migration-fixtures/435.json", + ".settings/migration-fixtures/436.json", + ".settings/migration-fixtures/437.json", + ".settings/migration-fixtures/438.json", + ".settings/migration-fixtures/439.json", + ".settings/migration-fixtures/44.json", + ".settings/migration-fixtures/440.json", + ".settings/migration-fixtures/441.json", + ".settings/migration-fixtures/442.json", + ".settings/migration-fixtures/443.json", + ".settings/migration-fixtures/444.json", + ".settings/migration-fixtures/445.json", + ".settings/migration-fixtures/446.json", + ".settings/migration-fixtures/447.json", + ".settings/migration-fixtures/448.json", + ".settings/migration-fixtures/449.json", + ".settings/migration-fixtures/45.json", + ".settings/migration-fixtures/450.json", + ".settings/migration-fixtures/451.json", + ".settings/migration-fixtures/452.json", + ".settings/migration-fixtures/453.json", + ".settings/migration-fixtures/454.json", + ".settings/migration-fixtures/455.json", + ".settings/migration-fixtures/456.json", + ".settings/migration-fixtures/457.json", + ".settings/migration-fixtures/458.json", + ".settings/migration-fixtures/459.json", + ".settings/migration-fixtures/46.json", + ".settings/migration-fixtures/460.json", + ".settings/migration-fixtures/461.json", + ".settings/migration-fixtures/462.json", + ".settings/migration-fixtures/463.json", + ".settings/migration-fixtures/464.json", + ".settings/migration-fixtures/465.json", + ".settings/migration-fixtures/466.json", + ".settings/migration-fixtures/467.json", + ".settings/migration-fixtures/468.json", + ".settings/migration-fixtures/469.json", + ".settings/migration-fixtures/47.json", + ".settings/migration-fixtures/470.json", + ".settings/migration-fixtures/471.json", + ".settings/migration-fixtures/472.json", + ".settings/migration-fixtures/473.json", + ".settings/migration-fixtures/474.json", + ".settings/migration-fixtures/475.json", + ".settings/migration-fixtures/476.json", + ".settings/migration-fixtures/477.json", + ".settings/migration-fixtures/478.json", + ".settings/migration-fixtures/479.json", + ".settings/migration-fixtures/48.json", + ".settings/migration-fixtures/480.json", + ".settings/migration-fixtures/481.json", + ".settings/migration-fixtures/482.json", + ".settings/migration-fixtures/483.json", + ".settings/migration-fixtures/484.json", + ".settings/migration-fixtures/485.json", + ".settings/migration-fixtures/486.json", + ".settings/migration-fixtures/487.json", + ".settings/migration-fixtures/488.json", + ".settings/migration-fixtures/489.json", + ".settings/migration-fixtures/49.json", + ".settings/migration-fixtures/490.json", + ".settings/migration-fixtures/491.json", + ".settings/migration-fixtures/492.json", + ".settings/migration-fixtures/493.json", + ".settings/migration-fixtures/494.json", + ".settings/migration-fixtures/495.json", + ".settings/migration-fixtures/496.json", + ".settings/migration-fixtures/497.json", + ".settings/migration-fixtures/498.json", + ".settings/migration-fixtures/499.json", + ".settings/migration-fixtures/5.json", + ".settings/migration-fixtures/50.json", + ".settings/migration-fixtures/51.json", + ".settings/migration-fixtures/52.json", + ".settings/migration-fixtures/53.json", + ".settings/migration-fixtures/54.json", + ".settings/migration-fixtures/55.json", + ".settings/migration-fixtures/56.json", + ".settings/migration-fixtures/57.json", + ".settings/migration-fixtures/58.json", + ".settings/migration-fixtures/59.json", + ".settings/migration-fixtures/6.json", + ".settings/migration-fixtures/60.json", + ".settings/migration-fixtures/61.json", + ".settings/migration-fixtures/62.json", + ".settings/migration-fixtures/63.json", + ".settings/migration-fixtures/64.json", + ".settings/migration-fixtures/65.json", + ".settings/migration-fixtures/66.json", + ".settings/migration-fixtures/67.json", + ".settings/migration-fixtures/68.json", + ".settings/migration-fixtures/69.json", + ".settings/migration-fixtures/7.json", + ".settings/migration-fixtures/70.json", + ".settings/migration-fixtures/71.json", + ".settings/migration-fixtures/72.json", + ".settings/migration-fixtures/73.json", + ".settings/migration-fixtures/74.json", + ".settings/migration-fixtures/75.json", + ".settings/migration-fixtures/76.json", + ".settings/migration-fixtures/77.json", + ".settings/migration-fixtures/78.json", + ".settings/migration-fixtures/79.json", + ".settings/migration-fixtures/8.json", + ".settings/migration-fixtures/80.json", + ".settings/migration-fixtures/81.json", + ".settings/migration-fixtures/82.json", + ".settings/migration-fixtures/83.json", + ".settings/migration-fixtures/84.json", + ".settings/migration-fixtures/85.json", + ".settings/migration-fixtures/86.json", + ".settings/migration-fixtures/87.json", + ".settings/migration-fixtures/88.json", + ".settings/migration-fixtures/89.json", + ".settings/migration-fixtures/9.json", + ".settings/migration-fixtures/90.json", + ".settings/migration-fixtures/91.json", + ".settings/migration-fixtures/92.json", + ".settings/migration-fixtures/93.json", + ".settings/migration-fixtures/94.json", + ".settings/migration-fixtures/95.json", + ".settings/migration-fixtures/96.json", + ".settings/migration-fixtures/97.json", + ".settings/migration-fixtures/98.json", + ".settings/migration-fixtures/99.json", + ".users/.dashboard-locations/01M75R7GF09K6AAC2SFQVDMCY3.json", + ".users/.dashboard-titles/2b9aa3aaf18aa9aa60fcc45d30833be66dc5907be190dfff57197c19883297e4.json", + ".users/8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918/dashboards/01M75R7GF09K6AAC2SFQVDMCY3.json", + ".users/8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918/filters/first/01M75R7GFTZ5J1962A6DH7F76H.json", + "first/.stream/.schema", + "first/.stream/.stream.json", + "first/date=2026-11-11/.manifest.json", + "first/date=2026-11-11/hour=16/minute=49/.data.01M75R8CC3NZ3FYFDNS7SYHNC4.parquet", + "first/date=2026-11-11/hour=16/minute=51/.data.01M75RC1EGSNZ3A495S8YYPXQS.parquet", + "partitioned/.stream/.schema", + "partitioned/.stream/.stream.json", + "partitioned/date=2026-11-11/.manifest.json", + "partitioned/date=2026-11-11/hour=16/minute=49/region=west/.data.01M75R8CC3M7ENK36HXV3K5Z40.parquet", + "pstats/.stream/.schema", + "pstats/.stream/.stream.json", + "pstats/date=2026-11-11/.manifest.json", + "pstats/date=2026-11-11/hour=16/minute=49/dataset_name=first/.data.01M75RA6Z3CTNEVGMEAQ4YX6MZ.parquet", + "pstats/date=2026-11-11/hour=16/minute=49/dataset_name=partitioned/.data.01M75RA6Z3CTNEVGMEAQ4YX6MZ.parquet", + "pstats/date=2026-11-11/hour=16/minute=49/dataset_name=second/.data.01M75RA6Z3CTNEVGMEAQ4YX6MZ.parquet", + "pstats/date=2026-11-11/hour=16/minute=51/dataset_name=first/.data.01M75RDW1HJ9H57GDBS6TY1PHN.parquet", + "second/.stream/.schema", + "second/.stream/.stream.json", + "second/date=2026-11-11/.manifest.json", + "second/date=2026-11-11/hour=16/minute=49/.data.01M75R8CC3AGT026GHYTM8M2CJ.parquet" + ] +} diff --git a/docs/validation/postgres-round2-checks.json b/docs/validation/postgres-round2-checks.json new file mode 100644 index 000000000..63560960a --- /dev/null +++ b/docs/validation/postgres-round2-checks.json @@ -0,0 +1,155 @@ +{ + "success": true, + "base_commit": "b96259b05238bae99ca48d5bee3cee9943b399d5", + "checks": [ + { + "name": "full-library-postgres-minio", + "command": "cargo test --locked --lib -- --test-threads=1", + "passed": 1405, + "failed": 0, + "ignored": 7 + }, + { + "name": "full-library-no-database", + "command": "cargo test --locked --lib -- --nocapture --test-threads=1", + "prerequisites_unset": [ + "P_TEST_POSTGRES_URL", + "P_TEST_REVIEW_S3_URL" + ], + "visible_skips": true, + "passed": 1405, + "failed": 0, + "ignored": 7 + }, + { + "name": "contracts-and-startup", + "command": "cargo test --locked --lib contract_tests:: -- --include-ignored --skip startup_cleanup_child --nocapture --test-threads=1", + "passed": 13, + "failed": 0, + "ignored": 0 + }, + { + "name": "focused-metastore", + "command": "cargo test --locked --lib metastore:: -- --nocapture --test-threads=1", + "passed": 37, + "failed": 0, + "ignored": 3 + }, + { + "name": "localfs-migration", + "command": "cargo test --lib migration::tests::migration_round_trip_postgres -- --nocapture", + "prerequisites_unset": [ + "P_TEST_REVIEW_S3_URL" + ], + "passed": 1, + "failed": 0, + "ignored": 0 + }, + { + "name": "paused-postgres-deadline", + "command": "cargo test --lib postgres_health_deadline_regression -- --nocapture --test-threads=1", + "passed": 1, + "failed": 0, + "ignored": 0 + }, + { + "name": "upload-sibling-drain", + "passed": 1, + "failed": 0, + "ignored": 0 + }, + { + "name": "build", + "command": "cargo build", + "exit_code": 0 + }, + { + "name": "smoke", + "command": "python3 scripts/postgres-metastore-smoke.py", + "modes": [ + "object-store", + "postgres", + "postgres-tls" + ], + "exit_code": 0, + "evidence": "docs/validation/postgres-metastore-smoke.json" + }, + { + "name": "migration-smoke", + "command": "python3 scripts/postgres-metastore-migration-smoke.py", + "modes": [ + "plaintext", + "verified-tls" + ], + "exit_code": 0, + "evidence": [ + "docs/validation/postgres-migration.json", + "docs/validation/postgres-migration-tls.json" + ] + }, + { + "name": "clippy", + "command": "cargo clippy --all-targets", + "exit_code": 0, + "new_warnings": 0, + "existing_library_test_warnings": 21 + }, + { + "name": "script-syntax-and-workflow-yaml", + "exit_code": 0 + }, + { + "name": "diff-whitespace", + "command": "git diff --check b96259b0", + "exit_code": 0 + } + ], + "first_hosted_ci": { + "conclusion": "failure", + "headSha": "b96259b05238bae99ca48d5bee3cee9943b399d5", + "status": "completed", + "url": "https://github.com/VibeEngineeringAI/parseable/actions/runs/43037438052", + "reason": "Runner setup could not resolve Ubuntu package mirror; compilation and tests never started.", + "code_failure": false, + "expanded_workflow_pushed": false + }, + "regression_fixture_note": "Functional metadata SQL tests use a 30-second budget after intermittent shared-host I/O timeouts. Dedicated short deadline and cancellation assertions retain their original budgets.", + "import_estimates": { + "assumptions": [ + "30 days", + "one manifest per stream/day", + "two stream documents per stream", + "two inventories", + "two metadata-body reads", + "snapshot-reference validation reads", + "150 milliseconds per request", + "16 concurrent reads" + ], + "deployments": [ + { + "streams": 20, + "requests": 3180, + "latency_only_lower_bound_seconds": 29.8125 + }, + { + "streams": 1000, + "requests": 159000, + "latency_only_lower_bound_seconds": 1490.625 + } + ], + "limitations": "Estimates, not benchmarks. Discovery dependencies, pagination, SQL, disk and body transfer add time." + }, + "uploader_policy": "Only after confirming PostgreSQL deletion pending: settle upload siblings, compensate remote parquet for the batch, then retire staged parquet and schema. Retain staging when deletion is unconfirmed or cleanup fails.", + "remaining_limits": [ + "Publication delayed beyond final SQL deletion requires stream incarnation fencing (issue 142).", + "SQL cancellation is best effort; timed-out writes and COMMIT can have ambiguous outcomes.", + "Transfers require all writers stopped and a direct or session-pooled PostgreSQL connection.", + "Literal EXPLAIN does not establish generic prepared-plan performance." + ], + "redaction": { + "home_paths": 0, + "checkout_paths": 0, + "hostname_occurrences": 0 + }, + "containers_removed": true +} diff --git a/scripts/postgres-metastore-migration-smoke.py b/scripts/postgres-metastore-migration-smoke.py index c94e45592..ad205f24a 100755 --- a/scripts/postgres-metastore-migration-smoke.py +++ b/scripts/postgres-metastore-migration-smoke.py @@ -10,6 +10,7 @@ import base64 import datetime as dt import urllib.request +import signal # Load only reusable definitions, never the existing smoke's executable driver. namespace = {'__file__': str(Path(__file__).with_name('postgres-metastore-smoke.py'))} @@ -19,7 +20,8 @@ root, out = namespace['ROOT'], namespace['OUT'] binary = str(Path(os.environ.get('CARGO_TARGET_DIR', root/'target'))/'debug/parseable') server = None -log = open(out/'migration.log','w') +transfer_processes = [] +log = open(out / ('migration-tls.log' if os.environ.get('P_MIGRATION_TLS') == '1' else 'migration-plain.log'), 'w') def stop(): global server if server: @@ -27,6 +29,8 @@ def stop(): try: server.wait(timeout=30) except sp.TimeoutExpired: server.kill(); server.wait() server = None +signal.signal(signal.SIGTERM, namespace['interrupted']) +signal.signal(signal.SIGINT, namespace['interrupted']) try: pgport, s3port, http = port(), port(), port() pg, cmd = container('postgres:17-alpine',POSTGRES_PASSWORD='smoke',POSTGRES_DB='migration') @@ -34,11 +38,13 @@ def stop(): mini, cmd = container('quay.io/parseablehq/minio:edge',MINIO_ROOT_USER='minioadmin',MINIO_ROOT_PASSWORD='minioadmin') run(*cmd,'-p',f'127.0.0.1:{s3port}:9000','quay.io/parseablehq/minio:edge','server','/data') run('docker','exec',mini,'mkdir','-p','/data/migration') - wait_for(lambda:'accepting connections' in run('docker','exec',pg,'pg_isready','-U','postgres')) + wait_for(lambda:'accepting connections' in run('docker','exec',pg,'pg_isready','-h','127.0.0.1','-U','postgres')) + wait_for(lambda: urllib.request.urlopen(f'http://127.0.0.1:{s3port}/minio/health/ready', timeout=3).status == 200) env = dict(os.environ,P_ADDR=f'127.0.0.1:{http}',P_GRPC_PORT=str(port()),P_FLIGHT_PORT=str(port()), P_STAGING_DIR=str(out/f'migration-staging-{os.getpid()}'),P_USERNAME='admin',P_PASSWORD='smoke-pass', P_S3_URL=f'http://127.0.0.1:{s3port}',P_S3_BUCKET='migration',P_S3_REGION='us-east-1', - P_S3_ACCESS_KEY='minioadmin',P_S3_SECRET_KEY='minioadmin',P_S3_PATH_STYLE='true',RUST_LOG='warn') + P_S3_ACCESS_KEY='minioadmin',P_S3_SECRET_KEY='minioadmin',P_S3_PATH_STYLE='true',RUST_LOG='warn', + P_METASTORE_POSTGRES_STATEMENT_TIMEOUT_MS='30000') env.pop('P_METASTORE_POSTGRES_URL',None) pgurl = f'postgresql://postgres:smoke@127.0.0.1:{pgport}/migration?sslmode=disable' if os.environ.get('P_MIGRATION_TLS') == '1': @@ -86,7 +92,26 @@ def drained(): def migrate(action,*flags,expected=0): result = sp.run([binary,'s3-store','--metadata-migration',action,'--acknowledge-server-stopped',*flags], env=dict(env,P_METASTORE_POSTGRES_URL=pgurl),stdout=log,stderr=log,timeout=180) + namespace['commands'].append(dict(argv=result.args, exit_code=result.returncode, expected_success=expected == 0)) assert (result.returncode == 0) == (expected == 0), (action,result.returncode) + def sql(statement): + return run('docker','exec',pg,'psql','-U','postgres','-d','migration','-qAt','-c',statement) + def refuse(label, postgres, overrides=None): + attempt_env = dict(env) + attempt_env.pop('P_METASTORE_POSTGRES_URL', None) + if postgres: attempt_env['P_METASTORE_POSTGRES_URL'] = pgurl + attempt_env.update(overrides or {}) + rejected = sp.run([binary,'s3-store'],env=attempt_env,text=True,stdout=sp.PIPE,stderr=sp.STDOUT,timeout=30) + log.write(rejected.stdout); log.flush() + assert rejected.returncode != 0, (label, postgres) + assert any(word in rejected.stdout.lower() for word in ['migration', 'exported', 'paired']), rejected.stdout + namespace['commands'].append(dict(argv=rejected.args, exit_code=rejected.returncode, refusal_state=label, postgres=postgres)) + def refuse_both(label): + for postgres in [False, True]: refuse(label, postgres) + def object_write(key, body): + return namespace['get_key'](env['P_S3_URL'],'migration',key,body=body) + def object_delete(key): + return namespace['get_key'](env['P_S3_URL'],'migration',key,method='DELETE') start() for stream in ['first','second','partitioned']: request('/logstream/'+stream,'PUT',headers={'X-P-Custom-Partition':'region'} if stream=='partitioned' else None) @@ -127,24 +152,52 @@ def snapshot(path): print('object-store fixture uploaded; starting offline import',flush=True) migrate('verify',expected=1) + missing_env = dict(env, P_METASTORE_POSTGRES_URL=pgurl, P_STAGING_DIR='missing-relative-staging') + attempt = sp.run([binary, 's3-store', '--metadata-migration', 'import', '--dry-run', '--acknowledge-server-stopped'], + cwd=out, env=missing_env, text=True, stdout=sp.PIPE, stderr=sp.STDOUT, timeout=30) + assert attempt.returncode != 0 and 'staging directory must exist' in attempt.stdout, attempt.stdout + config_dir = out / 'migration-env-fixture' + config_dir.mkdir(exist_ok=True) + (config_dir / 'staging').mkdir(exist_ok=True) + (config_dir / '.parseable.env').write_text('P_STAGING_DIR=./staging\n') + fixture_env = dict(env, P_METASTORE_POSTGRES_URL=pgurl) + fixture_env.pop('P_STAGING_DIR') + attempt = sp.run([binary, 's3-store', '--metadata-migration', 'import', '--dry-run', '--acknowledge-server-stopped'], + cwd=config_dir, env=fixture_env, text=True, stdout=sp.PIPE, stderr=sp.STDOUT, timeout=30) + assert attempt.returncode == 0, attempt.stdout + assert 'Resolved staging directory: ' + str((config_dir / 'staging').resolve()) in attempt.stdout, attempt.stdout + assert 'Environment file: ' + str(config_dir / '.parseable.env') in attempt.stdout, attempt.stdout + namespace['commands'].append(dict(argv=attempt.args, cwd=str(config_dir), exit_code=attempt.returncode, env_file_checked=True)) migrate('import','--dry-run') # Upload actual S3 metadata objects so interruption is observable on MinIO's XL layout. for i in range(500): namespace['get_key'](env['P_S3_URL'], 'migration', f'.settings/migration-fixtures/{i}.json', body=b'{}') interrupted=sp.Popen([binary,'s3-store','--metadata-migration','import','--acknowledge-server-stopped'], env=dict(env,P_METASTORE_POSTGRES_URL=pgurl),stdout=log,stderr=log) + transfer_processes.append(interrupted) deadline=time.monotonic()+120 while '.parseable/metastore-migration' not in list_keys(env['P_S3_URL'],'migration'): assert interrupted.poll() is None, 'import completed before interruption' assert time.monotonic()')] + hostname = socket.gethostname() + host_forms = {hostname, ''.join(c for c in hostname if c.isalnum() or c in '-_')} + replacements.extend((host, '') for host in sorted(host_forms, key=len, reverse=True) if host) + target = os.environ.get('CARGO_TARGET_DIR') + if target: + replacements.insert(0, (target, 'target')) + for original, replacement in replacements: + text = text.replace(original, replacement) + return text + def run(*args): attempt = sp.run(args, text=True, stdout=sp.PIPE, stderr=sp.PIPE) commands.append(dict(argv=list(args), exit_code=attempt.returncode)) @@ -85,13 +101,20 @@ def cleanup(): auxiliary = None stop() finally: + if sys.exc_info()[0] is not None: + results['container_logs'] = {} + for name in containers: + attempt = sp.run(['docker','logs',name],text=True,stdout=sp.PIPE,stderr=sp.STDOUT,timeout=10) + results['container_logs'][name] = attempt.stdout for name in reversed(containers): - attempt = sp.run(['docker', 'rm', '-f', name], stdout=sp.DEVNULL, stderr=sp.DEVNULL) - commands.append(dict(argv=['docker', 'rm', '-f', name], exit_code=attempt.returncode)) + attempt = sp.run(['docker', 'rm', '-fv', name], stdout=sp.DEVNULL, stderr=sp.DEVNULL) + commands.append(dict(argv=['docker', 'rm', '-fv', name], exit_code=attempt.returncode)) containers.clear() for process in processes: commands.append(dict(argv=process.args, exit_code=process.poll())) processes.clear() + for path in OUT.glob('*.log'): + path.write_text(redacted_text(path.read_text(errors='replace'))) results['commands'] = commands results['success'] = sys.exc_info()[0] is None if sys.exc_info()[0] is not None: @@ -99,11 +122,11 @@ def cleanup(): if (OUT/'checks.json').exists(): results['validation'] = json.loads((OUT/'checks.json').read_text()) results['http_requests'] = http_requests - evidence = json.dumps(results, indent=2) + '\n' + evidence = redacted_json(results) + '\n' (OUT/'results.json').write_text(evidence) (ARCHIVE/(hashlib.sha256(evidence.encode()).hexdigest()+'.json')).write_text(evidence) runs = [json.loads(path.read_text()) for path in sorted(ARCHIVE.glob('*.json'))] - (ROOT/'docs/validation/postgres-metastore-smoke.json').write_text(json.dumps({'runs': runs}, indent=2)+'\n') + (ROOT/'docs/validation/postgres-metastore-smoke.json').write_text(redacted_json({'runs': runs})+'\n') def interrupted(signum, frame): raise SystemExit(128 + signum) @@ -160,12 +183,12 @@ def list_keys(endpoint, bucket): assert root.findtext('{*}IsTruncated') != 'true', 'smoke fixture exceeded a listing page' return sorted(node.text for node in root.findall('{*}Contents/{*}Key')) -def get_key(endpoint, bucket, key, body=None): +def get_key(endpoint, bucket, key, body=None, method=None): """Read publication documents with SigV4 without an SDK dependency.""" now = dt.datetime.now(dt.timezone.utc) stamp, day = now.strftime('%Y%m%dT%H%M%SZ'), now.strftime('%Y%m%d') host = endpoint.removeprefix('http://') - method = 'GET' if body is None else 'PUT' + method = method or ('GET' if body is None else 'PUT') payload = hashlib.sha256(body or b'').hexdigest() headers = f'host:{host}\nx-amz-content-sha256:{payload}\nx-amz-date:{stamp}\n' signed = 'host;x-amz-content-sha256;x-amz-date' @@ -215,6 +238,7 @@ def wait_for(check, seconds=180): mini, cmd = container('quay.io/parseablehq/minio:edge', MINIO_ROOT_USER='minioadmin', MINIO_ROOT_PASSWORD='minioadmin') run(*cmd, '-p', f'127.0.0.1:{s3port}:9000', 'quay.io/parseablehq/minio:edge', 'server', '/data') wait_for(lambda: 'accepting connections' in run('docker', 'exec', pg, 'pg_isready', '-h', '127.0.0.1', '-U', 'postgres')) + wait_for(lambda: urllib.request.urlopen(f'http://127.0.0.1:{s3port}/minio/health/ready', timeout=3).status == 200) # An isolated CA and signed localhost server certificate; no host config changes. tls = OUT / 'tls' tls.mkdir(exist_ok=True) @@ -242,7 +266,8 @@ def wait_for(check, seconds=180): env = dict(os.environ, P_ADDR=f'127.0.0.1:{http}', P_GRPC_PORT=str(grpc), P_FLIGHT_PORT=str(flight), P_STAGING_DIR=str(OUT / (mode + '-staging-' + str(os.getpid()))), P_USERNAME='admin', P_PASSWORD='smoke-pass', P_S3_URL=f'http://127.0.0.1:{s3port}', P_S3_BUCKET=bucket, P_S3_REGION='us-east-1', - P_S3_ACCESS_KEY='minioadmin', P_S3_SECRET_KEY='minioadmin', P_S3_PATH_STYLE='true', RUST_LOG='warn') + P_S3_ACCESS_KEY='minioadmin', P_S3_SECRET_KEY='minioadmin', P_S3_PATH_STYLE='true', RUST_LOG='warn', + P_METASTORE_POSTGRES_STATEMENT_TIMEOUT_MS='30000') env.pop('P_METASTORE_POSTGRES_URL', None) if mode.startswith('postgres'): env['P_METASTORE_POSTGRES_URL'] = f'postgresql://postgres:smoke@127.0.0.1:{pgport}/parseable?sslmode=disable' @@ -255,7 +280,7 @@ def start(): global server server = sp.Popen([str(Path(os.environ.get('CARGO_TARGET_DIR', ROOT / 'target')) / 'debug/parseable'), 's3-store'], env=env, cwd=ROOT, stdout=log, stderr=log) processes.append(server) - def request(path, method='GET', body=None, headers=None, timeout=30): + def request(path, method='GET', body=None, headers=None, timeout=90): data = None if body is None else json.dumps(body).encode() req = urllib.request.Request(base + path, data=data, method=method, headers={'Authorization': 'Basic ' + base64.b64encode(b'admin:smoke-pass').decode(), @@ -375,6 +400,16 @@ def counts(): request_delta=delta, observed_requests_during_window_divided_by_queries=sum(delta.values())/5)) if mode.startswith('postgres'): assert results[mode]['non_parquet_keys'] == ['.parseable/postgres-metastore-id'] + database = 'tls' if mode == 'postgres-tls' else 'parseable' + sql = lambda text: run('docker', 'exec', pg, 'psql', '-v', 'ON_ERROR_STOP=1', '-U', 'postgres', '-d', database, '-Atc', text) + sql("INSERT INTO parseable_documents(tenant,key,kind,body) VALUES('','deletion-explicit/.stream/.deleting','document',''::bytea);") + try: + aux_request('/logstream/deletion-explicit', 'POST', [{'value':1}]) + except urllib.error.HTTPError as error: + assert error.code == 409, error.code + else: + raise AssertionError('explicit-stream ingest admitted a pending deletion') + sql("DELETE FROM parseable_documents WHERE tenant='' AND key='deletion-explicit/.stream/.deleting';") run('docker', 'stop', pg) started = time.monotonic() assert status('/readiness') == 503 @@ -382,6 +417,13 @@ def counts(): elapsed = time.monotonic()-started assert elapsed < 10, elapsed results[mode]['database_outage'] = dict(readiness=503, request_failed=True, seconds=elapsed) + try: + aux_request('/logstream/outage-explicit', 'POST', [{'value':1}]) + except urllib.error.HTTPError as error: + assert error.code == 503, error.code + else: + raise AssertionError('explicit-stream ingest hid a database outage') + results[mode]['explicit_stream_errors'] = dict(pending_deletion=409, database_outage=503) run('docker', 'start', pg) wait_for(lambda: status('/readiness') == 200) assert json.loads(request('/query', 'POST', query)) == json.loads(second) @@ -451,7 +493,7 @@ def counts(): rejected_env = env.copy() if case == 'future-schema': run('docker', 'exec', pg, 'psql', '-U', 'postgres', '-d', database, '-c', - 'UPDATE parseable_backend SET schema_version=3;') + 'UPDATE parseable_backend SET schema_version=999;') elif case == 'wrong-id': rejected_env['P_METASTORE_POSTGRES_ID'] = 'wrong-backend' else: @@ -463,12 +505,12 @@ def counts(): failures[case] = attempt.stdout if case == 'future-schema': run('docker', 'exec', pg, 'psql', '-U', 'postgres', '-d', database, '-c', - 'UPDATE parseable_backend SET schema_version=2;') + 'UPDATE parseable_backend SET schema_version=3;') results['startup_rejections'] = failures results['postgres_rows'] = run('docker','exec',pg,'psql','-U','postgres','-d',database,'-c', 'SELECT tenant,kind,key,version,octet_length(body) AS bytes FROM parseable_documents ORDER BY tenant,key;') - (OUT/'results.json').write_text(json.dumps(results,indent=2)) - print(json.dumps(results,indent=2)) + (OUT/'results.json').write_text(redacted_json(results)) + print(redacted_json(results)) finally: cleanup() atexit.unregister(cleanup) diff --git a/src/metastore/metastores/migration.rs b/src/metastore/metastores/migration.rs index 1f8238243..0f8a6b672 100644 --- a/src/metastore/metastores/migration.rs +++ b/src/metastore/metastores/migration.rs @@ -227,6 +227,12 @@ pub async fn run( "requires --acknowledge-server-stopped; stop ALL servers and writers first" ); eprintln!("OFFLINE migration: all servers must remain stopped. No writer fencing exists."); + #[cfg(test)] + eprintln!( + "TEST transfer: action={action:?} dry_run={} abort={} injection={:?}", + opts.migration_dry_run, opts.abort, opts.injected_failure + ); + let mut transfer_lock = db.transfer_lock().await?; db.migration_preflight(store.as_ref(), opts).await?; if opts.abort { @@ -640,6 +646,7 @@ mod tests { }; let options = PostgresOptions { metastore_postgres_url: Some(url.clone()), + metastore_postgres_statement_timeout_ms: 30000, ..Default::default() }; let admin = PostgresDocuments::new(&options, "test".into(), store.clone())?; @@ -655,6 +662,7 @@ mod tests { let opts = PostgresOptions { metastore_postgres_url: Some(url.to_string()), metastore_postgres_pool_size: 1, + metastore_postgres_statement_timeout_ms: 30000, ..Default::default() }; let db = PostgresDocuments::new(&opts, "test".into(), store.clone())?; diff --git a/src/metastore/metastores/postgres_documents.rs b/src/metastore/metastores/postgres_documents.rs index 0326b7ee3..40042563a 100644 --- a/src/metastore/metastores/postgres_documents.rs +++ b/src/metastore/metastores/postgres_documents.rs @@ -1055,24 +1055,34 @@ impl PostgresDocuments { .await .unwrap(); } - let client = self.client().await.unwrap(); + let timed = Self::new( + &PostgresOptions { + metastore_postgres_connect_timeout_ms: 1000, + metastore_postgres_statement_timeout_ms: 3000, + ..options.clone() + }, + self.bucket.clone(), + self.storage.clone(), + ) + .unwrap(); + let client = timed.client().await.unwrap(); let started = std::time::Instant::now(); assert!(client.query("SELECT pg_sleep(10)", &[]).await.is_err()); assert!(started.elapsed() < Duration::from_secs(5)); assert!(client.query("SELECT 1", &[]).await.is_ok()); drop(client); // Pool saturation is bounded and returned as an error. - let held = futures::future::join_all((0..4).map(|_| self.client())).await; + let held = futures::future::join_all((0..4).map(|_| timed.client())).await; assert!(held.iter().all(Result::is_ok)); let started = std::time::Instant::now(); - assert!(self.client().await.is_err()); + assert!(timed.client().await.is_err()); assert!(started.elapsed() < Duration::from_secs(3)); drop(held); let marker = RelativePath::new(".parseable/postgres-metastore-id"); let paired = self.storage.get_object(marker, &None).await.unwrap(); self.storage.delete_object(marker, &None).await.unwrap(); let started = std::time::Instant::now(); - assert!(self.check_health(&None).await.is_err()); + assert!(timed.check_health(&None).await.is_err()); assert!(started.elapsed() < Duration::from_secs(3)); self.storage .put_object(marker, Bytes::from_static(b"wrong-backend"), &None) diff --git a/src/metastore/metastores/postgres_metastore.rs b/src/metastore/metastores/postgres_metastore.rs index 6854f73b8..c85287530 100644 --- a/src/metastore/metastores/postgres_metastore.rs +++ b/src/metastore/metastores/postgres_metastore.rs @@ -1681,8 +1681,8 @@ mod contract_tests { let options = PostgresOptions { metastore_postgres_url: Some(url), metastore_postgres_pool_size: 4, - metastore_postgres_connect_timeout_ms: 1000, - metastore_postgres_statement_timeout_ms: 3000, + metastore_postgres_connect_timeout_ms: 5000, + metastore_postgres_statement_timeout_ms: 30000, metastore_postgres_id: None, metastore_postgres_ca: None, }; From 4dce91a15f339d81e3fd2fea0d7f47effb68b4f7 Mon Sep 17 00:00:00 2001 From: VibeSWE <338779468+VibeSWE@users.noreply.github.com> Date: Wed, 11 Nov 2026 11:48:20 -0600 Subject: [PATCH 8/9] ci: retry the MinIO readiness probe on every transient error The probe retried only refused connections. While MinIO is still starting, the published port accepts and then resets the connection, which curl reports as error 56 and does not retry without --retry-all-errors. The first hosted run of this workflow that got past setup failed there. Co-Authored-By: Claude Fable 5.2 --- .github/workflows/postgres-metastore.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/postgres-metastore.yaml b/.github/workflows/postgres-metastore.yaml index 5e17fe2e3..1a554aa81 100644 --- a/.github/workflows/postgres-metastore.yaml +++ b/.github/workflows/postgres-metastore.yaml @@ -51,7 +51,7 @@ jobs: run: | docker run -d --name migration-minio -p 127.0.0.1:59087:9000 -e MINIO_ROOT_USER=minioadmin -e MINIO_ROOT_PASSWORD=minioadmin quay.io/parseablehq/minio:edge server /data docker exec migration-minio mkdir -p /data/review /data/migration-unit - curl --fail --retry 30 --retry-connrefused --retry-delay 1 http://127.0.0.1:59087/minio/health/ready + curl --fail --retry 30 --retry-all-errors --retry-delay 1 http://127.0.0.1:59087/minio/health/ready - name: Migration and regressions on PostgreSQL 17 and MinIO run: 'cargo test --locked --lib metastore:: -- --nocapture --test-threads=1' - name: Full library suite with PostgreSQL From 6a57dac59d23f1a93fc7c1c588b06d0f05472405 Mon Sep 17 00:00:00 2001 From: VibeSWE <338779468+VibeSWE@users.noreply.github.com> Date: Wed, 11 Nov 2026 13:37:10 -0600 Subject: [PATCH 9/9] style: format the carried dataset probe test Co-Authored-By: Claude Fable 5.2 --- .../metastores/object_store_metastore.rs | 31 ++++++++++--------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/src/metastore/metastores/object_store_metastore.rs b/src/metastore/metastores/object_store_metastore.rs index 9ba1c01b7..d74ff6a6c 100644 --- a/src/metastore/metastores/object_store_metastore.rs +++ b/src/metastore/metastores/object_store_metastore.rs @@ -184,22 +184,23 @@ mod tests { for fail in [false, true] { let active = AtomicUsize::new(0); let peak = AtomicUsize::new(0); - let result = super::super::document_metastore::collect_dataset_probes((0..64).map(|i| { - let active = &active; - let peak = &peak; - async move { - let n = active.fetch_add(1, Ordering::SeqCst) + 1; - peak.fetch_max(n, Ordering::SeqCst); - tokio::time::sleep(std::time::Duration::from_millis(5)).await; - active.fetch_sub(1, Ordering::SeqCst); - if fail && i == 25 { - Err("failed probe") - } else { - Ok(i) + let result = + super::super::document_metastore::collect_dataset_probes((0..64).map(|i| { + let active = &active; + let peak = &peak; + async move { + let n = active.fetch_add(1, Ordering::SeqCst) + 1; + peak.fetch_max(n, Ordering::SeqCst); + tokio::time::sleep(std::time::Duration::from_millis(5)).await; + active.fetch_sub(1, Ordering::SeqCst); + if fail && i == 25 { + Err("failed probe") + } else { + Ok(i) + } } - } - })) - .await; + })) + .await; assert_eq!(peak.load(Ordering::SeqCst), 16); if fail { assert_eq!(result, Err("failed probe"));