From 4de5cc32d4669b5c0f8c4e77e91d437d64eb02e2 Mon Sep 17 00:00:00 2001 From: Jorge Rivera Date: Thu, 29 Oct 2026 16:39:04 +0100 Subject: [PATCH 1/2] fix(client-contract): ask the file selector only for the keys it reads A file-selector rule names format, bundle and layout; the validator, capture tool and selector scenarios derive the query from it. The validator compares values by one rule and reports malformed vectors, and the base-URL and client-identity rules close the cases where Python and WHATWG clients disagreed. --- CONTRIBUTING.md | 2 +- python/client-contract/CHANGELOG.md | 14 +- python/client-contract/README.md | 12 +- .../src/dfo_client_contract/contract.py | 1 + .../src/dfo_client_contract/rules/README.md | 14 +- .../dfo_client_contract/rules/base-url.json | 78 ++++ .../rules/client-identity.json | 11 +- .../rules/file-selector.json | 47 +++ .../rules/reading-order.json | 47 +++ .../rules/rules.schema.json | 62 ++- .../dfo_client_contract/scenario.schema.json | 6 +- .../choose-refused-label-in-two-groups.json | 395 ++++++++++++++++++ .../fetch-ref-artifact-unlisted.json | 2 +- .../fetch-ref-expired-file-selector.json | 2 +- .../scenarios/mint-async-build.json | 3 +- python/client-contract/tests/test_rules.py | 50 ++- python/spec-kit/CHANGELOG.md | 7 +- tools/README.md | 4 +- tools/capture_client_contract.py | 168 ++++++-- tools/tests/test_capture_client_contract.py | 105 ++++- tools/tests/test_validate_client_contract.py | 158 ++++++- tools/validate_client_contract.py | 178 ++++++-- 22 files changed, 1245 insertions(+), 121 deletions(-) create mode 100644 python/client-contract/src/dfo_client_contract/rules/file-selector.json create mode 100644 python/client-contract/src/dfo_client_contract/scenarios/choose-refused-label-in-two-groups.json diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 576963f..5468804 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -41,7 +41,7 @@ uv run tools/validate_unit_vectors.py uv run tools/validate_provenance_docs_vectors.py uv run tools/validate_descriptions_vectors.py uv run tools/validate_client_contract.py -uv run --with jsonschema --with pytest pytest tools/tests -q +uv run --with jsonschema --with pyarrow --with pytest pytest tools/tests -q ``` The last command tests the validators themselves. [tools/README.md](tools/README.md) says what each validator checks. Each package's tests drive its implementation against the same vectors or fixtures, and `uv run pytest` runs them. diff --git a/python/client-contract/CHANGELOG.md b/python/client-contract/CHANGELOG.md index 63f5868..9928610 100644 --- a/python/client-contract/CHANGELOG.md +++ b/python/client-contract/CHANGELOG.md @@ -11,7 +11,7 @@ All notable changes to the `dfo-spec-kit-client-contract` Python package will be - `strict-fields.json` and its schema: 29 members that stay required and non-nullable, each one a client reads, and the answer kinds, part statuses and operation statuses the clients branch on. - `actions.json` and its schema: the caller's next step for each of the 104 active refusal codes on `problem_details` or `no_data_reason`, with the gap kind of each no_data reason. - The status fallback for a refusal whose code has no row: 400 and 422 to `InvalidRequestError`, 401 to `AuthenticationError`, 403 to `PermissionDeniedError`, 404 and 410 to `NotFoundError`, 408 to `DfoTimeoutError`, 429 to `RateLimitError`, and any other 4xx or 5xx to `InvalidRequestError` or `ServerError`. -- `scenario.schema.json`, the scenario record, and 124 scenarios: 34 recorded against the deployed dfo-apis dev app at `d80df9f5f`, and 90 synthetic, each derived from a recorded one by a one-line edit, with 21 binary bodies totalling 1,134,603 bytes and records and bodies together 13,680,926 bytes. `search-place-ambiguous-twice` pins two entity_ambiguous groups of one text, each option and slot token naming its group by `entity_choice`. The scenarios cover the several-part choice entries and the assumptions each identity entry echoes, a picked reading whose assumption Search states in its answer, the place choice through a no_data part's options, the choice refusals, a part without variable_text, an unknown status without a restatement, a shared label, a one-part option whose data_query holds two parts, the separate and operations echo, an unsupported part's placeholder, the timed_out, unsupported and unknown gaps, an unknown answer kind, the `SearchWarning` rule, the stacked dataset's `part` column, the frame layout, a strict fetch's `DataWarning`, a spec fetch that rebinds one of two entity roles and keeps the other, one that drops a role bound to null, a spec whose destination is a named group fetched with a window or with its origin rebound, whose sentence lookup posts the groups of the roles the build left bound, a twice cut-off build, a cut-off build retried after its Retry-After, a build refused 504 then 503 and one refused 503 then 504, a refused build, a build refused 501 `capability_unavailable`, a digest mismatch, a signed URL refused on both rounds, a relative files-route Location, an artifact url on a foreign origin, on the storage host and absolute on the API's origin, a manifest definition without identity or schema_version, a Search spec without identity or resolved, nested specs no rule reads, an empty parts list, a Search body that is not an object, a codeless 503 problem document that is not retryable, a 401, an identity resubmit answering another spec, a part whose status is not a string, a Search spec whose definition names another spec_id, a Search response with no JSON body, an option with no data_query, empty entities and facets, a Sunset header issued once, `x-dfo-warning` messages and a Deprecation and Sunset pair, an unreadable versions page, a mint and a spec fetch whose build answers 202 and is polled to done, a resumed operation, a rate-limited poll, 503 polls retried within the budget an answered poll restores, a 202 that reads running, polls whose `retry_after_ms` changes from one to the next, a 404 poll that raises, a failed operation posted again, a failed operation with no error, and a ref whose stored file expired, read through the file selector, rebuilt at the same version, at a newer one, or refused with the newest version read from the versions route: the first version it lists with no `base_content_version`, the newest plain build, so a derived version listed before it is skipped. +- `scenario.schema.json`, the scenario record, and 125 scenarios: 34 recorded against the deployed dfo-apis dev app at `d80df9f5f`, and 91 synthetic, each derived from a recorded one by a one-line edit, with 21 binary bodies totalling 1,134,603 bytes and records and bodies together 13,693,847 bytes. `search-place-ambiguous-twice` pins two entity_ambiguous groups of one text, each option and slot token naming its group by `entity_choice`, and `choose-refused-label-in-two-groups` pins that a label both groups offer matches two options, so a choice there names its option by index. The scenarios cover the several-part choice entries and the assumptions each identity entry echoes, a picked reading whose assumption Search states in its answer, the place choice through a no_data part's options, the choice refusals, a part without variable_text, an unknown status without a restatement, a shared label, a one-part option whose data_query holds two parts, the separate and operations echo, an unsupported part's placeholder, the timed_out, unsupported and unknown gaps, an unknown answer kind, the `SearchWarning` rule, the stacked dataset's `part` column, the frame layout, a strict fetch's `DataWarning`, a spec fetch that rebinds one of two entity roles and keeps the other, one that drops a role bound to null, a spec whose destination is a named group fetched with a window or with its origin rebound, whose sentence lookup posts the groups of the roles the build left bound, a twice cut-off build, a cut-off build retried after its Retry-After, a build refused 504 then 503 and one refused 503 then 504, a refused build, a build refused 501 `capability_unavailable`, a digest mismatch, a signed URL refused on both rounds, a relative files-route Location, an artifact url on a foreign origin, on the storage host and absolute on the API's origin, a manifest definition without identity or schema_version, a Search spec without identity or resolved, nested specs no rule reads, an empty parts list, a Search body that is not an object, a codeless 503 problem document that is not retryable, a 401, an identity resubmit answering another spec, a part whose status is not a string, a Search spec whose definition names another spec_id, a Search response with no JSON body, an option with no data_query, empty entities and facets, a Sunset header issued once, `x-dfo-warning` messages and a Deprecation and Sunset pair, an unreadable versions page, a mint and a spec fetch whose build answers 202 and is polled to done, a resumed operation, a rate-limited poll, 503 polls retried within the budget an answered poll restores, a 202 that reads running, polls whose `retry_after_ms` changes from one to the next, a 404 poll that raises, a failed operation posted again, a failed operation with no error, and a ref whose stored file expired, read through the file selector, rebuilt at the same version, at a newer one, or refused with the newest version read from the versions route: the first version it lists with no `base_content_version`, the newest plain build, so a derived version listed before it is skipped. - `client-codes.json` and its schema: the 23 codes a client sets on the errors it raises itself, each with its error class, retryable flag and condition, disjoint from `actions.json` and `registries/codes.json`, and the `client_codes()` and `client_codes_schema()` accessors. Every scenario error the client raises itself carries its code, and `refusal_without_code` names a refusal whose body carries no code. - `proxy-codes.json` and its schema, the website proxy's `rate_limited` and `transient_unavailable` codes with their statuses, which the proxy sends in problem documents of the shape every refusal has, both retryable, with `Retry-After` on the 429, so a client reads one refusal shape, and `warning-codes.json` and its schema, the 6 codes a client sets on the warnings it issues itself, with the `proxy_codes()` and `warning_codes()` accessors. - `ref_digest_changed` in `warning-codes.json`, a `DfoWarning` for a ref member whose `content_version` serves a file with another digest than the ref's, because the server rendered the same data again. The client returns the data, with no error under strict. @@ -19,15 +19,17 @@ All notable changes to the `dfo-spec-kit-client-contract` Python package will be - A no_data part's gap kind comes from its own reason, whatever the answer's `diagnostics.timed_out` says. `answer-timed-out` gives its part the reason `timed_out`, and `answer-timed-out-keeps-reasons` pins a part that keeps `variable_not_resolved` as a `not_understood` gap in a timed-out answer. - Error outcomes give the error's `trace_id`, the body's, else the `x-dfo-trace-id` header's, else null. Scenario outcomes give `retry_waits`, the waits before each retry, and `warnings`, the warning codes a step's responses issued. Scenario steps take no `timeout`. - `dataset-ref.schema.json`, the frozen `DatasetRef` format, with accepted and rejected examples. -- `without_party_totals` in the ref's artifact rendering, `true` for a member fetched with party totals left out and absent for a plain rendering, with `ref-without-party-totals`, a default fetch's cut rendering, `ref-one-member`, a fetch with party totals and its plain rendering, and `ref-without-party-totals-false-rejected` as its examples. Each ref example's rendering is the one its scenario's build posted, and the validator checks it. -- `tools/capture_client_contract.py`, which captures `served/openapi.json` from a dfo-apis checkout in process and records the scenarios against a base URL, dev by default, or with `record --offline` derives their outcomes again from the records with no request. A local app's signed URLs are written as the storage URLs a deployment signs. +- `without_party_totals` in the ref's artifact rendering, `true` for a member fetched with party totals left out and absent for a plain rendering, with `ref-without-party-totals`, a default fetch's cut rendering, `ref-one-member`, a fetch with party totals and its plain rendering, and `ref-without-party-totals-false-rejected` as its examples. Each ref example's rendering is one its scenario's builds of its spec posted, and the validator checks it. +- `tools/capture_client_contract.py`, which captures `served/openapi.json` from a dfo-apis checkout in process and records the scenarios against a base URL, dev by default, or with `record --offline` derives their outcomes again from the records with no request. A local app's signed URLs are written as the storage URLs a deployment signs. `record` reads the rule tables from `--package-dir` before any request, and looks for each long build's spec from the year after the one its record built, stopping at the first start year whose estimate is within the sync budget. - `tools/validate_client_contract.py`, which checks the contract against its schemas, the served document and `registries/codes.json`, checks `client-codes.json` against them and against the reasons the key-lookup rule raises, and checks each scenario's `expect` against the outcomes its exchanges, `actions.json` and `client-codes.json` give, each Search request against the input its steps derive, a sentence lookup's `entity_groups` included, each build's `input.scope` against the narrowing its step names over the spec's bound roles, each build's options against its layout, each Search request's whole body, each download against its artifact's digest, each recorded scenario's commit against `served/SOURCE.json`, each problem document's code against `registries/codes.json` or, for the website proxy's, `proxy-codes.json`, each proxy problem document's content type, members and `retryable: true`, each 202 against an operation that reads `done`, and each absolute URL against the storage host, each ref example's members against the dataset members and downloaded files of the scenario it names, and the scenarios directory against a 16,000,000-byte budget for records and bodies together. A record that ends a run of retries while a retry remains does not fit. The outcomes include each long build's operation id, poll waits and resubmissions, and a `VersionUnavailableError`'s newest version. - The contract README states that a POST for a build identical to one pending or running returns that operation, so a client that stops waiting posts the same build to join it, and that `fetch` waits for a build with no whole-call deadline by default, with a download that delivers no byte for 30 s a retryable `network_error`. -- A fetch of an answer or a spec leaves out party totals by default: its build posts `without_party_totals: true` in `options`, so the server drops the group totals of the identity's open party roles, and a role the caller binds keeps what it binds. Fetch steps take `party_totals: true` for the client option, `party_totals` in Python and `partyTotals` in TypeScript, false by default, which posts no `without_party_totals`. The wire flag and the ref format are unchanged, and a ref records `without_party_totals: true` for a member fetched with the default. `fetch-spec-party-totals-left-out` and `fetch-spec-party-totals-kept` pin both on DAC2a ODA to Kenya in 2022 with the origin open. A mint and a resumed operation post the plain rendering. -- The `reading-order` rule and its vectors, which carry rows: a client presents a dataset's columns with the reading columns first, the time period, provider, recipient, subject, purpose, value, unit, flow type and provenance, each slot filled by the long layout's label or the frame layout's code, the flow type by CRS `crsTypeOfFlow` or else DAC2a `dac2aAidType`, then the columns whose values differ between rows, then those that hold one value, each group in the server's order, with `stat_var` last. It sorts each member's rows by the filled slots without `value` and the time period, then the varying columns, then the time period, ascending with nulls last and ties in served order, so each series' years read together. Scenario outcomes give `columns` in that order, and the validator checks it against the served files. -- The `client-identity` rule and its vectors: every request a client sends to the API carries `X-DFO-Client: / ()`, such as `dfo-python/0.1.0 (CPython 3.14)` or `dfo-ts/0.1.0 (Node 22.18)`, and `browser` as the runtime in a browser, with `pattern` matching every well-formed value. A request to another origin, a signed storage URL included, carries none. A caller's headers or header provider naming `X-DFO-Client` in any letter case raises `InvalidRequestError` `header_invalid`, whose condition now says so, and `User-Agent` stays as each client sends it. A scenario replay checks the header on each `path` and `origin_path` request and its absence on each `url` request. +- A fetch of an answer or a spec leaves out party totals by default: its build posts `without_party_totals: true` in `options`, so the server drops the group totals of the identity's open party roles, and a role the caller binds keeps what it binds. Fetch steps take `party_totals: true` for the client option, `party_totals` in Python and `partyTotals` in TypeScript, false by default, which posts no `without_party_totals`. The wire flag and the ref format are unchanged, and a ref records `without_party_totals: true` for a member fetched with the default. `fetch-spec-party-totals-left-out` and `fetch-spec-party-totals-kept` pin both on DAC2a ODA to Kenya in 2022 with the origin open. A mint leaves them out by default too, and its step takes `party_totals`, so its `result` answers the table a fetch of the spec answers; `mint-async-build` keeps them. +- The `reading-order` rule and its vectors, which carry rows: a client presents a dataset's columns with the reading columns first, the time period, provider, recipient, subject, purpose, value, unit, flow type and provenance, each slot filled by the long layout's label or the frame layout's code, the flow type by CRS `crsTypeOfFlow` or else DAC2a `dac2aAidType`, then the columns whose values differ between rows, then those that hold one value, each group in the server's order, with `stat_var` last. It sorts each member's rows by the filled slots without `value` and the time period, then the varying columns, then the time period, ascending with nulls last and ties in served order, so each series' years read together. Two cells hold the same value when they are equal as JSON values: numbers by value, so `1` and `1.0` are one value, booleans apart from numbers, and NaN the same as NaN. A vector's rows hold one cell per column and each column one JSON type, and `integer-and-float-are-one-value` pins `1` against `1.0`. Scenario outcomes give `columns` in that order, and the validator checks it against the served files, list and struct columns included, and reports a malformed vector. +- The `client-identity` rule and its vectors: every request a client sends to the API carries `X-DFO-Client: / ()`, such as `dfo-python/0.1.0 (CPython 3.14)` or `dfo-ts/0.1.0 (Node 22.18)`, `browser` as the runtime in a browser page or browser worker, and `other` in any other runtime, such as Cloudflare Workers, with `pattern` matching every well-formed value. A request to another origin, a signed storage URL included, carries none. A caller's headers or header provider naming `X-DFO-Client` in any letter case raises `InvalidRequestError` `header_invalid`, whose condition now says so, and `User-Agent` stays as each client sends it. A scenario replay checks the header on each `path` and `origin_path` request and its absence on each `url` request. - The `role-keys` rule and its vectors: a caller narrows a spec with `entities=` keyed by `place`, `origin` and `destination`, and a client posts `place` as the wire role `observationAbout`. `Spec.roles`, `specRoles(spec)` and the clients' errors name the caller keys, and a flow spec takes both directions. A key outside the spec's roles raises `InvalidRequestError` `invalid_argument` naming the key and the spec's roles, so `observationAbout` on GDP is refused naming `place` and `financingRecipient` on a flow spec naming `origin` and `destination`. A spec that names no role posts `place` as `observationAbout` and other keys as typed, and refuses `observationAbout` naming `place`. - The client rules and their vectors: the credentials path and file format, the key lookup order, with an empty `api_key` refused as `key_empty` and an empty profile key as `profile_without_key`, the key format check, the base URL order, `base_url_invalid` for an invalid value, and the cache directory and layout. - The `base-url` rule states the whole validity rule, and seven new vectors pin it: no query, fragment, `.` or `..` segment, character outside visible ASCII, empty userinfo or non-decimal IPv4 host; trailing slashes dropped; a relative URL valid only in a TypeScript client inside a browser page. +- The `base-url` rule also refuses a `.` or `..` segment written with `%2e`, a backslash, and a port outside 1 to 65535 or written other than in decimal digits, an empty one included, so a client on Python's `urlsplit` and one on the WHATWG URL parser accept the same values. Ten vectors pin them. +- The `file-selector` rule and its vectors: the file selector's query holds the `format`, `bundle` and `layout` of a ref member's rendering, the keys `query_keys` lists, in that order, and leaves out `without_party_totals`, which the selector refuses with 400 `request_invalid` because the version records its own cut. `fetch-ref-expired-file-selector` and `fetch-ref-artifact-unlisted` ask that query, and the validator and the capture build it from `query_keys`. [unreleased]: https://github.com/DevelopmentFinanceObservatory/spec-kit/commits/main/python diff --git a/python/client-contract/README.md b/python/client-contract/README.md index c57dfb6..03ca5f7 100644 --- a/python/client-contract/README.md +++ b/python/client-contract/README.md @@ -134,7 +134,7 @@ A scenario is one client-level call, the HTTP exchanges it makes, and the outcom | `exchanges` | The requests the call makes, in order, each with the response to give | | `expect` | One outcome per step: an `answer`, a `dataset`, an `operation`, a `newest_version` or an `error` | -A step that acts on an earlier result follows one: `choose` follows `search` or `choose`, a `fetch` with no target follows `search`, `choose` or another `fetch` with no target and acts on the last answer, and `result` follows `mint` or `operation`. A `fetch` of an answer or a spec, `mint` and `operation` take a `layout`, `long` by default or `frame`, and a `fetch` of an answer or a spec takes `strict` and `party_totals`. A `spec` in a step holds `spec_id`, the server's unmodified `definition`, the `sentence` when the client knows it, and the server's `resolved.entity_groups` as `entity_groups` when it names any. +A step that acts on an earlier result follows one: `choose` follows `search` or `choose`, a `fetch` with no target follows `search`, `choose` or another `fetch` with no target and acts on the last answer, and `result` follows `mint` or `operation`. A `fetch` of an answer or a spec, `mint` and `operation` take a `layout`, `long` by default or `frame`, a `fetch` of an answer or a spec takes `strict`, and that `fetch` and `mint` take `party_totals`. A `spec` in a step holds `spec_id`, the server's unmodified `definition`, the `sentence` when the client knows it, and the server's `resolved.entity_groups` as `entity_groups` when it names any. An exchange's request names its target in one of three ways. `path` is a path the client composes under the base URL, such as `/search/resolve`. `origin_path` is a root-relative URL the server returned, such as an artifact's `url`, or the path and query of an absolute one on the API's origin, which resolves against the base URL's origin, whatever path that URL has. `url` is an absolute URL outside the API, such as the signed storage URL a files route redirects to, which the client fetches with a bare request carrying no client header and no key. `json` holds the request body members that must match; a recorded scenario lists every member the client sent. The response gives `status`, the `headers` a client reads, and a `json` body or a `body_file`, a binary body in `scenarios/`. A scenario's own body files are named `.` and `-.`; a synthetic scenario may instead name a body file its parent names. @@ -161,15 +161,15 @@ The records also fix these client rules: - A fetch of a spec posts its narrowing as the build's `input.scope`, and posts no `scope` when it narrows nothing. A mint takes a layout and no narrowing, so it posts no `scope`. `window` posts `{"kind": "range", "start_year", "end_year"}` for a pair of years, or `{"kind": "all"}` or `{"kind": "latest"}`, and `facets` posts each axis as `{"ref": ...}`, and `facets: {}` names no axis and posts no `facets`. The server's `scope.entities` replaces the spec's whole entity binding, so `entities` posts a whole map: the roles the spec's `definition.identity.entities` binds, with each role the caller names replaced by its refs and each role the caller binds to null dropped. Naming `origin` alone on a spec bound to `origin` and `destination` keeps `destination`. A fetch that names no role, `entities: {}` included, posts no `entities`, so a fetch whose narrowing is all empty posts no `scope`; `fetch-spec-empty-narrowing` pins it. - A built `spec_id` that differs from the posted one, or a spec with no sentence, takes its sentence from Search: the client posts the built definition's `identity` as one `dataset_resubmit` identity entry, with the member's first part as `variable_text`, or `"dataset"` when no answer part names it, and checks the answer's `spec_id` equals the built one. The entry's `entity_groups` holds the spec's `resolved.entity_groups` for each role the built identity binds to the same entities as the spec's definition, so a window or facets narrowing keeps every group and a role the narrowing rebound or dropped loses its group. The entry leaves `entity_groups` out when it keeps no group, as for a resumed operation, whose spec the client does not hold. The server reads a role posted without its group as its members, so a narrowed spec of flows to the least developed countries would read "44 recipients" in place of "the 44 Least Developed Countries". - A dataset of several members puts the `part` column first, then the members' columns, gathered in member order with each column once and ordered by the reading-order rule, which also sorts each member's rows. A column that members hold in different types widens as pyarrow's permissive promotion widens it, and a pair that cannot widen raises `ContractError` `contract_violation`. A member file that already holds a `part` column raises `contract_violation` when members stack. Rows and columns are not derived, so no scenario pins these. The `part` value on a member's rows is the member's `parts` joined by `"; "`, so a member that answers two folded parts reads `"health ODA disbursements; education ODA disbursements"`, or the member's sentence when `parts` is empty. A definite part whose `variable_text` is absent or empty adds nothing to its member's `parts`, which `answer-folded-parts-unnamed` pins. -- A choice names its part by index, by variable text, or by a word that one gap's options mention, and its option, a candidate among them, by index or label. A boolean part or option is neither an index nor a label. A candidate's label is the `text` of its spec's `resolved.sentence_plan` tokens in the `heading` segment, a token whose `segment` is not a string counted in it, joined and stripped of surrounding white space, or else its `resolved.rendered_sentence`. A candidate with neither a heading nor a sentence has a null label; no scenario pins it, and the validator's tests do. A no_data option's label is matched among the options that mention the word the part was named by. With no part named, exactly one gap must await a choice: a `choice` gap, or any gap with options. +- A choice names its part by index, by variable text, or by a word that one gap's options mention, and its option, a candidate among them, by index or label. A boolean part or option is neither an index nor a label. A candidate's label is the `text` of its spec's `resolved.sentence_plan` tokens in the `heading` segment, a token whose `segment` is not a string counted in it, joined and stripped of surrounding white space, or else its `resolved.rendered_sentence`. A candidate with neither a heading nor a sentence has a null label; no scenario pins it, and the validator's tests do. A no_data option's label is matched among the options that mention the word the part was named by. An `entity_ambiguous` option carries `entity_choice`, the index into `no_data.ambiguous_entities` of the group it settles, and a menu groups the options by it. When one place name is ambiguous at two positions, as in "aid from Congo to Congo", each group offers the same labels, so a label matches an option in each group and the choice names its option by index, which `choose-refused-label-in-two-groups` pins. With no part named, exactly one gap must await a choice: a `choice` gap, or any gap with options. - A choice raises `InvalidRequestError` `choice_invalid` and makes no request when no gap, or more than one, awaits it, when the part is named by a boolean, is unknown, names more than one part or already has a reading, when the gap offers no options, or when the option is a boolean, is out of range or matches no label or several. No scenario can hold a boolean part or option, and the validator's tests pin both. It raises the same when an entry it would post cannot be built: a part with no `variable_text`, or a part of a status the client does not know that has no `restatement`. A chosen option whose `data_query` holds other than one part raises `ContractError`, and so does a chosen option with no `data_query`, in a one-part answer as in a several-part one, before any request. `choose-refused-shared-label`, `choose-refused-unknown-candidate-label`, `choose-refused-shared-candidate-label`, `choose-refused-part-without-variable-text`, `choose-refused-unknown-status`, `choose-option-two-part-query` and `choose-option-without-data-query` pin these, and `choose-candidate-by-label` pins a candidate chosen by its label. - Every pick of a candidate asks Search for the picked answer, and no client builds it locally. In a one-part answer the pick posts `dataset_resubmit`, without `raw_query`, with one identity entry: the candidate's identity and `entity_groups`, and the part's `variable_text`, `separate` and `operations` as below. The picked answer, its sentence with any "Assumed: …" clause, and its assumptions are Search's response. A choice of a no_data option in a one-part answer posts the option's `data_query` as the input. - In a several-part answer a choice posts `dataset_resubmit`, without `raw_query`, with one entry per part in part order. The chosen candidate posts an identity entry with its identity and `entity_groups`, and the chosen option its `data_query`'s one part as a parsed entry. A definite part posts an identity entry with its spec's identity and `entity_groups`. A candidates part left alone posts a `candidates` entry: its candidate set in the server's order, each with its identity and `entity_groups`, and its `ordering` and `truncated` echoed. The server cannot recompute that order, because it rests on the first answer's bind election and availability reads, and a parsed entry would cost an LLM bind. Any other part posts its `restatement` as a parsed entry, or a placeholder entry with its no_data `reason` or unsupported `code` when it has none. Every entry but a parsed one carries the part's `variable_text`, `separate` when true and `operations` when any of the part's operations reads. `entity_groups` is left out when the spec has none. `choose-echoes-separate-and-operations` pins the echo on a definite part's entry, and `choose-posts-unsupported-placeholder` an unsupported part's placeholder. - Each identity a choice posts echoes its reading's assumptions as `assumptions`: the chosen candidate's entry carries its part's `reading_assumptions` under the candidate's `spec_id`, a definite part's entry carries the part's `assumptions`, and each candidate of a `candidates` entry carries its own `reading_assumptions` entry. Search then states the echoed items that still hold, so the picked answer states what that reading assumed. `[]` states none. An identity with no `assumptions` reads as a fresh request, so a client that holds no list for a reading, from a part that carries no `assumptions` or `reading_assumptions`, posts none. The identity entry that asks for a built sentence posts no `assumptions`, since it reads only the spec's own `rendered_sentence`. An echoed `operations` or `assumptions` list posts exactly the items a client reads under Strict fields, in order, each built from the members it read: an optional member that read as absent is left out, and a `refs` list posts its string items. An `assumptions` list with no item read posts `[]`, and an `operations` list with none is left out. The item rule checks JSON types alone, so the echo posts what a client read, and the server still checks it against its own value constraints: the `verb` enum, the lengths of `phrase` and `text`, and at most one item in `refs`. `choose-echo-drops-malformed-items` pins both lists. - A strict fetch raises the manifest's first warning above `info` severity as a `DataWarning` with that warning's code, before the sentence lookup and the download. - An identity resubmit whose answer is not one definite part of the built `spec_id`, with a sentence, raises `ContractError`. -- The `frame` layout posts `{"format": "parquet", "bundle": false, "layout": "frame"}` as the build's `options` and downloads the artifact whose rendering's `layout` is `frame`; the `long` layout posts `{"format": "parquet", "bundle": false}` and downloads the bare Parquet artifact with any other layout. A fetch of an answer or a spec adds `"without_party_totals": true` to either, as the next rule gives. A client matches an artifact on its rendering's format, bundle and frame layout only, and `without_party_totals` plays no part in the match. A fetch of an answer with several members in the `frame` layout raises `InvalidRequestError` before any build. -- A fetch of an answer or a spec leaves out party totals by default. Its build posts `"without_party_totals": true` in `options`, and the server drops the rows whose party, on a flow party role the identity binds no entity on, the code index declares a group total, before any reshape. A role the caller binds keeps what it binds, so a total named by its dcid still comes back. The client option is `party_totals`, false by default, in Python, and `partyTotals` in TypeScript, and a fetch step's `party_totals: true` stands for it: the build posts no `without_party_totals` and keeps every total. The wire is the served rendering flag as it was; only the client's default changes. A cut that leaves rows out answers a version of its own, listed by the versions route with the plain build as its `base_content_version`, and a cut that leaves nothing out answers the plain version and its file. The manifest's artifact rendering carries `without_party_totals: true` in both cases. A mint and a resumed operation post the plain rendering. `fetch-spec-party-totals-left-out` pins the default on a DAC2a spec whose origin the fetch leaves open, where the cut drops the donor group totals, and `fetch-spec-party-totals-kept` the option. +- The `frame` layout posts `{"format": "parquet", "bundle": false, "layout": "frame"}` as the build's `options` and downloads the artifact whose rendering's `layout` is `frame`; the `long` layout posts `{"format": "parquet", "bundle": false}` and downloads the bare Parquet artifact with any other layout. A fetch of an answer or a spec, and a mint, add `"without_party_totals": true` to either, as the next rule gives. A client matches an artifact on its rendering's format, bundle and frame layout only, and `without_party_totals` plays no part in the match. A fetch of an answer with several members in the `frame` layout raises `InvalidRequestError` before any build. +- A fetch of an answer or a spec, and a mint, leave out party totals by default. The build posts `"without_party_totals": true` in `options`, and the server drops the rows whose party, on a flow party role the identity binds no entity on, the code index declares a group total, before any reshape. A role the caller binds keeps what it binds, so a total named by its dcid still comes back. The client option is `party_totals`, false by default, in Python, and `partyTotals` in TypeScript, and a fetch or mint step's `party_totals: true` stands for it: the build posts no `without_party_totals` and keeps every total. The wire is the served rendering flag as it was; only the client's default changes. A cut that leaves rows out answers a version of its own, listed by the versions route with the plain build as its `base_content_version`, and a cut that leaves nothing out answers the plain version and its file. The manifest's artifact rendering carries `without_party_totals: true` in both cases. A mint posts the options a fetch of its spec posts, so its `result` answers the table that fetch answers, which `mint-async-build` pins with `party_totals: true`. A resumed operation posts no build, and its `result` answers the rendering the operation's build posted. `fetch-spec-party-totals-left-out` pins the default on a DAC2a spec whose origin the fetch leaves open, where the cut drops the donor group totals, and `fetch-spec-party-totals-kept` the option. - A download asks the route that redirects to the signed URL, an artifact's `url` or the file selector, and fetches the URL its `location` names once. A files route answers 302 with the `location` the client fetches, or 200 with the file as its body, which the client hashes with no further request, which `fetch-spec-files-route-serves-file` pins. A refusal reads by the refusal rules, and any other status, another 2xx or another 3xx included, raises `ContractError` `unexpected_response` with that response's trace id, which `fetch-spec-files-route-no-content` pins with a 204 and `fetch-spec-files-route-temporary-redirect` with a 307. The route ask is an API request and follows the retry rules below, so a retryable refusal is retried within the budget and any other refusal, `artifact_expired` among them, raises at once. A failed fetch gets exactly one more round. A fetch that answers anything but 200, which `fetch-spec-signed-fetch-partial` pins with a 206, or bytes that do not hash to the artifact's digest ask the route again, under the same retry rules, and fetch again, with no wait between rounds. A second failure raises `ServerError` with `download_failed`, or `digest_mismatch` when the bytes are wrong. A network error or timeout on the fetch is retryable, so it starts the second round too, and when the second round fails the same way the client raises that failure as itself, `network_error` or `deadline_exceeded`. The replay holds no network failure, so no scenario pins that case. - A manifest artifact's `url` names its files route on the API's origin. A root-relative `url` resolves against the base URL's origin, and an absolute one on that origin is asked as its path and query, so both make the same `origin_path` request, which `fetch-spec-same-origin-artifact-url` pins. An artifact `url` on any other origin, the storage host's included, raises `ContractError` `contract_violation` before the sentence lookup and any request for the file, which `fetch-spec-foreign-artifact-url` and `fetch-spec-storage-artifact-url` pin. - A relative `location` resolves against the files route's URL, and the client fetches it on the API's origin as an `origin_path` request. A `location` on an origin other than the API's or `https://storage.googleapis.com` raises `ContractError` `contract_violation` before any fetch. A fetch on the storage host is bare and carries no API key. `fetch-spec-relative-location` pins the relative case. @@ -177,7 +177,7 @@ The records also fix these client rules: - A declined or catalogue answer is not complete, and fetching it raises `DeclinedError`. - Fetching an answer with a gap raises `IncompleteAnswerError` before any request, unless the fetch is partial and some part has a reading. - A fetch of an answer issues one `SearchWarning` when a definite part has an operation Search did not run, a departure, or a warning at severity `warn` or `degraded` with a `message`, or the answer's `diagnostics.warnings` holds such a warning. An `info` warning, a warning with no `message`, and an operation that does not read under Strict fields issue nothing. Each way is one line a client shows: `Not done: ()` coded by the verb, the phrase quoted as a JSON string that keeps non-ASCII characters as they are, the departure's `text` coded by its code, or `Note: ` coded by the warning's code, in part order with the diagnostics warnings last, and ways that show the same line count once, under the first one's code. A strict fetch raises the `SearchWarning` before any build, with the code of its one line, or a null code when there are several lines. A strict fetch raises it before the `frame` layout's several-member check. -- A ref's version is found in this order: the snapshot route and the files route at the stored digest; the file selector, `GET /datasets/{id}/{content_version}/file` with the ref's rendering as the query; and a build of the ref's spec in the ref's rendering, which must answer the ref's `spec_id` and the same `content_version`. Both recovery steps keep the ref's identity, so a recovered member carries the ref's `spec_id` and `content_version`: a snapshot manifest whose `definition.spec_id` or `content_version` is not the ref's, or a rebuild manifest whose `definition.spec_id` is not the ref's, raises `ContractError` `contract_violation` before any download, which `fetch-ref-expired-rebuild-other-spec` pins for the rebuild. A snapshot refusal other than `artifact_expired` ends the lookup, so `snapshot_not_found` raises `NotFoundError` with no further request. A files route or selector refusal of `artifact_expired`, `artifact_not_found` or `not_found` takes the next step, and a selector file of another digest is not the stored file. A snapshot manifest whose artifacts list no artifact of the ref member's digest sends the client to the selector in the ref member's own rendering, and a file that hashes to that digest gives the dataset, which `fetch-ref-artifact-unlisted` pins. A rebuild that answers another version, or that raises `InvalidRequestError`, `NotFoundError` or `VersionUnavailableError`, ends in `VersionUnavailableError` with the first refusal's code, or `version_unavailable` when nothing refused. Its `newest_version` is the version the rebuild answered, or else the newest stored version, as a `newest_version` step finds it: the first version `GET /datasets/{id}/versions?limit=100` lists with a null `base_content_version`, the newest plain build, read page by page through `&cursor=` with each page's `next_cursor` until one is listed. The route lists versions newest first and needs no key. A version with a `base_content_version` is derived from a plain build, a groupby collapse or a party-totals cut, and is skipped, which `fetch-ref-newest-version` pins. The newest version is null when the list ends without a plain version or the route refuses with `NotFoundError`. It names the newest plain build, which a fetch with the default party totals rule answers only when the cut leaves nothing out; when the cut leaves rows out, the fetch answers a version whose `base_content_version` is that build. So a client does not present `newest_version` as comparable to the `content_version` of a cut ref or a cut fetch: two different versions there need not mean newer data. Any other refusal of the versions route raises, and a page whose `versions` is not a list of versions each with a `content_version` and a `base_content_version` that is a string or null, or whose `next_cursor` is neither a string nor null, raises `ContractError` `contract_violation`, which `newest-version-unreadable-page` pins. +- A ref's version is found in this order: the snapshot route and the files route at the stored digest; the file selector, `GET /datasets/{id}/{content_version}/file` with the query the file-selector rule in `rules/README.md` builds from the ref's rendering, which leaves out `without_party_totals`; and a build of the ref's spec in the ref's rendering, which must answer the ref's `spec_id` and the same `content_version`. Both recovery steps keep the ref's identity, so a recovered member carries the ref's `spec_id` and `content_version`: a snapshot manifest whose `definition.spec_id` or `content_version` is not the ref's, or a rebuild manifest whose `definition.spec_id` is not the ref's, raises `ContractError` `contract_violation` before any download, which `fetch-ref-expired-rebuild-other-spec` pins for the rebuild. A snapshot refusal other than `artifact_expired` ends the lookup, so `snapshot_not_found` raises `NotFoundError` with no further request. A files route or selector refusal of `artifact_expired`, `artifact_not_found` or `not_found` takes the next step, and a selector file of another digest is not the stored file. A snapshot manifest whose artifacts list no artifact of the ref member's digest sends the client to the selector for the ref member's own rendering, and a file that hashes to that digest gives the dataset, which `fetch-ref-artifact-unlisted` pins. A rebuild that answers another version, or that raises `InvalidRequestError`, `NotFoundError` or `VersionUnavailableError`, ends in `VersionUnavailableError` with the first refusal's code, or `version_unavailable` when nothing refused. Its `newest_version` is the version the rebuild answered, or else the newest stored version, as a `newest_version` step finds it: the first version `GET /datasets/{id}/versions?limit=100` lists with a null `base_content_version`, the newest plain build, read page by page through `&cursor=` with each page's `next_cursor` until one is listed. The route lists versions newest first and needs no key. A version with a `base_content_version` is derived from a plain build, a groupby collapse or a party-totals cut, and is skipped, which `fetch-ref-newest-version` pins. The newest version is null when the list ends without a plain version or the route refuses with `NotFoundError`. It names the newest plain build, which a fetch with the default party totals rule answers only when the cut leaves nothing out; when the cut leaves rows out, the fetch answers a version whose `base_content_version` is that build. So a client does not present `newest_version` as comparable to the `content_version` of a cut ref or a cut fetch: two different versions there need not mean newer data. Any other refusal of the versions route raises, and a page whose `versions` is not a list of versions each with a `content_version` and a `base_content_version` that is a string or null, or whose `next_cursor` is neither a string nor null, raises `ContractError` `contract_violation`, which `newest-version-unreadable-page` pins. - A build answers 200 with the manifest, or 202 with an operation the client polls at `/datasets/operations/{operation_id}`, the id of the operation it holds. The 202 reads `pending` or `running`, because a submission starts a build or joins one under way, and a `done` operation left by an earlier submission of a rejected head is built again. A deterministic failure answers a 202 that reads `failed`. `fetch-spec-async-build` pins `pending` and `fetch-spec-accepted-running` pins `running`. A POST for a build identical to one that is pending or running returns that same operation, because the server derives the operation id from the request: dfo-apis `src/datasets_api/operation_store.py` `compute_operation_id` hashes the request identity, the origin catalogue build id and the rendering. A client that stops waiting for a build can post the same build again to join it. A mint's build answers 202. A record whose mint answers 200 fits no mint step. Before each poll it waits the `Retry-After` of the refusal that answered the last poll, whatever its status, as every retry of a refused request waits; else the last operation resource's `retry_after_ms`, else the `Retry-After` of the last answered poll, the response that carried that resource, else a backoff of 1, 2, 4 and then 8 seconds that advances one step each time it is used, a poll after a refusal included. `fetch-spec-poll-pacing-changes` pins a `retry_after_ms` that changes from one poll to the next, so a client waits each poll's own value. - A retryable 429 poll spends no retry, and a 429 whose body says it is not retryable raises at once, as a refusal's `retryable` outranks its status, which `fetch-spec-poll-rate-limited-not-retryable` pins. Any other retryable refused poll spends one of the two retries, which a poll that answers restores, and a refusal that is not retryable raises. `fetch-spec-poll-refused` pins two 503s polled again after an answered poll, `fetch-spec-poll-refused-backoff` a 503 with no `Retry-After` among polls no response paces, after which the backoff advances, `fetch-spec-poll-refused-retry-after` a 503 whose `Retry-After` paces the next poll, and `fetch-spec-poll-not-found` a 404 that raises at once. A poll that answers a failed operation whose error is retryable posts the same build once more and polls the operation it starts; a second failure, an error that is not retryable, and a resumed operation, whose build request the client does not hold, raise the operation's error. A refused poll whose code is `operation_failed` is a failed operation with that error. A failed operation whose resource carries no `error` raises `ServerError` with `operation_failed`, retryable by status 500 as a refusal without a document is, which `operation-failed-without-error` pins. A 202 or resumed operation that already reads `failed` raises its error before any poll. - Any other refusal is retried when it is retryable: as its problem document's `retryable` says, else when its status is 408, 429, 500, 502, 503 or 504. A client makes two retries by default, and the scenarios assume that default. A build's POST shares those two retries with every other refusal of it, and once a 504 cuts the build off the client posts it once more only, whatever that post answers: a build refused 504 then 503 raises the 503, a build refused 503 then 504 is posted once more after the 504, and a second 504 raises. `fetch-spec-504-then-503`, `fetch-spec-503-then-504` and `fetch-spec-504-twice` pin the three. A refusal that is not retryable, or one past the retries, raises its error. @@ -217,7 +217,7 @@ A record's `origin.base_url` is the base URL the run was given, a local app's in ## Rules -`rules/README.md` states the shared rules for credentials, key lookup and format, the base URL, disk caching, the order a client presents a dataset's columns and rows in, the keys a caller narrows a spec's roles by, and the `X-DFO-Client` header that identifies the client to the API. A rule applies where the client and runtime implement its capability. Python implements disk caching; TypeScript currently keeps no disk cache, so `cache-dir` and `cache-layout` apply to Python. Credentials-file rules apply to runtimes with local credentials support. Each rule's vectors are in `rules/.json`. +`rules/README.md` states the shared rules for credentials, key lookup and format, the base URL, disk caching, the order a client presents a dataset's columns and rows in, the keys a caller narrows a spec's roles by, the file selector's query, and the `X-DFO-Client` header that identifies the client to the API. A rule applies where the client and runtime implement its capability. Python implements disk caching; TypeScript currently keeps no disk cache, so `cache-dir` and `cache-layout` apply to Python. Credentials-file rules apply to runtimes with local credentials support. Each rule's vectors are in `rules/.json`. ## Validation diff --git a/python/client-contract/src/dfo_client_contract/contract.py b/python/client-contract/src/dfo_client_contract/contract.py index 944da09..e21412b 100644 --- a/python/client-contract/src/dfo_client_contract/contract.py +++ b/python/client-contract/src/dfo_client_contract/contract.py @@ -19,6 +19,7 @@ "reading-order", "client-identity", "role-keys", + "file-selector", ) diff --git a/python/client-contract/src/dfo_client_contract/rules/README.md b/python/client-contract/src/dfo_client_contract/rules/README.md index 3c52496..089940e 100644 --- a/python/client-contract/src/dfo_client_contract/rules/README.md +++ b/python/client-contract/src/dfo_client_contract/rules/README.md @@ -1,6 +1,6 @@ # Client rules -Clients implement these rules for the capabilities available in their runtime. Python implements every rule. TypeScript implements the key, credentials and base-URL rules in Node, and the reading-order, client-identity and role-keys rules in every runtime; browser runtimes use the key sources available there. TypeScript currently keeps no disk cache, so `cache-dir` and `cache-layout` apply to Python. Clients implementing the same capability use the same rules and formats. Each rule's vectors are in `.json`, validated by `rules.schema.json`. In a vector, `env` lists the environment variables set, and an empty value counts as unset everywhere below. +Clients implement these rules for the capabilities available in their runtime. Python implements every rule. TypeScript implements the key, credentials and base-URL rules in Node, and the reading-order, client-identity, role-keys and file-selector rules in every runtime; browser runtimes use the key sources available there. TypeScript currently keeps no disk cache, so `cache-dir` and `cache-layout` apply to Python. Clients implementing the same capability use the same rules and formats. Each rule's vectors are in `.json`, validated by `rules.schema.json`. In a vector, `env` lists the environment variables set, and an empty value counts as unset everywhere below. ## Credentials path (`credentials-path`) @@ -42,7 +42,7 @@ A client resolves its base URL when it is built. The first source found wins: 1. `DFO_BASE_URL`; 1. the public default, `https://developmentfinance.org/api/v1`. -Only the chosen source is checked. A base URL is valid when it is an absolute `http` or `https` URL with a host and none of these: userinfo, an empty one as in `https://@host/api` included; a query; a fragment; a `.` or `..` path segment; a character outside visible ASCII, a control character, a space or a non-ASCII character; or a host written as an IPv4 address in any form but four decimal numbers, such as `127.1`, `0x7f.1` or `2130706433`. An invalid one raises `base_url_invalid`, so an empty `base_url` argument raises it where an empty `DFO_BASE_URL` counts as unset. A relative URL is invalid, except in a TypeScript client inside a browser page, which resolves a relative base URL against the page's address. A valid value is used as given, less its trailing slashes, so the client's base URL reads `https://developmentfinance.org/api/v1` for `https://developmentfinance.org/api/v1/`; the rule fixes the source and the validity, and leaves the rest of the URL's form to the client. A key sent over plain `http` to a host that is not loopback is refused by `insecure_transport`, which this rule does not decide. +Only the chosen source is checked. A base URL is valid when it is an absolute `http` or `https` URL with a host and none of these: userinfo, an empty one as in `https://@host/api` included; a query; a fragment; a `.` or `..` path segment, with any of its dots written `%2e` or `%2E` included; a backslash; a port that is not 1 to 65535 in decimal digits, an empty port after a `:` included; a character outside visible ASCII, a control character, a space or a non-ASCII character; or a host written as an IPv4 address in any form but four decimal numbers, such as `127.1`, `0x7f.1` or `2130706433`. An invalid one raises `base_url_invalid`, so an empty `base_url` argument raises it where an empty `DFO_BASE_URL` counts as unset. A relative URL is invalid, except in a TypeScript client inside a browser page, which resolves a relative base URL against the page's address. A valid value is used as given, less its trailing slashes, so the client's base URL reads `https://developmentfinance.org/api/v1` for `https://developmentfinance.org/api/v1/`; the rule fixes the source and the validity, and leaves the rest of the URL's form to the client. A key sent over plain `http` to a host that is not loopback is refused by `insecure_transport`, which this rule does not decide. ## Key format (`key-format`) @@ -93,15 +93,15 @@ The columns come in four groups: 1. the other columns that hold one value in every row, null included, in the order the server wrote them; 1. `stat_var`, last whether or not it varies. -A table with no row or one row has no column that varies. A dataset of several members puts `part` first and judges the groups over the stacked table, where a member's rows hold null in a column that member lacks. So a CRS series of Germany's basic health ODA to Nigeria, split by co-operation modality, reads `time_period`, `financingProvider_label`, `financingRecipient_label`, `crsPurpose_label`, `value`, `unit_label`, `crsTypeOfFlow_label` and `provenance_label`, then `crsCooperationModality` and `crsCooperationModality_label`, which tell its rows apart, then its 15 constant columns, `stat_var` last. +A table with no row or one row has no column that varies. Two cells hold the same value when they are equal as JSON values: numbers by value, so `1` and `1.0` are one value, booleans apart from numbers, so `true` and `1` differ, and null the same as null. NaN, which a Parquet double column can hold, is the same as NaN. A list or struct cell is the same as another when their items are the same, in order. A dataset of several members puts `part` first and judges the groups over the stacked table, where a member's rows hold null in a column that member lacks. So a CRS series of Germany's basic health ODA to Nigeria, split by co-operation modality, reads `time_period`, `financingProvider_label`, `financingRecipient_label`, `crsPurpose_label`, `value`, `unit_label`, `crsTypeOfFlow_label` and `provenance_label`, then `crsCooperationModality` and `crsCooperationModality_label`, which tell its rows apart, then its 15 constant columns, `stat_var` last. The rows of each member are sorted by the filled slots in slot order, leaving out `value`, a measure, and the time period, then by the varying columns in presented order, `stat_var` after them when it varies, then by the time period, so each series' years read together. The keys of that series are the provider, recipient, purpose, unit, flow type and provenance labels, then `crsCooperationModality` and `crsCooperationModality_label`, then the time period, so it reads every A02 year, then every C01 year. Each key sorts ascending, numbers by value and strings by Unicode code point, with nulls last. Rows whose keys tie keep the order the server wrote them. The members follow one another in member order. -A vector gives the dataset's `members`, each with its `part` when there are several, its `columns` and its `rows` as the server wrote them, and the `columns` and `rows` a client presents. The order moves columns and rows and changes no value. A client hashes a downloaded file before it orders anything, so the digest and the ref keep the server's file. +A vector gives the dataset's `members`, each with its `part` when there are several, its `columns` and its `rows` as the server wrote them, and the `columns` and `rows` a client presents. Each row holds one cell per column, and the cells of a column, nulls aside, are all strings, all numbers or all booleans, since the rule orders no value against one of another type. The order moves columns and rows and changes no value. A client hashes a downloaded file before it orders anything, so the digest and the ref keep the server's file. ## Client identity (`client-identity`) -Every request a client sends to the API carries `X-DFO-Client: / ()`. `` is `dfo-python` or `dfo-ts`, and `` is the package version as the package states it. `` names the runtime and its major and minor version. In Python it is the implementation as `platform.python_implementation()` names it, then the interpreter's major and minor version, such as `CPython 3.14`. In TypeScript it is `Node`, `Deno` or `Bun` with the major and minor of that runtime's version, such as `Node 22.18`, and `browser` in any other runtime, a page or a worker. `pattern` in `client-identity.json` matches every well-formed value. A vector gives the `client`, the `version`, the `runtime` as its `name` and full `version`, or `browser` with no version, and the `value` the client sends. +Every request a client sends to the API carries `X-DFO-Client: / ()`. `` is `dfo-python` or `dfo-ts`, and `` is the package version as the package states it. `` names the runtime and its major and minor version. In Python it is the implementation as `platform.python_implementation()` names it, then the interpreter's major and minor version, such as `CPython 3.14`. In TypeScript it is `Node`, `Deno` or `Bun` with the major and minor of that runtime's version, such as `Node 22.18`, `browser` in a browser page or browser worker, and `other` in any other runtime, such as Cloudflare Workers. `pattern` in `client-identity.json` matches every well-formed value. A vector gives the `client`, the `version`, the `runtime` as its `name` and full `version`, or `browser` or `other` with no version, and the `value` the client sends. An API request is one to the base URL's origin, a `path` or `origin_path` request in a scenario. A request to any other origin, a signed storage URL included, carries no `X-DFO-Client`. In a browser, the files-route request the TypeScript client sends with `redirect: "follow"` carries the API key alone, because the browser forwards every other header to the storage host it is redirected to. @@ -116,3 +116,7 @@ The roles a spec takes are those its `definition.identity` binds in `entities` o A client checks each key the caller types before any request. A key outside the spec's roles raises `InvalidRequestError` `invalid_argument`, and the message names the key and the spec's roles. So GDP for Kenya refuses `origin` and `observationAbout` naming `place`, and a flow spec refuses `place` and a column name such as `financingRecipient` naming `origin` and `destination`. A spec that names no role leaves the check to the server, since the client cannot tell which roles it takes. Such a spec posts `place` as `observationAbout` and any other key as typed, and still refuses `observationAbout` naming `place`. The server replaces a spec's whole binding with the posted one, so a client posts the spec's own `identity.entities` with each role the caller names replaced by its dcids and each role the caller binds to null dropped. A vector gives the spec's `identity` and its `roles`. A vector that narrows adds the caller's `entities`, then either `posted`, the wire entities the build posts, or the `error` with its `reason` and the keys its message `names`, the typed key first. + +## File selector (`file-selector`) + +The file selector, `GET /datasets/{spec_id}/{content_version}/file`, redirects to a stored version's file in a rendering. A client asks it for a ref member's version when the files route at the stored digest refuses, or when the snapshot manifest lists no artifact of that digest. Its query holds the keys `query_keys` in `file-selector.json` lists, `format`, `bundle` and `layout`, in that order, each one the member's rendering holds, as `key=value` pairs joined by `&`, with booleans written `true` and `false`. The query leaves out `without_party_totals`: the version records the cut it was minted with, and the server lays that over the query. The server refuses any other key with 400 `request_invalid`, so a selector query that carried the cut would end the lookup. A vector gives the `rendering` and the `query` a client sends, without the leading `?`. diff --git a/python/client-contract/src/dfo_client_contract/rules/base-url.json b/python/client-contract/src/dfo_client_contract/rules/base-url.json index c53ffa5..b778e4e 100644 --- a/python/client-contract/src/dfo_client_contract/rules/base-url.json +++ b/python/client-contract/src/dfo_client_contract/rules/base-url.json @@ -172,6 +172,84 @@ "env": {}, "source": "argument", "url": "https://developmentfinance.org/api/v1" + }, + { + "name": "encoded-dot-segment-raises", + "base_url": "https://developmentfinance.org/api/%2e%2e/v1", + "env": {}, + "error": { + "reason": "base_url_invalid" + } + }, + { + "name": "mixed-encoded-dot-segment-raises", + "base_url": "https://developmentfinance.org/api/.%2E/v1", + "env": {}, + "error": { + "reason": "base_url_invalid" + } + }, + { + "name": "encoded-single-dot-segment-raises", + "base_url": "https://developmentfinance.org/api/%2E/v1", + "env": {}, + "error": { + "reason": "base_url_invalid" + } + }, + { + "name": "backslash-raises", + "base_url": "https://developmentfinance.org\\api\\v1", + "env": {}, + "error": { + "reason": "base_url_invalid" + } + }, + { + "name": "port-out-of-range-raises", + "base_url": "https://developmentfinance.org:99999/api/v1", + "env": {}, + "error": { + "reason": "base_url_invalid" + } + }, + { + "name": "port-zero-raises", + "base_url": "https://developmentfinance.org:0/api/v1", + "env": {}, + "error": { + "reason": "base_url_invalid" + } + }, + { + "name": "port-not-decimal-raises", + "base_url": "https://developmentfinance.org:abc/api/v1", + "env": {}, + "error": { + "reason": "base_url_invalid" + } + }, + { + "name": "empty-port-raises", + "base_url": "https://developmentfinance.org:/api/v1", + "env": {}, + "error": { + "reason": "base_url_invalid" + } + }, + { + "name": "highest-port-is-valid", + "base_url": "https://developmentfinance.org:65535/api/v1", + "env": {}, + "source": "argument", + "url": "https://developmentfinance.org:65535/api/v1" + }, + { + "name": "dot-inside-a-segment-is-valid", + "base_url": "https://developmentfinance.org/api/v1.%2e", + "env": {}, + "source": "argument", + "url": "https://developmentfinance.org/api/v1.%2e" } ] } diff --git a/python/client-contract/src/dfo_client_contract/rules/client-identity.json b/python/client-contract/src/dfo_client_contract/rules/client-identity.json index 1d7756c..480fb20 100644 --- a/python/client-contract/src/dfo_client_contract/rules/client-identity.json +++ b/python/client-contract/src/dfo_client_contract/rules/client-identity.json @@ -2,7 +2,7 @@ "format_version": 1, "rule": "client-identity", "header": "X-DFO-Client", - "pattern": "^(dfo-python|dfo-ts)/[0-9A-Za-z.+-]+ \\(([A-Za-z]+ [0-9]+\\.[0-9]+|browser)\\)$", + "pattern": "^(dfo-python|dfo-ts)/[0-9A-Za-z.+-]+ \\(([A-Za-z]+ [0-9]+\\.[0-9]+|browser|other)\\)$", "vectors": [ { "name": "python-cpython", @@ -93,6 +93,15 @@ }, "value": "dfo-ts/0.1.0 (browser)" }, + { + "name": "ts-other-runtime", + "client": "dfo-ts", + "version": "0.1.0", + "runtime": { + "name": "other" + }, + "value": "dfo-ts/0.1.0 (other)" + }, { "name": "ts-pre-release-package", "client": "dfo-ts", diff --git a/python/client-contract/src/dfo_client_contract/rules/file-selector.json b/python/client-contract/src/dfo_client_contract/rules/file-selector.json new file mode 100644 index 0000000..de7d35e --- /dev/null +++ b/python/client-contract/src/dfo_client_contract/rules/file-selector.json @@ -0,0 +1,47 @@ +{ + "format_version": 1, + "rule": "file-selector", + "query_keys": [ + "format", + "bundle", + "layout" + ], + "vectors": [ + { + "name": "long-cut", + "rendering": { + "format": "parquet", + "bundle": false, + "without_party_totals": true + }, + "query": "format=parquet&bundle=false" + }, + { + "name": "long-plain", + "rendering": { + "format": "parquet", + "bundle": false + }, + "query": "format=parquet&bundle=false" + }, + { + "name": "frame-cut", + "rendering": { + "format": "parquet", + "bundle": false, + "layout": "frame", + "without_party_totals": true + }, + "query": "format=parquet&bundle=false&layout=frame" + }, + { + "name": "keys-in-listed-order", + "rendering": { + "layout": "frame", + "bundle": true, + "format": "csv" + }, + "query": "format=csv&bundle=true&layout=frame" + } + ] +} diff --git a/python/client-contract/src/dfo_client_contract/rules/reading-order.json b/python/client-contract/src/dfo_client_contract/rules/reading-order.json index d4bb978..90c1f37 100644 --- a/python/client-contract/src/dfo_client_contract/rules/reading-order.json +++ b/python/client-contract/src/dfo_client_contract/rules/reading-order.json @@ -1394,6 +1394,53 @@ "dc/sv/gdp" ] ] + }, + { + "name": "integer-and-float-are-one-value", + "members": [ + { + "columns": [ + "time_period", + "value", + "currency", + "base_year" + ], + "rows": [ + [ + 2021, + 5, + "USD", + 2020 + ], + [ + 2020, + 4, + "USD", + 2020.0 + ] + ] + } + ], + "columns": [ + "time_period", + "value", + "currency", + "base_year" + ], + "rows": [ + [ + 2020, + 4, + "USD", + 2020.0 + ], + [ + 2021, + 5, + "USD", + 2020 + ] + ] } ] } diff --git a/python/client-contract/src/dfo_client_contract/rules/rules.schema.json b/python/client-contract/src/dfo_client_contract/rules/rules.schema.json index 978a167..e4680b6 100644 --- a/python/client-contract/src/dfo_client_contract/rules/rules.schema.json +++ b/python/client-contract/src/dfo_client_contract/rules/rules.schema.json @@ -34,6 +34,9 @@ }, { "$ref": "#/$defs/role_keys" + }, + { + "$ref": "#/$defs/file_selector" } ], "$defs": { @@ -90,6 +93,10 @@ }, "layout": { "const": "frame" + }, + "without_party_totals": { + "description": "Present, and true, for a rendering with party totals left out.", + "const": true } } }, @@ -721,7 +728,7 @@ "$ref": "#/$defs/one_line" }, "runtime": { - "description": "The runtime's name and its full version, or `browser` with no version.", + "description": "The runtime's name and its full version, or `browser` or `other` with no version.", "type": "object", "additionalProperties": false, "required": [ @@ -741,7 +748,10 @@ "if": { "properties": { "name": { - "const": "browser" + "enum": [ + "browser", + "other" + ] } } }, @@ -1121,6 +1131,54 @@ } } } + }, + "file_selector": { + "$ref": "#/$defs/suite", + "additionalProperties": false, + "required": [ + "query_keys" + ], + "properties": { + "format_version": true, + "rule": { + "const": "file-selector" + }, + "query_keys": { + "description": "The rendering keys the file selector reads, in the order a client writes them into its query.", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "type": "string", + "minLength": 1, + "$ref": "#/$defs/one_line" + } + }, + "vectors": { + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "rendering", + "query" + ], + "properties": { + "name": { + "$ref": "#/$defs/name" + }, + "rendering": { + "$ref": "#/$defs/rendering" + }, + "query": { + "description": "The selector's query a client sends for the rendering, without the leading `?`.", + "type": "string", + "minLength": 1 + } + } + } + } + } } } } diff --git a/python/client-contract/src/dfo_client_contract/scenario.schema.json b/python/client-contract/src/dfo_client_contract/scenario.schema.json index b44a8c5..c6e5752 100644 --- a/python/client-contract/src/dfo_client_contract/scenario.schema.json +++ b/python/client-contract/src/dfo_client_contract/scenario.schema.json @@ -398,7 +398,7 @@ } }, "mint_step": { - "description": "A mint of a spec, which starts its build and answers the operation; it takes a layout and no narrowing.", + "description": "A mint of a spec, which starts its build and answers the operation; it takes a layout and party totals as a fetch of a spec does, and no narrowing.", "type": "object", "additionalProperties": false, "required": [ @@ -414,6 +414,10 @@ }, "layout": { "$ref": "#/$defs/layout" + }, + "party_totals": { + "description": "True keeps the group totals of the identity's open party roles, so the build posts no `without_party_totals`. Absent or false leaves them out.", + "type": "boolean" } } }, diff --git a/python/client-contract/src/dfo_client_contract/scenarios/choose-refused-label-in-two-groups.json b/python/client-contract/src/dfo_client_contract/scenarios/choose-refused-label-in-two-groups.json new file mode 100644 index 0000000..fb234df --- /dev/null +++ b/python/client-contract/src/dfo_client_contract/scenarios/choose-refused-label-in-two-groups.json @@ -0,0 +1,395 @@ +{ + "format_version": 1, + "name": "choose-refused-label-in-two-groups", + "description": "Both entity_choice groups offer an option labelled Congo [DRC], so the label matches two options and the choice raises InvalidRequestError with no request. A choice in such a part names its option by index.", + "origin": { + "kind": "synthetic", + "derived_from": "search-place-ambiguous-twice", + "edit": "A choice of \"Congo [DRC]\" follows the search." + }, + "call": [ + { + "op": "search", + "question": "aid from Congo to Congo" + }, + { + "op": "choose", + "option": "Congo [DRC]" + } + ], + "exchanges": [ + { + "request": { + "method": "POST", + "path": "/search/resolve", + "json": { + "schema_version": 1, + "input": { + "kind": "raw_text", + "query": "aid from Congo to Congo" + } + } + }, + "response": { + "status": 200, + "headers": { + "content-type": "application/json", + "x-dfo-trace-id": "14c23c7a7a01430eb3f485e658ae7e51" + }, + "json": { + "schema_version": 1, + "query_echo": { + "entry_path": "raw_text", + "raw_query": "aid from Congo to Congo", + "normalized_query": "aid from Congo to Congo", + "extract_skipped": false + }, + "diagnostics": { + "engine_build": "d80df9f5fd598b7345255470932d202ee5d49942", + "trace_id": "14c23c7a7a01430eb3f485e658ae7e51", + "warnings": [], + "timing_ms": { + "total": 1535, + "by_step": { + "coverage": 0, + "extract": 1020, + "cache_lookup": 0 + } + }, + "llm_usage": { + "input_tokens": 2226, + "output_tokens": 106, + "cached_tokens": 840 + }, + "timed_out": false + }, + "kind": "resolution", + "parts": [ + { + "variable_text": "aid", + "warnings": [], + "timing_ms": { + "total": 377, + "by_step": { + "recall": 377, + "phase_resolve_places": 0, + "phase_check_entities": 0 + } + }, + "rendered_sentence": "\"Congo\" (from) and \"Congo\" (to) each match more than one entity: \"Congo [DRC]\" and \"Congo [Republic]\". Be more specific.", + "sentence_plan": [ + { + "kind": "text", + "text": "\"", + "segment": "heading", + "entity_refs": [], + "states": [] + }, + { + "kind": "slot", + "text": "Congo", + "segment": "heading", + "role": "origin", + "entity_choice": 0, + "entity_refs": [], + "states": [] + }, + { + "kind": "text", + "text": "\" (from) and \"", + "segment": "heading", + "entity_refs": [], + "states": [] + }, + { + "kind": "slot", + "text": "Congo", + "axis": "where", + "segment": "heading", + "role": "destination", + "entity_choice": 1, + "entity_refs": [], + "states": [] + }, + { + "kind": "text", + "text": "\" (to) each match more than one entity: \"Congo [DRC]\" and \"Congo [Republic]\". Be more specific.", + "segment": "heading", + "entity_refs": [], + "states": [] + } + ], + "restatement": { + "variable_text": "aid", + "entities": [ + { + "text": "Congo", + "name": "Democratic Republic of the Congo", + "role_text": "from" + }, + { + "text": "Congo", + "name": "Republic of the Congo", + "role_text": "to" + } + ], + "sources": [], + "operations": [] + }, + "entity_refs": [ + { + "text": "Congo", + "refs": [] + }, + { + "text": "Congo", + "refs": [] + } + ], + "operations": [], + "departures": [], + "assumptions": [], + "status": "no_data", + "no_data": { + "reason": "entity_ambiguous", + "category": "unresolved", + "message": "\"Congo\" (from) and \"Congo\" (to) each match more than one entity: \"Congo [DRC]\" and \"Congo [Republic]\". Be more specific.", + "entity_terms": [ + "Congo", + "Congo" + ], + "ambiguous_entities": [ + { + "text": "Congo", + "candidates": [ + { + "ref": "country/COD", + "label": "Congo [DRC]", + "kind": "country" + }, + { + "ref": "country/COG", + "label": "Congo [Republic]", + "kind": "country" + } + ] + }, + { + "text": "Congo", + "candidates": [ + { + "ref": "country/COD", + "label": "Congo [DRC]", + "kind": "country" + }, + { + "ref": "country/COG", + "label": "Congo [Republic]", + "kind": "country" + } + ] + } + ], + "source_terms": [], + "ambiguous_sources": [], + "unbound_qualifiers": [], + "options": [ + { + "mentions": [ + "Congo" + ], + "label": "Congo [DRC]", + "data_query": { + "kind": "parsed", + "parts": [ + { + "variable_text": "aid", + "entities": [ + { + "text": "Congo", + "role_text": "from", + "dcid": "country/COD" + }, + { + "text": "Congo", + "name": "Republic of the Congo", + "role_text": "to" + } + ], + "sources": [], + "operations": [] + } + ] + }, + "entity_choice": 0 + }, + { + "mentions": [ + "Congo" + ], + "label": "Congo [Republic]", + "data_query": { + "kind": "parsed", + "parts": [ + { + "variable_text": "aid", + "entities": [ + { + "text": "Congo", + "role_text": "from", + "dcid": "country/COG" + }, + { + "text": "Congo", + "name": "Republic of the Congo", + "role_text": "to" + } + ], + "sources": [], + "operations": [] + } + ] + }, + "entity_choice": 0 + }, + { + "mentions": [ + "Congo" + ], + "label": "Congo [DRC]", + "data_query": { + "kind": "parsed", + "parts": [ + { + "variable_text": "aid", + "entities": [ + { + "text": "Congo", + "name": "Democratic Republic of the Congo", + "role_text": "from" + }, + { + "text": "Congo", + "role_text": "to", + "dcid": "country/COD" + } + ], + "sources": [], + "operations": [] + } + ] + }, + "entity_choice": 1 + }, + { + "mentions": [ + "Congo" + ], + "label": "Congo [Republic]", + "data_query": { + "kind": "parsed", + "parts": [ + { + "variable_text": "aid", + "entities": [ + { + "text": "Congo", + "name": "Democratic Republic of the Congo", + "role_text": "from" + }, + { + "text": "Congo", + "role_text": "to", + "dcid": "country/COG" + } + ], + "sources": [], + "operations": [] + } + ] + }, + "entity_choice": 1 + } + ] + } + } + ], + "rendered_sentence": "\"Congo\" (from) and \"Congo\" (to) each match more than one entity: \"Congo [DRC]\" and \"Congo [Republic]\". Be more specific.", + "sentence_plan": [ + { + "kind": "text", + "text": "\"", + "segment": "heading", + "entity_refs": [], + "states": [], + "part_index": 0 + }, + { + "kind": "slot", + "text": "Congo", + "segment": "heading", + "role": "origin", + "entity_choice": 0, + "entity_refs": [], + "states": [], + "part_index": 0 + }, + { + "kind": "text", + "text": "\" (from) and \"", + "segment": "heading", + "entity_refs": [], + "states": [], + "part_index": 0 + }, + { + "kind": "slot", + "text": "Congo", + "axis": "where", + "segment": "heading", + "role": "destination", + "entity_choice": 1, + "entity_refs": [], + "states": [], + "part_index": 0 + }, + { + "kind": "text", + "text": "\" (to) each match more than one entity: \"Congo [DRC]\" and \"Congo [Republic]\". Be more specific.", + "segment": "heading", + "entity_refs": [], + "states": [], + "part_index": 0 + } + ], + "status": "no_data" + } + } + } + ], + "expect": [ + { + "answer": { + "kind": "resolution", + "status": "no_data", + "sentence": "\"Congo\" (from) and \"Congo\" (to) each match more than one entity: \"Congo [DRC]\" and \"Congo [Republic]\". Be more specific.", + "complete": false, + "specs": [], + "gaps": [ + { + "part": 0, + "kind": "choice", + "options": 4 + } + ] + } + }, + { + "error": { + "class": "InvalidRequestError", + "code": "choice_invalid", + "trace_id": null + } + } + ] +} diff --git a/python/client-contract/src/dfo_client_contract/scenarios/fetch-ref-artifact-unlisted.json b/python/client-contract/src/dfo_client_contract/scenarios/fetch-ref-artifact-unlisted.json index 2906e4f..f77ae51 100644 --- a/python/client-contract/src/dfo_client_contract/scenarios/fetch-ref-artifact-unlisted.json +++ b/python/client-contract/src/dfo_client_contract/scenarios/fetch-ref-artifact-unlisted.json @@ -951,7 +951,7 @@ { "request": { "method": "GET", - "path": "/datasets/spec_774d6c001a6250aa67ca676ab067895e/cv_b92d94774702c7f05f2075d0b03dd756/file?format=parquet&bundle=false&without_party_totals=true" + "path": "/datasets/spec_774d6c001a6250aa67ca676ab067895e/cv_b92d94774702c7f05f2075d0b03dd756/file?format=parquet&bundle=false" }, "response": { "status": 302, diff --git a/python/client-contract/src/dfo_client_contract/scenarios/fetch-ref-expired-file-selector.json b/python/client-contract/src/dfo_client_contract/scenarios/fetch-ref-expired-file-selector.json index 85ed981..d5debee 100644 --- a/python/client-contract/src/dfo_client_contract/scenarios/fetch-ref-expired-file-selector.json +++ b/python/client-contract/src/dfo_client_contract/scenarios/fetch-ref-expired-file-selector.json @@ -1002,7 +1002,7 @@ { "request": { "method": "GET", - "path": "/datasets/spec_774d6c001a6250aa67ca676ab067895e/cv_b92d94774702c7f05f2075d0b03dd756/file?format=parquet&bundle=false&without_party_totals=true" + "path": "/datasets/spec_774d6c001a6250aa67ca676ab067895e/cv_b92d94774702c7f05f2075d0b03dd756/file?format=parquet&bundle=false" }, "response": { "status": 302, diff --git a/python/client-contract/src/dfo_client_contract/scenarios/mint-async-build.json b/python/client-contract/src/dfo_client_contract/scenarios/mint-async-build.json index 9c27c02..4a1c656 100644 --- a/python/client-contract/src/dfo_client_contract/scenarios/mint-async-build.json +++ b/python/client-contract/src/dfo_client_contract/scenarios/mint-async-build.json @@ -243,7 +243,8 @@ "group_id": "groups/OECD.DAC" } } - } + }, + "party_totals": true }, { "op": "result" diff --git a/python/client-contract/tests/test_rules.py b/python/client-contract/tests/test_rules.py index fe7224f..e89a9e0 100644 --- a/python/client-contract/tests/test_rules.py +++ b/python/client-contract/tests/test_rules.py @@ -8,6 +8,7 @@ from __future__ import annotations import ipaddress +import math import ntpath import posixpath import re @@ -133,13 +134,20 @@ def look_up_key(vector: dict[str, Any]) -> tuple[str, str | None]: _NON_DECIMAL_IPV4 = re.compile(r"(?:[^:]*\.)?(?:0x[0-9a-f]*|[0-9]+)") +_PORT = re.compile(r"(?:\[[^\]]*\]|[^:]*)(?::([0-9]*))?") + + def is_valid_base_url(url: str) -> bool: - if re.search(r"[^\x21-\x7e]", url) or "?" in url or "#" in url: + if re.search(r"[^\x21-\x7e]|[?#\\]", url): return False parts = urlsplit(url) if parts.scheme not in ("http", "https") or "@" in parts.netloc: return False - if not parts.hostname or {".", ".."} & set(parts.path.split("/")): + segments = {s.lower().replace("%2e", ".") for s in parts.path.split("/")} + if not parts.hostname or {".", ".."} & segments: + return False + port = _PORT.fullmatch(parts.netloc) + if port is None or (port[1] is not None and not 1 <= int(port[1] or 0) <= 65535): return False host = parts.hostname.removesuffix(".") if _NON_DECIMAL_IPV4.fullmatch(host): @@ -195,6 +203,19 @@ def cache_entry(vector: dict[str, Any]) -> str: return f"manifests/{vector['spec_id']}/{vector['content_version']}/{key}.json" +def same_value(left: Any, right: Any) -> bool: + """Numbers by value, booleans apart from numbers, NaN as NaN.""" + if isinstance(left, bool) or isinstance(right, bool): + return type(left) is type(right) and left == right + if isinstance(left, float) and isinstance(right, float): + return left == right or (math.isnan(left) and math.isnan(right)) + return left == right + + +def varies(column: str, rows: list[dict[str, Any]]) -> bool: + return any(not same_value(row.get(column), rows[0].get(column)) for row in rows) + + def reading_order( members: list[dict[str, Any]], reading: list[list[str]] ) -> tuple[list[str], list[list[Any]]]: @@ -212,17 +233,17 @@ def reading_order( if (name := next((n for n in names if n in served), None)) is not None ] rest = [c for c in served if c not in first and c != "stat_var"] - varies = {c for c in rest if len({repr(row.get(c)) for row in stacked}) > 1} + varying = {c for c in rest if varies(c, stacked)} columns = [ *(["part"] if len(members) > 1 else []), *first, - *[c for c in rest if c in varies], - *[c for c in rest if c not in varies], + *[c for c in rest if c in varying], + *[c for c in rest if c not in varying], *(["stat_var"] if "stat_var" in served else []), ] keys = [c for c in first if c not in ("value", "time_period")] - keys += [c for c in columns if c in varies] - if "stat_var" in served and len({repr(row.get("stat_var")) for row in stacked}) > 1: + keys += [c for c in columns if c in varying] + if "stat_var" in served and varies("stat_var", stacked): keys.append("stat_var") keys += ["time_period"] if "time_period" in first else [] rows: list[list[Any]] = [] @@ -235,7 +256,7 @@ def reading_order( def client_identity(client: str, version: str, runtime: dict[str, str]) -> str: label = runtime["name"] - if label != "browser": + if label not in ("browser", "other"): match = _MAJOR_MINOR.match(runtime["version"]) assert match, runtime label += f" {match[1]}.{match[2]}" @@ -280,6 +301,14 @@ def posted_entities( return posted +def selector_query(rendering: dict[str, Any]) -> str: + return "&".join( + f"{key}={str(rendering[key]).lower()}" + for key in rule("file-selector")["query_keys"] + if key in rendering + ) + + @pytest.mark.parametrize("name", RULES) def test_every_rule_has_vectors(name: str) -> None: assert rule(name)["rule"] == name @@ -383,3 +412,8 @@ def test_role_keys(vector: dict[str, Any]) -> None: elif "entities" in vector: posted = posted_entities(vector["identity"], vector["entities"]) assert posted == vector["posted"] + + +@pytest.mark.parametrize("vector", _vectors("file-selector"), ids=_ids("file-selector")) +def test_file_selector(vector: dict[str, Any]) -> None: + assert selector_query(vector["rendering"]) == vector["query"] diff --git a/python/spec-kit/CHANGELOG.md b/python/spec-kit/CHANGELOG.md index 66d1003..d2b6879 100644 --- a/python/spec-kit/CHANGELOG.md +++ b/python/spec-kit/CHANGELOG.md @@ -79,8 +79,11 @@ and this package adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `release_mismatch` at 503, the statuses the Presentation write routes serve. `presentation.render` keeps 422 for the first two and 409 for the rest. `frame_unavailable` covers a frame that is not held in a state - Presentation can load, and its definition names each case. A frame store - that fails to answer is `service_unavailable`. + Presentation can load, and its definition names each case. A frame whose + stored artifact the store fails to read is also `frame_unavailable`, served + at 503 and retryable; the registry holds one status for each endpoint and + channel, so it records the 404 the write routes serve for every other + case. - Code `edit_conflict` maps to `presentation.render` at 409, for a render of a component that has no current revision, and its definition names that case. diff --git a/tools/README.md b/tools/README.md index d52b523..b2d73e8 100644 --- a/tools/README.md +++ b/tools/README.md @@ -38,13 +38,13 @@ uv run tools/validate_provenance_docs_vectors.py uv run tools/validate_descriptions_vectors.py ``` -`validate_client_contract.py` validates the client contract under `python/client-contract/src/dfo_client_contract/` against its schemas, then checks what they cannot express: every `strict-fields.json` pointer resolves in `served/openapi.json` and stays required and non-nullable, or keeps its enum values; `actions.json` holds one row, in code order, for each active refusal code in `registries/codes.json` that reaches a problem document or a no_data reason, with a gap kind exactly for the no_data reasons; `client-codes.json` lists each code a client raises itself once, in code order, none of them a code in `actions.json` or the registry, each with a class a scenario names and a retryable flag; `proxy-codes.json` and `warning-codes.json` list their codes once, in code order, apart from every other table, and the warning codes are those the client rules issue; `served/openapi.json` holds the Search, Datasets, Graph and Presentation surfaces and nothing outside them but the platform's `/codes` route; each recorded scenario's requests match a served route and its JSON bodies validate against that route's schemas, where a required member stays required when it admits null; every problem document in a scenario, synthetic ones and failed operations' errors included, carries an active code registered at its status or a code `proxy-codes.json` lists at its status, and one with a proxy code is `application/problem+json` with the common members, its own status and `retryable` true; no 202 in a scenario reads `done`; each recorded scenario names `served/SOURCE.json`'s commit; each scenario's `expect` equals the outcomes its steps reach over its exchanges, with refusals' error classes and gap kinds from `actions.json`, each error the client raises itself carrying its code and class from `client-codes.json`, a repeated refused request counted a retry only where the retry rules allow one, a record that stops retrying while a retry remains refused, and the waits before retries and the warnings response headers issue derived; each POST carries its body, each build posts the body its spec, narrowing, layout and, for a fetch, party totals give, each dataset's columns are the reading-order rule's order of the downloaded Parquet files' columns, each Search request posts the whole body its steps derive, each poll names the operation the client holds, and each signed download is the URL its redirect names; each body file a record names is a bare file name in `scenarios/` and no symlink; every absolute URL a record holds is a signed URL on the storage host with the redacted query, or a `location` under the base URL; each `DatasetRef` example gets the verdict it states, and each reading-order vector orders its columns and rows as the rule does. It does not import `dfo_client_contract`, whose `tests/test_rules.py` replays the rule vectors. Also a PEP 723 script: +`validate_client_contract.py` validates the client contract under `python/client-contract/src/dfo_client_contract/` against its schemas, then checks what they cannot express: every `strict-fields.json` pointer resolves in `served/openapi.json` and stays required and non-nullable, or keeps its enum values; `actions.json` holds one row, in code order, for each active refusal code in `registries/codes.json` that reaches a problem document or a no_data reason, with a gap kind exactly for the no_data reasons; `client-codes.json` lists each code a client raises itself once, in code order, none of them a code in `actions.json` or the registry, each with a class a scenario names and a retryable flag; `proxy-codes.json` and `warning-codes.json` list their codes once, in code order, apart from every other table, and the warning codes are those the client rules issue; `served/openapi.json` holds the Search, Datasets, Graph and Presentation surfaces and nothing outside them but the platform's `/codes` route; each recorded scenario's requests match a served route and its JSON bodies validate against that route's schemas, where a required member stays required when it admits null; every problem document in a scenario, synthetic ones and failed operations' errors included, carries an active code registered at its status or a code `proxy-codes.json` lists at its status, and one with a proxy code is `application/problem+json` with the common members, its own status and `retryable` true; no 202 in a scenario reads `done`; each recorded scenario names `served/SOURCE.json`'s commit; each scenario's `expect` equals the outcomes its steps reach over its exchanges, with refusals' error classes and gap kinds from `actions.json`, each error the client raises itself carrying its code and class from `client-codes.json`, a repeated refused request counted a retry only where the retry rules allow one, a record that stops retrying while a retry remains refused, and the waits before retries and the warnings response headers issue derived; each POST carries its body, each build posts the body its spec, narrowing, layout and, for a fetch or a mint, party totals give, each dataset's columns are the reading-order rule's order of the downloaded Parquet files' columns, each Search request posts the whole body its steps derive, each poll names the operation the client holds, and each signed download is the URL its redirect names; each body file a record names is a bare file name in `scenarios/` and no symlink; every absolute URL a record holds is a signed URL on the storage host with the redacted query, or a `location` under the base URL; each `DatasetRef` example gets the verdict it states, each reading-order vector orders its columns and rows as the rule does, and each file-selector vector's query is the one the rule builds. It does not import `dfo_client_contract`, whose `tests/test_rules.py` replays the rule vectors. Also a PEP 723 script: ```bash uv run tools/validate_client_contract.py ``` -`capture_client_contract.py` refreshes the client contract from dfo-apis. `openapi --dfo-apis PATH` builds the composed app in that checkout's environment, in process, and writes `served/openapi.json` and `served/SOURCE.json`, refusing a checkout with uncommitted changes or untracked files and an environment out of step with its `uv.lock`; `record` runs the contract's scenarios against a base URL, dev by default, writes the records with every absolute URL's query redacted and a local app's signed URLs written as storage URLs, refusing a record that still names any other host, derives the synthetic ones and validates the result, and `record --offline` derives the outcomes again from the records with no request. A recorded scenario holds only exchanges the server made. `record` refuses a recorded scenario that a canned exchange answered, and a synthetic derivation makes no request. Nothing in a record marks an exchange as canned, so `validate_client_contract.py` cannot check this rule and the capture enforces it. It reads and writes a binary body only through a bare file name beside its record, and refuses a name that is absolute or leaves that directory, or a symlink, before it reads or writes anything. It imports its outcome rules from `validate_client_contract.py`. `python/client-contract/README.md` has the recipe. A PEP 723 script whose tests run without network or pyarrow: +`capture_client_contract.py` refreshes the client contract from dfo-apis. `openapi --dfo-apis PATH` builds the composed app in that checkout's environment, in process, and writes `served/openapi.json` and `served/SOURCE.json`, refusing a checkout with uncommitted changes or untracked files and an environment out of step with its `uv.lock`; `record` runs the contract's scenarios against a base URL, dev by default, writes the records with every absolute URL's query redacted and a local app's signed URLs written as storage URLs, refusing a record that still names any other host, derives the synthetic ones and validates the result, and `record --offline` derives the outcomes again from the records with no request. A recorded scenario holds only exchanges the server made. `record` refuses a recorded scenario that a canned exchange answered, and a synthetic derivation makes no request. Nothing in a record marks an exchange as canned, so `validate_client_contract.py` cannot check this rule and the capture enforces it. It reads and writes a binary body only through a bare file name beside its record, and refuses a name that is absolute or leaves that directory, or a symlink, before it reads or writes anything. It imports its outcome rules from `validate_client_contract.py`. `python/client-contract/README.md` has the recipe. A PEP 723 script whose tests run without network: ```bash uv run tools/capture_client_contract.py record diff --git a/tools/capture_client_contract.py b/tools/capture_client_contract.py index 1f71bca..18492f6 100644 --- a/tools/capture_client_contract.py +++ b/tools/capture_client_contract.py @@ -110,6 +110,7 @@ header_warnings, held_spec, load_reading_columns, + load_selector_keys, may_retry, newest_plain, owns_body_file, @@ -150,9 +151,6 @@ # totals of open party roles left out, and so the rendering a ref it writes # records. RENDERING = fetch_rendering(None) -# The reading slots of the reading-order rule, which orders each dataset's -# columns as a client presents them, or None when the rule file cannot be read. -READING_COLUMNS = load_reading_columns(PACKAGE_DIR) # The response headers a client reads, and so the ones a record keeps, beside # a `cache-control` that marks the response shared-cacheable, which excuses it # from carrying `x-dfo-trace-id`. @@ -592,6 +590,7 @@ def ref(self) -> dict[str, Any]: def dataset_outcome( members: Sequence[Member], + reading: Sequence[Sequence[str]], *, missing: Sequence[Mapping[str, Any]] = (), search_warning: bool | None = None, @@ -610,13 +609,8 @@ def dataset_outcome( for member in members ] dataset: dict[str, Any] = {"rows": sum(member.rows for member in members)} - if READING_COLUMNS is None: - raise SystemExit( - "rules/reading-order.json is missing or holds no reading_columns, so " - "no dataset's columns can be ordered" - ) dataset["columns"] = presented_columns( - [member.table for member in members], READING_COLUMNS + [member.table for member in members], reading ) if len(members) > 1: dataset["part_rows"] = [ @@ -627,10 +621,6 @@ def dataset_outcome( return {"dataset": {**dataset, **summary}} -def parquet_shape(data: bytes) -> ServedTable: - return served_table(data) - - def _reached[T](results: list[T], step: str) -> T: """The result a step produced. A step that refused cannot be acted on.""" if not results: @@ -677,6 +667,36 @@ def _step_options( return step +@dataclass(frozen=True, slots=True) +class ClientRules: + """The rule tables a client reads from `rules/`: the reading-order rule's + reading slots and the file-selector rule's query keys.""" + + reading: Sequence[Sequence[str]] + selector_keys: Sequence[str] + + +def load_client_rules(package_dir: Path) -> ClientRules: + """The rule tables under `package_dir`, read before any request. + + Raises: + SystemExit: A rule file is missing or does not hold its table. + """ + reading = load_reading_columns(package_dir) + if reading is None: + raise SystemExit( + f"{package_dir / 'rules' / 'reading-order.json'} is missing or holds no " + "reading_columns, so no dataset's columns can be ordered" + ) + selector_keys = load_selector_keys(package_dir) + if selector_keys is None: + raise SystemExit( + f"{package_dir / 'rules' / 'file-selector.json'} is missing or holds no " + "query_keys, so no file selector query can be built" + ) + return ClientRules(reading, selector_keys) + + @dataclass class Session: """A client over a recorder: each step appends to `call` and `expect`.""" @@ -684,7 +704,8 @@ class Session: recorder: Recorder actions: Mapping[str, Mapping[str, Any]] client_codes: Mapping[str, Mapping[str, Any]] - read_shape: Callable[[bytes], ServedTable] = parquet_shape + rules: ClientRules + read_shape: Callable[[bytes], ServedTable] = served_table sleep: Callable[[float], None] = time.sleep call: list[dict[str, Any]] = field(default_factory=list) expect: list[dict[str, Any]] = field(default_factory=list) @@ -819,7 +840,9 @@ def run() -> dict[str, Any]: self.build(parts, spec, None, rendering, strict=strict) for parts, spec in readings ] - return dataset_outcome(members, missing=gaps, search_warning=warns) + return dataset_outcome( + members, self.rules.reading, missing=gaps, search_warning=warns + ) options = _step_options( step, layout=layout, strict=strict, party_totals=party_totals @@ -856,7 +879,7 @@ def fetch_spec( def run() -> dict[str, Any]: rendering = fetch_rendering(layout, party_totals=party_totals) built.append(self.build([], spec, scope, rendering, strict=strict)) - return dataset_outcome(built) + return dataset_outcome(built, self.rules.reading) options = _step_options( step, layout=layout, strict=strict, party_totals=party_totals @@ -868,7 +891,9 @@ def fetch_ref(self, ref: dict[str, Any]) -> None: """The exact versions a ref names, through the lookup order.""" def run() -> dict[str, Any]: - return dataset_outcome([self._stored(member) for member in ref["members"]]) + return dataset_outcome( + [self._stored(member) for member in ref["members"]], self.rules.reading + ) self._step({"op": "fetch", "ref": ref}, run) @@ -883,11 +908,13 @@ def run() -> dict[str, Any]: self._step({"op": "newest_version", "spec": spec.step()}, run) return found[0] if found else None - def mint(self, spec: Spec, *, layout: str | None = None) -> str: + def mint( + self, spec: Spec, *, layout: str | None = None, party_totals: bool = False + ) -> str: operation: list[str] = [] def run() -> dict[str, Any]: - rendering = rendering_for(layout) + rendering = fetch_rendering(layout, party_totals=party_totals) body = _build_body(spec, None, rendering) response = self._api("POST", "/datasets", body) resource = self._acted(response) @@ -900,7 +927,10 @@ def run() -> dict[str, Any]: self._step( _step_options( - {"op": "mint", "spec": spec.step()}, layout=layout, strict=False + {"op": "mint", "spec": spec.step()}, + layout=layout, + strict=False, + party_totals=party_totals, ), run, ) @@ -930,7 +960,7 @@ def result(self) -> None: def run() -> dict[str, Any]: manifest = self._wait(minted.resource, minted.headers, minted.request) member = self._member([], minted.spec, manifest, minted.rendering, False) - return dataset_outcome([member]) + return dataset_outcome([member], self.rules.reading) self._step({"op": "result"}, run) @@ -1136,6 +1166,7 @@ def _stored_file( member["spec"]["spec_id"], member["content_version"], member["artifact"]["rendering"], + self.rules.selector_keys, ) try: return self._download(path, digest, origin=False), None @@ -1243,7 +1274,9 @@ def _build_body( # version the server has built answers inline, so each long build needs a spec # the server has not built, and the run takes the first start year in # LONG_BUILD_YEARS whose spec it has not. Each record builds both for good, so -# the next record takes later start years. A flow side the question leaves unnamed +# the next record starts the scan at the year after the one LONG_BUILD_RECORDS +# holds, and a later start year estimates fewer rows, so the scan stops at the +# first year below the band. A flow side the question leaves unnamed # reads as its source's whole total, so both sides name groups read as their # members: about 3,700 donor and recipient pairs of one variable over some 60 # years. A variable broken down by sector would list thousands of variables in @@ -1253,6 +1286,8 @@ def _build_body( " from {year} to 2024" ) LONG_BUILD_YEARS = range(1960, 2001) +# The record holding each donor group's last long build. +LONG_BUILD_RECORDS = {"DAC": "mint-async-build", "OECD": "operation-resume"} NARROW_WINDOW = (2020, 2023) REBOUND_ORIGIN = "country/FRA" # The DAC donors' flow to the G77, narrowed to one recipient and year with its @@ -1274,7 +1309,10 @@ class Context: transport: Transport actions: Mapping[str, Mapping[str, Any]] client_codes: Mapping[str, Mapping[str, Any]] - read_shape: Callable[[bytes], ServedTable] = parquet_shape + rules: ClientRules + read_shape: Callable[[bytes], ServedTable] = served_table + # The first start year `async_spec` tries for each donor group. + long_build_starts: Mapping[str, int] = field(default_factory=dict) _cache: dict[str, Any] = field(default_factory=dict) def session(self, name: str, **recorder: Any) -> Session: @@ -1282,6 +1320,7 @@ def session(self, name: str, **recorder: Any) -> Session: Recorder(name, self.base_url, self.transport, **recorder), self.actions, self.client_codes, + self.rules, read_shape=self.read_shape, ) @@ -1330,25 +1369,37 @@ def definite_ref(self) -> dict[str, Any]: def async_spec(self, donors: str) -> Spec: """The spec the LONG_BUILD question for `donors` answers at the first - start year in LONG_BUILD_YEARS whose answer is one definite part, whose - estimate is just above the sync budget, and whose spec no earlier year - or other donors answered and the server has not built. No such year - raises ShapeError.""" + start year in LONG_BUILD_YEARS, from `long_build_starts`, whose answer + is one definite part, whose estimate is just above the sync budget, + and whose spec no earlier year or other donors answered and the server + has not built. A year whose estimate is at or below the budget ends + the scan, since later years estimate fewer rows. + + Raises: + ShapeError: No year gives such a spec. + """ picks: dict[str, Spec] = self._cache.setdefault("long builds", {}) if donors not in picks: session = self.session("prerequisite") seen = {spec.spec_id for spec in picks.values()} - for year in LONG_BUILD_YEARS: + first = max(self.long_build_starts.get(donors, 0), LONG_BUILD_YEARS[0]) + for year in range(first, LONG_BUILD_YEARS[-1] + 1): answer = session.search(LONG_BUILD.format(donors=donors, year=year)) if answer.status != "definite" or len(answer.parts) != 1: continue (part,) = answer.parts rows = part["spec"]["resolved"].get("download", {}).get("rows") or 0 spec = _part_spec(part["spec"]) + if rows <= SYNC_ROWS: + raise ShapeError( + f"the {donors} donors have no long build left: from {year} " + f"the estimate, {rows:,} rows, is within the {SYNC_ROWS:,}-row " + "sync budget; change LONG_BUILD to a question with more rows" + ) # A start year before the source's first year clips to the same # spec as the years after it, so a spec already seen is built or # is the other donors' pick. - if not SYNC_ROWS < rows < 2 * SYNC_ROWS or spec.spec_id in seen: + if rows >= 2 * SYNC_ROWS or spec.spec_id in seen: continue seen.add(spec.spec_id) # `_newest` and not `newest_version`, which reads a refused @@ -1358,7 +1409,7 @@ def async_spec(self, donors: str) -> Spec: break else: raise ShapeError( - f"no start year from {LONG_BUILD_YEARS[0]} to" + f"no start year from {first} to" f" {LONG_BUILD_YEARS[-1]} gives the {donors} donors a spec" " just above the sync budget that the server has not built" ) @@ -1540,7 +1591,7 @@ def _fetch_ref_unknown_version(s: Session, ctx: Context) -> None: def _mint_async(s: Session, ctx: Context) -> None: - s.mint(ctx.async_spec("DAC")) + s.mint(ctx.async_spec("DAC"), party_totals=True) s.result() @@ -2490,6 +2541,7 @@ def _expired_file_selector(parent: dict[str, Any], ctx: Context) -> Derived: member["spec"]["spec_id"], member["content_version"], member["artifact"]["rendering"], + ctx.rules.selector_keys, ) selector = { "request": {"method": "GET", "path": path}, @@ -2823,6 +2875,7 @@ def _artifact_unlisted(parent: dict[str, Any], ctx: Context) -> Derived: member["spec"]["spec_id"], member["content_version"], member["artifact"]["rendering"], + ctx.rules.selector_keys, ) exchanges[index]["request"] = {"method": "GET", "path": path} return _same(parent, exchanges) @@ -4425,6 +4478,15 @@ def _empty_narrowing(parent: dict[str, Any], ctx: Context) -> Derived: "request.", _choice_refused("Congo [Nowhere]"), ), + Synthetic( + "choose-refused-label-in-two-groups", + "search-place-ambiguous-twice", + 'A choice of "Congo [DRC]" follows the search.', + "Both entity_choice groups offer an option labelled Congo [DRC], so the label " + "matches two options and the choice raises InvalidRequestError with no request. " + "A choice in such a part names its option by index.", + _choice_refused("Congo [DRC]"), + ), Synthetic( "choose-candidate-by-label", "answer-candidates-choose-fetch", @@ -4690,6 +4752,7 @@ def drive(session: Session, call: Sequence[dict[str, Any]]) -> None: session.mint( _step_spec(spec), layout=step.get("layout"), + party_totals=step.get("party_totals", False), ) case {"op": "operation", "operation_id": operation_id}: session.resume(operation_id, layout=step.get("layout")) @@ -4709,6 +4772,7 @@ def rederive_recorded(name: str, ctx: Context, directory: Path) -> None: Recorder(name, served.base_url, served), ctx.actions, ctx.client_codes, + ctx.rules, read_shape=ctx.read_shape, sleep=lambda _: None, ) @@ -4728,6 +4792,37 @@ def rederive_recorded(name: str, ctx: Context, directory: Path) -> None: ) +def recorded_start_year(record: Mapping[str, Any]) -> int | None: + """The start year of the first window a record's spec definitions + request: a step's spec, then a manifest an exchange answers.""" + definitions = [ + step["spec"]["definition"] for step in record["call"] if "spec" in step + ] + for exchange in record["exchanges"]: + body = exchange["response"].get("json") + manifest = body.get("manifest", body) if isinstance(body, dict) else None + if isinstance(manifest, dict) and "definition" in manifest: + definitions.append(manifest["definition"]) + for definition in definitions: + window = definition.get("identity", {}).get("requested_window") or {} + if "start_year" in window: + return window["start_year"] + return None + + +def long_build_starts(directory: Path) -> dict[str, int]: + """The first start year to try for each donor group: the year after the + one its LONG_BUILD_RECORDS record built, which the server has built.""" + starts = {} + for donors, name in LONG_BUILD_RECORDS.items(): + path = directory / f"{name}.json" + if path.exists(): + year = recorded_start_year(json.loads(path.read_text(encoding="utf-8"))) + if year is not None: + starts[donors] = year + 1 + return starts + + def run_record(args: argparse.Namespace) -> int: package_dir: Path = args.package_dir directory: Path = args.scenarios_dir or package_dir / "scenarios" @@ -4741,7 +4836,18 @@ def run_record(args: argparse.Namespace) -> int: _offline if args.offline else urllib_transport, load_actions(package_dir), load_client_codes(package_dir), + load_client_rules(package_dir), + long_build_starts=long_build_starts(directory), ) + if not args.offline: + # A long build's spec is the scan most likely to fail, so it runs + # before any scenario is recorded. + for donors, name in LONG_BUILD_RECORDS.items(): + if name in chosen: + try: + ctx.async_spec(donors) + except ShapeError as exc: + raise SystemExit(f"{name}: {exc}") from exc count = 0 for definition in RECORDED: if definition.name not in chosen: diff --git a/tools/tests/test_capture_client_contract.py b/tools/tests/test_capture_client_contract.py index 9954ced..e546fc3 100644 --- a/tools/tests/test_capture_client_contract.py +++ b/tools/tests/test_capture_client_contract.py @@ -32,6 +32,7 @@ SCENARIOS = capture.PACKAGE_DIR / "scenarios" ACTIONS = capture.load_actions(capture.PACKAGE_DIR) CLIENT_CODES = capture.load_client_codes(capture.PACKAGE_DIR) +RULES = capture.load_client_rules(capture.PACKAGE_DIR) def _load(name: str) -> dict[str, Any]: @@ -75,7 +76,7 @@ def test_the_recorder_reproduces_each_record_from_its_steps(name: str) -> None: served = capture.Served(record, SCENARIOS) recorder = capture.Recorder(name, served.base_url, served) session = capture.Session( - recorder, ACTIONS, CLIENT_CODES, read_shape=_shape, sleep=lambda _: None + recorder, ACTIONS, CLIENT_CODES, RULES, read_shape=_shape, sleep=lambda _: None ) capture.drive(session, record["call"]) assert served.pending == [], "the client skipped a recorded exchange" @@ -236,7 +237,11 @@ def test_each_derived_record_follows_from_its_parent(name: str) -> None: definition = next(d for d in capture.SYNTHETIC if d.name == name) parent = _load(record["origin"]["derived_from"]) ctx = capture.Context( - capture.DEV_BASE_URL, capture.Served(parent, SCENARIOS), ACTIONS, CLIENT_CODES + capture.DEV_BASE_URL, + capture.Served(parent, SCENARIOS), + ACTIONS, + CLIENT_CODES, + RULES, ) derived = definition.derive(copy.deepcopy(parent), ctx) assert record["origin"]["edit"] == definition.edit @@ -253,7 +258,12 @@ def test_an_offline_rederivation_rewrites_only_the_expect(tmp_path: Path) -> Non stale["expect"][1]["error"]["code"] = None capture.write_record(tmp_path, stale, {}) ctx = capture.Context( - capture.DEV_BASE_URL, capture._offline, ACTIONS, CLIENT_CODES, read_shape=_shape + capture.DEV_BASE_URL, + capture._offline, + ACTIONS, + CLIENT_CODES, + RULES, + read_shape=_shape, ) capture.rederive_recorded(name, ctx, tmp_path) fresh = (tmp_path / f"{name}.json").read_text(encoding="utf-8") @@ -287,7 +297,12 @@ def test_an_offline_rederivation_refuses_a_body_file_outside_its_directory( exchange["response"]["body_file"] = body_file record.write_text(json.dumps(stale), encoding="utf-8") ctx = capture.Context( - capture.DEV_BASE_URL, capture._offline, ACTIONS, CLIENT_CODES, read_shape=_shape + capture.DEV_BASE_URL, + capture._offline, + ACTIONS, + CLIENT_CODES, + RULES, + read_shape=_shape, ) with pytest.raises(SystemExit) as refused: capture.rederive_recorded(name, ctx, scenarios) @@ -342,7 +357,7 @@ def transport(method: str, url: str, **kwargs: Any) -> Any: raise AssertionError(f"{method} {url} reached the network") definition = capture.Recorded("s", "None.", run) - ctx = capture.Context(capture.DEV_BASE_URL, transport, ACTIONS, CLIENT_CODES) + ctx = capture.Context(capture.DEV_BASE_URL, transport, ACTIONS, CLIENT_CODES, RULES) with pytest.raises(SystemExit, match="1 were canned"): capture.record_recorded(definition, ctx, tmp_path) assert list(tmp_path.iterdir()) == [] @@ -358,7 +373,7 @@ def transport(method: str, url: str, **kwargs: Any) -> Any: raise AssertionError(f"{method} {url} reached the network") definition = capture.Synthetic("s", "parent", "None.", "None.", derive) - ctx = capture.Context(capture.DEV_BASE_URL, transport, ACTIONS, CLIENT_CODES) + ctx = capture.Context(capture.DEV_BASE_URL, transport, ACTIONS, CLIENT_CODES, RULES) with pytest.raises(SystemExit, match="makes no request"): capture.record_synthetic(definition, ctx, tmp_path) @@ -427,15 +442,15 @@ def test_several_members_stack_with_a_part_column_first() -> None: 3, ["value", "time_period"], ) - dataset = capture.dataset_outcome([member, unnamed])["dataset"] + dataset = capture.dataset_outcome([member, unnamed], RULES.reading)["dataset"] assert dataset["columns"] == ["part", "time_period", "value", "x", "stat_var"] assert dataset["rows"] == 5 assert dataset["part_rows"] == [ {"part": "A.", "rows": 2}, {"part": "p; q", "rows": 3}, ] - assert "part_rows" not in capture.dataset_outcome([other])["dataset"] - columns = capture.dataset_outcome([member])["dataset"]["columns"] + assert "part_rows" not in capture.dataset_outcome([other], RULES.reading)["dataset"] + columns = capture.dataset_outcome([member], RULES.reading)["dataset"]["columns"] assert columns == ["value", "x", "stat_var"] @@ -553,10 +568,34 @@ def fake_run(args: list[str], **kwargs: Any) -> Any: } +@pytest.mark.parametrize("rule", ["reading-order", "file-selector"]) +def test_a_record_run_without_a_rule_table_stops_before_any_request( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, rule: str +) -> None: + """The rule tables load from --package-dir when the run starts, so a + broken one costs no live call.""" + for name in ("actions.json", "client-codes.json"): + (tmp_path / name).write_bytes((capture.PACKAGE_DIR / name).read_bytes()) + (tmp_path / "rules").mkdir() + for path in (capture.PACKAGE_DIR / "rules").glob("*.json"): + if path.stem != rule: + (tmp_path / "rules" / path.name).write_bytes(path.read_bytes()) + + def transport(method: str, url: str, **kwargs: Any) -> Any: + raise AssertionError(f"{method} {url} reached the network") + + monkeypatch.setattr(capture, "urllib_transport", transport) + with pytest.raises(SystemExit, match=f"{rule}.json is missing"): + capture.main( + ["--package-dir", str(tmp_path), "record", "--only", "answer-declined"] + ) + + def _long_build_ctx( monkeypatch: pytest.MonkeyPatch, answers: Mapping[tuple[str, int], tuple[str, int]], built: set[str], + starts: Mapping[str, int] | None = None, ) -> tuple[Any, list[str], list[str]]: """A context whose searches answer `answers`, (donors, year) to spec id and row estimate, and whose versions route lists a version of each spec @@ -582,7 +621,14 @@ def newest(self: Any, spec_id: str) -> str | None: monkeypatch.setattr(capture.Session, "search", search) monkeypatch.setattr(capture.Session, "_newest", newest) - ctx = capture.Context(capture.DEV_BASE_URL, capture._offline, ACTIONS, CLIENT_CODES) + ctx = capture.Context( + capture.DEV_BASE_URL, + capture._offline, + ACTIONS, + CLIENT_CODES, + RULES, + long_build_starts=starts or {}, + ) return ctx, questions, probed @@ -595,7 +641,7 @@ def test_a_long_build_takes_the_first_unbuilt_spec_in_the_band( ctx, questions, probed = _long_build_ctx( monkeypatch, { - ("DAC", 1960): ("spec_small", capture.SYNC_ROWS), + ("DAC", 1960): ("spec_large", 2 * capture.SYNC_ROWS), ("DAC", 1961): ("spec_built", in_band), ("DAC", 1962): ("spec_built", in_band), ("DAC", 1963): ("spec_dac", in_band), @@ -623,3 +669,40 @@ def test_a_long_build_with_no_unbuilt_year_is_refused( ) with pytest.raises(capture.ShapeError, match="from 1960 to 1961"): ctx.async_spec("DAC") + + +def test_a_long_build_scan_stops_at_the_first_year_within_the_budget( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A later start year estimates fewer rows, so none after it can be in + the band, and the scan stops without searching them.""" + ctx, questions, _ = _long_build_ctx( + monkeypatch, + { + ("DAC", 1960): ("spec_a", capture.SYNC_ROWS), + ("DAC", 1961): ("spec_b", capture.SYNC_ROWS + 1), + }, + set(), + ) + with pytest.raises(capture.ShapeError, match="no long build left: from 1960"): + ctx.async_spec("DAC") + assert len(questions) == 1 + + +def test_a_long_build_scan_starts_after_the_recorded_year( + monkeypatch: pytest.MonkeyPatch, +) -> None: + in_band = capture.SYNC_ROWS + 1 + ctx, questions, _ = _long_build_ctx( + monkeypatch, + {("DAC", 1960): ("spec_a", in_band), ("DAC", 1968): ("spec_b", in_band)}, + set(), + starts={"DAC": 1968}, + ) + assert ctx.async_spec("DAC").spec_id == "spec_b" + assert questions == [capture.LONG_BUILD.format(donors="DAC", year=1968)] + + +def test_the_long_build_scans_start_after_the_recorded_picks() -> None: + """The records hold the DAC pick at 1967 and the OECD pick at 1968.""" + assert capture.long_build_starts(SCENARIOS) == {"DAC": 1968, "OECD": 1969} diff --git a/tools/tests/test_validate_client_contract.py b/tools/tests/test_validate_client_contract.py index 9dffff3..c37f435 100644 --- a/tools/tests/test_validate_client_contract.py +++ b/tools/tests/test_validate_client_contract.py @@ -27,6 +27,8 @@ from typing import Any from urllib.parse import urlsplit +import pyarrow as pa +import pyarrow.parquet as pq import pytest _MODULE_PATH = Path(__file__).resolve().parent.parent / "validate_client_contract.py" @@ -165,14 +167,23 @@ def change(outcome: Any) -> None: return change +def _selector_query_with_cut(doc: Any) -> None: + """The file selector request asks for the ref member's whole rendering, + `without_party_totals` included, which the server refuses.""" + exchange = next( + e for e in doc["exchanges"] if "/file?" in e["request"].get("path", "") + ) + exchange["request"]["path"] += "&without_party_totals=true" + + def _vector(name: str, change: Callable[[Any], None]) -> Callable[[Any], None]: return lambda doc: change(next(v for v in doc["vectors"] if v["name"] == name)) def _served_order_after_reading_columns(holder: Any) -> None: """Moves the varying co-operation modality pair back to its served place, - just before `crsPurpose`, as the rule placed it before it put varying - columns ahead of constant ones.""" + just before `crsPurpose`, an order that ignores whether a column + varies.""" columns = holder["dataset"]["columns"] if "dataset" in holder else holder["columns"] varying = ["crsCooperationModality", "crsCooperationModality_label"] for name in varying: @@ -1715,6 +1726,14 @@ def change(doc: Any) -> None: "the steps: _UnfitError('exchange 0 posts" ), ), + ( + "scenarios/mint-async-build.json", + lambda doc: doc["call"][0].pop("party_totals"), + ( + "scenarios/mint-async-build.json: the exchanges do not fit the " + "steps: _UnfitError('exchange 0 posts" + ), + ), ( _FETCH, _set_outcome(0, lambda o: o["dataset"]["columns"].reverse()), @@ -1788,6 +1807,60 @@ def change(doc: Any) -> None: "reading_columns" ), ), + ( + "rules/reading-order.json", + _vector( + "ties-keep-served-order", lambda v: v["members"][0]["rows"][0].pop() + ), + "vector 'ties-keep-served-order' member 0 has a row of", + ), + ( + "rules/reading-order.json", + _vector( + "ties-keep-served-order", + lambda v: v["members"][0]["rows"][0].__setitem__(4, "2022"), + ), + "vector 'ties-keep-served-order' mixes JSON types in column", + ), + ( + "scenarios/fetch-ref-expired-file-selector.json", + _selector_query_with_cut, + ( + "scenarios/fetch-ref-expired-file-selector.json: the exchanges do " + "not fit the steps" + ), + ), + ( + "scenarios/fetch-ref-artifact-unlisted.json", + _selector_query_with_cut, + ( + "scenarios/fetch-ref-artifact-unlisted.json: the exchanges do not " + "fit the steps" + ), + ), + ( + "rules/file-selector.json", + _vector( + "long-cut", + lambda v: v.update( + query="format=parquet&bundle=false&without_party_totals=true" + ), + ), + "vector 'long-cut' gives a query other than selector_query's", + ), + ( + "rules/file-selector.json", + _vector( + "keys-in-listed-order", + lambda v: v.update(query="layout=frame&bundle=true&format=csv"), + ), + "vector 'keys-in-listed-order' gives a query other than selector_query's", + ), + ( + "rules/file-selector.json", + lambda doc: doc.pop("query_keys"), + "rules/file-selector.json: missing or unreadable, or holds no query_keys", + ), ], ids=[ "strict-pointer-unresolved", @@ -2031,6 +2104,7 @@ def change(doc: Any) -> None: "candidate-spec-not-an-object-read", "party-totals-left-in-by-default", "party-totals-kept-without-the-option", + "mint-party-totals-left-in-by-default", "columns-out-of-reading-order", "stacked-columns-out-of-reading-order", "served-order-of-other-columns-swapped", @@ -2042,6 +2116,13 @@ def change(doc: Any) -> None: "reading-order-series-interleaved-by-year", "reading-order-null-sorted-first", "reading-order-without-reading-columns", + "reading-order-vector-row-ragged", + "reading-order-vector-column-mixes-types", + "selector-query-carries-the-cut", + "unlisted-artifact-selector-query-carries-the-cut", + "file-selector-vector-carries-the-cut", + "file-selector-vector-out-of-key-order", + "file-selector-without-query-keys", ], ) def test_a_broken_contract_is_reported( @@ -2055,10 +2136,77 @@ def test_a_broken_contract_is_reported( assert any(expected in problem for problem in problems), problems -def test_a_missing_reading_order_rule_is_reported(package_dir: Path) -> None: - (package_dir / "rules" / "reading-order.json").unlink() +@pytest.mark.parametrize( + ("left", "right", "same"), + [ + (1, 1.0, True), + (True, 1, False), + (float("nan"), float("nan"), True), + (None, None, True), + ("1", 1, False), + ([1, True], [1.0, True], True), + ], +) +def test_cells_are_the_same_value_as_json_values( + left: Any, right: Any, same: bool +) -> None: + assert validator.same_value(left, right) is same + table = validator.table_from_rows(["c"], [[left], [right]]) + assert ("c" in table.same) is same + + +def _parquet(table: pa.Table) -> bytes: + sink = pa.BufferOutputStream() + pq.write_table(table, sink) + return sink.getvalue().to_pybytes() + + +def test_a_served_table_reads_list_struct_and_nan_columns() -> None: + """Columns `unique` cannot hash are judged cell by cell, and a NaN column + holds one value, in one table and over stacked ones.""" + data = _parquet( + pa.table( + { + "listed": [[1, 2], [1, 2]], + "struct": [{"a": 1}, {"a": 2}], + "nan": [float("nan"), float("nan")], + } + ) + ) + table = validator.served_table(data) + assert set(table.same) == {"listed", "nan"} + assert not validator._varies("nan", [table, table]) + assert validator._varies("struct", [table]) + + +def test_a_spec_built_in_two_renderings_keeps_both() -> None: + """A ref member may record either rendering a scenario built its spec in.""" + cut = {"format": "parquet", "bundle": False, "without_party_totals": True} + plain = {"format": "parquet", "bundle": False} + scenario = { + "exchanges": [ + { + "request": { + "method": "POST", + "path": "/datasets", + "json": {"options": o}, + }, + "response": { + "status": 200, + "json": {"definition": {"spec_id": "spec_a"}}, + }, + } + for o in (cut, plain) + ] + } + assert validator.posted_renderings(scenario) == {"spec_a": [cut, plain]} + + +@pytest.mark.parametrize("rule", ["reading-order", "file-selector"]) +def test_a_missing_rule_table_is_reported(package_dir: Path, rule: str) -> None: + (package_dir / "rules" / f"{rule}.json").unlink() assert any( - problem.startswith("rules/reading-order.json: missing") + problem.startswith(f"rules/{rule}.json: missing") for problem in validator.validate() ) diff --git a/tools/validate_client_contract.py b/tools/validate_client_contract.py index c84959f..20e64ce 100644 --- a/tools/validate_client_contract.py +++ b/tools/validate_client_contract.py @@ -63,14 +63,15 @@ REPLAY_ORIGIN with the recorded origin a response names moved there. A dataset's columns are the downloaded files' columns, read with pyarrow by `served_table` and ordered by `presented_columns`, which puts the - columns whose values vary before those that hold one value, its rows and + columns whose values vary, as `same_value` judges cells, before those + that hold one value, its rows and per-part row counts are left to the Parquet bodies, and each download hashes to its artifact's digest or the outcome says it does not. Each Search request a record holds posts the input the steps derive: the question, a choice's data_query or dataset_resubmit entries, or a member's identity with the entity groups of the roles its build left bound as they were, and each build posts the body `build_body` gives for the spec the step builds, its - narrowing, its layout and, for a fetch, its party totals, and each Search request the whole body + narrowing, its layout and, for a fetch or a mint, its party totals, and each Search request the whole body `search_body` gives. A POST carries its body, a refused request repeated is a retry only as `may_retry` allows, a record that ends a run of retries while `may_retry` holds does not fit, each poll names the operation the @@ -95,7 +96,9 @@ - Each rule file under `rules/` validates against `rules.schema.json`, names the rule its file name names, and names each vector once, and each reading-order vector's `columns` and `rows` are what `presented_rows` - gives. A missing or unreadable `rules/reading-order.json` is a problem. + gives, and each file-selector vector's `query` is what `selector_query` + gives. A missing or unreadable `rules/reading-order.json` or + `rules/file-selector.json` is a problem. It does not import `dfo_client_contract`, whose tests replay the rule vectors. `tools/capture_client_contract.py` imports the outcome @@ -106,6 +109,7 @@ import hashlib import json +import math import re import sys from collections.abc import Callable, Collection, Iterator, Mapping, Sequence @@ -1799,17 +1803,17 @@ class _UnfitError(Exception): def rendering_for(layout: str | None) -> dict[str, Any]: - """The build request's `options` for a mint's or an operation's - `layout`, long by default.""" + """The rendering a build posts for `layout`, long by default, with + party totals kept: what a resumed operation downloads.""" return dict(LAYOUT_RENDERINGS[layout or "long"]) def fetch_rendering( layout: str | None, *, party_totals: bool = False ) -> dict[str, Any]: - """The build request's `options` for a fetch of an answer or a spec: - the layout's rendering, with `without_party_totals` unless the caller - keeps party totals.""" + """The build request's `options` for a fetch of an answer or a spec, or + a mint: the layout's rendering, with `without_party_totals` unless the + caller keeps party totals.""" rendering = rendering_for(layout) if not party_totals: rendering[WITHOUT_PARTY_TOTALS] = True @@ -1827,6 +1831,29 @@ class ServedTable: same: Mapping[str, Any] +def same_value(left: Any, right: Any) -> bool: + """Whether two cells hold the same value, as the reading-order rule + judges it: numbers by value, so 1 and 1.0 are the same, booleans apart + from numbers, NaN the same as NaN, null the same as null, and lists and + structs item by item.""" + if isinstance(left, bool) or isinstance(right, bool): + return type(left) is type(right) and left == right + if isinstance(left, float) and isinstance(right, float): + return left == right or (math.isnan(left) and math.isnan(right)) + if isinstance(left, list) and isinstance(right, list): + return len(left) == len(right) and all(map(same_value, left, right)) + if isinstance(left, dict) and isinstance(right, dict): + return left.keys() == right.keys() and all( + same_value(value, right[key]) for key, value in left.items() + ) + return left == right + + +def _one_value(values: Sequence[Any]) -> bool: + """Whether `values` hold one value as `same_value` judges it.""" + return all(same_value(value, values[0]) for value in values[1:]) + + def table_from_rows( columns: Sequence[str], rows: Sequence[Sequence[Any]] ) -> ServedTable: @@ -1834,7 +1861,7 @@ def table_from_rows( same = { name: rows[0][index] for index, name in enumerate(columns) - if rows and all(row[index] == rows[0][index] for row in rows) + if rows and _one_value([row[index] for row in rows]) } return ServedTable(tuple(columns), len(rows), same) @@ -1856,9 +1883,9 @@ def served_table(data: bytes) -> ServedTable: ) from exc same = {} for name in table.column_names if table.num_rows else (): - values = table.column(name).unique() - if len(values) == 1: - same[name] = values[0].as_py() + values = table.column(name).to_pylist() + if _one_value(values): + same[name] = values[0] return ServedTable(tuple(table.column_names), table.num_rows, same) @@ -1884,7 +1911,7 @@ def _varies(name: str, tables: Sequence[ServedTable]) -> bool: if name in table.columns and name not in table.same: return True values.append(table.same.get(name)) - return any(value != values[0] for value in values[1:]) + return not _one_value(values) @dataclass(frozen=True, slots=True) @@ -1988,6 +2015,18 @@ def load_reading_columns(package_dir: Path) -> list[list[str]] | None: return None +def load_selector_keys(package_dir: Path) -> list[str] | None: + """The query keys `rules/file-selector.json` lists, or None when the file + does not hold them, which the validator reports.""" + try: + keys = _load(package_dir / "rules" / "file-selector.json")["query_keys"] + except (OSError, ValueError, KeyError, TypeError): + return None + if isinstance(keys, list) and keys and all(isinstance(k, str) for k in keys): + return keys + return None + + def posted_scope(step: Mapping[str, Any]) -> dict[str, Any] | None: """The `input.scope` a fetch of a spec posts for the step's narrowing, or None when it narrows nothing. The server's @@ -2172,16 +2211,25 @@ def digest_of(data: bytes) -> str: return "sha256:" + hashlib.sha256(data).hexdigest() +def selector_query(rendering: Mapping[str, Any], keys: Sequence[str]) -> str: + """The file selector's query for `rendering`, as the file-selector rule + builds it: each of `keys` the rendering holds, in that order.""" + return "&".join( + f"{key}={str(rendering[key]).lower()}" for key in keys if key in rendering + ) + + def selector_path( - spec_id: str, content_version: str, rendering: Mapping[str, Any] + spec_id: str, + content_version: str, + rendering: Mapping[str, Any], + keys: Sequence[str], ) -> str: - """The file selector's path for a stored version, its rendering as the - query.""" - query = "&".join( - f"{name}={str(value).lower() if isinstance(value, bool) else value}" - for name, value in rendering.items() + """The file selector's path for a stored version in `rendering`, its + query from `selector_query`.""" + return ( + f"/datasets/{spec_id}/{content_version}/file?{selector_query(rendering, keys)}" ) - return f"/datasets/{spec_id}/{content_version}/file?{query}" # The storage host a signed download names, the only host outside the API a @@ -2435,6 +2483,7 @@ class _Replay: client_codes: Mapping[str, Mapping[str, Any]] body_dir: Path reading: Sequence[Sequence[str]] | None + selector_keys: Sequence[str] | None cursor: int = 0 answer: Answer | None = None minted: _Minted | None = None @@ -2787,10 +2836,13 @@ def stored_file( if error.code not in FILE_GONE: raise refusal = error + if self.selector_keys is None: + raise _UnfitError("rules/file-selector.json lists no query_keys") path = selector_path( member["spec"]["spec_id"], member["content_version"], member["artifact"]["rendering"], + self.selector_keys, ) try: return self.download(path, digest, origin=False), None @@ -2913,7 +2965,9 @@ def _fetch_ref(self, step: dict[str, Any]) -> dict[str, Any]: return self.dataset(members, []) def _mint(self, step: dict[str, Any]) -> dict[str, Any]: - rendering = rendering_for(step.get("layout")) + rendering = fetch_rendering( + step.get("layout"), party_totals=step.get("party_totals", False) + ) exchange = self.posted_build(step["spec"], rendering, None) if exchange["response"]["status"] == 200: raise _UnfitError( @@ -3001,6 +3055,7 @@ def expected_outcomes( client_codes: Mapping[str, Mapping[str, Any]], body_dir: Path, reading: Sequence[Sequence[str]] | None, + selector_keys: Sequence[str] | None, ) -> list[dict[str, Any]]: """The outcome each of `record`'s steps reaches over its exchanges, its binary bodies read from `body_dir`, over @@ -3023,6 +3078,7 @@ def expected_outcomes( client_codes, body_dir, reading, + selector_keys, ) try: outcomes = [replay.step(step) for step in record["call"]] @@ -3278,6 +3334,7 @@ def _check_scenarios( proxy_codes: Mapping[str, Mapping[str, Any]], served_commit: str | None, reading: Sequence[Sequence[str]] | None, + selector_keys: Sequence[str] | None, ) -> list[str]: directory = PACKAGE_DIR / "scenarios" paths = sorted(directory.glob("*.json")) @@ -3375,6 +3432,7 @@ def _check_scenarios( client_codes=client_codes, body_dir=directory, reading=reading, + selector_keys=selector_keys, ) try: derived = replay(record, base_url=base_url) @@ -3556,11 +3614,12 @@ def _served_by(label: str, record: dict[str, Any]) -> list[str]: return problems -def posted_renderings(scenario: Mapping[str, Any]) -> dict[str, Any]: - """The rendering each build in `scenario` posted as its `options`, by - the spec_id the build answered: a manifest's own, or for a 202 the - manifest the poll of its operation answered done.""" - renderings: dict[str, Any] = {} +def posted_renderings(scenario: Mapping[str, Any]) -> dict[str, list[Any]]: + """The renderings the builds in `scenario` posted as their `options`, by + the spec_id each build answered: a manifest's own, or for a 202 the + manifest the poll of its operation answered done. A spec built in + several renderings lists each in build order.""" + renderings: dict[str, list[Any]] = {} by_operation: dict[str, Any] = {} for exchange in read_objects(scenario, "exchanges"): request = read_object(exchange, "request") or {} @@ -3573,11 +3632,11 @@ def posted_renderings(scenario: Mapping[str, Any]) -> dict[str, Any]: by_operation[operation_id] = options spec_id = read_string(read_object(manifest, "definition"), "spec_id") if spec_id is not None: - renderings[spec_id] = options + renderings.setdefault(spec_id, []).append(options) elif operation_id in by_operation and "manifest" in body: spec_id = read_string(read_object(manifest, "definition"), "spec_id") if spec_id is not None: - renderings[spec_id] = by_operation[operation_id] + renderings.setdefault(spec_id, []).append(by_operation[operation_id]) return renderings @@ -3601,10 +3660,10 @@ def _posted_by(label: str, record: dict[str, Any]) -> list[str]: problems.append( f"{label}: member {index}'s spec is built by no {name} build" ) - elif rendering != posted[spec_id]: + elif rendering not in posted[spec_id]: problems.append( f"{label}: member {index} records rendering {json.dumps(rendering)}, " - f"but {name} posted {json.dumps(posted[spec_id])}" + f"but {name} posted {' and '.join(map(json.dumps, posted[spec_id]))}" ) return problems @@ -3651,6 +3710,34 @@ def _check_refs(validator: jsonschema.Draft202012Validator) -> list[str]: # --- rules ------------------------------------------------------------------ +def _json_type(value: Any) -> str: + if isinstance(value, bool): + return "boolean" + return "number" if isinstance(value, int | float) else type(value).__name__ + + +def _unorderable(members: Sequence[Mapping[str, Any]]) -> str | None: + """Why a reading-order vector's members cannot be ordered, or None: a + row whose cells do not match its member's columns, or a column whose + cells, nulls aside, mix JSON types, which the rule gives no order.""" + types: dict[str, set[str]] = {} + for index, member in enumerate(members): + columns = member["columns"] + for row in member["rows"]: + if len(row) != len(columns): + return ( + f"member {index} has a row of {len(row)} cells for " + f"{len(columns)} columns" + ) + for name, cell in zip(columns, row, strict=True): + if cell is not None: + types.setdefault(name, set()).add(_json_type(cell)) + mixed = sorted(name for name, found in types.items() if len(found) > 1) + if mixed: + return f"mixes JSON types in column {mixed[0]!r}" + return None + + def _check_rules(validator: jsonschema.Draft202012Validator) -> list[str]: paths = sorted( p @@ -3673,18 +3760,28 @@ def _check_rules(validator: jsonschema.Draft202012Validator) -> list[str]: for name in sorted({n for n in names if names.count(n) > 1}) ) if document["rule"] == "reading-order": - problems.extend( - f"{label}: vector {vector['name']!r} orders its columns or rows " - f"otherwise than presented_rows" - for vector in document["vectors"] - if presented_rows( + for vector in document["vectors"]: + unfit = _unorderable(vector["members"]) + if unfit is not None: + problems.append(f"{label}: vector {vector['name']!r} {unfit}") + elif presented_rows( [ (member.get("part"), member["columns"], member["rows"]) for member in vector["members"] ], document["reading_columns"], - ) - != (vector["columns"], vector["rows"]) + ) != (vector["columns"], vector["rows"]): + problems.append( + f"{label}: vector {vector['name']!r} orders its columns or " + f"rows otherwise than presented_rows" + ) + if document["rule"] == "file-selector": + problems.extend( + f"{label}: vector {vector['name']!r} gives a query other than " + f"selector_query's" + for vector in document["vectors"] + if selector_query(vector["rendering"], document["query_keys"]) + != vector["query"] ) return problems @@ -3771,6 +3868,12 @@ def validate() -> list[str]: "reading_columns list of column-name lists, so no dataset's columns " "are checked" ) + selector_keys = load_selector_keys(PACKAGE_DIR) + if selector_keys is None: + problems.append( + "rules/file-selector.json: missing or unreadable, or holds no " + "query_keys list of names, so no file selector request is checked" + ) problems.extend( _check_scenarios( validators["scenario.schema.json"], @@ -3782,6 +3885,7 @@ def validate() -> list[str]: proxy_codes={row["code"]: row for row in proxy_rows}, served_commit=source.get("commit") if isinstance(source, dict) else None, reading=reading, + selector_keys=selector_keys, ) ) size = scenarios_size(PACKAGE_DIR / "scenarios") From 14555ff18e72c4ad526dfbc6b64959e2be177fed Mon Sep 17 00:00:00 2001 From: Jorge Rivera Date: Thu, 29 Oct 2026 16:55:22 +0100 Subject: [PATCH 2/2] fix(client-contract): read a resumed build's rendering and keep nested columns out of sort keys A resumed operation's result takes the rendering its build posted, and refs are checked against the build that served their file. List and struct columns never sort, the reference same_value is the validator's, selector queries lowercase booleans only, and the capture's OECD prerequisite keeps party totals. --- python/client-contract/CHANGELOG.md | 8 +- python/client-contract/README.md | 2 +- .../src/dfo_client_contract/rules/README.md | 6 +- .../rules/reading-order.json | 81 +++++++++++ .../rules/rules.schema.json | 8 +- python/client-contract/tests/test_rules.py | 39 +++++- tools/capture_client_contract.py | 22 +-- tools/tests/test_capture_client_contract.py | 35 ++++- tools/tests/test_validate_client_contract.py | 65 ++++++--- tools/validate_client_contract.py | 128 +++++++++++++----- 10 files changed, 316 insertions(+), 78 deletions(-) diff --git a/python/client-contract/CHANGELOG.md b/python/client-contract/CHANGELOG.md index 9928610..540c799 100644 --- a/python/client-contract/CHANGELOG.md +++ b/python/client-contract/CHANGELOG.md @@ -19,13 +19,13 @@ All notable changes to the `dfo-spec-kit-client-contract` Python package will be - A no_data part's gap kind comes from its own reason, whatever the answer's `diagnostics.timed_out` says. `answer-timed-out` gives its part the reason `timed_out`, and `answer-timed-out-keeps-reasons` pins a part that keeps `variable_not_resolved` as a `not_understood` gap in a timed-out answer. - Error outcomes give the error's `trace_id`, the body's, else the `x-dfo-trace-id` header's, else null. Scenario outcomes give `retry_waits`, the waits before each retry, and `warnings`, the warning codes a step's responses issued. Scenario steps take no `timeout`. - `dataset-ref.schema.json`, the frozen `DatasetRef` format, with accepted and rejected examples. -- `without_party_totals` in the ref's artifact rendering, `true` for a member fetched with party totals left out and absent for a plain rendering, with `ref-without-party-totals`, a default fetch's cut rendering, `ref-one-member`, a fetch with party totals and its plain rendering, and `ref-without-party-totals-false-rejected` as its examples. Each ref example's rendering is one its scenario's builds of its spec posted, and the validator checks it. +- `without_party_totals` in the ref's artifact rendering, `true` for a member fetched with party totals left out and absent for a plain rendering, with `ref-without-party-totals`, a default fetch's cut rendering, `ref-one-member`, a fetch with party totals and its plain rendering, and `ref-without-party-totals-false-rejected` as its examples. Each ref example's rendering is the one the build that answered its file posted, and the validator checks it. - `tools/capture_client_contract.py`, which captures `served/openapi.json` from a dfo-apis checkout in process and records the scenarios against a base URL, dev by default, or with `record --offline` derives their outcomes again from the records with no request. A local app's signed URLs are written as the storage URLs a deployment signs. `record` reads the rule tables from `--package-dir` before any request, and looks for each long build's spec from the year after the one its record built, stopping at the first start year whose estimate is within the sync budget. - `tools/validate_client_contract.py`, which checks the contract against its schemas, the served document and `registries/codes.json`, checks `client-codes.json` against them and against the reasons the key-lookup rule raises, and checks each scenario's `expect` against the outcomes its exchanges, `actions.json` and `client-codes.json` give, each Search request against the input its steps derive, a sentence lookup's `entity_groups` included, each build's `input.scope` against the narrowing its step names over the spec's bound roles, each build's options against its layout, each Search request's whole body, each download against its artifact's digest, each recorded scenario's commit against `served/SOURCE.json`, each problem document's code against `registries/codes.json` or, for the website proxy's, `proxy-codes.json`, each proxy problem document's content type, members and `retryable: true`, each 202 against an operation that reads `done`, and each absolute URL against the storage host, each ref example's members against the dataset members and downloaded files of the scenario it names, and the scenarios directory against a 16,000,000-byte budget for records and bodies together. A record that ends a run of retries while a retry remains does not fit. The outcomes include each long build's operation id, poll waits and resubmissions, and a `VersionUnavailableError`'s newest version. - The contract README states that a POST for a build identical to one pending or running returns that operation, so a client that stops waiting posts the same build to join it, and that `fetch` waits for a build with no whole-call deadline by default, with a download that delivers no byte for 30 s a retryable `network_error`. -- A fetch of an answer or a spec leaves out party totals by default: its build posts `without_party_totals: true` in `options`, so the server drops the group totals of the identity's open party roles, and a role the caller binds keeps what it binds. Fetch steps take `party_totals: true` for the client option, `party_totals` in Python and `partyTotals` in TypeScript, false by default, which posts no `without_party_totals`. The wire flag and the ref format are unchanged, and a ref records `without_party_totals: true` for a member fetched with the default. `fetch-spec-party-totals-left-out` and `fetch-spec-party-totals-kept` pin both on DAC2a ODA to Kenya in 2022 with the origin open. A mint leaves them out by default too, and its step takes `party_totals`, so its `result` answers the table a fetch of the spec answers; `mint-async-build` keeps them. -- The `reading-order` rule and its vectors, which carry rows: a client presents a dataset's columns with the reading columns first, the time period, provider, recipient, subject, purpose, value, unit, flow type and provenance, each slot filled by the long layout's label or the frame layout's code, the flow type by CRS `crsTypeOfFlow` or else DAC2a `dac2aAidType`, then the columns whose values differ between rows, then those that hold one value, each group in the server's order, with `stat_var` last. It sorts each member's rows by the filled slots without `value` and the time period, then the varying columns, then the time period, ascending with nulls last and ties in served order, so each series' years read together. Two cells hold the same value when they are equal as JSON values: numbers by value, so `1` and `1.0` are one value, booleans apart from numbers, and NaN the same as NaN. A vector's rows hold one cell per column and each column one JSON type, and `integer-and-float-are-one-value` pins `1` against `1.0`. Scenario outcomes give `columns` in that order, and the validator checks it against the served files, list and struct columns included, and reports a malformed vector. -- The `client-identity` rule and its vectors: every request a client sends to the API carries `X-DFO-Client: / ()`, such as `dfo-python/0.1.0 (CPython 3.14)` or `dfo-ts/0.1.0 (Node 22.18)`, `browser` as the runtime in a browser page or browser worker, and `other` in any other runtime, such as Cloudflare Workers, with `pattern` matching every well-formed value. A request to another origin, a signed storage URL included, carries none. A caller's headers or header provider naming `X-DFO-Client` in any letter case raises `InvalidRequestError` `header_invalid`, whose condition now says so, and `User-Agent` stays as each client sends it. A scenario replay checks the header on each `path` and `origin_path` request and its absence on each `url` request. +- A fetch of an answer or a spec leaves out party totals by default: its build posts `without_party_totals: true` in `options`, so the server drops the group totals of the identity's open party roles, and a role the caller binds keeps what it binds. Fetch steps take `party_totals: true` for the client option, `party_totals` in Python and `partyTotals` in TypeScript, false by default, which posts no `without_party_totals`. The wire flag and the ref format are unchanged, and a ref records `without_party_totals: true` for a member fetched with the default. `fetch-spec-party-totals-left-out` and `fetch-spec-party-totals-kept` pin both on DAC2a ODA to Kenya in 2022 with the origin open. A mint leaves them out by default too, and its step takes `party_totals`, so its `result` answers the table a fetch of the spec answers; `mint-async-build` keeps them. A resumed operation's `result` takes the rendering its build posted from the done manifest's artifact. +- The `reading-order` rule and its vectors, which carry rows: a client presents a dataset's columns with the reading columns first, the time period, provider, recipient, subject, purpose, value, unit, flow type and provenance, each slot filled by the long layout's label or the frame layout's code, the flow type by CRS `crsTypeOfFlow` or else DAC2a `dac2aAidType`, then the columns whose values differ between rows, then those that hold one value, each group in the server's order, with `stat_var` last. It sorts each member's rows by the filled slots without `value` and the time period, then the varying columns, then the time period, ascending with nulls last and ties in served order, so each series' years read together. Two cells hold the same value when they are equal as JSON values: numbers by value, so `1` and `1.0` are one value, booleans apart from numbers, NaN the same as NaN, lists item by item in order and structs field by field in any order. A column of lists or structs is no sort key. A vector's rows hold one cell per column and each column one JSON type, and `integer-and-float-are-one-value` and `nested-cells-compare-by-value` pin the comparison. Scenario outcomes give `columns` in that order, and the validator checks it against the served files, list and struct columns included, and reports a malformed vector. +- The `client-identity` rule and its vectors: every request a client sends to the API carries `X-DFO-Client: / ()`, such as `dfo-python/0.1.0 (CPython 3.14)` or `dfo-ts/0.1.0 (Node 22.18)`, `browser` as the runtime in a browser page or browser worker, and `other` in any other runtime, such as Cloudflare Workers, with `pattern` matching every well-formed value. A request to another origin, a signed storage URL included, carries none. A caller's headers or header provider naming `X-DFO-Client` in any letter case raises `InvalidRequestError` `header_invalid`, whose condition in `client-codes.json` names that case, and `User-Agent` stays as each client sends it. A scenario replay checks the header on each `path` and `origin_path` request and its absence on each `url` request. - The `role-keys` rule and its vectors: a caller narrows a spec with `entities=` keyed by `place`, `origin` and `destination`, and a client posts `place` as the wire role `observationAbout`. `Spec.roles`, `specRoles(spec)` and the clients' errors name the caller keys, and a flow spec takes both directions. A key outside the spec's roles raises `InvalidRequestError` `invalid_argument` naming the key and the spec's roles, so `observationAbout` on GDP is refused naming `place` and `financingRecipient` on a flow spec naming `origin` and `destination`. A spec that names no role posts `place` as `observationAbout` and other keys as typed, and refuses `observationAbout` naming `place`. - The client rules and their vectors: the credentials path and file format, the key lookup order, with an empty `api_key` refused as `key_empty` and an empty profile key as `profile_without_key`, the key format check, the base URL order, `base_url_invalid` for an invalid value, and the cache directory and layout. - The `base-url` rule states the whole validity rule, and seven new vectors pin it: no query, fragment, `.` or `..` segment, character outside visible ASCII, empty userinfo or non-decimal IPv4 host; trailing slashes dropped; a relative URL valid only in a TypeScript client inside a browser page. diff --git a/python/client-contract/README.md b/python/client-contract/README.md index 03ca5f7..312af1a 100644 --- a/python/client-contract/README.md +++ b/python/client-contract/README.md @@ -169,7 +169,7 @@ The records also fix these client rules: - A strict fetch raises the manifest's first warning above `info` severity as a `DataWarning` with that warning's code, before the sentence lookup and the download. - An identity resubmit whose answer is not one definite part of the built `spec_id`, with a sentence, raises `ContractError`. - The `frame` layout posts `{"format": "parquet", "bundle": false, "layout": "frame"}` as the build's `options` and downloads the artifact whose rendering's `layout` is `frame`; the `long` layout posts `{"format": "parquet", "bundle": false}` and downloads the bare Parquet artifact with any other layout. A fetch of an answer or a spec, and a mint, add `"without_party_totals": true` to either, as the next rule gives. A client matches an artifact on its rendering's format, bundle and frame layout only, and `without_party_totals` plays no part in the match. A fetch of an answer with several members in the `frame` layout raises `InvalidRequestError` before any build. -- A fetch of an answer or a spec, and a mint, leave out party totals by default. The build posts `"without_party_totals": true` in `options`, and the server drops the rows whose party, on a flow party role the identity binds no entity on, the code index declares a group total, before any reshape. A role the caller binds keeps what it binds, so a total named by its dcid still comes back. The client option is `party_totals`, false by default, in Python, and `partyTotals` in TypeScript, and a fetch or mint step's `party_totals: true` stands for it: the build posts no `without_party_totals` and keeps every total. The wire is the served rendering flag as it was; only the client's default changes. A cut that leaves rows out answers a version of its own, listed by the versions route with the plain build as its `base_content_version`, and a cut that leaves nothing out answers the plain version and its file. The manifest's artifact rendering carries `without_party_totals: true` in both cases. A mint posts the options a fetch of its spec posts, so its `result` answers the table that fetch answers, which `mint-async-build` pins with `party_totals: true`. A resumed operation posts no build, and its `result` answers the rendering the operation's build posted. `fetch-spec-party-totals-left-out` pins the default on a DAC2a spec whose origin the fetch leaves open, where the cut drops the donor group totals, and `fetch-spec-party-totals-kept` the option. +- A fetch of an answer or a spec, and a mint, leave out party totals by default. The build posts `"without_party_totals": true` in `options`, and the server drops the rows whose party, on a flow party role the identity binds no entity on, the code index declares a group total, before any reshape. A role the caller binds keeps what it binds, so a total named by its dcid still comes back. The client option is `party_totals`, false by default, in Python, and `partyTotals` in TypeScript, and a fetch or mint step's `party_totals: true` stands for it: the build posts no `without_party_totals` and keeps every total. The wire is the served rendering flag as it was; only the client's default changes. A cut that leaves rows out answers a version of its own, listed by the versions route with the plain build as its `base_content_version`, and a cut that leaves nothing out answers the plain version and its file. The manifest's artifact rendering carries `without_party_totals: true` in both cases. A mint posts the options a fetch of its spec posts, so its `result` answers the table that fetch answers, which `mint-async-build` pins with `party_totals: true`. A resumed operation posts no build, and its `result` answers the rendering the operation's build posted: the layout's rendering, with `without_party_totals: true` when the done manifest's artifact for that layout carries it. `fetch-spec-party-totals-left-out` pins the default on a DAC2a spec whose origin the fetch leaves open, where the cut drops the donor group totals, and `fetch-spec-party-totals-kept` the option. - A download asks the route that redirects to the signed URL, an artifact's `url` or the file selector, and fetches the URL its `location` names once. A files route answers 302 with the `location` the client fetches, or 200 with the file as its body, which the client hashes with no further request, which `fetch-spec-files-route-serves-file` pins. A refusal reads by the refusal rules, and any other status, another 2xx or another 3xx included, raises `ContractError` `unexpected_response` with that response's trace id, which `fetch-spec-files-route-no-content` pins with a 204 and `fetch-spec-files-route-temporary-redirect` with a 307. The route ask is an API request and follows the retry rules below, so a retryable refusal is retried within the budget and any other refusal, `artifact_expired` among them, raises at once. A failed fetch gets exactly one more round. A fetch that answers anything but 200, which `fetch-spec-signed-fetch-partial` pins with a 206, or bytes that do not hash to the artifact's digest ask the route again, under the same retry rules, and fetch again, with no wait between rounds. A second failure raises `ServerError` with `download_failed`, or `digest_mismatch` when the bytes are wrong. A network error or timeout on the fetch is retryable, so it starts the second round too, and when the second round fails the same way the client raises that failure as itself, `network_error` or `deadline_exceeded`. The replay holds no network failure, so no scenario pins that case. - A manifest artifact's `url` names its files route on the API's origin. A root-relative `url` resolves against the base URL's origin, and an absolute one on that origin is asked as its path and query, so both make the same `origin_path` request, which `fetch-spec-same-origin-artifact-url` pins. An artifact `url` on any other origin, the storage host's included, raises `ContractError` `contract_violation` before the sentence lookup and any request for the file, which `fetch-spec-foreign-artifact-url` and `fetch-spec-storage-artifact-url` pin. - A relative `location` resolves against the files route's URL, and the client fetches it on the API's origin as an `origin_path` request. A `location` on an origin other than the API's or `https://storage.googleapis.com` raises `ContractError` `contract_violation` before any fetch. A fetch on the storage host is bare and carries no API key. `fetch-spec-relative-location` pins the relative case. diff --git a/python/client-contract/src/dfo_client_contract/rules/README.md b/python/client-contract/src/dfo_client_contract/rules/README.md index 089940e..7eb40e4 100644 --- a/python/client-contract/src/dfo_client_contract/rules/README.md +++ b/python/client-contract/src/dfo_client_contract/rules/README.md @@ -93,11 +93,11 @@ The columns come in four groups: 1. the other columns that hold one value in every row, null included, in the order the server wrote them; 1. `stat_var`, last whether or not it varies. -A table with no row or one row has no column that varies. Two cells hold the same value when they are equal as JSON values: numbers by value, so `1` and `1.0` are one value, booleans apart from numbers, so `true` and `1` differ, and null the same as null. NaN, which a Parquet double column can hold, is the same as NaN. A list or struct cell is the same as another when their items are the same, in order. A dataset of several members puts `part` first and judges the groups over the stacked table, where a member's rows hold null in a column that member lacks. So a CRS series of Germany's basic health ODA to Nigeria, split by co-operation modality, reads `time_period`, `financingProvider_label`, `financingRecipient_label`, `crsPurpose_label`, `value`, `unit_label`, `crsTypeOfFlow_label` and `provenance_label`, then `crsCooperationModality` and `crsCooperationModality_label`, which tell its rows apart, then its 15 constant columns, `stat_var` last. +A table with no row or one row has no column that varies. Two cells hold the same value when they are equal as JSON values: numbers by value, so `1` and `1.0` are one value, booleans apart from numbers, so `true` and `1` differ, and null the same as null. NaN, which a Parquet double column can hold, is the same as NaN. A list cell is the same as another when their items are the same, in order, and a struct cell when it has the same fields with the same values, in any field order. A dataset of several members puts `part` first and judges the groups over the stacked table, where a member's rows hold null in a column that member lacks. So a CRS series of Germany's basic health ODA to Nigeria, split by co-operation modality, reads `time_period`, `financingProvider_label`, `financingRecipient_label`, `crsPurpose_label`, `value`, `unit_label`, `crsTypeOfFlow_label` and `provenance_label`, then `crsCooperationModality` and `crsCooperationModality_label`, which tell its rows apart, then its 15 constant columns, `stat_var` last. -The rows of each member are sorted by the filled slots in slot order, leaving out `value`, a measure, and the time period, then by the varying columns in presented order, `stat_var` after them when it varies, then by the time period, so each series' years read together. The keys of that series are the provider, recipient, purpose, unit, flow type and provenance labels, then `crsCooperationModality` and `crsCooperationModality_label`, then the time period, so it reads every A02 year, then every C01 year. Each key sorts ascending, numbers by value and strings by Unicode code point, with nulls last. Rows whose keys tie keep the order the server wrote them. The members follow one another in member order. +The rows of each member are sorted by the filled slots in slot order, leaving out `value`, a measure, and the time period, then by the varying columns in presented order, `stat_var` after them when it varies, then by the time period, so each series' years read together. A column that holds lists or structs is no sort key, since the rule gives them no order. The keys of that series are the provider, recipient, purpose, unit, flow type and provenance labels, then `crsCooperationModality` and `crsCooperationModality_label`, then the time period, so it reads every A02 year, then every C01 year. Each key sorts ascending, numbers by value and strings by Unicode code point, with nulls last. Rows whose keys tie keep the order the server wrote them. The members follow one another in member order. -A vector gives the dataset's `members`, each with its `part` when there are several, its `columns` and its `rows` as the server wrote them, and the `columns` and `rows` a client presents. Each row holds one cell per column, and the cells of a column, nulls aside, are all strings, all numbers or all booleans, since the rule orders no value against one of another type. The order moves columns and rows and changes no value. A client hashes a downloaded file before it orders anything, so the digest and the ref keep the server's file. +A vector gives the dataset's `members`, each with its `part` when there are several, its `columns` and its `rows` as the server wrote them, and the `columns` and `rows` a client presents. Each row holds one cell per column, and the cells of a column, nulls aside, are all strings, all numbers, all booleans, all lists or all structs, since the rule orders no value against one of another type. The order moves columns and rows and changes no value. A client hashes a downloaded file before it orders anything, so the digest and the ref keep the server's file. ## Client identity (`client-identity`) diff --git a/python/client-contract/src/dfo_client_contract/rules/reading-order.json b/python/client-contract/src/dfo_client_contract/rules/reading-order.json index 90c1f37..bd1db59 100644 --- a/python/client-contract/src/dfo_client_contract/rules/reading-order.json +++ b/python/client-contract/src/dfo_client_contract/rules/reading-order.json @@ -1441,6 +1441,87 @@ 2020 ] ] + }, + { + "name": "nested-cells-compare-by-value", + "members": [ + { + "columns": [ + "time_period", + "value", + "meta", + "tags", + "flags" + ], + "rows": [ + [ + 2021, + 5, + { + "a": 1, + "b": 2 + }, + [ + 1 + ], + [ + true + ] + ], + [ + 2020, + 4, + { + "b": 2, + "a": 1 + }, + [ + 1.0 + ], + [ + 1 + ] + ] + ] + } + ], + "columns": [ + "time_period", + "value", + "flags", + "meta", + "tags" + ], + "rows": [ + [ + 2020, + 4, + [ + 1 + ], + { + "b": 2, + "a": 1 + }, + [ + 1.0 + ] + ], + [ + 2021, + 5, + [ + true + ], + { + "a": 1, + "b": 2 + }, + [ + 1 + ] + ] + ] } ] } diff --git a/python/client-contract/src/dfo_client_contract/rules/rules.schema.json b/python/client-contract/src/dfo_client_contract/rules/rules.schema.json index e4680b6..8ccf5b4 100644 --- a/python/client-contract/src/dfo_client_contract/rules/rules.schema.json +++ b/python/client-contract/src/dfo_client_contract/rules/rules.schema.json @@ -875,7 +875,9 @@ "string", "number", "boolean", - "null" + "null", + "array", + "object" ] } } @@ -902,7 +904,9 @@ "string", "number", "boolean", - "null" + "null", + "array", + "object" ] } }, diff --git a/python/client-contract/tests/test_rules.py b/python/client-contract/tests/test_rules.py index e89a9e0..4dc86d9 100644 --- a/python/client-contract/tests/test_rules.py +++ b/python/client-contract/tests/test_rules.py @@ -204,11 +204,18 @@ def cache_entry(vector: dict[str, Any]) -> str: def same_value(left: Any, right: Any) -> bool: - """Numbers by value, booleans apart from numbers, NaN as NaN.""" + """Numbers by value, booleans apart from numbers, NaN as NaN, lists item + by item in order, structs field by field in any order.""" if isinstance(left, bool) or isinstance(right, bool): return type(left) is type(right) and left == right if isinstance(left, float) and isinstance(right, float): return left == right or (math.isnan(left) and math.isnan(right)) + if isinstance(left, list) and isinstance(right, list): + return len(left) == len(right) and all(map(same_value, left, right)) + if isinstance(left, dict) and isinstance(right, dict): + return left.keys() == right.keys() and all( + same_value(value, right[key]) for key, value in left.items() + ) return left == right @@ -246,6 +253,10 @@ def reading_order( if "stat_var" in served and varies("stat_var", stacked): keys.append("stat_var") keys += ["time_period"] if "time_period" in first else [] + nested = { + c for row in stacked for c, v in row.items() if isinstance(v, list | dict) + } + keys = [k for k in keys if k not in nested] rows: list[list[Any]] = [] for part in dict.fromkeys(row["part"] for row in stacked): block = [row for row in stacked if row["part"] == part] @@ -302,13 +313,37 @@ def posted_entities( def selector_query(rendering: dict[str, Any]) -> str: + def written(value: Any) -> str: + return ("true" if value else "false") if isinstance(value, bool) else value + return "&".join( - f"{key}={str(rendering[key]).lower()}" + f"{key}={written(rendering[key])}" for key in rule("file-selector")["query_keys"] if key in rendering ) +# The cells `same_value` judges, the same cases as the validator's tests; +# NaN has no JSON form, so no vector can carry it. +SAME_VALUE_CASES = [ + (1, 1.0, True), + (True, 1, False), + (float("nan"), float("nan"), True), + (None, None, True), + ("1", 1, False), + ([True], [1], False), + ([float("nan")], [float("nan")], True), + ([1], [1.0], True), + ({"a": 1, "b": 2}, {"b": 2, "a": 1.0}, True), + ({"a": True}, {"a": 1}, False), +] + + +@pytest.mark.parametrize(("left", "right", "same"), SAME_VALUE_CASES) +def test_same_value(left: Any, right: Any, same: bool) -> None: + assert same_value(left, right) is same + + @pytest.mark.parametrize("name", RULES) def test_every_rule_has_vectors(name: str) -> None: assert rule(name)["rule"] == name diff --git a/tools/capture_client_contract.py b/tools/capture_client_contract.py index 18492f6..d2f108d 100644 --- a/tools/capture_client_contract.py +++ b/tools/capture_client_contract.py @@ -121,7 +121,7 @@ problem_body, read_from, refusal_error, - rendering_for, + resumed_rendering, resubmitted_sentence, retry_wait, scenarios_size, @@ -642,13 +642,16 @@ class Minted: """The operation a mint started or a resume read: its resource, the headers that came with it, the build request a failed operation posts again (None for a resumed one), the spec the client holds, and the - rendering it downloads.""" + rendering the build posted, or for a resumed operation None and the + layout whose rendering `resumed_rendering` reads from the done + manifest.""" resource: dict[str, Any] headers: dict[str, str] request: dict[str, Any] | None spec: Spec | None - rendering: dict[str, Any] + rendering: dict[str, Any] | None + layout: str | None = None def _step_options( @@ -943,9 +946,7 @@ def run() -> dict[str, Any]: path = POLL_PATH.format(operation_id=operation_id) response = self._api("GET", path) resource = self._acted(response) - self.minted = Minted( - resource, response.headers, None, None, rendering_for(layout) - ) + self.minted = Minted(resource, response.headers, None, None, None, layout) return _operation_outcome(resource) step = {"op": "operation", "operation_id": operation_id} @@ -959,7 +960,8 @@ def result(self) -> None: def run() -> dict[str, Any]: manifest = self._wait(minted.resource, minted.headers, minted.request) - member = self._member([], minted.spec, manifest, minted.rendering, False) + rendering = minted.rendering or resumed_rendering(minted.layout, manifest) + member = self._member([], minted.spec, manifest, rendering, False) return dataset_outcome([member], self.rules.reading) self._step({"op": "result"}, run) @@ -1388,7 +1390,9 @@ def async_spec(self, donors: str) -> Spec: if answer.status != "definite" or len(answer.parts) != 1: continue (part,) = answer.parts - rows = part["spec"]["resolved"].get("download", {}).get("rows") or 0 + rows = part["spec"]["resolved"].get("download", {}).get("rows") + if rows is None: + continue spec = _part_spec(part["spec"]) if rows <= SYNC_ROWS: raise ShapeError( @@ -1419,7 +1423,7 @@ def running_operation(self) -> str: """The id of a long build another process started.""" if "operation" not in self._cache: self._cache["operation"] = self.session("prerequisite").mint( - self.async_spec("OECD") + self.async_spec("OECD"), party_totals=True ) return self._cache["operation"] diff --git a/tools/tests/test_capture_client_contract.py b/tools/tests/test_capture_client_contract.py index e546fc3..ba71bdf 100644 --- a/tools/tests/test_capture_client_contract.py +++ b/tools/tests/test_capture_client_contract.py @@ -593,7 +593,7 @@ def transport(method: str, url: str, **kwargs: Any) -> Any: def _long_build_ctx( monkeypatch: pytest.MonkeyPatch, - answers: Mapping[tuple[str, int], tuple[str, int]], + answers: Mapping[tuple[str, int], tuple[str, int | None]], built: set[str], starts: Mapping[str, int] | None = None, ) -> tuple[Any, list[str], list[str]]: @@ -706,3 +706,36 @@ def test_a_long_build_scan_starts_after_the_recorded_year( def test_the_long_build_scans_start_after_the_recorded_picks() -> None: """The records hold the DAC pick at 1967 and the OECD pick at 1968.""" assert capture.long_build_starts(SCENARIOS) == {"DAC": 1968, "OECD": 1969} + + +def test_a_long_build_scan_skips_a_year_without_an_estimate( + monkeypatch: pytest.MonkeyPatch, +) -> None: + ctx, _, _ = _long_build_ctx( + monkeypatch, + { + ("DAC", 1960): ("spec_a", None), + ("DAC", 1961): ("spec_b", capture.SYNC_ROWS + 1), + }, + set(), + ) + assert ctx.async_spec("DAC").spec_id == "spec_b" + + +def test_the_running_operation_keeps_party_totals( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The operation another process starts posts the plain rendering, which + the recorded resume and result read.""" + ctx, _, _ = _long_build_ctx( + monkeypatch, {("OECD", 1960): ("spec_oecd", capture.SYNC_ROWS + 1)}, set() + ) + minted: list[dict[str, Any]] = [] + + def mint(self: Any, spec: Any, **options: Any) -> str: + minted.append(options) + return "op_x" + + monkeypatch.setattr(capture.Session, "mint", mint) + assert ctx.running_operation() == "op_x" + assert minted == [{"party_totals": True}] diff --git a/tools/tests/test_validate_client_contract.py b/tools/tests/test_validate_client_contract.py index c37f435..ee02a33 100644 --- a/tools/tests/test_validate_client_contract.py +++ b/tools/tests/test_validate_client_contract.py @@ -2138,13 +2138,18 @@ def test_a_broken_contract_is_reported( @pytest.mark.parametrize( ("left", "right", "same"), + # The same cases as python/client-contract/tests/test_rules.py. [ (1, 1.0, True), (True, 1, False), (float("nan"), float("nan"), True), (None, None, True), ("1", 1, False), - ([1, True], [1.0, True], True), + ([True], [1], False), + ([float("nan")], [float("nan")], True), + ([1], [1.0], True), + ({"a": 1, "b": 2}, {"b": 2, "a": 1.0}, True), + ({"a": True}, {"a": 1}, False), ], ) def test_cells_are_the_same_value_as_json_values( @@ -2175,31 +2180,51 @@ def test_a_served_table_reads_list_struct_and_nan_columns() -> None: ) table = validator.served_table(data) assert set(table.same) == {"listed", "nan"} + assert table.nested == {"listed", "struct"} + groups = validator.column_groups([table], []) + assert groups.varying == ("struct",) and groups.sort_keys == [] assert not validator._varies("nan", [table, table]) assert validator._varies("struct", [table]) -def test_a_spec_built_in_two_renderings_keeps_both() -> None: - """A ref member may record either rendering a scenario built its spec in.""" +def test_a_spec_built_in_two_renderings_keys_each_by_its_file() -> None: + """A ref member's rendering is checked against the build that answered + its file, so each of a spec's two builds keeps its own rendering.""" cut = {"format": "parquet", "bundle": False, "without_party_totals": True} plain = {"format": "parquet", "bundle": False} - scenario = { - "exchanges": [ - { - "request": { - "method": "POST", - "path": "/datasets", - "json": {"options": o}, - }, - "response": { - "status": 200, - "json": {"definition": {"spec_id": "spec_a"}}, - }, - } - for o in (cut, plain) - ] - } - assert validator.posted_renderings(scenario) == {"spec_a": [cut, plain]} + + def build(options: dict[str, Any], digest: str) -> dict[str, Any]: + artifact = {"artifact_digest": digest, "rendering": options} + return { + "request": { + "method": "POST", + "path": "/datasets", + "json": {"options": options}, + }, + "response": { + "status": 200, + "json": {"definition": {"spec_id": "spec_a"}, "artifacts": [artifact]}, + }, + } + + scenario = {"exchanges": [build(cut, "sha256:a"), build(plain, "sha256:b")]} + posted = validator.posted_renderings(scenario) + assert posted == {"sha256:a": [cut], "sha256:b": [plain]} + + +def test_a_selector_query_writes_only_booleans_in_lower_case() -> None: + query = validator.selector_query( + {"format": "CSV", "bundle": True}, ["format", "bundle"] + ) + assert query == "format=CSV&bundle=true" + + +def test_a_resumed_result_takes_the_rendering_its_build_posted() -> None: + cut = {"format": "parquet", "bundle": False, "without_party_totals": True} + manifest = {"artifacts": [{"artifact_digest": "sha256:a", "rendering": cut}]} + assert validator.resumed_rendering(None, manifest) == cut + manifest["artifacts"][0]["rendering"] = {"format": "parquet", "bundle": False} + assert validator.resumed_rendering(None, manifest) == validator.rendering_for(None) @pytest.mark.parametrize("rule", ["reading-order", "file-selector"]) diff --git a/tools/validate_client_contract.py b/tools/validate_client_contract.py index 20e64ce..debb55d 100644 --- a/tools/validate_client_contract.py +++ b/tools/validate_client_contract.py @@ -1803,8 +1803,8 @@ class _UnfitError(Exception): def rendering_for(layout: str | None) -> dict[str, Any]: - """The rendering a build posts for `layout`, long by default, with - party totals kept: what a resumed operation downloads.""" + """The rendering for `layout`, long by default, with party totals + kept.""" return dict(LAYOUT_RENDERINGS[layout or "long"]) @@ -1820,22 +1820,39 @@ def fetch_rendering( return rendering +def resumed_rendering( + layout: str | None, manifest: Mapping[str, Any] +) -> dict[str, Any]: + """The rendering a resumed operation's build posted, as its done + manifest records it: the layout's rendering, with `without_party_totals` + when the artifact for that rendering carries the cut.""" + rendering = rendering_for(layout) + artifact = pick_artifact(manifest, rendering) + found = artifact.get("rendering") if artifact is not None else None + if isinstance(found, dict) and found.get(WITHOUT_PARTY_TOTALS) is True: + rendering[WITHOUT_PARTY_TOTALS] = True + return rendering + + @dataclass(frozen=True, slots=True) class ServedTable: """A served table as the reading-order rule reads it: its columns in the order the server wrote them, its row count, and the value of each column - that holds one value in every row. A column `same` leaves out varies.""" + that holds one value in every row. A column `same` leaves out varies. + `nested` names the columns holding lists or structs, which no row is + sorted by.""" columns: tuple[str, ...] rows: int same: Mapping[str, Any] + nested: frozenset[str] = frozenset() def same_value(left: Any, right: Any) -> bool: """Whether two cells hold the same value, as the reading-order rule judges it: numbers by value, so 1 and 1.0 are the same, booleans apart - from numbers, NaN the same as NaN, null the same as null, and lists and - structs item by item.""" + from numbers, NaN the same as NaN, null the same as null, lists item by + item in order, and structs field by field in any order.""" if isinstance(left, bool) or isinstance(right, bool): return type(left) is type(right) and left == right if isinstance(left, float) and isinstance(right, float): @@ -1863,7 +1880,12 @@ def table_from_rows( for index, name in enumerate(columns) if rows and _one_value([row[index] for row in rows]) } - return ServedTable(tuple(columns), len(rows), same) + nested = frozenset( + name + for index, name in enumerate(columns) + if any(isinstance(row[index], list | dict) for row in rows) + ) + return ServedTable(tuple(columns), len(rows), same, nested) @lru_cache(maxsize=64) @@ -1881,12 +1903,21 @@ def served_table(data: bytes) -> ServedTable: raise _UnfitError( f"a downloaded file is unreadable: not Parquet: {exc}" ) from exc + nested = frozenset( + field.name for field in table.schema if pa.types.is_nested(field.type) + ) same = {} for name in table.column_names if table.num_rows else (): - values = table.column(name).to_pylist() - if _one_value(values): - same[name] = values[0] - return ServedTable(tuple(table.column_names), table.num_rows, same) + column = table.column(name) + # Arrow's `unique` counts NaN once and cannot hash a nested type, + # whose cells `same_value` compares. + if name in nested: + values = column.to_pylist() + if _one_value(values): + same[name] = values[0] + elif len(distinct := column.unique()) == 1: + same[name] = distinct[0].as_py() + return ServedTable(tuple(table.column_names), table.num_rows, same, nested) def filled_slots( @@ -1928,6 +1959,7 @@ class ColumnGroups: same: tuple[str, ...] last: tuple[str, ...] last_varies: bool + nested: frozenset[str] = frozenset() @property def columns(self) -> list[str]: @@ -1939,10 +1971,12 @@ def sort_keys(self) -> list[str]: reading slots in slot order without `value`, a measure, and without the time period, then the varying columns in presented order, `stat_var` among them when it varies, then the time period, so each - series' years read together.""" + series' years read together. A column of lists or structs is no + key.""" keys = [c for c in self.first if c not in (VALUE, TIME_PERIOD)] keys += [*self.varying, *(self.last if self.last_varies else ())] - return keys + [TIME_PERIOD] * (TIME_PERIOD in self.first) + keys += [TIME_PERIOD] * (TIME_PERIOD in self.first) + return [key for key in keys if key not in self.nested] def column_groups( @@ -1962,6 +1996,7 @@ def column_groups( same=tuple(c for c in rest if c not in varying), last=(STAT_VAR,) if STAT_VAR in gathered else (), last_varies=STAT_VAR in gathered and _varies(STAT_VAR, tables), + nested=frozenset().union(*(table.nested for table in tables)), ) @@ -2215,10 +2250,17 @@ def selector_query(rendering: Mapping[str, Any], keys: Sequence[str]) -> str: """The file selector's query for `rendering`, as the file-selector rule builds it: each of `keys` the rendering holds, in that order.""" return "&".join( - f"{key}={str(rendering[key]).lower()}" for key in keys if key in rendering + f"{key}={_query_value(rendering[key])}" for key in keys if key in rendering ) +def _query_value(value: Any) -> str: + """A query value, a boolean written `true` or `false`.""" + if isinstance(value, bool): + return "true" if value else "false" + return str(value) + + def selector_path( spec_id: str, content_version: str, @@ -2461,13 +2503,16 @@ class _Minted: """An operation a mint started or a resume read, for the result step: its resource, the headers that came with it, the build request a failed operation resubmits (None for a resumed one), the spec the client holds, - and the rendering it downloads.""" + and the rendering the build posted, or for a resumed operation None and + the layout whose rendering `resumed_rendering` reads from the done + manifest.""" resource: dict[str, Any] headers: dict[str, str] request: dict[str, Any] | None spec: dict[str, Any] | None - rendering: dict[str, Any] + rendering: dict[str, Any] | None + layout: str | None = None @dataclass @@ -2984,8 +3029,7 @@ def _operation(self, step: dict[str, Any]) -> dict[str, Any]: exchange = self.call("GET", POLL_PATH.format(operation_id=step["operation_id"])) resource = self.acted_on(exchange, _OPERATION) headers = exchange["response"].get("headers", {}) - rendering = rendering_for(step.get("layout")) - self.minted = _Minted(resource, headers, None, None, rendering) + self.minted = _Minted(resource, headers, None, None, None, step.get("layout")) return _operation_outcome(resource) def _newest_version(self, step: dict[str, Any]) -> dict[str, Any]: @@ -2998,7 +3042,8 @@ def _result(self, step: dict[str, Any]) -> dict[str, Any]: raise _UnfitError("result follows no mint or operation") minted = self.minted manifest = self.long_build(minted.resource, minted.headers, minted.request) - member = self.member([], minted.spec, manifest, minted.rendering, False) + rendering = minted.rendering or resumed_rendering(minted.layout, manifest) + member = self.member([], minted.spec, manifest, rendering, False) return self.dataset([member], []) def dataset( @@ -3616,9 +3661,10 @@ def _served_by(label: str, record: dict[str, Any]) -> list[str]: def posted_renderings(scenario: Mapping[str, Any]) -> dict[str, list[Any]]: """The renderings the builds in `scenario` posted as their `options`, by - the spec_id each build answered: a manifest's own, or for a 202 the - manifest the poll of its operation answered done. A spec built in - several renderings lists each in build order.""" + the digest of the artifact each build's manifest lists for them: a + manifest's own, or for a 202 the manifest the poll of its operation + answered done. Builds that answered the same file list each rendering in + build order.""" renderings: dict[str, list[Any]] = {} by_operation: dict[str, Any] = {} for exchange in read_objects(scenario, "exchanges"): @@ -3630,20 +3676,30 @@ def posted_renderings(scenario: Mapping[str, Any]) -> dict[str, list[Any]]: options = (read_object(request, "json") or {}).get("options") if operation_id is not None: by_operation[operation_id] = options - spec_id = read_string(read_object(manifest, "definition"), "spec_id") - if spec_id is not None: - renderings.setdefault(spec_id, []).append(options) elif operation_id in by_operation and "manifest" in body: - spec_id = read_string(read_object(manifest, "definition"), "spec_id") - if spec_id is not None: - renderings.setdefault(spec_id, []).append(by_operation[operation_id]) + options = by_operation[operation_id] + else: + continue + digest = _artifact_digest(manifest, options) + if digest is not None: + renderings.setdefault(digest, []).append(options) return renderings +def _artifact_digest(manifest: Any, options: Any) -> str | None: + """The digest of the artifact `manifest` lists for the posted `options`, + or None when either does not read.""" + if not isinstance(options, dict) or not {"format", "bundle"} <= options.keys(): + return None + artifacts = read_objects(manifest, "artifacts") + artifact = pick_artifact({"artifacts": artifacts}, options) + return read_string(artifact, "artifact_digest") + + def _posted_by(label: str, record: dict[str, Any]) -> list[str]: """Where a ref example's member records a rendering other than the one - the build of its spec in its `scenario` posted, since a ref records the - rendering its fetch posted.""" + the build that answered its file in its `scenario` posted, since a ref + records the rendering its fetch posted.""" ref = record.get("ref") members = ref.get("members") if isinstance(ref, dict) else None name = record.get("scenario") @@ -3653,17 +3709,17 @@ def _posted_by(label: str, record: dict[str, Any]) -> list[str]: posted = posted_renderings(_load(path)) problems = [] for index, member in enumerate(members): - spec = read_object(member, "spec") - spec_id = read_string(spec, "spec_id") - rendering = read_object(read_object(member, "artifact"), "rendering") - if spec_id not in posted: + artifact = read_object(member, "artifact") + digest = read_string(artifact, "digest") + rendering = read_object(artifact, "rendering") + if digest not in posted: problems.append( - f"{label}: member {index}'s spec is built by no {name} build" + f"{label}: member {index}'s file is answered by no {name} build" ) - elif rendering not in posted[spec_id]: + elif rendering not in posted[digest]: problems.append( f"{label}: member {index} records rendering {json.dumps(rendering)}, " - f"but {name} posted {' and '.join(map(json.dumps, posted[spec_id]))}" + f"but {name} posted {' and '.join(map(json.dumps, posted[digest]))}" ) return problems