From 924e275b8f1fa1198270ef5267b658af958d87aa Mon Sep 17 00:00:00 2001 From: Jorge Rivera Date: Sun, 1 Nov 2026 09:08:45 +0100 Subject: [PATCH 1/2] feat(descriptions): carry each series' publisher codes A card or dataset version description can now state the publisher's own indicator codes behind its series (World Bank IDS's DT.DIS.BLAT.CB.CD and the like) as about.source_codes, and each drawn value its variables declare can carry its own. A code is {provenance, code, exact?} in Search's order and with its meaning of exact. A list holds 1 to 50 codes and is left out past that, as drawn.values is. A computed version and a source card take none, and a card drawn from a computed version takes no exact code. The components catalogue manifest carries the schema's new digest. --- python/descriptions/CHANGELOG.md | 7 + python/descriptions/README.md | 1 + .../schemas/description.schema.json | 97 +++++++++++- ...ion-card-computed-exact-code-rejected.json | 143 ++++++++++++++++++ .../description-card-source-codes.json | 138 +++++++++++++++++ ...scription-source-codes-empty-rejected.json | 131 ++++++++++++++++ .../catalogue/manifest.json | 2 +- 7 files changed, 514 insertions(+), 5 deletions(-) create mode 100644 python/descriptions/src/dfo_descriptions/vectors/description-card-computed-exact-code-rejected.json create mode 100644 python/descriptions/src/dfo_descriptions/vectors/description-card-source-codes.json create mode 100644 python/descriptions/src/dfo_descriptions/vectors/description-source-codes-empty-rejected.json diff --git a/python/descriptions/CHANGELOG.md b/python/descriptions/CHANGELOG.md index 6a59d01..3fb4ada 100644 --- a/python/descriptions/CHANGELOG.md +++ b/python/descriptions/CHANGELOG.md @@ -149,5 +149,12 @@ and this package adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 members of one side with no partner on the other. The `sum` and `compare` descriptions cover the paired form: members of `data` and `other` pair by role through `on`, and a call to `compare` returns `data` minus `other`. +- `about.source_codes` on a card and a dataset version, and `source_codes` on + a card's drawn value: the publishers' own series codes for the variables + covered, as `{provenance, code, exact?}` in the shape and order of Search's + `stat_vars.source_codes`, 1 to 50 of them, absent when there are none or + more. `exact: true` marks a code that names the whole of what lists it. A + computed version and a source card refuse `about.source_codes`, and a card + drawn from a computed version refuses `exact`. [unreleased]: https://github.com/DevelopmentFinanceObservatory/spec-kit/commits/main/python diff --git a/python/descriptions/README.md b/python/descriptions/README.md index 8bd92d1..5bb1845 100644 --- a/python/descriptions/README.md +++ b/python/descriptions/README.md @@ -151,6 +151,7 @@ A description carries `describes`, `title`, the five blocks `about`, `sources`, - An open dimension with a known `count` carries `top`, up to two values by variable count, `more`, the count of the rest, and `values`, the route that pages them, provider included. A dimension whose values are unknown has `count: null` with a gap and none of the three. - A source that is not an input carries `publisher`, `licence` and `series`, each `null` with a gap when unknown. A dataset version has no inputs, so its sources carry no `content_version`. - `about.measure` carries a `definition`, `null` with gap `about.measure.definition` when nothing defines the measure. Each `about.fixed[].value`, on a version or a card, may carry a `definition`, the graph's description of the value, 1 to 4096 characters, absent when the graph holds none. A card's `about.drawn.dimensions[]` may list the `values` it draws on that dimension, each with its `definition`, in the order the card first draws them. The list holds 1 to 50 values, and its presence means it names every value the card draws there. A list longer or shorter than the dimension's `count`, or one that repeats a `ref`, breaks the structure rules, and a card that draws none or more than 50 leaves `values` out. +- `about.source_codes` lists the codes publishers file the covered variables under, from each variable's `sourceIndicatorCode` arc, as `{provenance, code, exact?}`, sorted by provenance and then code, each once, as Search serves `stat_vars.source_codes`. A card lists those of every series it draws, and a series of a computed version takes the codes of the variables it was computed from. A dataset version lists those of the one variable its definition resolved to. A card's drawn value on a dimension the variables declare carries the codes of the drawn variables holding it. A list holds 1 to 50 codes and is absent when there are none or more. `exact: true` marks a code that names the whole of what lists it: every series there is read from that provenance alone and is one the code names, and the provenance gives none of them another code. A code without it names variables the series are built from or hold beside others. A computed version and a source card carry no `about.source_codes`, and a card drawn from a computed version carries no exact code. - Every reference is `{ref, label}`, and every unit, variable side and input version outside `steps` carries its `label` too. A label, source name, publisher or licence taken from the evidence stored at mint, in place of today's vocabulary, sits in an object carrying `at_mint: true`. - A computed version's inputs, in `sources`, `freshness.inputs` and `cite.inputs`, name each version it was computed from by `id` and `content_version`, with a `label`. An input that no step retrieves, as on a version published without steps, has an unknown id: it is served with `id: null`, or in `sources` with `ref: null` and no `description` link, and a gap naming that member. Its label is the one stored with it, else its content version with a gap. Only an input, which carries `content_version`, may leave its `ref` null. - `steps` is a step list whose every step adds `text`, the composer's sentence for that step, or `null` when the record holds none, with a gap. Both schemas build a step from `step_members` in `steps.schema.json`: a stored step closes it with `unevaluatedProperties: false`, so a stored list refuses `text`, and a served step requires `text` and closes it the same way. diff --git a/python/descriptions/src/dfo_descriptions/schemas/description.schema.json b/python/descriptions/src/dfo_descriptions/schemas/description.schema.json index 59af581..e9a95a8 100644 --- a/python/descriptions/src/dfo_descriptions/schemas/description.schema.json +++ b/python/descriptions/src/dfo_descriptions/schemas/description.schema.json @@ -124,7 +124,8 @@ "fixed": false, "varies": false, "variables": false, - "drawn": false + "drawn": false, + "source_codes": false } }, "freshness": { @@ -218,7 +219,8 @@ "measure": false, "fixed": false, "drawn": false, - "unit": false + "unit": false, + "source_codes": false } }, "freshness": { @@ -253,7 +255,29 @@ "about": { "properties": { "measure": false, - "fixed": false + "fixed": false, + "source_codes": { + "$ref": "#/$defs/built_from_codes" + }, + "drawn": { + "properties": { + "dimensions": { + "items": { + "properties": { + "values": { + "items": { + "properties": { + "source_codes": { + "$ref": "#/$defs/built_from_codes" + } + } + } + } + } + } + } + } + } } } } @@ -412,6 +436,67 @@ "minLength": 1, "maxLength": 4096 }, + "drawn_value": { + "description": "A value a card draws on a dimension: `ref`, `label`, `at_mint` and `definition` as in `defined_ref`, and `source_codes`, the codes of the drawn variables that hold the value, present only on a dimension the variables themselves declare, absent when none of them has a code.", + "type": "object", + "additionalProperties": false, + "required": ["ref", "label"], + "properties": { + "ref": { + "$ref": "steps.schema.json#/$defs/text" + }, + "label": { + "$ref": "steps.schema.json#/$defs/text" + }, + "at_mint": { + "const": true + }, + "definition": { + "$ref": "#/$defs/definition" + }, + "source_codes": { + "$ref": "#/$defs/source_codes" + } + } + }, + "source_codes": { + "description": "Publisher series codes, each once, sorted by provenance and then code, as Search serves `stat_vars.source_codes`.", + "type": "array", + "minItems": 1, + "maxItems": 50, + "uniqueItems": true, + "items": { + "$ref": "#/$defs/source_code" + } + }, + "built_from_codes": { + "description": "Codes none of which is exact, as a card drawn from a computed version lists them: its series are built from the variables the codes name.", + "items": { + "properties": { + "exact": false + } + } + }, + "source_code": { + "description": "A code a publisher files a series under, such as World Bank `DT.TDS.DEGG.CD`, from the variable's `sourceIndicatorCode` arc: `provenance`, the dcid of the provenance issuing it, and `code`, upper-cased. `exact: true` when the code names the whole of what lists it: every series there is read from `provenance` alone and is one the code names, `provenance` gives none of them another code, and the version is a dataset version. Absent, the series are built from the codes listed, or hold others beside them.", + "type": "object", + "additionalProperties": false, + "required": ["provenance", "code"], + "properties": { + "provenance": { + "$ref": "steps.schema.json#/$defs/text" + }, + "code": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "$ref": "steps.schema.json#/$defs/one_line" + }, + "exact": { + "const": true + } + } + }, "input_ref": { "description": "A computed version's input as served: its id, content version and label. `id` is `null` when the version holds no step that retrieves the input, with a gap naming it; the label is then the one stored with the input, else its content version with a gap.", "type": "object", @@ -715,6 +800,10 @@ "unit": { "$ref": "#/$defs/unit" }, + "source_codes": { + "description": "The publishers' own codes for the variables the description covers: on a card, those of every series it draws, a series of a computed version taking the codes of the variables it was computed from; on a dataset version, those of the one variable its definition resolved to. Absent when none of them has a code, and when they hold more than 50.", + "$ref": "#/$defs/source_codes" + }, "drawn": { "description": "A card's selection against its version: per dimension, how many values it draws of how many the version holds; and its window against the version's period.", "type": "object", @@ -743,7 +832,7 @@ "minItems": 1, "maxItems": 50, "items": { - "$ref": "#/$defs/defined_ref" + "$ref": "#/$defs/drawn_value" } } } diff --git a/python/descriptions/src/dfo_descriptions/vectors/description-card-computed-exact-code-rejected.json b/python/descriptions/src/dfo_descriptions/vectors/description-card-computed-exact-code-rejected.json new file mode 100644 index 0000000..00a2372 --- /dev/null +++ b/python/descriptions/src/dfo_descriptions/vectors/description-card-computed-exact-code-rejected.json @@ -0,0 +1,143 @@ +{ + "id": "description-card-computed-exact-code-rejected", + "description": "A card drawn from a computed version whose `about.source_codes` names an exact code is rejected by the schema, since its series are built from the variables the codes name.", + "schema": "description", + "input": { + "describes": { + "kind": "card_revision", + "component_id": "cmp_Ab3dE5gH7k", + "revision": 2 + }, + "title": "Gross ODA disbursements per gross national income, 2010 to 2024", + "about": { + "drawn": { + "dimensions": [ + { + "property": { + "ref": "crsPurpose", + "label": "sector" + }, + "count": 3, + "of": 19, + "values": [ + { + "ref": "CRSPurposeCode12220", + "label": "basic health care", + "definition": "Basic and primary health care programmes; paramedical and nursing care programmes; supply of drugs, medicines, and vaccines related to basic health care; and activities aimed at achieving universal health coverage." + }, + { + "ref": "CRSPurposeCode12110", + "label": "health policy and administrative management", + "definition": "Health sector policy, planning, and programmes; aid to health ministries and public health administration; institution capacity building and advice; medical insurance programmes; health system strengthening and health governance; and unspecified health activities." + }, + { + "ref": "CRSPurposeCode12250", + "label": "infectious disease control", + "definition": "Immunisation and prevention and control of infectious and parasitic diseases, including diarrhoeal diseases, vector-borne diseases, viral diseases, mycosis, helminthiasis, zoonosis, diseases caused by other bacteria and viruses, and pediculosis. Excludes malaria, tuberculosis, COVID-19, and HIV/AIDS and other sexually transmitted diseases." + } + ] + } + ], + "series": 3, + "period": { + "start": "2010", + "end": "2024", + "grain": "year" + } + }, + "unit": { + "ref": "CurrencyUSD", + "label": "US dollars", + "prices": "current" + }, + "source_codes": [ + { + "provenance": "provenance/WorldBankInternationalDebtStatistics", + "code": "DT.DIS.BLAT.CB.CD", + "exact": true + } + ] + }, + "sources": [ + { + "ref": "ops_3c9e1f7a2b8d4e6f", + "label": "Gross ODA disbursements per gross national income", + "content_version": "ocv_8d2a6f1c9e3b7a5d", + "description": "/api/presentation/frames/ops_3c9e1f7a2b8d4e6f/ocv_8d2a6f1c9e3b7a5d/description" + } + ], + "freshness": { + "latest_period": "2024", + "series_reaching_it": 2, + "estimated_values": null, + "provisional_values": null + }, + "steps": [ + { + "id": "s1", + "op": "select", + "input": { + "id": "ops_3c9e1f7a2b8d4e6f", + "content_version": "ocv_8d2a6f1c9e3b7a5d", + "label": "Gross ODA disbursements per gross national income", + "at_mint": true + }, + "policy": "card_selection", + "params": { + "coordinates": { + "crsPurpose": [ + "CRSPurposeCode12220", + "CRSPurposeCode12110", + "CRSPurposeCode12250" + ] + }, + "window": { + "start": "2010", + "end": "2024" + } + }, + "kept": { + "series": 3 + }, + "text": "Drew 3 series of gross disbursements to Nigeria for health, 2010 to 2024." + } + ], + "caveats": [ + { + "kind": "provisional", + "periods": [ + "2024" + ], + "series": [ + { + "ref": "CRSPurposeCode12250", + "label": "infectious disease control" + } + ], + "text": "Values of infectious disease control in 2024 are provisional." + } + ], + "cite": { + "title": "Gross ODA disbursements per gross national income", + "text": "Gross ODA disbursements and Gross national income (2026), processed by the Development Finance Observatory", + "component_id": "cmp_Ab3dE5gH7k", + "revision": 2, + "id": "ops_3c9e1f7a2b8d4e6f", + "content_version": "ocv_8d2a6f1c9e3b7a5d", + "retrieved": "2026-09-28", + "data": "/api/presentation/frames/ops_3c9e1f7a2b8d4e6f/ocv_8d2a6f1c9e3b7a5d" + }, + "gaps": [ + "about.measure.definition", + "freshness.estimated_values", + "freshness.provisional_values" + ], + "built_from": { + "facts_schema": 1, + "vocabulary": "voc_5e1d9a3c7b2f8e64" + } + }, + "error": { + "by": "schema" + } +} diff --git a/python/descriptions/src/dfo_descriptions/vectors/description-card-source-codes.json b/python/descriptions/src/dfo_descriptions/vectors/description-card-source-codes.json new file mode 100644 index 0000000..4fc628f --- /dev/null +++ b/python/descriptions/src/dfo_descriptions/vectors/description-card-source-codes.json @@ -0,0 +1,138 @@ +{ + "id": "description-card-source-codes", + "description": "A card drawn from a dataset version that lists the publisher codes of the series it draws, each code its drawn value's alone and so exact there, and both together, neither exact, under `about`, is accepted.", + "schema": "description", + "input": { + "describes": { + "kind": "card_revision", + "component_id": "cmp_Ab3dE5gH7k", + "revision": 2 + }, + "title": "Disbursements to Kenya's central bank, 2010 to 2024", + "about": { + "drawn": { + "dimensions": [ + { + "property": { + "ref": "idsCreditorType", + "label": "creditor type" + }, + "count": 2, + "of": 6, + "values": [ + { + "ref": "IDSCreditorTypeBLAT", + "label": "bilateral", + "definition": "A single government lending to another.", + "source_codes": [ + { + "provenance": "provenance/WorldBankInternationalDebtStatistics", + "code": "DT.DIS.BLAT.CB.CD", + "exact": true + } + ] + }, + { + "ref": "IDSCreditorTypeMLAT", + "label": "multilateral", + "source_codes": [ + { + "provenance": "provenance/WorldBankInternationalDebtStatistics", + "code": "DT.DIS.MLAT.CB.CD", + "exact": true + } + ] + } + ] + } + ], + "series": 2, + "period": { + "start": "2010", + "end": "2024", + "grain": "year" + } + }, + "unit": { + "ref": "CurrencyUSD", + "label": "US dollars", + "prices": "current" + }, + "source_codes": [ + { + "provenance": "provenance/WorldBankInternationalDebtStatistics", + "code": "DT.DIS.BLAT.CB.CD" + }, + { + "provenance": "provenance/WorldBankInternationalDebtStatistics", + "code": "DT.DIS.MLAT.CB.CD" + } + ] + }, + "sources": [ + { + "ref": "provenance/WorldBankInternationalDebtStatistics", + "label": "International Debt Statistics", + "publisher": "World Bank", + "licence": "CC BY 4.0", + "series": 2, + "documentation": "/api/graph/documentation/provenance/WorldBankInternationalDebtStatistics" + } + ], + "freshness": { + "latest_period": "2024", + "series_reaching_it": 2, + "estimated_values": null, + "provisional_values": null + }, + "steps": [ + { + "id": "s1", + "op": "select", + "input": { + "id": "spec_91d7345eec81a029629938b3d0ffd775", + "content_version": "cv_b785befb9f87471de96eb31cd7c56f5c", + "label": "Disbursements to Kenya's central bank", + "at_mint": true + }, + "policy": "card_selection", + "params": { + "coordinates": { + "variable": [ + "Amount_FinancialFlow_Disbursement_IDSCreditorTypeBLAT_IDSDebtorTypeCB_Nominal", + "Amount_FinancialFlow_Disbursement_IDSCreditorTypeMLAT_IDSDebtorTypeCB_Nominal" + ] + }, + "window": { + "start": "2010", + "end": "2024" + } + }, + "kept": { + "series": 2 + }, + "text": "Drew 2 series of disbursements to Kenya's central bank, 2010 to 2024." + } + ], + "caveats": [], + "cite": { + "title": "Disbursements to Kenya's central bank", + "text": "International Debt Statistics (2026), via the Development Finance Observatory", + "component_id": "cmp_Ab3dE5gH7k", + "revision": 2, + "id": "spec_91d7345eec81a029629938b3d0ffd775", + "content_version": "cv_b785befb9f87471de96eb31cd7c56f5c", + "retrieved": "2026-09-28", + "data": "/api/datasets/spec_91d7345eec81a029629938b3d0ffd775/cv_b785befb9f87471de96eb31cd7c56f5c" + }, + "gaps": [ + "freshness.estimated_values", + "freshness.provisional_values" + ], + "built_from": { + "facts_schema": 1, + "vocabulary": "voc_5e1d9a3c7b2f8e64" + } + }, + "expect": "accept" +} diff --git a/python/descriptions/src/dfo_descriptions/vectors/description-source-codes-empty-rejected.json b/python/descriptions/src/dfo_descriptions/vectors/description-source-codes-empty-rejected.json new file mode 100644 index 0000000..c9c13aa --- /dev/null +++ b/python/descriptions/src/dfo_descriptions/vectors/description-source-codes-empty-rejected.json @@ -0,0 +1,131 @@ +{ + "id": "description-source-codes-empty-rejected", + "description": "A card whose `about.source_codes` is an empty list is rejected by the schema, since a description without codes leaves the member out.", + "schema": "description", + "input": { + "describes": { + "kind": "card_revision", + "component_id": "cmp_Ab3dE5gH7k", + "revision": 2 + }, + "title": "Disbursements to Kenya's central bank, 2010 to 2024", + "about": { + "drawn": { + "dimensions": [ + { + "property": { + "ref": "idsCreditorType", + "label": "creditor type" + }, + "count": 2, + "of": 6, + "values": [ + { + "ref": "IDSCreditorTypeBLAT", + "label": "bilateral", + "definition": "A single government lending to another.", + "source_codes": [ + { + "provenance": "provenance/WorldBankInternationalDebtStatistics", + "code": "DT.DIS.BLAT.CB.CD", + "exact": true + } + ] + }, + { + "ref": "IDSCreditorTypeMLAT", + "label": "multilateral", + "source_codes": [ + { + "provenance": "provenance/WorldBankInternationalDebtStatistics", + "code": "DT.DIS.MLAT.CB.CD", + "exact": true + } + ] + } + ] + } + ], + "series": 2, + "period": { + "start": "2010", + "end": "2024", + "grain": "year" + } + }, + "unit": { + "ref": "CurrencyUSD", + "label": "US dollars", + "prices": "current" + }, + "source_codes": [] + }, + "sources": [ + { + "ref": "provenance/WorldBankInternationalDebtStatistics", + "label": "International Debt Statistics", + "publisher": "World Bank", + "licence": "CC BY 4.0", + "series": 2, + "documentation": "/api/graph/documentation/provenance/WorldBankInternationalDebtStatistics" + } + ], + "freshness": { + "latest_period": "2024", + "series_reaching_it": 2, + "estimated_values": null, + "provisional_values": null + }, + "steps": [ + { + "id": "s1", + "op": "select", + "input": { + "id": "spec_91d7345eec81a029629938b3d0ffd775", + "content_version": "cv_b785befb9f87471de96eb31cd7c56f5c", + "label": "Disbursements to Kenya's central bank", + "at_mint": true + }, + "policy": "card_selection", + "params": { + "coordinates": { + "variable": [ + "Amount_FinancialFlow_Disbursement_IDSCreditorTypeBLAT_IDSDebtorTypeCB_Nominal", + "Amount_FinancialFlow_Disbursement_IDSCreditorTypeMLAT_IDSDebtorTypeCB_Nominal" + ] + }, + "window": { + "start": "2010", + "end": "2024" + } + }, + "kept": { + "series": 2 + }, + "text": "Drew 2 series of disbursements to Kenya's central bank, 2010 to 2024." + } + ], + "caveats": [], + "cite": { + "title": "Disbursements to Kenya's central bank", + "text": "International Debt Statistics (2026), via the Development Finance Observatory", + "component_id": "cmp_Ab3dE5gH7k", + "revision": 2, + "id": "spec_91d7345eec81a029629938b3d0ffd775", + "content_version": "cv_b785befb9f87471de96eb31cd7c56f5c", + "retrieved": "2026-09-28", + "data": "/api/datasets/spec_91d7345eec81a029629938b3d0ffd775/cv_b785befb9f87471de96eb31cd7c56f5c" + }, + "gaps": [ + "freshness.estimated_values", + "freshness.provisional_values" + ], + "built_from": { + "facts_schema": 1, + "vocabulary": "voc_5e1d9a3c7b2f8e64" + } + }, + "error": { + "by": "schema" + } +} diff --git a/python/dfo-spec-kit-components/src/dfo_spec_kit_components/catalogue/manifest.json b/python/dfo-spec-kit-components/src/dfo_spec_kit_components/catalogue/manifest.json index 53516b9..c92a77e 100644 --- a/python/dfo-spec-kit-components/src/dfo_spec_kit_components/catalogue/manifest.json +++ b/python/dfo-spec-kit-components/src/dfo_spec_kit_components/catalogue/manifest.json @@ -73,7 +73,7 @@ "significant_digits": 3 }, "referenced_schemas": { - "https://developmentfinance.org/descriptions/schema/v1/description.schema.json": "sha256:ce1d5fbadcd804616d07f9b452624a52795acd27859213c9d9b93198828e8470", + "https://developmentfinance.org/descriptions/schema/v1/description.schema.json": "sha256:da94ffcd0db2fc2b7077a0e3e3bc0cbd9f53358cbb5ec54dce87c72226ba867f", "https://developmentfinance.org/descriptions/schema/v1/steps.schema.json": "sha256:910e3a5cdca9ffd42b94aac96ce52e2bfe9ccf57183110e1a7a96bbc71a371d1" }, "retry_modes": { From 6b0ce47fa348f3ed5bfd770e940e0947537004d0 Mon Sep 17 00:00:00 2001 From: Jorge Rivera Date: Sun, 1 Nov 2026 09:34:03 +0100 Subject: [PATCH 2/2] fix(descriptions): hold source code lists to one ascending order uniqueItems compares whole entries, so a source_codes list could name the same (provenance, code) twice when only exact differed, and could list codes in any order. check_description now refuses a list under about or on a drawn value that is not strictly ascending by (provenance, code), with the reason source_codes_not_ascending, which covers both. Two vectors pin a duplicate on a drawn value and a reversed about list. --- python/descriptions/CHANGELOG.md | 7 +- python/descriptions/README.md | 5 +- .../src/dfo_descriptions/structure.py | 31 +++- ...value-source-codes-duplicate-rejected.json | 146 ++++++++++++++++++ ...iption-source-codes-reversed-rejected.json | 142 +++++++++++++++++ .../vectors/descriptions-vector.schema.json | 3 +- 6 files changed, 329 insertions(+), 5 deletions(-) create mode 100644 python/descriptions/src/dfo_descriptions/vectors/description-drawn-value-source-codes-duplicate-rejected.json create mode 100644 python/descriptions/src/dfo_descriptions/vectors/description-source-codes-reversed-rejected.json diff --git a/python/descriptions/CHANGELOG.md b/python/descriptions/CHANGELOG.md index 3fb4ada..5ad82d5 100644 --- a/python/descriptions/CHANGELOG.md +++ b/python/descriptions/CHANGELOG.md @@ -155,6 +155,11 @@ and this package adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `stat_vars.source_codes`, 1 to 50 of them, absent when there are none or more. `exact: true` marks a code that names the whole of what lists it. A computed version and a source card refuse `about.source_codes`, and a card - drawn from a computed version refuses `exact`. + drawn from a computed version refuses `exact`. `check_description` raises + `source_codes_not_ascending` when a list, under `about` or on a drawn value, + is not strictly ascending by provenance and then code, which catches a + list out of order and a pair listed twice with and without `exact`. +- Vectors rejecting a drawn value's `source_codes` that list one pair twice, + once exact, and an `about.source_codes` list in descending order. [unreleased]: https://github.com/DevelopmentFinanceObservatory/spec-kit/commits/main/python diff --git a/python/descriptions/README.md b/python/descriptions/README.md index 5bb1845..6700b2c 100644 --- a/python/descriptions/README.md +++ b/python/descriptions/README.md @@ -95,7 +95,7 @@ A `pooled_entities` count names its graph role in `role`, such as `financingProv ### Structure rules -`check_steps` checks what the schema cannot, on a list that already validates against `steps.schema.json`, and raises on the first rule broken. `check_description` runs the count bound, the I-JSON check and the id, input and `sides` rules on a description's `steps` when they are a list, then checks the description's caveats and gaps and its drawn dimensions' `values`. +`check_steps` checks what the schema cannot, on a list that already validates against `steps.schema.json`, and raises on the first rule broken. `check_description` runs the count bound, the I-JSON check and the id, input and `sides` rules on a description's `steps` when they are a list, then checks the description's caveats and gaps, its drawn dimensions' `values` and its `source_codes` lists. | Reason | Meaning | | ----------------------------- | ------------------------------------------------------------------------------------------------------------------ | @@ -111,6 +111,7 @@ A `pooled_entities` count names its graph role in `role`, such as `financingProv | `unknown_gap_step` | A description's gap names a step its `steps` do not hold | | `drawn_values_count_mismatch` | A drawn dimension lists `values` whose number differs from its `count` | | `duplicate_drawn_value` | A drawn dimension's `values` repeat a `ref` | +| `source_codes_not_ascending` | A `source_codes` list is out of order by provenance and then code, or lists a pair twice | The size bound is on the stored list and `check_steps` checks it where the list is written, counting the list serialised by JCS (RFC 8785). `check_description` leaves it out, since a served step's labels come from the vocabulary and its `text` is added, so the served bytes differ from the stored list's. JCS spells numbers as ECMAScript does, so a TypeScript consumer counts the same bytes with `JSON.stringify` on the same members, once it refuses what I-JSON cannot carry (`String.prototype.isWellFormed`, `Number.isSafeInteger`, `Number.isFinite`). The bounds are checked first, on the whole list. Their errors, and those for a caveat or a gap, have no `step`. An input may name only an earlier step, so a list that passes cannot hold a cycle. Reasons are permanent once published. A changed meaning gets a new reason. @@ -151,7 +152,7 @@ A description carries `describes`, `title`, the five blocks `about`, `sources`, - An open dimension with a known `count` carries `top`, up to two values by variable count, `more`, the count of the rest, and `values`, the route that pages them, provider included. A dimension whose values are unknown has `count: null` with a gap and none of the three. - A source that is not an input carries `publisher`, `licence` and `series`, each `null` with a gap when unknown. A dataset version has no inputs, so its sources carry no `content_version`. - `about.measure` carries a `definition`, `null` with gap `about.measure.definition` when nothing defines the measure. Each `about.fixed[].value`, on a version or a card, may carry a `definition`, the graph's description of the value, 1 to 4096 characters, absent when the graph holds none. A card's `about.drawn.dimensions[]` may list the `values` it draws on that dimension, each with its `definition`, in the order the card first draws them. The list holds 1 to 50 values, and its presence means it names every value the card draws there. A list longer or shorter than the dimension's `count`, or one that repeats a `ref`, breaks the structure rules, and a card that draws none or more than 50 leaves `values` out. -- `about.source_codes` lists the codes publishers file the covered variables under, from each variable's `sourceIndicatorCode` arc, as `{provenance, code, exact?}`, sorted by provenance and then code, each once, as Search serves `stat_vars.source_codes`. A card lists those of every series it draws, and a series of a computed version takes the codes of the variables it was computed from. A dataset version lists those of the one variable its definition resolved to. A card's drawn value on a dimension the variables declare carries the codes of the drawn variables holding it. A list holds 1 to 50 codes and is absent when there are none or more. `exact: true` marks a code that names the whole of what lists it: every series there is read from that provenance alone and is one the code names, and the provenance gives none of them another code. A code without it names variables the series are built from or hold beside others. A computed version and a source card carry no `about.source_codes`, and a card drawn from a computed version carries no exact code. +- `about.source_codes` lists the codes publishers file the covered variables under, from each variable's `sourceIndicatorCode` arc, as `{provenance, code, exact?}`, sorted by provenance and then code, each once, as Search serves `stat_vars.source_codes`. The schema cannot hold the order, or tell a repeated pair that differs only in `exact`, so `check_description` raises `source_codes_not_ascending` on a list that is not strictly ascending by `(provenance, code)`. A card lists those of every series it draws, and a series of a computed version takes the codes of the variables it was computed from. A dataset version lists those of the one variable its definition resolved to. A card's drawn value on a dimension the variables declare carries the codes of the drawn variables holding it. A list holds 1 to 50 codes and is absent when there are none or more. `exact: true` marks a code that names the whole of what lists it: every series there is read from that provenance alone and is one the code names, and the provenance gives none of them another code. A code without it names variables the series are built from or hold beside others. A computed version and a source card carry no `about.source_codes`, and a card drawn from a computed version carries no exact code. - Every reference is `{ref, label}`, and every unit, variable side and input version outside `steps` carries its `label` too. A label, source name, publisher or licence taken from the evidence stored at mint, in place of today's vocabulary, sits in an object carrying `at_mint: true`. - A computed version's inputs, in `sources`, `freshness.inputs` and `cite.inputs`, name each version it was computed from by `id` and `content_version`, with a `label`. An input that no step retrieves, as on a version published without steps, has an unknown id: it is served with `id: null`, or in `sources` with `ref: null` and no `description` link, and a gap naming that member. Its label is the one stored with it, else its content version with a gap. Only an input, which carries `content_version`, may leave its `ref` null. - `steps` is a step list whose every step adds `text`, the composer's sentence for that step, or `null` when the record holds none, with a gap. Both schemas build a step from `step_members` in `steps.schema.json`: a stored step closes it with `unevaluatedProperties: false`, so a stored list refuses `text`, and a served step requires `text` and closes it the same way. diff --git a/python/descriptions/src/dfo_descriptions/structure.py b/python/descriptions/src/dfo_descriptions/structure.py index e1ab4d7..206c92a 100644 --- a/python/descriptions/src/dfo_descriptions/structure.py +++ b/python/descriptions/src/dfo_descriptions/structure.py @@ -5,6 +5,7 @@ from collections.abc import Mapping, Sequence from enum import StrEnum +from itertools import pairwise from typing import Any import rfc8785 @@ -73,6 +74,11 @@ class StepsErrorReason(StrEnum): DUPLICATE_DRAWN_VALUE = "duplicate_drawn_value" """A drawn dimension's `values` repeat a `ref`.""" + SOURCE_CODES_NOT_ASCENDING = "source_codes_not_ascending" + """A `source_codes` list, under `about` or on a drawn value, is not + strictly ascending by provenance and then code: it is out of order or + lists a `(provenance, code)` pair twice, with or without `exact`.""" + class StepsError(ValueError): """A step list or description breaks the structure rules. @@ -156,7 +162,9 @@ def check_description(description: Mapping[str, Any]) -> None: step in `steps`. A gap names a step when its path has at least three segments and starts `steps.`, as in `steps.s3.op`. A drawn dimension that lists `values` must list `count` of them, each - `ref` once. + `ref` once. Every `source_codes` list, under `about` and on each drawn + value, must be strictly ascending by provenance and then code, which + sorts it and holds each pair once. Args: description: A description that already validates against @@ -188,6 +196,7 @@ def check_description(description: Mapping[str, Any]) -> None: detail=f"gap {gap!r} names step {segments[1]!r}", ) _check_drawn_values(description) + _check_source_codes(description) def _check_count(steps: Sequence[Mapping[str, Any]]) -> None: @@ -276,3 +285,23 @@ def _check_drawn_values(description: Mapping[str, Any]) -> None: step=None, detail=f"drawn dimension {position} lists a ref twice", ) + + +def _check_source_codes(description: Mapping[str, Any]) -> None: + """Each `source_codes` list is strictly ascending by `(provenance, code)`.""" + about = description["about"] + lists = [("about.source_codes", about.get("source_codes"))] + drawn = about.get("drawn") or {} + for position, dimension in enumerate(drawn.get("dimensions", [])): + for index, value in enumerate(dimension.get("values") or []): + where = f"drawn dimension {position} value {index}" + lists.append((where, value.get("source_codes"))) + for where, codes in lists: + keys = [(code["provenance"], code["code"]) for code in codes or []] + for earlier, later in pairwise(keys): + if earlier >= later: + raise StepsError( + reason=StepsErrorReason.SOURCE_CODES_NOT_ASCENDING, + step=None, + detail=f"{where} lists {later} after {earlier}", + ) diff --git a/python/descriptions/src/dfo_descriptions/vectors/description-drawn-value-source-codes-duplicate-rejected.json b/python/descriptions/src/dfo_descriptions/vectors/description-drawn-value-source-codes-duplicate-rejected.json new file mode 100644 index 0000000..e298f9d --- /dev/null +++ b/python/descriptions/src/dfo_descriptions/vectors/description-drawn-value-source-codes-duplicate-rejected.json @@ -0,0 +1,146 @@ +{ + "id": "description-drawn-value-source-codes-duplicate-rejected", + "description": "A drawn value whose `source_codes` list the same provenance and code twice, once with `exact` and once without, breaks the structure rules.", + "schema": "description", + "input": { + "describes": { + "kind": "card_revision", + "component_id": "cmp_Ab3dE5gH7k", + "revision": 2 + }, + "title": "Disbursements to Kenya's central bank, 2010 to 2024", + "about": { + "drawn": { + "dimensions": [ + { + "property": { + "ref": "idsCreditorType", + "label": "creditor type" + }, + "count": 2, + "of": 6, + "values": [ + { + "ref": "IDSCreditorTypeBLAT", + "label": "bilateral", + "definition": "A single government lending to another.", + "source_codes": [ + { + "provenance": "provenance/WorldBankInternationalDebtStatistics", + "code": "DT.DIS.BLAT.CB.CD", + "exact": true + }, + { + "provenance": "provenance/WorldBankInternationalDebtStatistics", + "code": "DT.DIS.BLAT.CB.CD" + } + ] + }, + { + "ref": "IDSCreditorTypeMLAT", + "label": "multilateral", + "source_codes": [ + { + "provenance": "provenance/WorldBankInternationalDebtStatistics", + "code": "DT.DIS.MLAT.CB.CD", + "exact": true + } + ] + } + ] + } + ], + "series": 2, + "period": { + "start": "2010", + "end": "2024", + "grain": "year" + } + }, + "unit": { + "ref": "CurrencyUSD", + "label": "US dollars", + "prices": "current" + }, + "source_codes": [ + { + "provenance": "provenance/WorldBankInternationalDebtStatistics", + "code": "DT.DIS.BLAT.CB.CD" + }, + { + "provenance": "provenance/WorldBankInternationalDebtStatistics", + "code": "DT.DIS.MLAT.CB.CD" + } + ] + }, + "sources": [ + { + "ref": "provenance/WorldBankInternationalDebtStatistics", + "label": "International Debt Statistics", + "publisher": "World Bank", + "licence": "CC BY 4.0", + "series": 2, + "documentation": "/api/graph/documentation/provenance/WorldBankInternationalDebtStatistics" + } + ], + "freshness": { + "latest_period": "2024", + "series_reaching_it": 2, + "estimated_values": null, + "provisional_values": null + }, + "steps": [ + { + "id": "s1", + "op": "select", + "input": { + "id": "spec_91d7345eec81a029629938b3d0ffd775", + "content_version": "cv_b785befb9f87471de96eb31cd7c56f5c", + "label": "Disbursements to Kenya's central bank", + "at_mint": true + }, + "policy": "card_selection", + "params": { + "coordinates": { + "variable": [ + "Amount_FinancialFlow_Disbursement_IDSCreditorTypeBLAT_IDSDebtorTypeCB_Nominal", + "Amount_FinancialFlow_Disbursement_IDSCreditorTypeMLAT_IDSDebtorTypeCB_Nominal" + ] + }, + "window": { + "start": "2010", + "end": "2024" + } + }, + "kept": { + "series": 2 + }, + "text": "Drew 2 series of disbursements to Kenya's central bank, 2010 to 2024." + } + ], + "caveats": [], + "cite": { + "title": "Disbursements to Kenya's central bank", + "text": "International Debt Statistics (2026), via the Development Finance Observatory", + "component_id": "cmp_Ab3dE5gH7k", + "revision": 2, + "id": "spec_91d7345eec81a029629938b3d0ffd775", + "content_version": "cv_b785befb9f87471de96eb31cd7c56f5c", + "retrieved": "2026-09-28", + "data": "/api/datasets/spec_91d7345eec81a029629938b3d0ffd775/cv_b785befb9f87471de96eb31cd7c56f5c" + }, + "gaps": [ + "freshness.estimated_values", + "freshness.provisional_values" + ], + "built_from": { + "facts_schema": 1, + "vocabulary": "voc_5e1d9a3c7b2f8e64" + } + }, + "error": { + "by": "structure", + "reason": "source_codes_not_ascending", + "step": null + } +} diff --git a/python/descriptions/src/dfo_descriptions/vectors/description-source-codes-reversed-rejected.json b/python/descriptions/src/dfo_descriptions/vectors/description-source-codes-reversed-rejected.json new file mode 100644 index 0000000..759f7e8 --- /dev/null +++ b/python/descriptions/src/dfo_descriptions/vectors/description-source-codes-reversed-rejected.json @@ -0,0 +1,142 @@ +{ + "id": "description-source-codes-reversed-rejected", + "description": "A card whose `about.source_codes` list a provenance's codes in descending order breaks the structure rules.", + "schema": "description", + "input": { + "describes": { + "kind": "card_revision", + "component_id": "cmp_Ab3dE5gH7k", + "revision": 2 + }, + "title": "Disbursements to Kenya's central bank, 2010 to 2024", + "about": { + "drawn": { + "dimensions": [ + { + "property": { + "ref": "idsCreditorType", + "label": "creditor type" + }, + "count": 2, + "of": 6, + "values": [ + { + "ref": "IDSCreditorTypeBLAT", + "label": "bilateral", + "definition": "A single government lending to another.", + "source_codes": [ + { + "provenance": "provenance/WorldBankInternationalDebtStatistics", + "code": "DT.DIS.BLAT.CB.CD", + "exact": true + } + ] + }, + { + "ref": "IDSCreditorTypeMLAT", + "label": "multilateral", + "source_codes": [ + { + "provenance": "provenance/WorldBankInternationalDebtStatistics", + "code": "DT.DIS.MLAT.CB.CD", + "exact": true + } + ] + } + ] + } + ], + "series": 2, + "period": { + "start": "2010", + "end": "2024", + "grain": "year" + } + }, + "unit": { + "ref": "CurrencyUSD", + "label": "US dollars", + "prices": "current" + }, + "source_codes": [ + { + "provenance": "provenance/WorldBankInternationalDebtStatistics", + "code": "DT.DIS.MLAT.CB.CD" + }, + { + "provenance": "provenance/WorldBankInternationalDebtStatistics", + "code": "DT.DIS.BLAT.CB.CD" + } + ] + }, + "sources": [ + { + "ref": "provenance/WorldBankInternationalDebtStatistics", + "label": "International Debt Statistics", + "publisher": "World Bank", + "licence": "CC BY 4.0", + "series": 2, + "documentation": "/api/graph/documentation/provenance/WorldBankInternationalDebtStatistics" + } + ], + "freshness": { + "latest_period": "2024", + "series_reaching_it": 2, + "estimated_values": null, + "provisional_values": null + }, + "steps": [ + { + "id": "s1", + "op": "select", + "input": { + "id": "spec_91d7345eec81a029629938b3d0ffd775", + "content_version": "cv_b785befb9f87471de96eb31cd7c56f5c", + "label": "Disbursements to Kenya's central bank", + "at_mint": true + }, + "policy": "card_selection", + "params": { + "coordinates": { + "variable": [ + "Amount_FinancialFlow_Disbursement_IDSCreditorTypeBLAT_IDSDebtorTypeCB_Nominal", + "Amount_FinancialFlow_Disbursement_IDSCreditorTypeMLAT_IDSDebtorTypeCB_Nominal" + ] + }, + "window": { + "start": "2010", + "end": "2024" + } + }, + "kept": { + "series": 2 + }, + "text": "Drew 2 series of disbursements to Kenya's central bank, 2010 to 2024." + } + ], + "caveats": [], + "cite": { + "title": "Disbursements to Kenya's central bank", + "text": "International Debt Statistics (2026), via the Development Finance Observatory", + "component_id": "cmp_Ab3dE5gH7k", + "revision": 2, + "id": "spec_91d7345eec81a029629938b3d0ffd775", + "content_version": "cv_b785befb9f87471de96eb31cd7c56f5c", + "retrieved": "2026-09-28", + "data": "/api/datasets/spec_91d7345eec81a029629938b3d0ffd775/cv_b785befb9f87471de96eb31cd7c56f5c" + }, + "gaps": [ + "freshness.estimated_values", + "freshness.provisional_values" + ], + "built_from": { + "facts_schema": 1, + "vocabulary": "voc_5e1d9a3c7b2f8e64" + } + }, + "error": { + "by": "structure", + "reason": "source_codes_not_ascending", + "step": null + } +} diff --git a/python/descriptions/src/dfo_descriptions/vectors/descriptions-vector.schema.json b/python/descriptions/src/dfo_descriptions/vectors/descriptions-vector.schema.json index d653193..797929a 100644 --- a/python/descriptions/src/dfo_descriptions/vectors/descriptions-vector.schema.json +++ b/python/descriptions/src/dfo_descriptions/vectors/descriptions-vector.schema.json @@ -77,7 +77,8 @@ "unknown_caveat_step", "unknown_gap_step", "drawn_values_count_mismatch", - "duplicate_drawn_value" + "duplicate_drawn_value", + "source_codes_not_ascending" ] }, "step": {