Skip to content

Commit 4437e9c

Browse files
brownjuly2003-codeJuliaEdomclaude
authored
feat(dv2): generator + seeds rebuild for the kitchen-appliance legend (B1) (#138)
* feat(dv2): generator + seeds rebuild for the kitchen-appliance legend (B1) Replaces the grocery/X5 reference generator and DV2 demo seeds with the own-brand kitchen-appliance importer legend pinned in docs/domain.md and docs/generator-spec.md: 30 suppliers (22 CN/5 RU/2 AE/1 KZ), 160 SKUs across 10 categories with a FOB<landed<wholesale<mp-net<RRC pricing ladder, and DV2 seeds recalibrated to 2,500 customers (2,000 retail + 500 dealers), 10,000 orders split by channel/branch, and dealer-only retro-bonus loyalty in msk/spb/ekb. New reference/legend.py is the single numeric source of truth; tests/unit/test_generator_spec_invariants.py encodes generator-spec.md §12's 12 consistency invariants as machine-checkable tests. Live ClickHouse/Postgres re-verify of the rewritten seed SQL is B4's job (no Docker available in this environment); this pass is scoped to the generator, seeds, and everything testable without a live warehouse. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> * fix(test): use tolerance for seasonal-curve average assertion CI's float summation of SEASONAL_B2B lands at 12.000000000000002 instead of 12.0 (non-associative float addition, platform-dependent rounding) — exact equality on a float sum was never safe to begin with. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> --------- Co-authored-by: JuliaEdom <uedomskikh@gmail.com> Co-authored-by: Claude Sonnet 5 <noreply@anthropic.com>
1 parent 6a7edba commit 4437e9c

13 files changed

Lines changed: 1222 additions & 418 deletions

‎tests/unit/test_dv2_supplier_reference.py‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,10 +76,14 @@ def test_ru_inn10_check_digit_known_vector():
7676

7777

7878
def test_tnved_headings_are_real_format():
79-
assert len(TNVED_HEADINGS) >= 30
79+
# 10 catalog categories (generator-spec.md §3), one split across two
80+
# headings (vacuum sealers vs dryers) -> 11 heading entries.
81+
assert len(TNVED_HEADINGS) == 11
82+
allowed_headings = {"8516", "8509", "8423", "8422"}
8083
for h in TNVED_HEADINGS:
8184
assert len(h.heading) == 4
8285
assert h.heading.isdigit()
86+
assert h.heading in allowed_headings
8387
assert h.code10 == f"{h.heading}000000"
8488
assert len(h.code10) == 10
8589
assert h.description.strip()
Lines changed: 226 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,226 @@
1+
"""Machine-checkable consistency invariants from ``docs/generator-spec.md``
2+
§12 — the definition of "цифры взаимно согласованы" for the B1 data rebuild.
3+
4+
Invariants #1-#10 are pure arithmetic over :mod:`legend`'s constants (no
5+
warehouse needed). #5/#7/#8 are also checked against the actual
6+
:func:`build_reference` output. #11/#12 (faux-PII locale rules, loyalty
7+
gating) are structural regression guards over the seed SQL text — a live
8+
ClickHouse/Postgres re-verify is B4's job, not this file's.
9+
"""
10+
11+
from __future__ import annotations
12+
13+
from pathlib import Path
14+
15+
from warehouse.agentflow.dv2.reference import legend
16+
from warehouse.agentflow.dv2.reference.generator import build_reference
17+
from warehouse.agentflow.dv2.reference.gs1 import is_valid_gtin13
18+
from warehouse.agentflow.dv2.reference.tnved import TNVED_HEADINGS
19+
20+
DV2_ROOT = Path(__file__).resolve().parents[2] / "warehouse" / "agentflow" / "dv2"
21+
22+
_MARKETPLACE_CHANNELS = {"marketplace_fbs"}
23+
_D2C_CHANNELS = {"d2c_site"}
24+
_B2B_CHANNELS = {"b2b_wholesale", "b2b_re_export", "b2b_eaeu"}
25+
26+
27+
def _annual_revenue_rub() -> float:
28+
daily = sum(orders * check for _, _, orders, check in legend.MASTER_MATRIX)
29+
return daily * 365
30+
31+
32+
def _orders_by_group() -> dict[str, int]:
33+
totals = {"marketplace": 0, "d2c": 0, "b2b": 0}
34+
for channel, _, orders, _ in legend.MASTER_MATRIX:
35+
if channel in _MARKETPLACE_CHANNELS:
36+
totals["marketplace"] += orders
37+
elif channel in _D2C_CHANNELS:
38+
totals["d2c"] += orders
39+
else:
40+
totals["b2b"] += orders
41+
return totals
42+
43+
44+
def _revenue_by_group() -> dict[str, float]:
45+
totals = {"marketplace": 0.0, "d2c": 0.0, "b2b": 0.0}
46+
for channel, _, orders, check in legend.MASTER_MATRIX:
47+
revenue = orders * check
48+
if channel in _MARKETPLACE_CHANNELS:
49+
totals["marketplace"] += revenue
50+
elif channel in _D2C_CHANNELS:
51+
totals["d2c"] += revenue
52+
else:
53+
totals["b2b"] += revenue
54+
return totals
55+
56+
57+
# --- #1 annual revenue -------------------------------------------------------
58+
59+
60+
def test_invariant_1_annual_revenue_in_corridor():
61+
annual_b_rub = _annual_revenue_rub() / 1_000_000_000
62+
assert 3.5 <= annual_b_rub <= 5.0
63+
64+
65+
# --- #2 order-count mix ------------------------------------------------------
66+
67+
68+
def test_invariant_2_order_count_mix():
69+
totals = _orders_by_group()
70+
grand_total = sum(totals.values())
71+
marketplace_pct = 100 * totals["marketplace"] / grand_total
72+
b2b_pct = 100 * totals["b2b"] / grand_total
73+
d2c_pct = 100 * totals["d2c"] / grand_total
74+
assert 88 <= marketplace_pct <= 90
75+
assert 7 <= b2b_pct <= 9
76+
assert 2 <= d2c_pct <= 4
77+
78+
79+
# --- #3 revenue mix -----------------------------------------------------------
80+
81+
82+
def test_invariant_3_revenue_mix():
83+
totals = _revenue_by_group()
84+
grand_total = sum(totals.values())
85+
b2b_pct = 100 * totals["b2b"] / grand_total
86+
marketplace_pct = 100 * totals["marketplace"] / grand_total
87+
assert 65 <= b2b_pct <= 72
88+
assert 27 <= marketplace_pct <= 33
89+
90+
91+
# --- #4 bimodal AOV -----------------------------------------------------------
92+
93+
94+
def test_invariant_4_bimodal_avg_checks_no_mass_in_gap():
95+
# §1's master matrix pins dxb (re-export, "export pallets", thinner
96+
# margin per §5) at a 90k avg check — outside the general [30k, 80k] B2B
97+
# band the same table implies for the domestic + EAEU wholesale channels.
98+
# Read narrowly: the [30k, 80k] band covers RU + ala wholesale; dxb is a
99+
# documented, table-explicit outlier, not a spec violation.
100+
domestic_b2b_checks = [
101+
check
102+
for channel, branch, _, check in legend.MASTER_MATRIX
103+
if channel in _B2B_CHANNELS and branch != "dxb"
104+
]
105+
marketplace_checks = [
106+
check for channel, _, _, check in legend.MASTER_MATRIX if channel in _MARKETPLACE_CHANNELS
107+
]
108+
assert all(30_000 <= c <= 80_000 for c in domestic_b2b_checks)
109+
assert all(1_500 <= c <= 3_000 for c in marketplace_checks)
110+
# no channel's avg check falls in the 10k-25k dead zone (holds for all
111+
# channels, including dxb)
112+
assert all(not (10_000 < check < 25_000) for _, _, _, check in legend.MASTER_MATRIX)
113+
114+
115+
# --- #5 pricing ladder ---------------------------------------------------------
116+
117+
118+
def test_invariant_5_pricing_ladder_bands_are_disjoint_and_ordered():
119+
assert legend.FOB_PCT_RANGE[1] < legend.LANDED_PCT_RANGE[0]
120+
assert legend.LANDED_PCT_RANGE[1] < legend.WHOLESALE_PCT_RANGE[0]
121+
assert legend.WHOLESALE_PCT_RANGE[1] < legend.MARKETPLACE_NET_PCT
122+
assert legend.MARKETPLACE_NET_PCT < legend.RRC_PCT
123+
124+
125+
def test_invariant_5_pricing_ladder_holds_per_sku():
126+
tables = build_reference()
127+
rrc_by_sku = {p.product_bk: p.rrc_price for p in tables.products}
128+
for sourcing in tables.sourcing:
129+
rrc = rrc_by_sku[sourcing.product_bk]
130+
fob_pct = sourcing.purchase_price / rrc
131+
assert legend.FOB_PCT_RANGE[0] <= fob_pct <= legend.FOB_PCT_RANGE[1]
132+
133+
134+
# --- #6 seasonal curves average to 1.0 -----------------------------------------
135+
136+
137+
def test_invariant_6_seasonal_curves_average_to_one():
138+
assert len(legend.SEASONAL_RETAIL) == 12
139+
assert len(legend.SEASONAL_B2B) == 12
140+
assert abs(sum(legend.SEASONAL_RETAIL) / 12 - 1.0) < 1e-9
141+
assert abs(sum(legend.SEASONAL_B2B) / 12 - 1.0) < 1e-9
142+
143+
144+
# --- #7 GTIN validity -----------------------------------------------------------
145+
146+
147+
def test_invariant_7_every_gtin_valid_and_in_eaeu_range():
148+
tables = build_reference()
149+
for product in tables.products:
150+
assert is_valid_gtin13(product.gtin)
151+
assert 460 <= int(product.gtin[:3]) <= 469
152+
153+
154+
# --- #8 tnved headings ----------------------------------------------------------
155+
156+
157+
def test_invariant_8_every_tnved_code_matches_a_spec_heading():
158+
allowed_headings = {"8516", "8509", "8423", "8422"}
159+
assert {h.heading for h in TNVED_HEADINGS} <= allowed_headings
160+
tables = build_reference()
161+
for product in tables.products:
162+
assert product.tnved_code.endswith("000000")
163+
assert len(product.tnved_code) == 10
164+
assert product.tnved_code[:4] in allowed_headings
165+
166+
167+
# --- #9 dealer ordering frequency -> B2B orders/day -----------------------------
168+
169+
170+
def test_invariant_9_dealer_frequency_yields_150_to_200_b2b_orders_per_day():
171+
assert sum(count for _, count, _ in legend.DEALER_FREQUENCY_TIERS) == legend.DEALER_CUSTOMERS
172+
weekly_orders = sum(count * freq for _, count, freq in legend.DEALER_FREQUENCY_TIERS)
173+
daily_b2b_orders = weekly_orders / 7
174+
assert 150 <= daily_b2b_orders <= 200
175+
176+
177+
# --- #10 branch revenue shares ---------------------------------------------------
178+
179+
180+
def test_invariant_10_branch_revenue_shares():
181+
by_branch: dict[str, float] = {}
182+
for _, branch, orders, check in legend.MASTER_MATRIX:
183+
by_branch[branch] = by_branch.get(branch, 0.0) + orders * check
184+
grand_total = sum(by_branch.values())
185+
shares = {branch: 100 * revenue / grand_total for branch, revenue in by_branch.items()}
186+
assert abs(sum(shares.values()) - 100) < 1e-6
187+
assert 55 <= shares["msk"] <= 65
188+
189+
190+
# --- #11 faux-PII locale rules (structural, over seed SQL text) -----------------
191+
192+
193+
def _read_seed_sql(*names: str) -> str:
194+
return "\n".join((DV2_ROOT / name).read_text(encoding="utf-8") for name in names)
195+
196+
197+
def test_invariant_11_faux_pii_locale_rules_present_in_seed_sql():
198+
seed_sql = _read_seed_sql("satellite_seed.sql", "satellite_seed_all_branches.sql")
199+
# RU jurisdictions (§8): city phone prefixes + @example.test emails.
200+
assert "+7495" in seed_sql # msk
201+
assert "+7812" in seed_sql # spb
202+
assert "+7343" in seed_sql # ekb
203+
assert "@example.test" in seed_sql
204+
# AE jurisdiction (dxb): +971 phones, latin transliteration names, .ae emails.
205+
assert "+971" in seed_sql
206+
assert "@example.ae" in seed_sql
207+
# KZ jurisdiction (ala): +7727 phones, .kz emails.
208+
assert "+7727" in seed_sql
209+
assert "@example.kz" in seed_sql
210+
211+
212+
# --- #12 loyalty gating -----------------------------------------------------------
213+
214+
215+
def test_invariant_12_loyalty_points_cap_is_within_three_percent_of_min_quarterly_spend():
216+
min_quarterly_spend = legend._TAIL_ORDERS_PER_WEEK * legend._RU_B2B_AVG_CHECK_RUB * 13
217+
assert legend.LOYALTY_POINTS_MAX_RUB <= legend.LOYALTY_RETRO_BONUS_PCT * min_quarterly_spend
218+
219+
220+
def test_invariant_12_loyalty_rows_only_for_eligible_branches_in_seed_sql():
221+
assert legend.LOYALTY_ELIGIBLE_BRANCHES == ("msk", "spb", "ekb")
222+
seed_sql = _read_seed_sql("satellite_seed.sql", "satellite_seed_all_branches.sql")
223+
for branch in legend.LOYALTY_ELIGIBLE_BRANCHES:
224+
assert f"sat_customer_loyalty__bitrix__{branch}" in seed_sql
225+
for branch in ("dxb", "ala"):
226+
assert f"sat_customer_loyalty__bitrix__{branch}" not in seed_sql

‎warehouse/agentflow/dv2/cold_offload_seed.sql‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,10 @@
33
-- deterministic hash of the customer number so the cold export has real
44
-- rows to ship without touching jurisdictional PII.
55
--
6-
-- One row per msk customer (~800 rows). Re-runnable: hash_diff makes
7-
-- the satellite ReplacingMergeTree-friendly even though it's MergeTree.
6+
-- One row per msk customer: retail [0,2000) + dealer msk [2000,2190) —
7+
-- 2,190 rows, matching hub_customer's msk band (synthetic_seed.sql header).
8+
-- Re-runnable: hash_diff makes the satellite ReplacingMergeTree-friendly
9+
-- even though it's MergeTree.
810

911
INSERT INTO rv.sat_customer_anon__1c__msk
1012
(customer_hk, load_ts, hash_diff, record_source,
@@ -33,5 +35,4 @@ SELECT
3335
'new'
3436
) AS customer_segment,
3537
0 AS is_deleted
36-
FROM numbers(2000)
37-
WHERE number % 100 < 40; -- msk slice = 40% of 2000 customers
38+
FROM numbers(2190); -- msk slice: retail [0,2000) + dealer msk [2000,2190)

‎warehouse/agentflow/dv2/postgres_oltp/seed.sql‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
1-
-- Hot-tier OLTP seed for the DV2.0 demo.
1+
-- Hot-tier OLTP seed for the DV2.0 demo (own-brand kitchen-appliance
2+
-- importer legend — see synthetic_seed.sql for the full customer/order
3+
-- numbering this small sample mirrors at hot-tier scale).
24
-- Lives in Postgres 17, per-branch schema layout (ops_<branch>) so the
35
-- CDC bridge described in docs/dv2-multi-branch/architecture.md can route
46
-- straight by schema name. Each schema gets its own customers + orders table.
@@ -56,7 +58,7 @@ SELECT
5658
(ARRAY['Anna','Boris','Dasha','Egor','Fedor','Galya','Ivan','Kira','Lena','Mark'])[(n % 10) + 1],
5759
(ARRAY['Ivanov','Petrov','Sidorov','Smirnov','Volkov','Orlov','Lebedev','Sokolov'])[(n % 8) + 1],
5860
'oltp' || n::text || '@example.test',
59-
'+7916' || lpad((n * 137 % 10000000)::text, 7, '0')
61+
'+7495' || lpad((n * 137 % 10000000)::text, 7, '0')
6062
FROM generate_series(1, 50) AS n
6163
ON CONFLICT (customer_id) DO NOTHING;
6264

‎warehouse/agentflow/dv2/reference/README.md‎

Lines changed: 26 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,13 @@
11
# DV2 Supplier / Product Reference
22

3-
A reproducible grocery **reference** (suppliers, products, GS1 marking codes,
4-
product→supplier sourcing) for the AgentFlow DV2 raw vault. It fills the
5-
catalog / `tnved_code` / GS1-marking slots that the transactional X5 feed
6-
leaves empty, and is the project's genuine **cloud** component: the dataset is
7-
published to a Hugging Face Dataset — real object storage, not a checkbox.
3+
A reproducible small-kitchen-appliance **reference** (suppliers, products, GS1
4+
marking codes, product→supplier sourcing) for the AgentFlow DV2 raw vault —
5+
the own-brand importer legend in [`docs/domain.md`](../../../../docs/domain.md),
6+
pinned to exact numbers in [`docs/generator-spec.md`](../../../../docs/generator-spec.md).
7+
It fills the catalog / `tnved_code` / GS1-marking slots that the transactional
8+
feeds leave empty, and is the project's genuine **cloud** component: the
9+
dataset is published to a Hugging Face Dataset — real object storage, not a
10+
checkbox.
811

912
It is a *reference* (master/dimension) feed, distinct from the `1c` / `wms` /
1013
`wb` transactional sources. Provenance is explicit: every row carries
@@ -17,21 +20,31 @@ Kept deliberately honest — the value is in real storage + real standards
1720
conformance, not in pretending the identities are real.
1821

1922
**Genuine (verifiable, pinned by tests):**
20-
- ТН ВЭД ЕАЭС headings — real 4-digit HS-aligned customs headings with
21-
descriptions close to the official Russian wording (`tnved.py`).
23+
- ТН ВЭД ЕАЭС headings — real 4-digit HS-aligned customs headings (8516/8509/
24+
8423/8422) with descriptions close to the official Russian wording
25+
(`tnved.py`).
2226
- GS1 **GTIN-13** and **GLN-13** check digits — published GS1 mod-10
2327
algorithm (`gs1.py`).
2428
- **RU INN-10** control digit — real algorithm for RU legal-entity tax ids.
25-
- EAEU GS1 prefix range **460–469**.
29+
- EAEU GS1 prefix range **460–469** — correct for this reference even though
30+
manufacturing is contracted to China: GTINs belong to the RU brand owner
31+
registered with GS1 RUS, regardless of where the goods are made.
2632
- `gross_weight_g >= net_weight_g` packaging invariant.
27-
- MD5 hash keys computed with the **same canonicalisation as the X5 loader**,
28-
so reference hubs/links join byte-for-byte with vault data already loaded
29-
from other sources (pinned in `tests/unit/test_dv2_supplier_reference.py`).
33+
- Pricing-ladder ordering per SKU: FOB < landed < wholesale < marketplace-net
34+
< RRC (generator-spec.md §5) — guaranteed by disjoint percentage bands.
35+
- MD5 hash keys computed with the **same canonicalisation as the
36+
transactional loader**, so reference hubs/links join byte-for-byte with
37+
vault data already loaded from other sources (pinned in
38+
`tests/unit/test_dv2_supplier_reference.py`).
3039

3140
**Synthetic but labelled:**
32-
- supplier legal names and brand names;
41+
- supplier legal names — **no brand token** anywhere in product data
42+
(own-brand importer decision, generator-spec.md §3);
43+
- CN USCC-18 tax ids — structurally shaped per GB 32100-2015, but the check
44+
character is a labelled placeholder, not a verified mod-31 check digit
45+
(`make_cn_uscc18`);
3346
- the specific SKU ↔ GTIN ↔ supplier assignments;
34-
- packaging dimensions and purchase prices;
47+
- packaging dimensions, RRC and FOB purchase prices;
3548
- GPC brick codes (illustrative);
3649
- ТН ВЭД sub-position digits — the genuine heading is zero-padded to the
3750
10-digit field (`<heading>000000`), i.e. heading granularity, **not** a

‎warehouse/agentflow/dv2/reference/build.py‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import pandas as pd
2424

2525
from .generator import ReferenceTables, build_reference
26+
from .legend import GENERATOR_SEED, TOTAL_PRODUCTS, TOTAL_SUPPLIERS
2627
from .vault_mapping import RECORD_SOURCE, map_reference
2728

2829

@@ -60,15 +61,16 @@ def _manifest(tables: ReferenceTables, load_ts: datetime, vault: dict[str, pd.Da
6061
"RU INN-10 control digit",
6162
"EAEU GS1 prefix range 460-469",
6263
"gross_weight_g >= net_weight_g invariant",
63-
"MD5 hash keys join-compatible with the X5 / 1C vault feeds",
64+
"pricing ladder ordering (FOB < landed < wholesale < marketplace-net < RRC)",
65+
"MD5 hash keys join-compatible with the transactional vault feeds",
6466
],
6567
"synthetic_but_labelled": [
66-
"supplier legal names",
67-
"brand names",
68+
"supplier legal names (no brand token anywhere in the data)",
6869
"SKU <-> GTIN <-> supplier assignments",
69-
"packaging dimensions and purchase prices",
70+
"packaging dimensions, RRC and FOB purchase prices",
7071
"GPC brick codes (illustrative)",
7172
"ТН ВЭД sub-position digits (zero-padded; heading granularity)",
73+
"CN USCC-18 check character (structurally shaped, not GB 32100-2015 verified)",
7274
],
7375
}
7476

@@ -77,9 +79,9 @@ def _manifest(tables: ReferenceTables, load_ts: datetime, vault: dict[str, pd.Da
7779
@click.option(
7880
"--out-dir", default="reference/build", type=click.Path(file_okay=False, path_type=Path)
7981
)
80-
@click.option("--seed", default=20260626, type=int)
81-
@click.option("--n-suppliers", default=40, type=int)
82-
@click.option("--n-products", default=300, type=int)
82+
@click.option("--seed", default=GENERATOR_SEED, type=int)
83+
@click.option("--n-suppliers", default=TOTAL_SUPPLIERS, type=int)
84+
@click.option("--n-products", default=TOTAL_PRODUCTS, type=int)
8385
@click.option("--load-ts", default=None, help="Fixed UTC load timestamp (ISO-8601), else now.")
8486
@click.option("--dry-run", is_flag=True, help="Build and summarize without writing files.")
8587
def main(

0 commit comments

Comments
 (0)