Merge pull request #16 from opensource-observer/carl/ddp-notebook-freshness-doc-refs

docs(notebooks): refresh data-source references and hide_code compliance

Author: ccerv1

notebooks/claude.md

@@ -274,17 +274,22 @@ def _(mo, pyoso_db_conn):
 
 ### Developer models
 - `oso.int_ddp__developers` — Unified developer list (ODD + GHA), keyed by `user_id`
-- `oso.stg_opendevdata__developers` — ODD developers with GraphQL IDs
+- `oso.stg_opendevdata__canonical_developers` — ODD canonical developers, keyed to GitHub GraphQL user IDs (`primary_github_user_id`)
 
 ### Commit models
 - `oso.int_ddp__commits_unified` — Combined ODD + GHA commits (not deduped)
 - `oso.int_ddp__commits_deduped` — Deduplicated unified commits
 - `oso.stg_opendevdata__commits` — Raw ODD commits with identity resolution
 
-### GitHub Archive (event models)
+### GitHub events — unified models (current canonical)
+- `oso.int_events__github_unified` — Standardized GitHub events, one row per event. Keyed by OSO artifact IDs (`from_artifact_id` / `to_artifact_id`); original GitHub IDs live in the `*_source_id` columns. Canonical entrypoint used by `events.py`.
+- `oso.int_events_daily__github_unified` — Daily aggregation (`bucket_day` × artifact × `event_type`, with `amount`)
+
+### GitHub Archive — legacy DDP pipeline models
+Still queried by `repositories.py`, `timeseries-metrics.py`, `developers.py`, `github-archive.py`. Keyed by integer `actor_id` / `repo_id` (not artifact IDs).
 - `oso.stg_github__events` — Raw events with nested fields
-- `oso.int_gharchive__github_events` — Standardized events (canonical entrypoint)
-- `oso.int_ddp_github_events` — Curated subset of event types
+- `oso.int_gharchive__github_events` — Standardized events at PushEvent grain
+- `oso.int_ddp_github_events` — Curated subset of event types (`PushEvent`, `WatchEvent`)
 - `oso.int_ddp_github_events_daily` — Daily aggregation with normalized types
 - `oso.int_gharchive__developer_activities` — Daily rollup for MAD metrics
 
@@ -304,7 +309,7 @@ def _(mo, pyoso_db_conn):
 
 When writing or documenting model notebooks, be explicit about:
 
-- **Freshness**: GitHub Archive is ~3 days behind real-time
+- **Freshness**: GitHub Archive data typically lags real-time by ~1–2 weeks; ODD models (`eco_mads`, `repo_developer_28d_activities`) lag by ~2–3 weeks. Check `MAX(bucket_day)` / `MAX(day)` if exact currency matters.
 - **Completeness**: Public GitHub timeline only — no private repos, no deleted events
 - **Identity**: `actor_id` is a GitHub REST ID; `canonical_developer_id` is from Open Dev Data — don't conflate them
 - **Join keys**: Use `repo_id` (integer) for cross-source joins, not repo names

notebooks/data/models/developers.py

@@ -110,7 +110,7 @@ def _(mo):
 def _(mo):
     mo.mermaid("""
     graph TD
-        A[stg_opendevdata__developers<br/>ODD authors with GraphQL IDs] --> B[int_github__node_id_map<br/>Decode GraphQL → Database ID]
+        A[stg_opendevdata__canonical_developers<br/>ODD authors with GraphQL IDs] --> B[int_github__node_id_map<br/>Decode GraphQL → Database ID]
         B --> D{Union + Deduplicate by user_id}
         C[int_gharchive__github_events<br/>GHA actors with Database IDs] --> D
         D --> E[int_ddp__developers<br/>Unified developer list]

notebooks/data/models/events.py

@@ -977,7 +977,7 @@ def _(mo, pyoso_db_conn):
 
 
 
-@app.cell
+@app.cell(hide_code=True)
 def setup_pyoso():
     # This code sets up pyoso to be used as a database provider for this notebook
     # This code is autogenerated. Modification could lead to unexpected results :)

notebooks/data/sources/github-archive.py

@@ -4,7 +4,7 @@
 app = marimo.App(width="full", css_file="../../styles/data.css")
 
 
-@app.cell
+@app.cell(hide_code=True)
 def setup_pyoso():
     # This code sets up pyoso to be used as a database provider for this notebook
     # This code is autogenerated. Modification could lead to unexpected results :)

notebooks/data/sources/open-dev-data.py

@@ -4,7 +4,7 @@
 app = marimo.App(width="full", css_file="../../styles/data.css")
 
 
-@app.cell
+@app.cell(hide_code=True)
 def setup_pyoso():
     # This code sets up pyoso to be used as a database provider for this notebook
     # This code is autogenerated. Modification could lead to unexpected results :)

notebooks/data/sources/oss-directory.py

@@ -4,7 +4,7 @@
 app = marimo.App(width="full", css_file="../../styles/data.css")
 
 
-@app.cell
+@app.cell(hide_code=True)
 def setup_pyoso():
     # This code sets up pyoso to be used as a database provider for this notebook
     # This code is autogenerated. Modification could lead to unexpected results :)

notebooks/insights/ethereum-repo-rank.py

@@ -702,7 +702,7 @@ def _(mo):
 # =============================================================================
 
 
-@app.cell
+@app.cell(hide_code=True)
 def _(mo, pyoso_db_conn):
     with mo.persistent_cache("panel_devs_v5"):
         df_panel_devs = mo.sql(
@@ -723,7 +723,7 @@ def _(mo, pyoso_db_conn):
 # =============================================================================
 
 
-@app.cell
+@app.cell(hide_code=True)
 def _(mo, pd, pyoso_db_conn):
     with mo.persistent_cache("engagement_unified_v1"):
         df_engagement_raw = mo.sql(
@@ -743,7 +743,7 @@ def _(mo, pd, pyoso_db_conn):
     return (df_engagement_raw,)
 
 
-@app.cell
+@app.cell(hide_code=True)
 def _(mo, pyoso_db_conn):
     with mo.persistent_cache("repo_engagement_v1"):
         df_repo_engagement = mo.sql(
@@ -765,7 +765,7 @@ def _(mo, pyoso_db_conn):
     return (df_repo_engagement,)
 
 
-@app.cell
+@app.cell(hide_code=True)
 def _(mo, pyoso_db_conn):
     with mo.persistent_cache("crossover_devs_v1"):
         df_crossover_devs = mo.sql(
@@ -789,7 +789,7 @@ def _(mo, pyoso_db_conn):
 # =============================================================================
 
 
-@app.cell
+@app.cell(hide_code=True)
 def _(df_repo_engagement, df_engagement_raw, eth_dev_set, pd, REPO_CATEGORIES, REPO_DESCRIPTIONS):
     df_trending = df_repo_engagement.copy()
     df_trending.rename(columns={"repo_lower": "repo_name"}, inplace=True)
@@ -841,7 +841,7 @@ def _(df_repo_engagement, df_engagement_raw, eth_dev_set, pd, REPO_CATEGORIES, R
     return (df_trending,)
 
 
-@app.cell
+@app.cell(hide_code=True)
 def _(df_panel_devs):
     eth_dev_set = set(df_panel_devs["username"].str.lower().tolist())
     return (eth_dev_set,)
@@ -852,7 +852,7 @@ def _(df_panel_devs):
 # =============================================================================
 
 
-@app.cell
+@app.cell(hide_code=True)
 def _(df_trending):
     df_signal_strength = df_trending.copy()
     df_signal_strength.rename(columns={
@@ -868,7 +868,7 @@ def _(df_trending):
 # =============================================================================
 
 
-@app.cell
+@app.cell(hide_code=True)
 def _(df_engagement_raw, df_trending, np, pd, REPO_CATEGORIES):
     _eth_repos = [k.lower() for k, v in REPO_CATEGORIES.items() if v == "Ethereum & Crypto"]
     _eth_in_data = df_trending[df_trending["repo_name"].isin(_eth_repos)].sort_values("eth_devs_30d", ascending=False).head(12)["repo_name"].tolist()
@@ -905,7 +905,7 @@ def _(df_engagement_raw, df_trending, np, pd, REPO_CATEGORIES):
 # =============================================================================
 
 
-@app.cell
+@app.cell(hide_code=True)
 def _(df_engagement_raw):
     _df = df_engagement_raw.copy()
     _df["day"] = _df["ts"].dt.floor("D")