Merge pull request #1 from DevelopmentFinanceObservatory/chore/sync-website-v3.5.13

chore: sync website v3.5.13 for Data Commons Platform 1.1.6

Author: jm-rivera

.dfo-protected-paths

@@ -10,12 +10,21 @@ build/cdc_services/Dockerfile
 build/cdc_services/run.sh
 build/cdc_services/nginx.conf
 
-# Repo-level DFO files
+# Repo-level DFO files. Every upstream-tracked file DFO modifies, adds, or
+# deletes belongs here, or a conflict on it silently takes upstream's side.
 justfile
 .dockerignore
 .gcloudignore
 .dfo-protected-paths
 .dfo-overridden-files
+.gitattributes
+.gitignore
+README.md
+package-lock.json
+pyproject.toml
+CONTRIBUTING-DFO.md
+# Deleted in this fork; a conflict here means upstream changed it.
+.github/CODEOWNERS
 
 # CI/deploy workflows and deploy docs are DFO-owned even though they live
 # under upstream-owned directories (.github/, deploy/); an upstream file

.dockerignore

@@ -16,6 +16,7 @@ import/simple/sample/
 **/.data
 **/testdata
 **/test_data
+import/simple/tests/
 
 # ONE/DFO: exclude build artifacts from the Docker build context; keep source.
 **/.next

.gcloudignore

@@ -7,9 +7,6 @@
 #   $ gcloud topic gcloudignore
 #
 .gcloudignore
-# If you would like to upload your .git directory, .gitignore file or files
-# from your .gitignore file, remove the corresponding line
-# below:
 .git
 .gitignore
 #!include:.gitignore
@@ -23,13 +20,11 @@
 # Ignored by the build system
 /setup.cfg
 
-static/node_modules/
+**/node_modules/
 
 server/dist/
 
-**/.venv/
-**/.venv_website/
-**/.venv_nl/
+**/.venv*/
 
 # ONE/DFO: exclude build artifacts from Cloud Build context; keep source.
 # (.dockerignore above already propagates these via #!include, but listed explicitly for parity.)

.gitattributes

@@ -1,5 +1,5 @@
 # Keep DFO-owned copies of upstream-tracked files across `git merge upstream/master`.
 # `merge=ours` makes git keep our version and discard upstream's changes to these paths.
-# The driver is registered per-clone by `just setup` (git config merge.ours.driver true);
+# `just sync` and `just setup` register the driver per clone (git config merge.ours.driver true);
 # without it git falls back to a normal 3-way merge.
 README.md merge=ours

.github/workflows/build-and-pin.yml

@@ -110,11 +110,12 @@ jobs:
           fi
 
       - name: Free disk space
-        # ubuntu-latest has ~14 GB usable, and this build pulls large NL
-        # model/embedding files, installs torch CPU wheels, and exports every
-        # intermediate layer via mode=max cache-to. Removing the runner's
-        # preinstalled Android SDK, .NET, and GHC toolchains buys back room
-        # without a third-party action.
+        # ubuntu-latest has ~14 GB usable. The build holds four stages at
+        # once (Go toolchain and module cache, the website's Python venv,
+        # and two Node installs with their builds) on top of the runtime
+        # base, and mode=max cache-to exports every intermediate layer.
+        # Removing the runner's preinstalled Android SDK, .NET, and GHC
+        # toolchains buys back room without a third-party action.
         run: |
           sudo rm -rf /usr/local/lib/android
           sudo rm -rf /usr/share/dotnet

.github/workflows/ci.yml

@@ -8,10 +8,11 @@ name: CI
 # tree before it builds.
 #
 # Does not build the Docker image. build/cdc_services/Dockerfile compiles
-# the Go mixer, builds two Python venvs, and runs a Next.js production
-# build across four stages -- several minutes even with cache, so it only
-# runs on push to a deploy branch, in deploy-dev.yml and deploy-prod.yml,
-# with a registry-backed cache to keep that path fast.
+# the Go mixer, installs the website's Python venv, builds the website's
+# static assets, and runs a Next.js production build, in four build stages
+# -- several minutes even with cache, so it only runs on push to a deploy
+# branch, in deploy-dev.yml and deploy-prod.yml, with a registry-backed
+# cache to keep that path fast.
 
 on:
   pull_request:

CONTRIBUTING-DFO.md

@@ -9,7 +9,7 @@ The DFO's Data Commons instance, meaning the mixer and the graph-serving layer u
 Two kinds of code live here, and they behave differently.
 
 - **Upstream code.** Everything outside `custom_dc/` tracks datacommonsorg/website and the `mixer` and `import` submodules. We take changes from upstream; we do not write them here.
-- **ONE Data code.** Everything under `custom_dc/` is ours (for the DFO). `custom_dc/dfo/prototypes` is the frontend, and `custom_dc/api_server` holds a legacy copy of the Search API that stays until its removal PR lands. Upstream never touches these paths, so an upstream sync cannot conflict with work there.
+- **ONE Data code.** Everything under `custom_dc/` is ours (for the DFO). `custom_dc/dfo/prototypes` is the frontend. Upstream never touches these paths, so an upstream sync cannot conflict with work there.
 
 The repo builds one container image. A sibling repo, `infra`, deploys it to Cloud Run. This repo builds and pushes an image, and `infra` runs it.
 
@@ -25,8 +25,6 @@ feat/*  ──►  dev  ──►  main
          dfo-dev     dfo-prod
 ```
 
-`customdc_stable` is not a tier. It mirrors upstream's stable release channel and serves as the reference point for `just sync`.
-
 ## Where your work goes
 
 The loop is short.
@@ -44,34 +42,39 @@ The loop is short.
 
 ## Taking upstream changes
 
-Upstream sync is a step on `dev`, not a separate long-lived branch.
+`dev` tracks Data Commons Platform (DCP) releases. `infra` pins a DCP version as `dcp_version` in `infra/stacks/dcp/variables.tf`, and `dev` carries the website and mixer that release was built from. A sync is a merge of that website commit into `dev`, so `dev` shares history with datacommonsorg/website.
 
 ```sh
 git checkout dev
-git fetch upstream
-just sync                 # shows what changed since the last sync
-git merge upstream/master
-git submodule update --init --recursive
+just sync 1.1.6           # merge the website commit DCP 1.1.6 was built from
 ```
 
-Merging upstream's website also moves the `mixer` and `import` submodule pointers, because upstream pins them. **Website and submodules must move together.**
+`just sync` stops on conflicts. `just sync-resolve` takes upstream's side for every path outside `.dfo-protected-paths` and leaves the protected ones for you. After resolving those, `git add` them and run `just sync-finish`. `just sync-abort` drops the merge. Once committed, the recipe checks out the submodules and runs `just check-overrides`, which lists the seam files in `.dfo-overridden-files` that upstream changed across the sync.
 
 ### Matching a Data Commons Platform release
 
-`infra` pins a DCP version (`1.1.x`) for the Terraform stack, and the image this repo builds has to match it. A website tag alone will not tell you which mixer that release runs. Google builds the release image from a website tag plus a mixer commit that runs ahead of what the tag pins. DCP 1.1.2 was built from website `v3.5.7` with mixer `79627f8`, while `v3.5.7` itself pins mixer `051d8ff`.
+The website and mixer move as a pair, and the pairing comes from the release image. Each image `gcr.io/datcom-ci/datacommons-services:<version>` records its source commits as the `WEBSITE_HASH` and `MIXER_HASH` build args. A website tag pins one mixer, and Google can build a release with a newer one. DCP 1.1.2 was built from website `v3.5.7` with mixer `79627f8`, while `v3.5.7` pins mixer `051d8ff`.
+
+`just dcp-pair <version>` prints the pairing:
+
+```sh
+$ just dcp-pair 1.1.6
+website=9852134
+mixer=8118587
+```
 
-That mapping exists only inside the published image, as build args:
+It reads the image config from the registry, the same as:
 
 ```sh
 CFG=$(curl -s -H "Accept: application/vnd.docker.distribution.manifest.v2+json" \
-  https://gcr.io/v2/datcom-ci/datacommons-services/manifests/1.1.2 | jq -r .config.digest)
+  https://gcr.io/v2/datcom-ci/datacommons-services/manifests/1.1.6 | jq -r .config.digest)
 curl -sL https://gcr.io/v2/datcom-ci/datacommons-services/blobs/$CFG \
   | jq '.config.Env[] | select(contains("HASH"))'
 ```
 
-To move to a new DCP release, read `WEBSITE_HASH` and `MIXER_HASH` off that release's image, merge the website commit, then set the `mixer` pointer to `MIXER_HASH`. Taking the mixer the website tag carries instead builds a pairing Google never tested against that release's Terraform stack.
+`just sync` merges `WEBSITE_HASH`, sets the `mixer` pointer to `MIXER_HASH`, and sets the `import` pointer to the one the website commit pins. It fails if the image or either build arg is missing. Taking the mixer the website tag carries builds a pairing Google never tested against that release's Terraform stack.
 
-`.github/workflows/codeql-analysis.yml`, `dependabot-approval.yml`, and `submodule-update.yml` are deleted in this fork. All three guard their jobs on `github.repository == 'datacommonsorg/website'`, so they never run here. Upstream re-adds them on every sync that touches them. Keep them deleted rather than resolving the conflict toward upstream's version.
+`.github/workflows/` is DFO-owned. A workflow file upstream adds there would run on the fork, so a sync takes nothing from upstream under that directory. `just sync` and `just sync-finish` reset it to the pre-merge version and list every file upstream added, changed, or deleted there. Port one by hand in its own PR if the fork needs it. Upstream's `codeql-analysis.yml`, `dependabot-approval.yml`, and `submodule-update.yml` stay deleted this way.
 
 ## Releasing to production
 
@@ -109,5 +112,6 @@ always reproducible from a committed sha.
 
 - **Commits** follow [Conventional Commits](https://www.conventionalcommits.org): `feat:`, `fix:`, `chore:`, and so on.
 - **PRs target `dev`.** Only the release step writes to `main`.
+- **PRs into `dev` land as merge commits.** The `dev` ruleset allows only merges. A sync commit's second parent is the upstream website commit, and that parent is what keeps the fork's history shared with datacommonsorg/website. A squash or rebase drops it, and the next sync merges from an older base and replays upstream changes the fork already has as conflicts.
 - **Delete merged branches.** Feature branches are short-lived.
 - **Keep internal notes out of the repo.** Design docs, plans, and audits stay local. This guide and real reference docs are the exception.

README.md

@@ -6,7 +6,7 @@ New here? Read [CONTRIBUTING-DFO.md](./CONTRIBUTING-DFO.md) before opening a bra
 
 ## Layout
 
-- **`custom_dc/`** is ONE Data's DFO code. `custom_dc/dfo/prototypes` is the frontend, and `custom_dc/api_server` holds a legacy copy of the Search API that stays until its removal PR lands. The Search API and Datasets API are built in [apis](https://github.com/DevelopmentFinanceObservatory/apis). Upstream never touches these paths.
+- **`custom_dc/`** is ONE Data's DFO code. `custom_dc/dfo/prototypes` is the frontend. The Search API and Datasets API are built in [apis](https://github.com/DevelopmentFinanceObservatory/apis). Upstream never touches these paths.
 - **Everything else** tracks upstream `datacommonsorg/website`, with the graph server (`mixer`) and ingestion (`import`) as submodules.
 
 ## Quickstart

build/cdc_data/Dockerfile

@@ -20,16 +20,21 @@ ENV ENV=${ENV}
 
 WORKDIR /workspace
 
+# Apply security patches for OS base packages (OpenSSL, Perl, glibc, pcre2, etc.)
+RUN apt-get update && apt-get upgrade -y \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
 
 # Copy simple importer requirements.
 COPY import/simple/requirements.txt ./import/simple/requirements.txt
 
 ARG PIP_DISABLE_PIP_VERSION_CHECK=1
 ARG PIP_NO_CACHE_DIR=1
 
-# Create a virtual env, add it to path, and install all requirements.
+# Create a virtual env, add it to path, upgrade core packaging tools, and install all requirements.
 RUN python -m venv /workspace/venv
 ENV PATH="/workspace/venv/bin:$PATH"
+RUN pip3 install --no-cache-dir --upgrade pip "setuptools<70" wheel
 
 # Install simple importer requirements.
 RUN pip3 install -r ./import/simple/requirements.txt
@@ -47,4 +52,4 @@ RUN chmod +x run.sh
 ENV PATH="/workspace/venv/bin:$PATH"
 
 # Set the entrypoint to force bash execution
-ENTRYPOINT ["/bin/bash", "./run.sh"]
+ENTRYPOINT ["/bin/bash", "./run.sh"]

build/cdc_services/Dockerfile

@@ -49,7 +49,7 @@ RUN protoc \
 RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o bin/mixer cmd/main.go
 
 
-# #### Stage 2: Build python servers (website and NL). ####
+# #### Stage 2: Build python servers (website and MCP). ####
 FROM --platform=linux/amd64 python:3.11.14-slim as py-servers
 
 ARG PIP_DISABLE_PIP_VERSION_CHECK=1
@@ -61,33 +61,21 @@ WORKDIR /workspace
 RUN apt-get update && apt-get -y upgrade && apt-get install -y git
 
 # Copy requirements.
-# Copy nl_requirements.txt since it is referenced by embeddings requirements.txt
 COPY server/requirements.txt ./server/requirements.txt
-COPY nl_server/requirements.txt ./nl_server/requirements.txt
-COPY nl_requirements.txt ./nl_requirements.txt
-COPY torch_requirements.txt ./torch_requirements.txt
 
 # Create a virtual env and install requirements.
 RUN python -m venv ./venv
 ENV PATH="/workspace/venv/bin:$PATH"
-RUN pip3 install --upgrade pip setuptools \
+RUN pip3 install --no-cache-dir --upgrade pip setuptools \
     && pip3 install --no-cache-dir -r /workspace/server/requirements.txt \
-    && pip3 install --no-cache-dir -r /workspace/torch_requirements.txt --index-url https://download.pytorch.org/whl/cpu \
-    && pip3 install --no-cache-dir -r /workspace/nl_server/requirements.txt \
     && pip3 install --no-cache-dir datacommons-mcp
 
 # Copy website app script.
 COPY web_app.py ./web_app.py
 # Copy server module.
 COPY server/. ./server/
-# Copy NL app script.
-COPY nl_app.py ./nl_app.py
-# Copy nl_server module.
-COPY nl_server/. ./nl_server/
 # Copy the shared module.
 COPY shared/. ./shared/
-# Copy yaml files.
-COPY deploy/nl/. /datacommons/nl/
 
 
 # #### Stage 3: Build static website assets. ####
@@ -116,19 +104,6 @@ WORKDIR /workspace/static
 RUN npm run-script build
 
 
-# #### Stage 4: Download base dc model and embeddings from GCS. ####
-FROM --platform=linux/amd64 google/cloud-sdk:slim as download
-
-# Remove auth check since these are public buckets and we don't have credentials within the container.
-# Copy model and embeddings.
-RUN gcloud config set auth/disable_credentials True \
-    && mkdir -p /tmp/datcom-nl-models \
-    && gcloud storage cp --recursive \
-        gs://datcom-nl-models/ft_final_v20230717230459.all-MiniLM-L6-v2/ \
-        gs://datcom-nl-models/embeddings_medium_2024_05_09_18_01_32.ft_final_v20230717230459.all-MiniLM-L6-v2.csv \
-        /tmp/datcom-nl-models/
-
-
 # >>> DFO bake-in (custom_dc/dfo) >>>
 # ONE/DFO: Build the DFO prototyping Next.js app (custom_dc/dfo/prototypes).
 # Isolated stage; upstream never edits this. See custom_dc/dfo/README.md.
@@ -145,7 +120,10 @@ COPY custom_dc/dfo/prototypes/ ./
 # Wire /prototype/search to the live Search API. This is a NEXT_PUBLIC_* var, so
 # it is inlined into the browser bundle at build time — a runtime env on Cloud Run
 # cannot flip it. Set to 0 (--build-arg NEXT_PUBLIC_USE_SEARCH_API=0) to bake a
-# fixtures-only image. The engine still needs GEMINI_API_KEY set at runtime.
+# fixtures-only image. /api/search is rewritten to dfo-apis and needs no key here.
+# GEMINI_API_KEY (or GOOGLE_API_KEY) is read at runtime by /api/similarity, which
+# falls back to pseudo-embeddings without it, and by /api/skill-compare
+# (lib/skillAgent.ts), which returns an error without it.
 ARG NEXT_PUBLIC_USE_SEARCH_API=1
 ENV NEXT_PUBLIC_USE_SEARCH_API=${NEXT_PUBLIC_USE_SEARCH_API}
 # The website origin that next.config.mjs bakes into the /api/search and
@@ -194,13 +172,10 @@ COPY --from=mixer /workspace/deploy/featureflags deploy/featureflags
 COPY --from=mixer /workspace/mixer-grpc.pb .
 # nginx.
 COPY build/cdc_services/nginx.conf .
-# Website and NL servers.
+# Website server.
 COPY --from=py-servers /workspace/ .
-COPY --from=py-servers /datacommons/ /datacommons
 # Static website assets.
 COPY --from=static /workspace/server ./server
-# Downloaded model and embeddings.
-COPY --from=download /tmp/datcom-nl-models /tmp/datcom-nl-models
 
 # Copy executable script.
 COPY build/cdc_services/run.sh .