Skip to content

Commit 31d8f36

Browse files
authored
Merge branch 'main' into fix/index-worker-explicit-memory-budget
2 parents b98fd47 + 010569f commit 31d8f36

38 files changed

Lines changed: 1660 additions & 134 deletions

‎.github/workflows/scorecard.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,13 +20,13 @@ jobs:
2020
persist-credentials: false
2121

2222
- name: Run OpenSSF Scorecard
23-
uses: ossf/scorecard-action@4eaacf0543bb3f2c246792bd56e8cdeffafb205a # v2.4.3
23+
uses: ossf/scorecard-action@2d1146689b8cda280b9bc96326124645441f03bc # v2.4.4
2424
with:
2525
results_file: results.sarif
2626
results_format: sarif
2727
publish_results: true
2828

2929
- name: Upload SARIF results
30-
uses: github/codeql-action/upload-sarif@7188fc363630916deb702c7fdcf4e481b751f97a # v4
30+
uses: github/codeql-action/upload-sarif@5595ccaf912efad79be6eef63a5619ff05969be3 # v4
3131
with:
3232
sarif_file: results.sarif

‎.github/workflows/stale.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ jobs:
3434
permissions:
3535
issues: write
3636
steps:
37-
- uses: actions/stale@1e223db275d687790206a7acac4d1a11bd6fe629 # v10.4.0
37+
- uses: actions/stale@4391f3da665fdf50b6810c1a66712fb9ba21aa93 # v11.0.0
3838
with:
3939
only-labels: 'awaiting-reporter'
4040
stale-issue-label: 'stale'

‎Makefile.cbm‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -245,6 +245,7 @@ EXTRACTION_SRCS = \
245245
$(CBM_DIR)/extract_env_accesses.c \
246246
$(CBM_DIR)/extract_channels.c \
247247
$(CBM_DIR)/extract_k8s.c \
248+
$(CBM_DIR)/extract_dbt.c \
248249
$(CBM_DIR)/helpers.c \
249250
$(CBM_DIR)/lang_specs.c \
250251
$(CBM_DIR)/macro_table.c \

‎flake.nix‎

Lines changed: 52 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,57 @@
77
let
88
systems = [ "aarch64-darwin" "x86_64-darwin" "aarch64-linux" "x86_64-linux" ];
99
forAllSystems = f: nixpkgs.lib.genAttrs systems (system: f nixpkgs.legacyPackages.${system});
10+
11+
# Cross dev shell (macOS only): build the *other* darwin arch locally.
12+
# The native clang already cross-emits object code via -arch (scripts/env.sh
13+
# exports ARCHFLAGS), so the only thing missing for a cross link is zlib:
14+
# it is the product's single link-time dependency (`-lz`), and the host-arch
15+
# copy cannot satisfy an x86_64 link on arm64 or vice versa. This shell puts
16+
# the TARGET-arch zlib on the include and link paths, and nothing else.
17+
#
18+
# Building needs no Rosetta; *running* an x86_64 binary on Apple Silicon
19+
# does. The target-arch libs come from prebuilt substitutes, so entering the
20+
# shell is a download rather than a cross-build of the dependency tree.
21+
#
22+
# Returns {} off macOS, where there is no second darwin arch to target.
23+
crossDevShells = pkgs:
24+
let
25+
inherit (nixpkgs) lib;
26+
crossTarget =
27+
{
28+
"aarch64-darwin" = "x86_64-darwin";
29+
"x86_64-darwin" = "aarch64-darwin";
30+
}
31+
.${pkgs.system} or null;
32+
mkCrossShell =
33+
targetSystem:
34+
let
35+
tpkgs = nixpkgs.legacyPackages.${targetSystem};
36+
targetArch = if targetSystem == "x86_64-darwin" then "x86_64" else "arm64";
37+
in
38+
pkgs.mkShell {
39+
# Host toolchain only — clang comes from the shell stdenv. Deliberately
40+
# NOT inputsFrom the default package: that would put the HOST-arch zlib
41+
# on the link path, which is exactly the wrong-arch copy this shell
42+
# exists to displace.
43+
nativeBuildInputs = [ pkgs.gnumake ];
44+
shellHook = ''
45+
# zlib is consumed as a bare `-lz` / `#include <zlib.h>`, so its
46+
# include dir (the header is arch-independent) and lib dir are
47+
# supplied by hand. NIX_LDFLAGS is searched ahead of the link's own
48+
# -L, so the target-arch slice wins; a host-arch copy that still
49+
# reaches ld is skipped as "wrong architecture" rather than
50+
# producing a silently mislinked binary.
51+
export NIX_CFLAGS_COMPILE="-I${lib.getDev tpkgs.zlib}/include ''${NIX_CFLAGS_COMPILE:-}"
52+
export NIX_LDFLAGS="-L${lib.getLib tpkgs.zlib}/lib ''${NIX_LDFLAGS:-}"
53+
echo "[cross] target ${targetSystem} — build with: scripts/build.sh --arch ${targetArch}"
54+
'';
55+
};
56+
in
57+
lib.optionalAttrs (crossTarget != null) {
58+
# `nix develop .#cross` then `scripts/build.sh --arch <target>`.
59+
cross = mkCrossShell crossTarget;
60+
};
1061
in
1162
{
1263
packages = forAllSystems (pkgs: {
@@ -45,6 +96,6 @@
4596
default = pkgs.mkShell {
4697
inputsFrom = [ self.packages.${pkgs.system}.default ];
4798
};
48-
});
99+
} // crossDevShells pkgs);
49100
};
50101
}

‎internal/cbm/cbm.c‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1303,6 +1303,13 @@ CBMFileResult *cbm_extract_file_ex(const char *source, int source_len, CBMLangua
13031303
cbm_extract_k8s(&ctx);
13041304
}
13051305

1306+
// dbt lineage pass: a dbt model's dependencies live in Jinja ({{ ref(...) }}),
1307+
// which the SQL grammar cannot read. Self-gated — SQL files only, and only
1308+
// those carrying a real dbt builtin call.
1309+
if (ctx.language == CBM_LANG_SQL) {
1310+
cbm_extract_dbt(&ctx);
1311+
}
1312+
13061313
// LSP type-aware call/usage resolution (per-file). Runs in every mode;
13071314
// refines the tree-sitter + textual-resolution graph with type info.
13081315
uint64_t lsp_start = now_ns();

‎internal/cbm/cbm.h‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -726,6 +726,10 @@ void cbm_channels_push(CBMChannelArray *arr, CBMArena *a, CBMChannel ch);
726726
// --- Sub-extractor entry points ---
727727

728728
void cbm_extract_definitions(CBMExtractCtx *ctx);
729+
// dbt lineage for Jinja-templated SQL models: emits a Model def plus one usage
730+
// per ref()/source() call. No-op unless the file parses as SQL and actually
731+
// contains a dbt builtin call. Defined in extract_dbt.c.
732+
void cbm_extract_dbt(CBMExtractCtx *ctx);
729733
void cbm_extract_imports(CBMExtractCtx *ctx);
730734
void cbm_extract_usages(CBMExtractCtx *ctx);
731735
void cbm_extract_semantic(CBMExtractCtx *ctx);
@@ -753,4 +757,17 @@ void cbm_extract_k8s(CBMExtractCtx *ctx);
753757
// `label` may be NULL (returns false). Defined in helpers.c.
754758
bool cbm_label_is_type_like(const char *label);
755759

760+
// True for data-relation labels (Table, View — SQL DDL). Relations resolve as
761+
// lineage targets only: registry members, but never type-like and never valid
762+
// CALLS/THROWS/READS/WRITES targets. `label` may be NULL. Defined in helpers.c.
763+
bool cbm_label_is_relation(const char *label);
764+
765+
// True for labels admitted to the cross-file name registry: Function, Method,
766+
// every type-like container, Variable, Field, and the relation labels. Single
767+
// source of truth for registry seeding — the full (pass_definitions.c),
768+
// parallel (pass_parallel.c) and incremental (pipeline_incremental.c) pipelines
769+
// all seed through this predicate so their registries never diverge.
770+
// `label` may be NULL (returns false). Defined in helpers.c.
771+
bool cbm_label_is_registry_symbol(const char *label);
772+
756773
#endif // CBM_H

‎internal/cbm/extract_dbt.c‎

Lines changed: 219 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,219 @@
1+
// extract_dbt.c — dbt lineage extractor for Jinja-templated SQL models.
2+
//
3+
// A dbt model is an ordinary .sql file in the repository whose SELECT is
4+
// templated with Jinja: dependencies are written `{{ ref('other_model') }}`
5+
// (another model in the project) or `{{ source('group', 'table') }}` (a raw
6+
// warehouse table), never as literal table names. The SQL grammar cannot read
7+
// those — `FROM {{ ref('x') }}` is a parse error to it — so the dependency
8+
// structure of an entire dbt project is invisible to the code graph without
9+
// this pass.
10+
//
11+
// The vendored tree-sitter-jinja2 grammar models `{{ ... }}` expressions
12+
// (jinja_expression / fn_call / lit_string) but has no node types at all for
13+
// `{% ... %}` statements, so this extractor covers lineage only. `{% macro %}`
14+
// definitions would need a hand-written scanner and are deliberately left out
15+
// rather than recovered approximately.
16+
//
17+
// What it emits, per qualifying file:
18+
// - one Model definition, named by the file stem (dbt's own model identity)
19+
// - one usage per ref()/source() call, scoped to that Model
20+
// pass_usages then resolves each usage against the shared definition registry
21+
// and emits a `model -USAGE-> relation` lineage edge. Model is a relation label
22+
// (cbm_label_is_relation), so lineage joins dbt models to Table/View nodes
23+
// declared in plain DDL elsewhere in the same repository, and the registry's
24+
// relation veto keeps these names out of every non-lineage consumer.
25+
//
26+
// Gate: the file must parse as SQL, contain a Jinja expression delimiter, and
27+
// contain at least one real ref()/source() call. Generic templated SQL (an
28+
// Airflow `{{ ds }}` parameter, say) therefore produces nothing at all — the
29+
// dbt builtins are the evidence that this is a dbt model, so no dbt_project.yml
30+
// lookup is needed and no non-dbt repository pays for a Model node it did not
31+
// ask for.
32+
33+
#include "cbm.h"
34+
#include "arena.h"
35+
#include "helpers.h"
36+
#include "lang_specs.h"
37+
#include "tree_sitter/api.h"
38+
#include "foundation/constants.h"
39+
#include <stdbool.h>
40+
#include <stdint.h>
41+
#include <string.h>
42+
43+
/* Local constants. */
44+
enum { DBT_FIRST_LINE = 1 };
45+
46+
/* Cheap pre-filter: does the source contain a Jinja expression opener? Files
47+
* without one cannot hold a ref()/source() call, and skipping them keeps the
48+
* second parse off every ordinary .sql file in the repository. */
49+
static bool source_has_jinja_expr(const char *s, int len) {
50+
for (int i = 0; i + 1 < len; i++) {
51+
if (s[i] == '{' && s[i + 1] == '{') {
52+
return true;
53+
}
54+
}
55+
return false;
56+
}
57+
58+
/* Strip one pair of surrounding quotes from a jinja lit_string. The grammar
59+
* hands back the token with its quotes attached. */
60+
static char *dbt_unquote(CBMArena *a, char *s) {
61+
if (!s) {
62+
return NULL;
63+
}
64+
size_t n = strlen(s);
65+
if (n >= 2 && (s[0] == '\'' || s[0] == '"') && s[n - 1] == s[0]) {
66+
char *inner = cbm_arena_strdup(a, s + 1);
67+
if (!inner) {
68+
return s;
69+
}
70+
size_t m = strlen(inner);
71+
if (m > 0) {
72+
inner[m - 1] = '\0';
73+
}
74+
return inner;
75+
}
76+
return s;
77+
}
78+
79+
/* Rightmost lit_string under `node` in DFS order. Both dbt builtins put the
80+
* referenced relation last: ref('model'), ref('package', 'model') and
81+
* source('group', 'table') all name the relation in their final string
82+
* argument. The callee itself is an identifier, never a lit_string, so it
83+
* cannot be mistaken for one. */
84+
static TSNode dbt_last_lit_string(TSNode node, TSNode best, bool *found) {
85+
if (strcmp(ts_node_type(node), "lit_string") == 0) {
86+
best = node;
87+
*found = true;
88+
}
89+
uint32_t cc = ts_node_child_count(node);
90+
for (uint32_t i = 0; i < cc; i++) {
91+
best = dbt_last_lit_string(ts_node_child(node, i), best, found);
92+
}
93+
return best;
94+
}
95+
96+
/* Collect ref()/source() targets from a jinja2 parse tree into `out`, which is
97+
* the file's usage array. Usages are scoped to enclosing_qn (the Model). */
98+
static void collect_dbt_refs(CBMExtractCtx *ctx, TSNode node, const char *enclosing_qn,
99+
CBMUsageArray *out) {
100+
if (strcmp(ts_node_type(node), "fn_call") == 0) {
101+
TSNode fn = ts_node_child_by_field_name(node, "fn_name", (uint32_t)strlen("fn_name"));
102+
if (ts_node_is_null(fn)) {
103+
fn = cbm_find_child_by_kind(node, "identifier");
104+
}
105+
if (!ts_node_is_null(fn)) {
106+
char *fname = cbm_node_text(ctx->arena, fn, ctx->source);
107+
if (fname && (strcmp(fname, "ref") == 0 || strcmp(fname, "source") == 0)) {
108+
bool found = false;
109+
TSNode empty = {0};
110+
TSNode strn = dbt_last_lit_string(node, empty, &found);
111+
if (found) {
112+
char *name =
113+
dbt_unquote(ctx->arena, cbm_node_text(ctx->arena, strn, ctx->source));
114+
if (name && name[0]) {
115+
CBMUsage usage = {0};
116+
usage.ref_name = name;
117+
usage.enclosing_func_qn = enclosing_qn;
118+
usage.site_start_byte = ts_node_start_byte(strn);
119+
usage.site_end_byte = ts_node_end_byte(strn);
120+
cbm_usages_push(out, ctx->arena, usage);
121+
}
122+
}
123+
}
124+
}
125+
}
126+
uint32_t cc = ts_node_child_count(node);
127+
for (uint32_t i = 0; i < cc; i++) {
128+
collect_dbt_refs(ctx, ts_node_child(node, i), enclosing_qn, out);
129+
}
130+
}
131+
132+
/* dbt model identity is the file stem: models/staging/stg_users.sql is the
133+
* model `stg_users`, and `{{ ref('stg_users') }}` anywhere in the project
134+
* addresses it by that bare name (dbt requires model names to be unique across
135+
* a project, so the directory is deliberately not part of the identity). */
136+
static char *dbt_name_from_path(CBMArena *a, const char *rel_path) {
137+
if (!rel_path) {
138+
return NULL;
139+
}
140+
const char *base = rel_path;
141+
for (const char *p = rel_path; *p; p++) {
142+
if (*p == '/' || *p == '\\') {
143+
base = p + 1;
144+
}
145+
}
146+
const char *dot = NULL;
147+
for (const char *p = base; *p; p++) {
148+
if (*p == '.') {
149+
dot = p;
150+
}
151+
}
152+
size_t n = dot ? (size_t)(dot - base) : strlen(base);
153+
if (n == 0) {
154+
return NULL;
155+
}
156+
char *out = cbm_arena_strdup(a, base);
157+
if (!out) {
158+
return NULL;
159+
}
160+
out[n] = '\0';
161+
return out;
162+
}
163+
164+
void cbm_extract_dbt(CBMExtractCtx *ctx) {
165+
if (!ctx || ctx->language != CBM_LANG_SQL || !ctx->source || ctx->source_len <= 0) {
166+
return;
167+
}
168+
if (!source_has_jinja_expr(ctx->source, ctx->source_len)) {
169+
return;
170+
}
171+
const TSLanguage *jl = cbm_ts_language(CBM_LANG_JINJA2);
172+
if (!jl) {
173+
return;
174+
}
175+
char *model_name = dbt_name_from_path(ctx->arena, ctx->rel_path);
176+
if (!model_name || !model_name[0]) {
177+
return;
178+
}
179+
const char *model_qn = cbm_fqn_compute(ctx->arena, ctx->project, ctx->rel_path, model_name);
180+
if (!model_qn) {
181+
return;
182+
}
183+
184+
/* A fresh parser: the primary SQL pass owns the thread-local one, and this
185+
* runs inside its walk. */
186+
TSParser *parser = ts_parser_new();
187+
if (!parser) {
188+
return;
189+
}
190+
/* Refs are staged locally so a file with Jinja but no dbt builtins commits
191+
* nothing at all — neither usages nor a Model node. */
192+
CBMUsageArray staged = {0};
193+
if (ts_parser_set_language(parser, jl)) {
194+
TSTree *tree = ts_parser_parse_string(parser, NULL, ctx->source, (uint32_t)ctx->source_len);
195+
if (tree) {
196+
collect_dbt_refs(ctx, ts_tree_root_node(tree), model_qn, &staged);
197+
ts_tree_delete(tree);
198+
}
199+
}
200+
ts_parser_delete(parser);
201+
if (staged.count == 0) {
202+
return; /* templated SQL, but not dbt — emit nothing */
203+
}
204+
205+
CBMDefinition def;
206+
memset(&def, 0, sizeof(def));
207+
def.name = model_name;
208+
def.qualified_name = model_qn;
209+
def.label = "Model";
210+
def.file_path = ctx->rel_path;
211+
def.start_line = DBT_FIRST_LINE;
212+
def.end_line = ts_node_end_point(ctx->root).row + TS_LINE_OFFSET;
213+
def.is_exported = true;
214+
cbm_defs_push(&ctx->result->defs, ctx->arena, def);
215+
216+
for (int i = 0; i < staged.count; i++) {
217+
cbm_usages_push(&ctx->result->usages, ctx->arena, staged.items[i]);
218+
}
219+
}

0 commit comments

Comments
 (0)