Skip to content

Commit 34d18ae

Browse files
authored
Merge pull request DeusData#1756 from DeusData/feat/dbt-jinja-lineage
feat(dbt): dbt model lineage from Jinja-templated SQL
2 parents 4bbc978 + 43ecc09 commit 34d18ae

9 files changed

Lines changed: 386 additions & 10 deletions

File tree

‎Makefile.cbm‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -245,6 +245,7 @@ EXTRACTION_SRCS = \
245245
$(CBM_DIR)/extract_env_accesses.c \
246246
$(CBM_DIR)/extract_channels.c \
247247
$(CBM_DIR)/extract_k8s.c \
248+
$(CBM_DIR)/extract_dbt.c \
248249
$(CBM_DIR)/helpers.c \
249250
$(CBM_DIR)/lang_specs.c \
250251
$(CBM_DIR)/macro_table.c \

‎internal/cbm/cbm.c‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1303,6 +1303,13 @@ CBMFileResult *cbm_extract_file_ex(const char *source, int source_len, CBMLangua
13031303
cbm_extract_k8s(&ctx);
13041304
}
13051305

1306+
// dbt lineage pass: a dbt model's dependencies live in Jinja ({{ ref(...) }}),
1307+
// which the SQL grammar cannot read. Self-gated — SQL files only, and only
1308+
// those carrying a real dbt builtin call.
1309+
if (ctx.language == CBM_LANG_SQL) {
1310+
cbm_extract_dbt(&ctx);
1311+
}
1312+
13061313
// LSP type-aware call/usage resolution (per-file). Runs in every mode;
13071314
// refines the tree-sitter + textual-resolution graph with type info.
13081315
uint64_t lsp_start = now_ns();

‎internal/cbm/cbm.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -726,6 +726,10 @@ void cbm_channels_push(CBMChannelArray *arr, CBMArena *a, CBMChannel ch);
726726
// --- Sub-extractor entry points ---
727727

728728
void cbm_extract_definitions(CBMExtractCtx *ctx);
729+
// dbt lineage for Jinja-templated SQL models: emits a Model def plus one usage
730+
// per ref()/source() call. No-op unless the file parses as SQL and actually
731+
// contains a dbt builtin call. Defined in extract_dbt.c.
732+
void cbm_extract_dbt(CBMExtractCtx *ctx);
729733
void cbm_extract_imports(CBMExtractCtx *ctx);
730734
void cbm_extract_usages(CBMExtractCtx *ctx);
731735
void cbm_extract_semantic(CBMExtractCtx *ctx);

‎internal/cbm/extract_dbt.c‎

Lines changed: 219 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,219 @@
1+
// extract_dbt.c — dbt lineage extractor for Jinja-templated SQL models.
2+
//
3+
// A dbt model is an ordinary .sql file in the repository whose SELECT is
4+
// templated with Jinja: dependencies are written `{{ ref('other_model') }}`
5+
// (another model in the project) or `{{ source('group', 'table') }}` (a raw
6+
// warehouse table), never as literal table names. The SQL grammar cannot read
7+
// those — `FROM {{ ref('x') }}` is a parse error to it — so the dependency
8+
// structure of an entire dbt project is invisible to the code graph without
9+
// this pass.
10+
//
11+
// The vendored tree-sitter-jinja2 grammar models `{{ ... }}` expressions
12+
// (jinja_expression / fn_call / lit_string) but has no node types at all for
13+
// `{% ... %}` statements, so this extractor covers lineage only. `{% macro %}`
14+
// definitions would need a hand-written scanner and are deliberately left out
15+
// rather than recovered approximately.
16+
//
17+
// What it emits, per qualifying file:
18+
// - one Model definition, named by the file stem (dbt's own model identity)
19+
// - one usage per ref()/source() call, scoped to that Model
20+
// pass_usages then resolves each usage against the shared definition registry
21+
// and emits a `model -USAGE-> relation` lineage edge. Model is a relation label
22+
// (cbm_label_is_relation), so lineage joins dbt models to Table/View nodes
23+
// declared in plain DDL elsewhere in the same repository, and the registry's
24+
// relation veto keeps these names out of every non-lineage consumer.
25+
//
26+
// Gate: the file must parse as SQL, contain a Jinja expression delimiter, and
27+
// contain at least one real ref()/source() call. Generic templated SQL (an
28+
// Airflow `{{ ds }}` parameter, say) therefore produces nothing at all — the
29+
// dbt builtins are the evidence that this is a dbt model, so no dbt_project.yml
30+
// lookup is needed and no non-dbt repository pays for a Model node it did not
31+
// ask for.
32+
33+
#include "cbm.h"
34+
#include "arena.h"
35+
#include "helpers.h"
36+
#include "lang_specs.h"
37+
#include "tree_sitter/api.h"
38+
#include "foundation/constants.h"
39+
#include <stdbool.h>
40+
#include <stdint.h>
41+
#include <string.h>
42+
43+
/* Local constants. */
44+
enum { DBT_FIRST_LINE = 1 };
45+
46+
/* Cheap pre-filter: does the source contain a Jinja expression opener? Files
47+
* without one cannot hold a ref()/source() call, and skipping them keeps the
48+
* second parse off every ordinary .sql file in the repository. */
49+
static bool source_has_jinja_expr(const char *s, int len) {
50+
for (int i = 0; i + 1 < len; i++) {
51+
if (s[i] == '{' && s[i + 1] == '{') {
52+
return true;
53+
}
54+
}
55+
return false;
56+
}
57+
58+
/* Strip one pair of surrounding quotes from a jinja lit_string. The grammar
59+
* hands back the token with its quotes attached. */
60+
static char *dbt_unquote(CBMArena *a, char *s) {
61+
if (!s) {
62+
return NULL;
63+
}
64+
size_t n = strlen(s);
65+
if (n >= 2 && (s[0] == '\'' || s[0] == '"') && s[n - 1] == s[0]) {
66+
char *inner = cbm_arena_strdup(a, s + 1);
67+
if (!inner) {
68+
return s;
69+
}
70+
size_t m = strlen(inner);
71+
if (m > 0) {
72+
inner[m - 1] = '\0';
73+
}
74+
return inner;
75+
}
76+
return s;
77+
}
78+
79+
/* Rightmost lit_string under `node` in DFS order. Both dbt builtins put the
80+
* referenced relation last: ref('model'), ref('package', 'model') and
81+
* source('group', 'table') all name the relation in their final string
82+
* argument. The callee itself is an identifier, never a lit_string, so it
83+
* cannot be mistaken for one. */
84+
static TSNode dbt_last_lit_string(TSNode node, TSNode best, bool *found) {
85+
if (strcmp(ts_node_type(node), "lit_string") == 0) {
86+
best = node;
87+
*found = true;
88+
}
89+
uint32_t cc = ts_node_child_count(node);
90+
for (uint32_t i = 0; i < cc; i++) {
91+
best = dbt_last_lit_string(ts_node_child(node, i), best, found);
92+
}
93+
return best;
94+
}
95+
96+
/* Collect ref()/source() targets from a jinja2 parse tree into `out`, which is
97+
* the file's usage array. Usages are scoped to enclosing_qn (the Model). */
98+
static void collect_dbt_refs(CBMExtractCtx *ctx, TSNode node, const char *enclosing_qn,
99+
CBMUsageArray *out) {
100+
if (strcmp(ts_node_type(node), "fn_call") == 0) {
101+
TSNode fn = ts_node_child_by_field_name(node, "fn_name", (uint32_t)strlen("fn_name"));
102+
if (ts_node_is_null(fn)) {
103+
fn = cbm_find_child_by_kind(node, "identifier");
104+
}
105+
if (!ts_node_is_null(fn)) {
106+
char *fname = cbm_node_text(ctx->arena, fn, ctx->source);
107+
if (fname && (strcmp(fname, "ref") == 0 || strcmp(fname, "source") == 0)) {
108+
bool found = false;
109+
TSNode empty = {0};
110+
TSNode strn = dbt_last_lit_string(node, empty, &found);
111+
if (found) {
112+
char *name =
113+
dbt_unquote(ctx->arena, cbm_node_text(ctx->arena, strn, ctx->source));
114+
if (name && name[0]) {
115+
CBMUsage usage = {0};
116+
usage.ref_name = name;
117+
usage.enclosing_func_qn = enclosing_qn;
118+
usage.site_start_byte = ts_node_start_byte(strn);
119+
usage.site_end_byte = ts_node_end_byte(strn);
120+
cbm_usages_push(out, ctx->arena, usage);
121+
}
122+
}
123+
}
124+
}
125+
}
126+
uint32_t cc = ts_node_child_count(node);
127+
for (uint32_t i = 0; i < cc; i++) {
128+
collect_dbt_refs(ctx, ts_node_child(node, i), enclosing_qn, out);
129+
}
130+
}
131+
132+
/* dbt model identity is the file stem: models/staging/stg_users.sql is the
133+
* model `stg_users`, and `{{ ref('stg_users') }}` anywhere in the project
134+
* addresses it by that bare name (dbt requires model names to be unique across
135+
* a project, so the directory is deliberately not part of the identity). */
136+
static char *dbt_name_from_path(CBMArena *a, const char *rel_path) {
137+
if (!rel_path) {
138+
return NULL;
139+
}
140+
const char *base = rel_path;
141+
for (const char *p = rel_path; *p; p++) {
142+
if (*p == '/' || *p == '\\') {
143+
base = p + 1;
144+
}
145+
}
146+
const char *dot = NULL;
147+
for (const char *p = base; *p; p++) {
148+
if (*p == '.') {
149+
dot = p;
150+
}
151+
}
152+
size_t n = dot ? (size_t)(dot - base) : strlen(base);
153+
if (n == 0) {
154+
return NULL;
155+
}
156+
char *out = cbm_arena_strdup(a, base);
157+
if (!out) {
158+
return NULL;
159+
}
160+
out[n] = '\0';
161+
return out;
162+
}
163+
164+
void cbm_extract_dbt(CBMExtractCtx *ctx) {
165+
if (!ctx || ctx->language != CBM_LANG_SQL || !ctx->source || ctx->source_len <= 0) {
166+
return;
167+
}
168+
if (!source_has_jinja_expr(ctx->source, ctx->source_len)) {
169+
return;
170+
}
171+
const TSLanguage *jl = cbm_ts_language(CBM_LANG_JINJA2);
172+
if (!jl) {
173+
return;
174+
}
175+
char *model_name = dbt_name_from_path(ctx->arena, ctx->rel_path);
176+
if (!model_name || !model_name[0]) {
177+
return;
178+
}
179+
const char *model_qn = cbm_fqn_compute(ctx->arena, ctx->project, ctx->rel_path, model_name);
180+
if (!model_qn) {
181+
return;
182+
}
183+
184+
/* A fresh parser: the primary SQL pass owns the thread-local one, and this
185+
* runs inside its walk. */
186+
TSParser *parser = ts_parser_new();
187+
if (!parser) {
188+
return;
189+
}
190+
/* Refs are staged locally so a file with Jinja but no dbt builtins commits
191+
* nothing at all — neither usages nor a Model node. */
192+
CBMUsageArray staged = {0};
193+
if (ts_parser_set_language(parser, jl)) {
194+
TSTree *tree = ts_parser_parse_string(parser, NULL, ctx->source, (uint32_t)ctx->source_len);
195+
if (tree) {
196+
collect_dbt_refs(ctx, ts_tree_root_node(tree), model_qn, &staged);
197+
ts_tree_delete(tree);
198+
}
199+
}
200+
ts_parser_delete(parser);
201+
if (staged.count == 0) {
202+
return; /* templated SQL, but not dbt — emit nothing */
203+
}
204+
205+
CBMDefinition def;
206+
memset(&def, 0, sizeof(def));
207+
def.name = model_name;
208+
def.qualified_name = model_qn;
209+
def.label = "Model";
210+
def.file_path = ctx->rel_path;
211+
def.start_line = DBT_FIRST_LINE;
212+
def.end_line = ts_node_end_point(ctx->root).row + TS_LINE_OFFSET;
213+
def.is_exported = true;
214+
cbm_defs_push(&ctx->result->defs, ctx->arena, def);
215+
216+
for (int i = 0; i < staged.count; i++) {
217+
cbm_usages_push(&ctx->result->usages, ctx->arena, staged.items[i]);
218+
}
219+
}

‎internal/cbm/helpers.c‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -166,16 +166,20 @@ bool cbm_label_is_type_like(const char *label) {
166166
strcmp(label, "Type") == 0 || strcmp(label, "Trait") == 0;
167167
}
168168

169-
// True when `label` names a data relation (SQL CREATE TABLE / CREATE VIEW).
170-
// Relations live in the registry so FROM/JOIN lineage can resolve, but they are
171-
// deliberately NOT type-like: they must never satisfy inheritance, impl-receiver,
172-
// semantic-type, or LSP-registrar lookups, and resolver fallbacks treat them as
173-
// lineage-only targets (see cbm_label_is_registry_symbol call sites).
169+
// True when `label` names a data relation: SQL CREATE TABLE / CREATE VIEW, and
170+
// a dbt Model (a Jinja-templated .sql file, which materializes as a warehouse
171+
// table or view). Relations live in the registry so FROM/JOIN and dbt ref()
172+
// lineage can resolve, and sharing one label class is what lets a dbt model's
173+
// ref() reach a Table declared in plain DDL elsewhere in the same repository.
174+
// They are deliberately NOT type-like: they must never satisfy inheritance,
175+
// impl-receiver, semantic-type, or LSP-registrar lookups, and resolver
176+
// fallbacks treat them as lineage-only targets (see cbm_label_is_registry_symbol
177+
// call sites).
174178
bool cbm_label_is_relation(const char *label) {
175179
if (!label) {
176180
return false;
177181
}
178-
return strcmp(label, "Table") == 0 || strcmp(label, "View") == 0;
182+
return strcmp(label, "Table") == 0 || strcmp(label, "View") == 0 || strcmp(label, "Model") == 0;
179183
}
180184

181185
// True when `label` belongs in the cross-file name registry (see cbm.h). Single

‎src/foundation/constants.h‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -108,8 +108,8 @@ enum { SKIP_ONE = 1, PAIR_LEN = 2 };
108108
#define CBM_SQL_TYPE_LIKE_LABELS "'Class','Struct','Interface','Enum','Type','Trait'"
109109
#define CBM_SQL_CALLABLE_LABELS "'Function','Method'"
110110
#define CBM_SQL_CALLABLE_OR_TYPE_LABELS CBM_SQL_CALLABLE_LABELS "," CBM_SQL_TYPE_LIKE_LABELS
111-
/* SQL mirror of cbm_label_is_relation() (Table/View — data-lineage nodes),
112-
* pinned by tests/test_store_nodes.c the same way as the sets above. */
113-
#define CBM_SQL_RELATION_LABELS "'Table','View'"
111+
/* SQL mirror of cbm_label_is_relation() (Table/View/Model — data-lineage
112+
* nodes), pinned by tests/test_store_nodes.c the same way as the sets above. */
113+
#define CBM_SQL_RELATION_LABELS "'Table','View','Model'"
114114

115115
#endif /* CBM_CONSTANTS_H */

0 commit comments

Comments
 (0)