Skip to content

Commit 73edb7d

Browse files
committed
ci: compile every in-repo package the link step links
The "Link In-Repo Workspace Packages" step in pnpm-lockfile-check installs with link-workspace-packages=deep, then restores the committed lockfile. Turbo builds its graph from that lockfile, so it never compiles the packages the step linked, and the step compiles a hard-coded list instead (gaxios, teeny-request). Any other in-repo package that gets linked stays unbuilt. A devDependency on @google-cloud/bigquery-storage, which handwritten/bigquery-storage satisfies, is linked to source without a build/ directory, so requiring it fails with MODULE_NOT_FOUND. Replace the hard-coded list with linked-workspace-packages.cjs, which lists the workspace projects that symlinks in node_modules resolve to, including deep links inside registry packages' dependencies, and compile them with pnpm --filter.
1 parent 62559f1 commit 73edb7d

4 files changed

Lines changed: 206 additions & 3 deletions

File tree

‎.github/actions/pnpm-lockfile-check/action.yaml‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,13 @@ runs:
3636
pnpm install --config.link-workspace-packages=deep --config.prefer-workspace-packages=true --no-frozen-lockfile --resolution-only --ignore-scripts --prefer-offline
3737
rm -f node_modules/.pnpm-workspace-state.json
3838
pnpm install --config.link-workspace-packages=deep --config.prefer-workspace-packages=true --no-frozen-lockfile --ignore-scripts --prefer-offline
39+
# Turbo only sees workspace links recorded in the committed lockfile, so
40+
# compile the in-repo packages linked above, including deep links.
41+
linked_filters=()
42+
while IFS= read -r dir; do
43+
linked_filters+=(--filter "./${dir}")
44+
done < <(pnpm ls -r --depth -1 --json | node .github/scripts/linked-workspace-packages.cjs)
3945
git checkout -- pnpm-lock.yaml
40-
pnpm --dir core/packages/gaxios run compile
41-
pnpm --dir core/packages/teeny-request run compile
46+
if (( ${#linked_filters[@]} > 0 )); then
47+
pnpm "${linked_filters[@]}" run compile
48+
fi
Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
// Copyright 2026 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// https://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
'use strict';
16+
17+
// Lists the in-repo packages that `pnpm install --config.link-workspace-packages=deep`
18+
// linked into `node_modules`, one directory (relative to the repo root) per
19+
// line. Reads the workspace projects from `pnpm ls -r --depth -1 --json` on
20+
// stdin.
21+
//
22+
// The "Link In-Repo Workspace Packages" step restores the committed lockfile
23+
// after linking, so turbo doesn't see these links and never compiles their
24+
// targets. The step compiles them itself using this list. Deep linking also
25+
// links in-repo packages into registry packages' dependencies, so this reads
26+
// the symlinks Node resolves rather than the workspace projects' manifests.
27+
28+
const fs = require('fs');
29+
const path = require('path');
30+
31+
/**
32+
* Returns the paths of the package directories directly under `nodeModules`,
33+
* including scoped packages.
34+
*
35+
* @param {string} nodeModules Absolute path of a `node_modules` directory.
36+
* @returns {string[]}
37+
*/
38+
function packageEntries(nodeModules) {
39+
let names;
40+
try {
41+
names = fs.readdirSync(nodeModules);
42+
} catch {
43+
return [];
44+
}
45+
const entries = [];
46+
for (const name of names) {
47+
const entry = path.join(nodeModules, name);
48+
if (name.startsWith('@')) {
49+
try {
50+
for (const scoped of fs.readdirSync(entry)) {
51+
entries.push(path.join(entry, scoped));
52+
}
53+
} catch {
54+
// Not a directory.
55+
}
56+
} else if (!name.startsWith('.')) {
57+
entries.push(entry);
58+
}
59+
}
60+
return entries;
61+
}
62+
63+
/**
64+
* Returns the sorted directories, relative to `root`, of workspace projects
65+
* that a symlink in a workspace project's `node_modules` or in a
66+
* `node_modules/.pnpm/<package>/node_modules` directory resolves to.
67+
*
68+
* @param {string} root Absolute path of the repository root.
69+
* @param {string[]} projectDirs Absolute paths of the workspace projects.
70+
* @returns {string[]}
71+
*/
72+
function linkedWorkspaceDirs(root, projectDirs) {
73+
const realRoot = fs.realpathSync(root);
74+
const projects = new Set(projectDirs.map(dir => fs.realpathSync(dir)));
75+
const nodeModulesDirs = projectDirs.map(dir => path.join(dir, 'node_modules'));
76+
const store = path.join(root, 'node_modules', '.pnpm');
77+
for (const name of fs.existsSync(store) ? fs.readdirSync(store) : []) {
78+
// `.pnpm/node_modules` hoists every package for pnpm's own resolution;
79+
// nothing in the repository loads from it.
80+
if (name !== 'node_modules') {
81+
nodeModulesDirs.push(path.join(store, name, 'node_modules'));
82+
}
83+
}
84+
85+
const dirs = new Set();
86+
for (const nodeModules of nodeModulesDirs) {
87+
for (const entry of packageEntries(nodeModules)) {
88+
let target;
89+
try {
90+
if (!fs.lstatSync(entry).isSymbolicLink()) {
91+
continue;
92+
}
93+
target = fs.realpathSync(entry);
94+
} catch {
95+
continue;
96+
}
97+
if (projects.has(target) && target !== realRoot) {
98+
dirs.add(path.relative(realRoot, target).split(path.sep).join('/'));
99+
}
100+
}
101+
}
102+
return [...dirs].sort();
103+
}
104+
105+
module.exports = {linkedWorkspaceDirs};
106+
107+
if (require.main === module) {
108+
let input = '';
109+
process.stdin.on('data', chunk => (input += chunk));
110+
process.stdin.on('end', () => {
111+
const projectDirs = JSON.parse(input).map(project => project.path);
112+
for (const dir of linkedWorkspaceDirs(process.cwd(), projectDirs)) {
113+
console.log(dir);
114+
}
115+
});
116+
}

‎.github/scripts/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
"private": true,
44
"description": "tests for script",
55
"scripts": {
6-
"test": "mocha tests/close-invalid-link.test.cjs && mocha tests/close-or-remove-response-label.test.cjs && mocha tests/run-test.test.cjs",
6+
"test": "mocha tests/close-invalid-link.test.cjs && mocha tests/close-or-remove-response-label.test.cjs && mocha tests/run-test.test.cjs && mocha tests/linked-workspace-packages.test.cjs",
77
"samples-test": "echo 'no samples test!'",
88
"system-test": "echo 'no system test!'"
99
},
Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
// Copyright 2026 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// https://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
'use strict';
16+
17+
const assert = require('assert');
18+
const fs = require('fs');
19+
const os = require('os');
20+
const path = require('path');
21+
const {afterEach, beforeEach, describe, it} = require('mocha');
22+
const {linkedWorkspaceDirs} = require('../linked-workspace-packages.cjs');
23+
24+
describe('linkedWorkspaceDirs', () => {
25+
let root;
26+
27+
function mkdir(...parts) {
28+
const dir = path.join(root, ...parts);
29+
fs.mkdirSync(dir, {recursive: true});
30+
return dir;
31+
}
32+
33+
function link(target, ...parts) {
34+
const entry = path.join(root, ...parts);
35+
fs.mkdirSync(path.dirname(entry), {recursive: true});
36+
fs.symlinkSync(path.join(root, target), entry, 'dir');
37+
}
38+
39+
beforeEach(() => {
40+
root = fs.mkdtempSync(path.join(os.tmpdir(), 'linked-workspace-'));
41+
});
42+
43+
afterEach(() => {
44+
fs.rmSync(root, {recursive: true, force: true});
45+
});
46+
47+
it('lists in-repo packages linked into workspace projects and registry packages', () => {
48+
const bigquery = mkdir('handwritten/bigquery');
49+
const storage = mkdir('handwritten/bigquery-storage');
50+
const gaxios = mkdir('core/packages/gaxios');
51+
// A workspace project's devDependency linked to the in-repo package.
52+
link('handwritten/bigquery-storage', 'handwritten/bigquery/node_modules/@google-cloud/bigquery-storage');
53+
// A registry package's dependency deep-linked to the in-repo package.
54+
link('core/packages/gaxios', 'node_modules/.pnpm/google-auth-library@10.5.0/node_modules/gaxios');
55+
link('core/packages/gaxios', 'node_modules/.pnpm/gcp-metadata@8.1.2/node_modules/gaxios');
56+
57+
assert.deepStrictEqual(linkedWorkspaceDirs(root, [bigquery, storage, gaxios]), [
58+
'core/packages/gaxios',
59+
'handwritten/bigquery-storage',
60+
]);
61+
});
62+
63+
it('ignores registry packages, pnpm hoisting links, and links outside the workspace', () => {
64+
const bigquery = mkdir('handwritten/bigquery');
65+
const storage = mkdir('handwritten/bigquery-storage');
66+
mkdir('node_modules/.pnpm/teeny-request@11.0.1/node_modules/teeny-request');
67+
mkdir('not-a-project');
68+
// A dependency resolved from the registry.
69+
link(
70+
'node_modules/.pnpm/teeny-request@11.0.1/node_modules/teeny-request',
71+
'handwritten/bigquery/node_modules/teeny-request',
72+
);
73+
// pnpm hoists every package into .pnpm/node_modules for its own resolution.
74+
link('handwritten/bigquery-storage', 'node_modules/.pnpm/node_modules/@google-cloud/bigquery-storage');
75+
// A link to a directory that isn't a workspace project.
76+
link('not-a-project', 'handwritten/bigquery/node_modules/not-a-project');
77+
78+
assert.deepStrictEqual(linkedWorkspaceDirs(root, [bigquery, storage]), []);
79+
});
80+
});

0 commit comments

Comments
 (0)