Track workflow renames (#1596)

* deprecate the id field being written to workflow.yaml

* track workflow uuid in checkout.yaml

* track workflow UUIDs when loading from fs

* add helper to match workflows on merge

* hookup workflow tracking to deploy

* impove diff for users upon a workflow rename

* clarifying comments on test files

* couple of tests

* add more basic sync tests

* new tests

* restructure sync integration tests

* add step rename tests

* cli: use env var to skip no confirm

* refactor tests for cleaner commands#

* enable workflow tracking on rename

* new test

Author: josephjclark

.changeset/checkout-workflow-uuids.md

@@ -0,0 +1,5 @@
+---
+'@openfn/project': patch
+---
+
+Track workflow uuids in the checkout file, and load them onto workflows

.changeset/deploy-checkout-local-ids.md

@@ -0,0 +1,6 @@
+---
+'@openfn/cli': patch
+'@openfn/project': patch
+---
+
+Fix deploying an edit to a renamed workflow deleting it from Lightning

.changeset/diff-renamed-workflows.md

@@ -0,0 +1,6 @@
+---
+'@openfn/cli': patch
+'@openfn/project': patch
+---
+
+Show a renamed workflow as changed, not added and removed, in project diffs

.changeset/merge-match-workflows.md

@@ -0,0 +1,5 @@
+---
+'@openfn/project': patch
+---
+
+Match workflows by uuid, then id, then name when merging projects

.changeset/shaggy-bears-clean.md

@@ -0,0 +1,5 @@
+---
+'@openfn/cli': patch
+---
+
+Add OPENFN_NO_CONFIRM env var

.changeset/workflow-yaml-no-id.md

@@ -0,0 +1,5 @@
+---
+'@openfn/project': patch
+---
+
+Workflow ids in workflow.yaml files are now ignored on load: the file name is used instead

integration-tests/cli/src/run.ts

@@ -3,9 +3,12 @@ import * as path from 'node:path';
 
 const isProd = process.env.NODE_ENV === 'production';
 
-const getOptions = () => ({
+const getOptions = (env = {}) => ({
   env: {
+    // there's nobody to answer a prompt (a test can override this)
+    OPENFN_NO_CONFIRM: 'true',
     ...process.env,
+    ...env,
     OPENFN_REPO_DIR: path.resolve('repo'),
     // workspaces live inside the kit git repo: don't pick up its branch
     OPENFN_BRANCH: 'false',
@@ -23,10 +26,12 @@ const mapOpenFnPath = (cmd) => {
 };
 
 const run = async (
-  cmd: string
+  cmd: string,
+  // extra environment variables for this command only
+  env?: Record<string, string>
 ): Promise<{ stdout: string; stderr?: string; err?: any }> => {
   return new Promise((resolve) => {
-    exec(mapOpenFnPath(cmd), getOptions(), (err, stdout, stderr) => {
+    exec(mapOpenFnPath(cmd), getOptions(env), (err, stdout, stderr) => {
       resolve({ err, stdout, stderr });
     });
   });

integration-tests/cli/test/deploy.test.ts

@@ -1,3 +1,5 @@
+// Tests legacy `openfn deploy` and `openfn pull` commands against a mock Lightning
+
 import test from 'ava';
 import path from 'node:path';
 import fs, { rm } from 'node:fs/promises';
@@ -157,7 +159,6 @@ test.serial('deploy a local project', async (t) => {
     `openfn deploy \
       --project-path ${tmpDir}/project.yaml \
       --state-path ${tmpDir}/.state.json \
-      --no-confirm \
       --log-json \
       -l debug`
   );
@@ -188,7 +189,6 @@ test.serial('Update a project', async (t) => {
   const deployCmd = `openfn deploy \
       --project-path ${projectPath} \
       --state-path ${statePath} \
-      --no-confirm \
       --log-json -l debug`;
   const first = await run(deployCmd);
   t.falsy(first.stderr);
@@ -230,7 +230,6 @@ test.serial('pull a project', async (t) => {
   const deployCmd = `openfn deploy \
     --project-path ${projectPath} \
     --state-path ${statePath} \
-    --no-confirm \
     --log-json -l debug`;
 
   await run(deployCmd);
@@ -323,7 +322,7 @@ test.serial('redirect to v2 protocol if openfn.yaml is present', async (t) => {
 
   // v1 deploy -> should redirect to v2
   const { stdout, stderr } = await run(
-    `openfn deploy --workspace ${tmpDir} --no-confirm --log-json -l debug`
+    `openfn deploy --workspace ${tmpDir} --log-json -l debug`
   );
   t.falsy(stderr);
   assertLog(t, extractLogs(stdout), /Detected openfn.yaml file/i);
@@ -364,7 +363,6 @@ workflows:
     `openfn deploy \
       --project-path ${tmpDir}/project.yaml \
       --state-path ${tmpDir}/.state.json \
-      --no-confirm \
       --log-json \
       -l debug`
   );
@@ -397,7 +395,6 @@ test.serial('deploy then pull, changes one workflow, deploy', async (t) => {
   const deployCmd = `openfn deploy \
       --project-path ${projectPath} \
       --state-path ${statePath} \
-      --no-confirm \
       --log debug \
       --log-json`;
   await run(deployCmd);
@@ -451,7 +448,6 @@ test.serial('deploy a v2 project.yaml', async (t) => {
     `openfn deploy \
       --project-path ${tmpDir}/project.yaml \
       --state-path ${tmpDir}/.state.json \
-      --no-confirm \
       --log-json \
       -l debug`
   );
@@ -481,7 +477,6 @@ test.serial('deploy a new v2 project.yaml with credentials', async (t) => {
   const { stdout, stderr } = await run(
     `openfn deploy \
       --project-path ${tmpDir}/project.yaml \
-      --no-confirm \
       --log-json \
       -l debug`
   );
@@ -526,7 +521,6 @@ collections:
   const deployCmd = `openfn deploy \
     --project-path ${projectPath} \
     --state-path ${statePath} \
-    --no-confirm \
     --log-json -l debug`;
 
   // first deploy - no collections yet
@@ -573,7 +567,6 @@ ${collectionsWorkflow}
   const deployCmd = `openfn deploy \
     --project-path ${projectPath} \
     --state-path ${statePath} \
-    --no-confirm \
     --log-json -l debug`;
 
   // first deploy - creates the collection

integration-tests/cli/test/deploy.v2.test.ts

@@ -1,3 +1,5 @@
+// Tests v2 `openfn project pull` and `openfn project deploy` loop against a mock Lightning
+
 import test from 'ava';
 import path from 'node:path';
 import fs from 'node:fs/promises';
@@ -71,7 +73,7 @@ test.serial('pull, change and re-deploy', async (t) => {
 
   // deploy
   const { stdout, stderr } = await run(
-    `openfn project deploy --no-confirm --log-json -l debug`
+    `openfn project deploy --log-json -l debug`
   );
   t.falsy(stderr);
 
@@ -99,9 +101,7 @@ test.serial('pull, change and re-deploy twice', async (t) => {
 
   // first deploy
   await fs.writeFile(exprPath, 'fn(s => ({ ...s, v: 1 }))');
-  const first = await run(
-    `openfn project deploy --no-confirm --log-json -l debug`
-  );
+  const first = await run(`openfn project deploy --log-json -l debug`);
   t.falsy(first.stderr);
   assertLog(t, extractLogs(first.stdout), /Updated project/);
 
@@ -112,7 +112,7 @@ test.serial('pull, change and re-deploy twice', async (t) => {
   // second deploy after another update
   await fs.writeFile(exprPath, 'fn(s => ({ ...s, v: 2 }))');
   const { stdout, stderr } = await run(
-    `openfn project deploy --no-confirm --log-json -l debug`
+    `openfn project deploy --log-json -l debug`
   );
   t.falsy(stderr);
   const logs = extractLogs(stdout);
@@ -140,9 +140,7 @@ test.serial('deploy and pull to check version history', async (t) => {
   await fs.writeFile(exprPath, 'fn(s => ({ ...s, v: 1 }))');
 
   // deploy then pull
-  const { stderr } = await run(
-    `openfn project deploy --no-confirm --log-json -l debug`
-  );
+  const { stderr } = await run(`openfn project deploy --log-json -l debug`);
   t.falsy(stderr);
   await run(`openfn project pull ${projectId} --log-json -l debug`);
 
@@ -174,7 +172,7 @@ test.serial('deploy then pull, change one workflow, deploy', async (t) => {
 
   // deploy
   const { stdout, stderr } = await run(
-    `openfn project deploy --no-confirm --log-json -l debug`
+    `openfn project deploy --log-json -l debug`
   );
   t.falsy(stderr);
   const logs = extractLogs(stdout);
@@ -215,7 +213,7 @@ test.serial('Remove a whole workflow', async (t) => {
 
   // deploy
   const { stdout, stderr } = await run(
-    `openfn project deploy --no-confirm --log-json -l debug`
+    `openfn project deploy --log-json -l debug`
   );
   t.falsy(stderr);
 
@@ -262,7 +260,7 @@ steps:
   await rimraf(path.join(tmpDir, 'workflows/my-workflow/my-job.js'));
 
   const { stdout, stderr } = await run(
-    `openfn project deploy --no-confirm --log-json -l debug`
+    `openfn project deploy --log-json -l debug`
   );
   t.falsy(stderr);
 
@@ -317,7 +315,7 @@ steps:
   );
 
   const { stdout, stderr } = await run(
-    `openfn project deploy --no-confirm --log-json -l debug`
+    `openfn project deploy --log-json -l debug`
   );
   t.falsy(stderr);
 
@@ -372,9 +370,7 @@ test.serial.skip(
     await fs.writeFile(exprPath, "post('http://success.org')");
 
     // deploy
-    const { stdout, stderr } = await run(
-      `openfn project deploy --no-confirm  -l debug`
-    );
+    const { stdout, stderr } = await run(`openfn project deploy  -l debug`);
     console.log(stdout);
     t.falsy(stderr);
     const logs = extractLogs(stdout);
@@ -400,9 +396,7 @@ test.serial('warn when local and remote workflows have diverged', async (t) => {
   // base
   await run(`openfn project pull ${projectId} --log-json -l debug`);
   await fs.writeFile(exprPath, 'fn(s => ({ ...s, v: 1 }))');
-  const firstDeploy = await run(
-    `openfn project deploy --no-confirm --log-json -l debug`
-  );
+  const firstDeploy = await run(`openfn project deploy --log-json -l debug`);
   assertLog(t, extractLogs(firstDeploy.stdout), /Updated project/);
 
   // remote changed from base
@@ -424,7 +418,7 @@ test.serial('warn when local and remote workflows have diverged', async (t) => {
 
   // deploy with divergence
   const { stdout, err } = await run(
-    `openfn project deploy --no-confirm --log-json -l debug`
+    `openfn project deploy --log-json -l debug`
   );
   t.truthy(err);
   const logs = extractLogs(stdout);
@@ -448,7 +442,7 @@ test.serial('deploy a pulled v2 state file as a new project', async (t) => {
 
   // deploy that exact file back as a duplicate
   const { stdout, stderr } = await run(
-    `openfn project deploy ${pulledPath} --new --name my-duplicate --no-confirm --log-json -l debug`
+    `openfn project deploy ${pulledPath} --new --name my-duplicate --log-json -l debug`
   );
   t.falsy(stderr);
 
@@ -506,7 +500,7 @@ test.serial(
 
     // deploy main's pulled file, but target staging instead of main
     const { stdout, stderr } = await run(
-      `openfn project deploy ${mainPath} staging --no-confirm --log-json -l debug`
+      `openfn project deploy ${mainPath} staging --log-json -l debug`
     );
     t.falsy(stderr);
     assertLog(t, extractLogs(stdout), /Updated project/);
@@ -553,7 +547,7 @@ test.serial(
     await fs.writeFile(resourcesPath, jsonToYaml(before));
 
     const { stdout, stderr } = await run(
-      `openfn project deploy --no-confirm --log-json -l debug`
+      `openfn project deploy --log-json -l debug`
     );
     t.falsy(stderr);
 
@@ -596,7 +590,7 @@ workflows:
   await fs.writeFile(exportedPath, specYaml);
 
   const { stdout, stderr } = await run(
-    `openfn project deploy ${exportedPath} --name my-duplicate --no-confirm --log-json -l debug`
+    `openfn project deploy ${exportedPath} --name my-duplicate --log-json -l debug`
   );
   t.falsy(stderr);
 
@@ -659,7 +653,7 @@ test.serial(
     await fs.writeFile(resourcesPath, jsonToYaml(before));
 
     const { stdout, stderr } = await run(
-      `openfn project deploy --no-confirm --log-json -l debug`
+      `openfn project deploy --log-json -l debug`
     );
     t.falsy(stderr);
 

integration-tests/cli/test/project-v1.test.ts

@@ -1,3 +1,6 @@
+// Tests offline `openfn checkout`, `merge` and `projects` on static legacy v1 project yaml files
+// (no Lightning); see project-v2 for the same commands on v2 files
+
 import test from 'ava';
 import { rm, mkdir, writeFile, readFile } from 'node:fs/promises';
 import path from 'node:path';