diff --git a/.nextchanges/cli/aitools-telemetry-funnel.md b/.nextchanges/cli/aitools-telemetry-funnel.md new file mode 100644 index 00000000000..2b13b330f11 --- /dev/null +++ b/.nextchanges/cli/aitools-telemetry-funnel.md @@ -0,0 +1 @@ +* `databricks aitools` commands now send usage telemetry, and the CLI records when it recommends installing Databricks AI tools. When the configured workspace is slow or unreachable, `aitools install`, `update` and `uninstall` can take up to about 6 seconds longer, and `list`, `version` and runs that stop early up to about 3 seconds. Set `DATABRICKS_CLI_DISABLE_TELEMETRY=1` to opt out. ([#7467](https://github.com/databricks/cli/pull/7467), [#7469](https://github.com/databricks/cli/pull/7469), [#7470](https://github.com/databricks/cli/pull/7470)) diff --git a/acceptance/cmd/aitools/list/explicit-ref/output.txt b/acceptance/cmd/aitools/list/explicit-ref/output.txt index 5f057827c14..5d378356dc1 100644 --- a/acceptance/cmd/aitools/list/explicit-ref/output.txt +++ b/acceptance/cmd/aitools/list/explicit-ref/output.txt @@ -24,7 +24,7 @@ === Only the skills manifest reaches the mock server: no latest-release lookup ->>> print_requests.py --get +>>> print_requests.py --get ^//telemetry-ext ^//.well-known { "method": "GET", "path": "/test-ref/manifest.json" diff --git a/acceptance/cmd/aitools/list/explicit-ref/script b/acceptance/cmd/aitools/list/explicit-ref/script index 699d0c0c500..21f9a3e9a30 100644 --- a/acceptance/cmd/aitools/list/explicit-ref/script +++ b/acceptance/cmd/aitools/list/explicit-ref/script @@ -4,4 +4,4 @@ sethome home trace $CLI aitools list --output json | list_fields title "Only the skills manifest reaches the mock server: no latest-release lookup" -trace print_requests.py --get +trace print_requests.py --get '^//telemetry-ext' '^//.well-known' diff --git a/acceptance/cmd/aitools/list/latest-lookup-fails/output.txt b/acceptance/cmd/aitools/list/latest-lookup-fails/output.txt index ff293d47269..8856e2db261 100644 --- a/acceptance/cmd/aitools/list/latest-lookup-fails/output.txt +++ b/acceptance/cmd/aitools/list/latest-lookup-fails/output.txt @@ -4,7 +4,7 @@ Error: resolve latest skills release: HTTP 403 fetching latest skills release === The lookup fails, so the manifest is never fetched ->>> print_requests.py --get +>>> print_requests.py --get ^//telemetry-ext ^//.well-known { "method": "GET", "path": "/repos/databricks/databricks-agent-skills/releases/latest" diff --git a/acceptance/cmd/aitools/list/latest-lookup-fails/script b/acceptance/cmd/aitools/list/latest-lookup-fails/script index b37d6deb411..f419091ed28 100644 --- a/acceptance/cmd/aitools/list/latest-lookup-fails/script +++ b/acceptance/cmd/aitools/list/latest-lookup-fails/script @@ -7,4 +7,4 @@ use_compat_manifest trace musterr $CLI aitools list --output json title "The lookup fails, so the manifest is never fetched" -trace print_requests.py --get +trace print_requests.py --get '^//telemetry-ext' '^//.well-known' diff --git a/acceptance/cmd/aitools/list/latest/output.txt b/acceptance/cmd/aitools/list/latest/output.txt index 3e8dbfe030a..b6700496e81 100644 --- a/acceptance/cmd/aitools/list/latest/output.txt +++ b/acceptance/cmd/aitools/list/latest/output.txt @@ -24,7 +24,7 @@ === The latest-release lookup picks the manifest version ->>> print_requests.py --get +>>> print_requests.py --get ^//telemetry-ext ^//.well-known { "method": "GET", "path": "/repos/databricks/databricks-agent-skills/releases/latest" diff --git a/acceptance/cmd/aitools/list/latest/script b/acceptance/cmd/aitools/list/latest/script index aa052eb6643..67c45029d9c 100644 --- a/acceptance/cmd/aitools/list/latest/script +++ b/acceptance/cmd/aitools/list/latest/script @@ -5,4 +5,4 @@ use_compat_manifest trace $CLI aitools list --output json | list_fields title "The latest-release lookup picks the manifest version" -trace print_requests.py --get +trace print_requests.py --get '^//telemetry-ext' '^//.well-known' diff --git a/acceptance/cmd/aitools/list/pinned/output.txt b/acceptance/cmd/aitools/list/pinned/output.txt index 329eab653cb..37693710fa1 100644 --- a/acceptance/cmd/aitools/list/pinned/output.txt +++ b/acceptance/cmd/aitools/list/pinned/output.txt @@ -24,7 +24,7 @@ === A pinned version skips the latest-release lookup ->>> print_requests.py --get +>>> print_requests.py --get ^//telemetry-ext ^//.well-known { "method": "GET", "path": "/v0.2.32/manifest.json" diff --git a/acceptance/cmd/aitools/list/pinned/script b/acceptance/cmd/aitools/list/pinned/script index 0f5c2fe1bb7..d75b5d2f057 100644 --- a/acceptance/cmd/aitools/list/pinned/script +++ b/acceptance/cmd/aitools/list/pinned/script @@ -5,4 +5,4 @@ use_compat_manifest trace $CLI aitools list --output json | list_fields title "A pinned version skips the latest-release lookup" -trace print_requests.py --get +trace print_requests.py --get '^//telemetry-ext' '^//.well-known' diff --git a/acceptance/cmd/aitools/telemetry/update-uninstall/out.test.toml b/acceptance/cmd/aitools/telemetry/update-uninstall/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/update-uninstall/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/cmd/aitools/telemetry/update-uninstall/output.txt b/acceptance/cmd/aitools/telemetry/update-uninstall/output.txt new file mode 100644 index 00000000000..a9dd189e489 --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/update-uninstall/output.txt @@ -0,0 +1,18 @@ + +=== update: STARTED, FINISHED with SUCCESS + +>>> [CLI] aitools update --scope=global +Already up to date. + +>>> print_telemetry +{"command":"aitools_update","exit_code":0,"aitools_command_event":{"command":"UPDATE","stage":"STARTED","interactive":false}} +{"command":"aitools_update","exit_code":0,"aitools_command_event":{"command":"UPDATE","stage":"FINISHED","outcome":"SUCCESS","interactive":false,"scope":"GLOBAL"}} + +=== uninstall: STARTED, FINISHED with SUCCESS + +>>> [CLI] aitools uninstall --scope=global +Uninstalled 1 skill. + +>>> print_telemetry +{"command":"aitools_uninstall","exit_code":0,"aitools_command_event":{"command":"UNINSTALL","stage":"STARTED","interactive":false}} +{"command":"aitools_uninstall","exit_code":0,"aitools_command_event":{"command":"UNINSTALL","stage":"FINISHED","outcome":"SUCCESS","interactive":false,"scope":"GLOBAL"}} diff --git a/acceptance/cmd/aitools/telemetry/update-uninstall/script b/acceptance/cmd/aitools/telemetry/update-uninstall/script new file mode 100644 index 00000000000..83cc16a0f99 --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/update-uninstall/script @@ -0,0 +1,20 @@ +# Isolate HOME so parallel aitools tests don't race on a shared ~/.databricks. +sethome home +mkdir -p "${USERPROFILE:-$HOME}/.cursor" + +# Install one skill to update and uninstall. Its funnel is pinned by ../install; +# print_telemetry here only clears its upload. +$CLI aitools install --skills-only --scope=global --agents=cursor &> LOG.install +print_telemetry &> LOG.install-telemetry + +title "update: STARTED, FINISHED with SUCCESS" +trace $CLI aitools update --scope=global +trace print_telemetry + +# list, version, and update and uninstall with nothing installed upload their +# events only if auth has already resolved when they finish, so whether they +# upload here depends on timing; their funnels are covered by unit tests. + +title "uninstall: STARTED, FINISHED with SUCCESS" +trace $CLI aitools uninstall --scope=global +trace print_telemetry diff --git a/acceptance/cmd/aitools/telemetry/workspace-unreachable/out.test.toml b/acceptance/cmd/aitools/telemetry/workspace-unreachable/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/workspace-unreachable/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/cmd/aitools/telemetry/workspace-unreachable/output.txt b/acceptance/cmd/aitools/telemetry/workspace-unreachable/output.txt new file mode 100644 index 00000000000..a967a32a8c5 --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/workspace-unreachable/output.txt @@ -0,0 +1,18 @@ + +=== install: the background auth resolution prints nothing + +>>> [CLI] aitools install --skills-only --scope=global --agents=cursor +Installing Databricks skills for Cursor... +Using skills version test-ref +Fetching skills manifest... +Installed 1 skill. + +=== update: the background auth resolution prints nothing + +>>> [CLI] aitools update --scope=global +Already up to date. + +=== uninstall: the background auth resolution prints nothing + +>>> [CLI] aitools uninstall --scope=global +Uninstalled 1 skill. diff --git a/acceptance/cmd/aitools/telemetry/workspace-unreachable/script b/acceptance/cmd/aitools/telemetry/workspace-unreachable/script new file mode 100644 index 00000000000..d874aaa9cd0 --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/workspace-unreachable/script @@ -0,0 +1,22 @@ +# Isolate HOME so the real machine's agents and skills are not seen. +sethome home + +# The workspace host does not resolve, so the background telemetry auth +# resolution fails to fetch host metadata. The skills manifest still comes from +# the mock server (DATABRICKS_SKILLS_BASE_URL is set from the original host). +# A hostmetadata warning only reaches the output if the failed lookup ends +# before the command stops waiting for the resolution. .test resolution fails +# fast (NXDOMAIN), and install, update and uninstall wait for it once they get +# to their work, so they catch a regression; list and version never wait, and +# update and uninstall with nothing installed do not either. +export DATABRICKS_HOST=https://unreachable.test +mkdir -p "${USERPROFILE:-$HOME}/.cursor" + +title "install: the background auth resolution prints nothing" +trace $CLI aitools install --skills-only --scope=global --agents=cursor + +title "update: the background auth resolution prints nothing" +trace $CLI aitools update --scope=global + +title "uninstall: the background auth resolution prints nothing" +trace $CLI aitools uninstall --scope=global diff --git a/acceptance/cmd/aitools/telemetry/workspace-unreachable/test.toml b/acceptance/cmd/aitools/telemetry/workspace-unreachable/test.toml new file mode 100644 index 00000000000..ae8f52e3b91 --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/workspace-unreachable/test.toml @@ -0,0 +1,2 @@ +# Nothing reaches the mock server's telemetry endpoint, so there is nothing to record. +RecordRequests = false diff --git a/cmd/aitools/categorize.go b/cmd/aitools/categorize.go index 7f6f8f42dba..46a3785ee0e 100644 --- a/cmd/aitools/categorize.go +++ b/cmd/aitools/categorize.go @@ -60,6 +60,13 @@ func classifyInstallError(err error) protos.AitoolsErrorCategory { if tagged, ok := errors.AsType[*categorizedError](err); ok { return tagged.category } + if errors.Is(err, installer.ErrUnknownAgent) { + return protos.AitoolsErrorCategoryUnknownAgent + } + // Skills on disk without the install state that update and uninstall need. + if errors.Is(err, installer.ErrLegacyInstall) { + return protos.AitoolsErrorCategoryStateInvalid + } return stdlibErrorCategory(err) } diff --git a/cmd/aitools/categorize_test.go b/cmd/aitools/categorize_test.go index 4491fa3a2d2..9a6a2956f48 100644 --- a/cmd/aitools/categorize_test.go +++ b/cmd/aitools/categorize_test.go @@ -52,6 +52,16 @@ func TestClassifyInstallError(t *testing.T) { err: fmt.Errorf("resolve failed: %w", &installer.SkillError{Skill: "databricks", Reason: installer.ReasonSkillNotFound, Detail: "not found"}), want: protos.AitoolsErrorCategorySkillNotFound, }, + { + name: "legacy install without state", + err: fmt.Errorf("%w; run install first", installer.ErrLegacyInstall), + want: protos.AitoolsErrorCategoryStateInvalid, + }, + { + name: "unknown agent in the install state", + err: fmt.Errorf("skipped: %w", installer.ErrUnknownAgent), + want: protos.AitoolsErrorCategoryUnknownAgent, + }, { name: "version incompatible", err: &installer.SkillError{Skill: "databricks", Reason: installer.ReasonVersionIncompatible, Detail: "requires CLI version 0.5 (running 0.4)"}, diff --git a/cmd/aitools/funnel.go b/cmd/aitools/funnel.go index eeb579ac8e2..ccdf85696b9 100644 --- a/cmd/aitools/funnel.go +++ b/cmd/aitools/funnel.go @@ -6,9 +6,11 @@ import ( "github.com/charmbracelet/huh" "github.com/databricks/cli/libs/aitools/agents" + "github.com/databricks/cli/libs/aitools/installer" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/telemetry" "github.com/databricks/cli/libs/telemetry/protos" + "github.com/spf13/cobra" ) // commandFunnel emits the AitoolsCommandEvent funnel of one aitools subcommand @@ -38,7 +40,15 @@ func startFunnel(ctx context.Context, command protos.AitoolsCommand, installMode } // setScope records the resolved scope (installer.ScopeGlobal/ScopeProject). -func (f *commandFunnel) setScope(scope string) { f.scope = scopeType(scope) } +// An empty scope (none or both selected) leaves the field unset, so the event +// omits it rather than sending the Unspecified sentinel. +func (f *commandFunnel) setScope(scope string) { + if scope == "" { + f.scope = "" + return + } + f.scope = scopeType(scope) +} // setAgents records the plan's agents, reported on the FINISHED event. func (f *commandFunnel) setAgents(plan []agentPlanItem) { f.agents = agentsField(plan) } @@ -56,13 +66,16 @@ func (f *commandFunnel) step(ctx context.Context, s protos.AitoolsCommandStep) { } // finish logs the FINISHED event for a run that returned err: SUCCESS for nil, -// CANCELLED when the user aborted a prompt or picked no agents, FAILURE -// otherwise. No-op once the run has finished, so it is safe to defer after an -// explicit finishWith. +// NOTHING_TO_DO when update or uninstall found nothing installed (the command +// still exits non-zero), CANCELLED when the user aborted a prompt or picked no +// agents, FAILURE otherwise. No-op once the run has finished, so it is safe to +// defer after an explicit finishWith. func (f *commandFunnel) finish(ctx context.Context, err error) { switch { case err == nil: f.finishWith(ctx, protos.AitoolsCommandOutcomeSuccess, protos.AitoolsErrorCategoryUnspecified) + case errors.Is(err, installer.ErrNothingInstalled): + f.finishWith(ctx, protos.AitoolsCommandOutcomeNothingToDo, protos.AitoolsErrorCategoryUnspecified) case errors.Is(err, huh.ErrUserAborted): f.finishWith(ctx, protos.AitoolsCommandOutcomeCancelled, protos.AitoolsErrorCategoryUserAborted) case errors.Is(err, errNoAgentsSelected): @@ -102,6 +115,27 @@ func (f *commandFunnel) log(ctx context.Context, e protos.AitoolsCommandEvent) { telemetry.Log(ctx, protos.DatabricksCliLog{AitoolsCommandEvent: &e}) } +// withCommandFunnel wraps a subcommand's RunE so every run logs STARTED and +// FINISHED (outcome from the returned error). +func withCommandFunnel(command protos.AitoolsCommand, runE func(cmd *cobra.Command, args []string) error) func(cmd *cobra.Command, args []string) error { + return func(cmd *cobra.Command, args []string) (err error) { + f := startFunnel(cmd.Context(), command, "") + defer func() { f.finish(cmd.Context(), err) }() + cmd.SetContext(context.WithValue(cmd.Context(), funnelKey{}, f)) + return runE(cmd, args) + } +} + +type funnelKey struct{} + +// funnelFrom returns the funnel withCommandFunnel put on ctx. Only the RunE of +// a command wrapped by instrumentCommand calls it, for outcomes its returned +// error does not tell (see finishWith) and for the scope and agents it +// resolves. +func funnelFrom(ctx context.Context) *commandFunnel { + return ctx.Value(funnelKey{}).(*commandFunnel) +} + // installModeType maps the install delivery flags to the telemetry enum. func installModeType(skillsOnly bool, path string) protos.AitoolsInstallMode { switch { diff --git a/cmd/aitools/funnel_test.go b/cmd/aitools/funnel_test.go index e29156d08a1..ba9302a4354 100644 --- a/cmd/aitools/funnel_test.go +++ b/cmd/aitools/funnel_test.go @@ -5,10 +5,14 @@ import ( "errors" "fmt" "io" + "io/fs" + "maps" "net/http" "net/http/httptest" "path/filepath" + "strings" "testing" + "time" "github.com/charmbracelet/huh" "github.com/databricks/cli/cmd/root" @@ -461,6 +465,89 @@ func TestInstallFunnelPluginSuccess(t *testing.T) { assert.Equal(t, []protos.AitoolsAgentType{protos.AitoolsAgentTypeCodex}, events[len(events)-1].Agents) } +func TestInstrumentedCommandsLogStartAndFinish(t *testing.T) { + isolateCommandEnv(t) + + ctx := telemetry.WithNewLogger(cmdio.MockDiscard(t.Context())) + cmd := NewVersionCmd() + cmd.SetContext(ctx) + cmd.SetArgs([]string{}) + require.NoError(t, cmd.Execute()) + + events := funnelEvents(ctx) + assert.Equal(t, []string{"STARTED", "FINISHED:SUCCESS:"}, funnelShape(events)) + for _, e := range events { + assert.Equal(t, protos.AitoolsCommandVersion, e.Command) + assert.Empty(t, e.InstallMode) + } +} + +// Every instrumented command must start the best-effort auth resolution, or +// its events never upload (telemetry.Upload skips a command without auth +// config). Each run ends early; the resolution starts before RunE anyway. +func TestInstrumentedCommandsResolveAuth(t *testing.T) { + cases := []struct { + newCmd func() *cobra.Command + args []string + }{ + {newCmd: NewInstallCmd, args: []string{"--skills-only", "--path", "out"}}, + {newCmd: NewUpdateCmd, args: []string{"--scope=bogus"}}, + {newCmd: NewListCmd, args: []string{"--scope=bogus"}}, + {newCmd: NewVersionCmd}, + {newCmd: NewUninstallCmd, args: []string{"--scope=bogus"}}, + } + for _, tc := range cases { + cmd := tc.newCmd() + t.Run(cmd.Name(), func(t *testing.T) { + isolateCommandEnv(t) + resolved := make(chan struct{}, 1) + setTelemetryAuth(t, time.Second, func(*cobra.Command, []string) error { + resolved <- struct{}{} + return nil + }) + + _, _, _ = runWithTelemetry(t, cmd, tc.args...) + select { + case <-resolved: + case <-time.After(5 * time.Second): + t.Fatal("the telemetry auth resolution never ran") + } + }) + } +} + +// list reports the scope a flag selects, and no scope (not the Unspecified +// sentinel) when it shows both. +func TestListFunnelFailure(t *testing.T) { + cases := []struct { + name string + args []string + wantScope protos.AitoolsInstallScope + }{ + {name: "global", args: []string{"--scope=global"}, wantScope: protos.AitoolsInstallScopeGlobal}, + {name: "no scope flag"}, + {name: "both scopes", args: []string{"--scope=both"}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + isolateCommandEnv(t) + t.Setenv("DATABRICKS_SKILLS_REF", "v0.2.6") + t.Setenv("DATABRICKS_SKILLS_BASE_URL", "http://127.0.0.1:1") + + ctx := telemetry.WithNewLogger(cmdio.MockDiscard(t.Context())) + cmd := NewListCmd() + cmd.SetContext(ctx) + cmd.SilenceErrors = true + cmd.SilenceUsage = true + require.NoError(t, cmd.ParseFlags(tc.args)) + require.Error(t, cmd.RunE(cmd, nil)) + events := funnelEvents(ctx) + assert.Equal(t, []string{"STARTED", "FINISHED:FAILURE:NETWORK"}, funnelShape(events)) + assert.Equal(t, tc.wantScope, events[len(events)-1].Scope) + }) + } +} + // A plugin failure for an auto-detected agent is only a warning (exit 0), but // when nothing got installed the funnel must not report SUCCESS. func TestInstallFunnelDetectedPluginFailureIsNotSuccess(t *testing.T) { @@ -508,3 +595,318 @@ func TestNothingInstalledCategory(t *testing.T) { assert.Equal(t, []protos.AitoolsCommandStep{protos.AitoolsCommandStepPluginsInstalled}, deliverySteps([]agentOutcome{failed, {agent: codex, delivery: deliveryPlugin, status: outcomeInstalled}})) } + +// isolateCommandEnv gives a full command run an empty HOME and cwd, no calling +// agent (isolateAgents), the remaining config dir overrides derived from that +// HOME, and a no-op telemetry auth resolver, so it reads no host state and +// contacts nothing. +func isolateCommandEnv(t *testing.T) { + isolateAgents(t) + noTelemetryAuth(t) + home := t.TempDir() + t.Setenv("HOME", home) + t.Setenv("USERPROFILE", home) + t.Setenv("XDG_CONFIG_HOME", "") + t.Setenv("AIDEVKIT_HOME", "") + t.Chdir(t.TempDir()) +} + +func TestUninstallDeclinedIsCancelled(t *testing.T) { + isolateCommandEnv(t) + globalDir, err := installer.GlobalSkillsDir(t.Context()) + require.NoError(t, err) + require.NoError(t, installer.SaveState(globalDir, &installer.InstallState{ + SchemaVersion: 1, + Release: "v0.2.6", + Skills: map[string]string{"databricks-test": "1.0.0"}, + })) + calls := setupUninstallMock(t) + + ioCtx, test := cmdio.SetupTest(t.Context(), cmdio.TestOptions{PromptSupported: true}) + defer test.Done() + go drainReader(test.Stdout) + go drainReader(test.Stderr) + go func() { + _, _ = test.Stdin.WriteString("n\n") + _ = test.Stdin.Flush() + }() + + ctx := telemetry.WithNewLogger(ioCtx) + cmd := NewUninstallCmd() + cmd.SetContext(ctx) + cmd.SetArgs([]string{}) + require.NoError(t, cmd.Execute()) + + assert.Empty(t, *calls) + events := funnelEvents(ctx) + assert.Equal(t, []string{"STARTED", "FINISHED:CANCELLED:"}, funnelShape(events)) + assert.True(t, events[0].Interactive) +} + +// Update and uninstall with nothing installed keep their error (and exit code) +// but report NOTHING_TO_DO, not an uncategorized failure. +func TestNothingInstalledFinishesNothingToDo(t *testing.T) { + const projectNotInstalled = "no project-scoped skills found in the current directory." + const global, project = protos.AitoolsInstallScopeGlobal, protos.AitoolsInstallScopeProject + cases := []struct { + name string + newCmd func() *cobra.Command + args []string + command protos.AitoolsCommand + wantErr string + wantScope protos.AitoolsInstallScope + }{ + {name: "update", newCmd: NewUpdateCmd, command: protos.AitoolsCommandUpdate, wantErr: "no skills or plugins installed. Run 'databricks aitools install' to install", wantScope: global}, + {name: "uninstall", newCmd: NewUninstallCmd, command: protos.AitoolsCommandUninstall, wantErr: "no skills or plugins installed", wantScope: global}, + // The scope error comes before the scope is resolved; the flag still sets it. + {name: "update project scope", newCmd: NewUpdateCmd, args: []string{"--scope=project"}, command: protos.AitoolsCommandUpdate, wantErr: projectNotInstalled, wantScope: project}, + {name: "uninstall project scope", newCmd: NewUninstallCmd, args: []string{"--scope=project"}, command: protos.AitoolsCommandUninstall, wantErr: projectNotInstalled, wantScope: project}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + isolateCommandEnv(t) + // Nothing may be fetched; an unreachable source makes any fetch fail. + t.Setenv("DATABRICKS_SKILLS_REF", "v0.2.6") + t.Setenv("DATABRICKS_SKILLS_BASE_URL", "http://127.0.0.1:1") + + ctx := telemetry.WithNewLogger(cmdio.MockDiscard(t.Context())) + cmd := tc.newCmd() + cmd.SetContext(ctx) + cmd.SetArgs(append([]string{}, tc.args...)) + cmd.SilenceErrors = true + cmd.SilenceUsage = true + + // The exact project-scope text is pinned by TestScopeNotInstalledError*. + err := cmd.Execute() + require.Error(t, err) + assert.True(t, strings.HasPrefix(err.Error(), tc.wantErr), err.Error()) + if tc.args == nil { + assert.EqualError(t, err, tc.wantErr) + } + assert.ErrorIs(t, err, installer.ErrNothingInstalled) + + events := funnelEvents(ctx) + assert.Equal(t, []string{"STARTED", "FINISHED:NOTHING_TO_DO:"}, funnelShape(events)) + for _, e := range events { + assert.Equal(t, tc.command, e.Command) + } + assert.Equal(t, tc.wantScope, events[len(events)-1].Scope) + }) + } +} + +// Scope flag errors of update, list and uninstall finish INVALID_FLAGS, as +// install's do, and keep their message. +func TestScopeFlagErrorsFinishInvalidFlags(t *testing.T) { + cases := []struct { + name string + newCmd func() *cobra.Command + args []string + bothInstalled bool + wantErr string + }{ + {name: "list invalid scope", newCmd: NewListCmd, args: []string{"--scope=bogus"}, wantErr: `invalid --scope "bogus": must be one of project, global, both`}, + {name: "list project and global", newCmd: NewListCmd, args: []string{"--project", "--global"}, wantErr: "cannot use --global and --project together"}, + {name: "update scope with project", newCmd: NewUpdateCmd, args: []string{"--scope=global", "--project"}, wantErr: "cannot use --scope with --project or --global; --project and --global are deprecated aliases for --scope"}, + {name: "update both scopes installed", newCmd: NewUpdateCmd, bothInstalled: true, wantErr: "skills are installed in both global and project scopes; use --scope=global, --scope=project, or --scope=both to specify which to update"}, + {name: "uninstall scope both", newCmd: NewUninstallCmd, args: []string{"--scope=both"}, wantErr: "--scope=both is not supported for this command; use 'project' or 'global'"}, + {name: "uninstall project and global", newCmd: NewUninstallCmd, args: []string{"--project", "--global"}, wantErr: "cannot uninstall both scopes at once; run uninstall separately with --scope=global and --scope=project"}, + {name: "uninstall both scopes installed", newCmd: NewUninstallCmd, bothInstalled: true, wantErr: "skills are installed in both global and project scopes; use --scope=global or --scope=project to specify which to uninstall"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + isolateCommandEnv(t) + // Nothing may be fetched; an unreachable source makes any fetch fail. + t.Setenv("DATABRICKS_SKILLS_REF", "v0.2.6") + t.Setenv("DATABRICKS_SKILLS_BASE_URL", "http://127.0.0.1:1") + if tc.bothInstalled { + globalDir, err := installer.GlobalSkillsDir(t.Context()) + require.NoError(t, err) + projectDir, err := installer.ProjectSkillsDir(t.Context()) + require.NoError(t, err) + for _, dir := range []string{globalDir, projectDir} { + require.NoError(t, installer.SaveState(dir, &installer.InstallState{SchemaVersion: 2, Release: "v0.2.6", Skills: map[string]string{"databricks": "0.2.6"}})) + } + } + + ctx := telemetry.WithNewLogger(cmdio.MockDiscard(t.Context())) + cmd := tc.newCmd() + cmd.SetContext(ctx) + cmd.SetArgs(append([]string{}, tc.args...)) + cmd.SetOut(io.Discard) + cmd.SetErr(io.Discard) + cmd.SilenceErrors = true + cmd.SilenceUsage = true + + assert.EqualError(t, cmd.Execute(), tc.wantErr) + assert.Equal(t, []string{"STARTED", "FINISHED:FAILURE:INVALID_FLAGS"}, funnelShape(funnelEvents(ctx))) + }) + } +} + +var errCodexNotOnPath = &installer.BlockedError{Agent: agents.NameCodex, Reason: installer.ReasonCLINotOnPath} + +// testPluginRecords are the plugins saveGlobalState records: Codex and Claude Code. +var testPluginRecords = map[string]installer.PluginRecord{ + agents.NameCodex: {Plugin: "databricks", Version: "0.2.6"}, + agents.NameClaudeCode: {Marketplace: "claude-plugins-official", Plugin: "databricks", Version: "0.2.6"}, +} + +// saveGlobalState records a global install of testPluginRecords plus the given +// file skills (none for a plugin-only install), and returns its dir. +func saveGlobalState(t *testing.T, skills map[string]string) string { + dir, err := installer.GlobalSkillsDir(t.Context()) + require.NoError(t, err) + require.NoError(t, installer.SaveState(dir, &installer.InstallState{ + SchemaVersion: 2, + Release: "v0.2.6", + Skills: skills, + Plugins: testPluginRecords, + })) + return dir +} + +// Plugin failures are warnings (exit 0), but an update in which no plugin got +// updated and nothing else ran did nothing and must not report SUCCESS. +func TestUpdateFunnelWhenPluginsFail(t *testing.T) { + codexFailed := installer.PluginUpdate{Agent: "Codex CLI", Err: errCodexNotOnPath} + cases := []struct { + name string + skills map[string]string + noPlugins bool + results []installer.PluginUpdate + migrate bool + check bool + refErr error + checkErr error + want string + }{ + {name: "all plugins failed", results: []installer.PluginUpdate{codexFailed}, want: "FINISHED:FAILURE:CLI_NOT_ON_PATH"}, + {name: "one of two plugins failed", results: []installer.PluginUpdate{{Agent: "Claude Code", Version: "0.2.7"}, codexFailed}, want: "FINISHED:SUCCESS:"}, + {name: "plugin failed, file skills reconciled", skills: map[string]string{"databricks": "0.2.6"}, results: []installer.PluginUpdate{codexFailed}, want: "FINISHED:SUCCESS:"}, + {name: "plugin failed, raw skills migrated", results: []installer.PluginUpdate{codexFailed}, migrate: true, want: "FINISHED:SUCCESS:"}, + {name: "skills ref lookup failed", refErr: fmt.Errorf("resolve latest skills release: %w", context.DeadlineExceeded), want: "FINISHED:FAILURE:TIMEOUT"}, + {name: "check, plugins checked", check: true, want: "FINISHED:SUCCESS:"}, + {name: "check, skills ref lookup failed", check: true, refErr: fmt.Errorf("resolve latest skills release: %w", context.DeadlineExceeded), want: "FINISHED:FAILURE:TIMEOUT"}, + {name: "check, skills manifest fetch failed", skills: map[string]string{"databricks": "0.2.6"}, noPlugins: true, check: true, checkErr: fmt.Errorf("fetch manifest: %w", context.DeadlineExceeded), want: "FINISHED:FAILURE:TIMEOUT"}, + {name: "check, skills manifest fetch failed, plugins checked", skills: map[string]string{"databricks": "0.2.6"}, check: true, checkErr: fmt.Errorf("fetch manifest: %w", context.DeadlineExceeded), want: "FINISHED:SUCCESS:"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + isolateCommandEnv(t) + dir := saveGlobalState(t, tc.skills) + if tc.noPlugins { + require.NoError(t, installer.SaveState(dir, &installer.InstallState{SchemaVersion: 2, Release: "v0.2.6", Skills: tc.skills})) + } + resetUpdatePluginSeams(t) + origRef, origUpdateSkills := updateGetSkillsRefFn, updateSkillsFn + t.Cleanup(func() { updateGetSkillsRefFn, updateSkillsFn = origRef, origUpdateSkills }) + + updateGetSkillsRefFn = func(context.Context) (string, bool, error) { + return "v0.2.7", true, tc.refErr + } + updatePluginsFn = func(context.Context, string, string) ([]installer.PluginUpdate, error) { + assert.False(t, tc.check, "--check must not update plugins") + return tc.results, nil + } + reconciled := 0 + updateSkillsFn = func(context.Context, installer.ManifestSource, []*agents.Agent, installer.UpdateOptions) (*installer.UpdateResult, error) { + reconciled++ + return &installer.UpdateResult{CheckErr: tc.checkErr}, nil + } + // Migrate Copilot's raw skills to a plugin, leaving no file skills + // behind, so only the migration counts as work done. + hasManagedRawSkillsForAgentFn = func(_ context.Context, a *agents.Agent, _ string) (bool, error) { + return tc.migrate && a.Name == agents.NameCopilot, nil + } + updateInstallPluginForAgentFn = func(context.Context, *agents.Agent, string, string) (installer.PluginRecord, error) { + return installer.PluginRecord{Plugin: "databricks", Version: "0.2.7"}, nil + } + updateRecordPluginInstallsFn = func(_ context.Context, _ string, records map[string]installer.PluginRecord, _ string) error { + plugins := maps.Clone(testPluginRecords) + maps.Copy(plugins, records) + return installer.SaveState(dir, &installer.InstallState{SchemaVersion: 2, Release: "v0.2.7", Plugins: plugins}) + } + updateCleanupLegacyFn = func(context.Context, *agents.Agent, string) error { return nil } + + ctx := telemetry.WithNewLogger(cmdio.MockDiscard(t.Context())) + cmd := NewUpdateCmd() + cmd.SetContext(ctx) + args := []string{"--scope=global"} + if tc.check { + args = append(args, "--check") + } + cmd.SetArgs(args) + require.NoError(t, cmd.Execute()) + events := funnelEvents(ctx) + assert.Equal(t, []string{"STARTED", tc.want}, funnelShape(events)) + assert.Equal(t, protos.AitoolsInstallScopeGlobal, events[len(events)-1].Scope) + if tc.skills == nil { + assert.Zero(t, reconciled, "an install without file skills has nothing to reconcile") + } + }) + } +} + +// Plugin and skill removal failures are warnings (exit 0), but an uninstall +// that removed nothing because every removal failed must not report SUCCESS. +func TestUninstallFunnelWhenRemovalsFail(t *testing.T) { + skillFailed := fmt.Errorf("remove skill: %w", fs.ErrPermission) + cases := []struct { + name string + args []string + result installer.UninstallResult + want string + wantAgents []protos.AitoolsAgentType + }{ + {name: "all plugins failed", result: installer.UninstallResult{PluginFailures: []error{errCodexNotOnPath}}, want: "FINISHED:FAILURE:CLI_NOT_ON_PATH"}, + {name: "nothing to remove for the targeted agent", args: []string{"--agents=cursor"}, want: "FINISHED:NOTHING_TO_DO:", wantAgents: []protos.AitoolsAgentType{protos.AitoolsAgentTypeCursor}}, + {name: "skill removed for the targeted agent", args: []string{"--agents=cursor"}, result: installer.UninstallResult{SkillsRemoved: 1}, want: "FINISHED:SUCCESS:", wantAgents: []protos.AitoolsAgentType{protos.AitoolsAgentTypeCursor}}, + {name: "one of two plugins failed", result: installer.UninstallResult{PluginsRemoved: 1, PluginFailures: []error{errCodexNotOnPath}}, want: "FINISHED:SUCCESS:"}, + {name: "plugin failed, skills removed", result: installer.UninstallResult{SkillsRemoved: 1, PluginFailures: []error{errCodexNotOnPath}}, want: "FINISHED:SUCCESS:"}, + {name: "all skill removals failed", result: installer.UninstallResult{SkillFailures: []error{skillFailed}}, want: "FINISHED:FAILURE:PERMISSION_DENIED"}, + {name: "skill removal failed, plugin removed", result: installer.UninstallResult{PluginsRemoved: 1, SkillFailures: []error{skillFailed}}, want: "FINISHED:SUCCESS:"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + isolateCommandEnv(t) + saveGlobalState(t, nil) + orig := uninstallSkillsFn + t.Cleanup(func() { uninstallSkillsFn = orig }) + uninstallSkillsFn = func(context.Context, installer.UninstallOptions) (*installer.UninstallResult, error) { + return &tc.result, nil + } + + ctx := telemetry.WithNewLogger(cmdio.MockDiscard(t.Context())) + cmd := NewUninstallCmd() + cmd.SetContext(ctx) + cmd.SetArgs(append([]string{"--scope=global"}, tc.args...)) + require.NoError(t, cmd.Execute()) + events := funnelEvents(ctx) + assert.Equal(t, []string{"STARTED", tc.want}, funnelShape(events)) + finished := events[len(events)-1] + assert.Equal(t, protos.AitoolsInstallScopeGlobal, finished.Scope) + assert.Equal(t, tc.wantAgents, finished.Agents) + }) + } +} + +// A confirm prompt that fails (stdin closed, Ctrl-D) finishes PROMPT_FAILED, +// as install's prompts do, not as an uncategorized failure. +func TestUninstallConfirmPromptFailure(t *testing.T) { + isolateCommandEnv(t) + saveGlobalState(t, map[string]string{"databricks": "0.2.6"}) + calls := setupUninstallMock(t) + + ctx := cmdio.InContext(t.Context(), cmdio.NewTestIO(strings.NewReader(""), io.Discard, io.Discard)) + ctx = telemetry.WithNewLogger(ctx) + cmd := NewUninstallCmd() + cmd.SetContext(ctx) + cmd.SetArgs([]string{"--scope=global"}) + cmd.SilenceErrors = true + cmd.SilenceUsage = true + assert.ErrorIs(t, cmd.Execute(), io.EOF) + + assert.Empty(t, *calls) + assert.Equal(t, []string{"STARTED", "FINISHED:FAILURE:PROMPT_FAILED"}, funnelShape(funnelEvents(ctx))) +} diff --git a/cmd/aitools/list.go b/cmd/aitools/list.go index e6decb6a120..008bce4d0b0 100644 --- a/cmd/aitools/list.go +++ b/cmd/aitools/list.go @@ -16,6 +16,7 @@ import ( "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/flags" "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/telemetry/protos" "github.com/spf13/cobra" ) @@ -36,22 +37,17 @@ func NewListCmd() *cobra.Command { // doesn't silently degrade to --scope=both. Users who want both // scopes should use --scope=both (the new explicit spelling). if projectFlag && globalFlag && scopeFlag == "" { - return errors.New("cannot use --global and --project together") + return withCategory(errors.New("cannot use --global and --project together"), protos.AitoolsErrorCategoryInvalidFlags) } projectFlag, globalFlag, err := parseScopeFlag(scopeFlag, projectFlag, globalFlag, true) if err != nil { - return err + return withCategory(err, protos.AitoolsErrorCategoryInvalidFlags) } // list: empty scope = show both. --scope=both also lands here. - var scope string - switch { - case projectFlag && !globalFlag: - scope = installer.ScopeProject - case globalFlag && !projectFlag: - scope = installer.ScopeGlobal - } + scope := explicitScope(projectFlag, globalFlag) + funnelFrom(cmd.Context()).setScope(scope) return listSkillsFn(cmd, scope) }, } @@ -60,7 +56,7 @@ func NewListCmd() *cobra.Command { cmd.Flags().BoolVar(&projectFlag, "project", false, "Show only project-scoped skills") cmd.Flags().BoolVar(&globalFlag, "global", false, "Show only globally-scoped skills") markScopeBoolsDeprecated(cmd) - return cmd + return instrumentCommand(cmd, protos.AitoolsCommandList) } // listOutput is the structured representation of `aitools list` used by both diff --git a/cmd/aitools/list_test.go b/cmd/aitools/list_test.go index 72e236933b4..e1c8b42e033 100644 --- a/cmd/aitools/list_test.go +++ b/cmd/aitools/list_test.go @@ -567,6 +567,7 @@ func TestRenderListTextShowsPluginInstallsBeforeRawSkills(t *testing.T) { } func TestListScopeFlag(t *testing.T) { + noTelemetryAuth(t) tests := []struct { name string args []string diff --git a/cmd/aitools/scope.go b/cmd/aitools/scope.go index da9c7754185..8cb4dfeadb3 100644 --- a/cmd/aitools/scope.go +++ b/cmd/aitools/scope.go @@ -12,6 +12,7 @@ import ( "github.com/databricks/cli/libs/aitools/installer" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/env" + "github.com/databricks/cli/libs/telemetry/protos" "github.com/spf13/cobra" ) @@ -164,6 +165,19 @@ func detectInstalledScopes(globalDir, projectDir string) (global, project bool, return globalState != nil, projectState != nil, nil } +// explicitScope returns the single scope the flags parsed by parseScopeFlag +// select, or "" when they select both or none. +func explicitScope(projectFlag, globalFlag bool) string { + switch { + case projectFlag && !globalFlag: + return installer.ScopeProject + case globalFlag && !projectFlag: + return installer.ScopeGlobal + default: + return "" + } +} + // resolveScopeForUpdate resolves scopes for the update command. // Returns one or more scopes to update. When both flags are set, global always passes through // (for legacy install detection) and project is checked via state. @@ -199,7 +213,7 @@ func resolveScopeForUpdate(ctx context.Context, projectFlag, globalFlag bool, gl switch { case hasGlobal && hasProject: if !cmdio.IsPromptSupported(ctx) { - return nil, errors.New("skills are installed in both global and project scopes; use --scope=global, --scope=project, or --scope=both to specify which to update") + return nil, withCategory(errors.New("skills are installed in both global and project scopes; use --scope=global, --scope=project, or --scope=both to specify which to update"), protos.AitoolsErrorCategoryInvalidFlags) } scopes, err := promptUpdateScopeSelection(ctx) if err != nil { @@ -225,7 +239,7 @@ func resolveScopeForUpdate(ctx context.Context, projectFlag, globalFlag bool, gl // Unlike update, uninstall never allows "both" scopes at once. func resolveScopeForUninstall(ctx context.Context, projectFlag, globalFlag bool, globalDir, projectDir string) (string, error) { if projectFlag && globalFlag { - return "", errors.New("cannot uninstall both scopes at once; run uninstall separately with --scope=global and --scope=project") + return "", withCategory(errors.New("cannot uninstall both scopes at once; run uninstall separately with --scope=global and --scope=project"), protos.AitoolsErrorCategoryInvalidFlags) } hasGlobal, hasProject, err := detectInstalledScopes(globalDir, projectDir) @@ -249,7 +263,7 @@ func resolveScopeForUninstall(ctx context.Context, projectFlag, globalFlag bool, switch { case hasGlobal && hasProject: if !cmdio.IsPromptSupported(ctx) { - return "", errors.New("skills are installed in both global and project scopes; use --scope=global or --scope=project to specify which to uninstall") + return "", withCategory(errors.New("skills are installed in both global and project scopes; use --scope=global or --scope=project to specify which to uninstall"), protos.AitoolsErrorCategoryInvalidFlags) } scope, err := promptUninstallScopeSelection(ctx) if err != nil { @@ -308,9 +322,17 @@ func scopeNotInstalledError(scope, verb, projectDir string, hasGlobal, hasProjec msg += "\n\n" + hint } - return errors.New(msg) + return ¬hingInstalledInScopeError{msg: msg} } +// nothingInstalledInScopeError carries scopeNotInstalledError's message and +// matches installer.ErrNothingInstalled, so telemetry records the run as +// NOTHING_TO_DO like the installer's own "nothing installed" error. +type nothingInstalledInScopeError struct{ msg string } + +func (e *nothingInstalledInScopeError) Error() string { return e.msg } +func (e *nothingInstalledInScopeError) Unwrap() error { return installer.ErrNothingInstalled } + // crossScopeHint returns a hint string if the opposite scope has an installation. // The verb parameter (e.g. "update", "uninstall") controls the action in the hint message. func crossScopeHint(requestedScope, verb string, hasGlobal, hasProject bool) string { @@ -351,7 +373,7 @@ func defaultPromptUpdateScopeSelection(ctx context.Context) ([]string, error) { Value(&scope). Run() if err != nil { - return nil, err + return nil, promptError(err) } if scope == scopeBoth { @@ -386,7 +408,7 @@ func defaultPromptUninstallScopeSelection(ctx context.Context) (string, error) { Value(&scope). Run() if err != nil { - return "", err + return "", promptError(err) } return scope, nil diff --git a/cmd/aitools/telemetry.go b/cmd/aitools/telemetry.go index fbab971b8ed..4e451feb6eb 100644 --- a/cmd/aitools/telemetry.go +++ b/cmd/aitools/telemetry.go @@ -306,3 +306,29 @@ func scopeType(scope string) protos.AitoolsInstallScope { return protos.AitoolsInstallScopeUnspecified } } + +// instrumentCommand makes an aitools subcommand log its STARTED and FINISHED +// funnel events, and resolves auth best-effort so they can upload at exit. +// The end of a run does not wait for an unfinished resolution (the events +// upload only if it has already finished) unless RunE calls +// waitForTelemetryAuth, once the run gets past its cheap early exits. +func instrumentCommand(cmd *cobra.Command, command protos.AitoolsCommand) *cobra.Command { + runE := withCommandFunnel(command, cmd.RunE) + var auth *telemetryAuth + cmd.RunE = func(cmd *cobra.Command, args []string) error { + cmd.SetContext(context.WithValue(cmd.Context(), telemetryAuthKey{}, auth)) + return runE(cmd, args) + } + auth = withTelemetryAuth(cmd) + return cmd +} + +type telemetryAuthKey struct{} + +// waitForTelemetryAuth makes the end of an instrumented run wait for the auth +// resolution within telemetryAuthBudget, as install does past its plan +// confirmation. Only the RunE of a command wrapped by instrumentCommand calls +// it. +func waitForTelemetryAuth(ctx context.Context) { + ctx.Value(telemetryAuthKey{}).(*telemetryAuth).waitAtExit = true +} diff --git a/cmd/aitools/telemetry_auth_test.go b/cmd/aitools/telemetry_auth_test.go index ea59542176a..b6588d74287 100644 --- a/cmd/aitools/telemetry_auth_test.go +++ b/cmd/aitools/telemetry_auth_test.go @@ -33,6 +33,12 @@ func setTelemetryAuth(t *testing.T, budget time.Duration, resolve func(cmd *cobr telemetryAuthBudget = budget } +// noTelemetryAuth replaces the auth resolver with a no-op, so a test that runs +// an instrumented command through Execute never resolves the host's auth. +func noTelemetryAuth(t *testing.T) { + setTelemetryAuth(t, time.Second, func(*cobra.Command, []string) error { return nil }) +} + // authTestCmd returns a command wired with withTelemetryAuth whose RunE // records that it ran and logs a telemetry event, plus a --profile flag like // root's. Telemetry is enabled on its context whatever the outer environment @@ -297,3 +303,108 @@ func TestInstallPastPlanConfirmedWaitsForTelemetryAuth(t *testing.T) { require.True(t, cmdctx.HasConfigUsed(cmd.Context())) assert.Equal(t, "https://ws.test", cmdctx.ConfigUsed(cmd.Context()).Host) } + +// runWithTelemetry runs cmd with args, with telemetry enabled and its events +// buffered, and returns the context it ended with, how long it took and its +// error. +func runWithTelemetry(t *testing.T, cmd *cobra.Command, args ...string) (context.Context, time.Duration, error) { + t.Helper() + cmd.SetContext(telemetry.WithNewLogger(env.Set(cmdio.MockDiscard(t.Context()), "DATABRICKS_CLI_DISABLE_TELEMETRY", ""))) + cmd.SetArgs(args) + cmd.SilenceErrors = true + cmd.SilenceUsage = true + start := time.Now() + err := cmd.Execute() + return cmd.Context(), time.Since(start), err +} + +// list, version, and update and uninstall runs that end at a flag or scope +// error, find nothing installed, or name a skill that is not installed do not +// wait for an unfinished resolution, so +// their events are dropped instead of delaying exit. +func TestInstrumentedCommandsDoNotWaitForTelemetryAuth(t *testing.T) { + cases := []struct { + name string + newCmd func() *cobra.Command + args []string + seed bool // record one installed skill first + wantErr string + }{ + {name: "version", newCmd: NewVersionCmd}, + {name: "list", newCmd: NewListCmd, wantErr: "127.0.0.1:1"}, + {name: "update flag error", newCmd: NewUpdateCmd, args: []string{"--scope=global", "--project"}, wantErr: "cannot use --scope with --project or --global"}, + {name: "uninstall flag error", newCmd: NewUninstallCmd, args: []string{"--project", "--global"}, wantErr: "cannot uninstall both scopes at once"}, + {name: "update with nothing installed", newCmd: NewUpdateCmd, args: []string{"--scope=global"}, wantErr: "no skills or plugins installed"}, + {name: "uninstall with nothing installed", newCmd: NewUninstallCmd, args: []string{"--scope=global"}, wantErr: "no skills or plugins installed"}, + {name: "uninstall of a skill that is not installed", newCmd: NewUninstallCmd, args: []string{"--scope=global", "--skills=typo"}, seed: true, wantErr: `skill "typo" is not installed`}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + isolateCommandEnv(t) + t.Setenv("DATABRICKS_SKILLS_REF", "v0.2.6") + t.Setenv("DATABRICKS_SKILLS_BASE_URL", "http://127.0.0.1:1") + if tc.seed { + saveGlobalSkill(t) + } + blockTelemetryAuth(t) + + ctx, took, err := runWithTelemetry(t, tc.newCmd(), tc.args...) + if tc.wantErr == "" { + require.NoError(t, err) + } else { + require.ErrorContains(t, err, tc.wantErr) + } + assert.Less(t, took, 5*time.Second) + assert.False(t, cmdctx.HasConfigUsed(ctx), "an unfinished resolution leaves no config, so nothing uploads") + }) + } +} + +// saveGlobalSkill records one installed file skill in the global scope. +func saveGlobalSkill(t *testing.T) { + dir, err := installer.GlobalSkillsDir(t.Context()) + require.NoError(t, err) + require.NoError(t, installer.SaveState(dir, &installer.InstallState{SchemaVersion: 2, Release: "v0.2.6", Skills: map[string]string{"databricks": "0.2.6"}})) +} + +// update and uninstall runs that get to their work keep waiting for the +// resolution within the budget. The resolution finishes only some time after +// the last step of the work, so the config reaches the context only if the +// end of the run waits. +func TestUpdateAndUninstallWorkWaitsForTelemetryAuth(t *testing.T) { + t.Run("update", func(t *testing.T) { + isolateCommandEnv(t) + saveGlobalSkill(t) + release := blockTelemetryAuth(t) + resetUpdatePluginSeams(t) + origRef, origUpdateSkills := updateGetSkillsRefFn, updateSkillsFn + t.Cleanup(func() { updateGetSkillsRefFn, updateSkillsFn = origRef, origUpdateSkills }) + updateGetSkillsRefFn = func(context.Context) (string, bool, error) { + return "v0.2.6", true, nil + } + updateSkillsFn = func(context.Context, installer.ManifestSource, []*agents.Agent, installer.UpdateOptions) (*installer.UpdateResult, error) { + time.AfterFunc(200*time.Millisecond, release) + return &installer.UpdateResult{}, nil + } + + ctx, _, err := runWithTelemetry(t, NewUpdateCmd(), "--scope=global") + require.NoError(t, err) + require.True(t, cmdctx.HasConfigUsed(ctx)) + }) + + t.Run("uninstall", func(t *testing.T) { + isolateCommandEnv(t) + saveGlobalSkill(t) + release := blockTelemetryAuth(t) + orig := uninstallSkillsFn + t.Cleanup(func() { uninstallSkillsFn = orig }) + uninstallSkillsFn = func(context.Context, installer.UninstallOptions) (*installer.UninstallResult, error) { + time.AfterFunc(200*time.Millisecond, release) + return &installer.UninstallResult{SkillsRemoved: 1}, nil + } + + ctx, _, err := runWithTelemetry(t, NewUninstallCmd(), "--scope=global") + require.NoError(t, err) + require.True(t, cmdctx.HasConfigUsed(ctx)) + }) +} diff --git a/cmd/aitools/uninstall.go b/cmd/aitools/uninstall.go index 3336f92e6d8..8bbefd9f85a 100644 --- a/cmd/aitools/uninstall.go +++ b/cmd/aitools/uninstall.go @@ -3,17 +3,17 @@ package aitools import ( "context" "fmt" + "slices" "strings" "github.com/databricks/cli/libs/aitools/installer" "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/telemetry/protos" "github.com/spf13/cobra" ) // Package-level for testability. Tests override via uninstall_test.go. -var uninstallSkillsFn = func(ctx context.Context, opts installer.UninstallOptions) error { - return installer.UninstallSkillsOpts(ctx, opts) -} +var uninstallSkillsFn = installer.UninstallSkillsWithResult func NewUninstallCmd() *cobra.Command { var skillsFlag, agentsFlag, scopeFlag string @@ -31,8 +31,11 @@ By default, removes all skills. Use --skills to remove specific skills only.`, projectFlag, globalFlag, err := parseScopeFlag(scopeFlag, projectFlag, globalFlag, false) if err != nil { - return err + return withCategory(err, protos.AitoolsErrorCategoryInvalidFlags) } + // Record a scope flag before resolving, so a scope error still reports it. + funnel := funnelFrom(ctx) + funnel.setScope(explicitScope(projectFlag, globalFlag)) globalDir, err := installer.GlobalSkillsDir(ctx) if err != nil { @@ -47,6 +50,7 @@ By default, removes all skills. Use --skills to remove specific skills only.`, if err != nil { return err } + funnel.setScope(scope) opts := installer.UninstallOptions{ Scope: scope, @@ -58,6 +62,7 @@ By default, removes all skills. Use --skills to remove specific skills only.`, if err != nil { return err } + funnel.setTargets(targetAgents) opts.Agents = make([]string, 0, len(targetAgents)) for _, agent := range targetAgents { opts.Agents = append(opts.Agents, agent.Name) @@ -67,22 +72,50 @@ By default, removes all skills. Use --skills to remove specific skills only.`, // Uninstall is destructive, so confirm interactively before doing // anything. Non-interactive runs (no TTY) proceed unprompted so // automation is unaffected, matching how install only prompts on a TTY. + dir := globalDir + if scope == installer.ScopeProject { + dir = projectDir + } + state, err := installer.LoadState(dir) + if err != nil { + return fmt.Errorf("failed to load install state: %w", err) + } if cmdio.IsPromptSupported(ctx) { - dir := globalDir - if scope == installer.ScopeProject { - dir = projectDir - } - proceed, err := confirmUninstall(ctx, dir, opts) + proceed, err := confirmUninstall(ctx, state, opts) if err != nil { return err } if !proceed { cmdio.LogString(ctx, "Cancelled.") + funnel.finishWith(ctx, protos.AitoolsCommandOutcomeCancelled, protos.AitoolsErrorCategoryUnspecified) return nil } } - return uninstallSkillsFn(ctx, opts) + // With something installed the run removes files and runs agent CLIs, + // so it waits for the telemetry auth. The flag errors and the declined + // confirm above do not, nor does a run that only reports that nothing + // (or a legacy install) is there, or that --skills names a skill that + // is not installed (the installer returns that error). + if state != nil && skillsInstalled(state, opts.Skills) { + waitForTelemetryAuth(ctx) + } + result, err := uninstallSkillsFn(ctx, opts) + if err != nil { + return err + } + // A plugin or skill that could not be removed is only a warning. + failures := slices.Concat(result.PluginFailures, result.SkillFailures) + if result.PluginsRemoved == 0 && result.SkillsRemoved == 0 { + if len(failures) > 0 { + // Every removal failed: FAILURE, though it exits 0. + funnel.finishWith(ctx, protos.AitoolsCommandOutcomeFailure, classifyInstallError(failures[0])) + } else { + // E.g. --agents names agents that have none of the skills. + funnel.finishWith(ctx, protos.AitoolsCommandOutcomeNothingToDo, protos.AitoolsErrorCategoryUnspecified) + } + } + return nil }, } @@ -93,24 +126,24 @@ By default, removes all skills. Use --skills to remove specific skills only.`, cmd.Flags().BoolVar(&projectFlag, "project", false, "Uninstall project-scoped skills") cmd.Flags().BoolVar(&globalFlag, "global", false, "Uninstall globally-scoped skills") markScopeBoolsDeprecated(cmd) - return cmd + return instrumentCommand(cmd, protos.AitoolsCommandUninstall) } // confirmUninstall asks the user to confirm a destructive uninstall, summarizing // what will be removed. It returns true without prompting when nothing is -// recorded in the scope, so the installer can surface its own "nothing installed" -// or legacy guidance instead. -func confirmUninstall(ctx context.Context, dir string, opts installer.UninstallOptions) (bool, error) { - state, err := installer.LoadState(dir) - if err != nil { - return false, err - } +// recorded in the scope (state is nil), so the installer can surface its own +// "nothing installed" or legacy guidance instead. +func confirmUninstall(ctx context.Context, state *installer.InstallState, opts installer.UninstallOptions) (bool, error) { msg, ask := uninstallConfirmMessage(state, opts) if !ask { return true, nil } cmdio.LogString(ctx, msg) - return cmdio.AskYesOrNo(ctx, "Proceed?") + proceed, err := cmdio.AskYesOrNo(ctx, "Proceed?") + if err != nil { + return false, promptError(err) + } + return proceed, nil } // uninstallConfirmMessage builds the human summary of what an uninstall will @@ -139,6 +172,16 @@ func uninstallConfirmMessage(state *installer.InstallState, opts installer.Unins return fmt.Sprintf("This will remove %s from %s (%s scope).", strings.Join(parts, " and "), target, opts.Scope), true } +// skillsInstalled reports whether every name is recorded in state. +func skillsInstalled(state *installer.InstallState, names []string) bool { + for _, name := range names { + if _, ok := state.Skills[name]; !ok { + return false + } + } + return true +} + // plural returns noun for n == 1 and noun+"s" otherwise. func plural(n int, noun string) string { if n == 1 { diff --git a/cmd/aitools/uninstall_test.go b/cmd/aitools/uninstall_test.go index 04c70393c3e..6314254d727 100644 --- a/cmd/aitools/uninstall_test.go +++ b/cmd/aitools/uninstall_test.go @@ -16,14 +16,15 @@ func setupUninstallMock(t *testing.T) *[]installer.UninstallOptions { t.Cleanup(func() { uninstallSkillsFn = orig }) var calls []installer.UninstallOptions - uninstallSkillsFn = func(_ context.Context, opts installer.UninstallOptions) error { + uninstallSkillsFn = func(_ context.Context, opts installer.UninstallOptions) (*installer.UninstallResult, error) { calls = append(calls, opts) - return nil + return &installer.UninstallResult{}, nil } return &calls } func TestUninstallScopeFlag(t *testing.T) { + noTelemetryAuth(t) tests := []struct { name string args []string @@ -66,6 +67,7 @@ func TestUninstallScopeFlag(t *testing.T) { } func TestUninstallAgentsFlag(t *testing.T) { + noTelemetryAuth(t) setupTestAgents(t) calls := setupUninstallMock(t) @@ -81,6 +83,7 @@ func TestUninstallAgentsFlag(t *testing.T) { } func TestUninstallUnknownAgentErrors(t *testing.T) { + noTelemetryAuth(t) setupTestAgents(t) setupUninstallMock(t) diff --git a/cmd/aitools/update.go b/cmd/aitools/update.go index 977ad25ff13..dba5668bf58 100644 --- a/cmd/aitools/update.go +++ b/cmd/aitools/update.go @@ -11,6 +11,7 @@ import ( "github.com/databricks/cli/libs/aitools/installer" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/telemetry/protos" "github.com/spf13/cobra" ) @@ -20,6 +21,7 @@ var updateSkillsFn = func(ctx context.Context, src installer.ManifestSource, ins } var ( + updateGetSkillsRefFn = installer.GetSkillsRef updatePluginsFn = installer.UpdateInstalledPlugins updateInstallPluginForAgentFn = installer.InstallPluginForAgent updateRecordPluginInstallsFn = installer.RecordPluginInstalls @@ -65,13 +67,20 @@ preview what would change without downloading.`, projectFlag, globalFlag, err := parseScopeFlag(scopeFlag, projectFlag, globalFlag, true) if err != nil { - return err + return withCategory(err, protos.AitoolsErrorCategoryInvalidFlags) } + // Record a scope flag before resolving, so a scope error still reports it. + funnel := funnelFrom(ctx) + funnel.setScope(explicitScope(projectFlag, globalFlag)) scopes, err := resolveScopeForUpdate(ctx, projectFlag, globalFlag, globalDir, projectDir) if err != nil { return err } + // The scope enum has no "both"; a run over both scopes reports none. + if len(scopes) == 1 { + funnel.setScope(scopes[0]) + } installed := agents.DetectInstalled(ctx) src := &installer.GitHubManifestSource{} @@ -79,7 +88,13 @@ preview what would change without downloading.`, // The plugin version recorded on update tracks the skills release. // Tolerate a resolution error (skip plugin updates, still do skills). - ref, _, refErr := installer.GetSkillsRef(ctx) + ref, _, refErr := updateGetSkillsRefFn(ctx) + + // A failed or skipped plugin update, or a check whose manifest could + // not be fetched, is only a warning; telemetry still needs to know when + // every attempt failed and nothing else ran. + var firstErr error + updatedAnything := false for _, scope := range scopes { if len(scopes) > 1 { @@ -94,9 +109,16 @@ preview what would change without downloading.`, if err != nil { return err } + // A scope with something installed fetches releases and runs agent + // CLIs, so the run waits for the telemetry auth. Without state the + // run only reports that nothing (or a legacy install) is there. + if state != nil { + waitForTelemetryAuth(ctx) + } if check && refErr == nil && state != nil && len(state.Plugins) > 0 { printPluginCheckResults(ctx, state, installer.DisplaySkillsVersion(ref)) + updatedAnything = true } migrationCandidates, err := pluginMigrationCandidatesForScope(ctx, scope, state) @@ -107,6 +129,12 @@ preview what would change without downloading.`, printPluginMigrationCheckResults(ctx, migrationCandidates) } + // Without a ref the recorded plugins are skipped (with --check, + // not checked). + if refErr != nil && state != nil && len(state.Plugins) > 0 && firstErr == nil { + firstErr = refErr + } + // Update plugin agents through their own CLI (check mode skips this). if !check && refErr == nil { pluginUpdates, err := updatePluginsFn(ctx, scope, ref) @@ -114,6 +142,13 @@ preview what would change without downloading.`, return err } for _, pu := range pluginUpdates { + if pu.Err != nil { + if firstErr == nil { + firstErr = pu.Err + } + continue + } + updatedAnything = true cmdio.LogString(ctx, fmt.Sprintf(" %s databricks plugin %s", pu.Agent, versionToken(pu.Version))) } @@ -122,6 +157,7 @@ preview what would change without downloading.`, return err } if migrated { + updatedAnything = true state, err = installer.LoadState(dir) if err != nil { return err @@ -155,6 +191,13 @@ preview what would change without downloading.`, if err != nil { return err } + if result != nil && result.CheckErr != nil { + if firstErr == nil { + firstErr = result.CheckErr + } + } else { + updatedAnything = true + } if result != nil && (check || len(result.Updated) > 0 || len(result.Added) > 0 || len(result.Removed) > 0) { text := installer.FormatUpdateResult(result, check) if text != "No changes." || len(migrationCandidates) == 0 { @@ -163,6 +206,10 @@ preview what would change without downloading.`, } } } + if firstErr != nil && !updatedAnything { + // Every attempt failed or was skipped: FAILURE, though it exits 0. + funnel.finishWith(ctx, protos.AitoolsCommandOutcomeFailure, classifyInstallError(firstErr)) + } return nil }, } @@ -176,7 +223,7 @@ preview what would change without downloading.`, cmd.Flags().BoolVar(&projectFlag, "project", false, "Update project-scoped skills") cmd.Flags().BoolVar(&globalFlag, "global", false, "Update globally-scoped skills") markScopeBoolsDeprecated(cmd) - return cmd + return instrumentCommand(cmd, protos.AitoolsCommandUpdate) } func pluginMigrationCandidatesForScope(ctx context.Context, scope string, state *installer.InstallState) ([]pluginMigrationCandidate, error) { diff --git a/cmd/aitools/update_test.go b/cmd/aitools/update_test.go index 183cbc0205a..78e27d0d71b 100644 --- a/cmd/aitools/update_test.go +++ b/cmd/aitools/update_test.go @@ -44,6 +44,7 @@ func resetUpdatePluginSeams(t *testing.T) { } func TestUpdateCheckPrintsNoChanges(t *testing.T) { + noTelemetryAuth(t) setupTestAgents(t) t.Setenv("DATABRICKS_SKILLS_REF", "v0.2.6") setupUpdateMock(t) @@ -58,6 +59,7 @@ func TestUpdateCheckPrintsNoChanges(t *testing.T) { } func TestUpdateCheckPrintsPluginState(t *testing.T) { + noTelemetryAuth(t) setupTestAgents(t) t.Setenv("DATABRICKS_SKILLS_REF", "v0.2.6") @@ -96,6 +98,7 @@ func TestUpdateCheckPrintsPluginState(t *testing.T) { } func TestUpdateMigratesManagedRawSkillsToPlugins(t *testing.T) { + noTelemetryAuth(t) setupTestAgents(t) t.Setenv("DATABRICKS_SKILLS_REF", "v0.2.6") resetUpdatePluginSeams(t) @@ -170,6 +173,7 @@ func TestUpdateMigratesManagedRawSkillsToPlugins(t *testing.T) { } func TestUpdateKeepsRawSkillsWhenPluginMigrationStateWriteFails(t *testing.T) { + noTelemetryAuth(t) setupTestAgents(t) t.Setenv("DATABRICKS_SKILLS_REF", "v0.2.6") resetUpdatePluginSeams(t) @@ -234,6 +238,7 @@ func TestUpdateKeepsRawSkillsWhenPluginMigrationStateWriteFails(t *testing.T) { } func TestUpdateCheckPrintsPluginMigrationPlan(t *testing.T) { + noTelemetryAuth(t) setupTestAgents(t) t.Setenv("DATABRICKS_SKILLS_REF", "v0.2.6") resetUpdatePluginSeams(t) @@ -280,6 +285,7 @@ func TestUpdateCheckPrintsPluginMigrationPlan(t *testing.T) { } func TestUpdateDoesNotMigratePluginAgentWithoutManagedRawSkills(t *testing.T) { + noTelemetryAuth(t) setupTestAgents(t) t.Setenv("DATABRICKS_SKILLS_REF", "v0.2.6") resetUpdatePluginSeams(t) @@ -330,6 +336,7 @@ func TestUpdateDoesNotMigratePluginAgentWithoutManagedRawSkills(t *testing.T) { } func TestUpdateNoPruneFlag(t *testing.T) { + noTelemetryAuth(t) setupTestAgents(t) calls := setupUpdateMock(t) @@ -344,6 +351,7 @@ func TestUpdateNoPruneFlag(t *testing.T) { } func TestUpdateScopeFlag(t *testing.T) { + noTelemetryAuth(t) tests := []struct { name string args []string @@ -391,6 +399,7 @@ func TestUpdateScopeFlag(t *testing.T) { } func TestUpdateProjectIncludesProjectSkillAgents(t *testing.T) { + noTelemetryAuth(t) setupTestAgents(t) t.Setenv("DATABRICKS_SKILLS_REF", "v0.2.6") projectRoot := t.TempDir() diff --git a/cmd/aitools/version.go b/cmd/aitools/version.go index 739174a1864..e1d268487bf 100644 --- a/cmd/aitools/version.go +++ b/cmd/aitools/version.go @@ -10,6 +10,7 @@ import ( "github.com/databricks/cli/libs/aitools/installer" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/telemetry/protos" "github.com/spf13/cobra" "golang.org/x/mod/semver" ) @@ -74,7 +75,7 @@ func NewVersionCmd() *cobra.Command { }, } - return cmd + return instrumentCommand(cmd, protos.AitoolsCommandVersion) } // printPluginLines prints one line per plugin recorded in the scope's state, diff --git a/libs/aitools/installer/errors.go b/libs/aitools/installer/errors.go index a3c7bc41022..7b9991be7f7 100644 --- a/libs/aitools/installer/errors.go +++ b/libs/aitools/installer/errors.go @@ -1,12 +1,37 @@ package installer -import "fmt" +import ( + "errors" + "fmt" +) + +// ErrNothingInstalled reports that update or uninstall found no install state +// in the scope. The command layer matches it with errors.Is to tell "nothing to +// act on" apart from a failure. +var ErrNothingInstalled = errors.New("no skills or plugins installed") + +// ErrUnknownAgent reports a plugin record in the install state for an agent +// this CLI does not know (e.g. one written by a newer CLI). Update and +// uninstall leave such a record alone and report it as a failed plugin. +var ErrUnknownAgent = errors.New("unknown agent") + +// unknownAgentInStateError returns ErrUnknownAgent for the agent name recorded +// in the state. +func unknownAgentInStateError(name string) error { + return fmt.Errorf("%w %q in state", ErrUnknownAgent, name) +} + +// ErrLegacyInstall reports skills from a previous install that predates state +// tracking: update and uninstall need the install state, so they ask the user +// to run install first. +var ErrLegacyInstall = errors.New("found skills from a previous install without state tracking") // SkillError reports that a skill named via --skills could not be resolved from -// the manifest. Reason drives telemetry categorization (the command layer maps -// it with errors.AsType); Detail is the human-readable remainder of the message. -// Building the message from fields keeps a classification tag out of the -// user-facing string, so the error is stated exactly once. +// the manifest or, for uninstall, is not installed. Reason drives telemetry +// categorization (the command layer maps it with errors.AsType); Detail is the +// human-readable remainder of the message. Building the message from fields +// keeps a classification tag out of the user-facing string, so the error is +// stated exactly once. type SkillError struct { Skill string Reason string @@ -15,7 +40,8 @@ type SkillError struct { // Reasons a --skills entry can fail to resolve. const ( - // ReasonSkillNotFound: the named skill is absent from the resolved manifest. + // ReasonSkillNotFound: the named skill is absent from the resolved manifest + // or, for uninstall, from the install state. ReasonSkillNotFound = "skill-not-found" // ReasonVersionIncompatible: the skill requires a newer CLI than the one running. ReasonVersionIncompatible = "version-incompatible" diff --git a/libs/aitools/installer/installer_test.go b/libs/aitools/installer/installer_test.go index 8c395fad84a..781ae86cded 100644 --- a/libs/aitools/installer/installer_test.go +++ b/libs/aitools/installer/installer_test.go @@ -97,6 +97,15 @@ func setupTestHome(t *testing.T) string { return tmp } +// clearAgentDirOverrides unsets the env vars agents read their config dir +// from, so agent dirs derive from the temp HOME of setupTestHome rather than +// from the host's overrides. +func clearAgentDirOverrides(t *testing.T) { + for _, v := range []string{"CLAUDE_CONFIG_DIR", "PI_CODING_AGENT_DIR", "GEMINI_CLI_HOME", "GOOSE_PATH_ROOT", "APPDATA", "XDG_CONFIG_HOME"} { + t.Setenv(v, "") + } +} + func setBuildVersion(t *testing.T, version string) { t.Helper() orig := build.GetInfo().Version diff --git a/libs/aitools/installer/plugin_test.go b/libs/aitools/installer/plugin_test.go index 4b536e0e187..4ce12d5934e 100644 --- a/libs/aitools/installer/plugin_test.go +++ b/libs/aitools/installer/plugin_test.go @@ -1,6 +1,7 @@ package installer import ( + "context" "errors" "os/exec" "path/filepath" @@ -240,7 +241,7 @@ func TestUninstallPluginKeepsSharedMarketplace(t *testing.T) { } } -func TestUninstallSkillsOptsTearsDownPlugin(t *testing.T) { +func TestUninstallSkillsWithResultTearsDownPlugin(t *testing.T) { setupTestHome(t) stubAgentLookPath(t, true) ctx, stub := process.WithStub(t.Context()) @@ -257,7 +258,8 @@ func TestUninstallSkillsOptsTearsDownPlugin(t *testing.T) { }, })) - require.NoError(t, UninstallSkillsOpts(ctx, UninstallOptions{Scope: ScopeGlobal})) + _, err = UninstallSkillsWithResult(ctx, UninstallOptions{Scope: ScopeGlobal}) + require.NoError(t, err) cmds := stub.Commands() assert.Contains(t, cmds, "copilot plugin uninstall databricks@databricks-agent-skills") @@ -270,7 +272,7 @@ func TestUninstallSkillsOptsTearsDownPlugin(t *testing.T) { assert.Nil(t, state) } -func TestUninstallSkillsOptsTargetsPluginAgents(t *testing.T) { +func TestUninstallSkillsWithResultTargetsPluginAgents(t *testing.T) { setupTestHome(t) stubAgentLookPath(t, true) ctx, stub := process.WithStub(t.Context()) @@ -288,7 +290,8 @@ func TestUninstallSkillsOptsTargetsPluginAgents(t *testing.T) { }, })) - require.NoError(t, UninstallSkillsOpts(ctx, UninstallOptions{Scope: ScopeGlobal, Agents: []string{agents.NameClaudeCode}})) + _, err = UninstallSkillsWithResult(ctx, UninstallOptions{Scope: ScopeGlobal, Agents: []string{agents.NameClaudeCode}}) + require.NoError(t, err) cmds := stub.Commands() assert.Contains(t, cmds, "claude plugin uninstall databricks@claude-plugins-official --scope user") @@ -321,7 +324,8 @@ func TestUninstallNeverDeregistersSharedMarketplace(t *testing.T) { }, })) - require.NoError(t, UninstallSkillsOpts(ctx, UninstallOptions{Scope: ScopeGlobal})) + _, err = UninstallSkillsWithResult(ctx, UninstallOptions{Scope: ScopeGlobal}) + require.NoError(t, err) cmds := stub.Commands() assert.Contains(t, cmds, "claude plugin uninstall databricks@claude-plugins-official") @@ -349,7 +353,8 @@ func TestUninstallKeepMarketplace(t *testing.T) { }, })) - require.NoError(t, UninstallSkillsOpts(ctx, UninstallOptions{Scope: ScopeGlobal, KeepMarketplace: true})) + _, err = UninstallSkillsWithResult(ctx, UninstallOptions{Scope: ScopeGlobal, KeepMarketplace: true}) + require.NoError(t, err) for _, c := range stub.Commands() { assert.NotContains(t, c, "marketplace remove") @@ -371,7 +376,8 @@ func TestUninstallKeepsUnknownAgentRecord(t *testing.T) { // An unknown agent can't be torn down; its record (and the state file) must be // kept rather than silently dropped while the plugin may still exist. - require.NoError(t, UninstallSkillsOpts(ctx, UninstallOptions{Scope: ScopeGlobal})) + _, err = UninstallSkillsWithResult(ctx, UninstallOptions{Scope: ScopeGlobal}) + require.NoError(t, err) st, err := LoadState(dir) require.NoError(t, err) @@ -397,7 +403,8 @@ func TestUninstallClearsRecordWhenMarketplaceRemoveFails(t *testing.T) { }, })) - require.NoError(t, UninstallSkillsOpts(ctx, UninstallOptions{Scope: ScopeGlobal})) + _, err = UninstallSkillsWithResult(ctx, UninstallOptions{Scope: ScopeGlobal}) + require.NoError(t, err) // The plugin is gone, so the record is cleared even though the marketplace // de-register failed (otherwise a retry would be stuck). @@ -463,7 +470,111 @@ func TestUpdateInstalledPluginsUsesRecordedClaudeScope(t *testing.T) { } } -func TestUninstallSkillsOptsUsesRecordedClaudeScope(t *testing.T) { +// A plugin whose CLI is missing is skipped with a warning (no error), and +// reported so telemetry can tell a run that did nothing from a success. +func TestPluginOperationsReportFailures(t *testing.T) { + t.Run("update", func(t *testing.T) { + ctx, dir := saveCopilotPlugin(t) + results, err := UpdateInstalledPlugins(ctx, ScopeGlobal, "v0.2.7") + require.NoError(t, err) + require.Len(t, results, 1) + blocked, ok := errors.AsType[*BlockedError](results[0].Err) + require.True(t, ok, "got %v", results[0].Err) + assert.Equal(t, ReasonCLINotOnPath, blocked.Reason) + + state, err := LoadState(dir) + require.NoError(t, err) + assert.Equal(t, "0.2.6", state.Plugins[agents.NameCopilot].Version) + }) + + t.Run("uninstall", func(t *testing.T) { + ctx, dir := saveCopilotPlugin(t) + result, err := UninstallSkillsWithResult(ctx, UninstallOptions{Scope: ScopeGlobal}) + require.NoError(t, err) + assert.Equal(t, 0, result.PluginsRemoved) + assert.Equal(t, 0, result.SkillsRemoved) + require.Len(t, result.PluginFailures, 1) + blocked, ok := errors.AsType[*BlockedError](result.PluginFailures[0]) + require.True(t, ok, "got %v", result.PluginFailures[0]) + assert.Equal(t, ReasonCLINotOnPath, blocked.Reason) + + state, err := LoadState(dir) + require.NoError(t, err) + assert.Contains(t, state.Plugins, agents.NameCopilot) + }) +} + +// A plugin record for an agent this CLI does not know (e.g. written by a newer +// CLI) is kept and reported as a failure, so a run that acts on nothing else +// is not a success. +func TestPluginOperationsReportUnknownAgent(t *testing.T) { + t.Run("update", func(t *testing.T) { + ctx, dir := saveUnknownAgentPlugin(t) + results, err := UpdateInstalledPlugins(ctx, ScopeGlobal, "v0.2.7") + require.NoError(t, err) + require.Len(t, results, 1) + assert.Equal(t, "future-agent", results[0].Agent) + assert.EqualError(t, results[0].Err, `unknown agent "future-agent" in state`) + assert.ErrorIs(t, results[0].Err, ErrUnknownAgent) + + state, err := LoadState(dir) + require.NoError(t, err) + assert.Equal(t, "0.2.6", state.Plugins["future-agent"].Version) + }) + + t.Run("uninstall", func(t *testing.T) { + ctx, dir := saveUnknownAgentPlugin(t) + result, err := UninstallSkillsWithResult(ctx, UninstallOptions{Scope: ScopeGlobal}) + require.NoError(t, err) + assert.Equal(t, 0, result.PluginsRemoved) + require.Len(t, result.PluginFailures, 1) + assert.EqualError(t, result.PluginFailures[0], `unknown agent "future-agent" in state`) + assert.ErrorIs(t, result.PluginFailures[0], ErrUnknownAgent) + + state, err := LoadState(dir) + require.NoError(t, err) + assert.Contains(t, state.Plugins, "future-agent") + }) +} + +// saveUnknownAgentPlugin records a plugin for an agent this CLI does not know +// in the global state, and returns the context and the state dir. +func saveUnknownAgentPlugin(t *testing.T) (context.Context, string) { + t.Helper() + setupTestHome(t) + ctx := cmdio.MockDiscard(t.Context()) + dir, err := GlobalSkillsDir(ctx) + require.NoError(t, err) + require.NoError(t, SaveState(dir, &InstallState{ + SchemaVersion: schemaVersionV2, + Release: "v0.2.6", + Plugins: map[string]PluginRecord{ + "future-agent": {Marketplace: "databricks-agent-skills", Plugin: "databricks", Scope: "user", Version: "0.2.6"}, + }, + })) + return ctx, dir +} + +// saveCopilotPlugin records a Copilot plugin in the global state while the +// Copilot CLI is not on PATH, and returns the context and the state dir. +func saveCopilotPlugin(t *testing.T) (context.Context, string) { + t.Helper() + setupTestHome(t) + stubAgentLookPath(t, false) + ctx := cmdio.MockDiscard(t.Context()) + dir, err := GlobalSkillsDir(ctx) + require.NoError(t, err) + require.NoError(t, SaveState(dir, &InstallState{ + SchemaVersion: schemaVersionV2, + Release: "v0.2.6", + Plugins: map[string]PluginRecord{ + agents.NameCopilot: {Marketplace: "databricks-agent-skills", Plugin: "databricks", Scope: "user", Version: "0.2.6"}, + }, + })) + return ctx, dir +} + +func TestUninstallSkillsWithResultUsesRecordedClaudeScope(t *testing.T) { for _, scope := range []string{"project", "user", "local"} { t.Run(scope, func(t *testing.T) { setupTestHome(t) @@ -481,7 +592,8 @@ func TestUninstallSkillsOptsUsesRecordedClaudeScope(t *testing.T) { }, })) - require.NoError(t, UninstallSkillsOpts(ctx, UninstallOptions{Scope: ScopeGlobal})) + _, err = UninstallSkillsWithResult(ctx, UninstallOptions{Scope: ScopeGlobal}) + require.NoError(t, err) cmds := stub.Commands() assert.Contains(t, cmds, "claude plugin disable databricks@claude-plugins-official --scope "+scope) diff --git a/libs/aitools/installer/uninstall.go b/libs/aitools/installer/uninstall.go index 713ec27f39e..6e74b90386f 100644 --- a/libs/aitools/installer/uninstall.go +++ b/libs/aitools/installer/uninstall.go @@ -16,7 +16,7 @@ import ( "github.com/databricks/cli/libs/log" ) -// UninstallOptions controls the behavior of UninstallSkillsOpts. +// UninstallOptions controls the behavior of UninstallSkillsWithResult. type UninstallOptions struct { Skills []string // empty = all Agents []string // empty = all agents @@ -24,15 +24,32 @@ type UninstallOptions struct { KeepMarketplace bool // keep the marketplace registration when removing a plugin } -// UninstallSkills removes all installed skills, their symlinks, and the state file. -func UninstallSkills(ctx context.Context) error { - return UninstallSkillsOpts(ctx, UninstallOptions{}) +// removeAllFn removes a skill's canonical dir or a targeted agent's entry. A +// seam for tests. +var removeAllFn = os.RemoveAll + +// UninstallResult reports what an uninstall did. A plugin or skill that could +// not be removed is only warned about, so the run still returns no error; +// telemetry reads PluginFailures and SkillFailures to tell a run that removed +// nothing from a success. +type UninstallResult struct { + PluginsRemoved int + // SkillsRemoved counts skills whose canonical dir was removed or, when + // opts.Agents is set, whose entry was removed from one of those agents. With + // opts.Agents set, a skill whose entry could not be removed from one of them + // is not counted. + SkillsRemoved int + PluginFailures []error + // SkillFailures holds the errors from removing a skill's canonical dir or + // a targeted agent's entry. + SkillFailures []error } -// UninstallSkillsOpts removes installed skills based on options. -// When opts.Skills is empty, all skills are removed (same as UninstallSkills). +// UninstallSkillsWithResult removes installed skills based on options and +// reports what was removed and what could not be. +// When opts.Skills is empty, all skills (and plugins) are removed. // When opts.Skills is non-empty, only the named skills are removed. -func UninstallSkillsOpts(ctx context.Context, opts UninstallOptions) error { +func UninstallSkillsWithResult(ctx context.Context, opts UninstallOptions) (*UninstallResult, error) { scope := opts.Scope if scope == "" { scope = ScopeGlobal @@ -40,27 +57,27 @@ func UninstallSkillsOpts(ctx context.Context, opts UninstallOptions) error { baseDir, err := skillsDir(ctx, scope) if err != nil { - return err + return nil, err } var cwd string if scope == ScopeProject { cwd, err = os.Getwd() if err != nil { - return fmt.Errorf("failed to determine working directory: %w", err) + return nil, fmt.Errorf("failed to determine working directory: %w", err) } } state, err := LoadState(baseDir) if err != nil { - return fmt.Errorf("failed to load install state: %w", err) + return nil, fmt.Errorf("failed to load install state: %w", err) } if state == nil { if scope == ScopeGlobal && hasLegacyInstall(ctx, baseDir) { - return errors.New("found skills from a previous install without state tracking; run 'databricks aitools install' first, then uninstall") + return nil, fmt.Errorf("%w; run 'databricks aitools install' first, then uninstall", ErrLegacyInstall) } - return errors.New("no skills or plugins installed") + return nil, ErrNothingInstalled } // A full uninstall (no --skills filter) also tears down plugins. @@ -69,6 +86,7 @@ func UninstallSkillsOpts(ctx context.Context, opts UninstallOptions) error { targetedAgents := len(agentFilter) > 0 pluginCount := 0 + var pluginFailures []error if fullUninstall { for _, name := range slices.Sorted(maps.Keys(state.Plugins)) { if targetedAgents && !agentFilter[name] { @@ -79,11 +97,13 @@ func UninstallSkillsOpts(ctx context.Context, opts UninstallOptions) error { // We can't tear down an agent we don't know; keep the record (and // the state file) rather than silently dropping a live plugin. log.Warnf(ctx, "Leaving plugin record for unknown agent %q; remove its plugin manually", name) + pluginFailures = append(pluginFailures, unknownAgentInStateError(name)) continue } rec := state.Plugins[name] if err := UninstallPluginForAgent(ctx, agent, rec, opts.KeepMarketplace); err != nil { log.Warnf(ctx, "Skipped %s: %v", agent.DisplayName, err) + pluginFailures = append(pluginFailures, err) continue } delete(state.Plugins, name) @@ -106,10 +126,10 @@ func UninstallSkillsOpts(ctx context.Context, opts UninstallOptions) error { if _, ok := state.Skills[name]; !ok { if targetedAgents { if foundIn, found := untrackedSkillAgents(ctx, name, opts.Agents, scope, cwd); found { - return fmt.Errorf("skill %q is not tracked in Databricks aitools state but exists for %s. Run 'databricks aitools install --skills-only --agents %s' to rebuild state before uninstalling, or remove it manually", name, strings.Join(foundIn, ", "), strings.Join(opts.Agents, ",")) + return nil, fmt.Errorf("skill %q is not tracked in Databricks aitools state but exists for %s. Run 'databricks aitools install --skills-only --agents %s' to rebuild state before uninstalling, or remove it manually", name, strings.Join(foundIn, ", "), strings.Join(opts.Agents, ",")) } } - return fmt.Errorf("skill %q is not installed", name) + return nil, &SkillError{Skill: name, Reason: ReasonSkillNotFound, Detail: "is not installed"} } if seen[name] { continue @@ -118,27 +138,46 @@ func UninstallSkillsOpts(ctx context.Context, opts UninstallOptions) error { toRemove = append(toRemove, name) } } else { - for name := range state.Skills { - toRemove = append(toRemove, name) - } + // Sorted, like plugins above, so SkillFailures has a stable order. + toRemove = slices.Sorted(maps.Keys(state.Skills)) } - // Remove skill directories and symlinks for each skill. + // Remove skill directories and symlinks for each skill. skillsRemoved counts + // the skills dropped from the state on a full uninstall, and with --agents + // those whose canonical dir or a targeted agent's entry was removed. + skillsRemoved := 0 + var skillFailures []error for _, name := range toRemove { canonicalDir := filepath.Join(baseDir, name) removeCanonical := true + removed := false + var exposureFailures []error if targetedAgents { - removeSkillExposuresFromAgents(ctx, name, canonicalDir, scope, cwd, state, opts.Agents) + removed, exposureFailures = removeSkillExposuresFromAgents(ctx, name, canonicalDir, scope, cwd, state, opts.Agents) + skillFailures = append(skillFailures, exposureFailures...) removeCanonical = !hasRemainingSkillExposure(ctx, name, canonicalDir, scope, cwd, opts.Agents) } else { removeSymlinksFromAgents(ctx, name, canonicalDir, scope, cwd) } + if removeCanonical { + // RemoveAll succeeds on a missing dir. Without --agents the skill's + // state record is dropped either way, so it counts; with --agents + // only an existing dir does. + _, statErr := os.Lstat(canonicalDir) + if err := removeAllFn(canonicalDir); err != nil { + log.Warnf(ctx, "Failed to remove %s: %v", canonicalDir, err) + skillFailures = append(skillFailures, err) + } else if statErr == nil || !targetedAgents { + removed = true + } + } + // A targeted agent that keeps its entry still has the skill. + if removed && len(exposureFailures) == 0 { + skillsRemoved++ + } if !removeCanonical { continue } - if err := os.RemoveAll(canonicalDir); err != nil { - log.Warnf(ctx, "Failed to remove %s: %v", canonicalDir, err) - } delete(state.Skills, name) delete(state.RepoDirs, name) for path := range state.Files { @@ -154,11 +193,11 @@ func UninstallSkillsOpts(ctx context.Context, opts UninstallOptions) error { cleanOrphanedSymlinks(ctx, baseDir, scope, cwd) stateFile := filepath.Join(baseDir, stateFileName) if err := os.Remove(stateFile); err != nil && !errors.Is(err, fs.ErrNotExist) { - return fmt.Errorf("failed to remove state file: %w", err) + return nil, fmt.Errorf("failed to remove state file: %w", err) } } else { if err := SaveState(baseDir, state); err != nil { - return err + return nil, err } } @@ -176,7 +215,7 @@ func UninstallSkillsOpts(ctx context.Context, opts UninstallOptions) error { } cmdio.LogString(ctx, fmt.Sprintf("Uninstalled %d %s.", len(toRemove), noun)) } - return nil + return &UninstallResult{PluginsRemoved: pluginCount, SkillsRemoved: skillsRemoved, PluginFailures: pluginFailures, SkillFailures: skillFailures}, nil } func stringSet(items []string) map[string]bool { @@ -190,7 +229,12 @@ func stringSet(items []string) map[string]bool { return set } -func removeSkillExposuresFromAgents(ctx context.Context, skillName, canonicalDir, scope, cwd string, state *InstallState, agentNames []string) { +// removeSkillExposuresFromAgents removes the skill from the named agents' skills +// dirs and reports whether it removed an existing entry for any of them, plus +// the errors from entries it failed to remove. +func removeSkillExposuresFromAgents(ctx context.Context, skillName, canonicalDir, scope, cwd string, state *InstallState, agentNames []string) (bool, []error) { + removed := false + var failures []error for _, name := range agentNames { agent := agents.ByName(name) if agent == nil { @@ -208,10 +252,18 @@ func removeSkillExposuresFromAgents(ctx context.Context, skillName, canonicalDir log.Debugf(ctx, "Skipping unmanaged skill %s for %s", entry, agent.DisplayName) continue } - if err := os.RemoveAll(entry); err != nil { + // exposureRemovable also accepts a missing entry; there is nothing to remove. + if _, err := os.Lstat(entry); errors.Is(err, fs.ErrNotExist) { + continue + } + if err := removeAllFn(entry); err != nil { log.Warnf(ctx, "Failed to remove %s: %v", entry, err) + failures = append(failures, err) + continue } + removed = true } + return removed, failures } func hasRemainingSkillExposure(ctx context.Context, skillName, canonicalDir, scope, cwd string, removedAgents []string) bool { diff --git a/libs/aitools/installer/uninstall_test.go b/libs/aitools/installer/uninstall_test.go index 3bd7bfcc19e..97f13b09f85 100644 --- a/libs/aitools/installer/uninstall_test.go +++ b/libs/aitools/installer/uninstall_test.go @@ -2,6 +2,8 @@ package installer import ( "context" + "errors" + "fmt" "io/fs" "os" "path/filepath" @@ -33,7 +35,7 @@ func TestUninstallRemovesSkillDirectories(t *testing.T) { ctx, stderr := cmdio.NewTestContextWithStderr(t.Context()) - err := UninstallSkills(ctx) + _, err := UninstallSkillsWithResult(ctx, UninstallOptions{}) require.NoError(t, err) // Skill directories should be gone. @@ -75,7 +77,7 @@ func TestUninstallRemovesSymlinks(t *testing.T) { require.NoError(t, InstallSkillsForAgents(ctx, src, []*agents.Agent{claudeAgent, cursorAgent}, InstallOptions{})) ctx2, _ := cmdio.NewTestContextWithStderr(t.Context()) - err := UninstallSkills(ctx2) + _, err := UninstallSkillsWithResult(ctx2, UninstallOptions{}) require.NoError(t, err) // Check that agent skill directories are cleaned up. @@ -108,7 +110,7 @@ func TestUninstallCleansOrphanedSymlinks(t *testing.T) { require.NoError(t, os.Symlink(orphanTarget, orphanLink)) ctx, _ := cmdio.NewTestContextWithStderr(t.Context()) - err := UninstallSkills(ctx) + _, err := UninstallSkillsWithResult(ctx, UninstallOptions{}) require.NoError(t, err) // Orphaned symlink should be removed. @@ -125,7 +127,7 @@ func TestUninstallDeletesStateFile(t *testing.T) { require.NoError(t, err) ctx := cmdio.MockDiscard(t.Context()) - err = UninstallSkills(ctx) + _, err = UninstallSkillsWithResult(ctx, UninstallOptions{}) require.NoError(t, err) // State file should be gone. @@ -137,9 +139,9 @@ func TestUninstallNoStateReturnsError(t *testing.T) { setupTestHome(t) ctx := cmdio.MockDiscard(t.Context()) - err := UninstallSkills(ctx) - require.Error(t, err) - assert.Contains(t, err.Error(), "no skills or plugins installed") + _, err := UninstallSkillsWithResult(ctx, UninstallOptions{}) + assert.ErrorIs(t, err, ErrNothingInstalled) + assert.EqualError(t, err, "no skills or plugins installed") } func TestUninstallHandlesMissingDirectories(t *testing.T) { @@ -159,7 +161,7 @@ func TestUninstallHandlesMissingDirectories(t *testing.T) { require.NoError(t, SaveState(globalDir, state)) ctx, stderr := cmdio.NewTestContextWithStderr(t.Context()) - err := UninstallSkills(ctx) + _, err := UninstallSkillsWithResult(ctx, UninstallOptions{}) require.NoError(t, err) assert.Contains(t, stderr.String(), "Uninstalled 2 skills.") } @@ -188,7 +190,7 @@ func TestUninstallHandlesBrokenSymlinksToCanonicalDir(t *testing.T) { require.NoError(t, os.Symlink(canonicalTarget, link)) ctx, stderr := cmdio.NewTestContextWithStderr(t.Context()) - err := UninstallSkills(ctx) + _, err := UninstallSkillsWithResult(ctx, UninstallOptions{}) require.NoError(t, err) // Symlink pointing to canonical dir should be removed. @@ -219,7 +221,7 @@ func TestUninstallLeavesNonCanonicalSymlinks(t *testing.T) { require.NoError(t, os.Symlink("/some/other/place", externalLink)) ctx, stderr := cmdio.NewTestContextWithStderr(t.Context()) - err := UninstallSkills(ctx) + _, err := UninstallSkillsWithResult(ctx, UninstallOptions{}) require.NoError(t, err) // Symlink pointing outside canonical dir should be preserved. @@ -250,7 +252,7 @@ func TestUninstallLeavesNonSymlinkDirectories(t *testing.T) { require.NoError(t, os.WriteFile(filepath.Join(regularDir, "custom.md"), []byte("custom"), 0o644)) ctx, stderr := cmdio.NewTestContextWithStderr(t.Context()) - err := UninstallSkills(ctx) + _, err := UninstallSkillsWithResult(ctx, UninstallOptions{}) require.NoError(t, err) // Regular directory should be preserved (not a symlink to canonical dir). @@ -264,7 +266,7 @@ func TestUninstallSelectiveRemovesOnlyNamedSkills(t *testing.T) { globalDir := installTestSkills(t, tmp) ctx, stderr := cmdio.NewTestContextWithStderr(t.Context()) - err := UninstallSkillsOpts(ctx, UninstallOptions{Skills: []string{"databricks-sql"}}) + _, err := UninstallSkillsWithResult(ctx, UninstallOptions{Skills: []string{"databricks-sql"}}) require.NoError(t, err) // databricks-sql should be gone. @@ -285,14 +287,29 @@ func TestUninstallSelectiveRemovesOnlyNamedSkills(t *testing.T) { assert.Contains(t, stderr.String(), "Uninstalled 1 skill.") } +func TestUninstallLegacyInstallDetected(t *testing.T) { + tmp := setupTestHome(t) + ctx := cmdio.MockDiscard(t.Context()) + + // Create skills in canonical location but no state file. + globalDir := filepath.Join(tmp, ".databricks", "aitools", "skills") + require.NoError(t, os.MkdirAll(filepath.Join(globalDir, "databricks-sql"), 0o755)) + + _, err := UninstallSkillsWithResult(ctx, UninstallOptions{}) + assert.EqualError(t, err, "found skills from a previous install without state tracking; run 'databricks aitools install' first, then uninstall") + assert.ErrorIs(t, err, ErrLegacyInstall) +} + func TestUninstallSelectiveUnknownSkillErrors(t *testing.T) { tmp := setupTestHome(t) installTestSkills(t, tmp) ctx := cmdio.MockDiscard(t.Context()) - err := UninstallSkillsOpts(ctx, UninstallOptions{Skills: []string{"nonexistent"}}) - require.Error(t, err) - assert.Contains(t, err.Error(), "not installed") + _, err := UninstallSkillsWithResult(ctx, UninstallOptions{Skills: []string{"nonexistent"}}) + assert.EqualError(t, err, `skill "nonexistent" is not installed`) + skillErr, ok := errors.AsType[*SkillError](err) + require.True(t, ok, "got %T", err) + assert.Equal(t, ReasonSkillNotFound, skillErr.Reason) } func TestUninstallSelectiveDuplicateNamesDeduplicates(t *testing.T) { @@ -301,7 +318,7 @@ func TestUninstallSelectiveDuplicateNamesDeduplicates(t *testing.T) { ctx, stderr := cmdio.NewTestContextWithStderr(t.Context()) // Pass duplicate: should only remove databricks-sql once, not wipe state. - err := UninstallSkillsOpts(ctx, UninstallOptions{Skills: []string{"databricks-sql", "databricks-sql"}}) + _, err := UninstallSkillsWithResult(ctx, UninstallOptions{Skills: []string{"databricks-sql", "databricks-sql"}}) require.NoError(t, err) // databricks-sql should be gone. @@ -327,7 +344,7 @@ func TestUninstallSelectiveAllRemovesStateFile(t *testing.T) { globalDir := installTestSkills(t, tmp) ctx, _ := cmdio.NewTestContextWithStderr(t.Context()) - err := UninstallSkillsOpts(ctx, UninstallOptions{Skills: []string{"databricks-sql", "databricks-jobs"}}) + _, err := UninstallSkillsWithResult(ctx, UninstallOptions{Skills: []string{"databricks-sql", "databricks-jobs"}}) require.NoError(t, err) // State file should be gone since all skills were removed. @@ -351,9 +368,10 @@ func TestUninstallTargetedAgentKeepsRemainingRawExposure(t *testing.T) { globalDir := filepath.Join(tmp, ".databricks", "aitools", "skills") cursorLink := filepath.Join(tmp, ".cursor", "skills", "databricks-sql") claudeLink := filepath.Join(tmp, ".claude", "skills", "databricks-sql") - require.NoError(t, UninstallSkillsOpts(ctx, UninstallOptions{Agents: []string{agents.NameCursor}, Skills: []string{"databricks-sql"}})) + _, err := UninstallSkillsWithResult(ctx, UninstallOptions{Agents: []string{agents.NameCursor}, Skills: []string{"databricks-sql"}}) + require.NoError(t, err) - _, err := os.Lstat(cursorLink) + _, err = os.Lstat(cursorLink) assert.ErrorIs(t, err, fs.ErrNotExist) _, err = os.Lstat(claudeLink) assert.NoError(t, err) @@ -379,9 +397,10 @@ func TestUninstallTargetedAgentRemovesCanonicalWhenNoExposureRemains(t *testing. globalDir := filepath.Join(tmp, ".databricks", "aitools", "skills") cursorDir := filepath.Join(tmp, ".cursor", "skills", "databricks-sql") - require.NoError(t, UninstallSkillsOpts(ctx, UninstallOptions{Agents: []string{agents.NameCursor}, Skills: []string{"databricks-sql"}})) + _, err := UninstallSkillsWithResult(ctx, UninstallOptions{Agents: []string{agents.NameCursor}, Skills: []string{"databricks-sql"}}) + require.NoError(t, err) - _, err := os.Stat(cursorDir) + _, err = os.Stat(cursorDir) assert.ErrorIs(t, err, fs.ErrNotExist) _, err = os.Stat(filepath.Join(globalDir, "databricks-sql")) assert.ErrorIs(t, err, fs.ErrNotExist) @@ -390,6 +409,149 @@ func TestUninstallTargetedAgentRemovesCanonicalWhenNoExposureRemains(t *testing. assert.Nil(t, state) } +// Uninstall counts a skill as removed only when its canonical dir or a targeted +// agent's entry was removed, so telemetry can tell a run that removed nothing +// from a success. +func TestUninstallCountsRemovedSkills(t *testing.T) { + tmp := setupTestHome(t) + clearAgentDirOverrides(t) + ctx := cmdio.MockDiscard(t.Context()) + setupFetchMock(t) + t.Setenv("DATABRICKS_SKILLS_REF", testSkillsRef) + require.NoError(t, os.MkdirAll(filepath.Join(tmp, ".claude"), 0o755)) + require.NoError(t, os.MkdirAll(filepath.Join(tmp, ".cursor"), 0o755)) + + claudeLink := filepath.Join(tmp, ".claude", "skills", "databricks-sql") + cursorLink := filepath.Join(tmp, ".cursor", "skills", "databricks-sql") + + // Only Claude Code has the skill: there is nothing to remove for Cursor. + installSQLSkillForAgent(t, ctx, agents.NameClaudeCode) + result, err := UninstallSkillsWithResult(ctx, UninstallOptions{Agents: []string{agents.NameCursor}}) + require.NoError(t, err) + assert.Equal(t, 0, result.SkillsRemoved) + _, err = os.Lstat(claudeLink) + assert.NoError(t, err) + + // Both have it, but Cursor's entry cannot be removed: nothing counts as + // removed, the failure is reported, and Claude Code keeps the skill. + installSQLSkillForAgent(t, ctx, agents.NameCursor) + orig := removeAllFn + t.Cleanup(func() { removeAllFn = orig }) + removeAllFn = func(path string) error { + if path == cursorLink { + return fs.ErrPermission + } + return orig(path) + } + result, err = UninstallSkillsWithResult(ctx, UninstallOptions{Agents: []string{agents.NameCursor}}) + require.NoError(t, err) + assert.Equal(t, 0, result.SkillsRemoved) + require.Len(t, result.SkillFailures, 1) + assert.ErrorIs(t, result.SkillFailures[0], fs.ErrPermission) + _, err = os.Lstat(claudeLink) + assert.NoError(t, err) + removeAllFn = orig + + // Both have it: removed for Cursor, kept for Claude Code. + result, err = UninstallSkillsWithResult(ctx, UninstallOptions{Agents: []string{agents.NameCursor}}) + require.NoError(t, err) + assert.Equal(t, 1, result.SkillsRemoved) + assert.Empty(t, result.SkillFailures) + _, err = os.Lstat(cursorLink) + assert.ErrorIs(t, err, fs.ErrNotExist) + _, err = os.Lstat(claudeLink) + assert.NoError(t, err) + + // Full uninstall whose canonical dir removal fails: nothing counts as + // removed and the failure is reported. + removeAllFn = func(string) error { return fs.ErrPermission } + result, err = UninstallSkillsWithResult(ctx, UninstallOptions{}) + require.NoError(t, err) + assert.Equal(t, 0, result.SkillsRemoved) + require.Len(t, result.SkillFailures, 1) + assert.ErrorIs(t, result.SkillFailures[0], fs.ErrPermission) + removeAllFn = orig + + // Stale state whose skill dir is already gone: its state record is dropped, + // so the skill counts as removed. + globalDir := filepath.Join(tmp, ".databricks", "aitools", "skills") + require.NoError(t, os.RemoveAll(filepath.Join(globalDir, "databricks-sql"))) + require.NoError(t, SaveState(globalDir, &InstallState{SchemaVersion: schemaVersionV2, Skills: map[string]string{"databricks-sql": "0.1.0"}})) + result, err = UninstallSkillsWithResult(ctx, UninstallOptions{}) + require.NoError(t, err) + assert.Equal(t, 1, result.SkillsRemoved) + assert.Empty(t, result.SkillFailures) +} + +// A targeted agent whose entry cannot be removed keeps the skill, so the skill +// does not count as removed even though no other agent has it and its +// canonical dir is deleted. +func TestUninstallDoesNotCountSkillWithFailedTargetedRemoval(t *testing.T) { + tmp := setupTestHome(t) + clearAgentDirOverrides(t) + ctx := cmdio.MockDiscard(t.Context()) + setupFetchMock(t) + t.Setenv("DATABRICKS_SKILLS_REF", testSkillsRef) + require.NoError(t, os.MkdirAll(filepath.Join(tmp, ".cursor"), 0o755)) + installSQLSkillForAgent(t, ctx, agents.NameCursor) + + cursorEntry := filepath.Join(tmp, ".cursor", "skills", "databricks-sql") + orig := removeAllFn + t.Cleanup(func() { removeAllFn = orig }) + removeAllFn = func(path string) error { + if path == cursorEntry { + return fs.ErrPermission + } + return orig(path) + } + result, err := UninstallSkillsWithResult(ctx, UninstallOptions{Agents: []string{agents.NameCursor}}) + require.NoError(t, err) + assert.Equal(t, 0, result.SkillsRemoved) + require.Len(t, result.SkillFailures, 1) + assert.ErrorIs(t, result.SkillFailures[0], fs.ErrPermission) +} + +// installSQLSkillForAgent installs the databricks-sql test skill for one agent. +func installSQLSkillForAgent(t *testing.T, ctx context.Context, agentName string) { + t.Helper() + src := &mockManifestSource{manifest: testManifest()} + require.NoError(t, InstallSkillsForAgents(ctx, src, []*agents.Agent{agents.ByName(agentName)}, InstallOptions{SpecificSkills: []string{"databricks-sql"}})) +} + +// Telemetry reports the first skill failure's category, so the failures must +// come in a stable (sorted) order, not the state map's iteration order. +func TestUninstallSkillFailuresAreSorted(t *testing.T) { + tmp := setupTestHome(t) + clearAgentDirOverrides(t) + ctx := cmdio.MockDiscard(t.Context()) + globalDir := filepath.Join(tmp, ".databricks", "aitools", "skills") + names := []string{"skill-a", "skill-b", "skill-c", "skill-d", "skill-e", "skill-f"} + skills := map[string]string{} + for _, name := range names { + require.NoError(t, os.MkdirAll(filepath.Join(globalDir, name), 0o755)) + skills[name] = "0.1.0" + } + require.NoError(t, SaveState(globalDir, &InstallState{SchemaVersion: schemaVersionV2, Skills: skills})) + + orig := removeAllFn + t.Cleanup(func() { removeAllFn = orig }) + removeAllFn = func(path string) error { + return fmt.Errorf("%s: %w", filepath.Base(path), fs.ErrPermission) + } + result, err := UninstallSkillsWithResult(ctx, UninstallOptions{}) + require.NoError(t, err) + + var got []string + for _, err := range result.SkillFailures { + got = append(got, err.Error()) + } + var want []string + for _, name := range names { + want = append(want, name+": "+fs.ErrPermission.Error()) + } + assert.Equal(t, want, got) +} + func TestUninstallTargetedUntrackedSkillGivesRecoveryGuidance(t *testing.T) { tmp := setupTestHome(t) ctx := cmdio.MockDiscard(t.Context()) @@ -401,7 +563,7 @@ func TestUninstallTargetedUntrackedSkillGivesRecoveryGuidance(t *testing.T) { Skills: map[string]string{"databricks-jobs": "0.1.0"}, })) - err := UninstallSkillsOpts(ctx, UninstallOptions{Agents: []string{agents.NameCursor}, Skills: []string{"databricks-sql"}}) + _, err := UninstallSkillsWithResult(ctx, UninstallOptions{Agents: []string{agents.NameCursor}, Skills: []string{"databricks-sql"}}) require.Error(t, err) assert.Contains(t, err.Error(), "not tracked in Databricks aitools state") assert.Contains(t, err.Error(), "Cursor") diff --git a/libs/aitools/installer/update.go b/libs/aitools/installer/update.go index 5fa183e0eae..695edfb9e90 100644 --- a/libs/aitools/installer/update.go +++ b/libs/aitools/installer/update.go @@ -39,6 +39,9 @@ type UpdateResult struct { Removed []SkillUpdate // skills pruned because they vanished from the manifest Unchanged []string // skills at current version (or kept despite vanishing) Skipped []string // skills skipped (experimental, version constraint) + // CheckErr is set in check mode when the manifest could not be fetched: the + // check is skipped with a warning, so nothing was checked. + CheckErr error } // SkillUpdate describes a single skill version change. @@ -81,9 +84,9 @@ func UpdateSkills(ctx context.Context, src ManifestSource, targetAgents []*agent if state == nil { if scope == ScopeGlobal && hasLegacyInstall(ctx, baseDir) { - return nil, errors.New("found skills from a previous install without state tracking; run 'databricks aitools install' to refresh before updating") + return nil, fmt.Errorf("%w; run 'databricks aitools install' to refresh before updating", ErrLegacyInstall) } - return nil, errors.New("no skills or plugins installed. Run 'databricks aitools install' to install") + return nil, fmt.Errorf("%w. Run 'databricks aitools install' to install", ErrNothingInstalled) } installedRef, allowFallback := installedSkillsRelease(state) @@ -104,7 +107,7 @@ func UpdateSkills(ctx context.Context, src ManifestSource, targetAgents []*agent if err != nil { if opts.Check { log.Warnf(ctx, "Could not fetch manifest: %v", err) - return &UpdateResult{}, nil + return &UpdateResult{CheckErr: err}, nil } return nil, err } @@ -422,16 +425,19 @@ func dirMatchesRecords(dir, skillName string, state *InstallState) bool { return seen == len(recorded) } -// PluginUpdate records that a plugin was updated for an agent. +// PluginUpdate records a plugin update for an agent. Err is set when the +// update failed and the plugin was skipped with a warning. type PluginUpdate struct { Agent string Version string + Err error } // UpdateInstalledPlugins runs the plugin update for every plugin recorded in the // given scope's state, bumping each record's version to ref. A plugin that can't // be updated (CLI missing, etc.) is skipped with a warning, never failed, to -// keep the non-interactive update prompt-free and exit-0 on partial success. +// keep the non-interactive update prompt-free and exit-0 on partial success; +// it is returned after the updated ones with Err set. func UpdateInstalledPlugins(ctx context.Context, scope, ref string) ([]PluginUpdate, error) { dir, err := skillsDir(ctx, scope) if err != nil { @@ -446,16 +452,18 @@ func UpdateInstalledPlugins(ctx context.Context, scope, ref string) ([]PluginUpd } version := DisplaySkillsVersion(ref) - var updated []PluginUpdate + var updated, failed []PluginUpdate for _, name := range slices.Sorted(maps.Keys(state.Plugins)) { agent := agents.ByName(name) if agent == nil { log.Warnf(ctx, "Skipping unknown agent %q in state", name) + failed = append(failed, PluginUpdate{Agent: name, Err: unknownAgentInStateError(name)}) continue } rec := state.Plugins[name] if err := UpdatePluginForAgent(ctx, agent, rec); err != nil { log.Warnf(ctx, "Skipped %s: %v", agent.DisplayName, err) + failed = append(failed, PluginUpdate{Agent: agent.DisplayName, Err: err}) continue } rec.Version = version @@ -469,7 +477,7 @@ func UpdateInstalledPlugins(ctx context.Context, scope, ref string) ([]PluginUpd return nil, err } } - return updated, nil + return append(updated, failed...), nil } // FormatUpdateResult returns a human-readable summary of the update result. diff --git a/libs/aitools/installer/update_test.go b/libs/aitools/installer/update_test.go index 2a9326e4f82..dad9840ce0a 100644 --- a/libs/aitools/installer/update_test.go +++ b/libs/aitools/installer/update_test.go @@ -23,9 +23,8 @@ func TestUpdateNoStateReturnsInstallHint(t *testing.T) { src := &mockManifestSource{manifest: testManifest()} _, err := UpdateSkills(ctx, src, nil, UpdateOptions{}) - require.Error(t, err) - assert.Contains(t, err.Error(), "no skills or plugins installed") - assert.Contains(t, err.Error(), "databricks aitools install") + assert.ErrorIs(t, err, ErrNothingInstalled) + assert.EqualError(t, err, "no skills or plugins installed. Run 'databricks aitools install' to install") } func TestUpdateLegacyInstallDetected(t *testing.T) { @@ -38,9 +37,8 @@ func TestUpdateLegacyInstallDetected(t *testing.T) { src := &mockManifestSource{manifest: testManifest()} _, err := UpdateSkills(ctx, src, nil, UpdateOptions{}) - require.Error(t, err) - assert.Contains(t, err.Error(), "previous install without state tracking") - assert.Contains(t, err.Error(), "refresh before updating") + assert.EqualError(t, err, "found skills from a previous install without state tracking; run 'databricks aitools install' to refresh before updating") + assert.ErrorIs(t, err, ErrLegacyInstall) } func TestUpdateAlreadyUpToDate(t *testing.T) { diff --git a/libs/telemetry/protos/README.md b/libs/telemetry/protos/README.md index 232cd12e62a..176144d96a7 100644 --- a/libs/telemetry/protos/README.md +++ b/libs/telemetry/protos/README.md @@ -14,10 +14,13 @@ Claude Code session per hour, so when the command that printed it resolves no au `databricks version`), that session logs no install hint event for the next hour even though the agent saw the hint. -`databricks aitools install` logs an `aitools_command_event` funnel per run: STARTED, one -STEP_COMPLETED per install step reached, and FINISHED with the outcome and error category. The CLI -sets no calling-agent field on it: the upload's User-Agent carries the agent (`agent/`), its -Databricks tooling (`aitools-agent/...`) and the `cmd-exec-id` that every event of the run shares. It uploads at exit only when the command resolved -auth. A run that ends before the install plan is confirmed (a flag error, a cancelled prompt, no -agent detected, a `--path` dump) uploads only if auth has already resolved by then; it does not wait -for it. +`databricks aitools` install, update, list, version and uninstall log an `aitools_command_event` +funnel per run: STARTED, one STEP_COMPLETED per install step reached (install only), and FINISHED +with the outcome and error category. The CLI sets no calling-agent field on it: the upload's +User-Agent carries the agent (`agent/`), its Databricks tooling (`aitools-agent/...`) and the +`cmd-exec-id` that every event of the run shares. It uploads at exit only when the command resolved +auth. `list` and `version` upload only if auth has already resolved when they finish; they never +wait for it. Neither does a run of `install`, `update` or `uninstall` that ends before its plan or +confirmation step (a flag error, a cancelled prompt, no agent detected, a `--path` dump), nor an +`update` or `uninstall` that finds nothing installed. A run that does not wait still uploads, +within the upload timeout, when auth has already resolved. diff --git a/libs/telemetry/protos/aitools_events.go b/libs/telemetry/protos/aitools_events.go index 381862be3d3..501a0a93b1b 100644 --- a/libs/telemetry/protos/aitools_events.go +++ b/libs/telemetry/protos/aitools_events.go @@ -93,8 +93,9 @@ const ( // The user declined the confirm prompt, aborted a prompt, or confirmed the // agent picker with nothing selected. AitoolsCommandOutcomeCancelled AitoolsCommandOutcome = "CANCELLED" - // The command had nothing to act on (e.g. no coding agents detected, or a - // --path dump where every skill was filtered out). + // The command had nothing to act on (e.g. no coding agents detected, a + // --path dump where every skill was filtered out, or nothing installed to + // update or uninstall). AitoolsCommandOutcomeNothingToDo AitoolsCommandOutcome = "NOTHING_TO_DO" ) @@ -112,10 +113,11 @@ const ( AitoolsInstallModePath AitoolsInstallMode = "PATH" ) -// AitoolsCommandEvent is one funnel event of an aitools subcommand. Every -// install run that gets past flag parsing emits a STARTED event, a -// STEP_COMPLETED event per step reached and a FINISHED event. All events of one -// run share ExecutionContext.CmdExecID. +// AitoolsCommandEvent is one funnel event of an aitools subcommand. Every run +// of install, update, list, version and uninstall that gets past flag parsing +// emits a STARTED event and a FINISHED event; install also emits a +// STEP_COMPLETED event per step reached. All events of one run share +// ExecutionContext.CmdExecID. // The CLI sets no calling-agent field on this event: it reports the calling // agent through the User-Agent of the telemetry upload (`agent/`, and its // Databricks tooling as `aitools-agent/...`), together with the same @@ -140,11 +142,13 @@ type AitoolsCommandEvent struct { // Install only: delivery mode. Set on every install event once flags parse. InstallMode AitoolsInstallMode `json:"install_mode,omitempty"` - // Install only: agents the run targeted, deduplicated and sorted. Set on the - // FINISHED event once the targets are known. + // Install and uninstall: agents the run targeted (uninstall: from + // --agents), deduplicated and sorted. Set on the FINISHED event once the + // targets are known. Agents []AitoolsAgentType `json:"agents,omitempty"` - // Install only: scope the run acted on. Set on events after the scope is - // resolved. + // Install, update, list and uninstall: the single scope the run acts on. + // Set on events once a scope flag selects one or the scope is resolved; + // omitted when none or both scopes are selected. Scope AitoolsInstallScope `json:"scope,omitempty"` }