diff --git a/.nextchanges/cli/apps-init-plugin-tip.md b/.nextchanges/cli/apps-init-plugin-tip.md new file mode 100644 index 00000000000..728c582b23b --- /dev/null +++ b/.nextchanges/cli/apps-init-plugin-tip.md @@ -0,0 +1 @@ +* `apps init` no longer suggests installing skills when every detected coding agent already has the Databricks plugin. ([#7467](https://github.com/databricks/cli/pull/7467)) diff --git a/.nextchanges/cli/telemetry-upload-timeout.md b/.nextchanges/cli/telemetry-upload-timeout.md new file mode 100644 index 00000000000..2c3f4702e39 --- /dev/null +++ b/.nextchanges/cli/telemetry-upload-timeout.md @@ -0,0 +1 @@ +* Bound the whole telemetry upload at exit, including config resolution and authentication, by its timeout, so a stalled auth or telemetry endpoint no longer keeps a command from exiting. ([#7467](https://github.com/databricks/cli/pull/7467)) diff --git a/acceptance/cmd/aitools/telemetry/apps-init-tip/home/.claude/.gitkeep b/acceptance/cmd/aitools/telemetry/apps-init-tip/home/.claude/.gitkeep new file mode 100644 index 00000000000..e69de29bb2d diff --git a/acceptance/cmd/aitools/telemetry/apps-init-tip/out.test.toml b/acceptance/cmd/aitools/telemetry/apps-init-tip/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/apps-init-tip/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/cmd/aitools/telemetry/apps-init-tip/output.txt b/acceptance/cmd/aitools/telemetry/apps-init-tip/output.txt new file mode 100644 index 00000000000..335595f4d75 --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/apps-init-tip/output.txt @@ -0,0 +1,30 @@ + +=== apps init with flags prints the tip: one hint-shown event per hint printed + +>>> [CLI] apps init --template ./template --skip-install --name app-tip +Databricks skills are not installed. To work with Databricks reliably, first run: databricks aitools install +✔ Creating project + +✔ Project created successfully! + + Location: [TEST_TMP_DIR]/app-tip + Files: 3 + +Tip: coding agents detected without Databricks skills. Run 'databricks aitools install' to install them. + +>>> print_telemetry +{"command":"apps_init","exit_code":0,"aitools_hint_event":{"hint_kind":"INSTALL_TEXT"}} +{"command":"apps_init","exit_code":0,"aitools_hint_event":{"hint_kind":"APPS_INIT_TIP"}} + +=== plugin installed: no hint, no tip, no event + +>>> [CLI] apps init --template ./template --skip-install --name app-plugin +✔ Creating project + +✔ Project created successfully! + + Location: [TEST_TMP_DIR]/app-plugin + Files: 3 + + +>>> print_telemetry diff --git a/acceptance/cmd/aitools/telemetry/apps-init-tip/plugin-home/.claude/plugins/installed_plugins.json b/acceptance/cmd/aitools/telemetry/apps-init-tip/plugin-home/.claude/plugins/installed_plugins.json new file mode 100644 index 00000000000..ee67a12e806 --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/apps-init-tip/plugin-home/.claude/plugins/installed_plugins.json @@ -0,0 +1 @@ +{"plugins":{"databricks@claude-plugins-official":[{"scope":"user","version":"0.2.9"}]}} diff --git a/acceptance/cmd/aitools/telemetry/apps-init-tip/script b/acceptance/cmd/aitools/telemetry/apps-init-tip/script new file mode 100644 index 00000000000..5fa052bb0a4 --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/apps-init-tip/script @@ -0,0 +1,18 @@ +# Isolate HOME so the real machine's agents and skills are not seen. The +# committed home/.claude, a Claude Code config dir without Databricks skills, is +# what makes apps init print its tip. +sethome home +export CLAUDECODE=1 +unset CLAUDE_CODE_SESSION_ID +unset CLAUDE_CONFIG_DIR + +title "apps init with flags prints the tip: one hint-shown event per hint printed" +trace $CLI apps init --template ./template --skip-install --name app-tip > LOG.apps-init +trace print_telemetry +rm -rf app-tip + +title "plugin installed: no hint, no tip, no event" +sethome ./plugin-home +trace $CLI apps init --template ./template --skip-install --name app-plugin > LOG.apps-init-plugin +trace print_telemetry +rm -rf app-plugin diff --git a/acceptance/cmd/aitools/telemetry/apps-init-tip/template/app.yaml.tmpl b/acceptance/cmd/aitools/telemetry/apps-init-tip/template/app.yaml.tmpl new file mode 100644 index 00000000000..856b56d0a0b --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/apps-init-tip/template/app.yaml.tmpl @@ -0,0 +1 @@ +command: ['npm', 'run', 'start'] diff --git a/acceptance/cmd/aitools/telemetry/apps-init-tip/template/appkit.plugins.json b/acceptance/cmd/aitools/telemetry/apps-init-tip/template/appkit.plugins.json new file mode 100644 index 00000000000..678bd0ee554 --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/apps-init-tip/template/appkit.plugins.json @@ -0,0 +1,4 @@ +{ + "version": "1.0", + "plugins": {} +} diff --git a/acceptance/cmd/aitools/telemetry/apps-init-tip/template/databricks.yml.tmpl b/acceptance/cmd/aitools/telemetry/apps-init-tip/template/databricks.yml.tmpl new file mode 100644 index 00000000000..7f316afcd95 --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/apps-init-tip/template/databricks.yml.tmpl @@ -0,0 +1,2 @@ +bundle: + name: {{.projectName}} diff --git a/acceptance/cmd/aitools/telemetry/apps-init-tip/test.toml b/acceptance/cmd/aitools/telemetry/apps-init-tip/test.toml new file mode 100644 index 00000000000..def8da54843 --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/apps-init-tip/test.toml @@ -0,0 +1,8 @@ +[[Repls]] +# Fix path with reverse slashes in the output for Windows. +Old = 'TEST_TMP_DIR]\\app-tip' +New = 'TEST_TMP_DIR]/app-tip' + +[[Repls]] +Old = 'TEST_TMP_DIR]\\app-plugin' +New = 'TEST_TMP_DIR]/app-plugin' diff --git a/acceptance/cmd/aitools/telemetry/hint/installed-home/.claude/plugins/installed_plugins.json b/acceptance/cmd/aitools/telemetry/hint/installed-home/.claude/plugins/installed_plugins.json new file mode 100644 index 00000000000..ee67a12e806 --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/hint/installed-home/.claude/plugins/installed_plugins.json @@ -0,0 +1 @@ +{"plugins":{"databricks@claude-plugins-official":[{"scope":"user","version":"0.2.9"}]}} diff --git a/acceptance/cmd/aitools/telemetry/hint/out.test.toml b/acceptance/cmd/aitools/telemetry/hint/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/hint/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/cmd/aitools/telemetry/hint/output.txt b/acceptance/cmd/aitools/telemetry/hint/output.txt new file mode 100644 index 00000000000..1283eb52109 --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/hint/output.txt @@ -0,0 +1,22 @@ + +=== hint printed on an authenticated command: one hint-shown event is uploaded + +>>> [CLI] jobs list +Databricks skills are not installed. To work with Databricks reliably, first run: databricks aitools install + +>>> print_telemetry +{"command":"jobs_list","exit_code":0,"aitools_hint_event":{"hint_kind":"INSTALL_TEXT"}} + +=== hint printed on a command that resolves no auth: nothing is uploaded + +>>> [CLI] version +Databricks skills are not installed. To work with Databricks reliably, first run: databricks aitools install +Databricks CLI v[CLI_VERSION] + +>>> print_telemetry + +=== plugin installed: no hint, no event + +>>> [CLI] jobs list + +>>> print_telemetry diff --git a/acceptance/cmd/aitools/telemetry/hint/script b/acceptance/cmd/aitools/telemetry/hint/script new file mode 100644 index 00000000000..1289a90535e --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/hint/script @@ -0,0 +1,19 @@ +# Isolate HOME so the real machine's plugins and skills are not seen. +sethome home +export CLAUDECODE=1 +unset CLAUDE_CODE_SESSION_ID +# A host CLAUDE_CONFIG_DIR would move the plugin file away from the isolated HOME. +unset CLAUDE_CONFIG_DIR + +title "hint printed on an authenticated command: one hint-shown event is uploaded" +trace $CLI jobs list +trace print_telemetry + +title "hint printed on a command that resolves no auth: nothing is uploaded" +trace $CLI version +trace print_telemetry + +title "plugin installed: no hint, no event" +sethome ./installed-home +trace $CLI jobs list +trace print_telemetry diff --git a/acceptance/cmd/aitools/telemetry/script.prepare b/acceptance/cmd/aitools/telemetry/script.prepare new file mode 100644 index 00000000000..f55e1cef710 --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/script.prepare @@ -0,0 +1,17 @@ +# A host that disables telemetry or sets a zero upload timeout would upload +# nothing, so these cases would record no events. +unset DATABRICKS_CLI_DISABLE_TELEMETRY DATABRICKS_CLI_TELEMETRY_TIMEOUT + +# print_telemetry prints each telemetry event uploaded since the last call, one +# per line: the command it ran under, its exit code, and the event without the +# rest of the execution context (timings and versions vary). print_requests.py +# clears the recorded requests, so the next call sees only the next command's +# upload. It fails when no request was recorded, hence the file check. +print_telemetry() { + if [ -f "$OUT_REQUESTS" ]; then + print_requests.py --oneline //telemetry-ext | jq -c '.body.protoLogs[] | fromjson + | .entry.databricks_cli_log + | {command: .execution_context.command, exit_code: .execution_context.exit_code} + + (del(.execution_context))' + fi +} diff --git a/acceptance/cmd/aitools/telemetry/test.toml b/acceptance/cmd/aitools/telemetry/test.toml new file mode 100644 index 00000000000..ad4d6d69eec --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/test.toml @@ -0,0 +1,6 @@ +# Telemetry payloads of the install hint. +RecordRequests = true + +Ignore = [ + "home", +] diff --git a/cmd/apps/init.go b/cmd/apps/init.go index 7ba36f79999..6c02d9879dc 100644 --- a/cmd/apps/init.go +++ b/cmd/apps/init.go @@ -1913,8 +1913,8 @@ func runCreate(ctx context.Context, opts createOptions) (runErr error) { // Recommend skills installation if coding agents are detected without skills. // In flags mode, only print a hint — never prompt interactively. if flagsMode { - if !agents.HasDatabricksSkillsInstalled(ctx) { - cmdio.LogString(ctx, "Tip: coding agents detected without Databricks skills. Run 'databricks aitools install' to install them.") + if !agents.HasDatabricksToolingInstalled(ctx) { + agents.PrintSkillsTip(ctx) } } else if err := agents.RecommendSkillsInstall(ctx, installer.InstallAllSkills); err != nil { log.Warnf(ctx, "Skills recommendation failed: %v", err) diff --git a/cmd/root/root.go b/cmd/root/root.go index d304b62ed84..59b9777ef69 100644 --- a/cmd/root/root.go +++ b/cmd/root/root.go @@ -90,7 +90,8 @@ func New(ctx context.Context) *cobra.Command { cmd.SetContext(ctx) // Recommend installing Databricks AI tooling to Claude Code when it is - // driving the CLI without the tooling installed (best-effort, stderr only). + // driving the CLI without the tooling installed (best-effort: a stderr + // line and a telemetry event that is uploaded at exit). agents.MaybeHint(ctx, cmd) return nil } diff --git a/libs/aitools/agents/hint.go b/libs/aitools/agents/hint.go index 5cd331130f8..b37bcd6060c 100644 --- a/libs/aitools/agents/hint.go +++ b/libs/aitools/agents/hint.go @@ -2,13 +2,14 @@ package agents import ( "context" - "path/filepath" "time" "github.com/databricks/cli/libs/cache" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/telemetry" + "github.com/databricks/cli/libs/telemetry/protos" "github.com/databricks/databricks-sdk-go/useragent" "github.com/spf13/cobra" ) @@ -49,8 +50,9 @@ const hintSessionTTL = time.Hour // tooling when Claude Code drives the CLI without it installed. It is a no-op // for human callers, other agents, aitools commands themselves, when there is // no output stream, once the tooling is present, and when the session was -// hinted within the last hintSessionTTL. Best-effort: it writes only to stderr -// and never fails the command. +// hinted within the last hintSessionTTL. Best-effort: it never fails the +// command. Besides the stderr line, it logs a telemetry event for each hint +// shown, which makes a command that resolved auth upload telemetry at exit. func MaybeHint(ctx context.Context, cmd *cobra.Command) { if useragent.AgentProvider() != claudeAgentProvider { return @@ -72,6 +74,7 @@ func MaybeHint(ctx context.Context, cmd *cobra.Command) { return } cmdio.LogString(ctx, hintText) + logHintShown(ctx, protos.AitoolsHintKindInstallText) recordSessionHint(ctx) } @@ -110,12 +113,7 @@ func claudeToolingInstalled(ctx context.Context) bool { // The canonical dir always holds a real databricks* subdir after a skills // install, regardless of whether the agent copy is a symlink or a copy. The // legacy dir is checked too so users of an older install aren't hinted. - home, err := env.UserHomeDir(ctx) - if err != nil { - return false - } - return HasDatabricksSkillsIn(filepath.Join(home, CanonicalSkillsDir)) || - HasDatabricksSkillsIn(filepath.Join(home, LegacySkillsDir)) + return hasSharedSkills(ctx) } // alreadyHintedThisSession reports whether this Claude session was hinted @@ -142,3 +140,17 @@ func recordSessionHint(ctx context.Context) { c := cache.NewCache(ctx, hintCacheComponent, hintSessionTTL, nil) cache.Put(ctx, c, session, true) } + +// logHintShown records that a recommendation to install Databricks AI tools of +// the given kind was shown. Call it once per hint shown; for a prompt, call it +// before asking so a declined prompt still counts. The upload's +// ExecutionContext.Command names the command the hint was shown on. +// +// The event is buffered and uploaded by cmd/root at exit, which needs resolved +// auth: a hint shown on a command that never resolves auth (e.g. `databricks +// version`) is not uploaded. +func logHintShown(ctx context.Context, kind protos.AitoolsHintKind) { + telemetry.Log(ctx, protos.DatabricksCliLog{ + AitoolsHintEvent: &protos.AitoolsHintEvent{HintKind: kind}, + }) +} diff --git a/libs/aitools/agents/hint_test.go b/libs/aitools/agents/hint_test.go index b2a16ffb8e8..3add0417795 100644 --- a/libs/aitools/agents/hint_test.go +++ b/libs/aitools/agents/hint_test.go @@ -8,46 +8,22 @@ import ( "path/filepath" "testing" + "github.com/databricks/cli/internal/testutil" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/log" - "github.com/databricks/databricks-sdk-go/useragent" + "github.com/databricks/cli/libs/telemetry" + "github.com/databricks/cli/libs/telemetry/protos" "github.com/spf13/cobra" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -// agentEnvVars are the env vars useragent.AgentProvider() inspects. Kept in -// sync with listKnownAgents() in the SDK plus the AGENT/AI_AGENT fallbacks -// (mirrors acceptance/acceptance_test.go). -var agentEnvVars = []string{ - "AGENT", "AI_AGENT", "AMP_CURRENT_THREAD_ID", "ANTIGRAVITY_AGENT", - "AUGMENT_AGENT", "CLAUDECODE", "CLINE_ACTIVE", "CODEX_CI", "COPILOT_CLI", - "CURSOR_AGENT", "GEMINI_CLI", "GOOSE_TERMINAL", "KIRO", "OPENCLAW_SHELL", - "OPENCODE", "VSCODE_AGENT", "WINDSURF_AGENT", -} - -// clearAgentEnv unsets every agent-detection env var (restoring them after the -// test) so the host agent driving this test binary does not leak into -// AgentProvider(). t.Setenv cannot unset, hence the manual save/restore. -func clearAgentEnv(t *testing.T) { - for _, v := range agentEnvVars { - if orig, ok := os.LookupEnv(v); ok { - t.Cleanup(func() { os.Setenv(v, orig) }) //nolint:usetesting // restoring, not setting fresh - os.Unsetenv(v) //nolint:usetesting // t.Setenv cannot unset - } - } -} - // setClaudeAgent makes useragent.AgentProvider() report Claude Code for this -// test. It clears the ambient agent env, sets CLAUDECODE, and resets the SDK's -// process-global cache, so these tests must not run in parallel. The cleanup -// clears the cache again so the value never leaks into another test. +// test, with the host's environment cleared. These tests must not run in +// parallel (see testutil.SetAgentEnv). func setClaudeAgent(t *testing.T) { - clearAgentEnv(t) - t.Setenv("CLAUDECODE", "1") - useragent.ClearCache() - t.Cleanup(useragent.ClearCache) + testutil.SetAgentEnv(t, "CLAUDECODE", "1") } // claudeCtx returns a context with a captured stderr buffer and an isolated @@ -72,22 +48,35 @@ func writeCanonicalSkill(t *testing.T, home, name string) { require.NoError(t, os.MkdirAll(filepath.Join(home, ".databricks", "aitools", "skills", name), 0o755)) } +// hintKinds returns the kinds of the hint-shown events buffered on ctx, +// failing on any other event so a stray event cannot hide behind the filter. +func hintKinds(t *testing.T, ctx context.Context) []protos.AitoolsHintKind { + var out []protos.AitoolsHintKind + for _, e := range telemetry.BufferedEvents(ctx) { + require.NotNil(t, e.AitoolsHintEvent, "unexpected non-hint event: %+v", e) + out = append(out, e.AitoolsHintEvent.HintKind) + } + return out +} + func TestMaybeHint_HumanCallerSilent(t *testing.T) { // No agent env: AgentProvider() == "", so no hint even with nothing installed. - clearAgentEnv(t) - useragent.ClearCache() - t.Cleanup(useragent.ClearCache) + testutil.SetAgentEnv(t) ctx, stderr := cmdio.NewTestContextWithStderr(t.Context()) ctx = env.WithUserHomeDir(ctx, t.TempDir()) + ctx = telemetry.WithNewLogger(ctx) MaybeHint(ctx, regularCmd()) assert.Empty(t, stderr.String()) + assert.Empty(t, hintKinds(t, ctx)) } func TestMaybeHint_ClaudeNotInstalledHints(t *testing.T) { setClaudeAgent(t) ctx, stderr := claudeCtx(t, t.TempDir()) + ctx = telemetry.WithNewLogger(ctx) MaybeHint(ctx, regularCmd()) assert.Contains(t, stderr.String(), hintText) + assert.Equal(t, []protos.AitoolsHintKind{protos.AitoolsHintKindInstallText}, hintKinds(t, ctx)) } func TestMaybeHint_PluginInstalledSilent(t *testing.T) { @@ -97,9 +86,11 @@ func TestMaybeHint_PluginInstalledSilent(t *testing.T) { // The core regression: a plugin install writes only Claude's manifest, no // skills subdir. Detection must treat it as installed. writePluginManifest(t, home, `{"plugins":{"databricks@claude-plugins-official":[{"scope":"user"}]}}`) + ctx = telemetry.WithNewLogger(ctx) MaybeHint(ctx, regularCmd()) assert.Empty(t, stderr.String()) + assert.Empty(t, hintKinds(t, ctx)) } func TestClaudeToolingInstalled_LogsPluginVersion(t *testing.T) { @@ -191,6 +182,7 @@ func TestMaybeHint_NonDatabricksSkillStillHints(t *testing.T) { func TestMaybeHint_AIToolsCommandSilent(t *testing.T) { setClaudeAgent(t) ctx, stderr := claudeCtx(t, t.TempDir()) + ctx = telemetry.WithNewLogger(ctx) parent := &cobra.Command{Use: aiToolsCommandName} child := &cobra.Command{Use: "install"} @@ -198,6 +190,7 @@ func TestMaybeHint_AIToolsCommandSilent(t *testing.T) { MaybeHint(ctx, child) assert.Empty(t, stderr.String()) + assert.Empty(t, hintKinds(t, ctx)) } func TestMaybeHint_NoIONoPanic(t *testing.T) { @@ -205,7 +198,9 @@ func TestMaybeHint_NoIONoPanic(t *testing.T) { // Bare context without cmdio installed: cobra resolves --help and bare // invocations before PersistentPreRunE runs, so HasIO can be false here. ctx := env.WithUserHomeDir(t.Context(), t.TempDir()) + ctx = telemetry.WithNewLogger(ctx) assert.NotPanics(t, func() { MaybeHint(ctx, regularCmd()) }) + assert.Empty(t, hintKinds(t, ctx)) } func TestMaybeHint_ThrottledPerSession(t *testing.T) { @@ -217,15 +212,17 @@ func TestMaybeHint_ThrottledPerSession(t *testing.T) { ctx = env.WithUserHomeDir(ctx, home) ctx = env.Set(ctx, "DATABRICKS_CACHE_DIR", cacheDir) ctx = env.Set(ctx, claudeSessionEnv, "session-abc") - return ctx, stderr + return telemetry.WithNewLogger(ctx), stderr } ctx1, stderr1 := newCtx() MaybeHint(ctx1, regularCmd()) assert.Contains(t, stderr1.String(), hintText) + assert.Equal(t, []protos.AitoolsHintKind{protos.AitoolsHintKindInstallText}, hintKinds(t, ctx1)) - // Same session id, second command: must stay quiet. + // Same session id, second command: must stay quiet and log nothing. ctx2, stderr2 := newCtx() MaybeHint(ctx2, regularCmd()) assert.Empty(t, stderr2.String()) + assert.Empty(t, hintKinds(t, ctx2)) } diff --git a/libs/aitools/agents/recommend.go b/libs/aitools/agents/recommend.go index de906752316..0acc577c6c7 100644 --- a/libs/aitools/agents/recommend.go +++ b/libs/aitools/agents/recommend.go @@ -5,20 +5,31 @@ import ( "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/telemetry/protos" ) -// RecommendSkillsInstall checks if coding agents are detected but have no skills installed. +// PrintSkillsTip prints the non-interactive `apps init` tip to install skills +// and records that it was shown. Callers decide whether the tooling is missing +// (see HasDatabricksToolingInstalled). +func PrintSkillsTip(ctx context.Context) { + cmdio.LogString(ctx, "Tip: coding agents detected without Databricks skills. Run 'databricks aitools install' to install them.") + logHintShown(ctx, protos.AitoolsHintKindAppsInitTip) +} + +// RecommendSkillsInstall checks if coding agents are detected without Databricks tooling (skills +// or the databricks plugin) installed. // In interactive mode, prompts the user to install now using installFn. In non-interactive mode, prints a hint. func RecommendSkillsInstall(ctx context.Context, installFn func(context.Context) error) error { - if HasDatabricksSkillsInstalled(ctx) { + if HasDatabricksToolingInstalled(ctx) { return nil } if !cmdio.IsPromptSupported(ctx) { - cmdio.LogString(ctx, "Tip: coding agents detected without Databricks skills. Run 'databricks aitools install' to install them.") + PrintSkillsTip(ctx) return nil } + logHintShown(ctx, protos.AitoolsHintKindAppsInitPrompt) yes, err := cmdio.AskYesOrNo(ctx, "Coding agents detected without Databricks skills. Install skills now?") if err != nil { return err diff --git a/libs/aitools/agents/recommend_test.go b/libs/aitools/agents/recommend_test.go index e6952256627..9ecae2618d4 100644 --- a/libs/aitools/agents/recommend_test.go +++ b/libs/aitools/agents/recommend_test.go @@ -8,6 +8,8 @@ import ( "testing" "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/telemetry" + "github.com/databricks/cli/libs/telemetry/protos" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -34,9 +36,28 @@ func TestRecommendSkillsInstallSkipsWhenSkillsExist(t *testing.T) { } defer func() { Registry = origRegistry }() - ctx := cmdio.MockDiscard(t.Context()) + ctx := telemetry.WithNewLogger(cmdio.MockDiscard(t.Context())) + err := RecommendSkillsInstall(ctx, noopInstall) + assert.NoError(t, err) + assert.Empty(t, hintKinds(t, ctx)) +} + +func TestRecommendSkillsInstallSkipsWhenPluginInstalled(t *testing.T) { + tmpDir := t.TempDir() + t.Setenv("HOME", tmpDir) + t.Setenv("USERPROFILE", tmpDir) + writeManifest(t, tmpDir, `{"plugins":{"databricks@claude-plugins-official":[{"version":"0.2.6"}]}}`) + + origRegistry := Registry + Registry = []*Agent{agentWithConfigDir(tmpDir)} + defer func() { Registry = origRegistry }() + + ctx, stderr := cmdio.NewTestContextWithStderr(t.Context()) + ctx = telemetry.WithNewLogger(ctx) err := RecommendSkillsInstall(ctx, noopInstall) assert.NoError(t, err) + assert.Empty(t, stderr.String()) + assert.Empty(t, hintKinds(t, ctx)) } func TestRecommendSkillsInstallSkipsWhenNoAgents(t *testing.T) { @@ -69,9 +90,11 @@ func TestRecommendSkillsInstallNonInteractive(t *testing.T) { defer func() { Registry = origRegistry }() ctx, stderr := cmdio.NewTestContextWithStderr(t.Context()) + ctx = telemetry.WithNewLogger(ctx) err := RecommendSkillsInstall(ctx, noopInstall) require.NoError(t, err) assert.Contains(t, stderr.String(), "databricks aitools install") + assert.Equal(t, []protos.AitoolsHintKind{protos.AitoolsHintKindAppsInitTip}, hintKinds(t, ctx)) } func TestRecommendSkillsInstallInteractiveDecline(t *testing.T) { @@ -91,6 +114,7 @@ func TestRecommendSkillsInstallInteractiveDecline(t *testing.T) { ctx, testIO := cmdio.SetupTest(t.Context(), cmdio.TestOptions{PromptSupported: true}) defer testIO.Done() + ctx = telemetry.WithNewLogger(ctx) // Drain stderr so the prompt write doesn't block. go func() { _, _ = io.Copy(io.Discard, testIO.Stderr) }() @@ -105,4 +129,6 @@ func TestRecommendSkillsInstallInteractiveDecline(t *testing.T) { require.NoError(t, testIO.Stdin.Flush()) assert.NoError(t, <-errc) + // The prompt is logged when shown, so a declined prompt still counts. + assert.Equal(t, []protos.AitoolsHintKind{protos.AitoolsHintKindAppsInitPrompt}, hintKinds(t, ctx)) } diff --git a/libs/aitools/agents/skills.go b/libs/aitools/agents/skills.go index b575f825c2b..177f0fcfdf6 100644 --- a/libs/aitools/agents/skills.go +++ b/libs/aitools/agents/skills.go @@ -20,15 +20,34 @@ const ( LegacySkillsDir = ".databricks/agent-skills" ) -// HasDatabricksSkillsInstalled checks if Databricks skills are installed in the canonical location. -// Returns true if no agents are detected (nothing to recommend) or if skills exist in -// ~/.databricks/aitools/skills/ or the legacy ~/.databricks/agent-skills/. -func HasDatabricksSkillsInstalled(ctx context.Context) bool { +// HasDatabricksToolingInstalled reports whether every detected agent already +// has Databricks tooling, so `apps init` has nothing to recommend. Skills in +// the shared canonical (or legacy) dir cover every agent. Otherwise each +// detected agent needs the databricks plugin in its own plugin manifest, +// installed for every project or for the working directory (see +// pluginInstall.appliesTo): a plugin installed for another project does not +// load for the app being created. It also reports true when no agent is +// detected. +func HasDatabricksToolingInstalled(ctx context.Context) bool { installed := DetectInstalled(ctx) - if len(installed) == 0 { + if len(installed) == 0 || hasSharedSkills(ctx) { return true } + cwd := "" + if wd, err := os.Getwd(); err == nil { + cwd = resolvePath(wd) + } + for _, a := range installed { + if a.pluginInstalls == nil || !a.databricksPluginActiveIn(ctx, cwd) { + return false + } + } + return true +} +// hasSharedSkills reports whether Databricks skills are in the canonical +// skills dir or the legacy one. +func hasSharedSkills(ctx context.Context) bool { homeDir, err := env.UserHomeDir(ctx) if err != nil { return false diff --git a/libs/aitools/agents/skills_test.go b/libs/aitools/agents/skills_test.go index 4f58438b8b1..26ead3272c2 100644 --- a/libs/aitools/agents/skills_test.go +++ b/libs/aitools/agents/skills_test.go @@ -2,6 +2,7 @@ package agents import ( "context" + "fmt" "os" "path/filepath" "testing" @@ -10,15 +11,15 @@ import ( "github.com/stretchr/testify/require" ) -func TestHasDatabricksSkillsInstalledNoAgents(t *testing.T) { +func TestHasDatabricksToolingInstalledNoAgents(t *testing.T) { origRegistry := Registry Registry = []*Agent{} defer func() { Registry = origRegistry }() - assert.True(t, HasDatabricksSkillsInstalled(t.Context())) + assert.True(t, HasDatabricksToolingInstalled(t.Context())) } -func TestHasDatabricksSkillsInstalledCanonicalOnly(t *testing.T) { +func TestHasDatabricksToolingInstalledCanonicalOnly(t *testing.T) { tmpHome := t.TempDir() t.Setenv("HOME", tmpHome) t.Setenv("USERPROFILE", tmpHome) @@ -34,10 +35,10 @@ func TestHasDatabricksSkillsInstalledCanonicalOnly(t *testing.T) { } defer func() { Registry = origRegistry }() - assert.True(t, HasDatabricksSkillsInstalled(t.Context())) + assert.True(t, HasDatabricksToolingInstalled(t.Context())) } -func TestHasDatabricksSkillsInstalledIgnoresAgentDir(t *testing.T) { +func TestHasDatabricksToolingInstalledIgnoresAgentDir(t *testing.T) { tmpHome := t.TempDir() t.Setenv("HOME", tmpHome) t.Setenv("USERPROFILE", tmpHome) @@ -55,10 +56,10 @@ func TestHasDatabricksSkillsInstalledIgnoresAgentDir(t *testing.T) { } defer func() { Registry = origRegistry }() - assert.False(t, HasDatabricksSkillsInstalled(t.Context())) + assert.False(t, HasDatabricksToolingInstalled(t.Context())) } -func TestHasDatabricksSkillsInstalledWithOnlyNonDatabricksSkills(t *testing.T) { +func TestHasDatabricksToolingInstalledWithOnlyNonDatabricksSkills(t *testing.T) { tmpDir := t.TempDir() t.Setenv("HOME", tmpDir) t.Setenv("USERPROFILE", tmpDir) @@ -76,10 +77,10 @@ func TestHasDatabricksSkillsInstalledWithOnlyNonDatabricksSkills(t *testing.T) { } defer func() { Registry = origRegistry }() - assert.False(t, HasDatabricksSkillsInstalled(t.Context())) + assert.False(t, HasDatabricksToolingInstalled(t.Context())) } -func TestHasDatabricksSkillsInstalledNoSkillsDir(t *testing.T) { +func TestHasDatabricksToolingInstalledNoSkillsDir(t *testing.T) { tmpDir := t.TempDir() t.Setenv("HOME", tmpDir) t.Setenv("USERPROFILE", tmpDir) @@ -94,10 +95,10 @@ func TestHasDatabricksSkillsInstalledNoSkillsDir(t *testing.T) { } defer func() { Registry = origRegistry }() - assert.False(t, HasDatabricksSkillsInstalled(t.Context())) + assert.False(t, HasDatabricksToolingInstalled(t.Context())) } -func TestHasDatabricksSkillsInstalledCustomSubdirNotChecked(t *testing.T) { +func TestHasDatabricksToolingInstalledCustomSubdirNotChecked(t *testing.T) { tmpHome := t.TempDir() t.Setenv("HOME", tmpHome) t.Setenv("USERPROFILE", tmpHome) @@ -115,10 +116,10 @@ func TestHasDatabricksSkillsInstalledCustomSubdirNotChecked(t *testing.T) { } defer func() { Registry = origRegistry }() - assert.False(t, HasDatabricksSkillsInstalled(t.Context())) + assert.False(t, HasDatabricksToolingInstalled(t.Context())) } -func TestHasDatabricksSkillsInstalledDatabricksAppsCanonical(t *testing.T) { +func TestHasDatabricksToolingInstalledDatabricksAppsCanonical(t *testing.T) { tmpHome := t.TempDir() t.Setenv("HOME", tmpHome) t.Setenv("USERPROFILE", tmpHome) @@ -138,7 +139,7 @@ func TestHasDatabricksSkillsInstalledDatabricksAppsCanonical(t *testing.T) { } defer func() { Registry = origRegistry }() - assert.True(t, HasDatabricksSkillsInstalled(t.Context())) + assert.True(t, HasDatabricksToolingInstalled(t.Context())) } func TestHasDatabricksSkillsInFollowsSymlinks(t *testing.T) { @@ -165,7 +166,7 @@ func TestHasDatabricksSkillsInIgnoresDanglingSymlink(t *testing.T) { assert.False(t, HasDatabricksSkillsIn(skillsDir)) } -func TestHasDatabricksSkillsInstalledLegacyPath(t *testing.T) { +func TestHasDatabricksToolingInstalledLegacyPath(t *testing.T) { tmpHome := t.TempDir() t.Setenv("HOME", tmpHome) t.Setenv("USERPROFILE", tmpHome) @@ -185,5 +186,49 @@ func TestHasDatabricksSkillsInstalledLegacyPath(t *testing.T) { } defer func() { Registry = origRegistry }() - assert.True(t, HasDatabricksSkillsInstalled(t.Context())) + assert.True(t, HasDatabricksToolingInstalled(t.Context())) +} + +func TestHasDatabricksToolingInstalled(t *testing.T) { + const userPlugin = `{"plugins":{"databricks@claude-plugins-official":[{"scope":"user","version":"0.2.6"}]}}` + const otherProjectPlugin = `{"plugins":{"databricks@claude-plugins-official":[{"scope":"project","projectPath":"/work/other","version":"0.2.6"}]}}` + + for _, tc := range []struct { + name string + // manifests holds one plugin manifest per detected agent; "" means none. + manifests []string + skills bool + want bool + }{ + {name: "plugin for the only agent", manifests: []string{userPlugin}, want: true}, + {name: "plugin for every agent", manifests: []string{userPlugin, userPlugin}, want: true}, + {name: "plugin for one of two agents", manifests: []string{userPlugin, ""}, want: false}, + {name: "plugin for another project only", manifests: []string{otherProjectPlugin}, want: false}, + {name: "shared skills cover every agent", manifests: []string{userPlugin, ""}, skills: true, want: true}, + {name: "neither", manifests: []string{""}, want: false}, + } { + t.Run(tc.name, func(t *testing.T) { + tmpHome := t.TempDir() + t.Setenv("HOME", tmpHome) + t.Setenv("USERPROFILE", tmpHome) + if tc.skills { + require.NoError(t, os.MkdirAll(filepath.Join(tmpHome, CanonicalSkillsDir, "databricks"), 0o755)) + } + + var registry []*Agent + for i, manifest := range tc.manifests { + configDir := filepath.Join(tmpHome, fmt.Sprintf("agent-%d", i)) + require.NoError(t, os.MkdirAll(configDir, 0o755)) + if manifest != "" { + writeManifest(t, configDir, manifest) + } + registry = append(registry, agentWithConfigDir(configDir)) + } + origRegistry := Registry + Registry = registry + t.Cleanup(func() { Registry = origRegistry }) + + assert.Equal(t, tc.want, HasDatabricksToolingInstalled(t.Context())) + }) + } } diff --git a/libs/telemetry/logger.go b/libs/telemetry/logger.go index 509fdf65664..9ec42175c04 100644 --- a/libs/telemetry/logger.go +++ b/libs/telemetry/logger.go @@ -7,6 +7,7 @@ import ( "fmt" "net/http" "strconv" + "sync/atomic" "time" "uuid" @@ -17,6 +18,8 @@ import ( "github.com/databricks/cli/libs/telemetry/protos" "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/client" + "github.com/databricks/databricks-sdk-go/config" + "github.com/databricks/databricks-sdk-go/httpclient" ) const ( @@ -44,6 +47,10 @@ func Log(ctx context.Context, event protos.DatabricksCliLog) { type logger struct { logs []protos.FrontendLog stateLineageHash string + + // running counts the goroutines that Upload started and stopped waiting + // for, which may still be using the config (see within). + running atomic.Int32 } // SetStateLineageHash records the hashed lineage of the bundle state the command read, so that @@ -73,7 +80,14 @@ func (l *logger) log(event protos.DatabricksCliLog) { const ( defaultUploadTimeout = 3 * time.Second - waitBetweenRetries = 200 * time.Millisecond + + // hintOnlyUploadTimeout bounds an upload whose events are all install + // hint events. Most commands that show a hint log nothing else, so + // without the hint they would not upload at all; they wait less for it at + // exit. + hintOnlyUploadTimeout = 1 * time.Second + + waitBetweenRetries = 200 * time.Millisecond ) func Upload(ctx context.Context, ec protos.ExecutionContext) error { @@ -108,22 +122,19 @@ func Upload(ctx context.Context, ec protos.ExecutionContext) error { return nil } - apiClient, err := client.New(cmdctx.ConfigUsed(ctx)) - if err != nil { - return fmt.Errorf("failed to create API client: %w", err) + // The ssh agent shim uploads before it execs. If the exec fails, cmd/root + // uploads again at exit, while a goroutine of the timed-out first upload + // may still be resolving or authenticating the same config. + if l.running.Load() > 0 { + return errors.New("a previous telemetry upload is still running") } - uploadTimeout := defaultUploadTimeout - if v := env.Get(ctx, timeoutEnvVar); v != "" { - timeout, err := strconv.ParseFloat(v, 64) - if err != nil { - return fmt.Errorf("failed to parse timeout: %w", err) - } - - uploadTimeout = time.Duration(timeout * float64(time.Second)) + timeout, err := uploadTimeout(ctx, l.logs) + if err != nil { + return err } - ctx, cancel := context.WithTimeout(ctx, uploadTimeout) + ctx, cancel := context.WithTimeout(ctx, timeout) defer cancel() deadline, ok := ctx.Deadline() @@ -132,6 +143,13 @@ func Upload(ctx context.Context, ec protos.ExecutionContext) error { return nil } + apiClient, err := within(ctx, &l.running, func() (*client.DatabricksClient, error) { + return newAPIClient(cmdctx.ConfigUsed(ctx), &l.running) + }) + if err != nil { + return fmt.Errorf("failed to create API client: %w", err) + } + // Only try uploading logs for a maximum of 3 times. for i := range 3 { select { @@ -197,6 +215,100 @@ func Upload(ctx context.Context, ec protos.ExecutionContext) error { return errors.New("failed to upload telemetry logs after three attempts") } +// uploadTimeout returns how long Upload may take: DATABRICKS_CLI_TELEMETRY_TIMEOUT +// (in seconds) when set, else hintOnlyUploadTimeout when every log is an +// install hint event, else defaultUploadTimeout. +func uploadTimeout(ctx context.Context, logs []protos.FrontendLog) (time.Duration, error) { + if v := env.Get(ctx, timeoutEnvVar); v != "" { + timeout, err := strconv.ParseFloat(v, 64) + if err != nil { + return 0, fmt.Errorf("failed to parse timeout: %w", err) + } + return time.Duration(timeout * float64(time.Second)), nil + } + if onlyHintEvents(logs) { + return hintOnlyUploadTimeout, nil + } + return defaultUploadTimeout, nil +} + +// onlyHintEvents reports whether every log is an install hint event. +func onlyHintEvents(logs []protos.FrontendLog) bool { + for _, l := range logs { + if l.Entry.DatabricksCliLog.AitoolsHintEvent == nil { + return false + } + } + return true +} + +// within runs f on its own goroutine and returns its result, or ctx's error +// once ctx is done. Upload uses it for the SDK calls that ignore ctx: the SDK +// resolves the config (including host metadata discovery) and fetches auth +// tokens with context.Background(). The requests themselves honor ctx and run +// on the caller's goroutine. A panic in f is returned as an error, so it is +// logged like any other upload failure instead of crashing the process. +// +// After a timeout the goroutine keeps running until f returns or the process +// ends, and running counts it until then. It may still be writing to the +// config while resolving it, or refreshing an OAuth token that is lost if the +// process exits first. cmd/root runs Upload as the last step before the +// process exits; the ssh agent shim runs it just before exec, and Upload skips +// a second upload while running is non-zero. +func within[T any](ctx context.Context, running *atomic.Int32, f func() (T, error)) (T, error) { + type result struct { + v T + err error + } + resc := make(chan result, 1) + running.Add(1) + go func() { + defer running.Add(-1) + defer func() { + if r := recover(); r != nil { + resc <- result{err: fmt.Errorf("panic: %v", r)} + } + }() + v, err := f() + resc <- result{v, err} + }() + select { + case res := <-resc: + return res.v, res.err + case <-ctx.Done(): + var zero T + return zero, ctx.Err() + } +} + +// newAPIClient is client.New with authenticate as the auth visitor. The SDK +// runs the auth visitor on every request, including its own retries. +func newAPIClient(cfg *config.Config, running *atomic.Int32) (*client.DatabricksClient, error) { + clientCfg, err := config.HTTPClientConfigFromConfig(cfg) + if err != nil { + return nil, err + } + clientCfg.AuthVisitor = func(r *http.Request) error { + return authenticate(r, cfg, running) + } + return client.NewWithClient(cfg, httpclient.NewApiClient(clientCfg)) +} + +// authenticate sets the auth headers on r, giving up when r's context, which +// carries the upload deadline, is done. It authenticates a clone so that an +// abandoned goroutine never writes to r. +func authenticate(r *http.Request, cfg *config.Config, running *atomic.Int32) error { + authReq := r.Clone(r.Context()) + _, err := within(r.Context(), running, func() (struct{}, error) { + return struct{}{}, cfg.Authenticate(authReq) + }) + if err != nil { + return fmt.Errorf("authenticating telemetry upload: %w", err) + } + r.Header = authReq.Header + return nil +} + func attempt(ctx context.Context, apiClient *client.DatabricksClient, protoLogs []string) (*ResponseBody, error) { // Without the workspace routing header, telemetry on unified hosts is // recorded in a central shard instead of the user's workspace. @@ -219,3 +331,18 @@ func attempt(ctx context.Context, apiClient *client.DatabricksClient, protoLogs return resp, nil } + +// BufferedEvents returns the events logged on ctx, or nil when ctx has no +// logger. Upload does not clear them. Tests in other packages use it to assert +// what a command logs without uploading. +func BufferedEvents(ctx context.Context) []protos.DatabricksCliLog { + l, ok := loggerFromContext(ctx) + if !ok { + return nil + } + out := make([]protos.DatabricksCliLog, len(l.logs)) + for i, fl := range l.logs { + out[i] = fl.Entry.DatabricksCliLog + } + return out +} diff --git a/libs/telemetry/logger_test.go b/libs/telemetry/logger_test.go index abb7c198073..94673a9dd1a 100644 --- a/libs/telemetry/logger_test.go +++ b/libs/telemetry/logger_test.go @@ -1,15 +1,22 @@ package telemetry import ( + "context" + "errors" + "sync/atomic" "testing" + "time" "github.com/databricks/cli/libs/cmdctx" + "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/telemetry/protos" "github.com/databricks/cli/libs/testserver" "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/config" + "github.com/databricks/databricks-sdk-go/config/experimental/auth" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "golang.org/x/oauth2" ) func TestTelemetryUploadRetriesOnPartialSuccess(t *testing.T) { @@ -153,6 +160,127 @@ func TestTelemetryUploadMaxRetries(t *testing.T) { assert.Equal(t, 3, count) } +// The SDK fetches tokens with context.Background(), so a token source that +// retries ignores the deadline Upload passes to the request. The upload +// timeout must still bound Upload. +func TestTelemetryUploadTimeoutBoundsTokenAcquisition(t *testing.T) { + server := testserver.New(t) + t.Cleanup(server.Close) + + release := make(chan struct{}) + t.Cleanup(func() { close(release) }) + ts := auth.TokenSourceFn(func(context.Context) (*oauth2.Token, error) { + <-release + return nil, errors.New("token endpoint unavailable") + }) + + ctx := WithNewLogger(t.Context()) + ctx = env.Set(ctx, timeoutEnvVar, "0.1") + Log(ctx, protos.DatabricksCliLog{ + CliTestEvent: &protos.CliTestEvent{ + Name: protos.DummyCliEnumValue1, + }, + }) + ctx = cmdctx.SetConfigUsed(ctx, &config.Config{ + Host: server.URL, + Credentials: config.NewTokenSourceStrategy("blocking", ts), + }) + + assertUploadTimesOut(t, ctx) +} + +// A token that expires between attempts is fetched again on the retry; that +// fetch must be bounded by the upload timeout too. +func TestTelemetryUploadTimeoutBoundsTokenRefreshOnRetry(t *testing.T) { + server := testserver.New(t) + t.Cleanup(server.Close) + + // The token expires 500ms after it is issued, so the time Upload takes to + // get the first token does not matter. + var expiry atomic.Pointer[time.Time] + var count atomic.Int32 + server.Handle("POST", "/telemetry-ext", func(req testserver.Request) any { + count.Add(1) + // Fail the first attempt once the token has expired, so the retry + // has to fetch a new one. + time.Sleep(time.Until(*expiry.Load()) + 50*time.Millisecond) + return testserver.Response{ + StatusCode: 500, + Body: apierr.APIError{StatusCode: 500, Message: "Some error"}, + } + }) + + release := make(chan struct{}) + t.Cleanup(func() { close(release) }) + var calls atomic.Int32 + ts := auth.TokenSourceFn(func(context.Context) (*oauth2.Token, error) { + if calls.Add(1) == 1 { + e := time.Now().Add(500 * time.Millisecond) + expiry.Store(&e) + return &oauth2.Token{AccessToken: "token", Expiry: e}, nil + } + <-release + return nil, errors.New("token endpoint unavailable") + }) + + ctx := WithNewLogger(t.Context()) + ctx = env.Set(ctx, timeoutEnvVar, "2") + Log(ctx, protos.DatabricksCliLog{ + CliTestEvent: &protos.CliTestEvent{ + Name: protos.DummyCliEnumValue1, + }, + }) + ctx = cmdctx.SetConfigUsed(ctx, &config.Config{ + Host: server.URL, + Credentials: config.NewTokenSourceStrategy("expiring", ts), + }) + + assertUploadTimesOut(t, ctx) + assert.Equal(t, int32(1), count.Load()) +} + +// A config that the command never resolved is resolved when Upload creates +// its client, which includes host metadata discovery with +// context.Background(). The upload timeout must bound that too. +func TestTelemetryUploadTimeoutBoundsConfigResolution(t *testing.T) { + server := testserver.New(t) + t.Cleanup(server.Close) + + release := make(chan struct{}) + t.Cleanup(func() { close(release) }) + + ctx := WithNewLogger(t.Context()) + ctx = env.Set(ctx, timeoutEnvVar, "0.1") + Log(ctx, protos.DatabricksCliLog{ + CliTestEvent: &protos.CliTestEvent{ + Name: protos.DummyCliEnumValue1, + }, + }) + ctx = cmdctx.SetConfigUsed(ctx, &config.Config{ + Host: server.URL, + Token: "token", + HostMetadataResolver: func(context.Context, string) (*config.HostMetadata, error) { + <-release + return nil, errors.New("host unreachable") + }, + }) + + assertUploadTimesOut(t, ctx) +} + +// assertUploadTimesOut runs Upload and asserts it fails with its timeout, +// failing the test instead of hanging if Upload does not return. +func assertUploadTimesOut(t *testing.T, ctx context.Context) { + errc := make(chan error, 1) + go func() { errc <- Upload(ctx, protos.ExecutionContext{}) }() + select { + case err := <-errc: + assert.ErrorIs(t, err, context.DeadlineExceeded) + case <-time.After(10 * time.Second): + t.Fatal("Upload did not return after its timeout") + } +} + func TestLogWithoutLoggerDropsEvent(t *testing.T) { // A context without a telemetry logger must not panic; the event is dropped. assert.NotPanics(t, func() { @@ -176,3 +304,164 @@ func TestStateLineageHashWithoutLogger(t *testing.T) { SetStateLineageHash(t.Context(), "G0ZNUjK18Pa") assert.Empty(t, StateLineageHash(t.Context())) } + +func TestBufferedEvents(t *testing.T) { + assert.Nil(t, BufferedEvents(t.Context())) + + ctx := WithNewLogger(t.Context()) + assert.Empty(t, BufferedEvents(ctx)) + Log(ctx, protos.DatabricksCliLog{AitoolsHintEvent: &protos.AitoolsHintEvent{HintKind: protos.AitoolsHintKindInstallText}}) + got := BufferedEvents(ctx) + require.Len(t, got, 1) + assert.Equal(t, protos.AitoolsHintKindInstallText, got[0].AitoolsHintEvent.HintKind) +} + +func TestUploadTimeout(t *testing.T) { + hint := protos.FrontendLog{Entry: protos.FrontendLogEntry{DatabricksCliLog: protos.DatabricksCliLog{ + AitoolsHintEvent: &protos.AitoolsHintEvent{HintKind: protos.AitoolsHintKindInstallText}, + }}} + other := protos.FrontendLog{Entry: protos.FrontendLogEntry{DatabricksCliLog: protos.DatabricksCliLog{ + CliTestEvent: &protos.CliTestEvent{Name: protos.DummyCliEnumValue1}, + }}} + + for _, tc := range []struct { + name string + env string + logs []protos.FrontendLog + want time.Duration + }{ + {name: "only hint events", logs: []protos.FrontendLog{hint, hint}, want: hintOnlyUploadTimeout}, + {name: "hint and other events", logs: []protos.FrontendLog{hint, other}, want: defaultUploadTimeout}, + {name: "other events", logs: []protos.FrontendLog{other}, want: defaultUploadTimeout}, + {name: "env overrides hint events", env: "0.5", logs: []protos.FrontendLog{hint}, want: 500 * time.Millisecond}, + {name: "env overrides other events", env: "5", logs: []protos.FrontendLog{other}, want: 5 * time.Second}, + } { + t.Run(tc.name, func(t *testing.T) { + ctx := env.Set(t.Context(), timeoutEnvVar, tc.env) + got, err := uploadTimeout(ctx, tc.logs) + require.NoError(t, err) + assert.Equal(t, tc.want, got) + }) + } + + _, err := uploadTimeout(env.Set(t.Context(), timeoutEnvVar, "soon"), []protos.FrontendLog{other}) + assert.ErrorContains(t, err, "failed to parse timeout") +} + +// An upload of only hint events gives up after hintOnlyUploadTimeout, not +// the default timeout. +func TestTelemetryUploadHintOnlyUsesShortTimeout(t *testing.T) { + server := testserver.New(t) + t.Cleanup(server.Close) + release := make(chan struct{}) + t.Cleanup(func() { close(release) }) + server.Handle("POST", "/telemetry-ext", func(req testserver.Request) any { + <-release + return ResponseBody{NumProtoSuccess: 1} + }) + + ctx := WithNewLogger(t.Context()) + ctx = env.Set(ctx, timeoutEnvVar, "") + Log(ctx, protos.DatabricksCliLog{ + AitoolsHintEvent: &protos.AitoolsHintEvent{HintKind: protos.AitoolsHintKindInstallText}, + }) + ctx = cmdctx.SetConfigUsed(ctx, &config.Config{Host: server.URL, Token: "token"}) + + start := time.Now() + err := Upload(ctx, protos.ExecutionContext{}) + assert.ErrorIs(t, err, context.DeadlineExceeded) + assert.Less(t, time.Since(start), defaultUploadTimeout) +} + +// A hint-only upload runs for a config that never authenticated: inside a +// bundle directory the config stored for telemetry is not the one the +// command's client authenticated, so it has no resolved credentials yet. +func TestTelemetryUploadHintOnlyWithUnresolvedConfig(t *testing.T) { + server := testserver.New(t) + t.Cleanup(server.Close) + var calls atomic.Int32 + server.Handle("POST", "/telemetry-ext", func(req testserver.Request) any { + calls.Add(1) + return ResponseBody{NumProtoSuccess: 1} + }) + + ctx := WithNewLogger(t.Context()) + ctx = env.Set(ctx, timeoutEnvVar, "") + Log(ctx, protos.DatabricksCliLog{ + AitoolsHintEvent: &protos.AitoolsHintEvent{HintKind: protos.AitoolsHintKindInstallText}, + }) + ctx = cmdctx.SetConfigUsed(ctx, &config.Config{Host: server.URL, Token: "token"}) + + require.NoError(t, Upload(ctx, protos.ExecutionContext{})) + assert.Equal(t, int32(1), calls.Load()) +} + +// A hint-only upload whose credentials keep failing (an auth type preset by +// the profile, with a token source that cannot refresh) gives up within the +// hint-only timeout and sends nothing. +func TestTelemetryUploadHintOnlyWithFailingAuthIsBounded(t *testing.T) { + server := testserver.New(t) + t.Cleanup(server.Close) + var calls atomic.Int32 + server.Handle("POST", "/telemetry-ext", func(req testserver.Request) any { + calls.Add(1) + return ResponseBody{NumProtoSuccess: 1} + }) + + release := make(chan struct{}) + t.Cleanup(func() { close(release) }) + ts := auth.TokenSourceFn(func(context.Context) (*oauth2.Token, error) { + <-release + return nil, errors.New("refresh token revoked") + }) + + ctx := WithNewLogger(t.Context()) + ctx = env.Set(ctx, timeoutEnvVar, "") + Log(ctx, protos.DatabricksCliLog{ + AitoolsHintEvent: &protos.AitoolsHintEvent{HintKind: protos.AitoolsHintKindInstallText}, + }) + ctx = cmdctx.SetConfigUsed(ctx, &config.Config{ + Host: server.URL, + AuthType: "databricks-cli", + Credentials: config.NewTokenSourceStrategy("databricks-cli", ts), + }) + + start := time.Now() + err := Upload(ctx, protos.ExecutionContext{}) + assert.ErrorIs(t, err, context.DeadlineExceeded) + assert.Less(t, time.Since(start), defaultUploadTimeout) + assert.Equal(t, int32(0), calls.Load()) +} + +// While a timed-out upload's goroutine may still use the config, a second +// upload on the same logger does not touch it. +func TestTelemetryUploadSkipsWhileEarlierUploadRuns(t *testing.T) { + server := testserver.New(t) + t.Cleanup(server.Close) + + release := make(chan struct{}) + t.Cleanup(func() { close(release) }) + ts := auth.TokenSourceFn(func(context.Context) (*oauth2.Token, error) { + <-release + return nil, errors.New("token endpoint unavailable") + }) + + ctx := WithNewLogger(t.Context()) + ctx = env.Set(ctx, timeoutEnvVar, "0.1") + Log(ctx, protos.DatabricksCliLog{CliTestEvent: &protos.CliTestEvent{Name: protos.DummyCliEnumValue1}}) + ctx = cmdctx.SetConfigUsed(ctx, &config.Config{ + Host: server.URL, + Credentials: config.NewTokenSourceStrategy("blocking", ts), + }) + + assertUploadTimesOut(t, ctx) + err := Upload(ctx, protos.ExecutionContext{}) + assert.ErrorContains(t, err, "a previous telemetry upload is still running") +} + +func TestWithinReturnsPanicAsError(t *testing.T) { + var running atomic.Int32 + _, err := within(t.Context(), &running, func() (int, error) { panic("boom") }) + assert.EqualError(t, err, "panic: boom") + assert.Eventually(t, func() bool { return running.Load() == 0 }, time.Second, time.Millisecond) +} diff --git a/libs/telemetry/protos/README.md b/libs/telemetry/protos/README.md index 67964fc89d2..c82d4bc5589 100644 --- a/libs/telemetry/protos/README.md +++ b/libs/telemetry/protos/README.md @@ -5,3 +5,11 @@ You can find all lumberjack protos for the Databricks CLI in the `proto/logs/fro which config options were set, the outcome and latency, and how long packaging and uploading the code snapshot took. It never includes code, paths, names, parameters, or image references. Set `DATABRICKS_CLI_DISABLE_TELEMETRY=1` to opt out. + +The CLI logs an `aitools_hint_event` each time it prints the Claude Code install hint (any +non-aitools command) or the `apps init` skills tip or install prompt. It carries only the kind of +hint (`hint_kind`); the command it was shown on is the entry's `execution_context.command`. The +event uploads at exit only when the command resolved auth. The install hint prints at most once per +Claude Code session per hour, so when the command that printed it resolves no auth (e.g. +`databricks version`), that session logs no install hint event for the next hour even though the +agent saw the hint. diff --git a/libs/telemetry/protos/aitools_events.go b/libs/telemetry/protos/aitools_events.go new file mode 100644 index 00000000000..bc4db8f2948 --- /dev/null +++ b/libs/telemetry/protos/aitools_events.go @@ -0,0 +1,34 @@ +package protos + +// AitoolsHintKind mirrors AitoolsHintKind.Type in the databricks_cli lumberjack +// proto: which recommendation to install Databricks AI tools was shown. +type AitoolsHintKind string + +const ( + AitoolsHintKindUnspecified AitoolsHintKind = "TYPE_UNSPECIFIED" + // AitoolsHintKindInstallText is the generic stderr line asking the agent to + // run `databricks aitools install`. + AitoolsHintKindInstallText AitoolsHintKind = "INSTALL_TEXT" + // AitoolsHintKindClaudeCodeNative and AitoolsHintKindAgentInstallText + // mirror enum values of the proto for hints the CLI does not print. + // Nothing logs them today. + AitoolsHintKindClaudeCodeNative AitoolsHintKind = "CLAUDE_CODE_NATIVE" + AitoolsHintKindAgentInstallText AitoolsHintKind = "AGENT_INSTALL_TEXT" + // AitoolsHintKindAppsInitTip is the non-interactive tip printed by + // `databricks apps init`. + AitoolsHintKindAppsInitTip AitoolsHintKind = "APPS_INIT_TIP" + // AitoolsHintKindAppsInitPrompt is the interactive "install skills now?" + // prompt shown by `databricks apps init`. + AitoolsHintKindAppsInitPrompt AitoolsHintKind = "APPS_INIT_PROMPT" +) + +// AitoolsHintEvent is emitted once each time the CLI prints the Claude Code +// install hint or the `apps init` skills tip or install prompt; other messages +// that mention `databricks aitools install` log nothing. It carries no +// user-authored strings. The command it was shown on is ExecutionContext.Command +// of the enclosing log entry. The CLI sets no calling-agent field on this +// event: the calling agent is reported by the User-Agent of the telemetry +// upload (`agent/`). +type AitoolsHintEvent struct { + HintKind AitoolsHintKind `json:"hint_kind"` +} diff --git a/libs/telemetry/protos/aitools_events_test.go b/libs/telemetry/protos/aitools_events_test.go new file mode 100644 index 00000000000..78b3a903707 --- /dev/null +++ b/libs/telemetry/protos/aitools_events_test.go @@ -0,0 +1,19 @@ +package protos + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The lumberjack proto keys fields in snake_case; a wrong tag is dropped +// silently on ingestion. This pins the wire keys of the aitools hint event. +func TestAitoolsEventsWireKeys(t *testing.T) { + b, err := json.Marshal(DatabricksCliLog{ + AitoolsHintEvent: &AitoolsHintEvent{HintKind: AitoolsHintKindInstallText}, + }) + require.NoError(t, err) + assert.JSONEq(t, `{"aitools_hint_event":{"hint_kind":"INSTALL_TEXT"}}`, string(b)) +} diff --git a/libs/telemetry/protos/frontend_log.go b/libs/telemetry/protos/frontend_log.go index 197856bab05..355600e1896 100644 --- a/libs/telemetry/protos/frontend_log.go +++ b/libs/telemetry/protos/frontend_log.go @@ -26,4 +26,5 @@ type DatabricksCliLog struct { AitoolsInstallEvent *AitoolsInstallEvent `json:"aitools_install_event,omitempty"` SetupLocalEvent *SetupLocalEvent `json:"setup_local_event,omitempty"` AirRunEvent *AirRunEvent `json:"air_run_event,omitempty"` + AitoolsHintEvent *AitoolsHintEvent `json:"aitools_hint_event,omitempty"` } diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index 539a21fafa5..090eab4084a 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -608,10 +608,20 @@ func AddDefaultHandlers(server *Server) { return server.fakeOidc.OidcDeviceAuth(req) }) - server.Handle("POST", "/telemetry-ext", func(_ Request) any { + server.Handle("POST", "/telemetry-ext", func(req Request) any { + var request struct { + ProtoLogs []string `json:"protoLogs"` + } + if err := json.Unmarshal(req.Body, &request); err != nil { + return Response{ + Body: fmt.Sprintf("invalid request body: %s", err), + StatusCode: http.StatusBadRequest, + } + } + // Accept every log in the request, as the real endpoint does. return map[string]any{ "errors": []string{}, - "numProtoSuccess": 1, + "numProtoSuccess": len(request.ProtoLogs), } })