Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/cli/aitools-telemetry-funnel.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* `databricks aitools` commands now send usage telemetry, and the CLI records when it recommends installing Databricks AI tools. When the configured workspace is slow or unreachable, `aitools install`, `update` and `uninstall` can take up to about 6 seconds longer, and `list`, `version` and runs that stop early up to about 3 seconds. Set `DATABRICKS_CLI_DISABLE_TELEMETRY=1` to opt out. ([#7467](https://github.com/databricks/cli/pull/7467), [#7469](https://github.com/databricks/cli/pull/7469), [#7470](https://github.com/databricks/cli/pull/7470))
2 changes: 1 addition & 1 deletion acceptance/cmd/aitools/list/explicit-ref/output.txt
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@

=== Only the skills manifest reaches the mock server: no latest-release lookup

>>> print_requests.py --get
>>> print_requests.py --get ^//telemetry-ext ^//.well-known
{
"method": "GET",
"path": "/test-ref/manifest.json"
Expand Down
2 changes: 1 addition & 1 deletion acceptance/cmd/aitools/list/explicit-ref/script
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,4 @@ sethome home
trace $CLI aitools list --output json | list_fields

title "Only the skills manifest reaches the mock server: no latest-release lookup"
trace print_requests.py --get
trace print_requests.py --get '^//telemetry-ext' '^//.well-known'
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ Error: resolve latest skills release: HTTP 403 fetching latest skills release

=== The lookup fails, so the manifest is never fetched

>>> print_requests.py --get
>>> print_requests.py --get ^//telemetry-ext ^//.well-known
{
"method": "GET",
"path": "/repos/databricks/databricks-agent-skills/releases/latest"
Expand Down
2 changes: 1 addition & 1 deletion acceptance/cmd/aitools/list/latest-lookup-fails/script
Original file line number Diff line number Diff line change
Expand Up @@ -7,4 +7,4 @@ use_compat_manifest
trace musterr $CLI aitools list --output json

title "The lookup fails, so the manifest is never fetched"
trace print_requests.py --get
trace print_requests.py --get '^//telemetry-ext' '^//.well-known'
2 changes: 1 addition & 1 deletion acceptance/cmd/aitools/list/latest/output.txt
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@

=== The latest-release lookup picks the manifest version

>>> print_requests.py --get
>>> print_requests.py --get ^//telemetry-ext ^//.well-known
{
"method": "GET",
"path": "/repos/databricks/databricks-agent-skills/releases/latest"
Expand Down
2 changes: 1 addition & 1 deletion acceptance/cmd/aitools/list/latest/script
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@ use_compat_manifest
trace $CLI aitools list --output json | list_fields

title "The latest-release lookup picks the manifest version"
trace print_requests.py --get
trace print_requests.py --get '^//telemetry-ext' '^//.well-known'
2 changes: 1 addition & 1 deletion acceptance/cmd/aitools/list/pinned/output.txt
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@

=== A pinned version skips the latest-release lookup

>>> print_requests.py --get
>>> print_requests.py --get ^//telemetry-ext ^//.well-known
{
"method": "GET",
"path": "/v0.2.32/manifest.json"
Expand Down
2 changes: 1 addition & 1 deletion acceptance/cmd/aitools/list/pinned/script
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@ use_compat_manifest
trace $CLI aitools list --output json | list_fields

title "A pinned version skips the latest-release lookup"
trace print_requests.py --get
trace print_requests.py --get '^//telemetry-ext' '^//.well-known'
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Cloud = false
18 changes: 18 additions & 0 deletions acceptance/cmd/aitools/telemetry/update-uninstall/output.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@

=== update: STARTED, FINISHED with SUCCESS

>>> [CLI] aitools update --scope=global
Already up to date.

>>> print_telemetry
{"command":"aitools_update","exit_code":0,"aitools_command_event":{"command":"UPDATE","stage":"STARTED","interactive":false}}
{"command":"aitools_update","exit_code":0,"aitools_command_event":{"command":"UPDATE","stage":"FINISHED","outcome":"SUCCESS","interactive":false,"scope":"GLOBAL"}}

=== uninstall: STARTED, FINISHED with SUCCESS

>>> [CLI] aitools uninstall --scope=global
Uninstalled 1 skill.

>>> print_telemetry
{"command":"aitools_uninstall","exit_code":0,"aitools_command_event":{"command":"UNINSTALL","stage":"STARTED","interactive":false}}
{"command":"aitools_uninstall","exit_code":0,"aitools_command_event":{"command":"UNINSTALL","stage":"FINISHED","outcome":"SUCCESS","interactive":false,"scope":"GLOBAL"}}
20 changes: 20 additions & 0 deletions acceptance/cmd/aitools/telemetry/update-uninstall/script
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
# Isolate HOME so parallel aitools tests don't race on a shared ~/.databricks.
sethome home
mkdir -p "${USERPROFILE:-$HOME}/.cursor"

# Install one skill to update and uninstall. Its funnel is pinned by ../install;
# print_telemetry here only clears its upload.
$CLI aitools install --skills-only --scope=global --agents=cursor &> LOG.install
print_telemetry &> LOG.install-telemetry

title "update: STARTED, FINISHED with SUCCESS"
trace $CLI aitools update --scope=global
trace print_telemetry

# list, version, and update and uninstall with nothing installed upload their
# events only if auth has already resolved when they finish, so whether they
# upload here depends on timing; their funnels are covered by unit tests.

title "uninstall: STARTED, FINISHED with SUCCESS"
trace $CLI aitools uninstall --scope=global
trace print_telemetry
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Cloud = false
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@

=== install: the background auth resolution prints nothing

>>> [CLI] aitools install --skills-only --scope=global --agents=cursor
Installing Databricks skills for Cursor...
Using skills version test-ref
Fetching skills manifest...
Installed 1 skill.

=== update: the background auth resolution prints nothing

>>> [CLI] aitools update --scope=global
Already up to date.

=== uninstall: the background auth resolution prints nothing

>>> [CLI] aitools uninstall --scope=global
Uninstalled 1 skill.
22 changes: 22 additions & 0 deletions acceptance/cmd/aitools/telemetry/workspace-unreachable/script
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
# Isolate HOME so the real machine's agents and skills are not seen.
sethome home

# The workspace host does not resolve, so the background telemetry auth
# resolution fails to fetch host metadata. The skills manifest still comes from
# the mock server (DATABRICKS_SKILLS_BASE_URL is set from the original host).
# A hostmetadata warning only reaches the output if the failed lookup ends
# before the command stops waiting for the resolution. .test resolution fails
# fast (NXDOMAIN), and install, update and uninstall wait for it once they get
# to their work, so they catch a regression; list and version never wait, and
# update and uninstall with nothing installed do not either.
export DATABRICKS_HOST=https://unreachable.test
mkdir -p "${USERPROFILE:-$HOME}/.cursor"

title "install: the background auth resolution prints nothing"
trace $CLI aitools install --skills-only --scope=global --agents=cursor

title "update: the background auth resolution prints nothing"
trace $CLI aitools update --scope=global

title "uninstall: the background auth resolution prints nothing"
trace $CLI aitools uninstall --scope=global
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
# Nothing reaches the mock server's telemetry endpoint, so there is nothing to record.
RecordRequests = false
7 changes: 7 additions & 0 deletions cmd/aitools/categorize.go
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,13 @@ func classifyInstallError(err error) protos.AitoolsErrorCategory {
if tagged, ok := errors.AsType[*categorizedError](err); ok {
return tagged.category
}
if errors.Is(err, installer.ErrUnknownAgent) {
return protos.AitoolsErrorCategoryUnknownAgent
}
// Skills on disk without the install state that update and uninstall need.
if errors.Is(err, installer.ErrLegacyInstall) {
return protos.AitoolsErrorCategoryStateInvalid
}
return stdlibErrorCategory(err)
}

Expand Down
10 changes: 10 additions & 0 deletions cmd/aitools/categorize_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,16 @@ func TestClassifyInstallError(t *testing.T) {
err: fmt.Errorf("resolve failed: %w", &installer.SkillError{Skill: "databricks", Reason: installer.ReasonSkillNotFound, Detail: "not found"}),
want: protos.AitoolsErrorCategorySkillNotFound,
},
{
name: "legacy install without state",
err: fmt.Errorf("%w; run install first", installer.ErrLegacyInstall),
want: protos.AitoolsErrorCategoryStateInvalid,
},
{
name: "unknown agent in the install state",
err: fmt.Errorf("skipped: %w", installer.ErrUnknownAgent),
want: protos.AitoolsErrorCategoryUnknownAgent,
},
{
name: "version incompatible",
err: &installer.SkillError{Skill: "databricks", Reason: installer.ReasonVersionIncompatible, Detail: "requires CLI version 0.5 (running 0.4)"},
Expand Down
42 changes: 38 additions & 4 deletions cmd/aitools/funnel.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,11 @@ import (

"github.com/charmbracelet/huh"
"github.com/databricks/cli/libs/aitools/agents"
"github.com/databricks/cli/libs/aitools/installer"
"github.com/databricks/cli/libs/cmdio"
"github.com/databricks/cli/libs/telemetry"
"github.com/databricks/cli/libs/telemetry/protos"
"github.com/spf13/cobra"
)

// commandFunnel emits the AitoolsCommandEvent funnel of one aitools subcommand
Expand Down Expand Up @@ -38,7 +40,15 @@ func startFunnel(ctx context.Context, command protos.AitoolsCommand, installMode
}

// setScope records the resolved scope (installer.ScopeGlobal/ScopeProject).
func (f *commandFunnel) setScope(scope string) { f.scope = scopeType(scope) }
// An empty scope (none or both selected) leaves the field unset, so the event
// omits it rather than sending the Unspecified sentinel.
func (f *commandFunnel) setScope(scope string) {
if scope == "" {
f.scope = ""
return
}
f.scope = scopeType(scope)
}

// setAgents records the plan's agents, reported on the FINISHED event.
func (f *commandFunnel) setAgents(plan []agentPlanItem) { f.agents = agentsField(plan) }
Expand All @@ -56,13 +66,16 @@ func (f *commandFunnel) step(ctx context.Context, s protos.AitoolsCommandStep) {
}

// finish logs the FINISHED event for a run that returned err: SUCCESS for nil,
// CANCELLED when the user aborted a prompt or picked no agents, FAILURE
// otherwise. No-op once the run has finished, so it is safe to defer after an
// explicit finishWith.
// NOTHING_TO_DO when update or uninstall found nothing installed (the command
// still exits non-zero), CANCELLED when the user aborted a prompt or picked no
// agents, FAILURE otherwise. No-op once the run has finished, so it is safe to
// defer after an explicit finishWith.
func (f *commandFunnel) finish(ctx context.Context, err error) {
switch {
case err == nil:
f.finishWith(ctx, protos.AitoolsCommandOutcomeSuccess, protos.AitoolsErrorCategoryUnspecified)
case errors.Is(err, installer.ErrNothingInstalled):
f.finishWith(ctx, protos.AitoolsCommandOutcomeNothingToDo, protos.AitoolsErrorCategoryUnspecified)
case errors.Is(err, huh.ErrUserAborted):
f.finishWith(ctx, protos.AitoolsCommandOutcomeCancelled, protos.AitoolsErrorCategoryUserAborted)
case errors.Is(err, errNoAgentsSelected):
Expand Down Expand Up @@ -102,6 +115,27 @@ func (f *commandFunnel) log(ctx context.Context, e protos.AitoolsCommandEvent) {
telemetry.Log(ctx, protos.DatabricksCliLog{AitoolsCommandEvent: &e})
}

// withCommandFunnel wraps a subcommand's RunE so every run logs STARTED and
// FINISHED (outcome from the returned error).
func withCommandFunnel(command protos.AitoolsCommand, runE func(cmd *cobra.Command, args []string) error) func(cmd *cobra.Command, args []string) error {
return func(cmd *cobra.Command, args []string) (err error) {
f := startFunnel(cmd.Context(), command, "")
defer func() { f.finish(cmd.Context(), err) }()
cmd.SetContext(context.WithValue(cmd.Context(), funnelKey{}, f))
return runE(cmd, args)
}
}

type funnelKey struct{}

// funnelFrom returns the funnel withCommandFunnel put on ctx. Only the RunE of
// a command wrapped by instrumentCommand calls it, for outcomes its returned
// error does not tell (see finishWith) and for the scope and agents it
// resolves.
func funnelFrom(ctx context.Context) *commandFunnel {
return ctx.Value(funnelKey{}).(*commandFunnel)
}

// installModeType maps the install delivery flags to the telemetry enum.
func installModeType(skillsOnly bool, path string) protos.AitoolsInstallMode {
switch {
Expand Down
Loading
Loading