Skip to content

Commit 405e077

Browse files
anton-107Isaac
andcommitted
aitools: typed fetch, state and agent CLI timeout errors
`aitools install` failures that used to be `UNCATEGORIZED` or a plain `NETWORK` now get a specific `error_category`. The installer returns typed errors and `classifyInstallError` maps them: ```text classifyInstallError(err) BlockedError TimedOut -> PLUGIN_CLI_TIMEOUT (agent CLI killed by its timeout) FetchError{Stage, StatusCode, RateLimited} (release / manifest / skill file) context.Canceled -> COMMAND_INTERRUPTED RateLimited -> RATE_LIMITED (429, or 403 with GitHub's rate-limit headers) clicompat.ErrNotFound -> MANIFEST_NOT_FOUND Stage == release -> RELEASE_LOOKUP_FAILED JSON syntax/type error -> MANIFEST_INVALID otherwise -> SKILLS_FETCH_FAILED StateError{Path} -> STATE_INVALID (unparsable .state.json) ``` - `FetchError` and `StateError` keep the wrapped error's message, so what the user sees does not change. - An unreadable state file is not a `StateError`: it keeps its filesystem category. - `FetchManifest` reads the whole body before parsing it, so a connection dropped mid-body (`SKILLS_FETCH_FAILED`) is told apart from an empty or truncated manifest (`MANIFEST_INVALID`). - Changelog fragment: `.nextchanges/cli/aitools-install-fetch-error-categories.md`. Co-authored-by: Isaac <no-reply@databricks.com>
1 parent 3081662 commit 405e077

13 files changed

Lines changed: 416 additions & 38 deletions

File tree

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
* `databricks aitools install --output json` now reports `SKILLS_FETCH_FAILED`, `RELEASE_LOOKUP_FAILED` or `RATE_LIMITED` as the `error_category` when fetching skills fails, `MANIFEST_INVALID` for a skills manifest it cannot parse, `STATE_INVALID` for a corrupt install state file, and `PLUGIN_CLI_TIMEOUT` when a coding agent's CLI does not answer in time. ([#7475](https://github.com/databricks/cli/pull/7475))

‎acceptance/cmd/aitools/telemetry/install-failure/output.txt‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ Exit code: 1
5353
{"command":"aitools_install","exit_code":1,"aitools_command_event":{"command":"INSTALL","stage":"FINISHED","outcome":"FAILURE","error_category":"MANIFEST_NOT_FOUND","interactive":false,"install_mode":"SKILLS_ONLY","agents":["CURSOR"],"scope":"GLOBAL"}}
5454
{"command":"aitools_install","exit_code":1,"aitools_install_event":{"agents":["CURSOR"],"scope":"GLOBAL","error_category":"MANIFEST_NOT_FOUND"}}
5555

56-
=== any other HTTP error status from the manifest fetch is not classified yet (the error is untyped)
56+
=== any other HTTP error status from the manifest fetch is SKILLS_FETCH_FAILED
5757
Installing Databricks skills for Cursor...
5858
Using skills version broken-ref
5959
Fetching skills manifest...
@@ -67,8 +67,8 @@ Exit code: 1
6767
{"command":"aitools_install","exit_code":1,"aitools_command_event":{"command":"INSTALL","stage":"STEP_COMPLETED","step":"SCOPE_RESOLVED","interactive":false,"install_mode":"SKILLS_ONLY","scope":"GLOBAL"}}
6868
{"command":"aitools_install","exit_code":1,"aitools_command_event":{"command":"INSTALL","stage":"STEP_COMPLETED","step":"AGENTS_SELECTED","interactive":false,"install_mode":"SKILLS_ONLY","scope":"GLOBAL"}}
6969
{"command":"aitools_install","exit_code":1,"aitools_command_event":{"command":"INSTALL","stage":"STEP_COMPLETED","step":"PLAN_CONFIRMED","interactive":false,"install_mode":"SKILLS_ONLY","scope":"GLOBAL"}}
70-
{"command":"aitools_install","exit_code":1,"aitools_command_event":{"command":"INSTALL","stage":"FINISHED","outcome":"FAILURE","error_category":"UNCATEGORIZED","interactive":false,"install_mode":"SKILLS_ONLY","agents":["CURSOR"],"scope":"GLOBAL"}}
71-
{"command":"aitools_install","exit_code":1,"aitools_install_event":{"agents":["CURSOR"],"scope":"GLOBAL","error_category":"UNCATEGORIZED"}}
70+
{"command":"aitools_install","exit_code":1,"aitools_command_event":{"command":"INSTALL","stage":"FINISHED","outcome":"FAILURE","error_category":"SKILLS_FETCH_FAILED","interactive":false,"install_mode":"SKILLS_ONLY","agents":["CURSOR"],"scope":"GLOBAL"}}
71+
{"command":"aitools_install","exit_code":1,"aitools_install_event":{"agents":["CURSOR"],"scope":"GLOBAL","error_category":"SKILLS_FETCH_FAILED"}}
7272

7373
=== a named agent skipped for the scope takes its category from the agent's outcome
7474
Skipped Cursor: does not support project-scoped skills

‎acceptance/cmd/aitools/telemetry/install-failure/script‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ title "a manifest the server does not have is MANIFEST_NOT_FOUND\n"
1818
errcode env DATABRICKS_SKILLS_REF=missing-ref $CLI aitools install --skills-only --scope=global --agents=cursor
1919
trace print_telemetry
2020

21-
title "any other HTTP error status from the manifest fetch is not classified yet (the error is untyped)\n"
21+
title "any other HTTP error status from the manifest fetch is SKILLS_FETCH_FAILED\n"
2222
errcode env DATABRICKS_SKILLS_REF=broken-ref $CLI aitools install --skills-only --scope=global --agents=cursor
2323
trace print_telemetry
2424

‎cmd/aitools/categorize.go‎

Lines changed: 48 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package aitools
22

33
import (
44
"context"
5+
"encoding/json"
56
"errors"
67
"io/fs"
78
"net"
@@ -60,18 +61,27 @@ func classifyInstallError(err error) protos.AitoolsErrorCategory {
6061
if skill, ok := errors.AsType[*installer.SkillError](err); ok {
6162
return skillErrorCategory(skill)
6263
}
64+
// FetchError wraps the net error that stdlibErrorCategory would otherwise
65+
// report as NETWORK, so it has to be checked first.
66+
if fetch, ok := errors.AsType[*installer.FetchError](err); ok {
67+
return fetchErrorCategory(fetch)
68+
}
69+
if _, ok := errors.AsType[*installer.StateError](err); ok {
70+
return protos.AitoolsErrorCategoryStateInvalid
71+
}
6372
if tagged, ok := errors.AsType[*categorizedError](err); ok {
6473
return tagged.category
6574
}
6675
return stdlibErrorCategory(err)
6776
}
6877

6978
// stdlibErrorCategory classifies errors by their standard library type, which
70-
// the installer wraps with %w: network and timeout errors from fetching the
71-
// manifest or skill files, and filesystem errors from writing skills or the
72-
// install state. JSON decode errors stay UNCATEGORIZED: the manifest, the
73-
// release payload and the install state all decode JSON, so only typed
74-
// installer errors can tell MANIFEST_INVALID from STATE_INVALID.
79+
// the installer wraps with %w: filesystem errors from writing skills or the
80+
// install state, and network and timeout errors that no FetchError wraps (the
81+
// installer's own fetches return a FetchError, classified above). JSON decode
82+
// errors stay UNCATEGORIZED: the manifest, the release payload and the install
83+
// state all decode JSON, so only typed installer errors can tell
84+
// MANIFEST_INVALID from STATE_INVALID.
7585
func stdlibErrorCategory(err error) protos.AitoolsErrorCategory {
7686
switch {
7787
case errors.Is(err, huh.ErrUserAborted):
@@ -129,7 +139,40 @@ func skillErrorCategory(e *installer.SkillError) protos.AitoolsErrorCategory {
129139
}
130140
}
131141

142+
func fetchErrorCategory(e *installer.FetchError) protos.AitoolsErrorCategory {
143+
switch {
144+
// The command was interrupted while the request was in flight: the fetch
145+
// did not fail on its own.
146+
case errors.Is(e, context.Canceled):
147+
return protos.AitoolsErrorCategoryCommandInterrupted
148+
case e.RateLimited:
149+
return protos.AitoolsErrorCategoryRateLimited
150+
case errors.Is(e, clicompat.ErrNotFound):
151+
return protos.AitoolsErrorCategoryManifestNotFound
152+
// Before the decode check: a release payload that does not decode is a
153+
// failed release lookup, not an invalid manifest.
154+
case e.Stage == installer.FetchStageRelease:
155+
return protos.AitoolsErrorCategoryReleaseLookupFailed
156+
// The response arrived but was not a manifest: content, not transport.
157+
case isManifestDecodeError(e):
158+
return protos.AitoolsErrorCategoryManifestInvalid
159+
default:
160+
return protos.AitoolsErrorCategorySkillsFetchFailed
161+
}
162+
}
163+
164+
// isManifestDecodeError reports whether the response arrived but was not a
165+
// valid manifest.
166+
func isManifestDecodeError(err error) bool {
167+
_, syntax := errors.AsType[*json.SyntaxError](err)
168+
_, typ := errors.AsType[*json.UnmarshalTypeError](err)
169+
return syntax || typ
170+
}
171+
132172
func blockedErrorCategory(e *installer.BlockedError) protos.AitoolsErrorCategory {
173+
if e.TimedOut {
174+
return protos.AitoolsErrorCategoryPluginCLITimeout
175+
}
133176
switch e.Reason {
134177
case installer.ReasonCLINotOnPath:
135178
return protos.AitoolsErrorCategoryCLINotOnPath

‎cmd/aitools/categorize_test.go‎

Lines changed: 110 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,12 @@ import (
77
"fmt"
88
"io/fs"
99
"net"
10+
"net/http"
11+
"net/http/httptest"
1012
"net/url"
1113
"os"
1214
"path/filepath"
15+
"strconv"
1316
"syscall"
1417
"testing"
1518

@@ -42,6 +45,56 @@ func TestClassifyInstallError(t *testing.T) {
4245
err: &installer.BlockedError{Agent: "codex", Reason: installer.ReasonInstallFailed},
4346
want: protos.AitoolsErrorCategoryPluginInstallFailed,
4447
},
48+
{
49+
name: "blocked timeout wins over its reason",
50+
err: &installer.BlockedError{Agent: "claude-code", Reason: installer.ReasonCLINotOnPath, TimedOut: true},
51+
want: protos.AitoolsErrorCategoryPluginCLITimeout,
52+
},
53+
{
54+
name: "release lookup failed",
55+
err: fmt.Errorf("resolve latest skills release: %w", &installer.FetchError{Stage: installer.FetchStageRelease, Err: &net.DNSError{Err: "no such host"}}),
56+
want: protos.AitoolsErrorCategoryReleaseLookupFailed,
57+
},
58+
{
59+
name: "release payload that does not decode is not an invalid manifest",
60+
err: &installer.FetchError{Stage: installer.FetchStageRelease, StatusCode: 200, Err: fmt.Errorf("parse latest skills release: %w", &json.SyntaxError{})},
61+
want: protos.AitoolsErrorCategoryReleaseLookupFailed,
62+
},
63+
{
64+
name: "release lookup rate limited",
65+
err: &installer.FetchError{Stage: installer.FetchStageRelease, StatusCode: 403, RateLimited: true, Err: errors.New("HTTP 403")},
66+
want: protos.AitoolsErrorCategoryRateLimited,
67+
},
68+
{
69+
name: "release lookup denied without rate-limit evidence",
70+
err: &installer.FetchError{Stage: installer.FetchStageRelease, StatusCode: 403, Err: errors.New("HTTP 403")},
71+
want: protos.AitoolsErrorCategoryReleaseLookupFailed,
72+
},
73+
{
74+
name: "manifest fetch rate limited",
75+
err: &installer.FetchError{Stage: installer.FetchStageManifest, StatusCode: 429, RateLimited: true, Err: errors.New("HTTP 429")},
76+
want: protos.AitoolsErrorCategoryRateLimited,
77+
},
78+
{
79+
name: "manifest fetch network error is not plain NETWORK",
80+
err: &installer.FetchError{Stage: installer.FetchStageManifest, Err: &net.DNSError{Err: "no such host"}},
81+
want: protos.AitoolsErrorCategorySkillsFetchFailed,
82+
},
83+
{
84+
name: "skill file fetch failed",
85+
err: fmt.Errorf("failed to fetch skills/x/SKILL.md: %w", &installer.FetchError{Stage: installer.FetchStageFile, StatusCode: 500, Err: errors.New("HTTP 500")}),
86+
want: protos.AitoolsErrorCategorySkillsFetchFailed,
87+
},
88+
{
89+
name: "manifest missing at the pinned ref",
90+
err: &installer.FetchError{Stage: installer.FetchStageManifest, StatusCode: 404, Err: fmt.Errorf("skills manifest at x@v9: %w", clicompat.ErrNotFound)},
91+
want: protos.AitoolsErrorCategoryManifestNotFound,
92+
},
93+
{
94+
name: "state file invalid",
95+
err: fmt.Errorf("failed to load install state: %w", &installer.StateError{Path: "/s/.state.json", Err: errors.New("failed to parse state file")}),
96+
want: protos.AitoolsErrorCategoryStateInvalid,
97+
},
4598
{
4699
name: "blocked error with unknown reason is uncategorized",
47100
err: &installer.BlockedError{Agent: "codex", Reason: "some-future-reason"},
@@ -136,15 +189,14 @@ func TestWithCategoryKeepsMessage(t *testing.T) {
136189
assert.NoError(t, promptError(nil))
137190
}
138191

139-
// A corrupt install state must not be blamed on the remote manifest. It stays
140-
// UNCATEGORIZED until the installer returns a typed state error, which should
141-
// change this to STATE_INVALID.
142-
func TestClassifyInstallErrorCorruptStateIsUncategorized(t *testing.T) {
192+
// A corrupt install state must not be blamed on the remote manifest: the
193+
// installer's typed StateError makes it STATE_INVALID.
194+
func TestClassifyInstallErrorCorruptStateIsStateInvalid(t *testing.T) {
143195
dir := t.TempDir()
144196
require.NoError(t, os.WriteFile(filepath.Join(dir, ".state.json"), []byte("{not json"), 0o600))
145197
_, err := installer.LoadState(dir)
146198
require.Error(t, err)
147-
assert.Equal(t, protos.AitoolsErrorCategoryUncategorized, classifyInstallError(fmt.Errorf("failed to load install state: %w", err)))
199+
assert.Equal(t, protos.AitoolsErrorCategoryStateInvalid, classifyInstallError(fmt.Errorf("failed to load install state: %w", err)))
148200
}
149201

150202
// readMissingFile returns the error the OS gives for a missing file, so the
@@ -155,3 +207,56 @@ func readMissingFile(t *testing.T) error {
155207
require.Error(t, err)
156208
return err
157209
}
210+
211+
func TestClassifyCancelledManifestFetchIsCommandInterrupted(t *testing.T) {
212+
ctx, cancel := context.WithCancel(t.Context())
213+
server := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) {
214+
// Cancel while the request is in flight, then wait for the client to give up.
215+
cancel()
216+
<-r.Context().Done()
217+
}))
218+
t.Cleanup(server.Close)
219+
t.Setenv("DATABRICKS_SKILLS_BASE_URL", server.URL)
220+
221+
_, err := (&installer.GitHubManifestSource{}).FetchManifest(ctx, "v1")
222+
require.ErrorIs(t, err, context.Canceled)
223+
assert.Equal(t, protos.AitoolsErrorCategoryCommandInterrupted, classifyInstallError(err))
224+
}
225+
226+
func TestClassifyRealManifestFetchErrors(t *testing.T) {
227+
tests := []struct {
228+
name string
229+
status int
230+
body string
231+
// cutOff announces a longer body than is sent, so the connection
232+
// closes mid-body.
233+
cutOff bool
234+
want protos.AitoolsErrorCategory
235+
}{
236+
// A captive portal or broken mirror answers 200 with something else.
237+
{name: "200 with HTML", status: http.StatusOK, body: "<html>login</html>", want: protos.AitoolsErrorCategoryManifestInvalid},
238+
{name: "200 with the wrong JSON shape", status: http.StatusOK, body: `{"skills": []}`, want: protos.AitoolsErrorCategoryManifestInvalid},
239+
{name: "200 with an empty body", status: http.StatusOK, body: "", want: protos.AitoolsErrorCategoryManifestInvalid},
240+
{name: "200 with incomplete JSON", status: http.StatusOK, body: `{"skills":`, want: protos.AitoolsErrorCategoryManifestInvalid},
241+
{name: "connection closed mid-body", status: http.StatusOK, body: `{"skills":`, cutOff: true, want: protos.AitoolsErrorCategorySkillsFetchFailed},
242+
{name: "404", status: http.StatusNotFound, want: protos.AitoolsErrorCategoryManifestNotFound},
243+
{name: "503", status: http.StatusServiceUnavailable, want: protos.AitoolsErrorCategorySkillsFetchFailed},
244+
}
245+
for _, tc := range tests {
246+
t.Run(tc.name, func(t *testing.T) {
247+
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
248+
if tc.cutOff {
249+
w.Header().Set("Content-Length", strconv.Itoa(len(tc.body)+100))
250+
}
251+
w.WriteHeader(tc.status)
252+
_, _ = w.Write([]byte(tc.body))
253+
}))
254+
t.Cleanup(server.Close)
255+
t.Setenv("DATABRICKS_SKILLS_BASE_URL", server.URL)
256+
257+
_, err := (&installer.GitHubManifestSource{}).FetchManifest(t.Context(), "v1")
258+
require.Error(t, err)
259+
assert.Equal(t, tc.want, classifyInstallError(err))
260+
})
261+
}
262+
}

‎cmd/aitools/funnel_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -331,7 +331,7 @@ func TestInstallFunnelPathDump(t *testing.T) {
331331
ctx, err := runInstall(t, cmdio.MockDiscard(t.Context()), "--path", t.TempDir())
332332
require.Error(t, err)
333333
events := funnelEvents(ctx)
334-
assert.Equal(t, []string{"STARTED", "FLAGS_VALIDATED", "FINISHED:FAILURE:NETWORK"}, funnelShape(events))
334+
assert.Equal(t, []string{"STARTED", "FLAGS_VALIDATED", "FINISHED:FAILURE:SKILLS_FETCH_FAILED"}, funnelShape(events))
335335
assert.Equal(t, protos.AitoolsInstallModePath, events[0].InstallMode)
336336
}
337337

@@ -371,7 +371,7 @@ func TestListFunnelFailure(t *testing.T) {
371371
cmd.SilenceErrors = true
372372
cmd.SilenceUsage = true
373373
require.Error(t, cmd.RunE(cmd, nil))
374-
assert.Equal(t, []string{"STARTED", "FINISHED:FAILURE:NETWORK"}, funnelShape(funnelEvents(ctx)))
374+
assert.Equal(t, []string{"STARTED", "FINISHED:FAILURE:SKILLS_FETCH_FAILED"}, funnelShape(funnelEvents(ctx)))
375375
}
376376

377377
// A plugin failure for an auto-detected agent is only a warning (exit 0), but

‎libs/aitools/installer/errors.go‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package installer
33
import (
44
"errors"
55
"fmt"
6+
"net/http"
67
)
78

89
// ErrNothingInstalled reports that update or uninstall found no install state
@@ -39,3 +40,63 @@ func (e *SkillError) Error() string {
3940
}
4041
return fmt.Sprintf("skill %q %s", e.Skill, e.Reason)
4142
}
43+
44+
// Stages of fetching skills content, recorded on FetchError so the command
45+
// layer can name the host the user has to reach.
46+
const (
47+
// FetchStageRelease: resolving the latest skills release on api.github.com.
48+
FetchStageRelease = "release"
49+
// FetchStageManifest: downloading the skills manifest.
50+
FetchStageManifest = "manifest"
51+
// FetchStageFile: downloading one skill file.
52+
FetchStageFile = "file"
53+
)
54+
55+
// FetchError reports that skills content could not be downloaded, or that a
56+
// response arrived but could not be used (an unparsable manifest or release
57+
// payload). It wraps the cause without changing its message; StatusCode is 0
58+
// when no HTTP response arrived (DNS, proxy, TLS, timeout).
59+
type FetchError struct {
60+
Stage string
61+
StatusCode int
62+
// RateLimited is set when the server said the request exceeded a rate
63+
// limit, as opposed to an ordinary 403 from an access policy or proxy.
64+
RateLimited bool
65+
Err error
66+
}
67+
68+
func (e *FetchError) Error() string { return e.Err.Error() }
69+
func (e *FetchError) Unwrap() error { return e.Err }
70+
71+
// newHTTPFetchError builds the FetchError for a response with a non-OK status.
72+
func newHTTPFetchError(stage string, resp *http.Response, err error) *FetchError {
73+
return &FetchError{Stage: stage, StatusCode: resp.StatusCode, RateLimited: isRateLimited(resp), Err: err}
74+
}
75+
76+
// isRateLimited reports whether a response is a rate-limit refusal. 429 always
77+
// is; GitHub also answers 403 for exceeded limits, and then says so with
78+
// x-ratelimit-remaining: 0 (primary limit) or retry-after (secondary limit).
79+
// Any other 403 is an access failure.
80+
// See https://docs.github.com/en/rest/using-the-rest-api/rate-limits-for-the-rest-api#exceeding-the-rate-limit.
81+
func isRateLimited(resp *http.Response) bool {
82+
switch resp.StatusCode {
83+
case http.StatusTooManyRequests:
84+
return true
85+
case http.StatusForbidden:
86+
return resp.Header.Get("X-RateLimit-Remaining") == "0" || resp.Header.Get("Retry-After") != ""
87+
default:
88+
return false
89+
}
90+
}
91+
92+
// StateError reports that the install state file exists but its content is
93+
// invalid. Path is the file, so the user can be told which one to move aside.
94+
// A file that cannot be read is not a StateError: that is a filesystem problem
95+
// (usually permissions), and moving the file aside would not fix it.
96+
type StateError struct {
97+
Path string
98+
Err error
99+
}
100+
101+
func (e *StateError) Error() string { return e.Err.Error() }
102+
func (e *StateError) Unwrap() error { return e.Err }

0 commit comments

Comments
 (0)