diff --git a/.changesets/1792673500-0907a518.yaml b/.changesets/1792673500-0907a518.yaml new file mode 100644 index 000000000..37a78be18 --- /dev/null +++ b/.changesets/1792673500-0907a518.yaml @@ -0,0 +1,10 @@ +id: 1792673500-0907a518 +features: + - cliCommands +targets: + - cli +type: feat +bump: minor +description: declare terminal failure states with output.outcome so a command exits 1 when a successful response reports a declared failure state, for discrete JSON responses (statePointer) and streamed events (streamStatePointer), with the failure detail surfaced from errorPointer / streamErrorPointer and reported as CLI_OUTCOME_FAILED +author: AshGodfrey +date: "2026-10-22" diff --git a/internal/extensions/cli_commands.go b/internal/extensions/cli_commands.go index 78cdeebd5..ee4b3ed15 100644 --- a/internal/extensions/cli_commands.go +++ b/internal/extensions/cli_commands.go @@ -243,6 +243,30 @@ type CLICommandOutput struct { Projection *CLICommandProjection `json:"projection" yaml:"projection"` Artifact *CLICommandArtifact `json:"artifact,omitempty" yaml:"artifact,omitempty"` Stream *CLICommandStreamProjection `json:"stream,omitempty" yaml:"stream,omitempty"` + Outcome *CLICommandOutcome `json:"outcome,omitempty" yaml:"outcome,omitempty"` +} + +// CLICommandOutcome classifies the terminal state carried inside a +// successful response so the generated command can fail (exit 1) when the +// API reports failure with an HTTP success code. StatePointer addresses the +// state in the discrete JSON response and StreamStatePointer the same state +// inside streamed events; at least one is declared. States is keyed by the +// API's enum value, exactly as async.states, and only the "failure" +// classification changes runtime behavior. ErrorPointer optionally addresses +// the failure detail (an object or an array of objects) whose +// ErrorMessageField string is appended to the failure message. A +// single-route intent without its own table inherits its operation entry's. +type CLICommandOutcome struct { + StateFrom string `json:"stateFrom,omitempty" yaml:"stateFrom,omitempty"` + StatePointer string `json:"statePointer,omitempty" yaml:"statePointer,omitempty"` + StreamStateFrom string `json:"streamStateFrom,omitempty" yaml:"streamStateFrom,omitempty"` + StreamStatePointer string `json:"streamStatePointer,omitempty" yaml:"streamStatePointer,omitempty"` + States map[string]string `json:"states" yaml:"states"` + ErrorFrom string `json:"errorFrom,omitempty" yaml:"errorFrom,omitempty"` + ErrorPointer string `json:"errorPointer,omitempty" yaml:"errorPointer,omitempty"` + StreamErrorFrom string `json:"streamErrorFrom,omitempty" yaml:"streamErrorFrom,omitempty"` + StreamErrorPointer string `json:"streamErrorPointer,omitempty" yaml:"streamErrorPointer,omitempty"` + ErrorMessageField string `json:"errorMessageField,omitempty" yaml:"errorMessageField,omitempty"` } // CLIOperation augments the generated command for one operation. Unlike an @@ -509,6 +533,7 @@ var CLIRuntimeHintReasons = []string{ "CLI_ASYNC_FAILED", "CLI_ASYNC_TIMEOUT", "CLI_ASYNC_UNKNOWN_STATE", + "CLI_OUTCOME_FAILED", } // cliReservedCommandKeys are keys reserved for future capabilities. Each maps @@ -527,7 +552,9 @@ var cliCommandKeys = []string{ "args", "flags", "preset", "hints", "examples", "help", "jq", "output", "async", } -var cliOutputKeys = []string{"artifact", "stream"} +var cliOutputKeys = []string{"artifact", "stream", "outcome"} + +var cliOutcomeKeys = []string{"statePointer", "streamStatePointer", "states", "errorPointer", "streamErrorPointer", "errorMessageField"} var cliOutputStreamKeys = []string{"select", "metadata"} @@ -782,17 +809,31 @@ func (d *cliManifestDecoder) decodeOperation(opID string, node *yaml.Node) (*CLI return nil, err } if len(outputEntries) == 0 { - return nil, fmt.Errorf("line %d: operation %q output requires stream", entry.Value.Line, opID) + return nil, fmt.Errorf("line %d: operation %q output requires stream or outcome", entry.Value.Line, opID) } for _, outputEntry := range outputEntries { - if outputEntry.Key.Value != "stream" { - return nil, fmt.Errorf("line %d: operation %q output has unknown key %q%s", outputEntry.Key.Line, opID, outputEntry.Key.Value, cliDidYouMean(outputEntry.Key.Value, []string{"stream"})) - } - stream, err := d.decodeStreamProjection("operation "+opID, outputEntry.Value) - if err != nil { - return nil, err + switch outputEntry.Key.Value { + case "stream": + stream, err := d.decodeStreamProjection("operation "+opID, outputEntry.Value) + if err != nil { + return nil, err + } + if op.Output == nil { + op.Output = &CLICommandOutput{} + } + op.Output.Stream = stream + case "outcome": + outcome, err := d.decodeOutcome("operation "+opID, outputEntry.Value) + if err != nil { + return nil, err + } + if op.Output == nil { + op.Output = &CLICommandOutput{} + } + op.Output.Outcome = outcome + default: + return nil, fmt.Errorf("line %d: operation %q output has unknown key %q%s", outputEntry.Key.Line, opID, outputEntry.Key.Value, cliDidYouMean(outputEntry.Key.Value, []string{"stream", "outcome"})) } - op.Output = &CLICommandOutput{Stream: stream} } case "flags": flags, err := d.decodeInputs("operation "+opID, entry.Value, false) @@ -953,7 +994,7 @@ func (d *cliManifestDecoder) decodeCommand(key string, path []string, id string, } if outputNode != nil { - artifact, stream, err := d.decodeOutput(key, outputNode) + artifact, stream, outcome, err := d.decodeOutput(key, outputNode) if err != nil { return nil, err } @@ -962,6 +1003,7 @@ func (d *cliManifestDecoder) decodeCommand(key string, path []string, id string, } cmd.Output.Artifact = artifact cmd.Output.Stream = stream + cmd.Output.Outcome = outcome } if cmd.Output != nil && cmd.Output.Projection != nil && cmd.Output.Artifact != nil { return nil, fmt.Errorf("command %q declares both jq and output.artifact; the artifact declaration owns the command's default output (users can still project with --jq at runtime)", key) @@ -976,6 +1018,9 @@ func (d *cliManifestDecoder) decodeCommand(key string, path []string, id string, if cmd.Output != nil && cmd.Output.Stream != nil { return nil, fmt.Errorf("command %q declares both async and output.stream; async polling requires discrete JSON responses and cannot be combined with streamed output", key) } + if cmd.Output != nil && cmd.Output.Outcome != nil { + return nil, fmt.Errorf("command %q declares both async and output.outcome; async.states already classifies the terminal poll state", key) + } } if err := d.decodeSource(cmd, key, node, opNode, routesNode, plannedNode, customNode, groupNode, sourceNode); err != nil { @@ -1004,7 +1049,7 @@ func (d *cliManifestDecoder) decodeCommand(key string, path []string, id string, if cmd.Output != nil && cmd.Output.Projection != nil { illegal = append(illegal, "jq") } - if cmd.Output != nil && (cmd.Output.Artifact != nil || cmd.Output.Stream != nil) { + if cmd.Output != nil && (cmd.Output.Artifact != nil || cmd.Output.Stream != nil || cmd.Output.Outcome != nil) { illegal = append(illegal, "output") } if asyncNode != nil { @@ -1336,38 +1381,125 @@ func cliDecodeBackoff(node *yaml.Node) (float64, error) { // capabilities beyond the command-level jq sugar: artifact (the result is a // written media file) and stream (the selected field of each streamed event // is written raw to stdout as it arrives). -func (d *cliManifestDecoder) decodeOutput(cmdKey string, node *yaml.Node) (*CLICommandArtifact, *CLICommandStreamProjection, error) { +func (d *cliManifestDecoder) decodeOutput(cmdKey string, node *yaml.Node) (*CLICommandArtifact, *CLICommandStreamProjection, *CLICommandOutcome, error) { entries, err := cliMapEntries(node, fmt.Sprintf("command %q output", cmdKey)) if err != nil { - return nil, nil, err + return nil, nil, nil, err } var ( artifact *CLICommandArtifact stream *CLICommandStreamProjection + outcome *CLICommandOutcome ) for _, entry := range entries { switch entry.Key.Value { case "artifact": artifact, err = d.decodeArtifact(cmdKey, entry.Value) if err != nil { - return nil, nil, err + return nil, nil, nil, err } case "stream": stream, err = d.decodeStreamProjection(cmdKey, entry.Value) if err != nil { - return nil, nil, err + return nil, nil, nil, err + } + case "outcome": + outcome, err = d.decodeOutcome(fmt.Sprintf("command %q", cmdKey), entry.Value) + if err != nil { + return nil, nil, nil, err } case "jq": - return nil, nil, fmt.Errorf("line %d: command %q: jq is declared at the command level (jq: ), not under output", entry.Key.Line, cmdKey) + return nil, nil, nil, fmt.Errorf("line %d: command %q: jq is declared at the command level (jq: ), not under output", entry.Key.Line, cmdKey) default: - return nil, nil, fmt.Errorf("line %d: command %q output has unknown key %q%s", entry.Key.Line, cmdKey, entry.Key.Value, cliDidYouMean(entry.Key.Value, cliOutputKeys)) + return nil, nil, nil, fmt.Errorf("line %d: command %q output has unknown key %q%s", entry.Key.Line, cmdKey, entry.Key.Value, cliDidYouMean(entry.Key.Value, cliOutputKeys)) } } - if artifact == nil && stream == nil { - return nil, nil, fmt.Errorf("line %d: command %q: output must declare artifact or stream (the output capabilities beyond the command-level jq key)", node.Line, cmdKey) + if artifact == nil && stream == nil && outcome == nil { + return nil, nil, nil, fmt.Errorf("line %d: command %q: output must declare artifact, stream, or outcome (the output capabilities beyond the command-level jq key)", node.Line, cmdKey) } - return artifact, stream, nil + return artifact, stream, outcome, nil +} + +// decodeOutcome decodes an output.outcome declaration. Pointer resolvability +// and enum coverage are checked during linking; this stage validates the +// local grammar: at least one state pointer, a states table whose values are +// the async classifications, and member-name error bindings. +func (d *cliManifestDecoder) decodeOutcome(owner string, node *yaml.Node) (*CLICommandOutcome, error) { + entries, err := cliMapEntries(node, owner+" output.outcome") + if err != nil { + return nil, err + } + outcome := &CLICommandOutcome{States: map[string]string{}} + for _, entry := range entries { + switch entry.Key.Value { + case "statePointer": + outcome.StateFrom, outcome.StatePointer, _, err = cliDecodeAsyncPointer(entry.Value, "output.outcome.statePointer") + case "streamStatePointer": + outcome.StreamStateFrom, outcome.StreamStatePointer, _, err = cliDecodeAsyncPointer(entry.Value, "output.outcome.streamStatePointer") + case "states": + err = d.decodeOutcomeStates(owner, entry.Value, outcome.States) + case "errorPointer": + outcome.ErrorFrom, outcome.ErrorPointer, _, err = cliDecodeAsyncPointer(entry.Value, "output.outcome.errorPointer") + case "streamErrorPointer": + outcome.StreamErrorFrom, outcome.StreamErrorPointer, _, err = cliDecodeAsyncPointer(entry.Value, "output.outcome.streamErrorPointer") + case "errorMessageField": + outcome.ErrorMessageField, err = cliDecodeArtifactFieldName(entry.Value, "output.outcome.errorMessageField") + default: + return nil, fmt.Errorf("line %d: %s output.outcome has unknown key %q%s", entry.Key.Line, owner, entry.Key.Value, cliDidYouMean(entry.Key.Value, cliOutcomeKeys)) + } + if err != nil { + return nil, fmt.Errorf("line %d: %s: %w", entry.Value.Line, owner, err) + } + } + if outcome.StatePointer == "" && outcome.StreamStatePointer == "" { + return nil, fmt.Errorf("line %d: %s output.outcome requires statePointer (the state in the JSON response) or streamStatePointer (the state inside streamed events)", node.Line, owner) + } + if len(outcome.States) == 0 { + return nil, fmt.Errorf("line %d: %s output.outcome requires states (a map from every API state to pending, success, failure, or handoff)", node.Line, owner) + } + hasFailure := false + for _, class := range outcome.States { + hasFailure = hasFailure || class == "failure" + } + if !hasFailure { + return nil, fmt.Errorf("line %d: %s output.outcome.states must classify at least one state as failure; the declaration exists to fail the command on that state", node.Line, owner) + } + if outcome.ErrorPointer != "" && outcome.StatePointer == "" { + return nil, fmt.Errorf("line %d: %s output.outcome.errorPointer requires statePointer; use streamErrorPointer for streamed events", node.Line, owner) + } + if outcome.StreamErrorPointer != "" && outcome.StreamStatePointer == "" { + return nil, fmt.Errorf("line %d: %s output.outcome.streamErrorPointer requires streamStatePointer", node.Line, owner) + } + if outcome.ErrorMessageField != "" && outcome.ErrorPointer == "" && outcome.StreamErrorPointer == "" { + return nil, fmt.Errorf("line %d: %s output.outcome.errorMessageField requires errorPointer or streamErrorPointer", node.Line, owner) + } + if (outcome.ErrorPointer != "" || outcome.StreamErrorPointer != "") && outcome.ErrorMessageField == "" { + outcome.ErrorMessageField = cliAsyncDefaultErrorMessageField + } + return outcome, nil +} + +func (d *cliManifestDecoder) decodeOutcomeStates(owner string, node *yaml.Node, states map[string]string) error { + entries, err := cliMapEntries(node, owner+" output.outcome.states") + if err != nil { + return err + } + for _, entry := range entries { + state, err := cliScalarString(entry.Key, "outcome state name") + if err != nil || strings.TrimSpace(state) == "" { + return fmt.Errorf("line %d: %s output.outcome.states keys must be non-empty strings", entry.Key.Line, owner) + } + class, err := cliScalarString(entry.Value, fmt.Sprintf("classification for outcome state %q", state)) + if err != nil { + return err + } + if !cliContains(cliAsyncStateKind, class) { + return fmt.Errorf("line %d: %s output.outcome.states[%q] has unsupported classification %q (expected %s)", entry.Value.Line, owner, state, class, cliCandidateList(cliAsyncStateKind)) + } + states[state] = class + } + return nil } // decodeArtifact decodes and validates an output.artifact declaration. The diff --git a/internal/extensions/cli_commands_link.go b/internal/extensions/cli_commands_link.go index 217fd7025..73461ddc9 100644 --- a/internal/extensions/cli_commands_link.go +++ b/internal/extensions/cli_commands_link.go @@ -4,6 +4,7 @@ import ( "context" stderrors "errors" "fmt" + "maps" "math" "reflect" "slices" @@ -132,9 +133,43 @@ func (d *cliManifestDecoder) linkManifest(manifest *CLICommandManifest) error { return err } } + inheritOperationOutcomes(manifest) return nil } +// inheritOperationOutcomes gives a single-route intent without its own +// output.outcome the table its operation entry declared, so a failure state +// classified for the operation command also fails the intent. Async intents +// keep async.states as their only classification. +func inheritOperationOutcomes(manifest *CLICommandManifest) { + outcomes := map[string]*CLICommandOutcome{} + for i := range manifest.Operations { + if op := &manifest.Operations[i]; op.Output != nil && op.Output.Outcome != nil { + outcomes[op.OperationID] = op.Output.Outcome + } + } + if len(outcomes) == 0 { + return + } + for i := range manifest.Commands { + cmd := &manifest.Commands[i] + if cmd.Source.Type != CLICommandSourceOperation || len(cmd.Source.Routes) != 1 || cmd.Async != nil { + continue + } + if cmd.Output != nil && cmd.Output.Outcome != nil { + continue + } + outcome, ok := outcomes[cmd.Source.Routes[0].OperationID] + if !ok { + continue + } + if cmd.Output == nil { + cmd.Output = &CLICommandOutput{} + } + cmd.Output.Outcome = outcome + } +} + func (d *cliManifestDecoder) buildSchemaIndex() error { if d.schemaIndex != nil { return nil @@ -397,6 +432,11 @@ func (d *cliManifestDecoder) linkSingleRouteCommand(cmd *CLICommand) error { return err } } + if cmd.Output != nil && cmd.Output.Outcome != nil { + if err := d.linkOutcome(owner, cmd.Output.Outcome, route.OperationID, opInfo); err != nil { + return err + } + } artifactOpInfo := opInfo artifactOpID := route.OperationID projectionSchema := cliOperationProjectionSchema(opInfo) @@ -512,6 +552,11 @@ func (d *cliManifestDecoder) linkOperation(op *CLIOperation) error { return err } } + if op.Output != nil && op.Output.Outcome != nil { + if err := d.linkOutcome(fmt.Sprintf("operation %q", op.OperationID), op.Output.Outcome, op.OperationID, opInfo); err != nil { + return err + } + } if len(op.Flags) == 0 { return nil } @@ -556,6 +601,11 @@ func (d *cliManifestDecoder) linkDispatchCommand(cmd *CLICommand) error { return err } } + if cmd.Output != nil && cmd.Output.Outcome != nil { + if err := d.linkOutcome(fmt.Sprintf("command %q", key), cmd.Output.Outcome, operationID, opInfo); err != nil { + return err + } + } // An async recipe links exactly as for a pinned intent: the poll operation // is validated and an artifact extracts from the terminal poll response. artifactOpInfo := opInfo @@ -1002,6 +1052,9 @@ func (d *cliManifestDecoder) linkAsync(cmdKey string, cmd *CLICommand, createInf return nil, fmt.Errorf("command %q async.statePointer %s in poll operation %q response %s is an open enum; every possible state must be classifiable at generation time", cmdKey, recipe.StateFrom, recipe.OperationID, response.code) } for _, value := range facts.enum { + if value == nil { + continue + } state, ok := value.(string) if !ok { return nil, fmt.Errorf("command %q async.statePointer %s in poll operation %q response %s has a non-string enum member %v", cmdKey, recipe.StateFrom, recipe.OperationID, response.code, value) @@ -1032,6 +1085,148 @@ func (d *cliManifestDecoder) linkAsync(cmdKey string, cmd *CLICommand, createInf return pollInfo, nil } +// linkOutcome checks the state pointers against the operation's responses: +// the JSON state must be a closed string enum that the states table covers +// exactly, and the stream pointers must resolve in at least one event shape. +func (d *cliManifestDecoder) linkOutcome(owner string, outcome *CLICommandOutcome, opID string, opInfo *cliOperationInfo) error { + if outcome.StatePointer != "" { + if len(opInfo.jsonResponses) == 0 { + return fmt.Errorf("%s output.outcome.statePointer %s: operation %q has no application/json 2xx response to inspect; a streaming-only operation declares streamStatePointer instead", owner, outcome.StateFrom, opID) + } + enumStates := map[string]bool{} + for _, response := range opInfo.jsonResponses { + stateFacts, err := d.linkAsyncResponsePointer(outcome.StateFrom, response.schema, true) + if err != nil { + return fmt.Errorf("%s output.outcome.statePointer %s in operation %q response %s: %w", owner, outcome.StateFrom, opID, response.code, err) + } + for _, facts := range stateFacts { + if facts.openEnum { + return fmt.Errorf("%s output.outcome.statePointer %s in operation %q response %s is an open enum; every possible state must be classifiable at generation time", owner, outcome.StateFrom, opID, response.code) + } + for _, value := range facts.enum { + if value == nil { + continue + } + state, ok := value.(string) + if !ok { + return fmt.Errorf("%s output.outcome.statePointer %s in operation %q response %s has a non-string enum member %v", owner, outcome.StateFrom, opID, response.code, value) + } + enumStates[state] = true + } + } + } + for state := range outcome.States { + if !enumStates[state] { + return fmt.Errorf("%s output.outcome.states classifies %q, which is not an enum member at %s in operation %q%s", owner, state, outcome.StateFrom, opID, cliDidYouMean(state, cliSortedKeys(enumStates))) + } + } + for state := range enumStates { + if _, classified := outcome.States[state]; !classified { + return fmt.Errorf("%s output.outcome.states does not classify enum member %q from %s in operation %q; classify every member as pending, success, failure, or handoff", owner, state, outcome.StateFrom, opID) + } + } + if outcome.ErrorPointer != "" { + resolved := false + candidates := map[string]bool{} + for _, response := range opInfo.jsonResponses { + found, names, err := d.linkOutcomeErrorPointer(outcome.ErrorFrom, response.schema) + if err != nil { + return fmt.Errorf("%s output.outcome.errorPointer %s in operation %q response %s: %w", owner, outcome.ErrorFrom, opID, response.code, err) + } + resolved = resolved || found + for _, name := range names { + candidates[name] = true + } + } + if !resolved { + return fmt.Errorf("%s output.outcome.errorPointer %s does not resolve in any application/json 2xx response variant of operation %q%s", owner, outcome.ErrorFrom, opID, cliDidYouMean(cliLastPathSegmentName(outcome.ErrorFrom), cliSortedKeys(candidates))) + } + } + } + if outcome.StreamStatePointer != "" { + if !opInfo.streamMediaSeen { + return fmt.Errorf("%s declares output.outcome.streamStatePointer, but operation %q has no streaming (text/event-stream or JSONL) success response", owner, opID) + } + walker, err := d.walkStreamSelection(owner, "output.outcome.streamStatePointer", outcome.StreamStateFrom, opInfo) + if err != nil { + return err + } + if err := d.linkOutcomeStreamStates(owner, outcome, opID, walker); err != nil { + return err + } + if outcome.StreamErrorPointer != "" { + if err := d.linkStreamSelection(owner, cliStreamDetailField, outcome.StreamErrorFrom, opInfo); err != nil { + return err + } + } + } + return nil +} + +const cliStreamDetailField = "output.outcome.streamErrorPointer" + +// linkOutcomeErrorPointer accepts failure detail that exists in only some +// response variants (a failed union arm, say); the runtime treats absence as +// no detail. It reports whether the path resolved anywhere in this response +// and, when it did not, the property names seen where it went missing. +func (d *cliManifestDecoder) linkOutcomeErrorPointer(path string, responseSchema any) (bool, []string, error) { + views, candidates, err := d.walkResponsePathViews(path, responseSchema, true) + if err != nil { + return false, nil, err + } + if len(views) == 0 { + return false, candidates, nil + } + facts, err := d.responseLeafFacts(views) + if err != nil { + return false, nil, err + } + for _, propertyFacts := range facts { + if len(propertyFacts.kinds) != 1 || (propertyFacts.kinds[0] != "object" && propertyFacts.kinds[0] != "array") { + return false, nil, stderrors.New("selected failure detail must be an object or an array in every response variant") + } + } + return true, nil, nil +} + +// linkOutcomeStreamStates holds the states table to the enum the stream +// pointer selects. A stream-only table must match that enum exactly; when +// statePointer already pinned the table to the JSON enum, the stream enum may +// be a subset of it, but every member must still be classified. +func (d *cliManifestDecoder) linkOutcomeStreamStates(owner string, outcome *CLICommandOutcome, opID string, walker *cliStreamPathWalker) error { + if walker.stateOpen { + return fmt.Errorf("%s output.outcome.streamStatePointer %s in operation %q is an open enum; every possible state must be classifiable at generation time", owner, outcome.StreamStateFrom, opID) + } + if walker.resolved == 0 { + return nil + } + if walker.stateNoEnum || len(walker.stateEnum) == 0 { + d.warnf("%s output.outcome.streamStatePointer %s in operation %q does not select a closed string enum in every event shape; states cannot be verified at generation time, and a state missing from the table keeps exit 0 at runtime", owner, outcome.StreamStateFrom, opID) + return nil + } + if outcome.StatePointer == "" { + for _, state := range slices.Sorted(maps.Keys(outcome.States)) { + if !walker.stateEnum[state] { + return fmt.Errorf("%s output.outcome.states classifies %q, which is not an enum member at %s in operation %q%s", owner, state, outcome.StreamStateFrom, opID, cliDidYouMean(state, cliSortedKeys(walker.stateEnum))) + } + } + } + for _, state := range cliSortedKeys(walker.stateEnum) { + if _, classified := outcome.States[state]; !classified { + return fmt.Errorf("%s output.outcome.states does not classify enum member %q from %s in operation %q; classify every member as pending, success, failure, or handoff", owner, state, outcome.StreamStateFrom, opID) + } + } + return nil +} + +func cliLastPathSegmentName(path string) string { + segments, err := cliParseSingularPath(path) + if err != nil || len(segments) == 0 || segments[len(segments)-1].IsIndex { + return "" + } + return segments[len(segments)-1].Name +} + // linkAsyncErrorBindings validates explicitly declared failure-detail // bindings against a poll response schema. The errorField member must exist // at the root of at least one traversable variant (failure detail @@ -1123,65 +1318,147 @@ func cliAsyncErrorViewIsObject(view *cliResolvedSchema) bool { // a handle or state missing from even one response-union arm would otherwise // become a data-dependent runtime failure. func (d *cliManifestDecoder) linkAsyncResponsePointer(path string, responseSchema any, requireEnum bool) ([]*cliPropertyFacts, error) { - segments, err := cliParseSingularPath(path) + views, err := d.walkResponsePath(path, responseSchema) if err != nil { return nil, err } - views, err := d.artifactExpand(responseSchema, 0) + facts, err := d.responseLeafFacts(views) if err != nil { return nil, err } + for _, propertyFacts := range facts { + if len(propertyFacts.kinds) != 1 || propertyFacts.kinds[0] != "string" { + return nil, stderrors.New("selected value must be a string in every response variant") + } + if requireEnum && len(propertyFacts.enum) == 0 { + return nil, stderrors.New("selected state must declare a non-empty enum in every response variant") + } + } + return facts, nil +} + +// walkResponsePath follows a singular path through every traversable variant +// of a response schema and returns the leaf views. +func (d *cliManifestDecoder) walkResponsePath(path string, responseSchema any) ([]*cliResolvedSchema, error) { + views, _, err := d.walkResponsePathViews(path, responseSchema, false) + return views, err +} + +// walkResponsePathViews is walkResponsePath with a lenient mode that drops a +// variant the path does not resolve in instead of failing; it then also +// returns the property names available where the last variants were lost, +// for did-you-mean when nothing resolved. +func (d *cliManifestDecoder) walkResponsePathViews(path string, responseSchema any, lenient bool) ([]*cliResolvedSchema, []string, error) { + segments, err := cliParseSingularPath(path) + if err != nil { + return nil, nil, err + } + views, err := d.artifactExpand(responseSchema, 0) + if err != nil { + return nil, nil, err + } if len(views) == 0 { - return nil, stderrors.New("response schema has no traversable object variant") + return nil, nil, stderrors.New("response schema has no traversable object variant") } + var candidates []string for i, segment := range segments { var next []*cliResolvedSchema + candidates = nil for _, view := range views { var child any if segment.IsIndex { if view.raw == nil || view.raw["type"] != "array" { - return nil, fmt.Errorf("segment %d ([%d]) does not address an array in every response variant", i+1, segment.Index) + if lenient { + continue + } + return nil, nil, fmt.Errorf("segment %d ([%d]) does not address an array in every response variant", i+1, segment.Index) } child = view.raw["items"] if child == nil { - return nil, fmt.Errorf("segment %d ([%d]) addresses an array without an items schema", i+1, segment.Index) + return nil, nil, fmt.Errorf("segment %d ([%d]) addresses an array without an items schema", i+1, segment.Index) } } else { var ok bool child, ok = view.properties[segment.Name] if !ok { - return nil, fmt.Errorf("segment %d (.%s) does not resolve in every response variant%s", i+1, segment.Name, cliDidYouMean(segment.Name, view.propOrder)) + if lenient { + candidates = append(candidates, view.propOrder...) + continue + } + return nil, nil, fmt.Errorf("segment %d (.%s) does not resolve in every response variant%s", i+1, segment.Name, cliDidYouMean(segment.Name, view.propOrder)) } } expanded, err := d.artifactExpand(child, 0) if err != nil { - return nil, fmt.Errorf("segment %d: %w", i+1, err) + return nil, nil, fmt.Errorf("segment %d: %w", i+1, err) } if len(expanded) == 0 { - return nil, fmt.Errorf("segment %d does not resolve to a schema in every response variant", i+1) + if lenient { + continue + } + return nil, nil, fmt.Errorf("segment %d does not resolve to a schema in every response variant", i+1) } next = append(next, expanded...) } views = next + if len(views) == 0 { + break + } } + return views, candidates, nil +} +// responseLeafFacts reports the facts of each non-null leaf view; a null +// union arm or a "null" entry in a type list is accepted as absence. +func (d *cliManifestDecoder) responseLeafFacts(views []*cliResolvedSchema) ([]*cliPropertyFacts, error) { facts := make([]*cliPropertyFacts, 0, len(views)) for _, view := range views { - propertyFacts, err := d.propertyFacts(view.raw, nil) + raw := cliWithoutNullType(view.raw) + if raw == nil { + continue + } + propertyFacts, err := d.propertyFacts(raw, nil) if err != nil { return nil, err } - if len(propertyFacts.kinds) != 1 || propertyFacts.kinds[0] != "string" { - return nil, stderrors.New("selected value must be a string in every response variant") - } - if requireEnum && len(propertyFacts.enum) == 0 { - return nil, stderrors.New("selected state must declare a non-empty enum in every response variant") - } facts = append(facts, propertyFacts) } + if len(facts) == 0 { + return nil, stderrors.New("selected value is always null") + } return facts, nil } +func cliWithoutNullType(schema map[string]any) map[string]any { + switch typ := schema["type"].(type) { + case string: + if typ == "null" { + return nil + } + case []any: + kept := make([]any, 0, len(typ)) + for _, t := range typ { + if t != "null" { + kept = append(kept, t) + } + } + if len(kept) == len(typ) { + return schema + } + if len(kept) == 0 { + return nil + } + stripped := maps.Clone(schema) + if len(kept) == 1 { + stripped["type"] = kept[0] + } else { + stripped["type"] = kept + } + return stripped + } + return schema +} + func (d *cliManifestDecoder) requireStringSchema(schema any) error { if schema == nil { return stderrors.New("parameter schema is missing") @@ -1771,12 +2048,19 @@ func (d *cliManifestDecoder) linkStreamProjection(owner string, stream *CLIComma } func (d *cliManifestDecoder) linkStreamSelection(owner, field, selectPath string, opInfo *cliOperationInfo) error { + _, err := d.walkStreamSelection(owner, field, selectPath, opInfo) + return err +} + +// walkStreamSelection is linkStreamSelection returning the walker, so a +// caller can inspect what the resolved leaves declared. +func (d *cliManifestDecoder) walkStreamSelection(owner, field, selectPath string, opInfo *cliOperationInfo) (*cliStreamPathWalker, error) { segments, err := cliParseSingularPath(selectPath) if err != nil { - return fmt.Errorf("%s %s: %w", owner, field, err) + return nil, fmt.Errorf("%s %s: %w", owner, field, err) } - walker := &cliStreamPathWalker{d: d, owner: owner, selectPath: selectPath, field: field} + walker := &cliStreamPathWalker{d: d, owner: owner, selectPath: selectPath, field: field, detail: field == cliStreamDetailField} for _, schema := range opInfo.streamSchemas { if schema == nil { walker.unverifiable("the streaming response declares no schema") @@ -1785,22 +2069,26 @@ func (d *cliManifestDecoder) linkStreamSelection(owner, field, selectPath string walker.walk(schema, segments, nil) } if walker.err != nil { - return walker.err + return nil, walker.err + } + expected := "a string" + if walker.detail { + expected = "an object or array" } if walker.resolved == 0 { if walker.unverified == 0 { - return fmt.Errorf("%s %s %s does not resolve in any event shape of the streaming response%s", owner, field, selectPath, walker.missSuggestion()) + return nil, fmt.Errorf("%s %s %s does not resolve in any event shape of the streaming response%s", owner, field, selectPath, walker.missSuggestion()) } - d.warnf("%s %s %s could not be verified against the streaming response schema (%s); events without a string at that path are skipped at runtime", owner, field, selectPath, walker.firstReason) - return nil + d.warnf("%s %s %s could not be verified against the streaming response schema (%s); events without %s at that path are skipped at runtime", owner, field, selectPath, walker.firstReason, expected) + return walker, nil } if err := d.lintProjection(owner, field, cliSingularPathToJQ(segments), "the streaming response declares no event schema", cliOperationProjectionSchema(opInfo), true); err != nil { - return err + return nil, err } if walker.unverified > 0 { - d.warnf("%s %s %s resolves in %d event shape(s) but could not be verified in every arm (%s); unverified events without a string at that path are skipped at runtime", owner, field, selectPath, walker.resolved, walker.firstReason) + d.warnf("%s %s %s resolves in %d event shape(s) but could not be verified in every arm (%s); unverified events without %s at that path are skipped at runtime", owner, field, selectPath, walker.resolved, walker.firstReason, expected) } - return nil + return walker, nil } func cliSingularPathToJQ(segments []cliPathSegment) string { @@ -1831,12 +2119,17 @@ type cliStreamPathWalker struct { owner string selectPath string field string + detail bool // the leaf must be an object or array (failure detail) - resolved int // arms where the full path resolved to a string/null leaf + resolved int // arms where the full path resolved to an accepted leaf unverified int // arms the walk could not see through firstReason string err error + stateEnum map[string]bool // closed enum members declared at resolved string leaves + stateNoEnum bool // a resolved string leaf declared no closed enum + stateOpen bool // a resolved string leaf declared an open enum + // missCandidates records, for the deepest segment that failed to resolve, // the property names that were available (for did-you-mean). missDepth int @@ -1911,6 +2204,22 @@ func (w *cliStreamPathWalker) walk(schema any, segments []cliPathSegment, seen [ w.err = fmt.Errorf("%s %s %s: referenced schema %q does not exist in components.schemas%s", w.owner, w.field, w.selectPath, name, cliDidYouMean(name, cliComponentNames(w.d.schemaIndex.components))) return } + if cliRefHasSiblingConstraints(schemaMap) { + if len(segments) == 0 { + w.checkLeaf(schemaMap, seen) + return + } + // The rewrite keeps seen free of name so its $ref branch still + // resolves; the marker stops a union that re-enters this + // reference from recursing forever. + marker := name + " (with siblings)" + if cliContains(seen, marker) { + w.unverifiable("schema reference cycle at " + name) + return + } + w.walk(cliRefSiblingsAsAllOf(schemaMap), segments, append(seen, marker)) + return + } w.walk(component, segments, append(seen, name)) return } @@ -2031,21 +2340,34 @@ type leafClass int const ( leafUnknown leafClass = iota // untyped: nothing provable leafString // string or null - leafOther // provably not a string + leafOther // provably not the expected kind + leafDetail // object or array (failure detail) ) -// checkLeaf classifies the schema the full path resolved to. Strings and -// nulls are what the raw writer emits; anything provably not a string is a -// generation error naming the kind. +// checkLeaf classifies the schema the full path resolved to. func (w *cliStreamPathWalker) checkLeaf(schemaMap map[string]any, seen []string) { - class, kind := w.classifyLeaf(schemaMap, seen) + var ( + class leafClass + kind string + ) + if w.detail { + class, kind = w.classifyDetailLeaf(schemaMap, seen) + } else { + class, kind = w.classifyLeaf(schemaMap, seen) + } switch class { - case leafString: + case leafString, leafDetail: w.resolved++ + if !w.detail { + w.recordLeafEnum(schemaMap, seen) + } case leafOther: - if w.field == "output.stream.metadata.select" { + switch { + case w.detail: + w.err = fmt.Errorf("%s %s %s resolves to %s in one event shape; the failure detail must be an object or an array", w.owner, w.field, w.selectPath, kind) + case w.field == "output.stream.metadata.select": w.err = fmt.Errorf("%s %s %s resolves to %s in one event shape; metadata must address a string field", w.owner, w.field, w.selectPath, kind) - } else { + default: w.err = fmt.Errorf("%s %s %s resolves to %s in one event shape; the streamed projection writes string values raw, so select must address a string field", w.owner, w.field, w.selectPath, kind) } default: @@ -2053,6 +2375,266 @@ func (w *cliStreamPathWalker) checkLeaf(schemaMap map[string]any, seen []string) } } +func (w *cliStreamPathWalker) recordLeafEnum(schemaMap map[string]any, seen []string) { + values, closed, open := w.leafEnum(schemaMap, seen) + switch { + case open: + w.stateOpen = true + case !closed: + w.stateNoEnum = true + default: + if w.stateEnum == nil { + w.stateEnum = map[string]bool{} + } + for _, value := range values { + w.stateEnum[value] = true + } + } +} + +// leafEnum reports the string members a string leaf is closed over. Every +// keyword on the schema constrains conjunctively: a const, a closed enum, the +// referenced component and each allOf arm contribute a member set, and the +// leaf is closed over their intersection. Arms that only set a type constrain +// nothing. A oneOf/anyOf then filters those members, or supplies them when no +// other keyword does and every non-null arm is closed. +func (w *cliStreamPathWalker) leafEnum(schemaMap map[string]any, seen []string) (values []string, closed, open bool) { + if unknownValues, ok := schemaMap["x-speakeasy-unknown-values"].(string); ok && unknownValues == "allow" { + return nil, false, true + } + var sources [][]string + if ref, ok := schemaMap["$ref"].(string); ok { + name := strings.TrimPrefix(ref, cliComponentRefPrefix) + if strings.HasPrefix(ref, cliComponentRefPrefix) && !cliContains(seen, name) { + if component, ok := w.d.schemaIndex.components[name].(map[string]any); ok { + refValues, refClosed, refOpen := w.leafEnum(component, append(seen, name)) + if refOpen { + return nil, false, true + } + if refClosed { + sources = append(sources, refValues) + } + } + } + } + if constVal, ok := schemaMap["const"]; ok { + switch s := constVal.(type) { + case string: + sources = append(sources, []string{s}) + case nil: + sources = append(sources, nil) + default: + return nil, false, false + } + } + if enum, ok := schemaMap["enum"].([]any); ok && len(enum) > 0 { + var members []string + for _, value := range enum { + if s, isString := value.(string); isString { + members = append(members, s) + } + } + sources = append(sources, members) + } + if allOf, ok := schemaMap["allOf"].([]any); ok { + for _, arm := range allOf { + armMap, ok := arm.(map[string]any) + if !ok { + continue + } + armValues, armClosed, armOpen := w.leafEnum(armMap, seen) + if armOpen { + return nil, false, true + } + if armClosed { + sources = append(sources, armValues) + } + } + } + closed = len(sources) > 0 + if closed { + values = cliIntersectStrings(sources) + } + for _, keyword := range []string{"oneOf", "anyOf"} { + members, _ := schemaMap[keyword].([]any) + if len(members) == 0 { + continue + } + var arms []cliEnumArm + for _, member := range members { + memberMap, ok := member.(map[string]any) + if !ok { + continue + } + if types := cliSchemaTypes(memberMap); len(types) == 1 && types[0] == "null" { + continue + } + if cliSchemaIsAnnotationOnly(memberMap) { + continue + } + memberValues, memberClosed, memberOpen := w.leafEnum(memberMap, seen) + if memberOpen { + return nil, false, true + } + arms = append(arms, cliEnumArm{values: memberValues, closed: memberClosed}) + } + if !closed { + for _, arm := range arms { + if !arm.closed { + return nil, false, false + } + for _, value := range arm.values { + if !cliContains(values, value) { + values = append(values, value) + } + } + } + closed = true + } + values = cliFilterUnionMembers(values, arms, keyword == "oneOf") + } + if !closed { + return nil, false, false + } + return values, true, false +} + +// cliEnumArm is one oneOf/anyOf arm of a string leaf: the members it is +// closed over, or any string when it is not closed. +type cliEnumArm struct { + values []string + closed bool +} + +// cliFilterUnionMembers keeps the candidates the union accepts: at least one +// arm for anyOf, exactly one for oneOf. An arm that is not closed may carry +// constraints that cannot be evaluated here, so a oneOf containing one only +// requires a match, which keeps every possible member in the table. +func cliFilterUnionMembers(candidates []string, arms []cliEnumArm, exactlyOne bool) []string { + for _, arm := range arms { + if !arm.closed { + exactlyOne = false + } + } + var out []string + for _, candidate := range candidates { + matches := 0 + for _, arm := range arms { + if !arm.closed || cliContains(arm.values, candidate) { + matches++ + } + } + if matches > 0 && (!exactlyOne || matches == 1) { + out = append(out, candidate) + } + } + return out +} + +// cliIntersectStrings keeps the members of the first set present in every +// other set, in first-set order. +func cliIntersectStrings(sets [][]string) []string { + var out []string + for _, candidate := range sets[0] { + if cliContains(out, candidate) { + continue + } + keep := true + for _, other := range sets[1:] { + if !cliContains(other, candidate) { + keep = false + break + } + } + if keep { + out = append(out, candidate) + } + } + return out +} + +// cliRefHasSiblingConstraints reports a reference whose siblings narrow the +// referenced schema, so the leaf is checked here rather than at the target. +func cliRefHasSiblingConstraints(schemaMap map[string]any) bool { + siblings := make(map[string]any, len(schemaMap)) + for key, value := range schemaMap { + if key != "$ref" { + siblings[key] = value + } + } + return !cliSchemaIsAnnotationOnly(siblings) +} + +// cliSchemaIsAnnotationOnly reports a schema that constrains nothing (an +// allOf branch carrying only a description, say). +func cliSchemaIsAnnotationOnly(schemaMap map[string]any) bool { + for _, key := range []string{"$ref", "type", "enum", "const", "oneOf", "anyOf", "allOf", "properties", "items", "x-speakeasy-unknown-values"} { + if _, present := schemaMap[key]; present { + return false + } + } + return true +} + +// classifyDetailLeaf accepts object and array leaves; a null arm is neutral. +func (w *cliStreamPathWalker) classifyDetailLeaf(schemaMap map[string]any, seen []string) (leafClass, string) { + if ref, ok := schemaMap["$ref"].(string); ok { + name := strings.TrimPrefix(ref, cliComponentRefPrefix) + if !strings.HasPrefix(ref, cliComponentRefPrefix) || cliContains(seen, name) { + return leafUnknown, "" + } + component, ok := w.d.schemaIndex.components[name].(map[string]any) + if !ok { + return leafUnknown, "" + } + return w.classifyDetailLeaf(component, append(seen, name)) + } + + var branches []any + if members := cliUnionMembers(schemaMap); members != nil { + branches = append(branches, members...) + } + if allOf, ok := schemaMap["allOf"].([]any); ok { + branches = append(branches, allOf...) + } + if len(branches) > 0 { + result := leafUnknown + for _, branch := range branches { + branchMap, ok := branch.(map[string]any) + if !ok { + continue + } + class, kind := w.classifyDetailLeaf(branchMap, seen) + switch class { + case leafOther: + return leafOther, kind + case leafDetail: + result = leafDetail + } + } + return result, "" + } + + types := cliSchemaTypes(schemaMap) + _, hasProperties := schemaMap["properties"] + _, hasItems := schemaMap["items"] + if cliContains(types, "object") || cliContains(types, "array") || hasProperties || hasItems { + return leafDetail, "" + } + if constVal, ok := schemaMap["const"]; ok { + return leafOther, "a constant " + jsonValueKind(constVal) + } + if enum, ok := schemaMap["enum"].([]any); ok && len(enum) > 0 { + return leafOther, "an enum" + } + for _, typ := range types { + if typ != "null" { + return leafOther, cliKindArticle(cliSchemaTypeKind(typ)) + } + } + return leafUnknown, "" +} + // classifyLeaf inspects a leaf schema: const/enum values, type (scalar or // list), union arms (every arm must agree), and allOf branches (any typed // branch decides; conflicting branches are an error). @@ -2966,6 +3548,28 @@ func cliUnionMemberRefs(schema map[string]any) []string { return refs } +// cliRefSiblingsAsAllOf rewrites a reference with sibling keywords as an +// implicit allOf: JSON Schema 2020-12 applies keywords beside $ref +// conjunctively, and the existing composition (with its cycle detection via +// seen) then handles the reference branch alongside the siblings. +func cliRefSiblingsAsAllOf(schemaMap map[string]any) map[string]any { + branches := []any{map[string]any{"$ref": schemaMap["$ref"]}} + rest := map[string]any{} + for k, v := range schemaMap { + switch k { + case "$ref": + case "allOf": + if list, ok := v.([]any); ok { + branches = append(branches, list...) + } + default: + rest[k] = v + } + } + rest["allOf"] = branches + return rest +} + func cliTrimRefPrefixes(refs []string) []string { out := make([]string, 0, len(refs)) for _, ref := range refs { @@ -3006,25 +3610,7 @@ func (d *cliManifestDecoder) resolveObjectSchema(schema any, seen []string) (*cl return nil, fmt.Errorf("referenced schema %q does not exist in components.schemas%s", name, cliDidYouMean(name, cliComponentNames(d.schemaIndex.components))) } if len(schemaMap) > 1 { - // JSON Schema 2020-12: keywords beside $ref apply conjunctively. - // Rewrite as an implicit allOf so the existing composition (and - // its cycle detection via seen) handles the reference branch - // alongside the siblings instead of discarding them. - branches := []any{map[string]any{"$ref": ref}} - rest := map[string]any{} - for k, v := range schemaMap { - switch k { - case "$ref": - case "allOf": - if list, ok := v.([]any); ok { - branches = append(branches, list...) - } - default: - rest[k] = v - } - } - rest["allOf"] = branches - resolved, err := d.resolveObjectSchema(rest, seen) + resolved, err := d.resolveObjectSchema(cliRefSiblingsAsAllOf(schemaMap), seen) if err != nil { return nil, err } diff --git a/internal/extensions/cli_commands_test.go b/internal/extensions/cli_commands_test.go index be008bda0..3fbf0ed5c 100644 --- a/internal/extensions/cli_commands_test.go +++ b/internal/extensions/cli_commands_test.go @@ -2200,6 +2200,7 @@ commands: CLI_ASYNC_FAILED: async failed CLI_ASYNC_TIMEOUT: async timeout CLI_ASYNC_UNKNOWN_STATE: async unknown state + CLI_OUTCOME_FAILED: outcome failed `) require.NoError(t, err) require.Len(t, manifest.Commands, 1) @@ -3443,7 +3444,7 @@ commands: say: op: StreamTask output: {} -`, "output must declare artifact or stream") +`, "output must declare artifact, stream, or outcome") requireDecodeError(t, ` version: 1 @@ -5424,6 +5425,309 @@ operations: assert.Contains(t, warnings[1], "could not be verified") } +const validOutcomeManifest = ` +version: 1 +commands: + check: + op: GetTask + output: + outcome: + statePointer: $.status + states: + in_progress: pending + completed: success + failed: failure + requires_action: handoff + follow: + op: StreamTask + args: + prompt: {to: $.prompt, variadic: true} + output: + stream: + select: $.data.delta.text + outcome: + streamStatePointer: $.data.reason + streamErrorPointer: $.data.error + states: + done: success + cancelled: failure +operations: + GetTask: + output: + outcome: + statePointer: $.status + errorPointer: $.steps + states: {in_progress: pending, completed: success, failed: failure, requires_action: handoff} +` + +func TestCLICommands_OutcomeDecodeAndLink(t *testing.T) { + manifest, _, err := decodeCLITest(t, validOutcomeManifest) + require.NoError(t, err) + require.Len(t, manifest.Commands, 2) + + check := manifest.Commands[0].Output.Outcome + require.NotNil(t, check) + assert.Equal(t, "$.status", check.StateFrom) + assert.Equal(t, "/status", check.StatePointer) + assert.Empty(t, check.StreamStatePointer) + assert.Equal(t, map[string]string{"in_progress": "pending", "completed": "success", "failed": "failure", "requires_action": "handoff"}, check.States) + assert.Empty(t, check.ErrorPointer) + assert.Empty(t, check.ErrorMessageField) + + follow := manifest.Commands[1].Output.Outcome + require.NotNil(t, follow) + assert.Empty(t, follow.StatePointer) + assert.Equal(t, "$.data.reason", follow.StreamStateFrom) + assert.Equal(t, "/data/reason", follow.StreamStatePointer) + assert.Equal(t, "/data/error", follow.StreamErrorPointer) + assert.Equal(t, "message", follow.ErrorMessageField) + assert.Equal(t, "/data/delta/text", manifest.Commands[1].Output.Stream.Pointer) + + require.Len(t, manifest.Operations, 1) + op := manifest.Operations[0].Output.Outcome + require.NotNil(t, op) + assert.Equal(t, "/status", op.StatePointer) + assert.Equal(t, "/steps", op.ErrorPointer) + assert.Equal(t, "message", op.ErrorMessageField) +} + +func TestCLICommands_OutcomeNullableLeaves(t *testing.T) { + spec := strings.Replace(cliCommandsTestSpec, " status:\n type: string\n enum: [in_progress, completed, failed, requires_action]\n steps:\n type: array\n", " status:\n type: [string, \"null\"]\n enum: [in_progress, completed, failed, requires_action, null]\n steps:\n type: [array, \"null\"]\n", 1) + require.NotEqual(t, cliCommandsTestSpec, spec) + manifest, _, err := decodeCLIWithSpec(t, spec, validOutcomeManifest) + require.NoError(t, err) + require.Len(t, manifest.Operations, 1) + assert.Equal(t, "/steps", manifest.Operations[0].Output.Outcome.ErrorPointer) +} + +func TestCLICommands_AsyncNullableState(t *testing.T) { + spec := strings.Replace(cliCommandsTestSpec, " status:\n type: string\n enum: [in_progress, completed, failed, requires_action]\n", " status:\n type: [string, \"null\"]\n enum: [in_progress, completed, failed, requires_action, null]\n", 1) + require.NotEqual(t, cliCommandsTestSpec, spec) + manifest, _, err := decodeCLIWithSpec(t, spec, strings.Replace(validAsyncManifest, " params:\n stream: false\n", "", 1)) + require.NoError(t, err) + require.Len(t, manifest.Commands, 1) + require.NotNil(t, manifest.Commands[0].Async) + assert.Equal(t, "/status", manifest.Commands[0].Async.StatePointer) +} + +func TestCLICommands_OutcomeErrors(t *testing.T) { + base := validOutcomeManifest + for _, tc := range []struct { + name string + manifest string + want string + }{ + {"no pointer", strings.Replace(base, " statePointer: $.status\n states:\n in_progress: pending", " states:\n in_progress: pending", 1), "requires statePointer (the state in the JSON response) or streamStatePointer"}, + {"no failure class", strings.Replace(base, " failed: failure\n requires_action: handoff\n follow", " failed: success\n requires_action: handoff\n follow", 1), "must classify at least one state as failure"}, + {"bad classification", strings.Replace(base, "failed: failure\n requires_action", "failed: fatal\n requires_action", 1), `output.outcome.states["failed"] has unsupported classification "fatal"`}, + {"unknown key", strings.Replace(base, " statePointer: $.status\n states:", " pointer: $.status\n states:", 1), `output.outcome has unknown key "pointer"`}, + {"not an enum member", strings.Replace(base, " requires_action: handoff\n follow", " requires_action: handoff\n exploded: failure\n follow", 1), `classifies "exploded", which is not an enum member at $.status`}, + {"unclassified enum member", strings.Replace(base, " requires_action: handoff\n follow", " follow", 1), `does not classify enum member "requires_action"`}, + {"state pointer not an enum", strings.Replace(base, " statePointer: $.status\n states:\n in_progress", " statePointer: $.id\n states:\n in_progress", 1), "selected state must declare a non-empty enum"}, + {"stream pointer on discrete operation", strings.Replace(base, " statePointer: $.status\n states:\n in_progress", " streamStatePointer: $.status\n states:\n in_progress", 1), "has no streaming (text/event-stream or JSONL) success response"}, + {"stream pointer misses every event", strings.Replace(base, "streamStatePointer: $.data.reason", "streamStatePointer: $.data.outcome", 1), "does not resolve in any event shape"}, + {"error pointer must be object or array", strings.Replace(base, "errorPointer: $.steps", "errorPointer: $.id", 1), "selected failure detail must be an object or an array"}, + {"error pointer misses every response variant", strings.Replace(base, "errorPointer: $.steps", "errorPointer: $.step", 1), `output.outcome.errorPointer $.step does not resolve in any application/json 2xx response variant of operation "GetTask" (did you mean "steps"?)`}, + {"stream state not an enum member", strings.Replace(base, " done: success\n cancelled: failure", " done: success\n canceled: failure", 1), `output.outcome.states classifies "canceled", which is not an enum member at $.data.reason in operation "StreamTask" (did you mean "cancelled"?)`}, + {"stream enum member unclassified", strings.Replace(base, " done: success\n cancelled: failure", " cancelled: failure", 1), `output.outcome.states does not classify enum member "done" from $.data.reason in operation "StreamTask"`}, + {"stream error pointer misses every event", strings.Replace(base, "streamErrorPointer: $.data.error", "streamErrorPointer: $.data.failure", 1), "output.outcome.streamErrorPointer $.data.failure does not resolve in any event shape"}, + {"stream error pointer must be object or array", strings.Replace(base, "streamErrorPointer: $.data.error", "streamErrorPointer: $.data.event_id", 1), "resolves to a string in one event shape; the failure detail must be an object or an array"}, + {"error pointer without state pointer", strings.Replace(base, " streamStatePointer: $.data.reason\n", " streamStatePointer: $.data.reason\n errorPointer: $.data.reason\n", 1), "output.outcome.errorPointer requires statePointer"}, + {"message field without pointer", strings.Replace(base, " statePointer: $.status\n states:\n in_progress", " statePointer: $.status\n errorMessageField: detail\n states:\n in_progress", 1), "errorMessageField requires errorPointer or streamErrorPointer"}, + {"async conflict", strings.TrimSuffix(validAsyncManifest, "\n") + "\n output:\n outcome:\n statePointer: $.status\n states: {in_progress: pending, completed: success, failed: failure, requires_action: handoff}\n", "declares both async and output.outcome"}, + {"planned command", "version: 1\ncommands:\n later:\n planned: later\n output:\n outcome:\n statePointer: $.status\n states: {failed: failure}\n", "planned command and cannot declare output"}, + } { + t.Run(tc.name, func(t *testing.T) { + _, _, err := decodeCLITest(t, tc.manifest) + require.Error(t, err) + assert.Contains(t, err.Error(), tc.want) + }) + } +} + +const cliOutcomeUnionTaskResult = ` TaskResult: + oneOf: + - $ref: '#/components/schemas/TaskSettled' + - $ref: '#/components/schemas/TaskFailed' + TaskSettled: + type: object + properties: + id: + type: string + status: + type: string + enum: [in_progress, completed, failed, requires_action] + TaskFailed: + type: object + properties: + id: + type: string + status: + type: string + enum: [in_progress, completed, failed, requires_action] + steps: + type: array + items: + oneOf: + - $ref: '#/components/schemas/NoteStep' + - $ref: '#/components/schemas/OutputStep' +` + +func TestCLICommands_OutcomeErrorPointerAnyVariant(t *testing.T) { + start := strings.Index(cliCommandsTestSpec, " TaskResult:\n") + end := strings.Index(cliCommandsTestSpec, " NoteStep:\n") + require.True(t, start >= 0 && end > start) + spec := cliCommandsTestSpec[:start] + cliOutcomeUnionTaskResult + cliCommandsTestSpec[end:] + + manifest, _, err := decodeCLIWithSpec(t, spec, validOutcomeManifest) + require.NoError(t, err) + require.Len(t, manifest.Operations, 1) + assert.Equal(t, "/steps", manifest.Operations[0].Output.Outcome.ErrorPointer) + + _, _, err = decodeCLIWithSpec(t, spec, strings.Replace(validOutcomeManifest, "errorPointer: $.steps", "errorPointer: $.id", 1)) + require.Error(t, err) + assert.Contains(t, err.Error(), "selected failure detail must be an object or an array") + + _, _, err = decodeCLIWithSpec(t, spec, strings.Replace(validOutcomeManifest, "errorPointer: $.steps", "errorPointer: $.detail", 1)) + require.Error(t, err) + assert.Contains(t, err.Error(), `output.outcome.errorPointer $.detail does not resolve in any application/json 2xx response variant of operation "GetTask"`) +} + +func TestCLICommands_OutcomeStreamStatesWithoutEnum(t *testing.T) { + manifest, warnings, err := decodeCLITest(t, strings.Replace(validOutcomeManifest, "streamStatePointer: $.data.reason", "streamStatePointer: $.data.event_id", 1)) + require.NoError(t, err) + require.Len(t, manifest.Commands, 2) + assert.Equal(t, "/data/event_id", manifest.Commands[1].Output.Outcome.StreamStatePointer) + require.Len(t, warnings, 1, warnings) + assert.Equal(t, `command "follow" output.outcome.streamStatePointer $.data.event_id in operation "StreamTask" does not select a closed string enum in every event shape; states cannot be verified at generation time, and a state missing from the table keeps exit 0 at runtime`, warnings[0]) + + _, warnings, err = decodeCLITest(t, validOutcomeManifest) + require.NoError(t, err) + assert.Empty(t, warnings) +} + +func TestCLICommands_OutcomeInheritedFromOperation(t *testing.T) { + manifest, _, err := decodeCLITest(t, ` +version: 1 +commands: + check: + op: GetTask + own: + op: GetTask + output: + outcome: + statePointer: $.status + states: {in_progress: pending, completed: success, failed: failure, requires_action: failure} +operations: + GetTask: + output: + outcome: + statePointer: $.status + errorPointer: $.steps + states: {in_progress: pending, completed: success, failed: failure, requires_action: handoff} +`) + require.NoError(t, err) + require.Len(t, manifest.Commands, 2) + require.Len(t, manifest.Operations, 1) + + inherited := manifest.Commands[0].Output.Outcome + require.NotNil(t, inherited) + assert.Equal(t, manifest.Operations[0].Output.Outcome, inherited) + assert.Equal(t, "/steps", inherited.ErrorPointer) + + own := manifest.Commands[1].Output.Outcome + require.NotNil(t, own) + assert.Equal(t, "failure", own.States["requires_action"]) + assert.Empty(t, own.ErrorPointer) + + asyncManifest := strings.Replace(validAsyncManifest, " params:\n stream: false\n", "", 1) + async, _, err := decodeCLITest(t, strings.TrimSuffix(asyncManifest, "\n")+` +operations: + CreateTask: + output: + outcome: + statePointer: $.status + states: {in_progress: pending, completed: success, failed: failure, requires_action: handoff} +`) + require.NoError(t, err) + require.Len(t, async.Commands, 1) + require.NotNil(t, async.Commands[0].Async) + if async.Commands[0].Output != nil { + assert.Nil(t, async.Commands[0].Output.Outcome, "an async intent keeps async.states as its only classification") + } +} + +func TestCLICommands_OutcomeStreamStateOneOfExcludesOverlap(t *testing.T) { + const stopReason = " StopReason:\n type: string\n enum: [done, cancelled]\n" + require.Contains(t, cliCommandsTestSpec, stopReason) + const table = " done: success\n cancelled: failure" + require.Contains(t, validOutcomeManifest, table) + + spec := strings.Replace(cliCommandsTestSpec, stopReason, " StopReason:\n type: string\n oneOf:\n - enum: [done, cancelled]\n - enum: [cancelled, extra]\n", 1) + _, warnings, err := decodeCLIWithSpec(t, spec, strings.Replace(validOutcomeManifest, table, " done: success\n extra: failure", 1)) + require.NoError(t, err) + assert.Empty(t, warnings) + + _, _, err = decodeCLIWithSpec(t, spec, strings.Replace(validOutcomeManifest, table, " done: success\n cancelled: failure\n extra: failure", 1)) + require.Error(t, err) + assert.Contains(t, err.Error(), `output.outcome.states classifies "cancelled", which is not an enum member at $.data.reason`) + + anyOf := strings.Replace(cliCommandsTestSpec, stopReason, " StopReason:\n type: string\n anyOf:\n - enum: [done, cancelled]\n - enum: [cancelled, extra]\n", 1) + _, _, err = decodeCLIWithSpec(t, anyOf, strings.Replace(validOutcomeManifest, table, " done: success\n cancelled: failure\n extra: failure", 1)) + require.NoError(t, err) + + unbounded := strings.Replace(cliCommandsTestSpec, stopReason, " StopReason:\n type: string\n enum: [done, cancelled]\n oneOf:\n - enum: [done, cancelled]\n - type: string\n", 1) + _, _, err = decodeCLIWithSpec(t, unbounded, validOutcomeManifest) + require.NoError(t, err) +} + +func TestCLICommands_OutcomeStreamStateRefSiblingCycle(t *testing.T) { + const stopArm = " - $ref: '#/components/schemas/StepStopEvent'\n" + require.Contains(t, cliCommandsTestSpec, stopArm) + spec := strings.Replace(cliCommandsTestSpec, stopArm, stopArm+" - $ref: '#/components/schemas/TaskEvent'\n properties:\n reason:\n enum: [done]\n", 1) + + _, warnings, err := decodeCLIWithSpec(t, spec, validOutcomeManifest) + require.NoError(t, err) + require.NotEmpty(t, warnings) + for _, warning := range warnings { + assert.Contains(t, warning, "schema reference cycle at TaskEvent") + } +} + +func TestCLICommands_OutcomeStreamStateAllOfIntersectsEnums(t *testing.T) { + const stopReason = " StopReason:\n type: string\n enum: [done, cancelled]\n" + require.Contains(t, cliCommandsTestSpec, stopReason) + typoTable := strings.Replace(validOutcomeManifest, " done: success\n cancelled: failure", " done: success\n canceled: failure", 1) + + const stopArm = " - $ref: '#/components/schemas/StepStopEvent'\n" + require.Contains(t, cliCommandsTestSpec, stopArm) + + for _, tc := range []struct { + name string + schema string + stopArmSiblings string + }{ + {"type-only arm", " StopReason:\n allOf:\n - type: string\n - enum: [done, cancelled]\n", ""}, + {"two enum arms", " StopReason:\n allOf:\n - type: string\n enum: [done, cancelled, extra]\n - enum: [done, cancelled]\n", ""}, + {"enum beside ref", " StopReason:\n $ref: '#/components/schemas/StopReasonBase'\n enum: [done, cancelled]\n StopReasonBase:\n type: string\n enum: [done, cancelled, extra]\n", ""}, + {"properties beside ref on the path", " StopReason:\n type: string\n enum: [done, cancelled, extra]\n", " properties:\n reason:\n enum: [done, cancelled]\n"}, + {"enum beside union with an unbounded arm", " StopReason:\n type: string\n enum: [done, cancelled]\n anyOf:\n - type: string\n - enum: [done]\n", ""}, + } { + t.Run(tc.name, func(t *testing.T) { + spec := strings.Replace(cliCommandsTestSpec, stopReason, tc.schema, 1) + spec = strings.Replace(spec, stopArm, stopArm+tc.stopArmSiblings, 1) + + _, warnings, err := decodeCLIWithSpec(t, spec, validOutcomeManifest) + require.NoError(t, err) + assert.Empty(t, warnings) + + _, _, err = decodeCLIWithSpec(t, spec, typoTable) + require.Error(t, err) + assert.Contains(t, err.Error(), `output.outcome.states classifies "canceled", which is not an enum member at $.data.reason in operation "StreamTask" (did you mean "cancelled"?)`) + }) + } +} + func TestCLICommands_FlagImplies(t *testing.T) { manifest, _, err := decodeCLITest(t, ` version: 1 diff --git a/pkg/generate/snapshots/cli_release_go_test.go b/pkg/generate/snapshots/cli_release_go_test.go index 79b39b0b3..ba34dbe62 100644 --- a/pkg/generate/snapshots/cli_release_go_test.go +++ b/pkg/generate/snapshots/cli_release_go_test.go @@ -464,7 +464,7 @@ Outside agent mode, explicit JSON and ` + "`" + `--jq` + "`" + ` preserve the co } ` + "`" + `` + "`" + `` + "`" + ` -` + "`" + `error_type` + "`" + ` is one of ` + "`" + `authentication_error` + "`" + `, ` + "`" + `authorization_error` + "`" + `, ` + "`" + `not_found` + "`" + `, ` + "`" + `validation_error` + "`" + `, ` + "`" + `rate_limit_error` + "`" + `, ` + "`" + `server_error` + "`" + `, ` + "`" + `api_error` + "`" + `, ` + "`" + `connection_error` + "`" + `, ` + "`" + `protocol_error` + "`" + `, ` + "`" + `runtime_error` + "`" + `, ` + "`" + `unsupported_error` + "`" + `, ` + "`" + `async_failed` + "`" + `, ` + "`" + `async_timeout` + "`" + `, ` + "`" + `async_unknown_state` + "`" + `. Classification derives from the HTTP status and transport evidence; ` + "`" + `error_reason` + "`" + ` is absent for API errors. Status-less local failures may use ` + "`" + `CLI_VALIDATION` + "`" + `, ` + "`" + `CLI_CONNECTION` + "`" + `, ` + "`" + `CLI_PROTOCOL` + "`" + `, ` + "`" + `CLI_RUNTIME` + "`" + `, ` + "`" + `CLI_UNAVAILABLE` + "`" + `, ` + "`" + `CLI_AUTHENTICATION` + "`" + `, or the async polling reasons ` + "`" + `CLI_ASYNC_FAILED` + "`" + `, ` + "`" + `CLI_ASYNC_TIMEOUT` + "`" + `, and ` + "`" + `CLI_ASYNC_UNKNOWN_STATE` + "`" + `. ` + "`" + `hints` + "`" + ` preserves server guidance first, adds the most specific local taxonomy guidance, then typed CLI and command-specific guidance, removing exact duplicates. ` + "`" + `exit_code` + "`" + ` is always the code for the final ` + "`" + `error_type` + "`" + ` shown in the envelope: 1 runtime, 2 usage, or 3 authentication/authorization. +` + "`" + `error_type` + "`" + ` is one of ` + "`" + `authentication_error` + "`" + `, ` + "`" + `authorization_error` + "`" + `, ` + "`" + `not_found` + "`" + `, ` + "`" + `validation_error` + "`" + `, ` + "`" + `rate_limit_error` + "`" + `, ` + "`" + `server_error` + "`" + `, ` + "`" + `api_error` + "`" + `, ` + "`" + `connection_error` + "`" + `, ` + "`" + `protocol_error` + "`" + `, ` + "`" + `runtime_error` + "`" + `, ` + "`" + `unsupported_error` + "`" + `, ` + "`" + `async_failed` + "`" + `, ` + "`" + `async_timeout` + "`" + `, ` + "`" + `async_unknown_state` + "`" + `. Classification derives from the HTTP status and transport evidence; ` + "`" + `error_reason` + "`" + ` is absent for API errors. Status-less local failures may use ` + "`" + `CLI_VALIDATION` + "`" + `, ` + "`" + `CLI_CONNECTION` + "`" + `, ` + "`" + `CLI_PROTOCOL` + "`" + `, ` + "`" + `CLI_RUNTIME` + "`" + `, ` + "`" + `CLI_UNAVAILABLE` + "`" + `, ` + "`" + `CLI_AUTHENTICATION` + "`" + `, the async polling reasons ` + "`" + `CLI_ASYNC_FAILED` + "`" + `, ` + "`" + `CLI_ASYNC_TIMEOUT` + "`" + `, and ` + "`" + `CLI_ASYNC_UNKNOWN_STATE` + "`" + `, or ` + "`" + `CLI_OUTCOME_FAILED` + "`" + ` when a successful response reports a declared failure state; that envelope also carries ` + "`" + `state` + "`" + ` (the reported state) and, when the command declares an error pointer, ` + "`" + `error` + "`" + ` (the failure detail taken from the response). ` + "`" + `hints` + "`" + ` preserves server guidance first, adds the most specific local taxonomy guidance, then typed CLI and command-specific guidance, removing exact duplicates. ` + "`" + `exit_code` + "`" + ` is always the code for the final ` + "`" + `error_type` + "`" + ` shown in the envelope: 1 runtime, 2 usage, or 3 authentication/authorization. @@ -1514,7 +1514,7 @@ Outside agent mode, explicit JSON and ` + "`" + `--jq` + "`" + ` preserve the co } ` + "`" + `` + "`" + `` + "`" + ` -` + "`" + `error_type` + "`" + ` is one of ` + "`" + `authentication_error` + "`" + `, ` + "`" + `authorization_error` + "`" + `, ` + "`" + `not_found` + "`" + `, ` + "`" + `validation_error` + "`" + `, ` + "`" + `rate_limit_error` + "`" + `, ` + "`" + `server_error` + "`" + `, ` + "`" + `api_error` + "`" + `, ` + "`" + `connection_error` + "`" + `, ` + "`" + `protocol_error` + "`" + `, ` + "`" + `runtime_error` + "`" + `, ` + "`" + `unsupported_error` + "`" + `, ` + "`" + `async_failed` + "`" + `, ` + "`" + `async_timeout` + "`" + `, ` + "`" + `async_unknown_state` + "`" + `. Classification derives from the HTTP status and transport evidence; ` + "`" + `error_reason` + "`" + ` is absent for API errors. Status-less local failures may use ` + "`" + `CLI_VALIDATION` + "`" + `, ` + "`" + `CLI_CONNECTION` + "`" + `, ` + "`" + `CLI_PROTOCOL` + "`" + `, ` + "`" + `CLI_RUNTIME` + "`" + `, ` + "`" + `CLI_UNAVAILABLE` + "`" + `, ` + "`" + `CLI_AUTHENTICATION` + "`" + `, or the async polling reasons ` + "`" + `CLI_ASYNC_FAILED` + "`" + `, ` + "`" + `CLI_ASYNC_TIMEOUT` + "`" + `, and ` + "`" + `CLI_ASYNC_UNKNOWN_STATE` + "`" + `. ` + "`" + `hints` + "`" + ` preserves server guidance first, adds the most specific local taxonomy guidance, then typed CLI and command-specific guidance, removing exact duplicates. ` + "`" + `exit_code` + "`" + ` is always the code for the final ` + "`" + `error_type` + "`" + ` shown in the envelope: 1 runtime, 2 usage, or 3 authentication/authorization. +` + "`" + `error_type` + "`" + ` is one of ` + "`" + `authentication_error` + "`" + `, ` + "`" + `authorization_error` + "`" + `, ` + "`" + `not_found` + "`" + `, ` + "`" + `validation_error` + "`" + `, ` + "`" + `rate_limit_error` + "`" + `, ` + "`" + `server_error` + "`" + `, ` + "`" + `api_error` + "`" + `, ` + "`" + `connection_error` + "`" + `, ` + "`" + `protocol_error` + "`" + `, ` + "`" + `runtime_error` + "`" + `, ` + "`" + `unsupported_error` + "`" + `, ` + "`" + `async_failed` + "`" + `, ` + "`" + `async_timeout` + "`" + `, ` + "`" + `async_unknown_state` + "`" + `. Classification derives from the HTTP status and transport evidence; ` + "`" + `error_reason` + "`" + ` is absent for API errors. Status-less local failures may use ` + "`" + `CLI_VALIDATION` + "`" + `, ` + "`" + `CLI_CONNECTION` + "`" + `, ` + "`" + `CLI_PROTOCOL` + "`" + `, ` + "`" + `CLI_RUNTIME` + "`" + `, ` + "`" + `CLI_UNAVAILABLE` + "`" + `, ` + "`" + `CLI_AUTHENTICATION` + "`" + `, the async polling reasons ` + "`" + `CLI_ASYNC_FAILED` + "`" + `, ` + "`" + `CLI_ASYNC_TIMEOUT` + "`" + `, and ` + "`" + `CLI_ASYNC_UNKNOWN_STATE` + "`" + `, or ` + "`" + `CLI_OUTCOME_FAILED` + "`" + ` when a successful response reports a declared failure state; that envelope also carries ` + "`" + `state` + "`" + ` (the reported state) and, when the command declares an error pointer, ` + "`" + `error` + "`" + ` (the failure detail taken from the response). ` + "`" + `hints` + "`" + ` preserves server guidance first, adds the most specific local taxonomy guidance, then typed CLI and command-specific guidance, removing exact duplicates. ` + "`" + `exit_code` + "`" + ` is always the code for the final ` + "`" + `error_type` + "`" + ` shown in the envelope: 1 runtime, 2 usage, or 3 authentication/authorization. @@ -2623,7 +2623,7 @@ Outside agent mode, explicit JSON and ` + "`" + `--jq` + "`" + ` preserve the co } ` + "`" + `` + "`" + `` + "`" + ` -` + "`" + `error_type` + "`" + ` is one of ` + "`" + `authentication_error` + "`" + `, ` + "`" + `authorization_error` + "`" + `, ` + "`" + `not_found` + "`" + `, ` + "`" + `validation_error` + "`" + `, ` + "`" + `rate_limit_error` + "`" + `, ` + "`" + `server_error` + "`" + `, ` + "`" + `api_error` + "`" + `, ` + "`" + `connection_error` + "`" + `, ` + "`" + `protocol_error` + "`" + `, ` + "`" + `runtime_error` + "`" + `, ` + "`" + `unsupported_error` + "`" + `, ` + "`" + `async_failed` + "`" + `, ` + "`" + `async_timeout` + "`" + `, ` + "`" + `async_unknown_state` + "`" + `. Classification derives from the HTTP status and transport evidence; ` + "`" + `error_reason` + "`" + ` is absent for API errors. Status-less local failures may use ` + "`" + `CLI_VALIDATION` + "`" + `, ` + "`" + `CLI_CONNECTION` + "`" + `, ` + "`" + `CLI_PROTOCOL` + "`" + `, ` + "`" + `CLI_RUNTIME` + "`" + `, ` + "`" + `CLI_UNAVAILABLE` + "`" + `, ` + "`" + `CLI_AUTHENTICATION` + "`" + `, or the async polling reasons ` + "`" + `CLI_ASYNC_FAILED` + "`" + `, ` + "`" + `CLI_ASYNC_TIMEOUT` + "`" + `, and ` + "`" + `CLI_ASYNC_UNKNOWN_STATE` + "`" + `. ` + "`" + `hints` + "`" + ` preserves server guidance first, adds the most specific local taxonomy guidance, then typed CLI and command-specific guidance, removing exact duplicates. ` + "`" + `exit_code` + "`" + ` is always the code for the final ` + "`" + `error_type` + "`" + ` shown in the envelope: 1 runtime, 2 usage, or 3 authentication/authorization. +` + "`" + `error_type` + "`" + ` is one of ` + "`" + `authentication_error` + "`" + `, ` + "`" + `authorization_error` + "`" + `, ` + "`" + `not_found` + "`" + `, ` + "`" + `validation_error` + "`" + `, ` + "`" + `rate_limit_error` + "`" + `, ` + "`" + `server_error` + "`" + `, ` + "`" + `api_error` + "`" + `, ` + "`" + `connection_error` + "`" + `, ` + "`" + `protocol_error` + "`" + `, ` + "`" + `runtime_error` + "`" + `, ` + "`" + `unsupported_error` + "`" + `, ` + "`" + `async_failed` + "`" + `, ` + "`" + `async_timeout` + "`" + `, ` + "`" + `async_unknown_state` + "`" + `. Classification derives from the HTTP status and transport evidence; ` + "`" + `error_reason` + "`" + ` is absent for API errors. Status-less local failures may use ` + "`" + `CLI_VALIDATION` + "`" + `, ` + "`" + `CLI_CONNECTION` + "`" + `, ` + "`" + `CLI_PROTOCOL` + "`" + `, ` + "`" + `CLI_RUNTIME` + "`" + `, ` + "`" + `CLI_UNAVAILABLE` + "`" + `, ` + "`" + `CLI_AUTHENTICATION` + "`" + `, the async polling reasons ` + "`" + `CLI_ASYNC_FAILED` + "`" + `, ` + "`" + `CLI_ASYNC_TIMEOUT` + "`" + `, and ` + "`" + `CLI_ASYNC_UNKNOWN_STATE` + "`" + `, or ` + "`" + `CLI_OUTCOME_FAILED` + "`" + ` when a successful response reports a declared failure state; that envelope also carries ` + "`" + `state` + "`" + ` (the reported state) and, when the command declares an error pointer, ` + "`" + `error` + "`" + ` (the failure detail taken from the response). ` + "`" + `hints` + "`" + ` preserves server guidance first, adds the most specific local taxonomy guidance, then typed CLI and command-specific guidance, removing exact duplicates. ` + "`" + `exit_code` + "`" + ` is always the code for the final ` + "`" + `error_type` + "`" + ` shown in the envelope: 1 runtime, 2 usage, or 3 authentication/authorization. diff --git a/templates/perms.go b/templates/perms.go index acbb9d0d3..ca046ab3a 100644 --- a/templates/perms.go +++ b/templates/perms.go @@ -59,6 +59,7 @@ var fileMode = map[string]fs.FileMode{ "templates/cli/auxiliary/internal/output/errortable.go.stmpl": 0o644, "templates/cli/auxiliary/internal/output/exitcodes.go.stmpl": 0o644, "templates/cli/auxiliary/internal/output/jq.go.stmpl": 0o644, + "templates/cli/auxiliary/internal/output/outcome.go.stmpl": 0o644, "templates/cli/auxiliary/internal/output/output.go.stmpl": 0o644, "templates/cli/auxiliary/internal/output/output_test.go.stmpl": 0o644, "templates/cli/auxiliary/internal/output/outputitems.go.stmpl": 0o644, @@ -202,6 +203,7 @@ var fileMode = map[string]fs.FileMode{ "templates/cli/tests/primary/keyring_timeout_additional_test.go.stmpl": 0o644, "templates/cli/tests/primary/methods_additional_test.go.stmpl": 0o644, "templates/cli/tests/primary/optionalnullable_additional_test.go.stmpl": 0o644, + "templates/cli/tests/primary/outcome_additional_test.go.stmpl": 0o644, "templates/cli/tests/primary/pagination_test.go.stmpl": 0o644, "templates/cli/tests/primary/parameter_additional_test.go.stmpl": 0o644, "templates/cli/tests/primary/parameter_encoding_additional_test.go.stmpl": 0o644, diff --git a/templates/templates/cli/artifact_test.go.stmpl b/templates/templates/cli/artifact_test.go.stmpl index 3c40010ea..f19c12dc1 100644 --- a/templates/templates/cli/artifact_test.go.stmpl +++ b/templates/templates/cli/artifact_test.go.stmpl @@ -602,4 +602,22 @@ func TestArtifact_InteractiveSuppressedByNonInteractiveAndAgentModes(t *testing. } } {{- end}} + +{{- if templateArtifactTestOutcomeFailureState}} +// A declared failure state (output.outcome) fails the command before any +// media is written. +func TestArtifact_DeclaredFailureStateWritesNothing(t *testing.T) { + server := artifactTestServer(t, `{{templateArtifactTestOutcomeFailureJSON templateArtifactTestMime}}`) + dir := t.TempDir() + h := NewCLITestHarness(t) + + err := runArtifact(t, h, server.URL, dir) + require.Error(t, err) + assert.Contains(t, err.Error(), "ended in failure state "+{{printf "%q" (printf "%q" templateArtifactTestOutcomeFailureState)}}) + assert.Empty(t, strings.TrimSpace(h.GetStdout())) + entries, err := os.ReadDir(dir) + require.NoError(t, err) + assert.Empty(t, entries, "a failure state must not write the artifact") +} +{{- end}} {{- end}} diff --git a/templates/templates/cli/auxiliary/internal/output/binary.go.stmpl b/templates/templates/cli/auxiliary/internal/output/binary.go.stmpl index 9fe243877..5cdb4dd39 100644 --- a/templates/templates/cli/auxiliary/internal/output/binary.go.stmpl +++ b/templates/templates/cli/auxiliary/internal/output/binary.go.stmpl @@ -22,6 +22,7 @@ func WriteToFile(cmd *cobra.Command, res interface{}, path string) error { if flagutil.DidDryRunRequest(cmd) { return nil } + failure := outcomeResultFailure(cmd, res) content := extractResultContent(res) f, err := os.Create(path) @@ -73,10 +74,16 @@ func WriteToFile(cmd *cobra.Command, res interface{}, path string) error { if !IsMachineMode(cmd) { fmt.Fprintf(cmd.ErrOrStderr(), "Wrote %s (%d bytes)%s", absPath, written.Size(), string(rune(10))) } - return renderPathEnvelope(cmd, absPath, map[string]interface{}{ + if err := renderPathEnvelope(cmd, absPath, map[string]interface{}{ "path": absPath, "size_bytes": written.Size(), - }) + }); err != nil { + return err + } + if failure != nil { + return Error(cmd, failure) + } + return nil } // WriteBase64 encodes binary response content as base64 and prints it to stdout. @@ -86,6 +93,7 @@ func WriteBase64(cmd *cobra.Command, res interface{}) error { if flagutil.DidDryRunRequest(cmd) { return nil } + failure := outcomeResultFailure(cmd, res) content := extractResultContent(res) if content == nil { return nil @@ -115,6 +123,9 @@ func WriteBase64(cmd *cobra.Command, res interface{}) error { fmt.Fprintln(w, base64.StdEncoding.EncodeToString(data)) } + if failure != nil { + return Error(cmd, failure) + } return nil } diff --git a/templates/templates/cli/auxiliary/internal/output/classify.go.stmpl b/templates/templates/cli/auxiliary/internal/output/classify.go.stmpl index d6ed673a6..08b684a58 100644 --- a/templates/templates/cli/auxiliary/internal/output/classify.go.stmpl +++ b/templates/templates/cli/auxiliary/internal/output/classify.go.stmpl @@ -702,6 +702,7 @@ const ( ReasonCLIAsyncFailed = "CLI_ASYNC_FAILED" ReasonCLIAsyncTimeout = "CLI_ASYNC_TIMEOUT" ReasonCLIAsyncUnknownState = "CLI_ASYNC_UNKNOWN_STATE" + ReasonCLIOutcomeFailed = "CLI_OUTCOME_FAILED" ) type cliReasonError struct { @@ -723,7 +724,8 @@ func normalizeCLIReason(reason string) string { switch reason { case ReasonCLIValidation, ReasonCLIConnection, ReasonCLIProtocol, ReasonCLIRuntime, ReasonCLIUnavailable, ReasonCLIAuthentication, - ReasonCLIAsyncFailed, ReasonCLIAsyncTimeout, ReasonCLIAsyncUnknownState: + ReasonCLIAsyncFailed, ReasonCLIAsyncTimeout, ReasonCLIAsyncUnknownState, + ReasonCLIOutcomeFailed: return reason } return ReasonCLIRuntime diff --git a/templates/templates/cli/auxiliary/internal/output/outcome.go.stmpl b/templates/templates/cli/auxiliary/internal/output/outcome.go.stmpl new file mode 100644 index 000000000..e70123c9f --- /dev/null +++ b/templates/templates/cli/auxiliary/internal/output/outcome.go.stmpl @@ -0,0 +1,186 @@ +package output + +import ( + "encoding/json" + "fmt" + "io" + "strconv" + "strings" + + "github.com/spf13/cobra" +) + +const outcomeAnnotation = "speakeasy_outcome" + +type outcomeConfig struct { + StatePointer string `json:"statePointer"` + StreamStatePointer string `json:"streamStatePointer"` + States map[string]string `json:"states"` + ErrorPointer string `json:"errorPointer"` + StreamErrorPointer string `json:"streamErrorPointer"` + ErrorMessageField string `json:"errorMessageField"` +} + +func activeOutcome(cmd *cobra.Command) *outcomeConfig { + if cmd == nil || cmd.Annotations == nil { + return nil + } + raw := cmd.Annotations[outcomeAnnotation] + if raw == "" { + return nil + } + var cfg outcomeConfig + if err := json.Unmarshal([]byte(raw), &cfg); err != nil || len(cfg.States) == 0 { + return nil + } + if cfg.ErrorMessageField == "" { + cfg.ErrorMessageField = "message" + } + return &cfg +} + +type outcomeError struct { + message string + state string + detail interface{} +} + +func (e *outcomeError) Error() string { return e.message } + +func (e *outcomeError) CLIReason() string { return ReasonCLIOutcomeFailed } + +func (e *outcomeError) MachineErrorFields() map[string]interface{} { + fields := map[string]interface{}{"state": e.state} + if e.detail != nil { + fields["error"] = e.detail + } + return fields +} + +func outcomeResultFailure(cmd *cobra.Command, res interface{}) *outcomeError { + cfg := activeOutcome(cmd) + if cfg == nil || cfg.StatePointer == "" { + return nil + } + payload, ok := outcomeResultPayload(res) + if !ok { + return nil + } + return outcomeFailure(cmd, cfg, payload, cfg.StatePointer, cfg.ErrorPointer) +} + +func outcomeFailure(cmd *cobra.Command, cfg *outcomeConfig, payload interface{}, statePointer, errorPointer string) *outcomeError { + if cfg == nil || statePointer == "" { + return nil + } + value, ok := outcomePointer(payload, statePointer) + state, isString := value.(string) + if !ok || !isString || state == "" || cfg.States[state] != "failure" { + return nil + } + failure := &outcomeError{state: state, message: fmt.Sprintf("%s ended in failure state %q", outcomeCommandLabel(cmd), state)} + if errorPointer != "" { + if detail, found := outcomePointer(payload, errorPointer); found && detail != nil { + failure.detail = detail + if message := outcomeDetailMessage(detail, cfg.ErrorMessageField); message != "" { + failure.message += ": " + message + } + } + } + return failure +} + +func outcomeDetailMessage(detail interface{}, field string) string { + switch value := detail.(type) { + case string: + return strings.TrimSpace(value) + case map[string]interface{}: + message, _ := value[field].(string) + return strings.TrimSpace(message) + case []interface{}: + for _, item := range value { + if message := outcomeDetailMessage(item, field); message != "" { + return message + } + } + } + return "" +} + +func outcomeCommandLabel(cmd *cobra.Command) string { + path := cmd.CommandPath() + if root := cmd.Root(); root != nil && root != cmd { + path = strings.TrimPrefix(path, root.Name()+" ") + } + return path +} + +func outcomeResultPayload(res interface{}) (interface{}, bool) { + switch extractResultContent(res).(type) { + case io.ReadCloser, []byte: + return nil, false + } + raw := peekRawBody(res) + if len(raw) == 0 { + content := extractResultContent(res) + if content == nil { + return nil, false + } + var err error + if raw, err = marshalJSON(content); err != nil { + return nil, false + } + } + parsed, err := decodeWithNumbers(raw) + return parsed, err == nil +} + + +// outcomePointer resolves an RFC 6901 pointer against decoded JSON. +func outcomePointer(root interface{}, pointer string) (interface{}, bool) { + if pointer == "" { + return root, true + } + if !strings.HasPrefix(pointer, "/") { + return nil, false + } + current := root + for _, raw := range strings.Split(strings.TrimPrefix(pointer, "/"), "/") { + segment := strings.ReplaceAll(strings.ReplaceAll(raw, "~1", "/"), "~0", "~") + switch value := current.(type) { + case map[string]interface{}: + var ok bool + if current, ok = value[segment]; !ok { + return nil, false + } + case []interface{}: + index, err := strconv.Atoi(segment) + if err != nil || index < 0 || index >= len(value) { + return nil, false + } + current = value[index] + default: + return nil, false + } + } + return current, true +} + +func outcomeStreamFailure(cmd *cobra.Command, cfg *outcomeConfig, item interface{}, frame map[string]interface{}) *outcomeError { + if cfg == nil || cfg.StreamStatePointer == "" { + return nil + } + var payload interface{} = frame + if frame == nil { + data, err := marshalJSON(item) + if err != nil { + return nil + } + decoded, err := decodeWithNumbers(data) + if err != nil { + return nil + } + payload = decoded + } + return outcomeFailure(cmd, cfg, payload, cfg.StreamStatePointer, cfg.StreamErrorPointer) +} diff --git a/templates/templates/cli/auxiliary/internal/output/output.go.stmpl b/templates/templates/cli/auxiliary/internal/output/output.go.stmpl index fabc121ef..514ffcade 100644 --- a/templates/templates/cli/auxiliary/internal/output/output.go.stmpl +++ b/templates/templates/cli/auxiliary/internal/output/output.go.stmpl @@ -398,6 +398,22 @@ func LocalResult(cmd *cobra.Command, res interface{}) error { } func result(cmd *cobra.Command, res interface{}, allowArtifact bool) error { + failure := outcomeResultFailure(cmd, res) + {{- if hasCliArtifacts}} + if failure != nil && allowArtifact && activeArtifact(cmd) != nil { + return Error(cmd, failure) + } + {{- end}} + if err := renderResult(cmd, res, allowArtifact); err != nil { + return err + } + if failure != nil { + return Error(cmd, failure) + } + return nil +} + +func renderResult(cmd *cobra.Command, res interface{}, allowArtifact bool) error { out := cmd.OutOrStdout() colorFlag, _ := flagutil.GetStringFlag(cmd, "color") colorize := ShouldColorize(colorFlag) diff --git a/templates/templates/cli/auxiliary/internal/output/paginated.go.stmpl b/templates/templates/cli/auxiliary/internal/output/paginated.go.stmpl index 1fcdf8006..0da1744b0 100644 --- a/templates/templates/cli/auxiliary/internal/output/paginated.go.stmpl +++ b/templates/templates/cli/auxiliary/internal/output/paginated.go.stmpl @@ -134,6 +134,11 @@ func PaginatedResult(cmd *cobra.Command, res interface{}, contentFieldName, resu } } + if failure := outcomeResultFailure(cmd, page); failure != nil { + _ = flushRows(true) + return Error(cmd, failure) + } + pageCount++ if maxPages > 0 && pageCount >= maxPages { if HasMorePages(page, probe) && !IsMachineMode(cmd) { diff --git a/templates/templates/cli/auxiliary/internal/output/streaming.go.stmpl b/templates/templates/cli/auxiliary/internal/output/streaming.go.stmpl index 30d43f0a3..1f0384619 100644 --- a/templates/templates/cli/auxiliary/internal/output/streaming.go.stmpl +++ b/templates/templates/cli/auxiliary/internal/output/streaming.go.stmpl @@ -86,6 +86,11 @@ func StreamResult(cmd *cobra.Command, res interface{}, streamFieldName string, s } projector := newStreamProjector(cmd) + outcomeCfg := activeOutcome(cmd) + if outcomeCfg != nil && outcomeCfg.StreamStatePointer == "" { + outcomeCfg = nil + } + var streamFailure *outcomeError // Projected output already written must end with its terminating // newline before any error is rendered. The primary error keeps the @@ -158,9 +163,12 @@ func StreamResult(cmd *cobra.Command, res interface{}, streamFieldName string, s } } - if raw != nil { - item = wireStreamItem(item, raw, format == "table" && jqExpr == "") + var frame map[string]interface{} + if outcomeCfg != nil && streamFailure == nil { + frame, _ = decodeStreamFrame(raw) + streamFailure = outcomeStreamFailure(cmd, outcomeCfg, item, frame) } + item = wireStreamItem(item, raw, frame, format == "table" && jqExpr == "") if projector != nil { if err := projector.emit(out, item); err != nil { @@ -180,6 +188,9 @@ func StreamResult(cmd *cobra.Command, res interface{}, streamFieldName string, s return renderAfterFinish(errResults[0].Interface().(error)) } } + if streamFailure != nil { + return renderAfterFinish(streamFailure) + } if projector != nil { if err := projector.finish(out); err != nil { @@ -201,7 +212,9 @@ func (w wireEvent) MarshalJSON() ([]byte, error) { } // Re-encoding the decoded model drops undeclared fields and zero-fills absent ones. -func wireStreamItem(item interface{}, raw []byte, table bool) interface{} { +// frame is the wire object when the caller already decoded raw; nil decodes +// it here, and only once the item is known to need it. +func wireStreamItem(item interface{}, raw []byte, frame map[string]interface{}, table bool) interface{} { keys, ok := envelopeKeys(reflect.ValueOf(item)) if len(raw) == 0 || !ok { return item @@ -211,10 +224,11 @@ func wireStreamItem(item interface{}, raw []byte, table bool) interface{} { if err == nil && table { return item } - frame, err := decodeWithNumbers(raw) - wire, isObject := frame.(map[string]interface{}) - if err != nil || !isObject { - return item + wire := frame + if wire == nil { + if wire, ok = decodeStreamFrame(raw); !ok { + return item + } } for name := range wire { if keys != nil && !keys[name] { @@ -228,6 +242,19 @@ func wireStreamItem(item interface{}, raw []byte, table bool) interface{} { return wireEvent{data: data} } +func decodeStreamFrame(raw []byte) (map[string]interface{}, bool) { + if len(raw) == 0 { + return nil, false + } + frame, err := decodeWithNumbers(raw) + wire, isObject := frame.(map[string]interface{}) + if err != nil || !isObject { + return nil, false + } + return wire, true +} + + // nil keys keep the whole frame: a union event that matched no member. func envelopeKeys(v reflect.Value) (map[string]bool, bool) { v = derefValue(v) @@ -326,7 +353,7 @@ func streamErrorEvent(item interface{}, raw []byte) (json.RawMessage, bool) { return data, true } if isUnionStruct(derefValue(reflect.ValueOf(item)).Type()) { - data, err := json.Marshal(wireStreamItem(item, raw, false)) + data, err := json.Marshal(wireStreamItem(item, raw, nil, false)) return data, err == nil } var frame map[string]json.RawMessage diff --git a/templates/templates/cli/auxiliary/internal/output/streaming_test.go.stmpl b/templates/templates/cli/auxiliary/internal/output/streaming_test.go.stmpl index 3743ed3bb..7ba3564b2 100644 --- a/templates/templates/cli/auxiliary/internal/output/streaming_test.go.stmpl +++ b/templates/templates/cli/auxiliary/internal/output/streaming_test.go.stmpl @@ -41,7 +41,7 @@ func TestWireStreamItemUnwrapsNestedEnvelopeUnions(t *testing.T) { "inner member unknown": {outerEnvelopeUnion{Inner: &innerEnvelopeUnion{}}, `{"data":"hello","event":"tick","id":"7"}`}, } { t.Run(name, func(t *testing.T) { - got, err := json.Marshal(wireStreamItem(tc.item, raw, false)) + got, err := json.Marshal(wireStreamItem(tc.item, raw, nil, false)) if err != nil || string(got) != tc.want { t.Fatalf("got %s, error %v; want %s", got, err, tc.want) } @@ -57,7 +57,7 @@ type openEnvelope struct { func TestWireStreamItemKeepsFrameOfEnvelopeWithAdditionalProperties(t *testing.T) { raw := []byte(`{"data":"hello","event":"tick","id":"7"}`) want := `{"data":"hello","event":"tick","id":"7"}` - got, err := json.Marshal(wireStreamItem(openEnvelope{Data: "hello"}, raw, false)) + got, err := json.Marshal(wireStreamItem(openEnvelope{Data: "hello"}, raw, nil, false)) if err != nil || string(got) != want { t.Fatalf("got %s, error %v; want %s", got, err, want) } @@ -138,7 +138,7 @@ func TestWireStreamItemTableKeepsModelLayout(t *testing.T) { } { t.Run(name, func(t *testing.T) { var buf bytes.Buffer - if err := printTable(&buf, wireStreamItem(tc.item, []byte(tc.raw), true)); err != nil || buf.String() != tc.want { + if err := printTable(&buf, wireStreamItem(tc.item, []byte(tc.raw), nil, true)); err != nil || buf.String() != tc.want { t.Fatalf("got %q, error %v; want %q", buf.String(), err, tc.want) } }) diff --git a/templates/templates/cli/includes/intents.ts b/templates/templates/cli/includes/intents.ts index 7d60d406b..36cb9feaa 100644 --- a/templates/templates/cli/includes/intents.ts +++ b/templates/templates/cli/includes/intents.ts @@ -137,6 +137,7 @@ interface IntentCmdCtx { AsyncMissingHandleTestJSON: string; AsyncFailureHint: string; StreamMetadataJSON: string; + OutcomeJSON: string; // terminal-state table carried via cobra annotation ("" = none) StreamSelect: string; // RFC 6901 pointer of output.stream.select ("" = none) StreamToggle: { Name: string; Key: string } | null; StreamKind: string; // "sse" | "jsonl" | "" — how the backing operation streams @@ -202,6 +203,7 @@ interface CLIOperationFlagCtx { interface CLIOperationCtx { OperationID: string; StreamMetadataJSON: string; + OutcomeJSON: string; StreamSelect: string; Flags: CLIOperationFlagCtx[]; BodyFlags: string[]; @@ -1352,6 +1354,7 @@ function collectIntentManifest(): IntentManifestCtx { AsyncFailureHint: cmd.Hints?.CLI_ASYNC_FAILED?.[0] || "", StreamSelect: cmd.Output?.Stream?.Pointer || "", StreamMetadataJSON: streamMetadataJSON(cmd.Output?.Stream?.Metadata), + OutcomeJSON: outcomeJSON(cmd.Output?.Outcome), StreamToggle: !dispatch && intentStreamKind(found.op).kind ? intentDefaultOnStreamToggle(flags, presetJSON) @@ -1390,6 +1393,32 @@ function intentStreamKind(op: Operation): { kind: string; sentinel: string } { return { kind: "", sentinel: "" }; } +// outcomeJSON serializes output.outcome for the runtime. The states map +// arrives from Go map iteration, so it is sorted for a deterministic file. +function outcomeJSON(outcome: CLICommandOutcome | null | undefined): string { + if (!outcome) { + return ""; + } + const states: Record = Object.create(null); + for (const state of Object.keys(outcome.States || {}).sort()) { + states[state] = outcome.States[state]; + } + return JSON.stringify({ + ...(outcome.StatePointer ? { statePointer: outcome.StatePointer } : {}), + ...(outcome.StreamStatePointer + ? { streamStatePointer: outcome.StreamStatePointer } + : {}), + states, + ...(outcome.ErrorPointer ? { errorPointer: outcome.ErrorPointer } : {}), + ...(outcome.StreamErrorPointer + ? { streamErrorPointer: outcome.StreamErrorPointer } + : {}), + ...(outcome.ErrorMessageField + ? { errorMessageField: outcome.ErrorMessageField } + : {}), + }); +} + function streamMetadataJSON( fields: CLICommandStreamMetadata[] | null | undefined, ): string { @@ -1600,6 +1629,7 @@ function getCLIOperationCtx(op: Operation): CLIOperationCtx | null { OperationID: declared.OperationID, StreamSelect: declared.Output?.Stream?.Pointer || "", StreamMetadataJSON: streamMetadataJSON(declared.Output?.Stream?.Metadata), + OutcomeJSON: outcomeJSON(declared.Output?.Outcome), Flags: flags, BodyFlags: bodyFlags, CanonicalBodyFlag: bodyFlags[bodyFlags.length - 1] || "", @@ -2049,6 +2079,53 @@ registerTemplateFunc( templateArtifactTestMockJSON, ); +// The artifact test command's first declared failure state and the discrete +// pointer that carries it, when the command declares output.outcome. +function artifactTestOutcome(): { pointer: string; state: string } | null { + const cmd = getArtifactTestCommand(); + if (!cmd?.OutcomeJSON) return null; + const outcome = JSON.parse(cmd.OutcomeJSON); + if (!outcome.statePointer) return null; + const state = Object.keys(outcome.states || {}) + .sort() + .find((candidate) => outcome.states[candidate] === "failure"); + return state ? { pointer: outcome.statePointer, state } : null; +} + +function templateArtifactTestOutcomeFailureState(): string { + return artifactTestOutcome()?.state || ""; +} +registerTemplateFunc( + "templateArtifactTestOutcomeFailureState", + templateArtifactTestOutcomeFailureState, +); + +// The inline artifact mock with the declared failure state set at the +// outcome's state pointer. +function templateArtifactTestOutcomeFailureJSON(mime: string): string { + const cmd = getArtifactTestCommand(); + const outcome = artifactTestOutcome(); + if (!cmd || !outcome) return "{}"; + const body = JSON.parse( + artifactTestMockJSON(JSON.parse(cmd.ArtifactJSON), "inline", mime), + ); + let value: any = outcome.state; + for (const segment of outcome.pointer.split("/").slice(1).reverse()) { + if (/^(0|[1-9][0-9]*)$/.test(segment)) { + const list: any[] = []; + list[Number(segment)] = value; + value = list; + continue; + } + value = { [segment.replace(/~1/g, "/").replace(/~0/g, "~")]: value }; + } + return JSON.stringify(mergeIntentTestValue(body, value)); +} +registerTemplateFunc( + "templateArtifactTestOutcomeFailureJSON", + templateArtifactTestOutcomeFailureJSON, +); + // Whether the artifact test command's pointer contains a [*]; only then can // a response carry more than one candidate block. function templateArtifactTestPointerHasWild(): boolean { diff --git a/templates/templates/cli/intentcmd.go.stmpl b/templates/templates/cli/intentcmd.go.stmpl index 744ade996..6eedb5f8f 100644 --- a/templates/templates/cli/intentcmd.go.stmpl +++ b/templates/templates/cli/intentcmd.go.stmpl @@ -71,6 +71,9 @@ func InitIntent{{$c.FuncName}}(parent *cobra.Command) error { {{- if $c.AsyncJSON}} "speakeasy_async": "{{escapeGoString $c.AsyncJSON}}", {{- end}} + {{- if $c.OutcomeJSON}} + "speakeasy_outcome": "{{escapeGoString $c.OutcomeJSON}}", + {{- end}} {{- if $c.StreamSelect}} "speakeasy_stream_select": "{{escapeGoString $c.StreamSelect}}", {{- if $c.StreamMetadataJSON}} diff --git a/templates/templates/cli/opcmd.go.stmpl b/templates/templates/cli/opcmd.go.stmpl index 35381d055..0e4624a92 100644 --- a/templates/templates/cli/opcmd.go.stmpl +++ b/templates/templates/cli/opcmd.go.stmpl @@ -46,6 +46,9 @@ func {{$initFuncName}}(parent *cobra.Command) error { parent.Annotations["speakeasy_stream_metadata"] = "{{escapeGoString $cliOp.StreamMetadataJSON}}" {{- end}} {{- end}} + {{- if and $cliOp $cliOp.OutcomeJSON}} + parent.Annotations["speakeasy_outcome"] = "{{escapeGoString $cliOp.OutcomeJSON}}" + {{- end}} {{- if or $hasSecurity $op.Request (hasPagination $op) (hasBinaryResponse $op) (and $cliOp $cliOp.Flags)}} cmd := parent // register flags on the parent {{- end}} @@ -73,6 +76,9 @@ func {{$initFuncName}}(parent *cobra.Command) error { "speakeasy_stream_metadata": "{{escapeGoString $cliOp.StreamMetadataJSON}}", {{- end}} {{- end}} + {{- if and $cliOp $cliOp.OutcomeJSON}} + "speakeasy_outcome": "{{escapeGoString $cliOp.OutcomeJSON}}", + {{- end}} }, } {{- end}} diff --git a/templates/templates/cli/readme/agents.stmpl b/templates/templates/cli/readme/agents.stmpl index 3a1bce2f4..07ee488e4 100644 --- a/templates/templates/cli/readme/agents.stmpl +++ b/templates/templates/cli/readme/agents.stmpl @@ -145,7 +145,7 @@ Outside agent mode, explicit JSON and `--jq` preserve the compatibility envelope {{- $carriers := cliErrorCarrierDisplayList}} -`error_type` is one of `authentication_error`, `authorization_error`, `not_found`, `validation_error`, `rate_limit_error`, `server_error`, `api_error`, `connection_error`, `protocol_error`, {{if cliEmitsServiceDisabledErrorType}}`service_disabled`, {{end}}{{if cliEmitsBillingDisabledErrorType}}`billing_disabled`, {{end}}`runtime_error`, `unsupported_error`, `async_failed`, `async_timeout`, `async_unknown_state`. {{if $carriers}}Classification reads the structured reason code at {{$carriers}} in the error body (resolved against the nested `error` object when the body has one) before HTTP status, so a declared credential reason sent with HTTP 400 is not mistaken for request validation. {{if cliErrorAnyCarrierDeclared}}`error_reason` carries the reason code found there, verbatim from a declared carrier when no declared rule matches it{{else}}`error_reason` carries the reason code found there when a declared rule matches it (an undeclared body reason is never promoted){{end}}; it is absent for status-only API errors.{{else}}Classification derives from the HTTP status and transport evidence; `error_reason` is absent for API errors.{{end}} Status-less local failures may use `CLI_VALIDATION`, `CLI_CONNECTION`, `CLI_PROTOCOL`, `CLI_RUNTIME`, `CLI_UNAVAILABLE`, `CLI_AUTHENTICATION`, or the async polling reasons `CLI_ASYNC_FAILED`, `CLI_ASYNC_TIMEOUT`, and `CLI_ASYNC_UNKNOWN_STATE`. `hints` preserves server guidance first, adds the most specific local taxonomy guidance, then typed CLI and command-specific guidance, removing exact duplicates. `exit_code` is always the code for the final `error_type` shown in the envelope: 1 runtime, 2 usage, or 3 authentication/authorization. +`error_type` is one of `authentication_error`, `authorization_error`, `not_found`, `validation_error`, `rate_limit_error`, `server_error`, `api_error`, `connection_error`, `protocol_error`, {{if cliEmitsServiceDisabledErrorType}}`service_disabled`, {{end}}{{if cliEmitsBillingDisabledErrorType}}`billing_disabled`, {{end}}`runtime_error`, `unsupported_error`, `async_failed`, `async_timeout`, `async_unknown_state`. {{if $carriers}}Classification reads the structured reason code at {{$carriers}} in the error body (resolved against the nested `error` object when the body has one) before HTTP status, so a declared credential reason sent with HTTP 400 is not mistaken for request validation. {{if cliErrorAnyCarrierDeclared}}`error_reason` carries the reason code found there, verbatim from a declared carrier when no declared rule matches it{{else}}`error_reason` carries the reason code found there when a declared rule matches it (an undeclared body reason is never promoted){{end}}; it is absent for status-only API errors.{{else}}Classification derives from the HTTP status and transport evidence; `error_reason` is absent for API errors.{{end}} Status-less local failures may use `CLI_VALIDATION`, `CLI_CONNECTION`, `CLI_PROTOCOL`, `CLI_RUNTIME`, `CLI_UNAVAILABLE`, `CLI_AUTHENTICATION`, the async polling reasons `CLI_ASYNC_FAILED`, `CLI_ASYNC_TIMEOUT`, and `CLI_ASYNC_UNKNOWN_STATE`, or `CLI_OUTCOME_FAILED` when a successful response reports a declared failure state; that envelope also carries `state` (the reported state) and, when the command declares an error pointer, `error` (the failure detail taken from the response). `hints` preserves server guidance first, adds the most specific local taxonomy guidance, then typed CLI and command-specific guidance, removing exact duplicates. `exit_code` is always the code for the final `error_type` shown in the envelope: 1 runtime, 2 usage, or 3 authentication/authorization. {{- if $c.Hints}} ```bash diff --git a/templates/templates/cli/tests/primary/outcome_additional_test.go.stmpl b/templates/templates/cli/tests/primary/outcome_additional_test.go.stmpl new file mode 100644 index 000000000..a696b1aa3 --- /dev/null +++ b/templates/templates/cli/tests/primary/outcome_additional_test.go.stmpl @@ -0,0 +1,298 @@ +package tests + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "{{getGolangPackage}}/internal/cli" + "{{getGolangPackage}}/internal/clierrors" + "{{getGolangPackage}}/internal/config" +) + +func outcomeInteractionJSON(status string, withErrors bool) string { + errorsJSON := "" + if withErrors { + errorsJSON = `,"errors":[{"code":"blocked","message":"content was blocked"}]` + } + return fmt.Sprintf(`{"id":"int_1","status":%q,"output_text":"Hello"%s}`, status, errorsJSON) +} + +func newOutcomeServer(t *testing.T, status string, withErrors bool) *httptest.Server { + t.Helper() + return newOutcomeServerFirstPage(t, status, withErrors, "in_progress") +} + +func newOutcomeServerFirstPage(t *testing.T, status string, withErrors bool, firstPageStatus string) *httptest.Server { + t.Helper() + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.HasPrefix(r.URL.Path, "/outcome/interactions/"), strings.HasPrefix(r.URL.Path, "/outcome/exports/"): + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, outcomeInteractionJSON(status, withErrors)) + case r.URL.Path == "/outcome/interactions": + w.Header().Set("Content-Type", "application/json") + if r.URL.Query().Get("cursor") == "" { + fmt.Fprintf(w, `{"status":%q,"next_cursor":"page-2","items":[{"id":"int_1","status":"completed"}]}`, firstPageStatus) + return + } + errorsJSON := "" + if withErrors { + errorsJSON = `,"errors":[{"code":"blocked","message":"content was blocked"}]` + } + fmt.Fprintf(w, `{"status":%q,"items":[{"id":"int_2","status":%q}]%s}`, status, status, errorsJSON) + case r.URL.Path == "/outcome/stream": + w.Header().Set("Content-Type", "text/event-stream") + fmt.Fprint(w, "data: {\"event_type\":\"delta\",\"content\":\"Hello\"}\n\n") + if status != "" { + fmt.Fprintf(w, "data: {\"event_type\":\"completed\",\"interaction\":%s}\n\n", outcomeInteractionJSON(status, withErrors)) + } + fmt.Fprint(w, "data: [DONE]\n\n") + default: + http.NotFound(w, r) + } + })) +} + +func outcomeErrorEnvelope(t *testing.T, stderr string) map[string]interface{} { + t.Helper() + var envelope map[string]interface{} + require.NoError(t, json.Unmarshal([]byte(stderr), &envelope), stderr) + return envelope +} + +func runOutcome(t *testing.T, serverURL string, args ...string) (*CLITestHarness, error) { + t.Helper() + t.Cleanup(config.ResetKeyring) + h := NewCLITestHarness(t) + h.resetAndSetupEnv() + root, err := cli.NewRootCommand() + require.NoError(t, err) + root.SetOut(h.stdout) + root.SetErr(h.stderr) + root.SetIn(strings.NewReader("")) + fullArgs := append([]string{"--server-url", serverURL, "--color", "never", "--no-interactive", "--no-keyring"}, args...) + return h, cli.ExecuteRoot(context.Background(), root, fullArgs) +} + +func TestOutcomeDiscreteResponse(t *testing.T) { + for _, command := range [][]string{ + {"outcome", "outcome-get-interaction", "--id", "int_1"}, + {"outcome-check", "--id", "int_1"}, + } { + name := strings.Join(command[:1], " ") + for _, tc := range []struct { + status string + withErrors bool + wantExit int + wantStderr string + }{ + {"completed", false, clierrors.ExitOK, ""}, + {"incomplete", false, clierrors.ExitOK, ""}, + {"in_progress", false, clierrors.ExitOK, ""}, + {"failed", true, clierrors.ExitRuntime, `ended in failure state "failed": content was blocked`}, + {"failed", false, clierrors.ExitRuntime, `ended in failure state "failed"`}, + {"cancelled", false, clierrors.ExitRuntime, `ended in failure state "cancelled"`}, + } { + t.Run(name+"/"+tc.status+"/json", func(t *testing.T) { + srv := newOutcomeServer(t, tc.status, tc.withErrors) + defer srv.Close() + h, err := runOutcome(t, srv.URL, append(command, "--output-format", "json")...) + assert.Equal(t, tc.wantExit, clierrors.ExitCode(err), h.GetStderr()) + var payload map[string]interface{} + require.NoError(t, json.Unmarshal([]byte(h.GetStdout()), &payload), h.GetStdout()) + assert.Equal(t, tc.status, payload["status"]) + if tc.wantStderr == "" { + assert.Empty(t, h.GetStderr()) + return + } + envelope := outcomeErrorEnvelope(t, h.GetStderr()) + assert.Contains(t, envelope["message"], tc.wantStderr) + assert.Equal(t, "runtime_error", envelope["error_type"]) + assert.Equal(t, "CLI_OUTCOME_FAILED", envelope["error_reason"]) + }) + } + t.Run(name+"/pretty", func(t *testing.T) { + srv := newOutcomeServer(t, "failed", true) + defer srv.Close() + h, err := runOutcome(t, srv.URL, command...) + assert.Equal(t, clierrors.ExitRuntime, clierrors.ExitCode(err)) + assert.Contains(t, h.GetStdout(), "failed") + assert.Contains(t, h.GetStderr(), `ended in failure state "failed": content was blocked`) + }) + t.Run(name+"/agent-mode", func(t *testing.T) { + srv := newOutcomeServer(t, "failed", true) + defer srv.Close() + h, err := runOutcome(t, srv.URL, append(command, "--agent-mode")...) + assert.Equal(t, clierrors.ExitRuntime, clierrors.ExitCode(err)) + var envelope map[string]interface{} + require.NoError(t, json.Unmarshal([]byte(h.GetStderr()), &envelope), h.GetStderr()) + assert.Equal(t, "CLI_OUTCOME_FAILED", envelope["error_reason"]) + assert.Equal(t, "runtime_error", envelope["error_type"]) + assert.Equal(t, float64(clierrors.ExitRuntime), envelope["exit_code"]) + assert.Equal(t, "failed", envelope["state"]) + detail, _ := envelope["error"].([]interface{}) + require.Len(t, detail, 1) + assert.Equal(t, "content was blocked", detail[0].(map[string]interface{})["message"]) + }) + } +} + +func TestOutcomeBinaryOutputFlags(t *testing.T) { + for _, tc := range []struct { + status string + withErrors bool + wantExit int + wantStderr string + }{ + {"completed", false, clierrors.ExitOK, ""}, + {"failed", true, clierrors.ExitRuntime, `ended in failure state "failed": content was blocked`}, + {"cancelled", false, clierrors.ExitRuntime, `ended in failure state "cancelled"`}, + } { + t.Run(tc.status+"/output-file", func(t *testing.T) { + srv := newOutcomeServer(t, tc.status, tc.withErrors) + defer srv.Close() + path := filepath.Join(t.TempDir(), "out.json") + h, err := runOutcome(t, srv.URL, "outcome", "outcome-export-interaction", "--id", "int_1", "--output-file", path) + assert.Equal(t, tc.wantExit, clierrors.ExitCode(err), h.GetStderr()) + written, readErr := os.ReadFile(path) + require.NoError(t, readErr) + assert.Contains(t, string(written), `"status":`+fmt.Sprintf("%q", tc.status)) + if tc.wantStderr == "" { + assert.NotContains(t, h.GetStderr(), "failure state") + return + } + assert.Contains(t, h.GetStderr(), tc.wantStderr) + }) + t.Run(tc.status+"/output-b64", func(t *testing.T) { + srv := newOutcomeServer(t, tc.status, tc.withErrors) + defer srv.Close() + h, err := runOutcome(t, srv.URL, "outcome", "outcome-export-interaction", "--id", "int_1", "--output-b64") + assert.Equal(t, tc.wantExit, clierrors.ExitCode(err), h.GetStderr()) + decoded, decodeErr := base64.StdEncoding.DecodeString(strings.TrimSpace(h.GetStdout())) + require.NoError(t, decodeErr, h.GetStdout()) + assert.Contains(t, string(decoded), `"status":`+fmt.Sprintf("%q", tc.status)) + if tc.wantStderr == "" { + assert.Empty(t, h.GetStderr()) + return + } + assert.Contains(t, h.GetStderr(), tc.wantStderr) + }) + } +} + +func TestOutcomePaginatedResponse(t *testing.T) { + for _, tc := range []struct { + status string + withErrors bool + wantExit int + wantStderr string + }{ + {"completed", false, clierrors.ExitOK, ""}, + {"failed", true, clierrors.ExitRuntime, `ended in failure state "failed": content was blocked`}, + {"cancelled", false, clierrors.ExitRuntime, `ended in failure state "cancelled"`}, + } { + t.Run(tc.status+"/all-pages", func(t *testing.T) { + srv := newOutcomeServer(t, tc.status, tc.withErrors) + defer srv.Close() + h, err := runOutcome(t, srv.URL, "outcome", "outcome-list-interactions", "--all", "--output-format", "json") + assert.Equal(t, tc.wantExit, clierrors.ExitCode(err), h.GetStderr()) + pages := strings.Split(strings.TrimSpace(h.GetStdout()), "\n") + require.Len(t, pages, 2, h.GetStdout()) + assert.Contains(t, pages[0], `"status":"in_progress"`) + assert.Contains(t, pages[1], `"status":`+fmt.Sprintf("%q", tc.status)) + if tc.wantStderr == "" { + assert.Empty(t, h.GetStderr()) + return + } + envelope := outcomeErrorEnvelope(t, h.GetStderr()) + assert.Contains(t, envelope["message"], tc.wantStderr) + assert.Equal(t, "CLI_OUTCOME_FAILED", envelope["error_reason"]) + }) + } + for _, tc := range []struct { + firstPage string + wantExit int + }{ + {"in_progress", clierrors.ExitOK}, + {"failed", clierrors.ExitRuntime}, + } { + t.Run("first page only/"+tc.firstPage, func(t *testing.T) { + srv := newOutcomeServerFirstPage(t, "completed", false, tc.firstPage) + defer srv.Close() + h, err := runOutcome(t, srv.URL, "outcome", "outcome-list-interactions", "--output-format", "json") + assert.Equal(t, tc.wantExit, clierrors.ExitCode(err), h.GetStderr()) + assert.Contains(t, h.GetStdout(), `"status":`+fmt.Sprintf("%q", tc.firstPage)) + assert.NotContains(t, h.GetStdout(), "int_2", "only the first page is fetched without --all") + if tc.wantExit == clierrors.ExitOK { + assert.Empty(t, h.GetStderr()) + return + } + assert.Contains(t, outcomeErrorEnvelope(t, h.GetStderr())["message"], `ended in failure state "failed"`) + }) + } +} + +func TestOutcomeStreamedResponse(t *testing.T) { + for _, tc := range []struct { + name string + status string + withErrors bool + wantExit int + wantStderr string + }{ + {"completed", "completed", false, clierrors.ExitOK, ""}, + {"incomplete", "incomplete", false, clierrors.ExitOK, ""}, + {"failed with detail", "failed", true, clierrors.ExitRuntime, `ended in failure state "failed": content was blocked`}, + {"cancelled", "cancelled", false, clierrors.ExitRuntime, `ended in failure state "cancelled"`}, + {"no completion event", "", false, clierrors.ExitOK, ""}, + } { + t.Run(tc.name+"/projected", func(t *testing.T) { + srv := newOutcomeServer(t, tc.status, tc.withErrors) + defer srv.Close() + h, err := runOutcome(t, srv.URL, "outcome-say", "hello") + assert.Equal(t, tc.wantExit, clierrors.ExitCode(err), h.GetStderr()) + assert.Equal(t, "Hello\n", h.GetStdout()) + if tc.wantStderr == "" { + assert.Empty(t, h.GetStderr()) + return + } + assert.Contains(t, h.GetStderr(), tc.wantStderr) + }) + t.Run(tc.name+"/json", func(t *testing.T) { + srv := newOutcomeServer(t, tc.status, tc.withErrors) + defer srv.Close() + h, err := runOutcome(t, srv.URL, "outcome-say", "hello", "--output-format", "json") + assert.Equal(t, tc.wantExit, clierrors.ExitCode(err), h.GetStderr()) + events := strings.Split(strings.TrimSpace(h.GetStdout()), "\n") + if tc.status == "" { + require.Len(t, events, 1) + } else { + require.Len(t, events, 2) + assert.Contains(t, events[1], `"status":`+fmt.Sprintf("%q", tc.status)) + } + if tc.wantStderr != "" { + assert.Contains(t, outcomeErrorEnvelope(t, h.GetStderr())["message"], tc.wantStderr) + } + }) + } + t.Run("agent-mode envelope", func(t *testing.T) { + srv := newOutcomeServer(t, "failed", true) + defer srv.Close() + h, err := runOutcome(t, srv.URL, "outcome-say", "hello", "--agent-mode") + assert.Equal(t, clierrors.ExitRuntime, clierrors.ExitCode(err)) + var envelope map[string]interface{} + require.NoError(t, json.Unmarshal([]byte(h.GetStderr()), &envelope), h.GetStderr()) + assert.Equal(t, "CLI_OUTCOME_FAILED", envelope["error_reason"]) + assert.Equal(t, "failed", envelope["state"]) + }) +} diff --git a/templates/templates/common/common/typeDefs.d.ts b/templates/templates/common/common/typeDefs.d.ts index 03d8b95a4..b9aaf856a 100644 --- a/templates/templates/common/common/typeDefs.d.ts +++ b/templates/templates/common/common/typeDefs.d.ts @@ -2338,10 +2338,24 @@ declare global { Pointer: string; }; + type CLICommandOutcome = { + StateFrom?: string; + StatePointer?: string; + StreamStateFrom?: string; + StreamStatePointer?: string; + States: Record; + ErrorFrom?: string; + ErrorPointer?: string; + StreamErrorFrom?: string; + StreamErrorPointer?: string; + ErrorMessageField?: string; + }; + type CLICommandOutput = { Projection: CLICommandProjection | null; Artifact?: CLICommandArtifact | null; Stream?: CLICommandStreamProjection | null; + Outcome?: CLICommandOutcome | null; }; type CLICommandAsyncParameter = { diff --git a/tests/overlays/primary/cli/overlay.yaml b/tests/overlays/primary/cli/overlay.yaml index 33b664455..99300da9b 100644 --- a/tests/overlays/primary/cli/overlay.yaml +++ b/tests/overlays/primary/cli/overlay.yaml @@ -121,7 +121,69 @@ actions: type: bool defaultFrom: schema summary: Stream events as they arrive + outcomeGetInteraction: + output: + outcome: + statePointer: $.status + errorPointer: $.errors + states: + queued: pending + in_progress: pending + completed: success + incomplete: success + failed: failure + cancelled: failure + outcomeExportInteraction: + output: + outcome: + statePointer: $.status + errorPointer: $.errors + states: + queued: pending + in_progress: pending + completed: success + incomplete: success + failed: failure + cancelled: failure + outcomeListInteractions: + output: + outcome: + statePointer: $.status + errorPointer: $.errors + states: + queued: pending + in_progress: pending + completed: success + incomplete: success + failed: failure + cancelled: failure commands: + outcome-check: + category: Create + summary: Fetch an interaction and fail when it ended in a failure state (output.outcome inherited from the operation entry) + op: outcomeGetInteraction + outcome-say: + category: Create + summary: Stream an interaction and fail when it completes in a failure state + op: outcomeStream + args: + prompt: + to: $.input + variadic: true + summary: Prompt text + output: + stream: + select: $.data.content + outcome: + streamStatePointer: $.data.interaction.status + streamErrorPointer: $.data.interaction.errors + states: + queued: pending + in_progress: pending + completed: success + incomplete: success + failed: failure + cancelled: failure create-widget: op: cliCatalogOptionsPost preset: @@ -553,3 +615,156 @@ actions: type: string cursor: type: string + +- target: $.paths + update: + /outcome/interactions/{id}: + get: + operationId: outcomeGetInteraction + tags: [outcome] + description: Discrete lookup whose success response carries the interaction's terminal status, so the CLI can classify a failure reported with HTTP 200 (output.outcome). + servers: + - url: http://localhost:35456 + parameters: + - name: id + in: path + required: true + schema: + type: string + responses: + "200": + description: The interaction. + content: + application/json: + schema: + $ref: '#/components/schemas/outcomeInteraction' + /outcome/exports/{id}: + get: + operationId: outcomeExportInteraction + tags: [outcome] + description: Lookup whose success response is either the interaction as JSON or its exported binary, so output.outcome is classified when the JSON response is saved with --output-file or printed with --output-b64. + servers: + - url: http://localhost:35456 + parameters: + - name: id + in: path + required: true + schema: + type: string + responses: + "200": + description: The interaction, or its exported binary. + content: + application/json: + schema: + $ref: '#/components/schemas/outcomeInteraction' + application/octet-stream: + schema: + type: string + format: binary + /outcome/interactions: + get: + operationId: outcomeListInteractions + tags: [outcome] + description: Cursor-paginated listing whose every page carries the batch's terminal status, so output.outcome classifies each page fetched with --all and not only the first response. + servers: + - url: http://localhost:35456 + parameters: + - name: cursor + in: query + schema: + type: string + x-speakeasy-pagination: + type: cursor + inputs: + - name: cursor + in: parameters + type: cursor + outputs: + nextCursor: $.next_cursor + responses: + "200": + description: One page of interactions. + content: + application/json: + schema: + $ref: '#/components/schemas/outcomeInteractionPage' + /outcome/stream: + post: + operationId: outcomeStream + tags: [outcome] + description: SSE stream whose completion event carries the interaction's terminal status (output.outcome on streamed events). + servers: + - url: http://localhost:35456 + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [input] + properties: + input: + type: string + responses: + "200": + description: Streamed interaction events. + content: + text/event-stream: + x-speakeasy-sse-sentinel: "[DONE]" + schema: + $ref: '#/components/schemas/outcomeStreamEvent' +- target: $.components.schemas + update: + outcomeInteraction: + type: object + required: [id, status] + properties: + id: + type: string + status: + type: string + enum: [queued, in_progress, completed, incomplete, failed, cancelled] + output_text: + type: string + errors: + type: array + items: + $ref: '#/components/schemas/outcomeError' + outcomeError: + type: object + properties: + code: + type: string + message: + type: string + outcomeInteractionPage: + type: object + required: [status, items] + properties: + status: + type: string + enum: [queued, in_progress, completed, incomplete, failed, cancelled] + next_cursor: + type: string + items: + type: array + items: + $ref: '#/components/schemas/outcomeInteraction' + errors: + type: array + items: + $ref: '#/components/schemas/outcomeError' + outcomeStreamEvent: + type: object + required: [data] + properties: + data: + type: object + properties: + event_type: + type: string + content: + type: string + interaction: + $ref: '#/components/schemas/outcomeInteraction' diff --git a/tests/overlays/review/cli/overlay.yaml b/tests/overlays/review/cli/overlay.yaml index 7ecac3f72..592eb3c4f 100644 --- a/tests/overlays/review/cli/overlay.yaml +++ b/tests/overlays/review/cli/overlay.yaml @@ -50,3 +50,25 @@ actions: update: resource_id: type: string + - target: $['x-speakeasy-cli-commands'].commands.render.output + update: + outcome: + statePointer: $.status + states: + in_progress: pending + completed: success + failed: failure + requires_action: handoff + - target: $['x-speakeasy-cli-commands'].commands.say.output + update: + outcome: + streamStatePointer: $.data.status + states: + streaming: pending + completed: success + failed: failure + - target: $.paths['/chat'].post.responses['200'].content['text/event-stream'].schema.properties.data.properties + update: + status: + type: string + enum: [streaming, completed, failed] diff --git a/zSDKs/sdk-cli/.speakeasy/gen.lock b/zSDKs/sdk-cli/.speakeasy/gen.lock index 66c7e6b17..099667bde 100644 --- a/zSDKs/sdk-cli/.speakeasy/gen.lock +++ b/zSDKs/sdk-cli/.speakeasy/gen.lock @@ -1,7 +1,7 @@ lockVersion: 2.0.0 id: review-sdk-test-id management: - docChecksum: b76f0639e425901fecb124d1ffa0f638 + docChecksum: 542812afcb058d0bf283cdd12d4e3e4a docVersion: 0.1.0 speakeasyVersion: internal generationVersion: internal @@ -124,9 +124,9 @@ trackedFiles: internal/cli/intent_produce.go: last_write_checksum: sha1:bd40954296ab4b5dd4265987e5c8918d46209528 internal/cli/intent_render.go: - last_write_checksum: sha1:9b211c0615b4df7cad91ca9434669eebcc7282e3 + last_write_checksum: sha1:78811930e627878b1fcaf41dd2a6fcd208a3003d internal/cli/intent_say.go: - last_write_checksum: sha1:b0e2860aa03588884166695a4904b025757f5b47 + last_write_checksum: sha1:9a0baf8595e51aef6bdbbd381cfede03ea73b55e internal/cli/intents.go: last_write_checksum: sha1:997b951af6d215576d30574a54022937beedd890 internal/cli/intents_order_test.go: @@ -272,9 +272,9 @@ trackedFiles: internal/output/async.go: last_write_checksum: sha1:3fee77ef0cdf3414e9a903222a4a671b5ad90f2c internal/output/binary.go: - last_write_checksum: sha1:843f1fafc38ee03782e978f617a07bb543f06d3b + last_write_checksum: sha1:6fd77172cad1bd29dc9f157b931345f96a172223 internal/output/classify.go: - last_write_checksum: sha1:2d17c649096fbd0384e2a8be2e1e5efe189ed40d + last_write_checksum: sha1:db7e072a1c68803b44a68ab0c0a620170bb5c6d1 internal/output/color.go: last_write_checksum: sha1:c6f41f9691f6209f480d153c15c3424d8ca5883d internal/output/errortable.go: @@ -283,22 +283,24 @@ trackedFiles: last_write_checksum: sha1:313fa20ec561de744a818df99d4659ba715e0a86 internal/output/jq.go: last_write_checksum: sha1:9e59e92439a8c6269a76be1fa44e85cbbf2dee59 + internal/output/outcome.go: + last_write_checksum: sha1:5b7a3ad24255889f7a5cdc53478c237e7f358b85 internal/output/output.go: - last_write_checksum: sha1:387be8156f26d964a2b8f7eb679899dfa17b6478 + last_write_checksum: sha1:ad43078f33c1cd610c08a6417d4b5143e9caace6 internal/output/output_test.go: last_write_checksum: sha1:84a8a6cf79bfa62ae9635f992bc7578a2bbbcce3 internal/output/outputitems.go: last_write_checksum: sha1:d327eee8d9a395821fef201257e033bae50cda51 internal/output/paginated.go: - last_write_checksum: sha1:d060d669c8f1ca4390e46c5b4b684baf8b169898 + last_write_checksum: sha1:ef1ce0609772febe04419419ab06da121e649f27 internal/output/pretty.go: last_write_checksum: sha1:b7c74bd93d4e8a464956f098724dc653e9f8e594 internal/output/progress.go: last_write_checksum: sha1:f8b839f9394602fe9c30feb3d7155af6235663be internal/output/streaming.go: - last_write_checksum: sha1:3ffdcd4e995640502eaf0b10086fb4d1324ae971 + last_write_checksum: sha1:0bab1d6cbb6a5d6dca122c8d57a14467dd3f9237 internal/output/streaming_test.go: - last_write_checksum: sha1:9bb53612aa2a3e6d3da96f0c8f8e2ce4ba99b509 + last_write_checksum: sha1:ed4ffe1ebf27f5b32e69d0d2e4a377d47e1655c4 internal/output/streamproject.go: last_write_checksum: sha1:19d7dec64b1f78cbcfb5420cc4e968aa7aedadd0 internal/sdk/.gitattributes: @@ -442,7 +444,7 @@ trackedFiles: internal/sdk/models/operations/binaryandstringupload.go: last_write_checksum: sha1:7fb4d81daaf907f9894a74cbcda0474399461414 internal/sdk/models/operations/chat.go: - last_write_checksum: sha1:aa4756ca1e28c2fbefbbf10e588652e842e67e0d + last_write_checksum: sha1:601be5db5ea03d1206810ce26998975748283dd5 internal/sdk/models/operations/createnamespaceconflict.go: last_write_checksum: sha1:fa39c1f2ddb12ccafded76afcf4cd754aa6b4d21 internal/sdk/models/operations/createsinglenamespacefoopet.go: @@ -668,7 +670,7 @@ trackedFiles: scripts/install.sh: last_write_checksum: sha1:ba255d220bc830375d4f2bc34a7ca01e0a0a8fe2 tests/artifact_test.go: - last_write_checksum: sha1:933a9f1df6a4192d6decfac193e1f93af3eb63f1 + last_write_checksum: sha1:4ca3d41f09f6dc66d58a3c2b8de36722bb7e9dc3 tests/async_test.go: last_write_checksum: sha1:7feb37653f240d6b9c625f4476365b018f738a03 tests/cli_harness_test.go: @@ -934,7 +936,7 @@ trackedFiles: tests/mockserver/internal/sdk/models/operations/binaryandstringupload.go: last_write_checksum: sha1:abbd174263b69f5b19baaec3685c1cfd4375aea6 tests/mockserver/internal/sdk/models/operations/chat.go: - last_write_checksum: sha1:d51dfc8a83c2049c9573f1336a4c420888065383 + last_write_checksum: sha1:68425147994d2985f0a230579ce88d0b6d18704d tests/mockserver/internal/sdk/models/operations/createnamespaceconflict.go: last_write_checksum: sha1:c78a9d0dbb60e9e2e14aeb9d8d9ac5a6faf3c001 tests/mockserver/internal/sdk/models/operations/createsinglenamespacefoopet.go: diff --git a/zSDKs/sdk-cli/.speakeasy/logs/naming.log b/zSDKs/sdk-cli/.speakeasy/logs/naming.log index ed43ac982..3c16a1efb 100644 --- a/zSDKs/sdk-cli/.speakeasy/logs/naming.log +++ b/zSDKs/sdk-cli/.speakeasy/logs/naming.log @@ -465,7 +465,8 @@ ChatRequest (union) ChatResponse (HttpMeta: HTTPMetadata, object: class, ChatStream: event-stream) ChatResponseBody (data: string) ChatStream (data: class) - ChatData (content: string, resource_id: string) + ChatData (content: string, resource_id: string, status: enum) + Status (enum: streaming, completed, failed) GetBinaryDefaultResponseResponse (HttpMeta: HTTPMetadata, bytes: response-stream) TestEnumFormatsRequest (stringArrayFormat: enum, stringMapFormat: enum, stringPartialMapFormat: enum ...) StringArrayFormat (enum: pending_review, approved_final, rejected_permanently ...) diff --git a/zSDKs/sdk-cli/README.md b/zSDKs/sdk-cli/README.md index a239759a4..5b24e550a 100644 --- a/zSDKs/sdk-cli/README.md +++ b/zSDKs/sdk-cli/README.md @@ -243,7 +243,7 @@ In agent mode interactive prompts never launch, output defaults to TOON, and eve } ``` -`error_type` is one of `authentication_error`, `authorization_error`, `not_found`, `validation_error`, `rate_limit_error`, `server_error`, `api_error`, `connection_error`, `protocol_error`, `service_disabled`, `billing_disabled`, `runtime_error`, `unsupported_error`, `async_failed`, `async_timeout`, `async_unknown_state`. Classification reads the structured reason code at `$.details[*].reason`, then `$.status` in the error body (resolved against the nested `error` object when the body has one) before HTTP status, so a declared credential reason sent with HTTP 400 is not mistaken for request validation. `error_reason` carries the reason code found there, verbatim from a declared carrier when no declared rule matches it; it is absent for status-only API errors. Status-less local failures may use `CLI_VALIDATION`, `CLI_CONNECTION`, `CLI_PROTOCOL`, `CLI_RUNTIME`, `CLI_UNAVAILABLE`, `CLI_AUTHENTICATION`, or the async polling reasons `CLI_ASYNC_FAILED`, `CLI_ASYNC_TIMEOUT`, and `CLI_ASYNC_UNKNOWN_STATE`. `hints` preserves server guidance first, adds the most specific local taxonomy guidance, then typed CLI and command-specific guidance, removing exact duplicates. `exit_code` is always the code for the final `error_type` shown in the envelope: 1 runtime, 2 usage, or 3 authentication/authorization. +`error_type` is one of `authentication_error`, `authorization_error`, `not_found`, `validation_error`, `rate_limit_error`, `server_error`, `api_error`, `connection_error`, `protocol_error`, `service_disabled`, `billing_disabled`, `runtime_error`, `unsupported_error`, `async_failed`, `async_timeout`, `async_unknown_state`. Classification reads the structured reason code at `$.details[*].reason`, then `$.status` in the error body (resolved against the nested `error` object when the body has one) before HTTP status, so a declared credential reason sent with HTTP 400 is not mistaken for request validation. `error_reason` carries the reason code found there, verbatim from a declared carrier when no declared rule matches it; it is absent for status-only API errors. Status-less local failures may use `CLI_VALIDATION`, `CLI_CONNECTION`, `CLI_PROTOCOL`, `CLI_RUNTIME`, `CLI_UNAVAILABLE`, `CLI_AUTHENTICATION`, the async polling reasons `CLI_ASYNC_FAILED`, `CLI_ASYNC_TIMEOUT`, and `CLI_ASYNC_UNKNOWN_STATE`, or `CLI_OUTCOME_FAILED` when a successful response reports a declared failure state; that envelope also carries `state` (the reported state) and, when the command declares an error pointer, `error` (the failure detail taken from the response). `hints` preserves server guidance first, adds the most specific local taxonomy guidance, then typed CLI and command-specific guidance, removing exact duplicates. `exit_code` is always the code for the final `error_type` shown in the envelope: 1 runtime, 2 usage, or 3 authentication/authorization. ```bash # "invite" declares hints for CLI_VALIDATION; a failing request returns them in the envelope diff --git a/zSDKs/sdk-cli/internal/cli/intent_render.go b/zSDKs/sdk-cli/internal/cli/intent_render.go index 695eca04a..dfb9691aa 100644 --- a/zSDKs/sdk-cli/internal/cli/intent_render.go +++ b/zSDKs/sdk-cli/internal/cli/intent_render.go @@ -27,6 +27,7 @@ func InitIntentRender(parent *cobra.Command) error { flagutil.AnnotationWholeBodyFlag: "body", "speakeasy_strict_body_keys": "true", "speakeasy_artifact": "{\"pointer\":[{\"field\":\"steps\"},{\"wild\":true},{\"field\":\"content\"},{\"wild\":true}],\"kind\":\"image\",\"defaultPath\":\"render-{timestamp}-{rand}.{ext}\"}", + "speakeasy_outcome": "{\"statePointer\":\"/status\",\"states\":{\"completed\":\"success\",\"failed\":\"failure\",\"in_progress\":\"pending\",\"requires_action\":\"handoff\"\x7d\x7d", }, } intentMeta := flagutil.NonBodyMeta(renderAssetCmdMeta, "") diff --git a/zSDKs/sdk-cli/internal/cli/intent_say.go b/zSDKs/sdk-cli/internal/cli/intent_say.go index fb47c8ead..631c72e37 100644 --- a/zSDKs/sdk-cli/internal/cli/intent_say.go +++ b/zSDKs/sdk-cli/internal/cli/intent_say.go @@ -26,6 +26,7 @@ func InitIntentSay(parent *cobra.Command) error { "speakeasy_operation": "chat", flagutil.AnnotationWholeBodyFlag: "request", "speakeasy_strict_body_keys": "true", + "speakeasy_outcome": "{\"streamStatePointer\":\"/data/status\",\"states\":{\"completed\":\"success\",\"failed\":\"failure\",\"streaming\":\"pending\"\x7d\x7d", "speakeasy_stream_select": "/data/content", "speakeasy_stream_metadata": "[{\"label\":\"Resource ID\",\"pointers\":[\"/data/resource_id\"]}]", }, diff --git a/zSDKs/sdk-cli/internal/output/binary.go b/zSDKs/sdk-cli/internal/output/binary.go index b8c6839a9..3923e2a20 100644 --- a/zSDKs/sdk-cli/internal/output/binary.go +++ b/zSDKs/sdk-cli/internal/output/binary.go @@ -23,6 +23,7 @@ func WriteToFile(cmd *cobra.Command, res interface{}, path string) error { if flagutil.DidDryRunRequest(cmd) { return nil } + failure := outcomeResultFailure(cmd, res) content := extractResultContent(res) f, err := os.Create(path) @@ -74,10 +75,16 @@ func WriteToFile(cmd *cobra.Command, res interface{}, path string) error { if !IsMachineMode(cmd) { fmt.Fprintf(cmd.ErrOrStderr(), "Wrote %s (%d bytes)%s", absPath, written.Size(), string(rune(10))) } - return renderPathEnvelope(cmd, absPath, map[string]interface{}{ + if err := renderPathEnvelope(cmd, absPath, map[string]interface{}{ "path": absPath, "size_bytes": written.Size(), - }) + }); err != nil { + return err + } + if failure != nil { + return Error(cmd, failure) + } + return nil } // WriteBase64 encodes binary response content as base64 and prints it to stdout. @@ -87,6 +94,7 @@ func WriteBase64(cmd *cobra.Command, res interface{}) error { if flagutil.DidDryRunRequest(cmd) { return nil } + failure := outcomeResultFailure(cmd, res) content := extractResultContent(res) if content == nil { return nil @@ -116,6 +124,9 @@ func WriteBase64(cmd *cobra.Command, res interface{}) error { fmt.Fprintln(w, base64.StdEncoding.EncodeToString(data)) } + if failure != nil { + return Error(cmd, failure) + } return nil } diff --git a/zSDKs/sdk-cli/internal/output/classify.go b/zSDKs/sdk-cli/internal/output/classify.go index 90d24516b..b2f510e5a 100644 --- a/zSDKs/sdk-cli/internal/output/classify.go +++ b/zSDKs/sdk-cli/internal/output/classify.go @@ -688,6 +688,7 @@ const ( ReasonCLIAsyncFailed = "CLI_ASYNC_FAILED" ReasonCLIAsyncTimeout = "CLI_ASYNC_TIMEOUT" ReasonCLIAsyncUnknownState = "CLI_ASYNC_UNKNOWN_STATE" + ReasonCLIOutcomeFailed = "CLI_OUTCOME_FAILED" ) type cliReasonError struct { @@ -709,7 +710,8 @@ func normalizeCLIReason(reason string) string { switch reason { case ReasonCLIValidation, ReasonCLIConnection, ReasonCLIProtocol, ReasonCLIRuntime, ReasonCLIUnavailable, ReasonCLIAuthentication, - ReasonCLIAsyncFailed, ReasonCLIAsyncTimeout, ReasonCLIAsyncUnknownState: + ReasonCLIAsyncFailed, ReasonCLIAsyncTimeout, ReasonCLIAsyncUnknownState, + ReasonCLIOutcomeFailed: return reason } return ReasonCLIRuntime diff --git a/zSDKs/sdk-cli/internal/output/outcome.go b/zSDKs/sdk-cli/internal/output/outcome.go new file mode 100644 index 000000000..ba67e397b --- /dev/null +++ b/zSDKs/sdk-cli/internal/output/outcome.go @@ -0,0 +1,187 @@ +// Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT. + +package output + +import ( + "encoding/json" + "fmt" + "io" + "strconv" + "strings" + + "github.com/spf13/cobra" +) + +const outcomeAnnotation = "speakeasy_outcome" + +type outcomeConfig struct { + StatePointer string `json:"statePointer"` + StreamStatePointer string `json:"streamStatePointer"` + States map[string]string `json:"states"` + ErrorPointer string `json:"errorPointer"` + StreamErrorPointer string `json:"streamErrorPointer"` + ErrorMessageField string `json:"errorMessageField"` +} + +func activeOutcome(cmd *cobra.Command) *outcomeConfig { + if cmd == nil || cmd.Annotations == nil { + return nil + } + raw := cmd.Annotations[outcomeAnnotation] + if raw == "" { + return nil + } + var cfg outcomeConfig + if err := json.Unmarshal([]byte(raw), &cfg); err != nil || len(cfg.States) == 0 { + return nil + } + if cfg.ErrorMessageField == "" { + cfg.ErrorMessageField = "message" + } + return &cfg +} + +type outcomeError struct { + message string + state string + detail interface{} +} + +func (e *outcomeError) Error() string { return e.message } + +func (e *outcomeError) CLIReason() string { return ReasonCLIOutcomeFailed } + +func (e *outcomeError) MachineErrorFields() map[string]interface{} { + fields := map[string]interface{}{"state": e.state} + if e.detail != nil { + fields["error"] = e.detail + } + return fields +} + +func outcomeResultFailure(cmd *cobra.Command, res interface{}) *outcomeError { + cfg := activeOutcome(cmd) + if cfg == nil || cfg.StatePointer == "" { + return nil + } + payload, ok := outcomeResultPayload(res) + if !ok { + return nil + } + return outcomeFailure(cmd, cfg, payload, cfg.StatePointer, cfg.ErrorPointer) +} + +func outcomeFailure(cmd *cobra.Command, cfg *outcomeConfig, payload interface{}, statePointer, errorPointer string) *outcomeError { + if cfg == nil || statePointer == "" { + return nil + } + value, ok := outcomePointer(payload, statePointer) + state, isString := value.(string) + if !ok || !isString || state == "" || cfg.States[state] != "failure" { + return nil + } + failure := &outcomeError{state: state, message: fmt.Sprintf("%s ended in failure state %q", outcomeCommandLabel(cmd), state)} + if errorPointer != "" { + if detail, found := outcomePointer(payload, errorPointer); found && detail != nil { + failure.detail = detail + if message := outcomeDetailMessage(detail, cfg.ErrorMessageField); message != "" { + failure.message += ": " + message + } + } + } + return failure +} + +func outcomeDetailMessage(detail interface{}, field string) string { + switch value := detail.(type) { + case string: + return strings.TrimSpace(value) + case map[string]interface{}: + message, _ := value[field].(string) + return strings.TrimSpace(message) + case []interface{}: + for _, item := range value { + if message := outcomeDetailMessage(item, field); message != "" { + return message + } + } + } + return "" +} + +func outcomeCommandLabel(cmd *cobra.Command) string { + path := cmd.CommandPath() + if root := cmd.Root(); root != nil && root != cmd { + path = strings.TrimPrefix(path, root.Name()+" ") + } + return path +} + +func outcomeResultPayload(res interface{}) (interface{}, bool) { + switch extractResultContent(res).(type) { + case io.ReadCloser, []byte: + return nil, false + } + raw := peekRawBody(res) + if len(raw) == 0 { + content := extractResultContent(res) + if content == nil { + return nil, false + } + var err error + if raw, err = marshalJSON(content); err != nil { + return nil, false + } + } + parsed, err := decodeWithNumbers(raw) + return parsed, err == nil +} + +// outcomePointer resolves an RFC 6901 pointer against decoded JSON. +func outcomePointer(root interface{}, pointer string) (interface{}, bool) { + if pointer == "" { + return root, true + } + if !strings.HasPrefix(pointer, "/") { + return nil, false + } + current := root + for _, raw := range strings.Split(strings.TrimPrefix(pointer, "/"), "/") { + segment := strings.ReplaceAll(strings.ReplaceAll(raw, "~1", "/"), "~0", "~") + switch value := current.(type) { + case map[string]interface{}: + var ok bool + if current, ok = value[segment]; !ok { + return nil, false + } + case []interface{}: + index, err := strconv.Atoi(segment) + if err != nil || index < 0 || index >= len(value) { + return nil, false + } + current = value[index] + default: + return nil, false + } + } + return current, true +} + +func outcomeStreamFailure(cmd *cobra.Command, cfg *outcomeConfig, item interface{}, frame map[string]interface{}) *outcomeError { + if cfg == nil || cfg.StreamStatePointer == "" { + return nil + } + var payload interface{} = frame + if frame == nil { + data, err := marshalJSON(item) + if err != nil { + return nil + } + decoded, err := decodeWithNumbers(data) + if err != nil { + return nil + } + payload = decoded + } + return outcomeFailure(cmd, cfg, payload, cfg.StreamStatePointer, cfg.StreamErrorPointer) +} diff --git a/zSDKs/sdk-cli/internal/output/output.go b/zSDKs/sdk-cli/internal/output/output.go index 26ded673f..edc016104 100644 --- a/zSDKs/sdk-cli/internal/output/output.go +++ b/zSDKs/sdk-cli/internal/output/output.go @@ -398,6 +398,20 @@ func LocalResult(cmd *cobra.Command, res interface{}) error { } func result(cmd *cobra.Command, res interface{}, allowArtifact bool) error { + failure := outcomeResultFailure(cmd, res) + if failure != nil && allowArtifact && activeArtifact(cmd) != nil { + return Error(cmd, failure) + } + if err := renderResult(cmd, res, allowArtifact); err != nil { + return err + } + if failure != nil { + return Error(cmd, failure) + } + return nil +} + +func renderResult(cmd *cobra.Command, res interface{}, allowArtifact bool) error { out := cmd.OutOrStdout() colorFlag, _ := flagutil.GetStringFlag(cmd, "color") colorize := ShouldColorize(colorFlag) diff --git a/zSDKs/sdk-cli/internal/output/paginated.go b/zSDKs/sdk-cli/internal/output/paginated.go index c41ec7ec5..a5e480469 100644 --- a/zSDKs/sdk-cli/internal/output/paginated.go +++ b/zSDKs/sdk-cli/internal/output/paginated.go @@ -135,6 +135,11 @@ func PaginatedResult(cmd *cobra.Command, res interface{}, contentFieldName, resu } } + if failure := outcomeResultFailure(cmd, page); failure != nil { + _ = flushRows(true) + return Error(cmd, failure) + } + pageCount++ if maxPages > 0 && pageCount >= maxPages { if HasMorePages(page, probe) && !IsMachineMode(cmd) { diff --git a/zSDKs/sdk-cli/internal/output/streaming.go b/zSDKs/sdk-cli/internal/output/streaming.go index 3ace53f12..c5b89175d 100644 --- a/zSDKs/sdk-cli/internal/output/streaming.go +++ b/zSDKs/sdk-cli/internal/output/streaming.go @@ -87,6 +87,11 @@ func StreamResult(cmd *cobra.Command, res interface{}, streamFieldName string, s } projector := newStreamProjector(cmd) + outcomeCfg := activeOutcome(cmd) + if outcomeCfg != nil && outcomeCfg.StreamStatePointer == "" { + outcomeCfg = nil + } + var streamFailure *outcomeError // Projected output already written must end with its terminating // newline before any error is rendered. The primary error keeps the @@ -159,9 +164,12 @@ func StreamResult(cmd *cobra.Command, res interface{}, streamFieldName string, s } } - if raw != nil { - item = wireStreamItem(item, raw, format == "table" && jqExpr == "") + var frame map[string]interface{} + if outcomeCfg != nil && streamFailure == nil { + frame, _ = decodeStreamFrame(raw) + streamFailure = outcomeStreamFailure(cmd, outcomeCfg, item, frame) } + item = wireStreamItem(item, raw, frame, format == "table" && jqExpr == "") if projector != nil { if err := projector.emit(out, item); err != nil { @@ -181,6 +189,9 @@ func StreamResult(cmd *cobra.Command, res interface{}, streamFieldName string, s return renderAfterFinish(errResults[0].Interface().(error)) } } + if streamFailure != nil { + return renderAfterFinish(streamFailure) + } if projector != nil { if err := projector.finish(out); err != nil { @@ -202,7 +213,9 @@ func (w wireEvent) MarshalJSON() ([]byte, error) { } // Re-encoding the decoded model drops undeclared fields and zero-fills absent ones. -func wireStreamItem(item interface{}, raw []byte, table bool) interface{} { +// frame is the wire object when the caller already decoded raw; nil decodes +// it here, and only once the item is known to need it. +func wireStreamItem(item interface{}, raw []byte, frame map[string]interface{}, table bool) interface{} { keys, ok := envelopeKeys(reflect.ValueOf(item)) if len(raw) == 0 || !ok { return item @@ -212,10 +225,11 @@ func wireStreamItem(item interface{}, raw []byte, table bool) interface{} { if err == nil && table { return item } - frame, err := decodeWithNumbers(raw) - wire, isObject := frame.(map[string]interface{}) - if err != nil || !isObject { - return item + wire := frame + if wire == nil { + if wire, ok = decodeStreamFrame(raw); !ok { + return item + } } for name := range wire { if keys != nil && !keys[name] { @@ -229,6 +243,18 @@ func wireStreamItem(item interface{}, raw []byte, table bool) interface{} { return wireEvent{data: data} } +func decodeStreamFrame(raw []byte) (map[string]interface{}, bool) { + if len(raw) == 0 { + return nil, false + } + frame, err := decodeWithNumbers(raw) + wire, isObject := frame.(map[string]interface{}) + if err != nil || !isObject { + return nil, false + } + return wire, true +} + // nil keys keep the whole frame: a union event that matched no member. func envelopeKeys(v reflect.Value) (map[string]bool, bool) { v = derefValue(v) @@ -327,7 +353,7 @@ func streamErrorEvent(item interface{}, raw []byte) (json.RawMessage, bool) { return data, true } if isUnionStruct(derefValue(reflect.ValueOf(item)).Type()) { - data, err := json.Marshal(wireStreamItem(item, raw, false)) + data, err := json.Marshal(wireStreamItem(item, raw, nil, false)) return data, err == nil } var frame map[string]json.RawMessage diff --git a/zSDKs/sdk-cli/internal/output/streaming_test.go b/zSDKs/sdk-cli/internal/output/streaming_test.go index 31a436535..c71b17931 100644 --- a/zSDKs/sdk-cli/internal/output/streaming_test.go +++ b/zSDKs/sdk-cli/internal/output/streaming_test.go @@ -42,7 +42,7 @@ func TestWireStreamItemUnwrapsNestedEnvelopeUnions(t *testing.T) { "inner member unknown": {outerEnvelopeUnion{Inner: &innerEnvelopeUnion{}}, `{"data":"hello","event":"tick","id":"7"}`}, } { t.Run(name, func(t *testing.T) { - got, err := json.Marshal(wireStreamItem(tc.item, raw, false)) + got, err := json.Marshal(wireStreamItem(tc.item, raw, nil, false)) if err != nil || string(got) != tc.want { t.Fatalf("got %s, error %v; want %s", got, err, tc.want) } @@ -58,7 +58,7 @@ type openEnvelope struct { func TestWireStreamItemKeepsFrameOfEnvelopeWithAdditionalProperties(t *testing.T) { raw := []byte(`{"data":"hello","event":"tick","id":"7"}`) want := `{"data":"hello","event":"tick","id":"7"}` - got, err := json.Marshal(wireStreamItem(openEnvelope{Data: "hello"}, raw, false)) + got, err := json.Marshal(wireStreamItem(openEnvelope{Data: "hello"}, raw, nil, false)) if err != nil || string(got) != want { t.Fatalf("got %s, error %v; want %s", got, err, want) } @@ -139,7 +139,7 @@ func TestWireStreamItemTableKeepsModelLayout(t *testing.T) { } { t.Run(name, func(t *testing.T) { var buf bytes.Buffer - if err := printTable(&buf, wireStreamItem(tc.item, []byte(tc.raw), true)); err != nil || buf.String() != tc.want { + if err := printTable(&buf, wireStreamItem(tc.item, []byte(tc.raw), nil, true)); err != nil || buf.String() != tc.want { t.Fatalf("got %q, error %v; want %q", buf.String(), err, tc.want) } }) diff --git a/zSDKs/sdk-cli/internal/sdk/models/operations/chat.go b/zSDKs/sdk-cli/internal/sdk/models/operations/chat.go index c55a4ccbf..24e1c40ad 100644 --- a/zSDKs/sdk-cli/internal/sdk/models/operations/chat.go +++ b/zSDKs/sdk-cli/internal/sdk/models/operations/chat.go @@ -106,9 +106,33 @@ func (u ChatRequest) MarshalJSON() ([]byte, error) { return nil, errors.New("could not marshal union type ChatRequest: all fields are null") } +type Status string + +const ( + StatusStreaming Status = "streaming" + StatusCompleted Status = "completed" + StatusFailed Status = "failed" +) + +func (e Status) ToPointer() *Status { + return &e +} + +// IsExact returns true if the value matches a known enum value, false otherwise. +func (e *Status) IsExact() bool { + if e != nil { + switch *e { + case "streaming", "completed", "failed": + return true + } + } + return false +} + type ChatData struct { Content *string `json:"content,omitzero"` ResourceID *string `json:"resource_id,omitzero"` + Status *Status `json:"status,omitzero"` } func (c *ChatData) GetContent() *string { @@ -125,6 +149,13 @@ func (c *ChatData) GetResourceID() *string { return c.ResourceID } +func (c *ChatData) GetStatus() *Status { + if c == nil { + return nil + } + return c.Status +} + // ChatStream - A stream containing chat completion tokens type ChatStream struct { Data ChatData `json:"data"` diff --git a/zSDKs/sdk-cli/tests/artifact_test.go b/zSDKs/sdk-cli/tests/artifact_test.go index 365de0b40..c660c1fea 100644 --- a/zSDKs/sdk-cli/tests/artifact_test.go +++ b/zSDKs/sdk-cli/tests/artifact_test.go @@ -588,3 +588,19 @@ func TestArtifact_InteractiveSuppressedByNonInteractiveAndAgentModes(t *testing. }) } } + +// A declared failure state (output.outcome) fails the command before any +// media is written. +func TestArtifact_DeclaredFailureStateWritesNothing(t *testing.T) { + server := artifactTestServer(t, `{"id":"task-123","status":"failed","steps":[{"content":[{"type":"image","data":"YXJ0aWZhY3QtYnl0ZXM=","mime_type":"image/png"}]}]}`) + dir := t.TempDir() + h := NewCLITestHarness(t) + + err := runArtifact(t, h, server.URL, dir) + require.Error(t, err) + assert.Contains(t, err.Error(), "ended in failure state "+"\"failed\"") + assert.Empty(t, strings.TrimSpace(h.GetStdout())) + entries, err := os.ReadDir(dir) + require.NoError(t, err) + assert.Empty(t, entries, "a failure state must not write the artifact") +} diff --git a/zSDKs/sdk-cli/tests/mockserver/internal/sdk/models/operations/chat.go b/zSDKs/sdk-cli/tests/mockserver/internal/sdk/models/operations/chat.go index bc2e6be36..1087370ec 100644 --- a/zSDKs/sdk-cli/tests/mockserver/internal/sdk/models/operations/chat.go +++ b/zSDKs/sdk-cli/tests/mockserver/internal/sdk/models/operations/chat.go @@ -3,6 +3,7 @@ package operations import ( + "encoding/json" "errors" "fmt" "mockserver/internal/sdk/models/components" @@ -73,9 +74,39 @@ func (u ChatRequest) MarshalJSON() ([]byte, error) { return nil, errors.New("could not marshal union type ChatRequest: all fields are null") } +type Status string + +const ( + StatusStreaming Status = "streaming" + StatusCompleted Status = "completed" + StatusFailed Status = "failed" +) + +func (e Status) ToPointer() *Status { + return &e +} +func (e *Status) UnmarshalJSON(data []byte) error { + var v string + if err := json.Unmarshal(data, &v); err != nil { + return err + } + switch v { + case "streaming": + fallthrough + case "completed": + fallthrough + case "failed": + *e = Status(v) + return nil + default: + return fmt.Errorf("invalid value for Status: %v", v) + } +} + type ChatData struct { Content *string `json:"content,omitempty"` ResourceID *string `json:"resource_id,omitempty"` + Status *Status `json:"status,omitempty"` } func (o *ChatData) GetContent() *string { @@ -92,6 +123,13 @@ func (o *ChatData) GetResourceID() *string { return o.ResourceID } +func (o *ChatData) GetStatus() *Status { + if o == nil { + return nil + } + return o.Status +} + // ChatStream - A stream containing chat completion tokens type ChatStream struct { Data ChatData `json:"data"`