Skip to content

Commit 4464f3f

Browse files
committed
air: show workspace-supported accelerators in compute help
1 parent 80d4ffb commit 4464f3f

9 files changed

Lines changed: 258 additions & 5 deletions

File tree

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
* Show only the accelerator types available in the current workspace in `databricks air run -h config.compute`. ([#6993](https://github.com/databricks/cli/pull/6993))

‎acceptance/air/config-help/output.txt‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,10 @@ Use "-h config.<field>" for details on a field.
7474
config.compute
7575
Which accelerators to run on and how many.
7676

77+
Supported accelerator types:
78+
GPU_1xH100 - 1 accelerator per node [Private Preview]
79+
GPU_8xH100 - 8 accelerators per node
80+
7781
Fields:
7882
num_accelerators Total number of GPUs to allocate.
7983
accelerator_type Which accelerator to run on, e.g. GPU_1xA10.
Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
# The SDK occasionally probes host reachability with a HEAD request; stub it so
2+
# the test is deterministic.
3+
[[Server]]
4+
Pattern = "HEAD /"
5+
Response.Body = ''
6+
7+
[[Server]]
8+
Pattern = "GET /api/2.0/ai-training/compute-options"
9+
Response.Body = '''
10+
{
11+
"compute_options": [
12+
{"hardware_accelerator": "GPU_1xH100", "per_node_accelerator_count": 1, "launch_stage": "PRIVATE_PREVIEW"},
13+
{"hardware_accelerator": "GPU_8xH100", "per_node_accelerator_count": 8, "launch_stage": "GA"}
14+
]
15+
}
16+
'''

‎cmd/air/aitraining.go‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,34 @@ import (
1818
// with a raw client.Do because the SDK does not model the AiTrainingService.
1919
const aiTrainingWorkflowsPath = "/api/2.0/ai-training/workflows"
2020

21+
const computeOptionsPath = "/api/2.0/ai-training/compute-options"
22+
23+
type computeOption struct {
24+
HardwareAccelerator string `json:"hardware_accelerator"`
25+
DisplayName string `json:"display_name"`
26+
MultiNodeSupported *bool `json:"multi_node_supported"`
27+
PerNodeAcceleratorCount *int `json:"per_node_accelerator_count"`
28+
LaunchStage string `json:"launch_stage"`
29+
}
30+
31+
type computeOptionsResponse struct {
32+
ComputeOptions []computeOption `json:"compute_options"`
33+
}
34+
35+
func listWorkspaceComputeOptions(ctx context.Context, w *databricks.WorkspaceClient) ([]computeOption, error) {
36+
apiClient, err := client.New(w.Config)
37+
if err != nil {
38+
return nil, fmt.Errorf("failed to create API client: %w", err)
39+
}
40+
41+
var resp computeOptionsResponse
42+
err = apiClient.Do(ctx, http.MethodGet, computeOptionsPath, auth.WorkspaceIDHeaders(w.Config), nil, nil, &resp)
43+
if err != nil {
44+
return nil, fmt.Errorf("failed to list compute options: %w", err)
45+
}
46+
return resp.ComputeOptions, nil
47+
}
48+
2149
// workflowRef is one run from the index: its Jobs run id and submission time.
2250
type workflowRef struct {
2351
jobRunID int64

‎cmd/air/aitraining_test.go‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,31 @@ func TestParseSubmitTimeMs(t *testing.T) {
3030
}
3131
}
3232

33+
func TestListWorkspaceComputeOptions(t *testing.T) {
34+
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
35+
if r.URL.Path == "/.well-known/databricks-config" {
36+
_, _ = w.Write([]byte(`{}`))
37+
return
38+
}
39+
assert.Equal(t, http.MethodGet, r.Method)
40+
assert.Equal(t, computeOptionsPath, r.URL.Path)
41+
assert.Equal(t, "123", r.Header.Get("X-Databricks-Workspace-Id"))
42+
_, _ = w.Write([]byte(`{"compute_options":[{"hardware_accelerator":"GPU_8xH100","display_name":"8x H100","multi_node_supported":true,"per_node_accelerator_count":8,"launch_stage":"PUBLIC_PREVIEW"}]}`))
43+
}))
44+
t.Cleanup(srv.Close)
45+
w := newTestWorkspaceClient(t, srv.URL)
46+
w.Config.WorkspaceID = "123"
47+
48+
options, err := listWorkspaceComputeOptions(t.Context(), w)
49+
require.NoError(t, err)
50+
require.Len(t, options, 1)
51+
assert.Equal(t, "GPU_8xH100", options[0].HardwareAccelerator)
52+
assert.Equal(t, "8x H100", options[0].DisplayName)
53+
assert.Equal(t, true, *options[0].MultiNodeSupported)
54+
assert.Equal(t, 8, *options[0].PerNodeAcceleratorCount)
55+
assert.Equal(t, "PUBLIC_PREVIEW", options[0].LaunchStage)
56+
}
57+
3358
// indexServer serves paginated AiTrainingService responses, one body per call,
3459
// tracking whether the index was hit.
3560
func indexServer(t *testing.T, hit *bool, bodies ...string) *httptest.Server {

‎cmd/air/run.go‎

Lines changed: 39 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,9 @@ import (
2424
// dryRunValidationTimeout bounds only the config:validate request.
2525
const dryRunValidationTimeout = 15 * time.Second
2626

27+
// Bounds the workspace lookup performed by config help.
28+
const computeOptionsHelpTimeout = 5 * time.Second
29+
2730
// runResult is the JSON payload for `air run`.
2831
type runResult struct {
2932
Status string `json:"status"`
@@ -112,9 +115,17 @@ The path must be a separate argument: cobra reserves -h as a boolean, so
112115
_ = c.Usage()
113116
return
114117
}
115-
if err := writeConfigFieldHelp(c.OutOrStdout(), fields[0]); err != nil {
118+
field, err := resolveConfigField(fields[0])
119+
if err != nil {
116120
c.PrintErrln("Error:", err)
121+
return
122+
}
123+
124+
var computeOptions []computeOption
125+
if field.path == "config.compute" {
126+
computeOptions = loadComputeOptionsForHelp(c, args)
117127
}
128+
renderConfigField(c.OutOrStdout(), field, computeOptions)
118129
})
119130

120131
cmd.Flags().StringVarP(&file, "file", "f", "", "Path to the workload YAML config")
@@ -265,6 +276,33 @@ The path must be a separate argument: cobra reserves -h as a boolean, so
265276
return cmd
266277
}
267278

279+
func loadComputeOptionsForHelp(cmd *cobra.Command, args []string) []computeOption {
280+
ctx, cancel := context.WithTimeout(cmd.Context(), computeOptionsHelpTimeout)
281+
defer cancel()
282+
cmd.SetContext(root.SkipLoadBundle(root.SkipPrompt(ctx)))
283+
284+
if !cmdctx.HasWorkspaceClient(cmd.Context()) {
285+
if err := root.MustWorkspaceClient(cmd, args); err != nil {
286+
cmd.PrintErrf("Warning: couldn't resolve a workspace to list accelerator types; showing the built-in list instead: %v\n", err)
287+
return fallbackComputeOptions()
288+
}
289+
}
290+
291+
options, err := listWorkspaceComputeOptions(cmd.Context(), cmdctx.WorkspaceClient(cmd.Context()))
292+
if err != nil {
293+
if !endpointUnavailable(err) {
294+
cmd.PrintErrf("Warning: couldn't fetch the accelerator types available in this workspace; showing the built-in list instead: %v\n", err)
295+
}
296+
return fallbackComputeOptions()
297+
}
298+
options = filterKnownComputeOptions(options)
299+
if len(options) == 0 {
300+
cmd.PrintErrln("Warning: the workspace reported no supported accelerator types; showing the built-in list instead")
301+
return fallbackComputeOptions()
302+
}
303+
return options
304+
}
305+
268306
func airLogsCommand(profile, runID string) string {
269307
args := []string{"databricks", "air", "logs", shellquote.BashArg(runID)}
270308
if profile != "" {

‎cmd/air/runconfig.go‎

Lines changed: 65 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -610,7 +610,7 @@ func writeConfigFieldHelp(w io.Writer, path string) error {
610610
if err != nil {
611611
return err
612612
}
613-
renderConfigField(w, field)
613+
renderConfigField(w, field, nil)
614614
return nil
615615
}
616616

@@ -857,11 +857,17 @@ func underlyingConfigStruct(t reflect.Type) reflect.Type {
857857

858858
// renderConfigField writes a resolved field's documentation. An object lists its
859859
// immediate children; a leaf gets its type, required-ness, and description.
860-
func renderConfigField(w io.Writer, f configField) {
860+
func renderConfigField(w io.Writer, f configField, computeOptions []computeOption) {
861861
fmt.Fprintf(w, "%s\n", f.path)
862862
if f.help != "" {
863863
fmt.Fprintf(w, " %s\n", f.help)
864864
}
865+
if f.path == "config.compute" && len(computeOptions) > 0 {
866+
fmt.Fprintln(w, "\n Supported accelerator types:")
867+
for _, option := range computeOptions {
868+
fmt.Fprintf(w, " %s - %s%s\n", option.HardwareAccelerator, formatPerNodeAcceleratorCount(option), launchStageBadge(option.LaunchStage))
869+
}
870+
}
865871

866872
if len(f.children) == 0 {
867873
fmt.Fprintf(w, "\n Type: %s\n", f.typeName)
@@ -890,6 +896,63 @@ func renderConfigField(w io.Writer, f configField) {
890896
fmt.Fprintf(w, "\nUse \"-h %s.<field>\" for details on a field.\n", f.path)
891897
}
892898

899+
func filterKnownComputeOptions(options []computeOption) []computeOption {
900+
result := make([]computeOption, 0, len(options))
901+
for _, option := range options {
902+
if _, err := parseGPUType(option.HardwareAccelerator); err == nil {
903+
result = append(result, option)
904+
}
905+
}
906+
return result
907+
}
908+
909+
func fallbackComputeOptions() []computeOption {
910+
options := make([]computeOption, 0, len(gpuTypes))
911+
for _, acceleratorType := range gpuTypes {
912+
perNode, err := gpusPerNode(acceleratorType)
913+
if err != nil {
914+
continue
915+
}
916+
options = append(options, computeOption{
917+
HardwareAccelerator: string(acceleratorType),
918+
PerNodeAcceleratorCount: &perNode,
919+
})
920+
}
921+
return options
922+
}
923+
924+
func formatPerNodeAcceleratorCount(option computeOption) string {
925+
count := option.PerNodeAcceleratorCount
926+
if count == nil {
927+
acceleratorType, err := parseGPUType(option.HardwareAccelerator)
928+
if err == nil {
929+
if fallback, err := gpusPerNode(acceleratorType); err == nil {
930+
count = &fallback
931+
}
932+
}
933+
}
934+
if count == nil {
935+
return "accelerator count per node unavailable"
936+
}
937+
unit := "accelerators"
938+
if *count == 1 {
939+
unit = "accelerator"
940+
}
941+
return fmt.Sprintf("%d %s per node", *count, unit)
942+
}
943+
944+
func launchStageBadge(stage string) string {
945+
labels := map[string]string{
946+
"PRIVATE_PREVIEW": "Private Preview",
947+
"PUBLIC_BETA": "Beta",
948+
"PUBLIC_PREVIEW": "Public Preview",
949+
}
950+
if label := labels[stage]; label != "" {
951+
return " [" + label + "]"
952+
}
953+
return ""
954+
}
955+
893956
// configFieldSummary is the one-line description used in a field listing: the
894957
// first sentence of the help text, annotated when the field is required.
895958
func configFieldSummary(f configField) string {

‎cmd/air/runconfig_test.go‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,14 @@ package aircmd
22

33
import (
44
"fmt"
5+
"net/http"
6+
"net/http/httptest"
57
"os"
68
"path/filepath"
79
"strings"
810
"testing"
911

12+
"github.com/databricks/cli/libs/cmdctx"
1013
"github.com/stretchr/testify/assert"
1114
"github.com/stretchr/testify/require"
1215
)
@@ -756,6 +759,82 @@ func TestWriteConfigFieldHelp(t *testing.T) {
756759
require.Error(t, writeConfigFieldHelp(&strings.Builder{}, "config.nope"))
757760
}
758761

762+
func TestRunCommandHelpComputeOptions(t *testing.T) {
763+
builtInOptions := ` GPU_1xA10 - 1 accelerator per node
764+
GPU_1xH100 - 1 accelerator per node
765+
GPU_8xH100 - 8 accelerators per node
766+
GPU_8xB300 - 8 accelerators per node`
767+
tests := []struct {
768+
name string
769+
statusCode int
770+
response string
771+
wantOptions string
772+
wantWarning bool
773+
}{
774+
{
775+
name: "workspace subset",
776+
response: `{"compute_options":[{"hardware_accelerator":"GPU_8xH100","per_node_accelerator_count":4,"launch_stage":"PUBLIC_PREVIEW"},{"hardware_accelerator":"GPU_UNKNOWN"},{"hardware_accelerator":"GPU_1xA10"}]}`,
777+
wantOptions: " GPU_8xH100 - 4 accelerators per node [Public Preview]\n GPU_1xA10 - 1 accelerator per node",
778+
},
779+
{
780+
name: "empty response falls back",
781+
response: `{}`,
782+
wantOptions: builtInOptions,
783+
wantWarning: true,
784+
},
785+
{
786+
name: "disabled endpoint falls back",
787+
statusCode: http.StatusBadRequest,
788+
response: `{"error_code":"FEATURE_DISABLED","message":"ListComputeOptions is not yet enabled."}`,
789+
wantOptions: builtInOptions,
790+
},
791+
{
792+
name: "missing endpoint falls back",
793+
statusCode: http.StatusNotFound,
794+
response: `{"error_code":"ENDPOINT_NOT_FOUND","message":"Not found."}`,
795+
wantOptions: builtInOptions,
796+
},
797+
{
798+
name: "unexpected API error warns and falls back",
799+
statusCode: http.StatusInternalServerError,
800+
response: `{"error_code":"INTERNAL_ERROR","message":"Internal error."}`,
801+
wantOptions: builtInOptions,
802+
wantWarning: true,
803+
},
804+
}
805+
806+
for _, tt := range tests {
807+
t.Run(tt.name, func(t *testing.T) {
808+
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
809+
if tt.statusCode != 0 {
810+
w.WriteHeader(tt.statusCode)
811+
}
812+
_, _ = w.Write([]byte(tt.response))
813+
}))
814+
t.Cleanup(srv.Close)
815+
816+
var out, errOut strings.Builder
817+
cmd := newRunCommand()
818+
cmd.SetContext(cmdctx.SetWorkspaceClient(t.Context(), newTestWorkspaceClient(t, srv.URL)))
819+
cmd.SetOut(&out)
820+
cmd.SetErr(&errOut)
821+
cmd.SetArgs([]string{"-h", "config.compute"})
822+
require.NoError(t, cmd.Execute())
823+
824+
options, found := strings.CutPrefix(out.String(), "config.compute\n Which accelerators to run on and how many.\n\n Supported accelerator types:\n")
825+
require.True(t, found)
826+
options, _, found = strings.Cut(options, "\n\n Fields:")
827+
require.True(t, found)
828+
assert.Equal(t, tt.wantOptions, options)
829+
if tt.wantWarning {
830+
assert.Contains(t, errOut.String(), "showing the built-in list")
831+
} else {
832+
assert.Empty(t, errOut.String())
833+
}
834+
})
835+
}
836+
}
837+
759838
// Guards against adding a schema field without a help: tag.
760839
func TestConfigFieldsAllDocumented(t *testing.T) {
761840
root, err := resolveConfigField("config")

‎cmd/air/validateconfig.go‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -246,8 +246,7 @@ func putOpt[T any](m map[string]any, key string, value *T) {
246246
}
247247
}
248248

249-
// endpointUnavailable reports that the validation endpoint could not answer
250-
// because it is disabled or absent.
249+
// endpointUnavailable reports that an API endpoint is disabled or absent.
251250
func endpointUnavailable(err error) bool {
252251
apiErr, ok := errors.AsType[*apierr.APIError](err)
253252
return ok && (apiErr.ErrorCode == "FEATURE_DISABLED" ||

0 commit comments

Comments
 (0)