diff --git a/.nextchanges/bundles/bundle-generate-all-resources.md b/.nextchanges/bundles/bundle-generate-all-resources.md new file mode 100644 index 00000000000..a5a0bccdedb --- /dev/null +++ b/.nextchanges/bundles/bundle-generate-all-resources.md @@ -0,0 +1 @@ +* Add `databricks bundle generate ` subcommands for every bundle resource type. ([#7365](https://github.com/databricks/cli/pull/7365)) diff --git a/acceptance/bundle/generate/catalog/databricks.yml b/acceptance/bundle/generate/catalog/databricks.yml new file mode 100644 index 00000000000..3d13d105288 --- /dev/null +++ b/acceptance/bundle/generate/catalog/databricks.yml @@ -0,0 +1,5 @@ +bundle: + name: generate-catalog + +include: + - resources/*.yml diff --git a/acceptance/bundle/generate/catalog/out.test.toml b/acceptance/bundle/generate/catalog/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/generate/catalog/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/generate/catalog/output.txt b/acceptance/bundle/generate/catalog/output.txt new file mode 100644 index 00000000000..82bfd7d1005 --- /dev/null +++ b/acceptance/bundle/generate/catalog/output.txt @@ -0,0 +1,19 @@ + +>>> [CLI] bundle generate catalog --existing-id my_catalog +Catalog configuration successfully saved to [TEST_TMP_DIR]/resources/my_catalog.catalog.yml + +>>> cat resources/my_catalog.catalog.yml +resources: + catalogs: + my_catalog: + comment: primary catalog + name: my_catalog + +>>> [CLI] bundle validate +Name: generate-catalog +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/generate-catalog/default + +Validation OK! diff --git a/acceptance/bundle/generate/catalog/script b/acceptance/bundle/generate/catalog/script new file mode 100644 index 00000000000..024066e01ef --- /dev/null +++ b/acceptance/bundle/generate/catalog/script @@ -0,0 +1,10 @@ +$CLI catalogs create my_catalog --comment "primary catalog" > /dev/null + +trace $CLI bundle generate catalog --existing-id my_catalog + +trace cat resources/my_catalog.catalog.yml + +# The generated configuration loads as part of the bundle. +trace $CLI bundle validate + +rm -r resources diff --git a/acceptance/bundle/generate/catalog/test.toml b/acceptance/bundle/generate/catalog/test.toml new file mode 100644 index 00000000000..601384fdf96 --- /dev/null +++ b/acceptance/bundle/generate/catalog/test.toml @@ -0,0 +1 @@ +Ignore = [".databricks"] diff --git a/acceptance/bundle/help/bundle-generate-dashboard/output.txt b/acceptance/bundle/help/bundle-generate-dashboard/output.txt index 41ed2d6c02a..7560f71d608 100644 --- a/acceptance/bundle/help/bundle-generate-dashboard/output.txt +++ b/acceptance/bundle/help/bundle-generate-dashboard/output.txt @@ -35,6 +35,9 @@ bundle files automatically, useful during active dashboard development. Usage: databricks bundle generate dashboard [flags] +Aliases: + dashboard, dashboards + Flags: -s, --dashboard-dir string directory to write the dashboard representation to (default "src") --existing-id string ID of the dashboard to generate configuration for diff --git a/acceptance/bundle/help/bundle-generate-job/output.txt b/acceptance/bundle/help/bundle-generate-job/output.txt index 092babc9af9..2518421f40e 100644 --- a/acceptance/bundle/help/bundle-generate-job/output.txt +++ b/acceptance/bundle/help/bundle-generate-job/output.txt @@ -28,6 +28,9 @@ After generation, you can deploy this job to other targets using: Usage: databricks bundle generate job [flags] +Aliases: + job, jobs + Flags: -d, --config-dir string Dir path where the output config will be stored (default "resources") --download-spark-python-files download workspace files referenced by spark_python_task and rewrite them to a relative path diff --git a/acceptance/bundle/help/bundle-generate-pipeline/output.txt b/acceptance/bundle/help/bundle-generate-pipeline/output.txt index 884cd8614e8..74940bb8446 100644 --- a/acceptance/bundle/help/bundle-generate-pipeline/output.txt +++ b/acceptance/bundle/help/bundle-generate-pipeline/output.txt @@ -27,6 +27,9 @@ like catalogs, schemas, and compute configurations per target. Usage: databricks bundle generate pipeline [flags] +Aliases: + pipeline, pipelines + Flags: -d, --config-dir string Dir path where the output config will be stored (default "resources") --existing-pipeline-id string ID of the pipeline to generate config for diff --git a/acceptance/bundle/help/bundle-generate/output.txt b/acceptance/bundle/help/bundle-generate/output.txt index 39ce9293539..1a38b85be7e 100644 --- a/acceptance/bundle/help/bundle-generate/output.txt +++ b/acceptance/bundle/help/bundle-generate/output.txt @@ -27,12 +27,47 @@ Usage: databricks bundle generate [command] Available Commands: - alert Generate configuration for an alert - app Generate bundle configuration for a Databricks app - dashboard Generate configuration for a dashboard - genie-space Generate configuration for a Genie space - job Generate bundle configuration for a job - pipeline Generate bundle configuration for a pipeline + alert Generate configuration for an alert + app Generate bundle configuration for a Databricks app + catalog Generate bundle configuration for a catalog + cluster Generate bundle configuration for a cluster + cluster-policy Generate bundle configuration for a cluster policy + dashboard Generate configuration for a dashboard + data-quality-monitor Generate bundle configuration for a data quality monitor + database-catalog Generate bundle configuration for a database catalog + database-instance Generate bundle configuration for a database instance + experiment Generate bundle configuration for an experiment + external-location Generate bundle configuration for an external location + feature Generate bundle configuration for a feature + genie-space Generate configuration for a Genie space + instance-pool Generate bundle configuration for an instance pool + job Generate bundle configuration for a job + materialization-set Generate bundle configuration for a materialization set + mcp-service Generate bundle configuration for a mcp service + model Generate bundle configuration for a model + model-provider-service Generate bundle configuration for a model provider service + model-service Generate bundle configuration for a model service + model-serving-endpoint Generate bundle configuration for a model serving endpoint + pipeline Generate bundle configuration for a pipeline + postgres-branch Generate bundle configuration for a postgres branch + postgres-catalog Generate bundle configuration for a postgres catalog + postgres-data-api Generate bundle configuration for a postgres data api + postgres-database Generate bundle configuration for a postgres database + postgres-endpoint Generate bundle configuration for a postgres endpoint + postgres-project Generate bundle configuration for a postgres project + postgres-role Generate bundle configuration for a postgres role + postgres-snapshot-schedule Generate bundle configuration for a postgres snapshot schedule + postgres-synced-table Generate bundle configuration for a postgres synced table + quality-monitor Generate bundle configuration for a quality monitor + registered-model Generate bundle configuration for a registered model + schema Generate bundle configuration for a schema + secret Generate bundle configuration for a secret + secret-scope Generate bundle configuration for a secret scope + sql-warehouse Generate bundle configuration for a sql warehouse + synced-database-table Generate bundle configuration for a synced database table + vector-search-endpoint Generate bundle configuration for a vector search endpoint + vector-search-index Generate bundle configuration for a vector search index + volume Generate bundle configuration for a volume Flags: -h, --help help for generate diff --git a/bundle/generate/resource.go b/bundle/generate/resource.go new file mode 100644 index 00000000000..2c6dd92006b --- /dev/null +++ b/bundle/generate/resource.go @@ -0,0 +1,12 @@ +package generate + +import ( + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/yamlsaver" +) + +// ConvertResourceToValue serializes a bundle resource for a generated YAML file. +// Nil fields are dropped and the remaining keys are sorted alphabetically. +func ConvertResourceToValue(resource any) (dyn.Value, error) { + return yamlsaver.ConvertToMapValue(resource, nil, nil, make(map[string]dyn.Value)) +} diff --git a/bundle/generate/resource_test.go b/bundle/generate/resource_test.go new file mode 100644 index 00000000000..47875f5b057 --- /dev/null +++ b/bundle/generate/resource_test.go @@ -0,0 +1,30 @@ +package generate + +import ( + "testing" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestConvertResourceToValueDropsEmptyFieldsAndSortsKeys(t *testing.T) { + c := resources.Catalog{ + StorageRoot: "s3://bucket", + Name: "main", + Comment: "primary", + } + + v, err := ConvertResourceToValue(&c) + require.NoError(t, err) + + m, ok := v.AsMap() + require.True(t, ok) + // The YAML encoder orders entries by line, so the lines encode alphabetical order. + // BaseResource, ID, and Grants are zero and must not appear. + lines := make(map[string]int) + for _, p := range m.Pairs() { + lines[p.Key.MustString()] = p.Value.Location().Line + } + assert.Equal(t, map[string]int{"comment": 1, "name": 2, "storage_root": 3}, lines) +} diff --git a/cmd/bundle/generate.go b/cmd/bundle/generate.go index 12452293bc6..508769e21e9 100644 --- a/cmd/bundle/generate.go +++ b/cmd/bundle/generate.go @@ -35,12 +35,7 @@ Use --key to specify the resource name in your bundle configuration. Use --bind to automatically bind the generated resource to the existing workspace resource.`, } - cmd.AddCommand(generate.NewGenerateJobCommand()) - cmd.AddCommand(generate.NewGeneratePipelineCommand()) - cmd.AddCommand(generate.NewGenerateDashboardCommand()) - cmd.AddCommand(generate.NewGenerateAlertCommand()) - cmd.AddCommand(generate.NewGenerateAppCommand()) - cmd.AddCommand(generate.NewGenerateGenieSpaceCommand()) + cmd.AddCommand(generate.NewCommands()...) cmd.PersistentFlags().StringVar(&key, "key", "", `resource key to use for the generated configuration`) return cmd } diff --git a/cmd/bundle/generate/generic.go b/cmd/bundle/generate/generic.go new file mode 100644 index 00000000000..a5739cf0dba --- /dev/null +++ b/cmd/bundle/generate/generic.go @@ -0,0 +1,268 @@ +package generate + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "maps" + "path/filepath" + "reflect" + "slices" + "strings" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/direct/dresources" + "github.com/databricks/cli/bundle/generate" + "github.com/databricks/cli/cmd/bundle/deployment" + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/yamlsaver" + "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/textutil" + "github.com/databricks/databricks-sdk-go" + "github.com/spf13/cobra" + "github.com/spf13/pflag" +) + +// idFlag is the lookup flag of the generic command. Hand-written generators that +// named theirs differently accept it as an alias (see generators). +const idFlag = "existing-id" + +// generator is a hand-written generate command. They exist because these types +// also download files (notebooks, dashboard JSON, app sources) or have a watch +// mode, which the generic command does not do yet. Every other resource type in +// dresources.SupportedResources gets the generic command. +type generator struct { + newCommand func() *cobra.Command + // idFlagName is the command's own lookup flag when it is not idFlag. + idFlagName string +} + +var generators = map[string]generator{ + "alerts": {newCommand: NewGenerateAlertCommand}, + "apps": {newCommand: NewGenerateAppCommand, idFlagName: "existing-app-name"}, + "dashboards": {newCommand: NewGenerateDashboardCommand}, + "genie_spaces": {newCommand: NewGenerateGenieSpaceCommand}, + "jobs": {newCommand: NewGenerateJobCommand, idFlagName: "existing-job-id"}, + "pipelines": {newCommand: NewGeneratePipelineCommand, idFlagName: "existing-pipeline-id"}, +} + +// notGenerated lists the resource groups without a generate command: job runs are +// triggered rather than imported, and snapshots are internal. +var notGenerated = map[string]bool{ + "job_runs": true, + "internal_immutable_snapshots": true, +} + +// NewCommands returns a generate subcommand for every bundle resource type. Each +// command also answers to its resource group ("catalogs"), so callers that hold a +// key from deployment state can address it without a singular/plural mapping. +func NewCommands() []*cobra.Command { + descriptions := config.SupportedResources() + var cmds []*cobra.Command + for _, group := range slices.Sorted(maps.Keys(dresources.SupportedResources)) { + // Permissions and grants ("jobs.permissions") are not standalone resources. + if notGenerated[group] || strings.Contains(group, ".") { + continue + } + var cmd *cobra.Command + if g, ok := generators[group]; ok { + cmd = g.newCommand() + if g.idFlagName != "" { + aliasFlag(cmd, idFlag, g.idFlagName) + } + } else { + cmd = newResourceCommand(group, descriptions[group]) + } + cmd.Aliases = append(cmd.Aliases, group) + cmds = append(cmds, cmd) + } + return cmds +} + +// aliasFlag makes --alias parse as --name on cmd. pflag resolves names through the +// normalize function, so the flag keeps its own name in help and in the required +// flag check. +func aliasFlag(cmd *cobra.Command, alias, name string) { + cmd.Flags().SetNormalizeFunc(func(f *pflag.FlagSet, n string) pflag.NormalizedName { + if n == alias { + return pflag.NormalizedName(name) + } + return pflag.NormalizedName(n) + }) +} + +// resourceOptions carries the flag values of a generic generate subcommand. +type resourceOptions struct { + id string + key string + configDir string + force bool +} + +func newResourceCommand(group string, desc resources.ResourceDescription) *cobra.Command { + var opts resourceOptions + var bind bool + + use := strings.ReplaceAll(desc.SingularName, "_", "-") + title := strings.ToLower(desc.SingularTitle) + cmd := &cobra.Command{ + Use: use, + Short: "Generate bundle configuration for " + article(title) + " " + title, + Long: fmt.Sprintf(`Generate bundle configuration for an existing %s. + +The %s is read from the workspace and written as a bundle resource to +/.%s.yml so that you can deploy it to other targets or manage +it as code. --existing-id takes the identifier that "databricks bundle deployment +bind" uses for this resource type. + +Example: + databricks bundle generate %s --existing-id ID --key my_%s`, + title, title, desc.SingularName, use, desc.SingularName), + } + + cmd.Flags().StringVar(&opts.id, idFlag, "", "ID of the "+title+" to generate configuration for") + cmd.MarkFlagRequired(idFlag) + cmd.Flags().StringVarP(&opts.configDir, "config-dir", "d", "resources", `directory to write the configuration to`) + cmd.Flags().BoolVarP(&opts.force, "force", "f", false, `force overwrite existing files in the output directory`) + cmd.Flags().BoolVarP(&bind, "bind", "b", false, `automatically bind the generated resource to the existing resource`) + cmd.Flags().MarkHidden("bind") + + cmd.RunE = func(cmd *cobra.Command, args []string) error { + ctx := logdiag.InitContext(cmd.Context()) + cmd.SetContext(ctx) + + b := root.MustConfigureBundle(cmd) + if b == nil || logdiag.HasError(ctx) { + return root.ErrAlreadyPrinted + } + + opts.key = cmd.Flag("key").Value.String() + key, err := generateResource(ctx, b, group, desc, opts) + if err != nil { + return err + } + + if bind { + return deployment.BindResource(cmd, key, opts.id, true, false, true) + } + return nil + } + + return cmd +} + +// generateResource reads the resource, writes it as /..yml under +// the bundle root, and returns the resource key it used. +func generateResource(ctx context.Context, b *bundle.Bundle, group string, desc resources.ResourceDescription, opts resourceOptions) (string, error) { + resource, err := readResource(ctx, b.WorkspaceClient(ctx), group, opts.id) + if err != nil { + return "", err + } + + key := opts.key + if key == "" { + key = textutil.NormalizeString(resource.GetName()) + } + if key == "" { + return "", fmt.Errorf("cannot derive a resource key for %s %q, pass --key", desc.SingularName, opts.id) + } + + v, err := generate.ConvertResourceToValue(resource) + if err != nil { + return "", err + } + + configDir := opts.configDir + if !filepath.IsAbs(configDir) { + configDir = filepath.Join(b.BundleRootPath, configDir) + } + filename := filepath.Join(configDir, key+"."+desc.SingularName+".yml") + + result := map[string]dyn.Value{ + "resources": dyn.V(map[string]dyn.Value{ + group: dyn.V(map[string]dyn.Value{ + key: v, + }), + }), + } + err = yamlsaver.NewSaver().SaveAsYAML(result, filename, opts.force) + if err != nil { + return "", err + } + + cmdio.LogString(ctx, desc.SingularTitle+" configuration successfully saved to "+filepath.ToSlash(filename)) + warnIfNotIncluded(ctx, b, filename) + return key, nil +} + +// readResource reads a resource through its deployment-engine adapter and maps the +// engine state onto the bundle resource type via JSON. The mapping is only trusted +// when feeding the result back through PrepareState reproduces the state; a type +// whose state is shaped differently from its configuration (secret scopes, for +// example) is rejected instead of being written as YAML that would not deploy. +func readResource(ctx context.Context, w *databricks.WorkspaceClient, group, id string) (config.ConfigResource, error) { + adapter, err := dresources.NewAdapter(dresources.SupportedResources[group], group, w) + if err != nil { + return nil, err + } + remote, err := adapter.DoRead(ctx, id) + if err != nil { + return nil, err + } + state, err := adapter.RemapState(remote) + if err != nil { + return nil, err + } + + data, err := json.Marshal(state) + if err != nil { + return nil, err + } + resource, err := newConfigResource(group) + if err != nil { + return nil, err + } + err = json.Unmarshal(data, resource) + if err != nil { + return nil, err + } + + again, err := adapter.PrepareState(resource) + if err != nil { + return nil, err + } + check, err := json.Marshal(again) + if err != nil { + return nil, err + } + if !bytes.Equal(data, check) { + return nil, fmt.Errorf("generating %s is not supported yet: the deployment engine state does not map onto bundle configuration", group) + } + return resource, nil +} + +// newConfigResource returns a zero bundle resource of the type config.Resources +// declares for group. +func newConfigResource(group string) (config.ConfigResource, error) { + for field := range reflect.TypeFor[config.Resources]().Fields() { + name, _, _ := strings.Cut(field.Tag.Get("json"), ",") + if name == group { + return reflect.New(field.Type.Elem().Elem()).Interface().(config.ConfigResource), nil + } + } + return nil, fmt.Errorf("internal error: no bundle resource type for %q", group) +} + +// article picks the indefinite article for a lowercase title such as "experiment". +func article(title string) string { + switch title[0] { + case 'a', 'e', 'i', 'o', 'u': + return "an" + } + return "a" +} diff --git a/cmd/bundle/generate/generic_test.go b/cmd/bundle/generate/generic_test.go new file mode 100644 index 00000000000..6bdbb66b2f1 --- /dev/null +++ b/cmd/bundle/generate/generic_test.go @@ -0,0 +1,196 @@ +package generate + +import ( + "context" + "errors" + "io" + "os" + "path/filepath" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/databricks-sdk-go/experimental/mocks" + "github.com/databricks/databricks-sdk-go/service/catalog" + "github.com/databricks/databricks-sdk-go/service/ml" + "github.com/databricks/databricks-sdk-go/service/workspace" + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +func testBundle(t *testing.T) (context.Context, *bundle.Bundle, *mocks.MockWorkspaceClient) { + t.Helper() + m := mocks.NewMockWorkspaceClient(t) + b := &bundle.Bundle{BundleRootPath: t.TempDir()} + b.SetWorkpaceClient(m.WorkspaceClient) + ctx := logdiag.InitContext(cmdio.MockDiscard(t.Context())) + logdiag.SetCollect(ctx, true) + return ctx, b, m +} + +func generateCatalog(t *testing.T, opts resourceOptions) (context.Context, *bundle.Bundle, string, error) { + t.Helper() + ctx, b, m := testBundle(t) + m.GetMockCatalogsAPI().EXPECT().GetByName(mock.Anything, opts.id).Return(&catalog.CatalogInfo{ + Name: opts.id, + Comment: "primary", + Owner: "someone@example.com", + CatalogType: catalog.CatalogTypeManagedCatalog, + StorageRoot: "s3://bucket", + }, nil).Maybe() + key, err := generateResource(ctx, b, "catalogs", config.SupportedResources()["catalogs"], opts) + return ctx, b, key, err +} + +func TestGenerateResourceWritesConfig(t *testing.T) { + ctx, b, key, err := generateCatalog(t, resourceOptions{id: "My Catalog", configDir: "resources"}) + require.NoError(t, err) + assert.Equal(t, "my_catalog", key) + + data, err := os.ReadFile(filepath.Join(b.BundleRootPath, "resources", "my_catalog.catalog.yml")) + require.NoError(t, err) + // Owner and catalog type are not part of the create request and are dropped. + assert.Equal(t, `resources: + catalogs: + my_catalog: + comment: primary + name: My Catalog + storage_root: s3://bucket +`, string(data)) + + // The bundle has no include patterns, so the file is reported as not included. + diags := logdiag.FlushCollected(ctx) + require.Len(t, diags, 1) + assert.Equal(t, "Generated configuration is not included in the bundle", diags[0].Summary) +} + +func TestGenerateResourceHonorsKeyAndConfigDir(t *testing.T) { + _, b, key, err := generateCatalog(t, resourceOptions{id: "main", key: "k", configDir: "conf"}) + require.NoError(t, err) + assert.Equal(t, "k", key) + assert.FileExists(t, filepath.Join(b.BundleRootPath, "conf", "k.catalog.yml")) +} + +func TestGenerateResourceRefusesToOverwriteWithoutForce(t *testing.T) { + ctx, b, m := testBundle(t) + m.GetMockCatalogsAPI().EXPECT().GetByName(mock.Anything, "main").Return(&catalog.CatalogInfo{Name: "main"}, nil) + existing := filepath.Join(b.BundleRootPath, "resources", "main.catalog.yml") + require.NoError(t, os.MkdirAll(filepath.Dir(existing), 0o755)) + require.NoError(t, os.WriteFile(existing, []byte("keep"), 0o644)) + desc := config.SupportedResources()["catalogs"] + + opts := resourceOptions{id: "main", configDir: "resources"} + _, err := generateResource(ctx, b, "catalogs", desc, opts) + assert.ErrorContains(t, err, "already exists") + + opts.force = true + _, err = generateResource(ctx, b, "catalogs", desc, opts) + require.NoError(t, err) + data, err := os.ReadFile(existing) + require.NoError(t, err) + assert.Contains(t, string(data), "name: main") +} + +func TestGenerateResourceReturnsReadError(t *testing.T) { + ctx, b, m := testBundle(t) + m.GetMockCatalogsAPI().EXPECT().GetByName(mock.Anything, "missing").Return(nil, errors.New("catalog not found")) + + _, err := generateResource(ctx, b, "catalogs", config.SupportedResources()["catalogs"], resourceOptions{id: "missing", configDir: "resources"}) + assert.EqualError(t, err, "catalog not found") +} + +func TestGenerateResourceUnwrapsExperimentResponse(t *testing.T) { + ctx, b, m := testBundle(t) + m.GetMockExperimentsAPI().EXPECT().GetExperiment(mock.Anything, ml.GetExperimentRequest{ExperimentId: "123"}).Return(&ml.GetExperimentResponse{ + Experiment: &ml.Experiment{ + ExperimentId: "123", + Name: "/Users/someone/My Experiment", + ArtifactLocation: "dbfs:/artifacts", + LifecycleStage: "active", + }, + }, nil) + + key, err := generateResource(ctx, b, "experiments", config.SupportedResources()["experiments"], resourceOptions{id: "123", configDir: "resources"}) + require.NoError(t, err) + assert.Equal(t, "users_someone_my_experiment", key) + + data, err := os.ReadFile(filepath.Join(b.BundleRootPath, "resources", "users_someone_my_experiment.experiment.yml")) + require.NoError(t, err) + assert.Equal(t, `resources: + experiments: + users_someone_my_experiment: + artifact_location: dbfs:/artifacts + name: /Users/someone/My Experiment +`, string(data)) +} + +func TestGenerateResourceRejectsStateThatDoesNotMapOntoConfig(t *testing.T) { + ctx, b, m := testBundle(t) + // Secret scope state uses "scope" where the bundle resource uses "name". + m.GetMockSecretsAPI().EXPECT().ListScopesAll(mock.Anything).Return([]workspace.SecretScope{ + {Name: "my_scope", BackendType: workspace.ScopeBackendTypeDatabricks}, + }, nil) + + _, err := generateResource(ctx, b, "secret_scopes", config.SupportedResources()["secret_scopes"], resourceOptions{id: "my_scope", configDir: "resources"}) + assert.ErrorContains(t, err, "generating secret_scopes is not supported yet") + assert.NoFileExists(t, filepath.Join(b.BundleRootPath, "resources", "my_scope.secret_scope.yml")) +} + +func TestNewCommandsCoversEveryResourceType(t *testing.T) { + names := make(map[string]*cobra.Command) + for _, cmd := range NewCommands() { + names[cmd.Name()] = cmd + } + + // Hand-written generators keep their own flag and accept --existing-id as an alias. + assert.Contains(t, names["job"].Flags().FlagUsages(), "existing-job-id") + assert.NotContains(t, names["job"].Flags().FlagUsages(), "--existing-id") + assert.Equal(t, "existing-job-id", names["job"].Flags().Lookup("existing-id").Name) + assert.Equal(t, "existing-app-name", names["app"].Flags().Lookup("existing-id").Name) + // Generic ones take --existing-id. + assert.Contains(t, names["catalog"].Flags().FlagUsages(), "existing-id") + assert.Contains(t, names, "external-location") + + // Every command answers to its resource group as well. + assert.Equal(t, []string{"catalogs"}, names["catalog"].Aliases) + assert.Equal(t, []string{"jobs"}, names["job"].Aliases) + assert.True(t, names["external-location"].HasAlias("external_locations")) + + assert.NotContains(t, names, "job-run") + for name := range names { + assert.NotContains(t, name, ".") + } +} + +func TestAliasFlagParsesAliasAsTarget(t *testing.T) { + cmd := NewGenerateJobCommand() + aliasFlag(cmd, "existing-id", "existing-job-id") + require.NoError(t, cmd.Flags().Parse([]string{"--existing-id", "123"})) + + id, err := cmd.Flags().GetInt64("existing-job-id") + require.NoError(t, err) + assert.Equal(t, int64(123), id) +} + +func TestResourceCommandFlagsAndHelp(t *testing.T) { + cmd := newResourceCommand("external_locations", config.SupportedResources()["external_locations"]) + assert.Equal(t, "external-location", cmd.Use) + assert.Equal(t, "Generate bundle configuration for an external location", cmd.Short) + assert.Contains(t, cmd.Long, "generate external-location --existing-id ID --key my_external_location") + assert.True(t, cmd.Flags().Lookup("bind").Hidden) + + cmd.SetOut(io.Discard) + cmd.SetErr(io.Discard) + cmd.SetArgs(nil) + err := cmd.Execute() + assert.ErrorContains(t, err, `required flag(s) "existing-id" not set`) +} + +func TestArticle(t *testing.T) { + assert.Equal(t, "an", article("experiment")) + assert.Equal(t, "a", article("catalog")) +} diff --git a/cmd/bundle/generate_test.go b/cmd/bundle/generate_test.go new file mode 100644 index 00000000000..911e2c884d6 --- /dev/null +++ b/cmd/bundle/generate_test.go @@ -0,0 +1,47 @@ +package bundle + +import ( + "slices" + "strings" + "testing" + + "github.com/databricks/cli/bundle/config" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// generateUnsupported lists the resource groups that deliberately have no generate +// subcommand. A group that is in neither this list nor the command tree fails the +// test, so every new resource type gets an explicit decision. +var generateUnsupported = []string{ + "internal_immutable_snapshots", + "job_runs", +} + +func TestGenerateCommandCoversAllResourceGroups(t *testing.T) { + groups := config.SupportedResources() + + // Subcommands are named after the resource type's singular name with dashes. + byCommand := make(map[string]string, len(groups)) + for group, d := range groups { + byCommand[strings.ReplaceAll(d.SingularName, "_", "-")] = group + } + + covered := make(map[string]bool) + for _, sub := range newGenerateCommand().Commands() { + group, ok := byCommand[sub.Name()] + require.True(t, ok, "generate subcommand %q does not match the singular name of any resource type", sub.Name()) + covered[group] = true + } + + for _, group := range generateUnsupported { + assert.Contains(t, groups, group, "%q is not a resource group; drop it from generateUnsupported", group) + assert.False(t, covered[group], "%q has a generate subcommand; drop it from generateUnsupported", group) + } + for group := range groups { + if slices.Contains(generateUnsupported, group) { + continue + } + assert.True(t, covered[group], "%q has no generate subcommand; add one or list it in generateUnsupported", group) + } +} diff --git a/libs/dyn/yamlsaver/utils.go b/libs/dyn/yamlsaver/utils.go index 61f41790029..b4b846cc63a 100644 --- a/libs/dyn/yamlsaver/utils.go +++ b/libs/dyn/yamlsaver/utils.go @@ -65,6 +65,10 @@ func sortMapAlphabetically(mv dyn.Value) (dyn.Value, error) { } func skipAndOrder(mv dyn.Value, order *Order, skipFields []string, dst map[string]dyn.Value) (dyn.Value, error) { + // The YAML encoder orders map entries by Location.Line. Without an explicit + // order, number the entries so the alphabetical order from + // sortMapAlphabetically survives instead of Go map iteration order. + line := 0 for _, pair := range mv.MustMap().Pairs() { k := pair.Key.MustString() v := pair.Value @@ -84,7 +88,8 @@ func skipAndOrder(mv dyn.Value, order *Order, skipFields []string, dst map[strin } if order == nil { - dst[k] = v + line++ + dst[k] = dyn.NewValue(v.Value(), []diag.Location{{Line: line}}) } else { dst[k] = dyn.NewValue(v.Value(), []diag.Location{{Line: order.Get(k)}}) } diff --git a/libs/dyn/yamlsaver/utils_test.go b/libs/dyn/yamlsaver/utils_test.go index 8b056e0076f..9188edc69de 100644 --- a/libs/dyn/yamlsaver/utils_test.go +++ b/libs/dyn/yamlsaver/utils_test.go @@ -34,23 +34,23 @@ func TestConvertToMap(t *testing.T) { result, err := ConvertToMapValue(v, nil, []string{"format"}, map[string]dyn.Value{}) assert.NoError(t, err) dynassert.Equal(t, dyn.V(map[string]dyn.Value{ - "list": dyn.V( + "list": dyn.NewValue( []dyn.Value{ dyn.V("a"), dyn.V("b"), dyn.V("c"), }, + []diag.Location{{Line: 1}}, ), - "long_name_field": dyn.V("long name goes here"), - "map": dyn.V( + "long_name_field": dyn.NewValue("long name goes here", []diag.Location{{Line: 2}}), + "map": dyn.NewValue( map[string]dyn.Value{ "key1": dyn.V("value1"), "key2": dyn.V("value2"), }, + []diag.Location{{Line: 3}}, ), - "name": dyn.V( - "test", - ), + "name": dyn.NewValue("test", []diag.Location{{Line: 4}}), }), result) }