From 793bf2b99514cdb4d37d0b37d2c0b2a4406ef6ae Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Mon, 23 Dec 2024 13:08:01 +0100 Subject: [PATCH 01/19] fix: Empty schema fields in OpenAPI spec (#2045) ## Changes 1. Removes default yaml-fields during schema generation, caused by [this PR](https://github.com/databricks/cli/pull/2032) (current yaml package can't read `json` annotations in struct fields) 2. Addresses missing annotations for fields from OpenAPI spec, which are named differently in go SDK 3. Adds filtering for annotations.yaml to include only CLI package fields 4. Implements alphabetical sort for yaml keys to avoid unnecessary diff in PRs ## Tests Manually tested --- bundle/internal/schema/annotations.go | 62 +- bundle/internal/schema/annotations.yml | 65 +- .../internal/schema/annotations_openapi.yml | 3657 +++++++++-------- bundle/internal/schema/main_test.go | 3 +- bundle/internal/schema/parser.go | 60 +- bundle/schema/jsonschema.json | 19 + 6 files changed, 2097 insertions(+), 1769 deletions(-) diff --git a/bundle/internal/schema/annotations.go b/bundle/internal/schema/annotations.go index aec5e68b0..91aaa4555 100644 --- a/bundle/internal/schema/annotations.go +++ b/bundle/internal/schema/annotations.go @@ -6,6 +6,7 @@ import ( "os" "reflect" "regexp" + "slices" "strings" yaml3 "gopkg.in/yaml.v3" @@ -119,7 +120,15 @@ func (d *annotationHandler) syncWithMissingAnnotations(outputPath string) error if err != nil { return err } - missingAnnotations, err := convert.FromTyped(&d.missingAnnotations, dyn.NilValue) + + for k := range d.missingAnnotations { + if !isCliPath(k) { + delete(d.missingAnnotations, k) + fmt.Printf("Missing annotations for `%s` that are not in CLI package, try to fetch latest OpenAPI spec and regenerate annotations", k) + } + } + + missingAnnotations, err := convert.FromTyped(d.missingAnnotations, dyn.NilValue) if err != nil { return err } @@ -129,7 +138,13 @@ func (d *annotationHandler) syncWithMissingAnnotations(outputPath string) error return err } - err = saveYamlWithStyle(outputPath, output) + var outputTyped annotationFile + err = convert.ToTyped(&outputTyped, output) + if err != nil { + return err + } + + err = saveYamlWithStyle(outputPath, outputTyped) if err != nil { return err } @@ -153,21 +168,50 @@ func assignAnnotation(s *jsonschema.Schema, a annotation) { s.Enum = a.Enum } -func saveYamlWithStyle(outputPath string, input dyn.Value) error { +func saveYamlWithStyle(outputPath string, annotations annotationFile) error { + annotationOrder := yamlsaver.NewOrder([]string{"description", "markdown_description", "title", "default", "enum"}) style := map[string]yaml3.Style{} - file, _ := input.AsMap() - for _, v := range file.Keys() { - style[v.MustString()] = yaml3.LiteralStyle + + order := getAlphabeticalOrder(annotations) + dynMap := map[string]dyn.Value{} + for k, v := range annotations { + style[k] = yaml3.LiteralStyle + + properties := map[string]dyn.Value{} + propertiesOrder := getAlphabeticalOrder(v) + for key, value := range v { + d, err := convert.FromTyped(value, dyn.NilValue) + if d.Kind() == dyn.KindNil || err != nil { + properties[key] = dyn.NewValue(map[string]dyn.Value{}, []dyn.Location{{Line: propertiesOrder.Get(key)}}) + continue + } + val, err := yamlsaver.ConvertToMapValue(value, annotationOrder, []string{}, map[string]dyn.Value{}) + if err != nil { + return err + } + properties[key] = val.WithLocations([]dyn.Location{{Line: propertiesOrder.Get(key)}}) + } + + dynMap[k] = dyn.NewValue(properties, []dyn.Location{{Line: order.Get(k)}}) } saver := yamlsaver.NewSaverWithStyle(style) - err := saver.SaveAsYAML(file, outputPath, true) + err := saver.SaveAsYAML(dynMap, outputPath, true) if err != nil { return err } return nil } +func getAlphabeticalOrder[T any](mapping map[string]T) *yamlsaver.Order { + order := []string{} + for k := range mapping { + order = append(order, k) + } + slices.Sort(order) + return yamlsaver.NewOrder(order) +} + func convertLinksToAbsoluteUrl(s string) string { if s == "" { return s @@ -207,3 +251,7 @@ func convertLinksToAbsoluteUrl(s string) string { return result } + +func isCliPath(path string) bool { + return !strings.HasPrefix(path, "github.com/databricks/databricks-sdk-go") +} diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index e52189daa..84f6753e3 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -417,10 +417,10 @@ github.com/databricks/cli/bundle/config/variable.TargetVariable: "lookup": "description": |- The name of the alert, cluster_policy, cluster, dashboard, instance_pool, job, metastore, pipeline, query, service_principal, or warehouse object for which to retrieve an ID. - "type": + "markdown_description": "description": |- The type of the variable. - "markdown_description": + "type": "description": |- The type of the variable. github.com/databricks/cli/bundle/config/variable.Variable: @@ -438,64 +438,3 @@ github.com/databricks/cli/bundle/config/variable.Variable: "type": "description": |- The type of the variable. -github.com/databricks/databricks-sdk-go/service/serving.Ai21LabsConfig: - "ai21labs_api_key": - "description": |- - PLACEHOLDER - "ai21labs_api_key_plaintext": - "description": |- - PLACEHOLDER -github.com/databricks/databricks-sdk-go/service/serving.GoogleCloudVertexAiConfig: - "private_key": - "description": |- - PLACEHOLDER - "private_key_plaintext": - "description": |- - PLACEHOLDER - "project_id": - "description": |- - PLACEHOLDER - "region": - "description": |- - PLACEHOLDER -github.com/databricks/databricks-sdk-go/service/serving.OpenAiConfig: - "microsoft_entra_client_id": - "description": |- - PLACEHOLDER - "microsoft_entra_client_secret": - "description": |- - PLACEHOLDER - "microsoft_entra_client_secret_plaintext": - "description": |- - PLACEHOLDER - "microsoft_entra_tenant_id": - "description": |- - PLACEHOLDER - "openai_api_base": - "description": |- - PLACEHOLDER - "openai_api_key": - "description": |- - PLACEHOLDER - "openai_api_key_plaintext": - "description": |- - PLACEHOLDER - "openai_api_type": - "description": |- - PLACEHOLDER - "openai_api_version": - "description": |- - PLACEHOLDER - "openai_deployment_name": - "description": |- - PLACEHOLDER - "openai_organization": - "description": |- - PLACEHOLDER -github.com/databricks/databricks-sdk-go/service/serving.PaLmConfig: - "palm_api_key": - "description": |- - PLACEHOLDER - "palm_api_key_plaintext": - "description": |- - PLACEHOLDER diff --git a/bundle/internal/schema/annotations_openapi.yml b/bundle/internal/schema/annotations_openapi.yml index d8f681a28..e9c893c87 100644 --- a/bundle/internal/schema/annotations_openapi.yml +++ b/bundle/internal/schema/annotations_openapi.yml @@ -1,89 +1,89 @@ # This file is auto-generated. DO NOT EDIT. github.com/databricks/cli/bundle/config/resources.Cluster: - apply_policy_default_values: - description: When set to true, fixed and default values from the policy will be - used for fields that are omitted. When set to false, only fixed values from - the policy will be applied. - autoscale: - description: |- + "apply_policy_default_values": + "description": |- + When set to true, fixed and default values from the policy will be used for fields that are omitted. When set to false, only fixed values from the policy will be applied. + "autoscale": + "description": |- Parameters needed in order to automatically scale clusters up and down based on load. Note: autoscaling works best with DB runtime versions 3.0 or later. - autotermination_minutes: - description: |- + "autotermination_minutes": + "description": |- Automatically terminates the cluster after it is inactive for this time in minutes. If not set, this cluster will not be automatically terminated. If specified, the threshold must be between 10 and 10000 minutes. Users can also set this value to 0 to explicitly disable automatic termination. - aws_attributes: - description: |- + "aws_attributes": + "description": |- Attributes related to clusters running on Amazon Web Services. If not specified at cluster creation, a set of default values will be used. - azure_attributes: - description: |- + "azure_attributes": + "description": |- Attributes related to clusters running on Microsoft Azure. If not specified at cluster creation, a set of default values will be used. - cluster_log_conf: - description: |- + "cluster_log_conf": + "description": |- The configuration for delivering spark logs to a long-term storage destination. Two kinds of destinations (dbfs and s3) are supported. Only one destination can be specified for one cluster. If the conf is given, the logs will be delivered to the destination every `5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while the destination of executor logs is `$destination/$clusterId/executor`. - cluster_name: - description: | + "cluster_name": + "description": | Cluster name requested by the user. This doesn't have to be unique. If not specified at creation, the cluster name will be an empty string. - custom_tags: - description: |- + "custom_tags": + "description": |- Additional tags for cluster resources. Databricks will tag all cluster resources (e.g., AWS instances and EBS volumes) with these tags in addition to `default_tags`. Notes: - Currently, Databricks allows at most 45 custom tags - Clusters can only reuse cloud resources if the resources' tags are a subset of the cluster tags - data_security_mode: {} - docker_image: {} - driver_instance_pool_id: - description: |- + "data_security_mode": {} + "docker_image": {} + "driver_instance_pool_id": + "description": |- The optional ID of the instance pool for the driver of the cluster belongs. The pool cluster uses the instance pool with id (instance_pool_id) if the driver pool is not assigned. - driver_node_type_id: - description: | + "driver_node_type_id": + "description": | The node type of the Spark driver. Note that this field is optional; if unset, the driver node type will be set as the same value as `node_type_id` defined above. - enable_elastic_disk: - description: |- + "enable_elastic_disk": + "description": |- Autoscaling Local Storage: when enabled, this cluster will dynamically acquire additional disk space when its Spark workers are running low on disk space. This feature requires specific AWS permissions to function correctly - refer to the User Guide for more details. - enable_local_disk_encryption: - description: Whether to enable LUKS on cluster VMs' local disks - gcp_attributes: - description: |- + "enable_local_disk_encryption": + "description": |- + Whether to enable LUKS on cluster VMs' local disks + "gcp_attributes": + "description": |- Attributes related to clusters running on Google Cloud Platform. If not specified at cluster creation, a set of default values will be used. - init_scripts: - description: The configuration for storing init scripts. Any number of destinations - can be specified. The scripts are executed sequentially in the order provided. - If `cluster_log_conf` is specified, init script logs are sent to `//init_scripts`. - instance_pool_id: - description: The optional ID of the instance pool to which the cluster belongs. - is_single_node: - description: | + "init_scripts": + "description": |- + The configuration for storing init scripts. Any number of destinations can be specified. The scripts are executed sequentially in the order provided. If `cluster_log_conf` is specified, init script logs are sent to `//init_scripts`. + "instance_pool_id": + "description": |- + The optional ID of the instance pool to which the cluster belongs. + "is_single_node": + "description": | This field can only be used with `kind`. When set to true, Databricks will automatically set single node related `custom_tags`, `spark_conf`, and `num_workers` - kind: {} - node_type_id: - description: | + "kind": {} + "node_type_id": + "description": | This field encodes, through a single value, the resources available to each of the Spark nodes in this cluster. For example, the Spark nodes can be provisioned and optimized for memory or compute intensive workloads. A list of available node types can be retrieved by using the :method:clusters/listNodeTypes API call. - num_workers: - description: |- + "num_workers": + "description": |- Number of worker nodes that this cluster should have. A cluster has one Spark Driver and `num_workers` Executors for a total of `num_workers` + 1 Spark nodes. @@ -92,18 +92,20 @@ github.com/databricks/cli/bundle/config/resources.Cluster: is resized from 5 to 10 workers, this field will immediately be updated to reflect the target size of 10 workers, whereas the workers listed in `spark_info` will gradually increase from 5 to 10 as the new nodes are provisioned. - policy_id: - description: The ID of the cluster policy used to create the cluster if applicable. - runtime_engine: {} - single_user_name: - description: Single user name if data_security_mode is `SINGLE_USER` - spark_conf: - description: | + "policy_id": + "description": |- + The ID of the cluster policy used to create the cluster if applicable. + "runtime_engine": {} + "single_user_name": + "description": |- + Single user name if data_security_mode is `SINGLE_USER` + "spark_conf": + "description": | An object containing a set of optional, user-specified Spark configuration key-value pairs. Users can also pass in a string of extra JVM options to the driver and the executors via `spark.driver.extraJavaOptions` and `spark.executor.extraJavaOptions` respectively. - spark_env_vars: - description: |- + "spark_env_vars": + "description": |- An object containing a set of optional, user-specified environment variable key-value pairs. Please note that key-value pair of the form (X,Y) will be exported as is (i.e., `export X='Y'`) while launching the driver and workers. @@ -115,399 +117,460 @@ github.com/databricks/cli/bundle/config/resources.Cluster: Example Spark environment variables: `{"SPARK_WORKER_MEMORY": "28000m", "SPARK_LOCAL_DIRS": "/local_disk0"}` or `{"SPARK_DAEMON_JAVA_OPTS": "$SPARK_DAEMON_JAVA_OPTS -Dspark.shuffle.service.enabled=true"}` - spark_version: - description: | + "spark_version": + "description": | The Spark version of the cluster, e.g. `3.3.x-scala2.11`. A list of available Spark versions can be retrieved by using the :method:clusters/sparkVersions API call. - ssh_public_keys: - description: |- + "ssh_public_keys": + "description": |- SSH public key contents that will be added to each Spark node in this cluster. The corresponding private keys can be used to login with the user name `ubuntu` on port `2200`. Up to 10 keys can be specified. - use_ml_runtime: - description: | + "use_ml_runtime": + "description": | This field can only be used with `kind`. `effective_spark_version` is determined by `spark_version` (DBR release), this field `use_ml_runtime`, and whether `node_type_id` is gpu node or not. - workload_type: {} + "workload_type": {} github.com/databricks/cli/bundle/config/resources.Dashboard: - create_time: - description: The timestamp of when the dashboard was created. - dashboard_id: - description: UUID identifying the dashboard. - display_name: - description: The display name of the dashboard. - etag: - description: |- + "create_time": + "description": |- + The timestamp of when the dashboard was created. + "dashboard_id": + "description": |- + UUID identifying the dashboard. + "display_name": + "description": |- + The display name of the dashboard. + "etag": + "description": |- The etag for the dashboard. Can be optionally provided on updates to ensure that the dashboard has not been modified since the last read. This field is excluded in List Dashboards responses. - lifecycle_state: - description: The state of the dashboard resource. Used for tracking trashed status. - parent_path: - description: |- + "lifecycle_state": + "description": |- + The state of the dashboard resource. Used for tracking trashed status. + "parent_path": + "description": |- The workspace path of the folder containing the dashboard. Includes leading slash and no trailing slash. This field is excluded in List Dashboards responses. - path: - description: |- + "path": + "description": |- The workspace path of the dashboard asset, including the file name. Exported dashboards always have the file extension `.lvdash.json`. This field is excluded in List Dashboards responses. - serialized_dashboard: - description: |- + "serialized_dashboard": + "description": |- The contents of the dashboard in serialized string form. This field is excluded in List Dashboards responses. Use the [get dashboard API](https://docs.databricks.com/api/workspace/lakeview/get) to retrieve an example response, which includes the `serialized_dashboard` field. This field provides the structure of the JSON string that represents the dashboard's layout and components. - update_time: - description: |- + "update_time": + "description": |- The timestamp of when the dashboard was last updated by the user. This field is excluded in List Dashboards responses. - warehouse_id: - description: The warehouse ID used to run the dashboard. + "warehouse_id": + "description": |- + The warehouse ID used to run the dashboard. github.com/databricks/cli/bundle/config/resources.Job: - budget_policy_id: - description: |- + "budget_policy_id": + "description": |- The id of the user specified budget policy to use for this job. If not specified, a default budget policy may be applied when creating or modifying the job. See `effective_budget_policy_id` for the budget policy used by this workload. - continuous: - description: An optional continuous property for this job. The continuous property - will ensure that there is always one run executing. Only one of `schedule` and - `continuous` can be used. - deployment: - description: Deployment information for jobs managed by external sources. - description: - description: An optional description for the job. The maximum length is 27700 - characters in UTF-8 encoding. - edit_mode: - description: |- + "continuous": + "description": |- + An optional continuous property for this job. The continuous property will ensure that there is always one run executing. Only one of `schedule` and `continuous` can be used. + "deployment": + "description": |- + Deployment information for jobs managed by external sources. + "description": + "description": |- + An optional description for the job. The maximum length is 27700 characters in UTF-8 encoding. + "edit_mode": + "description": |- Edit mode of the job. * `UI_LOCKED`: The job is in a locked UI state and cannot be modified. * `EDITABLE`: The job is in an editable state and can be modified. - email_notifications: - description: An optional set of email addresses that is notified when runs of - this job begin or complete as well as when this job is deleted. - environments: - description: |- + "email_notifications": + "description": |- + An optional set of email addresses that is notified when runs of this job begin or complete as well as when this job is deleted. + "environments": + "description": |- A list of task execution environment specifications that can be referenced by serverless tasks of this job. An environment is required to be present for serverless tasks. For serverless notebook tasks, the environment is accessible in the notebook environment panel. For other serverless tasks, the task environment is required to be specified using environment_key in the task settings. - format: - description: Used to tell what is the format of the job. This field is ignored - in Create/Update/Reset calls. When using the Jobs API 2.1 this value is always - set to `"MULTI_TASK"`. - git_source: - description: |- + "format": + "description": |- + Used to tell what is the format of the job. This field is ignored in Create/Update/Reset calls. When using the Jobs API 2.1 this value is always set to `"MULTI_TASK"`. + "git_source": + "description": |- An optional specification for a remote Git repository containing the source code used by tasks. Version-controlled source code is supported by notebook, dbt, Python script, and SQL File tasks. If `git_source` is set, these tasks retrieve the file from the remote repository by default. However, this behavior can be overridden by setting `source` to `WORKSPACE` on the task. Note: dbt and SQL File tasks support only version-controlled sources. If dbt or SQL File tasks are used, `git_source` must be defined on the job. - health: {} - job_clusters: - description: A list of job cluster specifications that can be shared and reused - by tasks of this job. Libraries cannot be declared in a shared job cluster. - You must declare dependent libraries in task settings. - max_concurrent_runs: - description: |- + "health": {} + "job_clusters": + "description": |- + A list of job cluster specifications that can be shared and reused by tasks of this job. Libraries cannot be declared in a shared job cluster. You must declare dependent libraries in task settings. + "max_concurrent_runs": + "description": |- An optional maximum allowed number of concurrent runs of the job. Set this value if you want to be able to execute multiple runs of the same job concurrently. This is useful for example if you trigger your job on a frequent schedule and want to allow consecutive runs to overlap with each other, or if you want to trigger multiple runs which differ by their input parameters. This setting affects only new runs. For example, suppose the job’s concurrency is 4 and there are 4 concurrent active runs. Then setting the concurrency to 3 won’t kill any of the active runs. However, from then on, new runs are skipped unless there are fewer than 3 active runs. This value cannot exceed 1000. Setting this value to `0` causes all new runs to be skipped. - name: - description: An optional name for the job. The maximum length is 4096 bytes in - UTF-8 encoding. - notification_settings: - description: Optional notification settings that are used when sending notifications - to each of the `email_notifications` and `webhook_notifications` for this job. - parameters: - description: Job-level parameter definitions - queue: - description: The queue settings of the job. - run_as: {} - schedule: - description: An optional periodic schedule for this job. The default behavior - is that the job only runs when triggered by clicking “Run Now” in the Jobs UI - or sending an API request to `runNow`. - tags: - description: A map of tags associated with the job. These are forwarded to the - cluster as cluster tags for jobs clusters, and are subject to the same limitations - as cluster tags. A maximum of 25 tags can be added to the job. - tasks: - description: A list of task specifications to be executed by this job. - timeout_seconds: - description: An optional timeout applied to each run of this job. A value of `0` - means no timeout. - trigger: - description: A configuration to trigger a run when certain conditions are met. - The default behavior is that the job runs only when triggered by clicking “Run - Now” in the Jobs UI or sending an API request to `runNow`. - webhook_notifications: - description: A collection of system notification IDs to notify when runs of this - job begin or complete. + "name": + "description": |- + An optional name for the job. The maximum length is 4096 bytes in UTF-8 encoding. + "notification_settings": + "description": |- + Optional notification settings that are used when sending notifications to each of the `email_notifications` and `webhook_notifications` for this job. + "parameters": + "description": |- + Job-level parameter definitions + "queue": + "description": |- + The queue settings of the job. + "run_as": {} + "schedule": + "description": |- + An optional periodic schedule for this job. The default behavior is that the job only runs when triggered by clicking “Run Now” in the Jobs UI or sending an API request to `runNow`. + "tags": + "description": |- + A map of tags associated with the job. These are forwarded to the cluster as cluster tags for jobs clusters, and are subject to the same limitations as cluster tags. A maximum of 25 tags can be added to the job. + "tasks": + "description": |- + A list of task specifications to be executed by this job. + "timeout_seconds": + "description": |- + An optional timeout applied to each run of this job. A value of `0` means no timeout. + "trigger": + "description": |- + A configuration to trigger a run when certain conditions are met. The default behavior is that the job runs only when triggered by clicking “Run Now” in the Jobs UI or sending an API request to `runNow`. + "webhook_notifications": + "description": |- + A collection of system notification IDs to notify when runs of this job begin or complete. github.com/databricks/cli/bundle/config/resources.MlflowExperiment: - artifact_location: - description: Location where artifacts for the experiment are stored. - creation_time: - description: Creation time - experiment_id: - description: Unique identifier for the experiment. - last_update_time: - description: Last update time - lifecycle_stage: - description: |- + "artifact_location": + "description": |- + Location where artifacts for the experiment are stored. + "creation_time": + "description": |- + Creation time + "experiment_id": + "description": |- + Unique identifier for the experiment. + "last_update_time": + "description": |- + Last update time + "lifecycle_stage": + "description": |- Current life cycle stage of the experiment: "active" or "deleted". Deleted experiments are not returned by APIs. - name: - description: Human readable name that identifies the experiment. - tags: - description: 'Tags: Additional metadata key-value pairs.' + "name": + "description": |- + Human readable name that identifies the experiment. + "tags": + "description": |- + Tags: Additional metadata key-value pairs. github.com/databricks/cli/bundle/config/resources.MlflowModel: - creation_timestamp: - description: Timestamp recorded when this `registered_model` was created. - description: - description: Description of this `registered_model`. - last_updated_timestamp: - description: Timestamp recorded when metadata for this `registered_model` was - last updated. - latest_versions: - description: |- + "creation_timestamp": + "description": |- + Timestamp recorded when this `registered_model` was created. + "description": + "description": |- + Description of this `registered_model`. + "last_updated_timestamp": + "description": |- + Timestamp recorded when metadata for this `registered_model` was last updated. + "latest_versions": + "description": |- Collection of latest model versions for each stage. Only contains models with current `READY` status. - name: - description: Unique name for the model. - tags: - description: 'Tags: Additional metadata key-value pairs for this `registered_model`.' - user_id: - description: User that created this `registered_model` + "name": + "description": |- + Unique name for the model. + "tags": + "description": |- + Tags: Additional metadata key-value pairs for this `registered_model`. + "user_id": + "description": |- + User that created this `registered_model` github.com/databricks/cli/bundle/config/resources.ModelServingEndpoint: - ai_gateway: - description: 'The AI Gateway configuration for the serving endpoint. NOTE: only - external model endpoints are supported as of now.' - config: - description: The core config of the serving endpoint. - name: - description: | + "ai_gateway": + "description": |- + The AI Gateway configuration for the serving endpoint. NOTE: only external model endpoints are supported as of now. + "config": + "description": |- + The core config of the serving endpoint. + "name": + "description": | The name of the serving endpoint. This field is required and must be unique across a Databricks workspace. An endpoint name can consist of alphanumeric characters, dashes, and underscores. - rate_limits: - description: 'Rate limits to be applied to the serving endpoint. NOTE: this field - is deprecated, please use AI Gateway to manage rate limits.' - route_optimized: - description: Enable route optimization for the serving endpoint. - tags: - description: Tags to be attached to the serving endpoint and automatically propagated - to billing logs. + "rate_limits": + "description": |- + Rate limits to be applied to the serving endpoint. NOTE: this field is deprecated, please use AI Gateway to manage rate limits. + "route_optimized": + "description": |- + Enable route optimization for the serving endpoint. + "tags": + "description": |- + Tags to be attached to the serving endpoint and automatically propagated to billing logs. github.com/databricks/cli/bundle/config/resources.Pipeline: - budget_policy_id: - description: Budget policy of this pipeline. - catalog: - description: A catalog in Unity Catalog to publish data from this pipeline to. - If `target` is specified, tables in this pipeline are published to a `target` - schema inside `catalog` (for example, `catalog`.`target`.`table`). If `target` - is not specified, no data is published to Unity Catalog. - channel: - description: DLT Release Channel that specifies which version to use. - clusters: - description: Cluster settings for this pipeline deployment. - configuration: - description: String-String configuration for this pipeline execution. - continuous: - description: Whether the pipeline is continuous or triggered. This replaces `trigger`. - deployment: - description: Deployment type of this pipeline. - development: - description: Whether the pipeline is in Development mode. Defaults to false. - edition: - description: Pipeline product edition. - filters: - description: Filters on which Pipeline packages to include in the deployed graph. - gateway_definition: - description: The definition of a gateway pipeline to support change data capture. - id: - description: Unique identifier for this pipeline. - ingestion_definition: - description: The configuration for a managed ingestion pipeline. These settings - cannot be used with the 'libraries', 'target' or 'catalog' settings. - libraries: - description: Libraries or code needed by this deployment. - name: - description: Friendly identifier for this pipeline. - notifications: - description: List of notification settings for this pipeline. - photon: - description: Whether Photon is enabled for this pipeline. - restart_window: - description: Restart window of this pipeline. - schema: - description: The default schema (database) where tables are read from or published - to. The presence of this field implies that the pipeline is in direct publishing - mode. - serverless: - description: Whether serverless compute is enabled for this pipeline. - storage: - description: DBFS root directory for storing checkpoints and tables. - target: - description: Target schema (database) to add tables in this pipeline to. If not - specified, no data is published to the Hive metastore or Unity Catalog. To publish - to Unity Catalog, also specify `catalog`. - trigger: - description: 'Which pipeline trigger to use. Deprecated: Use `continuous` instead.' + "budget_policy_id": + "description": |- + Budget policy of this pipeline. + "catalog": + "description": |- + A catalog in Unity Catalog to publish data from this pipeline to. If `target` is specified, tables in this pipeline are published to a `target` schema inside `catalog` (for example, `catalog`.`target`.`table`). If `target` is not specified, no data is published to Unity Catalog. + "channel": + "description": |- + DLT Release Channel that specifies which version to use. + "clusters": + "description": |- + Cluster settings for this pipeline deployment. + "configuration": + "description": |- + String-String configuration for this pipeline execution. + "continuous": + "description": |- + Whether the pipeline is continuous or triggered. This replaces `trigger`. + "deployment": + "description": |- + Deployment type of this pipeline. + "development": + "description": |- + Whether the pipeline is in Development mode. Defaults to false. + "edition": + "description": |- + Pipeline product edition. + "filters": + "description": |- + Filters on which Pipeline packages to include in the deployed graph. + "gateway_definition": + "description": |- + The definition of a gateway pipeline to support change data capture. + "id": + "description": |- + Unique identifier for this pipeline. + "ingestion_definition": + "description": |- + The configuration for a managed ingestion pipeline. These settings cannot be used with the 'libraries', 'target' or 'catalog' settings. + "libraries": + "description": |- + Libraries or code needed by this deployment. + "name": + "description": |- + Friendly identifier for this pipeline. + "notifications": + "description": |- + List of notification settings for this pipeline. + "photon": + "description": |- + Whether Photon is enabled for this pipeline. + "restart_window": + "description": |- + Restart window of this pipeline. + "schema": + "description": |- + The default schema (database) where tables are read from or published to. The presence of this field implies that the pipeline is in direct publishing mode. + "serverless": + "description": |- + Whether serverless compute is enabled for this pipeline. + "storage": + "description": |- + DBFS root directory for storing checkpoints and tables. + "target": + "description": |- + Target schema (database) to add tables in this pipeline to. If not specified, no data is published to the Hive metastore or Unity Catalog. To publish to Unity Catalog, also specify `catalog`. + "trigger": + "description": |- + Which pipeline trigger to use. Deprecated: Use `continuous` instead. github.com/databricks/cli/bundle/config/resources.QualityMonitor: - assets_dir: - description: The directory to store monitoring assets (e.g. dashboard, metric - tables). - baseline_table_name: - description: | + "assets_dir": + "description": |- + The directory to store monitoring assets (e.g. dashboard, metric tables). + "baseline_table_name": + "description": | Name of the baseline table from which drift metrics are computed from. Columns in the monitored table should also be present in the baseline table. - custom_metrics: - description: | + "custom_metrics": + "description": | Custom metrics to compute on the monitored table. These can be aggregate metrics, derived metrics (from already computed aggregate metrics), or drift metrics (comparing metrics across time windows). - data_classification_config: - description: The data classification config for the monitor. - inference_log: - description: Configuration for monitoring inference logs. - notifications: - description: The notification settings for the monitor. - output_schema_name: - description: Schema where output metric tables are created. - schedule: - description: The schedule for automatically updating and refreshing metric tables. - skip_builtin_dashboard: - description: Whether to skip creating a default dashboard summarizing data quality - metrics. - slicing_exprs: - description: | + "data_classification_config": + "description": |- + The data classification config for the monitor. + "inference_log": + "description": |- + Configuration for monitoring inference logs. + "notifications": + "description": |- + The notification settings for the monitor. + "output_schema_name": + "description": |- + Schema where output metric tables are created. + "schedule": + "description": |- + The schedule for automatically updating and refreshing metric tables. + "skip_builtin_dashboard": + "description": |- + Whether to skip creating a default dashboard summarizing data quality metrics. + "slicing_exprs": + "description": | List of column expressions to slice data with for targeted analysis. The data is grouped by each expression independently, resulting in a separate slice for each predicate and its complements. For high-cardinality columns, only the top 100 unique values by frequency will generate slices. - snapshot: - description: Configuration for monitoring snapshot tables. - time_series: - description: Configuration for monitoring time series tables. - warehouse_id: - description: | + "snapshot": + "description": |- + Configuration for monitoring snapshot tables. + "time_series": + "description": |- + Configuration for monitoring time series tables. + "warehouse_id": + "description": | Optional argument to specify the warehouse for dashboard creation. If not specified, the first running warehouse will be used. github.com/databricks/cli/bundle/config/resources.RegisteredModel: - catalog_name: - description: The name of the catalog where the schema and the registered model - reside - comment: - description: The comment attached to the registered model - name: - description: The name of the registered model - schema_name: - description: The name of the schema where the registered model resides - storage_location: - description: The storage location on the cloud under which model version data - files are stored + "catalog_name": + "description": |- + The name of the catalog where the schema and the registered model reside + "comment": + "description": |- + The comment attached to the registered model + "name": + "description": |- + The name of the registered model + "schema_name": + "description": |- + The name of the schema where the registered model resides + "storage_location": + "description": |- + The storage location on the cloud under which model version data files are stored github.com/databricks/cli/bundle/config/resources.Schema: - catalog_name: - description: Name of parent catalog. - comment: - description: User-provided free-form text description. - name: - description: Name of schema, relative to parent catalog. - properties: {} - storage_root: - description: Storage root URL for managed tables within schema. + "catalog_name": + "description": |- + Name of parent catalog. + "comment": + "description": |- + User-provided free-form text description. + "name": + "description": |- + Name of schema, relative to parent catalog. + "properties": {} + "storage_root": + "description": |- + Storage root URL for managed tables within schema. github.com/databricks/cli/bundle/config/resources.Volume: - catalog_name: - description: The name of the catalog where the schema and the volume are - comment: - description: The comment attached to the volume - name: - description: The name of the volume - schema_name: - description: The name of the schema where the volume is - storage_location: - description: The storage location on the cloud - volume_type: {} + "catalog_name": + "description": |- + The name of the catalog where the schema and the volume are + "comment": + "description": |- + The comment attached to the volume + "name": + "description": |- + The name of the volume + "schema_name": + "description": |- + The name of the schema where the volume is + "storage_location": + "description": |- + The storage location on the cloud + "volume_type": {} github.com/databricks/databricks-sdk-go/service/catalog.MonitorCronSchedule: - pause_status: - description: Read only field that indicates whether a schedule is paused or not. - quartz_cron_expression: - description: | + "pause_status": + "description": |- + Read only field that indicates whether a schedule is paused or not. + "quartz_cron_expression": + "description": | The expression that determines when to run the monitor. See [examples](https://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html). - timezone_id: - description: | + "timezone_id": + "description": | The timezone id (e.g., ``"PST"``) in which to evaluate the quartz expression. github.com/databricks/databricks-sdk-go/service/catalog.MonitorCronSchedulePauseStatus: - _: - description: Read only field that indicates whether a schedule is paused or not. - enum: - - UNPAUSED - - PAUSED + "_": + "description": |- + Read only field that indicates whether a schedule is paused or not. + "enum": + - |- + UNPAUSED + - |- + PAUSED github.com/databricks/databricks-sdk-go/service/catalog.MonitorDataClassificationConfig: - enabled: - description: Whether data classification is enabled. + "enabled": + "description": |- + Whether data classification is enabled. github.com/databricks/databricks-sdk-go/service/catalog.MonitorDestination: - email_addresses: - description: The list of email addresses to send the notification to. A maximum - of 5 email addresses is supported. + "email_addresses": + "description": |- + The list of email addresses to send the notification to. A maximum of 5 email addresses is supported. github.com/databricks/databricks-sdk-go/service/catalog.MonitorInferenceLog: - granularities: - description: | + "granularities": + "description": | Granularities for aggregating data into time windows based on their timestamp. Currently the following static granularities are supported: {``"5 minutes"``, ``"30 minutes"``, ``"1 hour"``, ``"1 day"``, ``" week(s)"``, ``"1 month"``, ``"1 year"``}. - label_col: - description: Optional column that contains the ground truth for the prediction. - model_id_col: - description: | + "label_col": + "description": |- + Optional column that contains the ground truth for the prediction. + "model_id_col": + "description": | Column that contains the id of the model generating the predictions. Metrics will be computed per model id by default, and also across all model ids. - prediction_col: - description: Column that contains the output/prediction from the model. - prediction_proba_col: - description: | + "prediction_col": + "description": |- + Column that contains the output/prediction from the model. + "prediction_proba_col": + "description": | Optional column that contains the prediction probabilities for each class in a classification problem type. The values in this column should be a map, mapping each class label to the prediction probability for a given sample. The map should be of PySpark MapType(). - problem_type: - description: Problem type the model aims to solve. Determines the type of model-quality - metrics that will be computed. - timestamp_col: - description: | + "problem_type": + "description": |- + Problem type the model aims to solve. Determines the type of model-quality metrics that will be computed. + "timestamp_col": + "description": | Column that contains the timestamps of requests. The column must be one of the following: - A ``TimestampType`` column - A column whose values can be converted to timestamps through the pyspark ``to_timestamp`` [function](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.to_timestamp.html). github.com/databricks/databricks-sdk-go/service/catalog.MonitorInferenceLogProblemType: - _: - description: Problem type the model aims to solve. Determines the type of model-quality - metrics that will be computed. - enum: - - PROBLEM_TYPE_CLASSIFICATION - - PROBLEM_TYPE_REGRESSION + "_": + "description": |- + Problem type the model aims to solve. Determines the type of model-quality metrics that will be computed. + "enum": + - |- + PROBLEM_TYPE_CLASSIFICATION + - |- + PROBLEM_TYPE_REGRESSION github.com/databricks/databricks-sdk-go/service/catalog.MonitorMetric: - definition: - description: Jinja template for a SQL expression that specifies how to compute - the metric. See [create metric definition](https://docs.databricks.com/en/lakehouse-monitoring/custom-metrics.html#create-definition). - input_columns: - description: | + "definition": + "description": |- + Jinja template for a SQL expression that specifies how to compute the metric. See [create metric definition](https://docs.databricks.com/en/lakehouse-monitoring/custom-metrics.html#create-definition). + "input_columns": + "description": | A list of column names in the input table the metric should be computed for. Can use ``":table"`` to indicate that the metric needs information from multiple columns. - name: - description: Name of the metric in the output tables. - output_data_type: - description: The output type of the custom metric. - type: - description: | + "name": + "description": |- + Name of the metric in the output tables. + "output_data_type": + "description": |- + The output type of the custom metric. + "type": + "description": | Can only be one of ``"CUSTOM_METRIC_TYPE_AGGREGATE"``, ``"CUSTOM_METRIC_TYPE_DERIVED"``, or ``"CUSTOM_METRIC_TYPE_DRIFT"``. The ``"CUSTOM_METRIC_TYPE_AGGREGATE"`` and ``"CUSTOM_METRIC_TYPE_DERIVED"`` metrics are computed on a single table, whereas the ``"CUSTOM_METRIC_TYPE_DRIFT"`` compare metrics across @@ -516,8 +579,8 @@ github.com/databricks/databricks-sdk-go/service/catalog.MonitorMetric: - CUSTOM_METRIC_TYPE_DERIVED: depend on previously computed aggregate metrics - CUSTOM_METRIC_TYPE_DRIFT: depend on previously computed aggregate or derived metrics github.com/databricks/databricks-sdk-go/service/catalog.MonitorMetricType: - _: - description: | + "_": + "description": | Can only be one of ``"CUSTOM_METRIC_TYPE_AGGREGATE"``, ``"CUSTOM_METRIC_TYPE_DERIVED"``, or ``"CUSTOM_METRIC_TYPE_DRIFT"``. The ``"CUSTOM_METRIC_TYPE_AGGREGATE"`` and ``"CUSTOM_METRIC_TYPE_DERIVED"`` metrics are computed on a single table, whereas the ``"CUSTOM_METRIC_TYPE_DRIFT"`` compare metrics across @@ -525,50 +588,57 @@ github.com/databricks/databricks-sdk-go/service/catalog.MonitorMetricType: - CUSTOM_METRIC_TYPE_AGGREGATE: only depend on the existing columns in your table - CUSTOM_METRIC_TYPE_DERIVED: depend on previously computed aggregate metrics - CUSTOM_METRIC_TYPE_DRIFT: depend on previously computed aggregate or derived metrics - enum: - - CUSTOM_METRIC_TYPE_AGGREGATE - - CUSTOM_METRIC_TYPE_DERIVED - - CUSTOM_METRIC_TYPE_DRIFT + "enum": + - |- + CUSTOM_METRIC_TYPE_AGGREGATE + - |- + CUSTOM_METRIC_TYPE_DERIVED + - |- + CUSTOM_METRIC_TYPE_DRIFT github.com/databricks/databricks-sdk-go/service/catalog.MonitorNotifications: - on_failure: - description: Who to send notifications to on monitor failure. - on_new_classification_tag_detected: - description: Who to send notifications to when new data classification tags are - detected. + "on_failure": + "description": |- + Who to send notifications to on monitor failure. + "on_new_classification_tag_detected": + "description": |- + Who to send notifications to when new data classification tags are detected. github.com/databricks/databricks-sdk-go/service/catalog.MonitorSnapshot: {} github.com/databricks/databricks-sdk-go/service/catalog.MonitorTimeSeries: - granularities: - description: | + "granularities": + "description": | Granularities for aggregating data into time windows based on their timestamp. Currently the following static granularities are supported: {``"5 minutes"``, ``"30 minutes"``, ``"1 hour"``, ``"1 day"``, ``" week(s)"``, ``"1 month"``, ``"1 year"``}. - timestamp_col: - description: | + "timestamp_col": + "description": | Column that contains the timestamps of requests. The column must be one of the following: - A ``TimestampType`` column - A column whose values can be converted to timestamps through the pyspark ``to_timestamp`` [function](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.to_timestamp.html). github.com/databricks/databricks-sdk-go/service/catalog.VolumeType: - _: - enum: - - EXTERNAL - - MANAGED + "_": + "enum": + - |- + EXTERNAL + - |- + MANAGED github.com/databricks/databricks-sdk-go/service/compute.Adlsgen2Info: - destination: - description: abfss destination, e.g. `abfss://@.dfs.core.windows.net/`. + "destination": + "description": |- + abfss destination, e.g. `abfss://@.dfs.core.windows.net/`. github.com/databricks/databricks-sdk-go/service/compute.AutoScale: - max_workers: - description: |- + "max_workers": + "description": |- The maximum number of workers to which the cluster can scale up when overloaded. Note that `max_workers` must be strictly greater than `min_workers`. - min_workers: - description: |- + "min_workers": + "description": |- The minimum number of workers to which the cluster can scale down when underutilized. It is also the initial number of workers the cluster will have after creation. github.com/databricks/databricks-sdk-go/service/compute.AwsAttributes: - availability: {} - ebs_volume_count: - description: |- + "availability": {} + "ebs_volume_count": + "description": |- The number of volumes launched for each instance. Users can choose up to 10 volumes. This feature is only enabled for supported node types. Legacy node types cannot specify custom EBS volumes. @@ -585,22 +655,20 @@ github.com/databricks/databricks-sdk-go/service/compute.AwsAttributes: Please note that if EBS volumes are specified, then the Spark configuration `spark.local.dir` will be overridden. - ebs_volume_iops: - description: If using gp3 volumes, what IOPS to use for the disk. If this is not - set, the maximum performance of a gp2 volume with the same volume size will - be used. - ebs_volume_size: - description: |- + "ebs_volume_iops": + "description": |- + If using gp3 volumes, what IOPS to use for the disk. If this is not set, the maximum performance of a gp2 volume with the same volume size will be used. + "ebs_volume_size": + "description": |- The size of each EBS volume (in GiB) launched for each instance. For general purpose SSD, this value must be within the range 100 - 4096. For throughput optimized HDD, this value must be within the range 500 - 4096. - ebs_volume_throughput: - description: If using gp3 volumes, what throughput to use for the disk. If this - is not set, the maximum performance of a gp2 volume with the same volume size - will be used. - ebs_volume_type: {} - first_on_demand: - description: |- + "ebs_volume_throughput": + "description": |- + If using gp3 volumes, what throughput to use for the disk. If this is not set, the maximum performance of a gp2 volume with the same volume size will be used. + "ebs_volume_type": {} + "first_on_demand": + "description": |- The first `first_on_demand` nodes of the cluster will be placed on on-demand instances. If this value is greater than 0, the cluster driver node in particular will be placed on an on-demand instance. If this value is greater than or equal to the current cluster size, all @@ -608,8 +676,8 @@ github.com/databricks/databricks-sdk-go/service/compute.AwsAttributes: size, `first_on_demand` nodes will be placed on on-demand instances and the remainder will be placed on `availability` instances. Note that this value does not affect cluster size and cannot currently be mutated over the lifetime of a cluster. - instance_profile_arn: - description: |- + "instance_profile_arn": + "description": |- Nodes for this cluster will only be placed on AWS instances with this instance profile. If ommitted, nodes will be placed on instances without an IAM instance profile. The instance profile must have previously been added to the Databricks environment by an account @@ -618,8 +686,8 @@ github.com/databricks/databricks-sdk-go/service/compute.AwsAttributes: This feature may only be available to certain customer plans. If this field is ommitted, we will pull in the default from the conf if it exists. - spot_bid_price_percent: - description: |- + "spot_bid_price_percent": + "description": |- The bid price for AWS spot instances, as a percentage of the corresponding instance type's on-demand price. For example, if this field is set to 50, and the cluster needs a new `r3.xlarge` spot @@ -632,8 +700,8 @@ github.com/databricks/databricks-sdk-go/service/compute.AwsAttributes: The default value and documentation here should be kept consistent with CommonConf.defaultSpotBidPricePercent and CommonConf.maxSpotBidPricePercent. - zone_id: - description: |- + "zone_id": + "description": |- Identifier for the availability zone/datacenter in which the cluster resides. This string will be of a form like "us-west-2a". The provided availability zone must be in the same region as the Databricks deployment. For example, "us-west-2a" @@ -644,19 +712,22 @@ github.com/databricks/databricks-sdk-go/service/compute.AwsAttributes: The list of available zones as well as the default value can be found by using the `List Zones` method. github.com/databricks/databricks-sdk-go/service/compute.AwsAvailability: - _: - description: | + "_": + "description": | Availability type used for all subsequent nodes past the `first_on_demand` ones. Note: If `first_on_demand` is zero, this availability type will be used for the entire cluster. - enum: - - SPOT - - ON_DEMAND - - SPOT_WITH_FALLBACK + "enum": + - |- + SPOT + - |- + ON_DEMAND + - |- + SPOT_WITH_FALLBACK github.com/databricks/databricks-sdk-go/service/compute.AzureAttributes: - availability: {} - first_on_demand: - description: |- + "availability": {} + "first_on_demand": + "description": |- The first `first_on_demand` nodes of the cluster will be placed on on-demand instances. This value should be greater than 0, to make sure the cluster driver node is placed on an on-demand instance. If this value is greater than or equal to the current cluster size, all @@ -664,126 +735,131 @@ github.com/databricks/databricks-sdk-go/service/compute.AzureAttributes: size, `first_on_demand` nodes will be placed on on-demand instances and the remainder will be placed on `availability` instances. Note that this value does not affect cluster size and cannot currently be mutated over the lifetime of a cluster. - log_analytics_info: - description: Defines values necessary to configure and run Azure Log Analytics - agent - spot_bid_max_price: - description: |- + "log_analytics_info": + "description": |- + Defines values necessary to configure and run Azure Log Analytics agent + "spot_bid_max_price": + "description": |- The max bid price to be used for Azure spot instances. The Max price for the bid cannot be higher than the on-demand price of the instance. If not specified, the default value is -1, which specifies that the instance cannot be evicted on the basis of price, and only on the basis of availability. Further, the value should > 0 or -1. github.com/databricks/databricks-sdk-go/service/compute.AzureAvailability: - _: - description: |- + "_": + "description": |- Availability type used for all subsequent nodes past the `first_on_demand` ones. Note: If `first_on_demand` is zero (which only happens on pool clusters), this availability type will be used for the entire cluster. - enum: - - SPOT_AZURE - - ON_DEMAND_AZURE - - SPOT_WITH_FALLBACK_AZURE + "enum": + - |- + SPOT_AZURE + - |- + ON_DEMAND_AZURE + - |- + SPOT_WITH_FALLBACK_AZURE github.com/databricks/databricks-sdk-go/service/compute.ClientsTypes: - jobs: - description: With jobs set, the cluster can be used for jobs - notebooks: - description: With notebooks set, this cluster can be used for notebooks + "jobs": + "description": |- + With jobs set, the cluster can be used for jobs + "notebooks": + "description": |- + With notebooks set, this cluster can be used for notebooks github.com/databricks/databricks-sdk-go/service/compute.ClusterLogConf: - dbfs: - description: |- + "dbfs": + "description": |- destination needs to be provided. e.g. `{ "dbfs" : { "destination" : "dbfs:/home/cluster_log" } }` - s3: - description: |- + "s3": + "description": |- destination and either the region or endpoint need to be provided. e.g. `{ "s3": { "destination" : "s3://cluster_log_bucket/prefix", "region" : "us-west-2" } }` Cluster iam role is used to access s3, please make sure the cluster iam role in `instance_profile_arn` has permission to write data to the s3 destination. github.com/databricks/databricks-sdk-go/service/compute.ClusterSpec: - apply_policy_default_values: - description: When set to true, fixed and default values from the policy will be - used for fields that are omitted. When set to false, only fixed values from - the policy will be applied. - autoscale: - description: |- + "apply_policy_default_values": + "description": |- + When set to true, fixed and default values from the policy will be used for fields that are omitted. When set to false, only fixed values from the policy will be applied. + "autoscale": + "description": |- Parameters needed in order to automatically scale clusters up and down based on load. Note: autoscaling works best with DB runtime versions 3.0 or later. - autotermination_minutes: - description: |- + "autotermination_minutes": + "description": |- Automatically terminates the cluster after it is inactive for this time in minutes. If not set, this cluster will not be automatically terminated. If specified, the threshold must be between 10 and 10000 minutes. Users can also set this value to 0 to explicitly disable automatic termination. - aws_attributes: - description: |- + "aws_attributes": + "description": |- Attributes related to clusters running on Amazon Web Services. If not specified at cluster creation, a set of default values will be used. - azure_attributes: - description: |- + "azure_attributes": + "description": |- Attributes related to clusters running on Microsoft Azure. If not specified at cluster creation, a set of default values will be used. - cluster_log_conf: - description: |- + "cluster_log_conf": + "description": |- The configuration for delivering spark logs to a long-term storage destination. Two kinds of destinations (dbfs and s3) are supported. Only one destination can be specified for one cluster. If the conf is given, the logs will be delivered to the destination every `5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while the destination of executor logs is `$destination/$clusterId/executor`. - cluster_name: - description: | + "cluster_name": + "description": | Cluster name requested by the user. This doesn't have to be unique. If not specified at creation, the cluster name will be an empty string. - custom_tags: - description: |- + "custom_tags": + "description": |- Additional tags for cluster resources. Databricks will tag all cluster resources (e.g., AWS instances and EBS volumes) with these tags in addition to `default_tags`. Notes: - Currently, Databricks allows at most 45 custom tags - Clusters can only reuse cloud resources if the resources' tags are a subset of the cluster tags - data_security_mode: {} - docker_image: {} - driver_instance_pool_id: - description: |- + "data_security_mode": {} + "docker_image": {} + "driver_instance_pool_id": + "description": |- The optional ID of the instance pool for the driver of the cluster belongs. The pool cluster uses the instance pool with id (instance_pool_id) if the driver pool is not assigned. - driver_node_type_id: - description: | + "driver_node_type_id": + "description": | The node type of the Spark driver. Note that this field is optional; if unset, the driver node type will be set as the same value as `node_type_id` defined above. - enable_elastic_disk: - description: |- + "enable_elastic_disk": + "description": |- Autoscaling Local Storage: when enabled, this cluster will dynamically acquire additional disk space when its Spark workers are running low on disk space. This feature requires specific AWS permissions to function correctly - refer to the User Guide for more details. - enable_local_disk_encryption: - description: Whether to enable LUKS on cluster VMs' local disks - gcp_attributes: - description: |- + "enable_local_disk_encryption": + "description": |- + Whether to enable LUKS on cluster VMs' local disks + "gcp_attributes": + "description": |- Attributes related to clusters running on Google Cloud Platform. If not specified at cluster creation, a set of default values will be used. - init_scripts: - description: The configuration for storing init scripts. Any number of destinations - can be specified. The scripts are executed sequentially in the order provided. - If `cluster_log_conf` is specified, init script logs are sent to `//init_scripts`. - instance_pool_id: - description: The optional ID of the instance pool to which the cluster belongs. - is_single_node: - description: | + "init_scripts": + "description": |- + The configuration for storing init scripts. Any number of destinations can be specified. The scripts are executed sequentially in the order provided. If `cluster_log_conf` is specified, init script logs are sent to `//init_scripts`. + "instance_pool_id": + "description": |- + The optional ID of the instance pool to which the cluster belongs. + "is_single_node": + "description": | This field can only be used with `kind`. When set to true, Databricks will automatically set single node related `custom_tags`, `spark_conf`, and `num_workers` - kind: {} - node_type_id: - description: | + "kind": {} + "node_type_id": + "description": | This field encodes, through a single value, the resources available to each of the Spark nodes in this cluster. For example, the Spark nodes can be provisioned and optimized for memory or compute intensive workloads. A list of available node types can be retrieved by using the :method:clusters/listNodeTypes API call. - num_workers: - description: |- + "num_workers": + "description": |- Number of worker nodes that this cluster should have. A cluster has one Spark Driver and `num_workers` Executors for a total of `num_workers` + 1 Spark nodes. @@ -792,18 +868,20 @@ github.com/databricks/databricks-sdk-go/service/compute.ClusterSpec: is resized from 5 to 10 workers, this field will immediately be updated to reflect the target size of 10 workers, whereas the workers listed in `spark_info` will gradually increase from 5 to 10 as the new nodes are provisioned. - policy_id: - description: The ID of the cluster policy used to create the cluster if applicable. - runtime_engine: {} - single_user_name: - description: Single user name if data_security_mode is `SINGLE_USER` - spark_conf: - description: | + "policy_id": + "description": |- + The ID of the cluster policy used to create the cluster if applicable. + "runtime_engine": {} + "single_user_name": + "description": |- + Single user name if data_security_mode is `SINGLE_USER` + "spark_conf": + "description": | An object containing a set of optional, user-specified Spark configuration key-value pairs. Users can also pass in a string of extra JVM options to the driver and the executors via `spark.driver.extraJavaOptions` and `spark.executor.extraJavaOptions` respectively. - spark_env_vars: - description: |- + "spark_env_vars": + "description": |- An object containing a set of optional, user-specified environment variable key-value pairs. Please note that key-value pair of the form (X,Y) will be exported as is (i.e., `export X='Y'`) while launching the driver and workers. @@ -815,25 +893,25 @@ github.com/databricks/databricks-sdk-go/service/compute.ClusterSpec: Example Spark environment variables: `{"SPARK_WORKER_MEMORY": "28000m", "SPARK_LOCAL_DIRS": "/local_disk0"}` or `{"SPARK_DAEMON_JAVA_OPTS": "$SPARK_DAEMON_JAVA_OPTS -Dspark.shuffle.service.enabled=true"}` - spark_version: - description: | + "spark_version": + "description": | The Spark version of the cluster, e.g. `3.3.x-scala2.11`. A list of available Spark versions can be retrieved by using the :method:clusters/sparkVersions API call. - ssh_public_keys: - description: |- + "ssh_public_keys": + "description": |- SSH public key contents that will be added to each Spark node in this cluster. The corresponding private keys can be used to login with the user name `ubuntu` on port `2200`. Up to 10 keys can be specified. - use_ml_runtime: - description: | + "use_ml_runtime": + "description": | This field can only be used with `kind`. `effective_spark_version` is determined by `spark_version` (DBR release), this field `use_ml_runtime`, and whether `node_type_id` is gpu node or not. - workload_type: {} + "workload_type": {} github.com/databricks/databricks-sdk-go/service/compute.DataSecurityMode: - _: - description: | + "_": + "description": | Data security mode decides what data governance model to use when accessing data from a cluster. @@ -854,193 +932,218 @@ github.com/databricks/databricks-sdk-go/service/compute.DataSecurityMode: * `LEGACY_PASSTHROUGH`: This mode is for users migrating from legacy Passthrough on high concurrency clusters. * `LEGACY_SINGLE_USER`: This mode is for users migrating from legacy Passthrough on standard clusters. * `LEGACY_SINGLE_USER_STANDARD`: This mode provides a way that doesn’t have UC nor passthrough enabled. - enum: - - DATA_SECURITY_MODE_AUTO - - DATA_SECURITY_MODE_STANDARD - - DATA_SECURITY_MODE_DEDICATED - - NONE - - SINGLE_USER - - USER_ISOLATION - - LEGACY_TABLE_ACL - - LEGACY_PASSTHROUGH - - LEGACY_SINGLE_USER - - LEGACY_SINGLE_USER_STANDARD + "enum": + - |- + DATA_SECURITY_MODE_AUTO + - |- + DATA_SECURITY_MODE_STANDARD + - |- + DATA_SECURITY_MODE_DEDICATED + - |- + NONE + - |- + SINGLE_USER + - |- + USER_ISOLATION + - |- + LEGACY_TABLE_ACL + - |- + LEGACY_PASSTHROUGH + - |- + LEGACY_SINGLE_USER + - |- + LEGACY_SINGLE_USER_STANDARD github.com/databricks/databricks-sdk-go/service/compute.DbfsStorageInfo: - destination: - description: dbfs destination, e.g. `dbfs:/my/path` + "destination": + "description": |- + dbfs destination, e.g. `dbfs:/my/path` github.com/databricks/databricks-sdk-go/service/compute.DockerBasicAuth: - password: - description: Password of the user - username: - description: Name of the user + "password": + "description": |- + Password of the user + "username": + "description": |- + Name of the user github.com/databricks/databricks-sdk-go/service/compute.DockerImage: - basic_auth: {} - url: - description: URL of the docker image. + "basic_auth": {} + "url": + "description": |- + URL of the docker image. github.com/databricks/databricks-sdk-go/service/compute.EbsVolumeType: - _: - description: The type of EBS volumes that will be launched with this cluster. - enum: - - GENERAL_PURPOSE_SSD - - THROUGHPUT_OPTIMIZED_HDD + "_": + "description": |- + The type of EBS volumes that will be launched with this cluster. + "enum": + - |- + GENERAL_PURPOSE_SSD + - |- + THROUGHPUT_OPTIMIZED_HDD github.com/databricks/databricks-sdk-go/service/compute.Environment: - _: - description: |- + "_": + "description": |- The environment entity used to preserve serverless environment side panel and jobs' environment for non-notebook task. In this minimal environment spec, only pip dependencies are supported. - client: - description: |- + "client": + "description": |- Client version used by the environment The client is the user-facing environment of the runtime. Each client comes with a specific set of pre-installed libraries. The version is a string, consisting of the major client version. - dependencies: - description: |- + "dependencies": + "description": |- List of pip dependencies, as supported by the version of pip in this environment. Each dependency is a pip requirement file line https://pip.pypa.io/en/stable/reference/requirements-file-format/ Allowed dependency could be , , (WSFS or Volumes in Databricks), E.g. dependencies: ["foo==0.0.1", "-r /Workspace/test/requirements.txt"] github.com/databricks/databricks-sdk-go/service/compute.GcpAttributes: - availability: {} - boot_disk_size: - description: boot disk size in GB - google_service_account: - description: |- + "availability": {} + "boot_disk_size": + "description": |- + boot disk size in GB + "google_service_account": + "description": |- If provided, the cluster will impersonate the google service account when accessing gcloud services (like GCS). The google service account must have previously been added to the Databricks environment by an account administrator. - local_ssd_count: - description: If provided, each node (workers and driver) in the cluster will have - this number of local SSDs attached. Each local SSD is 375GB in size. Refer to - [GCP documentation](https://cloud.google.com/compute/docs/disks/local-ssd#choose_number_local_ssds) - for the supported number of local SSDs for each instance type. - use_preemptible_executors: - description: |- + "local_ssd_count": + "description": |- + If provided, each node (workers and driver) in the cluster will have this number of local SSDs attached. Each local SSD is 375GB in size. Refer to [GCP documentation](https://cloud.google.com/compute/docs/disks/local-ssd#choose_number_local_ssds) for the supported number of local SSDs for each instance type. + "use_preemptible_executors": + "description": |- This field determines whether the spark executors will be scheduled to run on preemptible VMs (when set to true) versus standard compute engine VMs (when set to false; default). Note: Soon to be deprecated, use the availability field instead. - zone_id: - description: |- + "zone_id": + "description": |- Identifier for the availability zone in which the cluster resides. This can be one of the following: - "HA" => High availability, spread nodes across availability zones for a Databricks deployment region [default] - "AUTO" => Databricks picks an availability zone to schedule the cluster on. - A GCP availability zone => Pick One of the available zones for (machine type + region) from https://cloud.google.com/compute/docs/regions-zones. github.com/databricks/databricks-sdk-go/service/compute.GcpAvailability: - _: - description: |- + "_": + "description": |- This field determines whether the instance pool will contain preemptible VMs, on-demand VMs, or preemptible VMs with a fallback to on-demand VMs if the former is unavailable. - enum: - - PREEMPTIBLE_GCP - - ON_DEMAND_GCP - - PREEMPTIBLE_WITH_FALLBACK_GCP + "enum": + - |- + PREEMPTIBLE_GCP + - |- + ON_DEMAND_GCP + - |- + PREEMPTIBLE_WITH_FALLBACK_GCP github.com/databricks/databricks-sdk-go/service/compute.GcsStorageInfo: - destination: - description: GCS destination/URI, e.g. `gs://my-bucket/some-prefix` + "destination": + "description": |- + GCS destination/URI, e.g. `gs://my-bucket/some-prefix` github.com/databricks/databricks-sdk-go/service/compute.InitScriptInfo: - abfss: - description: |- + "abfss": + "description": |- destination needs to be provided. e.g. `{ "abfss" : { "destination" : "abfss://@.dfs.core.windows.net/" } } - dbfs: - description: |- + "dbfs": + "description": |- destination needs to be provided. e.g. `{ "dbfs" : { "destination" : "dbfs:/home/cluster_log" } }` - file: - description: |- + "file": + "description": |- destination needs to be provided. e.g. `{ "file" : { "destination" : "file:/my/local/file.sh" } }` - gcs: - description: |- + "gcs": + "description": |- destination needs to be provided. e.g. `{ "gcs": { "destination": "gs://my-bucket/file.sh" } }` - s3: - description: |- + "s3": + "description": |- destination and either the region or endpoint need to be provided. e.g. `{ "s3": { "destination" : "s3://cluster_log_bucket/prefix", "region" : "us-west-2" } }` Cluster iam role is used to access s3, please make sure the cluster iam role in `instance_profile_arn` has permission to write data to the s3 destination. - volumes: - description: |- + "volumes": + "description": |- destination needs to be provided. e.g. `{ "volumes" : { "destination" : "/Volumes/my-init.sh" } }` - workspace: - description: |- + "workspace": + "description": |- destination needs to be provided. e.g. `{ "workspace" : { "destination" : "/Users/user1@databricks.com/my-init.sh" } }` github.com/databricks/databricks-sdk-go/service/compute.Library: - cran: - description: Specification of a CRAN library to be installed as part of the library - egg: - description: Deprecated. URI of the egg library to install. Installing Python - egg files is deprecated and is not supported in Databricks Runtime 14.0 and - above. - jar: - description: |- + "cran": + "description": |- + Specification of a CRAN library to be installed as part of the library + "egg": + "description": |- + Deprecated. URI of the egg library to install. Installing Python egg files is deprecated and is not supported in Databricks Runtime 14.0 and above. + "jar": + "description": |- URI of the JAR library to install. Supported URIs include Workspace paths, Unity Catalog Volumes paths, and S3 URIs. For example: `{ "jar": "/Workspace/path/to/library.jar" }`, `{ "jar" : "/Volumes/path/to/library.jar" }` or `{ "jar": "s3://my-bucket/library.jar" }`. If S3 is used, please make sure the cluster has read access on the library. You may need to launch the cluster with an IAM role to access the S3 URI. - maven: - description: |- + "maven": + "description": |- Specification of a maven library to be installed. For example: `{ "coordinates": "org.jsoup:jsoup:1.7.2" }` - pypi: - description: |- + "pypi": + "description": |- Specification of a PyPi library to be installed. For example: `{ "package": "simplejson" }` - requirements: - description: |- + "requirements": + "description": |- URI of the requirements.txt file to install. Only Workspace paths and Unity Catalog Volumes paths are supported. For example: `{ "requirements": "/Workspace/path/to/requirements.txt" }` or `{ "requirements" : "/Volumes/path/to/requirements.txt" }` - whl: - description: |- + "whl": + "description": |- URI of the wheel library to install. Supported URIs include Workspace paths, Unity Catalog Volumes paths, and S3 URIs. For example: `{ "whl": "/Workspace/path/to/library.whl" }`, `{ "whl" : "/Volumes/path/to/library.whl" }` or `{ "whl": "s3://my-bucket/library.whl" }`. If S3 is used, please make sure the cluster has read access on the library. You may need to launch the cluster with an IAM role to access the S3 URI. github.com/databricks/databricks-sdk-go/service/compute.LocalFileInfo: - destination: - description: local file destination, e.g. `file:/my/local/file.sh` + "destination": + "description": |- + local file destination, e.g. `file:/my/local/file.sh` github.com/databricks/databricks-sdk-go/service/compute.LogAnalyticsInfo: - log_analytics_primary_key: - description: - log_analytics_workspace_id: - description: + "log_analytics_primary_key": + "description": |- + + "log_analytics_workspace_id": + "description": |- + github.com/databricks/databricks-sdk-go/service/compute.MavenLibrary: - coordinates: - description: 'Gradle-style maven coordinates. For example: "org.jsoup:jsoup:1.7.2".' - exclusions: - description: |- + "coordinates": + "description": |- + Gradle-style maven coordinates. For example: "org.jsoup:jsoup:1.7.2". + "exclusions": + "description": |- List of dependences to exclude. For example: `["slf4j:slf4j", "*:hadoop-client"]`. Maven dependency exclusions: https://maven.apache.org/guides/introduction/introduction-to-optional-and-excludes-dependencies.html. - repo: - description: |- + "repo": + "description": |- Maven repo to install the Maven package from. If omitted, both Maven Central Repository and Spark Packages are searched. github.com/databricks/databricks-sdk-go/service/compute.PythonPyPiLibrary: - package: - description: |- + "package": + "description": |- The name of the pypi package to install. An optional exact version specification is also supported. Examples: "simplejson" and "simplejson==3.8.0". - repo: - description: |- + "repo": + "description": |- The repository where the package can be found. If not specified, the default pip index is used. github.com/databricks/databricks-sdk-go/service/compute.RCranLibrary: - package: - description: The name of the CRAN package to install. - repo: - description: The repository where the package can be found. If not specified, - the default CRAN repo is used. + "package": + "description": |- + The name of the CRAN package to install. + "repo": + "description": |- + The repository where the package can be found. If not specified, the default CRAN repo is used. github.com/databricks/databricks-sdk-go/service/compute.RuntimeEngine: - _: - description: | + "_": + "description": | Determines the cluster's runtime engine, either standard or Photon. This field is not compatible with legacy `spark_version` values that contain `-photon-`. @@ -1048,13 +1151,16 @@ github.com/databricks/databricks-sdk-go/service/compute.RuntimeEngine: If left unspecified, the runtime engine defaults to standard unless the spark_version contains -photon-, in which case Photon will be used. - enum: - - "NULL" - - STANDARD - - PHOTON + "enum": + - |- + NULL + - |- + STANDARD + - |- + PHOTON github.com/databricks/databricks-sdk-go/service/compute.S3StorageInfo: - canned_acl: - description: |- + "canned_acl": + "description": |- (Optional) Set canned access control list for the logs, e.g. `bucket-owner-full-control`. If `canned_cal` is set, please make sure the cluster iam role has `s3:PutObjectAcl` permission on the destination bucket and prefix. The full list of possible canned acl can be found at @@ -1062,329 +1168,352 @@ github.com/databricks/databricks-sdk-go/service/compute.S3StorageInfo: Please also note that by default only the object owner gets full controls. If you are using cross account role for writing data, you may want to set `bucket-owner-full-control` to make bucket owner able to read the logs. - destination: - description: |- + "destination": + "description": |- S3 destination, e.g. `s3://my-bucket/some-prefix` Note that logs will be delivered using cluster iam role, please make sure you set cluster iam role and the role has write access to the destination. Please also note that you cannot use AWS keys to deliver logs. - enable_encryption: - description: (Optional) Flag to enable server side encryption, `false` by default. - encryption_type: - description: |- + "enable_encryption": + "description": |- + (Optional) Flag to enable server side encryption, `false` by default. + "encryption_type": + "description": |- (Optional) The encryption type, it could be `sse-s3` or `sse-kms`. It will be used only when encryption is enabled and the default type is `sse-s3`. - endpoint: - description: |- + "endpoint": + "description": |- S3 endpoint, e.g. `https://s3-us-west-2.amazonaws.com`. Either region or endpoint needs to be set. If both are set, endpoint will be used. - kms_key: - description: (Optional) Kms key which will be used if encryption is enabled and - encryption type is set to `sse-kms`. - region: - description: |- + "kms_key": + "description": |- + (Optional) Kms key which will be used if encryption is enabled and encryption type is set to `sse-kms`. + "region": + "description": |- S3 region, e.g. `us-west-2`. Either region or endpoint needs to be set. If both are set, endpoint will be used. github.com/databricks/databricks-sdk-go/service/compute.VolumesStorageInfo: - destination: - description: Unity Catalog Volumes file destination, e.g. `/Volumes/my-init.sh` + "destination": + "description": |- + Unity Catalog Volumes file destination, e.g. `/Volumes/my-init.sh` github.com/databricks/databricks-sdk-go/service/compute.WorkloadType: - clients: - description: ' defined what type of clients can use the cluster. E.g. Notebooks, - Jobs' + "clients": + "description": |2- + defined what type of clients can use the cluster. E.g. Notebooks, Jobs github.com/databricks/databricks-sdk-go/service/compute.WorkspaceStorageInfo: - destination: - description: workspace files destination, e.g. `/Users/user1@databricks.com/my-init.sh` + "destination": + "description": |- + workspace files destination, e.g. `/Users/user1@databricks.com/my-init.sh` github.com/databricks/databricks-sdk-go/service/dashboards.LifecycleState: - _: - enum: - - ACTIVE - - TRASHED + "_": + "enum": + - |- + ACTIVE + - |- + TRASHED github.com/databricks/databricks-sdk-go/service/jobs.CleanRoomsNotebookTask: - clean_room_name: - description: The clean room that the notebook belongs to. - etag: - description: |- + "clean_room_name": + "description": |- + The clean room that the notebook belongs to. + "etag": + "description": |- Checksum to validate the freshness of the notebook resource (i.e. the notebook being run is the latest version). It can be fetched by calling the :method:cleanroomassets/get API. - notebook_base_parameters: - description: Base parameters to be used for the clean room notebook job. - notebook_name: - description: Name of the notebook being run. + "notebook_base_parameters": + "description": |- + Base parameters to be used for the clean room notebook job. + "notebook_name": + "description": |- + Name of the notebook being run. github.com/databricks/databricks-sdk-go/service/jobs.Condition: - _: - enum: - - ANY_UPDATED - - ALL_UPDATED + "_": + "enum": + - |- + ANY_UPDATED + - |- + ALL_UPDATED github.com/databricks/databricks-sdk-go/service/jobs.ConditionTask: - left: - description: The left operand of the condition task. Can be either a string value - or a job state or parameter reference. - op: - description: |- + "left": + "description": |- + The left operand of the condition task. Can be either a string value or a job state or parameter reference. + "op": + "description": |- * `EQUAL_TO`, `NOT_EQUAL` operators perform string comparison of their operands. This means that `“12.0” == “12”` will evaluate to `false`. * `GREATER_THAN`, `GREATER_THAN_OR_EQUAL`, `LESS_THAN`, `LESS_THAN_OR_EQUAL` operators perform numeric comparison of their operands. `“12.0” >= “12”` will evaluate to `true`, `“10.0” >= “12”` will evaluate to `false`. The boolean comparison to task values can be implemented with operators `EQUAL_TO`, `NOT_EQUAL`. If a task value was set to a boolean value, it will be serialized to `“true”` or `“false”` for the comparison. - right: - description: The right operand of the condition task. Can be either a string value - or a job state or parameter reference. + "right": + "description": |- + The right operand of the condition task. Can be either a string value or a job state or parameter reference. github.com/databricks/databricks-sdk-go/service/jobs.ConditionTaskOp: - _: - description: |- + "_": + "description": |- * `EQUAL_TO`, `NOT_EQUAL` operators perform string comparison of their operands. This means that `“12.0” == “12”` will evaluate to `false`. * `GREATER_THAN`, `GREATER_THAN_OR_EQUAL`, `LESS_THAN`, `LESS_THAN_OR_EQUAL` operators perform numeric comparison of their operands. `“12.0” >= “12”` will evaluate to `true`, `“10.0” >= “12”` will evaluate to `false`. The boolean comparison to task values can be implemented with operators `EQUAL_TO`, `NOT_EQUAL`. If a task value was set to a boolean value, it will be serialized to `“true”` or `“false”` for the comparison. - enum: - - EQUAL_TO - - GREATER_THAN - - GREATER_THAN_OR_EQUAL - - LESS_THAN - - LESS_THAN_OR_EQUAL - - NOT_EQUAL + "enum": + - |- + EQUAL_TO + - |- + GREATER_THAN + - |- + GREATER_THAN_OR_EQUAL + - |- + LESS_THAN + - |- + LESS_THAN_OR_EQUAL + - |- + NOT_EQUAL github.com/databricks/databricks-sdk-go/service/jobs.Continuous: - pause_status: - description: Indicate whether the continuous execution of the job is paused or - not. Defaults to UNPAUSED. + "pause_status": + "description": |- + Indicate whether the continuous execution of the job is paused or not. Defaults to UNPAUSED. github.com/databricks/databricks-sdk-go/service/jobs.CronSchedule: - pause_status: - description: Indicate whether this schedule is paused or not. - quartz_cron_expression: - description: A Cron expression using Quartz syntax that describes the schedule - for a job. See [Cron Trigger](http://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html) - for details. This field is required. - timezone_id: - description: A Java timezone ID. The schedule for a job is resolved with respect - to this timezone. See [Java TimeZone](https://docs.oracle.com/javase/7/docs/api/java/util/TimeZone.html) - for details. This field is required. + "pause_status": + "description": |- + Indicate whether this schedule is paused or not. + "quartz_cron_expression": + "description": |- + A Cron expression using Quartz syntax that describes the schedule for a job. See [Cron Trigger](http://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html) for details. This field is required. + "timezone_id": + "description": |- + A Java timezone ID. The schedule for a job is resolved with respect to this timezone. See [Java TimeZone](https://docs.oracle.com/javase/7/docs/api/java/util/TimeZone.html) for details. This field is required. github.com/databricks/databricks-sdk-go/service/jobs.DbtTask: - catalog: - description: Optional name of the catalog to use. The value is the top level in - the 3-level namespace of Unity Catalog (catalog / schema / relation). The catalog - value can only be specified if a warehouse_id is specified. Requires dbt-databricks - >= 1.1.1. - commands: - description: A list of dbt commands to execute. All commands must start with `dbt`. - This parameter must not be empty. A maximum of up to 10 commands can be provided. - profiles_directory: - description: Optional (relative) path to the profiles directory. Can only be specified - if no warehouse_id is specified. If no warehouse_id is specified and this folder - is unset, the root directory is used. - project_directory: - description: |- + "catalog": + "description": |- + Optional name of the catalog to use. The value is the top level in the 3-level namespace of Unity Catalog (catalog / schema / relation). The catalog value can only be specified if a warehouse_id is specified. Requires dbt-databricks >= 1.1.1. + "commands": + "description": |- + A list of dbt commands to execute. All commands must start with `dbt`. This parameter must not be empty. A maximum of up to 10 commands can be provided. + "profiles_directory": + "description": |- + Optional (relative) path to the profiles directory. Can only be specified if no warehouse_id is specified. If no warehouse_id is specified and this folder is unset, the root directory is used. + "project_directory": + "description": |- Path to the project directory. Optional for Git sourced tasks, in which case if no value is provided, the root of the Git repository is used. - schema: - description: Optional schema to write to. This parameter is only used when a warehouse_id - is also provided. If not provided, the `default` schema is used. - source: - description: |- + "schema": + "description": |- + Optional schema to write to. This parameter is only used when a warehouse_id is also provided. If not provided, the `default` schema is used. + "source": + "description": |- Optional location type of the project directory. When set to `WORKSPACE`, the project will be retrieved from the local Databricks workspace. When set to `GIT`, the project will be retrieved from a Git repository defined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise. * `WORKSPACE`: Project is located in Databricks workspace. * `GIT`: Project is located in cloud Git provider. - warehouse_id: - description: ID of the SQL warehouse to connect to. If provided, we automatically - generate and provide the profile and connection details to dbt. It can be overridden - on a per-command basis by using the `--profiles-dir` command line argument. + "warehouse_id": + "description": |- + ID of the SQL warehouse to connect to. If provided, we automatically generate and provide the profile and connection details to dbt. It can be overridden on a per-command basis by using the `--profiles-dir` command line argument. github.com/databricks/databricks-sdk-go/service/jobs.FileArrivalTriggerConfiguration: - min_time_between_triggers_seconds: - description: |- + "min_time_between_triggers_seconds": + "description": |- If set, the trigger starts a run only after the specified amount of time passed since the last time the trigger fired. The minimum allowed value is 60 seconds - url: - description: URL to be monitored for file arrivals. The path must point to the - root or a subpath of the external location. - wait_after_last_change_seconds: - description: |- + "url": + "description": |- + URL to be monitored for file arrivals. The path must point to the root or a subpath of the external location. + "wait_after_last_change_seconds": + "description": |- If set, the trigger starts a run only after no file activity has occurred for the specified amount of time. This makes it possible to wait for a batch of incoming files to arrive before triggering a run. The minimum allowed value is 60 seconds. github.com/databricks/databricks-sdk-go/service/jobs.ForEachTask: - concurrency: - description: |- + "concurrency": + "description": |- An optional maximum allowed number of concurrent runs of the task. Set this value if you want to be able to execute multiple runs of the task concurrently. - inputs: - description: |- + "inputs": + "description": |- Array for task to iterate on. This can be a JSON string or a reference to an array parameter. - task: - description: Configuration for the task that will be run for each element in the - array + "task": + "description": |- + Configuration for the task that will be run for each element in the array github.com/databricks/databricks-sdk-go/service/jobs.Format: - _: - enum: - - SINGLE_TASK - - MULTI_TASK + "_": + "enum": + - |- + SINGLE_TASK + - |- + MULTI_TASK github.com/databricks/databricks-sdk-go/service/jobs.GitProvider: - _: - enum: - - gitHub - - bitbucketCloud - - azureDevOpsServices - - gitHubEnterprise - - bitbucketServer - - gitLab - - gitLabEnterpriseEdition - - awsCodeCommit + "_": + "enum": + - |- + gitHub + - |- + bitbucketCloud + - |- + azureDevOpsServices + - |- + gitHubEnterprise + - |- + bitbucketServer + - |- + gitLab + - |- + gitLabEnterpriseEdition + - |- + awsCodeCommit github.com/databricks/databricks-sdk-go/service/jobs.GitSnapshot: - _: - description: Read-only state of the remote repository at the time the job was - run. This field is only included on job runs. - used_commit: - description: Commit that was used to execute the run. If git_branch was specified, - this points to the HEAD of the branch at the time of the run; if git_tag was - specified, this points to the commit the tag points to. + "_": + "description": |- + Read-only state of the remote repository at the time the job was run. This field is only included on job runs. + "used_commit": + "description": |- + Commit that was used to execute the run. If git_branch was specified, this points to the HEAD of the branch at the time of the run; if git_tag was specified, this points to the commit the tag points to. github.com/databricks/databricks-sdk-go/service/jobs.GitSource: - _: - description: |- + "_": + "description": |- An optional specification for a remote Git repository containing the source code used by tasks. Version-controlled source code is supported by notebook, dbt, Python script, and SQL File tasks. If `git_source` is set, these tasks retrieve the file from the remote repository by default. However, this behavior can be overridden by setting `source` to `WORKSPACE` on the task. Note: dbt and SQL File tasks support only version-controlled sources. If dbt or SQL File tasks are used, `git_source` must be defined on the job. - git_branch: - description: Name of the branch to be checked out and used by this job. This field - cannot be specified in conjunction with git_tag or git_commit. - git_commit: - description: Commit to be checked out and used by this job. This field cannot - be specified in conjunction with git_branch or git_tag. - git_provider: - description: Unique identifier of the service used to host the Git repository. - The value is case insensitive. - git_snapshot: {} - git_tag: - description: Name of the tag to be checked out and used by this job. This field - cannot be specified in conjunction with git_branch or git_commit. - git_url: - description: URL of the repository to be cloned by this job. - job_source: - description: The source of the job specification in the remote repository when - the job is source controlled. + "git_branch": + "description": |- + Name of the branch to be checked out and used by this job. This field cannot be specified in conjunction with git_tag or git_commit. + "git_commit": + "description": |- + Commit to be checked out and used by this job. This field cannot be specified in conjunction with git_branch or git_tag. + "git_provider": + "description": |- + Unique identifier of the service used to host the Git repository. The value is case insensitive. + "git_snapshot": {} + "git_tag": + "description": |- + Name of the tag to be checked out and used by this job. This field cannot be specified in conjunction with git_branch or git_commit. + "git_url": + "description": |- + URL of the repository to be cloned by this job. + "job_source": + "description": |- + The source of the job specification in the remote repository when the job is source controlled. github.com/databricks/databricks-sdk-go/service/jobs.JobCluster: - job_cluster_key: - description: |- + "job_cluster_key": + "description": |- A unique name for the job cluster. This field is required and must be unique within the job. `JobTaskSettings` may refer to this field to determine which cluster to launch for the task execution. - new_cluster: - description: If new_cluster, a description of a cluster that is created for each - task. + "new_cluster": + "description": |- + If new_cluster, a description of a cluster that is created for each task. github.com/databricks/databricks-sdk-go/service/jobs.JobDeployment: - kind: - description: |- + "kind": + "description": |- The kind of deployment that manages the job. * `BUNDLE`: The job is managed by Databricks Asset Bundle. - metadata_file_path: - description: Path of the file that contains deployment metadata. + "metadata_file_path": + "description": |- + Path of the file that contains deployment metadata. github.com/databricks/databricks-sdk-go/service/jobs.JobDeploymentKind: - _: - description: '* `BUNDLE`: The job is managed by Databricks Asset Bundle.' - enum: - - BUNDLE + "_": + "description": |- + * `BUNDLE`: The job is managed by Databricks Asset Bundle. + "enum": + - |- + BUNDLE github.com/databricks/databricks-sdk-go/service/jobs.JobEditMode: - _: - description: |- + "_": + "description": |- Edit mode of the job. * `UI_LOCKED`: The job is in a locked UI state and cannot be modified. * `EDITABLE`: The job is in an editable state and can be modified. - enum: - - UI_LOCKED - - EDITABLE + "enum": + - |- + UI_LOCKED + - |- + EDITABLE github.com/databricks/databricks-sdk-go/service/jobs.JobEmailNotifications: - no_alert_for_skipped_runs: - description: |- + "no_alert_for_skipped_runs": + "description": |- If true, do not send email to recipients specified in `on_failure` if the run is skipped. This field is `deprecated`. Please use the `notification_settings.no_alert_for_skipped_runs` field. - on_duration_warning_threshold_exceeded: - description: A list of email addresses to be notified when the duration of a run - exceeds the threshold specified for the `RUN_DURATION_SECONDS` metric in the - `health` field. If no rule for the `RUN_DURATION_SECONDS` metric is specified - in the `health` field for the job, notifications are not sent. - on_failure: - description: A list of email addresses to be notified when a run unsuccessfully - completes. A run is considered to have completed unsuccessfully if it ends with - an `INTERNAL_ERROR` `life_cycle_state` or a `FAILED`, or `TIMED_OUT` result_state. - If this is not specified on job creation, reset, or update the list is empty, - and notifications are not sent. - on_start: - description: A list of email addresses to be notified when a run begins. If not - specified on job creation, reset, or update, the list is empty, and notifications - are not sent. - on_streaming_backlog_exceeded: - description: |- + "on_duration_warning_threshold_exceeded": + "description": |- + A list of email addresses to be notified when the duration of a run exceeds the threshold specified for the `RUN_DURATION_SECONDS` metric in the `health` field. If no rule for the `RUN_DURATION_SECONDS` metric is specified in the `health` field for the job, notifications are not sent. + "on_failure": + "description": |- + A list of email addresses to be notified when a run unsuccessfully completes. A run is considered to have completed unsuccessfully if it ends with an `INTERNAL_ERROR` `life_cycle_state` or a `FAILED`, or `TIMED_OUT` result_state. If this is not specified on job creation, reset, or update the list is empty, and notifications are not sent. + "on_start": + "description": |- + A list of email addresses to be notified when a run begins. If not specified on job creation, reset, or update, the list is empty, and notifications are not sent. + "on_streaming_backlog_exceeded": + "description": |- A list of email addresses to notify when any streaming backlog thresholds are exceeded for any stream. Streaming backlog thresholds can be set in the `health` field using the following metrics: `STREAMING_BACKLOG_BYTES`, `STREAMING_BACKLOG_RECORDS`, `STREAMING_BACKLOG_SECONDS`, or `STREAMING_BACKLOG_FILES`. Alerting is based on the 10-minute average of these metrics. If the issue persists, notifications are resent every 30 minutes. - on_success: - description: A list of email addresses to be notified when a run successfully - completes. A run is considered to have completed successfully if it ends with - a `TERMINATED` `life_cycle_state` and a `SUCCESS` result_state. If not specified - on job creation, reset, or update, the list is empty, and notifications are - not sent. + "on_success": + "description": |- + A list of email addresses to be notified when a run successfully completes. A run is considered to have completed successfully if it ends with a `TERMINATED` `life_cycle_state` and a `SUCCESS` result_state. If not specified on job creation, reset, or update, the list is empty, and notifications are not sent. github.com/databricks/databricks-sdk-go/service/jobs.JobEnvironment: - environment_key: - description: The key of an environment. It has to be unique within a job. - spec: {} + "environment_key": + "description": |- + The key of an environment. It has to be unique within a job. + "spec": {} github.com/databricks/databricks-sdk-go/service/jobs.JobNotificationSettings: - no_alert_for_canceled_runs: - description: If true, do not send notifications to recipients specified in `on_failure` - if the run is canceled. - no_alert_for_skipped_runs: - description: If true, do not send notifications to recipients specified in `on_failure` - if the run is skipped. + "no_alert_for_canceled_runs": + "description": |- + If true, do not send notifications to recipients specified in `on_failure` if the run is canceled. + "no_alert_for_skipped_runs": + "description": |- + If true, do not send notifications to recipients specified in `on_failure` if the run is skipped. github.com/databricks/databricks-sdk-go/service/jobs.JobParameterDefinition: - default: - description: Default value of the parameter. - name: - description: The name of the defined parameter. May only contain alphanumeric - characters, `_`, `-`, and `.` + "default": + "description": |- + Default value of the parameter. + "name": + "description": |- + The name of the defined parameter. May only contain alphanumeric characters, `_`, `-`, and `.` github.com/databricks/databricks-sdk-go/service/jobs.JobRunAs: - _: - description: |- + "_": + "description": |- Write-only setting. Specifies the user or service principal that the job runs as. If not specified, the job runs as the user who created the job. Either `user_name` or `service_principal_name` should be specified. If not, an error is thrown. - service_principal_name: - description: Application ID of an active service principal. Setting this field - requires the `servicePrincipal/user` role. - user_name: - description: The email of an active workspace user. Non-admin users can only set - this field to their own email. + "service_principal_name": + "description": |- + Application ID of an active service principal. Setting this field requires the `servicePrincipal/user` role. + "user_name": + "description": |- + The email of an active workspace user. Non-admin users can only set this field to their own email. github.com/databricks/databricks-sdk-go/service/jobs.JobSource: - _: - description: The source of the job specification in the remote repository when - the job is source controlled. - dirty_state: - description: |- + "_": + "description": |- + The source of the job specification in the remote repository when the job is source controlled. + "dirty_state": + "description": |- Dirty state indicates the job is not fully synced with the job specification in the remote repository. Possible values are: * `NOT_SYNCED`: The job is not yet synced with the remote job specification. Import the remote job specification from UI to make the job fully synced. * `DISCONNECTED`: The job is temporary disconnected from the remote job specification and is allowed for live edit. Import the remote job specification again from UI to make the job fully synced. - import_from_git_branch: - description: Name of the branch which the job is imported from. - job_config_path: - description: Path of the job YAML file that contains the job specification. + "import_from_git_branch": + "description": |- + Name of the branch which the job is imported from. + "job_config_path": + "description": |- + Path of the job YAML file that contains the job specification. github.com/databricks/databricks-sdk-go/service/jobs.JobSourceDirtyState: - _: - description: |- + "_": + "description": |- Dirty state indicates the job is not fully synced with the job specification in the remote repository. Possible values are: * `NOT_SYNCED`: The job is not yet synced with the remote job specification. Import the remote job specification from UI to make the job fully synced. * `DISCONNECTED`: The job is temporary disconnected from the remote job specification and is allowed for live edit. Import the remote job specification again from UI to make the job fully synced. - enum: - - NOT_SYNCED - - DISCONNECTED + "enum": + - |- + NOT_SYNCED + - |- + DISCONNECTED github.com/databricks/databricks-sdk-go/service/jobs.JobsHealthMetric: - _: - description: |- + "_": + "description": |- Specifies the health metric that is being evaluated for a particular health rule. * `RUN_DURATION_SECONDS`: Expected total time for a run in seconds. @@ -1392,31 +1521,38 @@ github.com/databricks/databricks-sdk-go/service/jobs.JobsHealthMetric: * `STREAMING_BACKLOG_RECORDS`: An estimate of the maximum offset lag across all streams. This metric is in Public Preview. * `STREAMING_BACKLOG_SECONDS`: An estimate of the maximum consumer delay across all streams. This metric is in Public Preview. * `STREAMING_BACKLOG_FILES`: An estimate of the maximum number of outstanding files across all streams. This metric is in Public Preview. - enum: - - RUN_DURATION_SECONDS - - STREAMING_BACKLOG_BYTES - - STREAMING_BACKLOG_RECORDS - - STREAMING_BACKLOG_SECONDS - - STREAMING_BACKLOG_FILES + "enum": + - |- + RUN_DURATION_SECONDS + - |- + STREAMING_BACKLOG_BYTES + - |- + STREAMING_BACKLOG_RECORDS + - |- + STREAMING_BACKLOG_SECONDS + - |- + STREAMING_BACKLOG_FILES github.com/databricks/databricks-sdk-go/service/jobs.JobsHealthOperator: - _: - description: Specifies the operator used to compare the health metric value with - the specified threshold. - enum: - - GREATER_THAN + "_": + "description": |- + Specifies the operator used to compare the health metric value with the specified threshold. + "enum": + - |- + GREATER_THAN github.com/databricks/databricks-sdk-go/service/jobs.JobsHealthRule: - metric: {} - op: {} - value: - description: Specifies the threshold value that the health metric should obey - to satisfy the health rule. + "metric": {} + "op": {} + "value": + "description": |- + Specifies the threshold value that the health metric should obey to satisfy the health rule. github.com/databricks/databricks-sdk-go/service/jobs.JobsHealthRules: - _: - description: An optional set of health rules that can be defined for this job. - rules: {} + "_": + "description": |- + An optional set of health rules that can be defined for this job. + "rules": {} github.com/databricks/databricks-sdk-go/service/jobs.NotebookTask: - base_parameters: - description: |- + "base_parameters": + "description": |- Base parameters to be used for each run of this job. If the run is initiated by a call to :method:jobs/run Now with parameters specified, the two parameters maps are merged. If the same key is specified in `base_parameters` and in `run-now`, the value from `run-now` is used. @@ -1428,65 +1564,76 @@ github.com/databricks/databricks-sdk-go/service/jobs.NotebookTask: Retrieve these parameters in a notebook using [dbutils.widgets.get](https://docs.databricks.com/dev-tools/databricks-utils.html#dbutils-widgets). The JSON representation of this field cannot exceed 1MB. - notebook_path: - description: |- + "notebook_path": + "description": |- The path of the notebook to be run in the Databricks workspace or remote repository. For notebooks stored in the Databricks workspace, the path must be absolute and begin with a slash. For notebooks stored in a remote repository, the path must be relative. This field is required. - source: - description: |- + "source": + "description": |- Optional location type of the notebook. When set to `WORKSPACE`, the notebook will be retrieved from the local Databricks workspace. When set to `GIT`, the notebook will be retrieved from a Git repository defined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise. * `WORKSPACE`: Notebook is located in Databricks workspace. * `GIT`: Notebook is located in cloud Git provider. - warehouse_id: - description: |- + "warehouse_id": + "description": |- Optional `warehouse_id` to run the notebook on a SQL warehouse. Classic SQL warehouses are NOT supported, please use serverless or pro SQL warehouses. Note that SQL warehouses only support SQL cells; if the notebook contains non-SQL cells, the run will fail. github.com/databricks/databricks-sdk-go/service/jobs.PauseStatus: - _: - enum: - - UNPAUSED - - PAUSED + "_": + "enum": + - |- + UNPAUSED + - |- + PAUSED github.com/databricks/databricks-sdk-go/service/jobs.PeriodicTriggerConfiguration: - interval: - description: The interval at which the trigger should run. - unit: - description: The unit of time for the interval. + "interval": + "description": |- + The interval at which the trigger should run. + "unit": + "description": |- + The unit of time for the interval. github.com/databricks/databricks-sdk-go/service/jobs.PeriodicTriggerConfigurationTimeUnit: - _: - enum: - - HOURS - - DAYS - - WEEKS + "_": + "enum": + - |- + HOURS + - |- + DAYS + - |- + WEEKS github.com/databricks/databricks-sdk-go/service/jobs.PipelineParams: - full_refresh: - description: If true, triggers a full refresh on the delta live table. + "full_refresh": + "description": |- + If true, triggers a full refresh on the delta live table. github.com/databricks/databricks-sdk-go/service/jobs.PipelineTask: - full_refresh: - description: If true, triggers a full refresh on the delta live table. - pipeline_id: - description: The full name of the pipeline task to execute. + "full_refresh": + "description": |- + If true, triggers a full refresh on the delta live table. + "pipeline_id": + "description": |- + The full name of the pipeline task to execute. github.com/databricks/databricks-sdk-go/service/jobs.PythonWheelTask: - entry_point: - description: Named entry point to use, if it does not exist in the metadata of - the package it executes the function from the package directly using `$packageName.$entryPoint()` - named_parameters: - description: Command-line parameters passed to Python wheel task in the form of - `["--name=task", "--data=dbfs:/path/to/data.json"]`. Leave it empty if `parameters` - is not null. - package_name: - description: Name of the package to execute - parameters: - description: Command-line parameters passed to Python wheel task. Leave it empty - if `named_parameters` is not null. + "entry_point": + "description": |- + Named entry point to use, if it does not exist in the metadata of the package it executes the function from the package directly using `$packageName.$entryPoint()` + "named_parameters": + "description": |- + Command-line parameters passed to Python wheel task in the form of `["--name=task", "--data=dbfs:/path/to/data.json"]`. Leave it empty if `parameters` is not null. + "package_name": + "description": |- + Name of the package to execute + "parameters": + "description": |- + Command-line parameters passed to Python wheel task. Leave it empty if `named_parameters` is not null. github.com/databricks/databricks-sdk-go/service/jobs.QueueSettings: - enabled: - description: If true, enable queueing for the job. This is a required field. + "enabled": + "description": |- + If true, enable queueing for the job. This is a required field. github.com/databricks/databricks-sdk-go/service/jobs.RunIf: - _: - description: |- + "_": + "description": |- An optional value indicating the condition that determines whether the task should be run once its dependencies have been completed. When omitted, defaults to `ALL_SUCCESS`. Possible values are: @@ -1496,20 +1643,25 @@ github.com/databricks/databricks-sdk-go/service/jobs.RunIf: * `ALL_DONE`: All dependencies have been completed * `AT_LEAST_ONE_FAILED`: At least one dependency failed * `ALL_FAILED`: ALl dependencies have failed - enum: - - ALL_SUCCESS - - ALL_DONE - - NONE_FAILED - - AT_LEAST_ONE_SUCCESS - - ALL_FAILED - - AT_LEAST_ONE_FAILED + "enum": + - |- + ALL_SUCCESS + - |- + ALL_DONE + - |- + NONE_FAILED + - |- + AT_LEAST_ONE_SUCCESS + - |- + ALL_FAILED + - |- + AT_LEAST_ONE_FAILED github.com/databricks/databricks-sdk-go/service/jobs.RunJobTask: - dbt_commands: - description: 'An array of commands to execute for jobs with the dbt task, for - example `"dbt_commands": ["dbt deps", "dbt seed", "dbt deps", "dbt seed", "dbt - run"]`' - jar_params: - description: |- + "dbt_commands": + "description": |- + An array of commands to execute for jobs with the dbt task, for example `"dbt_commands": ["dbt deps", "dbt seed", "dbt deps", "dbt seed", "dbt run"]` + "jar_params": + "description": |- A list of parameters for jobs with Spark JAR tasks, for example `"jar_params": ["john doe", "35"]`. The parameters are used to invoke the main function of the main class specified in the Spark JAR task. If not specified upon `run-now`, it defaults to an empty list. @@ -1517,12 +1669,14 @@ github.com/databricks/databricks-sdk-go/service/jobs.RunJobTask: The JSON representation of this field (for example `{"jar_params":["john doe","35"]}`) cannot exceed 10,000 bytes. Use [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs. - job_id: - description: ID of the job to trigger. - job_parameters: - description: Job-level parameters used to trigger the job. - notebook_params: - description: |- + "job_id": + "description": |- + ID of the job to trigger. + "job_parameters": + "description": |- + Job-level parameters used to trigger the job. + "notebook_params": + "description": |- A map from keys to values for jobs with notebook task, for example `"notebook_params": {"name": "john doe", "age": "35"}`. The map is passed to the notebook and is accessible through the [dbutils.widgets.get](https://docs.databricks.com/dev-tools/databricks-utils.html) function. @@ -1533,11 +1687,12 @@ github.com/databricks/databricks-sdk-go/service/jobs.RunJobTask: Use [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs. The JSON representation of this field (for example `{"notebook_params":{"name":"john doe","age":"35"}}`) cannot exceed 10,000 bytes. - pipeline_params: - description: Controls whether the pipeline should perform a full refresh - python_named_params: {} - python_params: - description: |- + "pipeline_params": + "description": |- + Controls whether the pipeline should perform a full refresh + "python_named_params": {} + "python_params": + "description": |- A list of parameters for jobs with Python tasks, for example `"python_params": ["john doe", "35"]`. The parameters are passed to Python file as command-line parameters. If specified upon `run-now`, it would overwrite the parameters specified in job setting. The JSON representation of this field (for example `{"python_params":["john doe","35"]}`) @@ -1549,8 +1704,8 @@ github.com/databricks/databricks-sdk-go/service/jobs.RunJobTask: These parameters accept only Latin characters (ASCII character set). Using non-ASCII characters returns an error. Examples of invalid, non-ASCII characters are Chinese, Japanese kanjis, and emojis. - spark_submit_params: - description: |- + "spark_submit_params": + "description": |- A list of parameters for jobs with spark submit task, for example `"spark_submit_params": ["--class", "org.apache.spark.examples.SparkPi"]`. The parameters are passed to spark-submit script as command-line parameters. If specified upon `run-now`, it would overwrite the parameters specified in job setting. The JSON representation of this field (for example `{"python_params":["john doe","35"]}`) @@ -1562,50 +1717,48 @@ github.com/databricks/databricks-sdk-go/service/jobs.RunJobTask: These parameters accept only Latin characters (ASCII character set). Using non-ASCII characters returns an error. Examples of invalid, non-ASCII characters are Chinese, Japanese kanjis, and emojis. - sql_params: - description: 'A map from keys to values for jobs with SQL task, for example `"sql_params": - {"name": "john doe", "age": "35"}`. The SQL alert task does not support custom - parameters.' + "sql_params": + "description": |- + A map from keys to values for jobs with SQL task, for example `"sql_params": {"name": "john doe", "age": "35"}`. The SQL alert task does not support custom parameters. github.com/databricks/databricks-sdk-go/service/jobs.Source: - _: - description: |- + "_": + "description": |- Optional location type of the SQL file. When set to `WORKSPACE`, the SQL file will be retrieved\ from the local Databricks workspace. When set to `GIT`, the SQL file will be retrieved from a Git repository defined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise. * `WORKSPACE`: SQL file is located in Databricks workspace. * `GIT`: SQL file is located in cloud Git provider. - enum: - - WORKSPACE - - GIT + "enum": + - |- + WORKSPACE + - |- + GIT github.com/databricks/databricks-sdk-go/service/jobs.SparkJarTask: - jar_uri: - description: Deprecated since 04/2016. Provide a `jar` through the `libraries` - field instead. For an example, see :method:jobs/create. - main_class_name: - description: |- + "jar_uri": + "description": |- + Deprecated since 04/2016. Provide a `jar` through the `libraries` field instead. For an example, see :method:jobs/create. + "main_class_name": + "description": |- The full name of the class containing the main method to be executed. This class must be contained in a JAR provided as a library. The code must use `SparkContext.getOrCreate` to obtain a Spark context; otherwise, runs of the job fail. - parameters: - description: |- + "parameters": + "description": |- Parameters passed to the main method. Use [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs. github.com/databricks/databricks-sdk-go/service/jobs.SparkPythonTask: - parameters: - description: |- + "parameters": + "description": |- Command line parameters passed to the Python file. Use [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs. - python_file: - description: The Python file to be executed. Cloud file URIs (such as dbfs:/, - s3:/, adls:/, gcs:/) and workspace paths are supported. For python files stored - in the Databricks workspace, the path must be absolute and begin with `/`. For - files stored in a remote repository, the path must be relative. This field is - required. - source: - description: |- + "python_file": + "description": |- + The Python file to be executed. Cloud file URIs (such as dbfs:/, s3:/, adls:/, gcs:/) and workspace paths are supported. For python files stored in the Databricks workspace, the path must be absolute and begin with `/`. For files stored in a remote repository, the path must be relative. This field is required. + "source": + "description": |- Optional location type of the Python file. When set to `WORKSPACE` or not specified, the file will be retrieved from the local Databricks workspace or cloud location (if the `python_file` has a URI format). When set to `GIT`, the Python file will be retrieved from a Git repository defined in `git_source`. @@ -1613,52 +1766,59 @@ github.com/databricks/databricks-sdk-go/service/jobs.SparkPythonTask: * `WORKSPACE`: The Python file is located in a Databricks workspace or at a cloud filesystem URI. * `GIT`: The Python file is located in a remote Git repository. github.com/databricks/databricks-sdk-go/service/jobs.SparkSubmitTask: - parameters: - description: |- + "parameters": + "description": |- Command-line parameters passed to spark submit. Use [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs. github.com/databricks/databricks-sdk-go/service/jobs.SqlTask: - alert: - description: If alert, indicates that this job must refresh a SQL alert. - dashboard: - description: If dashboard, indicates that this job must refresh a SQL dashboard. - file: - description: If file, indicates that this job runs a SQL file in a remote Git - repository. - parameters: - description: Parameters to be used for each run of this job. The SQL alert task - does not support custom parameters. - query: - description: If query, indicates that this job must execute a SQL query. - warehouse_id: - description: The canonical identifier of the SQL warehouse. Recommended to use - with serverless or pro SQL warehouses. Classic SQL warehouses are only supported - for SQL alert, dashboard and query tasks and are limited to scheduled single-task - jobs. + "alert": + "description": |- + If alert, indicates that this job must refresh a SQL alert. + "dashboard": + "description": |- + If dashboard, indicates that this job must refresh a SQL dashboard. + "file": + "description": |- + If file, indicates that this job runs a SQL file in a remote Git repository. + "parameters": + "description": |- + Parameters to be used for each run of this job. The SQL alert task does not support custom parameters. + "query": + "description": |- + If query, indicates that this job must execute a SQL query. + "warehouse_id": + "description": |- + The canonical identifier of the SQL warehouse. Recommended to use with serverless or pro SQL warehouses. Classic SQL warehouses are only supported for SQL alert, dashboard and query tasks and are limited to scheduled single-task jobs. github.com/databricks/databricks-sdk-go/service/jobs.SqlTaskAlert: - alert_id: - description: The canonical identifier of the SQL alert. - pause_subscriptions: - description: If true, the alert notifications are not sent to subscribers. - subscriptions: - description: If specified, alert notifications are sent to subscribers. + "alert_id": + "description": |- + The canonical identifier of the SQL alert. + "pause_subscriptions": + "description": |- + If true, the alert notifications are not sent to subscribers. + "subscriptions": + "description": |- + If specified, alert notifications are sent to subscribers. github.com/databricks/databricks-sdk-go/service/jobs.SqlTaskDashboard: - custom_subject: - description: Subject of the email sent to subscribers of this task. - dashboard_id: - description: The canonical identifier of the SQL dashboard. - pause_subscriptions: - description: If true, the dashboard snapshot is not taken, and emails are not - sent to subscribers. - subscriptions: - description: If specified, dashboard snapshots are sent to subscriptions. + "custom_subject": + "description": |- + Subject of the email sent to subscribers of this task. + "dashboard_id": + "description": |- + The canonical identifier of the SQL dashboard. + "pause_subscriptions": + "description": |- + If true, the dashboard snapshot is not taken, and emails are not sent to subscribers. + "subscriptions": + "description": |- + If specified, dashboard snapshots are sent to subscriptions. github.com/databricks/databricks-sdk-go/service/jobs.SqlTaskFile: - path: - description: Path of the SQL file. Must be relative if the source is a remote - Git repository and absolute for workspace paths. - source: - description: |- + "path": + "description": |- + Path of the SQL file. Must be relative if the source is a remote Git repository and absolute for workspace paths. + "source": + "description": |- Optional location type of the SQL file. When set to `WORKSPACE`, the SQL file will be retrieved from the local Databricks workspace. When set to `GIT`, the SQL file will be retrieved from a Git repository defined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise. @@ -1666,107 +1826,104 @@ github.com/databricks/databricks-sdk-go/service/jobs.SqlTaskFile: * `WORKSPACE`: SQL file is located in Databricks workspace. * `GIT`: SQL file is located in cloud Git provider. github.com/databricks/databricks-sdk-go/service/jobs.SqlTaskQuery: - query_id: - description: The canonical identifier of the SQL query. + "query_id": + "description": |- + The canonical identifier of the SQL query. github.com/databricks/databricks-sdk-go/service/jobs.SqlTaskSubscription: - destination_id: - description: The canonical identifier of the destination to receive email notification. - This parameter is mutually exclusive with user_name. You cannot set both destination_id - and user_name for subscription notifications. - user_name: - description: The user name to receive the subscription email. This parameter is - mutually exclusive with destination_id. You cannot set both destination_id and - user_name for subscription notifications. + "destination_id": + "description": |- + The canonical identifier of the destination to receive email notification. This parameter is mutually exclusive with user_name. You cannot set both destination_id and user_name for subscription notifications. + "user_name": + "description": |- + The user name to receive the subscription email. This parameter is mutually exclusive with destination_id. You cannot set both destination_id and user_name for subscription notifications. github.com/databricks/databricks-sdk-go/service/jobs.TableUpdateTriggerConfiguration: - condition: - description: The table(s) condition based on which to trigger a job run. - min_time_between_triggers_seconds: - description: |- + "condition": + "description": |- + The table(s) condition based on which to trigger a job run. + "min_time_between_triggers_seconds": + "description": |- If set, the trigger starts a run only after the specified amount of time has passed since the last time the trigger fired. The minimum allowed value is 60 seconds. - table_names: - description: A list of Delta tables to monitor for changes. The table name must - be in the format `catalog_name.schema_name.table_name`. - wait_after_last_change_seconds: - description: |- + "table_names": + "description": |- + A list of Delta tables to monitor for changes. The table name must be in the format `catalog_name.schema_name.table_name`. + "wait_after_last_change_seconds": + "description": |- If set, the trigger starts a run only after no table updates have occurred for the specified time and can be used to wait for a series of table updates before triggering a run. The minimum allowed value is 60 seconds. github.com/databricks/databricks-sdk-go/service/jobs.Task: - clean_rooms_notebook_task: - description: |- + "clean_rooms_notebook_task": + "description": |- The task runs a [clean rooms](https://docs.databricks.com/en/clean-rooms/index.html) notebook when the `clean_rooms_notebook_task` field is present. - condition_task: - description: |- + "condition_task": + "description": |- The task evaluates a condition that can be used to control the execution of other tasks when the `condition_task` field is present. The condition task does not require a cluster to execute and does not support retries or notifications. - dbt_task: - description: The task runs one or more dbt commands when the `dbt_task` field - is present. The dbt task requires both Databricks SQL and the ability to use - a serverless or a pro SQL warehouse. - depends_on: - description: |- + "dbt_task": + "description": |- + The task runs one or more dbt commands when the `dbt_task` field is present. The dbt task requires both Databricks SQL and the ability to use a serverless or a pro SQL warehouse. + "depends_on": + "description": |- An optional array of objects specifying the dependency graph of the task. All tasks specified in this field must complete before executing this task. The task will run only if the `run_if` condition is true. The key is `task_key`, and the value is the name assigned to the dependent task. - description: - description: An optional description for this task. - disable_auto_optimization: - description: An option to disable auto optimization in serverless - email_notifications: - description: An optional set of email addresses that is notified when runs of - this task begin or complete as well as when this task is deleted. The default - behavior is to not send any emails. - environment_key: - description: The key that references an environment spec in a job. This field - is required for Python script, Python wheel and dbt tasks when using serverless - compute. - existing_cluster_id: - description: |- + "description": + "description": |- + An optional description for this task. + "disable_auto_optimization": + "description": |- + An option to disable auto optimization in serverless + "email_notifications": + "description": |- + An optional set of email addresses that is notified when runs of this task begin or complete as well as when this task is deleted. The default behavior is to not send any emails. + "environment_key": + "description": |- + The key that references an environment spec in a job. This field is required for Python script, Python wheel and dbt tasks when using serverless compute. + "existing_cluster_id": + "description": |- If existing_cluster_id, the ID of an existing cluster that is used for all runs. When running jobs or tasks on an existing cluster, you may need to manually restart the cluster if it stops responding. We suggest running jobs and tasks on new clusters for greater reliability - for_each_task: - description: The task executes a nested task for every input provided when the - `for_each_task` field is present. - health: {} - job_cluster_key: - description: If job_cluster_key, this task is executed reusing the cluster specified - in `job.settings.job_clusters`. - libraries: - description: |- + "for_each_task": + "description": |- + The task executes a nested task for every input provided when the `for_each_task` field is present. + "health": {} + "job_cluster_key": + "description": |- + If job_cluster_key, this task is executed reusing the cluster specified in `job.settings.job_clusters`. + "libraries": + "description": |- An optional list of libraries to be installed on the cluster. The default value is an empty list. - max_retries: - description: An optional maximum number of times to retry an unsuccessful run. - A run is considered to be unsuccessful if it completes with the `FAILED` result_state - or `INTERNAL_ERROR` `life_cycle_state`. The value `-1` means to retry indefinitely - and the value `0` means to never retry. - min_retry_interval_millis: - description: An optional minimal interval in milliseconds between the start of - the failed run and the subsequent retry run. The default behavior is that unsuccessful - runs are immediately retried. - new_cluster: - description: If new_cluster, a description of a new cluster that is created for - each run. - notebook_task: - description: The task runs a notebook when the `notebook_task` field is present. - notification_settings: - description: Optional notification settings that are used when sending notifications - to each of the `email_notifications` and `webhook_notifications` for this task. - pipeline_task: - description: The task triggers a pipeline update when the `pipeline_task` field - is present. Only pipelines configured to use triggered more are supported. - python_wheel_task: - description: The task runs a Python wheel when the `python_wheel_task` field is - present. - retry_on_timeout: - description: |- + "max_retries": + "description": |- + An optional maximum number of times to retry an unsuccessful run. A run is considered to be unsuccessful if it completes with the `FAILED` result_state or `INTERNAL_ERROR` `life_cycle_state`. The value `-1` means to retry indefinitely and the value `0` means to never retry. + "min_retry_interval_millis": + "description": |- + An optional minimal interval in milliseconds between the start of the failed run and the subsequent retry run. The default behavior is that unsuccessful runs are immediately retried. + "new_cluster": + "description": |- + If new_cluster, a description of a new cluster that is created for each run. + "notebook_task": + "description": |- + The task runs a notebook when the `notebook_task` field is present. + "notification_settings": + "description": |- + Optional notification settings that are used when sending notifications to each of the `email_notifications` and `webhook_notifications` for this task. + "pipeline_task": + "description": |- + The task triggers a pipeline update when the `pipeline_task` field is present. Only pipelines configured to use triggered more are supported. + "python_wheel_task": + "description": |- + The task runs a Python wheel when the `python_wheel_task` field is present. + "retry_on_timeout": + "description": |- An optional policy to specify whether to retry a job when it times out. The default behavior is to not retry on timeout. - run_if: - description: |- + "run_if": + "description": |- An optional value specifying the condition determining whether the task is run once its dependencies have been completed. * `ALL_SUCCESS`: All dependencies have executed and succeeded @@ -1775,15 +1932,17 @@ github.com/databricks/databricks-sdk-go/service/jobs.Task: * `ALL_DONE`: All dependencies have been completed * `AT_LEAST_ONE_FAILED`: At least one dependency failed * `ALL_FAILED`: ALl dependencies have failed - run_job_task: - description: The task triggers another job when the `run_job_task` field is present. - spark_jar_task: - description: The task runs a JAR when the `spark_jar_task` field is present. - spark_python_task: - description: The task runs a Python file when the `spark_python_task` field is - present. - spark_submit_task: - description: |- + "run_job_task": + "description": |- + The task triggers another job when the `run_job_task` field is present. + "spark_jar_task": + "description": |- + The task runs a JAR when the `spark_jar_task` field is present. + "spark_python_task": + "description": |- + The task runs a Python file when the `spark_python_task` field is present. + "spark_submit_task": + "description": |- (Legacy) The task runs the spark-submit script when the `spark_submit_task` field is present. This task can run only on new clusters and is not compatible with serverless compute. In the `new_cluster` specification, `libraries` and `spark_conf` are not supported. Instead, use `--jars` and `--py-files` to add Java and Python libraries and `--conf` to set the Spark configurations. @@ -1793,220 +1952,238 @@ github.com/databricks/databricks-sdk-go/service/jobs.Task: By default, the Spark submit job uses all available memory (excluding reserved memory for Databricks services). You can set `--driver-memory`, and `--executor-memory` to a smaller value to leave some room for off-heap usage. The `--jars`, `--py-files`, `--files` arguments support DBFS and S3 paths. - sql_task: - description: The task runs a SQL query or file, or it refreshes a SQL alert or - a legacy SQL dashboard when the `sql_task` field is present. - task_key: - description: |- + "sql_task": + "description": |- + The task runs a SQL query or file, or it refreshes a SQL alert or a legacy SQL dashboard when the `sql_task` field is present. + "task_key": + "description": |- A unique name for the task. This field is used to refer to this task from other tasks. This field is required and must be unique within its parent job. On Update or Reset, this field is used to reference the tasks to be updated or reset. - timeout_seconds: - description: An optional timeout applied to each run of this job task. A value - of `0` means no timeout. - webhook_notifications: - description: A collection of system notification IDs to notify when runs of this - task begin or complete. The default behavior is to not send any system notifications. + "timeout_seconds": + "description": |- + An optional timeout applied to each run of this job task. A value of `0` means no timeout. + "webhook_notifications": + "description": |- + A collection of system notification IDs to notify when runs of this task begin or complete. The default behavior is to not send any system notifications. github.com/databricks/databricks-sdk-go/service/jobs.TaskDependency: - outcome: - description: Can only be specified on condition task dependencies. The outcome - of the dependent task that must be met for this task to run. - task_key: - description: The name of the task this task depends on. + "outcome": + "description": |- + Can only be specified on condition task dependencies. The outcome of the dependent task that must be met for this task to run. + "task_key": + "description": |- + The name of the task this task depends on. github.com/databricks/databricks-sdk-go/service/jobs.TaskEmailNotifications: - no_alert_for_skipped_runs: - description: |- + "no_alert_for_skipped_runs": + "description": |- If true, do not send email to recipients specified in `on_failure` if the run is skipped. This field is `deprecated`. Please use the `notification_settings.no_alert_for_skipped_runs` field. - on_duration_warning_threshold_exceeded: - description: A list of email addresses to be notified when the duration of a run - exceeds the threshold specified for the `RUN_DURATION_SECONDS` metric in the - `health` field. If no rule for the `RUN_DURATION_SECONDS` metric is specified - in the `health` field for the job, notifications are not sent. - on_failure: - description: A list of email addresses to be notified when a run unsuccessfully - completes. A run is considered to have completed unsuccessfully if it ends with - an `INTERNAL_ERROR` `life_cycle_state` or a `FAILED`, or `TIMED_OUT` result_state. - If this is not specified on job creation, reset, or update the list is empty, - and notifications are not sent. - on_start: - description: A list of email addresses to be notified when a run begins. If not - specified on job creation, reset, or update, the list is empty, and notifications - are not sent. - on_streaming_backlog_exceeded: - description: |- + "on_duration_warning_threshold_exceeded": + "description": |- + A list of email addresses to be notified when the duration of a run exceeds the threshold specified for the `RUN_DURATION_SECONDS` metric in the `health` field. If no rule for the `RUN_DURATION_SECONDS` metric is specified in the `health` field for the job, notifications are not sent. + "on_failure": + "description": |- + A list of email addresses to be notified when a run unsuccessfully completes. A run is considered to have completed unsuccessfully if it ends with an `INTERNAL_ERROR` `life_cycle_state` or a `FAILED`, or `TIMED_OUT` result_state. If this is not specified on job creation, reset, or update the list is empty, and notifications are not sent. + "on_start": + "description": |- + A list of email addresses to be notified when a run begins. If not specified on job creation, reset, or update, the list is empty, and notifications are not sent. + "on_streaming_backlog_exceeded": + "description": |- A list of email addresses to notify when any streaming backlog thresholds are exceeded for any stream. Streaming backlog thresholds can be set in the `health` field using the following metrics: `STREAMING_BACKLOG_BYTES`, `STREAMING_BACKLOG_RECORDS`, `STREAMING_BACKLOG_SECONDS`, or `STREAMING_BACKLOG_FILES`. Alerting is based on the 10-minute average of these metrics. If the issue persists, notifications are resent every 30 minutes. - on_success: - description: A list of email addresses to be notified when a run successfully - completes. A run is considered to have completed successfully if it ends with - a `TERMINATED` `life_cycle_state` and a `SUCCESS` result_state. If not specified - on job creation, reset, or update, the list is empty, and notifications are - not sent. + "on_success": + "description": |- + A list of email addresses to be notified when a run successfully completes. A run is considered to have completed successfully if it ends with a `TERMINATED` `life_cycle_state` and a `SUCCESS` result_state. If not specified on job creation, reset, or update, the list is empty, and notifications are not sent. github.com/databricks/databricks-sdk-go/service/jobs.TaskNotificationSettings: - alert_on_last_attempt: - description: If true, do not send notifications to recipients specified in `on_start` - for the retried runs and do not send notifications to recipients specified in - `on_failure` until the last retry of the run. - no_alert_for_canceled_runs: - description: If true, do not send notifications to recipients specified in `on_failure` - if the run is canceled. - no_alert_for_skipped_runs: - description: If true, do not send notifications to recipients specified in `on_failure` - if the run is skipped. + "alert_on_last_attempt": + "description": |- + If true, do not send notifications to recipients specified in `on_start` for the retried runs and do not send notifications to recipients specified in `on_failure` until the last retry of the run. + "no_alert_for_canceled_runs": + "description": |- + If true, do not send notifications to recipients specified in `on_failure` if the run is canceled. + "no_alert_for_skipped_runs": + "description": |- + If true, do not send notifications to recipients specified in `on_failure` if the run is skipped. github.com/databricks/databricks-sdk-go/service/jobs.TriggerSettings: - file_arrival: - description: File arrival trigger settings. - pause_status: - description: Whether this trigger is paused or not. - periodic: - description: Periodic trigger settings. - table: - description: Old table trigger settings name. Deprecated in favor of `table_update`. - table_update: {} + "file_arrival": + "description": |- + File arrival trigger settings. + "pause_status": + "description": |- + Whether this trigger is paused or not. + "periodic": + "description": |- + Periodic trigger settings. + "table": + "description": |- + Old table trigger settings name. Deprecated in favor of `table_update`. + "table_update": {} github.com/databricks/databricks-sdk-go/service/jobs.Webhook: - id: {} + "id": {} github.com/databricks/databricks-sdk-go/service/jobs.WebhookNotifications: - on_duration_warning_threshold_exceeded: - description: An optional list of system notification IDs to call when the duration - of a run exceeds the threshold specified for the `RUN_DURATION_SECONDS` metric - in the `health` field. A maximum of 3 destinations can be specified for the - `on_duration_warning_threshold_exceeded` property. - on_failure: - description: An optional list of system notification IDs to call when the run - fails. A maximum of 3 destinations can be specified for the `on_failure` property. - on_start: - description: An optional list of system notification IDs to call when the run - starts. A maximum of 3 destinations can be specified for the `on_start` property. - on_streaming_backlog_exceeded: - description: |- + "on_duration_warning_threshold_exceeded": + "description": |- + An optional list of system notification IDs to call when the duration of a run exceeds the threshold specified for the `RUN_DURATION_SECONDS` metric in the `health` field. A maximum of 3 destinations can be specified for the `on_duration_warning_threshold_exceeded` property. + "on_failure": + "description": |- + An optional list of system notification IDs to call when the run fails. A maximum of 3 destinations can be specified for the `on_failure` property. + "on_start": + "description": |- + An optional list of system notification IDs to call when the run starts. A maximum of 3 destinations can be specified for the `on_start` property. + "on_streaming_backlog_exceeded": + "description": |- An optional list of system notification IDs to call when any streaming backlog thresholds are exceeded for any stream. Streaming backlog thresholds can be set in the `health` field using the following metrics: `STREAMING_BACKLOG_BYTES`, `STREAMING_BACKLOG_RECORDS`, `STREAMING_BACKLOG_SECONDS`, or `STREAMING_BACKLOG_FILES`. Alerting is based on the 10-minute average of these metrics. If the issue persists, notifications are resent every 30 minutes. A maximum of 3 destinations can be specified for the `on_streaming_backlog_exceeded` property. - on_success: - description: An optional list of system notification IDs to call when the run - completes successfully. A maximum of 3 destinations can be specified for the - `on_success` property. + "on_success": + "description": |- + An optional list of system notification IDs to call when the run completes successfully. A maximum of 3 destinations can be specified for the `on_success` property. github.com/databricks/databricks-sdk-go/service/ml.ExperimentTag: - key: - description: The tag key. - value: - description: The tag value. + "key": + "description": |- + The tag key. + "value": + "description": |- + The tag value. github.com/databricks/databricks-sdk-go/service/ml.ModelTag: - key: - description: The tag key. - value: - description: The tag value. + "key": + "description": |- + The tag key. + "value": + "description": |- + The tag value. github.com/databricks/databricks-sdk-go/service/ml.ModelVersion: - creation_timestamp: - description: Timestamp recorded when this `model_version` was created. - current_stage: - description: Current stage for this `model_version`. - description: - description: Description of this `model_version`. - last_updated_timestamp: - description: Timestamp recorded when metadata for this `model_version` was last - updated. - name: - description: Unique name of the model - run_id: - description: |- + "creation_timestamp": + "description": |- + Timestamp recorded when this `model_version` was created. + "current_stage": + "description": |- + Current stage for this `model_version`. + "description": + "description": |- + Description of this `model_version`. + "last_updated_timestamp": + "description": |- + Timestamp recorded when metadata for this `model_version` was last updated. + "name": + "description": |- + Unique name of the model + "run_id": + "description": |- MLflow run ID used when creating `model_version`, if `source` was generated by an experiment run stored in MLflow tracking server. - run_link: - description: 'Run Link: Direct link to the run that generated this version' - source: - description: URI indicating the location of the source model artifacts, used when - creating `model_version` - status: - description: Current status of `model_version` - status_message: - description: Details on current `status`, if it is pending or failed. - tags: - description: 'Tags: Additional metadata key-value pairs for this `model_version`.' - user_id: - description: User that created this `model_version`. - version: - description: Model's version number. + "run_link": + "description": |- + Run Link: Direct link to the run that generated this version + "source": + "description": |- + URI indicating the location of the source model artifacts, used when creating `model_version` + "status": + "description": |- + Current status of `model_version` + "status_message": + "description": |- + Details on current `status`, if it is pending or failed. + "tags": + "description": |- + Tags: Additional metadata key-value pairs for this `model_version`. + "user_id": + "description": |- + User that created this `model_version`. + "version": + "description": |- + Model's version number. github.com/databricks/databricks-sdk-go/service/ml.ModelVersionStatus: - _: - description: Current status of `model_version` - enum: - - PENDING_REGISTRATION - - FAILED_REGISTRATION - - READY + "_": + "description": |- + Current status of `model_version` + "enum": + - |- + PENDING_REGISTRATION + - |- + FAILED_REGISTRATION + - |- + READY github.com/databricks/databricks-sdk-go/service/ml.ModelVersionTag: - key: - description: The tag key. - value: - description: The tag value. + "key": + "description": |- + The tag key. + "value": + "description": |- + The tag value. github.com/databricks/databricks-sdk-go/service/pipelines.CronTrigger: - quartz_cron_schedule: {} - timezone_id: {} + "quartz_cron_schedule": {} + "timezone_id": {} github.com/databricks/databricks-sdk-go/service/pipelines.DeploymentKind: - _: - description: | + "_": + "description": | The deployment method that manages the pipeline: - BUNDLE: The pipeline is managed by a Databricks Asset Bundle. - enum: - - BUNDLE + "enum": + - |- + BUNDLE github.com/databricks/databricks-sdk-go/service/pipelines.FileLibrary: - path: - description: The absolute path of the file. + "path": + "description": |- + The absolute path of the file. github.com/databricks/databricks-sdk-go/service/pipelines.Filters: - exclude: - description: Paths to exclude. - include: - description: Paths to include. + "exclude": + "description": |- + Paths to exclude. + "include": + "description": |- + Paths to include. github.com/databricks/databricks-sdk-go/service/pipelines.IngestionConfig: - report: - description: Select a specific source report. - schema: - description: Select all tables from a specific source schema. - table: - description: Select a specific source table. + "report": + "description": |- + Select a specific source report. + "schema": + "description": |- + Select all tables from a specific source schema. + "table": + "description": |- + Select a specific source table. github.com/databricks/databricks-sdk-go/service/pipelines.IngestionGatewayPipelineDefinition: - connection_id: - description: '[Deprecated, use connection_name instead] Immutable. The Unity Catalog - connection that this gateway pipeline uses to communicate with the source.' - connection_name: - description: Immutable. The Unity Catalog connection that this gateway pipeline - uses to communicate with the source. - gateway_storage_catalog: - description: Required, Immutable. The name of the catalog for the gateway pipeline's - storage location. - gateway_storage_name: - description: | + "connection_id": + "description": |- + [Deprecated, use connection_name instead] Immutable. The Unity Catalog connection that this gateway pipeline uses to communicate with the source. + "connection_name": + "description": |- + Immutable. The Unity Catalog connection that this gateway pipeline uses to communicate with the source. + "gateway_storage_catalog": + "description": |- + Required, Immutable. The name of the catalog for the gateway pipeline's storage location. + "gateway_storage_name": + "description": | Optional. The Unity Catalog-compatible name for the gateway storage location. This is the destination to use for the data that is extracted by the gateway. Delta Live Tables system will automatically create the storage location under the catalog and schema. - gateway_storage_schema: - description: Required, Immutable. The name of the schema for the gateway pipelines's - storage location. + "gateway_storage_schema": + "description": |- + Required, Immutable. The name of the schema for the gateway pipelines's storage location. github.com/databricks/databricks-sdk-go/service/pipelines.IngestionPipelineDefinition: - connection_name: - description: Immutable. The Unity Catalog connection that this ingestion pipeline - uses to communicate with the source. This is used with connectors for applications - like Salesforce, Workday, and so on. - ingestion_gateway_id: - description: Immutable. Identifier for the gateway that is used by this ingestion - pipeline to communicate with the source database. This is used with connectors - to databases like SQL Server. - objects: - description: Required. Settings specifying tables to replicate and the destination - for the replicated tables. - table_configuration: - description: Configuration settings to control the ingestion of tables. These - settings are applied to all tables in the pipeline. + "connection_name": + "description": |- + Immutable. The Unity Catalog connection that this ingestion pipeline uses to communicate with the source. This is used with connectors for applications like Salesforce, Workday, and so on. + "ingestion_gateway_id": + "description": |- + Immutable. Identifier for the gateway that is used by this ingestion pipeline to communicate with the source database. This is used with connectors to databases like SQL Server. + "objects": + "description": |- + Required. Settings specifying tables to replicate and the destination for the replicated tables. + "table_configuration": + "description": |- + Configuration settings to control the ingestion of tables. These settings are applied to all tables in the pipeline. github.com/databricks/databricks-sdk-go/service/pipelines.ManualTrigger: {} github.com/databricks/databricks-sdk-go/service/pipelines.NotebookLibrary: - path: - description: The absolute path of the notebook. + "path": + "description": |- + The absolute path of the notebook. github.com/databricks/databricks-sdk-go/service/pipelines.Notifications: - alerts: - description: | + "alerts": + "description": | A list of alerts that trigger the sending of notifications to the configured destinations. The supported alerts are: @@ -2014,74 +2191,74 @@ github.com/databricks/databricks-sdk-go/service/pipelines.Notifications: * `on-update-failure`: Each time a pipeline update fails. * `on-update-fatal-failure`: A pipeline update fails with a non-retryable (fatal) error. * `on-flow-failure`: A single data flow fails. - email_recipients: - description: | + "email_recipients": + "description": | A list of email addresses notified when a configured alert is triggered. github.com/databricks/databricks-sdk-go/service/pipelines.PipelineCluster: - apply_policy_default_values: - description: 'Note: This field won''t be persisted. Only API users will check - this field.' - autoscale: - description: |- + "apply_policy_default_values": + "description": |- + Note: This field won't be persisted. Only API users will check this field. + "autoscale": + "description": |- Parameters needed in order to automatically scale clusters up and down based on load. Note: autoscaling works best with DB runtime versions 3.0 or later. - aws_attributes: - description: |- + "aws_attributes": + "description": |- Attributes related to clusters running on Amazon Web Services. If not specified at cluster creation, a set of default values will be used. - azure_attributes: - description: |- + "azure_attributes": + "description": |- Attributes related to clusters running on Microsoft Azure. If not specified at cluster creation, a set of default values will be used. - cluster_log_conf: - description: | + "cluster_log_conf": + "description": | The configuration for delivering spark logs to a long-term storage destination. Only dbfs destinations are supported. Only one destination can be specified for one cluster. If the conf is given, the logs will be delivered to the destination every `5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while the destination of executor logs is `$destination/$clusterId/executor`. - custom_tags: - description: |- + "custom_tags": + "description": |- Additional tags for cluster resources. Databricks will tag all cluster resources (e.g., AWS instances and EBS volumes) with these tags in addition to `default_tags`. Notes: - Currently, Databricks allows at most 45 custom tags - Clusters can only reuse cloud resources if the resources' tags are a subset of the cluster tags - driver_instance_pool_id: - description: |- + "driver_instance_pool_id": + "description": |- The optional ID of the instance pool for the driver of the cluster belongs. The pool cluster uses the instance pool with id (instance_pool_id) if the driver pool is not assigned. - driver_node_type_id: - description: |- + "driver_node_type_id": + "description": |- The node type of the Spark driver. Note that this field is optional; if unset, the driver node type will be set as the same value as `node_type_id` defined above. - enable_local_disk_encryption: - description: Whether to enable local disk encryption for the cluster. - gcp_attributes: - description: |- + "enable_local_disk_encryption": + "description": |- + Whether to enable local disk encryption for the cluster. + "gcp_attributes": + "description": |- Attributes related to clusters running on Google Cloud Platform. If not specified at cluster creation, a set of default values will be used. - init_scripts: - description: The configuration for storing init scripts. Any number of destinations - can be specified. The scripts are executed sequentially in the order provided. - If `cluster_log_conf` is specified, init script logs are sent to `//init_scripts`. - instance_pool_id: - description: The optional ID of the instance pool to which the cluster belongs. - label: - description: A label for the cluster specification, either `default` to configure - the default cluster, or `maintenance` to configure the maintenance cluster. - This field is optional. The default value is `default`. - node_type_id: - description: | + "init_scripts": + "description": |- + The configuration for storing init scripts. Any number of destinations can be specified. The scripts are executed sequentially in the order provided. If `cluster_log_conf` is specified, init script logs are sent to `//init_scripts`. + "instance_pool_id": + "description": |- + The optional ID of the instance pool to which the cluster belongs. + "label": + "description": |- + A label for the cluster specification, either `default` to configure the default cluster, or `maintenance` to configure the maintenance cluster. This field is optional. The default value is `default`. + "node_type_id": + "description": | This field encodes, through a single value, the resources available to each of the Spark nodes in this cluster. For example, the Spark nodes can be provisioned and optimized for memory or compute intensive workloads. A list of available node types can be retrieved by using the :method:clusters/listNodeTypes API call. - num_workers: - description: |- + "num_workers": + "description": |- Number of worker nodes that this cluster should have. A cluster has one Spark Driver and `num_workers` Executors for a total of `num_workers` + 1 Spark nodes. @@ -2090,14 +2267,15 @@ github.com/databricks/databricks-sdk-go/service/pipelines.PipelineCluster: is resized from 5 to 10 workers, this field will immediately be updated to reflect the target size of 10 workers, whereas the workers listed in `spark_info` will gradually increase from 5 to 10 as the new nodes are provisioned. - policy_id: - description: The ID of the cluster policy used to create the cluster if applicable. - spark_conf: - description: | + "policy_id": + "description": |- + The ID of the cluster policy used to create the cluster if applicable. + "spark_conf": + "description": | An object containing a set of optional, user-specified Spark configuration key-value pairs. See :method:clusters/create for more details. - spark_env_vars: - description: |- + "spark_env_vars": + "description": |- An object containing a set of optional, user-specified environment variable key-value pairs. Please note that key-value pair of the form (X,Y) will be exported as is (i.e., `export X='Y'`) while launching the driver and workers. @@ -2109,523 +2287,638 @@ github.com/databricks/databricks-sdk-go/service/pipelines.PipelineCluster: Example Spark environment variables: `{"SPARK_WORKER_MEMORY": "28000m", "SPARK_LOCAL_DIRS": "/local_disk0"}` or `{"SPARK_DAEMON_JAVA_OPTS": "$SPARK_DAEMON_JAVA_OPTS -Dspark.shuffle.service.enabled=true"}` - ssh_public_keys: - description: |- + "ssh_public_keys": + "description": |- SSH public key contents that will be added to each Spark node in this cluster. The corresponding private keys can be used to login with the user name `ubuntu` on port `2200`. Up to 10 keys can be specified. github.com/databricks/databricks-sdk-go/service/pipelines.PipelineClusterAutoscale: - max_workers: - description: The maximum number of workers to which the cluster can scale up when - overloaded. `max_workers` must be strictly greater than `min_workers`. - min_workers: - description: |- + "max_workers": + "description": |- + The maximum number of workers to which the cluster can scale up when overloaded. `max_workers` must be strictly greater than `min_workers`. + "min_workers": + "description": |- The minimum number of workers the cluster can scale down to when underutilized. It is also the initial number of workers the cluster will have after creation. - mode: - description: | + "mode": + "description": | Databricks Enhanced Autoscaling optimizes cluster utilization by automatically allocating cluster resources based on workload volume, with minimal impact to the data processing latency of your pipelines. Enhanced Autoscaling is available for `updates` clusters only. The legacy autoscaling feature is used for `maintenance` clusters. github.com/databricks/databricks-sdk-go/service/pipelines.PipelineClusterAutoscaleMode: - _: - description: | + "_": + "description": | Databricks Enhanced Autoscaling optimizes cluster utilization by automatically allocating cluster resources based on workload volume, with minimal impact to the data processing latency of your pipelines. Enhanced Autoscaling is available for `updates` clusters only. The legacy autoscaling feature is used for `maintenance` clusters. - enum: - - ENHANCED - - LEGACY + "enum": + - |- + ENHANCED + - |- + LEGACY github.com/databricks/databricks-sdk-go/service/pipelines.PipelineDeployment: - kind: - description: The deployment method that manages the pipeline. - metadata_file_path: - description: The path to the file containing metadata about the deployment. + "kind": + "description": |- + The deployment method that manages the pipeline. + "metadata_file_path": + "description": |- + The path to the file containing metadata about the deployment. github.com/databricks/databricks-sdk-go/service/pipelines.PipelineLibrary: - file: - description: | + "file": + "description": | The path to a file that defines a pipeline and is stored in the Databricks Repos. - jar: - description: | + "jar": + "description": | URI of the jar to be installed. Currently only DBFS is supported. - maven: - description: | + "maven": + "description": | Specification of a maven library to be installed. - notebook: - description: | + "notebook": + "description": | The path to a notebook that defines a pipeline and is stored in the Databricks workspace. - whl: - description: URI of the whl to be installed. + "whl": + "description": |- + URI of the whl to be installed. github.com/databricks/databricks-sdk-go/service/pipelines.PipelineTrigger: - cron: {} - manual: {} + "cron": {} + "manual": {} github.com/databricks/databricks-sdk-go/service/pipelines.ReportSpec: - destination_catalog: - description: Required. Destination catalog to store table. - destination_schema: - description: Required. Destination schema to store table. - destination_table: - description: Required. Destination table name. The pipeline fails if a table with - that name already exists. - source_url: - description: Required. Report URL in the source system. - table_configuration: - description: Configuration settings to control the ingestion of tables. These - settings override the table_configuration defined in the IngestionPipelineDefinition - object. + "destination_catalog": + "description": |- + Required. Destination catalog to store table. + "destination_schema": + "description": |- + Required. Destination schema to store table. + "destination_table": + "description": |- + Required. Destination table name. The pipeline fails if a table with that name already exists. + "source_url": + "description": |- + Required. Report URL in the source system. + "table_configuration": + "description": |- + Configuration settings to control the ingestion of tables. These settings override the table_configuration defined in the IngestionPipelineDefinition object. github.com/databricks/databricks-sdk-go/service/pipelines.RestartWindow: - days_of_week: - description: |- + "days_of_week": + "description": |- Days of week in which the restart is allowed to happen (within a five-hour window starting at start_hour). If not specified all days of the week will be used. - start_hour: - description: |- + "start_hour": + "description": |- An integer between 0 and 23 denoting the start hour for the restart window in the 24-hour day. Continuous pipeline restart is triggered only within a five-hour window starting at this hour. - time_zone_id: - description: |- + "time_zone_id": + "description": |- Time zone id of restart window. See https://docs.databricks.com/sql/language-manual/sql-ref-syntax-aux-conf-mgmt-set-timezone.html for details. If not specified, UTC will be used. github.com/databricks/databricks-sdk-go/service/pipelines.RestartWindowDaysOfWeek: - _: - description: |- + "_": + "description": |- Days of week in which the restart is allowed to happen (within a five-hour window starting at start_hour). If not specified all days of the week will be used. - enum: - - MONDAY - - TUESDAY - - WEDNESDAY - - THURSDAY - - FRIDAY - - SATURDAY - - SUNDAY + "enum": + - |- + MONDAY + - |- + TUESDAY + - |- + WEDNESDAY + - |- + THURSDAY + - |- + FRIDAY + - |- + SATURDAY + - |- + SUNDAY github.com/databricks/databricks-sdk-go/service/pipelines.SchemaSpec: - destination_catalog: - description: Required. Destination catalog to store tables. - destination_schema: - description: Required. Destination schema to store tables in. Tables with the - same name as the source tables are created in this destination schema. The pipeline - fails If a table with the same name already exists. - source_catalog: - description: The source catalog name. Might be optional depending on the type - of source. - source_schema: - description: Required. Schema name in the source database. - table_configuration: - description: Configuration settings to control the ingestion of tables. These - settings are applied to all tables in this schema and override the table_configuration - defined in the IngestionPipelineDefinition object. + "destination_catalog": + "description": |- + Required. Destination catalog to store tables. + "destination_schema": + "description": |- + Required. Destination schema to store tables in. Tables with the same name as the source tables are created in this destination schema. The pipeline fails If a table with the same name already exists. + "source_catalog": + "description": |- + The source catalog name. Might be optional depending on the type of source. + "source_schema": + "description": |- + Required. Schema name in the source database. + "table_configuration": + "description": |- + Configuration settings to control the ingestion of tables. These settings are applied to all tables in this schema and override the table_configuration defined in the IngestionPipelineDefinition object. github.com/databricks/databricks-sdk-go/service/pipelines.TableSpec: - destination_catalog: - description: Required. Destination catalog to store table. - destination_schema: - description: Required. Destination schema to store table. - destination_table: - description: Optional. Destination table name. The pipeline fails if a table with - that name already exists. If not set, the source table name is used. - source_catalog: - description: Source catalog name. Might be optional depending on the type of source. - source_schema: - description: Schema name in the source database. Might be optional depending on - the type of source. - source_table: - description: Required. Table name in the source database. - table_configuration: - description: Configuration settings to control the ingestion of tables. These - settings override the table_configuration defined in the IngestionPipelineDefinition - object and the SchemaSpec. + "destination_catalog": + "description": |- + Required. Destination catalog to store table. + "destination_schema": + "description": |- + Required. Destination schema to store table. + "destination_table": + "description": |- + Optional. Destination table name. The pipeline fails if a table with that name already exists. If not set, the source table name is used. + "source_catalog": + "description": |- + Source catalog name. Might be optional depending on the type of source. + "source_schema": + "description": |- + Schema name in the source database. Might be optional depending on the type of source. + "source_table": + "description": |- + Required. Table name in the source database. + "table_configuration": + "description": |- + Configuration settings to control the ingestion of tables. These settings override the table_configuration defined in the IngestionPipelineDefinition object and the SchemaSpec. github.com/databricks/databricks-sdk-go/service/pipelines.TableSpecificConfig: - primary_keys: - description: The primary key of the table used to apply changes. - salesforce_include_formula_fields: - description: If true, formula fields defined in the table are included in the - ingestion. This setting is only valid for the Salesforce connector - scd_type: - description: The SCD type to use to ingest the table. - sequence_by: - description: The column names specifying the logical order of events in the source - data. Delta Live Tables uses this sequencing to handle change events that arrive - out of order. + "primary_keys": + "description": |- + The primary key of the table used to apply changes. + "salesforce_include_formula_fields": + "description": |- + If true, formula fields defined in the table are included in the ingestion. This setting is only valid for the Salesforce connector + "scd_type": + "description": |- + The SCD type to use to ingest the table. + "sequence_by": + "description": |- + The column names specifying the logical order of events in the source data. Delta Live Tables uses this sequencing to handle change events that arrive out of order. github.com/databricks/databricks-sdk-go/service/pipelines.TableSpecificConfigScdType: - _: - description: The SCD type to use to ingest the table. - enum: - - SCD_TYPE_1 - - SCD_TYPE_2 + "_": + "description": |- + The SCD type to use to ingest the table. + "enum": + - |- + SCD_TYPE_1 + - |- + SCD_TYPE_2 +github.com/databricks/databricks-sdk-go/service/serving.Ai21LabsConfig: + "ai21labs_api_key": + "description": |- + The Databricks secret key reference for an AI21 Labs API key. If you prefer to paste your API key directly, see `ai21labs_api_key_plaintext`. You must provide an API key using one of the following fields: `ai21labs_api_key` or `ai21labs_api_key_plaintext`. + "ai21labs_api_key_plaintext": + "description": |- + An AI21 Labs API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `ai21labs_api_key`. You must provide an API key using one of the following fields: `ai21labs_api_key` or `ai21labs_api_key_plaintext`. github.com/databricks/databricks-sdk-go/service/serving.AiGatewayConfig: - guardrails: - description: Configuration for AI Guardrails to prevent unwanted data and unsafe - data in requests and responses. - inference_table_config: - description: Configuration for payload logging using inference tables. Use these - tables to monitor and audit data being sent to and received from model APIs - and to improve model quality. - rate_limits: - description: Configuration for rate limits which can be set to limit endpoint - traffic. - usage_tracking_config: - description: Configuration to enable usage tracking using system tables. These - tables allow you to monitor operational usage on endpoints and their associated - costs. + "guardrails": + "description": |- + Configuration for AI Guardrails to prevent unwanted data and unsafe data in requests and responses. + "inference_table_config": + "description": |- + Configuration for payload logging using inference tables. Use these tables to monitor and audit data being sent to and received from model APIs and to improve model quality. + "rate_limits": + "description": |- + Configuration for rate limits which can be set to limit endpoint traffic. + "usage_tracking_config": + "description": |- + Configuration to enable usage tracking using system tables. These tables allow you to monitor operational usage on endpoints and their associated costs. github.com/databricks/databricks-sdk-go/service/serving.AiGatewayGuardrailParameters: - invalid_keywords: - description: List of invalid keywords. AI guardrail uses keyword or string matching - to decide if the keyword exists in the request or response content. - pii: - description: Configuration for guardrail PII filter. - safety: - description: Indicates whether the safety filter is enabled. - valid_topics: - description: The list of allowed topics. Given a chat request, this guardrail - flags the request if its topic is not in the allowed topics. + "invalid_keywords": + "description": |- + List of invalid keywords. AI guardrail uses keyword or string matching to decide if the keyword exists in the request or response content. + "pii": + "description": |- + Configuration for guardrail PII filter. + "safety": + "description": |- + Indicates whether the safety filter is enabled. + "valid_topics": + "description": |- + The list of allowed topics. Given a chat request, this guardrail flags the request if its topic is not in the allowed topics. github.com/databricks/databricks-sdk-go/service/serving.AiGatewayGuardrailPiiBehavior: - behavior: - description: Behavior for PII filter. Currently only 'BLOCK' is supported. If - 'BLOCK' is set for the input guardrail and the request contains PII, the request - is not sent to the model server and 400 status code is returned; if 'BLOCK' - is set for the output guardrail and the model response contains PII, the PII - info in the response is redacted and 400 status code is returned. + "behavior": + "description": |- + Behavior for PII filter. Currently only 'BLOCK' is supported. If 'BLOCK' is set for the input guardrail and the request contains PII, the request is not sent to the model server and 400 status code is returned; if 'BLOCK' is set for the output guardrail and the model response contains PII, the PII info in the response is redacted and 400 status code is returned. github.com/databricks/databricks-sdk-go/service/serving.AiGatewayGuardrailPiiBehaviorBehavior: - _: - description: Behavior for PII filter. Currently only 'BLOCK' is supported. If - 'BLOCK' is set for the input guardrail and the request contains PII, the request - is not sent to the model server and 400 status code is returned; if 'BLOCK' - is set for the output guardrail and the model response contains PII, the PII - info in the response is redacted and 400 status code is returned. - enum: - - NONE - - BLOCK + "_": + "description": |- + Behavior for PII filter. Currently only 'BLOCK' is supported. If 'BLOCK' is set for the input guardrail and the request contains PII, the request is not sent to the model server and 400 status code is returned; if 'BLOCK' is set for the output guardrail and the model response contains PII, the PII info in the response is redacted and 400 status code is returned. + "enum": + - |- + NONE + - |- + BLOCK github.com/databricks/databricks-sdk-go/service/serving.AiGatewayGuardrails: - input: - description: Configuration for input guardrail filters. - output: - description: Configuration for output guardrail filters. + "input": + "description": |- + Configuration for input guardrail filters. + "output": + "description": |- + Configuration for output guardrail filters. github.com/databricks/databricks-sdk-go/service/serving.AiGatewayInferenceTableConfig: - catalog_name: - description: 'The name of the catalog in Unity Catalog. Required when enabling - inference tables. NOTE: On update, you have to disable inference table first - in order to change the catalog name.' - enabled: - description: Indicates whether the inference table is enabled. - schema_name: - description: 'The name of the schema in Unity Catalog. Required when enabling - inference tables. NOTE: On update, you have to disable inference table first - in order to change the schema name.' - table_name_prefix: - description: 'The prefix of the table in Unity Catalog. NOTE: On update, you have - to disable inference table first in order to change the prefix name.' + "catalog_name": + "description": |- + The name of the catalog in Unity Catalog. Required when enabling inference tables. NOTE: On update, you have to disable inference table first in order to change the catalog name. + "enabled": + "description": |- + Indicates whether the inference table is enabled. + "schema_name": + "description": |- + The name of the schema in Unity Catalog. Required when enabling inference tables. NOTE: On update, you have to disable inference table first in order to change the schema name. + "table_name_prefix": + "description": |- + The prefix of the table in Unity Catalog. NOTE: On update, you have to disable inference table first in order to change the prefix name. github.com/databricks/databricks-sdk-go/service/serving.AiGatewayRateLimit: - calls: - description: Used to specify how many calls are allowed for a key within the renewal_period. - key: - description: Key field for a rate limit. Currently, only 'user' and 'endpoint' - are supported, with 'endpoint' being the default if not specified. - renewal_period: - description: Renewal period field for a rate limit. Currently, only 'minute' is - supported. + "calls": + "description": |- + Used to specify how many calls are allowed for a key within the renewal_period. + "key": + "description": |- + Key field for a rate limit. Currently, only 'user' and 'endpoint' are supported, with 'endpoint' being the default if not specified. + "renewal_period": + "description": |- + Renewal period field for a rate limit. Currently, only 'minute' is supported. github.com/databricks/databricks-sdk-go/service/serving.AiGatewayRateLimitKey: - _: - description: Key field for a rate limit. Currently, only 'user' and 'endpoint' - are supported, with 'endpoint' being the default if not specified. - enum: - - user - - endpoint + "_": + "description": |- + Key field for a rate limit. Currently, only 'user' and 'endpoint' are supported, with 'endpoint' being the default if not specified. + "enum": + - |- + user + - |- + endpoint github.com/databricks/databricks-sdk-go/service/serving.AiGatewayRateLimitRenewalPeriod: - _: - description: Renewal period field for a rate limit. Currently, only 'minute' is - supported. - enum: - - minute + "_": + "description": |- + Renewal period field for a rate limit. Currently, only 'minute' is supported. + "enum": + - |- + minute github.com/databricks/databricks-sdk-go/service/serving.AiGatewayUsageTrackingConfig: - enabled: - description: Whether to enable usage tracking. + "enabled": + "description": |- + Whether to enable usage tracking. github.com/databricks/databricks-sdk-go/service/serving.AmazonBedrockConfig: - aws_access_key_id: - description: 'The Databricks secret key reference for an AWS access key ID with - permissions to interact with Bedrock services. If you prefer to paste your API - key directly, see `aws_access_key_id`. You must provide an API key using one - of the following fields: `aws_access_key_id` or `aws_access_key_id_plaintext`.' - aws_access_key_id_plaintext: - description: 'An AWS access key ID with permissions to interact with Bedrock services - provided as a plaintext string. If you prefer to reference your key using Databricks - Secrets, see `aws_access_key_id`. You must provide an API key using one of the - following fields: `aws_access_key_id` or `aws_access_key_id_plaintext`.' - aws_region: - description: The AWS region to use. Bedrock has to be enabled there. - aws_secret_access_key: - description: 'The Databricks secret key reference for an AWS secret access key - paired with the access key ID, with permissions to interact with Bedrock services. - If you prefer to paste your API key directly, see `aws_secret_access_key_plaintext`. - You must provide an API key using one of the following fields: `aws_secret_access_key` - or `aws_secret_access_key_plaintext`.' - aws_secret_access_key_plaintext: - description: 'An AWS secret access key paired with the access key ID, with permissions - to interact with Bedrock services provided as a plaintext string. If you prefer - to reference your key using Databricks Secrets, see `aws_secret_access_key`. - You must provide an API key using one of the following fields: `aws_secret_access_key` - or `aws_secret_access_key_plaintext`.' - bedrock_provider: - description: 'The underlying provider in Amazon Bedrock. Supported values (case - insensitive) include: Anthropic, Cohere, AI21Labs, Amazon.' + "aws_access_key_id": + "description": |- + The Databricks secret key reference for an AWS access key ID with permissions to interact with Bedrock services. If you prefer to paste your API key directly, see `aws_access_key_id`. You must provide an API key using one of the following fields: `aws_access_key_id` or `aws_access_key_id_plaintext`. + "aws_access_key_id_plaintext": + "description": |- + An AWS access key ID with permissions to interact with Bedrock services provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `aws_access_key_id`. You must provide an API key using one of the following fields: `aws_access_key_id` or `aws_access_key_id_plaintext`. + "aws_region": + "description": |- + The AWS region to use. Bedrock has to be enabled there. + "aws_secret_access_key": + "description": |- + The Databricks secret key reference for an AWS secret access key paired with the access key ID, with permissions to interact with Bedrock services. If you prefer to paste your API key directly, see `aws_secret_access_key_plaintext`. You must provide an API key using one of the following fields: `aws_secret_access_key` or `aws_secret_access_key_plaintext`. + "aws_secret_access_key_plaintext": + "description": |- + An AWS secret access key paired with the access key ID, with permissions to interact with Bedrock services provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `aws_secret_access_key`. You must provide an API key using one of the following fields: `aws_secret_access_key` or `aws_secret_access_key_plaintext`. + "bedrock_provider": + "description": |- + The underlying provider in Amazon Bedrock. Supported values (case insensitive) include: Anthropic, Cohere, AI21Labs, Amazon. github.com/databricks/databricks-sdk-go/service/serving.AmazonBedrockConfigBedrockProvider: - _: - description: 'The underlying provider in Amazon Bedrock. Supported values (case - insensitive) include: Anthropic, Cohere, AI21Labs, Amazon.' - enum: - - anthropic - - cohere - - ai21labs - - amazon + "_": + "description": |- + The underlying provider in Amazon Bedrock. Supported values (case insensitive) include: Anthropic, Cohere, AI21Labs, Amazon. + "enum": + - |- + anthropic + - |- + cohere + - |- + ai21labs + - |- + amazon github.com/databricks/databricks-sdk-go/service/serving.AnthropicConfig: - anthropic_api_key: - description: 'The Databricks secret key reference for an Anthropic API key. If - you prefer to paste your API key directly, see `anthropic_api_key_plaintext`. - You must provide an API key using one of the following fields: `anthropic_api_key` - or `anthropic_api_key_plaintext`.' - anthropic_api_key_plaintext: - description: 'The Anthropic API key provided as a plaintext string. If you prefer - to reference your key using Databricks Secrets, see `anthropic_api_key`. You - must provide an API key using one of the following fields: `anthropic_api_key` - or `anthropic_api_key_plaintext`.' + "anthropic_api_key": + "description": |- + The Databricks secret key reference for an Anthropic API key. If you prefer to paste your API key directly, see `anthropic_api_key_plaintext`. You must provide an API key using one of the following fields: `anthropic_api_key` or `anthropic_api_key_plaintext`. + "anthropic_api_key_plaintext": + "description": |- + The Anthropic API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `anthropic_api_key`. You must provide an API key using one of the following fields: `anthropic_api_key` or `anthropic_api_key_plaintext`. github.com/databricks/databricks-sdk-go/service/serving.AutoCaptureConfigInput: - catalog_name: - description: 'The name of the catalog in Unity Catalog. NOTE: On update, you cannot - change the catalog name if the inference table is already enabled.' - enabled: - description: Indicates whether the inference table is enabled. - schema_name: - description: 'The name of the schema in Unity Catalog. NOTE: On update, you cannot - change the schema name if the inference table is already enabled.' - table_name_prefix: - description: 'The prefix of the table in Unity Catalog. NOTE: On update, you cannot - change the prefix name if the inference table is already enabled.' + "catalog_name": + "description": |- + The name of the catalog in Unity Catalog. NOTE: On update, you cannot change the catalog name if the inference table is already enabled. + "enabled": + "description": |- + Indicates whether the inference table is enabled. + "schema_name": + "description": |- + The name of the schema in Unity Catalog. NOTE: On update, you cannot change the schema name if the inference table is already enabled. + "table_name_prefix": + "description": |- + The prefix of the table in Unity Catalog. NOTE: On update, you cannot change the prefix name if the inference table is already enabled. github.com/databricks/databricks-sdk-go/service/serving.CohereConfig: - cohere_api_base: - description: "This is an optional field to provide a customized base URL for the - Cohere API. \nIf left unspecified, the standard Cohere base URL is used.\n" - cohere_api_key: - description: 'The Databricks secret key reference for a Cohere API key. If you - prefer to paste your API key directly, see `cohere_api_key_plaintext`. You must - provide an API key using one of the following fields: `cohere_api_key` or `cohere_api_key_plaintext`.' - cohere_api_key_plaintext: - description: 'The Cohere API key provided as a plaintext string. If you prefer - to reference your key using Databricks Secrets, see `cohere_api_key`. You must - provide an API key using one of the following fields: `cohere_api_key` or `cohere_api_key_plaintext`.' + "cohere_api_base": + "description": "This is an optional field to provide a customized base URL for the Cohere API. \nIf left unspecified, the standard Cohere base URL is used.\n" + "cohere_api_key": + "description": |- + The Databricks secret key reference for a Cohere API key. If you prefer to paste your API key directly, see `cohere_api_key_plaintext`. You must provide an API key using one of the following fields: `cohere_api_key` or `cohere_api_key_plaintext`. + "cohere_api_key_plaintext": + "description": |- + The Cohere API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `cohere_api_key`. You must provide an API key using one of the following fields: `cohere_api_key` or `cohere_api_key_plaintext`. github.com/databricks/databricks-sdk-go/service/serving.DatabricksModelServingConfig: - databricks_api_token: - description: | + "databricks_api_token": + "description": | The Databricks secret key reference for a Databricks API token that corresponds to a user or service principal with Can Query access to the model serving endpoint pointed to by this external model. If you prefer to paste your API key directly, see `databricks_api_token_plaintext`. You must provide an API key using one of the following fields: `databricks_api_token` or `databricks_api_token_plaintext`. - databricks_api_token_plaintext: - description: | + "databricks_api_token_plaintext": + "description": | The Databricks API token that corresponds to a user or service principal with Can Query access to the model serving endpoint pointed to by this external model provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `databricks_api_token`. You must provide an API key using one of the following fields: `databricks_api_token` or `databricks_api_token_plaintext`. - databricks_workspace_url: - description: | + "databricks_workspace_url": + "description": | The URL of the Databricks workspace containing the model serving endpoint pointed to by this external model. github.com/databricks/databricks-sdk-go/service/serving.EndpointCoreConfigInput: - auto_capture_config: - description: Configuration for Inference Tables which automatically logs requests - and responses to Unity Catalog. - served_entities: - description: A list of served entities for the endpoint to serve. A serving endpoint - can have up to 15 served entities. - served_models: - description: (Deprecated, use served_entities instead) A list of served models - for the endpoint to serve. A serving endpoint can have up to 15 served models. - traffic_config: - description: The traffic config defining how invocations to the serving endpoint - should be routed. + "auto_capture_config": + "description": |- + Configuration for Inference Tables which automatically logs requests and responses to Unity Catalog. + "served_entities": + "description": |- + A list of served entities for the endpoint to serve. A serving endpoint can have up to 15 served entities. + "served_models": + "description": |- + (Deprecated, use served_entities instead) A list of served models for the endpoint to serve. A serving endpoint can have up to 15 served models. + "traffic_config": + "description": |- + The traffic config defining how invocations to the serving endpoint should be routed. github.com/databricks/databricks-sdk-go/service/serving.EndpointTag: - key: - description: Key field for a serving endpoint tag. - value: - description: Optional value field for a serving endpoint tag. + "key": + "description": |- + Key field for a serving endpoint tag. + "value": + "description": |- + Optional value field for a serving endpoint tag. github.com/databricks/databricks-sdk-go/service/serving.ExternalModel: - ai21labs_config: - description: AI21Labs Config. Only required if the provider is 'ai21labs'. - amazon_bedrock_config: - description: Amazon Bedrock Config. Only required if the provider is 'amazon-bedrock'. - anthropic_config: - description: Anthropic Config. Only required if the provider is 'anthropic'. - cohere_config: - description: Cohere Config. Only required if the provider is 'cohere'. - databricks_model_serving_config: - description: Databricks Model Serving Config. Only required if the provider is - 'databricks-model-serving'. - google_cloud_vertex_ai_config: - description: Google Cloud Vertex AI Config. Only required if the provider is 'google-cloud-vertex-ai'. - name: - description: The name of the external model. - openai_config: - description: OpenAI Config. Only required if the provider is 'openai'. - palm_config: - description: PaLM Config. Only required if the provider is 'palm'. - provider: - description: | + "ai21labs_config": + "description": |- + AI21Labs Config. Only required if the provider is 'ai21labs'. + "amazon_bedrock_config": + "description": |- + Amazon Bedrock Config. Only required if the provider is 'amazon-bedrock'. + "anthropic_config": + "description": |- + Anthropic Config. Only required if the provider is 'anthropic'. + "cohere_config": + "description": |- + Cohere Config. Only required if the provider is 'cohere'. + "databricks_model_serving_config": + "description": |- + Databricks Model Serving Config. Only required if the provider is 'databricks-model-serving'. + "google_cloud_vertex_ai_config": + "description": |- + Google Cloud Vertex AI Config. Only required if the provider is 'google-cloud-vertex-ai'. + "name": + "description": |- + The name of the external model. + "openai_config": + "description": |- + OpenAI Config. Only required if the provider is 'openai'. + "palm_config": + "description": |- + PaLM Config. Only required if the provider is 'palm'. + "provider": + "description": | The name of the provider for the external model. Currently, the supported providers are 'ai21labs', 'anthropic', 'amazon-bedrock', 'cohere', 'databricks-model-serving', 'google-cloud-vertex-ai', 'openai', and 'palm'.", - task: - description: The task type of the external model. + "task": + "description": |- + The task type of the external model. github.com/databricks/databricks-sdk-go/service/serving.ExternalModelProvider: - _: - description: | + "_": + "description": | The name of the provider for the external model. Currently, the supported providers are 'ai21labs', 'anthropic', 'amazon-bedrock', 'cohere', 'databricks-model-serving', 'google-cloud-vertex-ai', 'openai', and 'palm'.", - enum: - - ai21labs - - anthropic - - amazon-bedrock - - cohere - - databricks-model-serving - - google-cloud-vertex-ai - - openai - - palm + "enum": + - |- + ai21labs + - |- + anthropic + - |- + amazon-bedrock + - |- + cohere + - |- + databricks-model-serving + - |- + google-cloud-vertex-ai + - |- + openai + - |- + palm +github.com/databricks/databricks-sdk-go/service/serving.GoogleCloudVertexAiConfig: + "private_key": + "description": |- + The Databricks secret key reference for a private key for the service account which has access to the Google Cloud Vertex AI Service. See [Best practices for managing service account keys](https://cloud.google.com/iam/docs/best-practices-for-managing-service-account-keys). If you prefer to paste your API key directly, see `private_key_plaintext`. You must provide an API key using one of the following fields: `private_key` or `private_key_plaintext` + "private_key_plaintext": + "description": |- + The private key for the service account which has access to the Google Cloud Vertex AI Service provided as a plaintext secret. See [Best practices for managing service account keys](https://cloud.google.com/iam/docs/best-practices-for-managing-service-account-keys). If you prefer to reference your key using Databricks Secrets, see `private_key`. You must provide an API key using one of the following fields: `private_key` or `private_key_plaintext`. + "project_id": + "description": |- + This is the Google Cloud project id that the service account is associated with. + "region": + "description": |- + This is the region for the Google Cloud Vertex AI Service. See [supported regions](https://cloud.google.com/vertex-ai/docs/general/locations) for more details. Some models are only available in specific regions. +github.com/databricks/databricks-sdk-go/service/serving.OpenAiConfig: + "microsoft_entra_client_id": + "description": | + This field is only required for Azure AD OpenAI and is the Microsoft Entra Client ID. + "microsoft_entra_client_secret": + "description": | + The Databricks secret key reference for a client secret used for Microsoft Entra ID authentication. + If you prefer to paste your client secret directly, see `microsoft_entra_client_secret_plaintext`. + You must provide an API key using one of the following fields: `microsoft_entra_client_secret` or `microsoft_entra_client_secret_plaintext`. + "microsoft_entra_client_secret_plaintext": + "description": | + The client secret used for Microsoft Entra ID authentication provided as a plaintext string. + If you prefer to reference your key using Databricks Secrets, see `microsoft_entra_client_secret`. + You must provide an API key using one of the following fields: `microsoft_entra_client_secret` or `microsoft_entra_client_secret_plaintext`. + "microsoft_entra_tenant_id": + "description": | + This field is only required for Azure AD OpenAI and is the Microsoft Entra Tenant ID. + "openai_api_base": + "description": | + This is a field to provide a customized base URl for the OpenAI API. + For Azure OpenAI, this field is required, and is the base URL for the Azure OpenAI API service + provided by Azure. + For other OpenAI API types, this field is optional, and if left unspecified, the standard OpenAI base URL is used. + "openai_api_key": + "description": |- + The Databricks secret key reference for an OpenAI API key using the OpenAI or Azure service. If you prefer to paste your API key directly, see `openai_api_key_plaintext`. You must provide an API key using one of the following fields: `openai_api_key` or `openai_api_key_plaintext`. + "openai_api_key_plaintext": + "description": |- + The OpenAI API key using the OpenAI or Azure service provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `openai_api_key`. You must provide an API key using one of the following fields: `openai_api_key` or `openai_api_key_plaintext`. + "openai_api_type": + "description": | + This is an optional field to specify the type of OpenAI API to use. + For Azure OpenAI, this field is required, and adjust this parameter to represent the preferred security + access validation protocol. For access token validation, use azure. For authentication using Azure Active + Directory (Azure AD) use, azuread. + "openai_api_version": + "description": | + This is an optional field to specify the OpenAI API version. + For Azure OpenAI, this field is required, and is the version of the Azure OpenAI service to + utilize, specified by a date. + "openai_deployment_name": + "description": | + This field is only required for Azure OpenAI and is the name of the deployment resource for the + Azure OpenAI service. + "openai_organization": + "description": | + This is an optional field to specify the organization in OpenAI or Azure OpenAI. +github.com/databricks/databricks-sdk-go/service/serving.PaLmConfig: + "palm_api_key": + "description": |- + The Databricks secret key reference for a PaLM API key. If you prefer to paste your API key directly, see `palm_api_key_plaintext`. You must provide an API key using one of the following fields: `palm_api_key` or `palm_api_key_plaintext`. + "palm_api_key_plaintext": + "description": |- + The PaLM API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `palm_api_key`. You must provide an API key using one of the following fields: `palm_api_key` or `palm_api_key_plaintext`. github.com/databricks/databricks-sdk-go/service/serving.RateLimit: - calls: - description: Used to specify how many calls are allowed for a key within the renewal_period. - key: - description: Key field for a serving endpoint rate limit. Currently, only 'user' - and 'endpoint' are supported, with 'endpoint' being the default if not specified. - renewal_period: - description: Renewal period field for a serving endpoint rate limit. Currently, - only 'minute' is supported. + "calls": + "description": |- + Used to specify how many calls are allowed for a key within the renewal_period. + "key": + "description": |- + Key field for a serving endpoint rate limit. Currently, only 'user' and 'endpoint' are supported, with 'endpoint' being the default if not specified. + "renewal_period": + "description": |- + Renewal period field for a serving endpoint rate limit. Currently, only 'minute' is supported. github.com/databricks/databricks-sdk-go/service/serving.RateLimitKey: - _: - description: Key field for a serving endpoint rate limit. Currently, only 'user' - and 'endpoint' are supported, with 'endpoint' being the default if not specified. - enum: - - user - - endpoint + "_": + "description": |- + Key field for a serving endpoint rate limit. Currently, only 'user' and 'endpoint' are supported, with 'endpoint' being the default if not specified. + "enum": + - |- + user + - |- + endpoint github.com/databricks/databricks-sdk-go/service/serving.RateLimitRenewalPeriod: - _: - description: Renewal period field for a serving endpoint rate limit. Currently, - only 'minute' is supported. - enum: - - minute + "_": + "description": |- + Renewal period field for a serving endpoint rate limit. Currently, only 'minute' is supported. + "enum": + - |- + minute github.com/databricks/databricks-sdk-go/service/serving.Route: - served_model_name: - description: The name of the served model this route configures traffic for. - traffic_percentage: - description: The percentage of endpoint traffic to send to this route. It must - be an integer between 0 and 100 inclusive. + "served_model_name": + "description": |- + The name of the served model this route configures traffic for. + "traffic_percentage": + "description": |- + The percentage of endpoint traffic to send to this route. It must be an integer between 0 and 100 inclusive. github.com/databricks/databricks-sdk-go/service/serving.ServedEntityInput: - entity_name: - description: | + "entity_name": + "description": | The name of the entity to be served. The entity may be a model in the Databricks Model Registry, a model in the Unity Catalog (UC), or a function of type FEATURE_SPEC in the UC. If it is a UC object, the full name of the object should be given in the form of __catalog_name__.__schema_name__.__model_name__. - entity_version: - description: The version of the model in Databricks Model Registry to be served - or empty if the entity is a FEATURE_SPEC. - environment_vars: - description: "An object containing a set of optional, user-specified environment - variable key-value pairs used for serving this entity.\nNote: this is an experimental - feature and subject to change. \nExample entity environment variables that refer - to Databricks secrets: `{\"OPENAI_API_KEY\": \"{{secrets/my_scope/my_key}}\", - \"DATABRICKS_TOKEN\": \"{{secrets/my_scope2/my_key2}}\"}`" - external_model: - description: | + "entity_version": + "description": |- + The version of the model in Databricks Model Registry to be served or empty if the entity is a FEATURE_SPEC. + "environment_vars": + "description": "An object containing a set of optional, user-specified environment variable key-value pairs used for serving this entity.\nNote: this is an experimental feature and subject to change. \nExample entity environment variables that refer to Databricks secrets: `{\"OPENAI_API_KEY\": \"{{secrets/my_scope/my_key}}\", \"DATABRICKS_TOKEN\": \"{{secrets/my_scope2/my_key2}}\"}`" + "external_model": + "description": | The external model to be served. NOTE: Only one of external_model and (entity_name, entity_version, workload_size, workload_type, and scale_to_zero_enabled) can be specified with the latter set being used for custom model serving for a Databricks registered model. For an existing endpoint with external_model, it cannot be updated to an endpoint without external_model. If the endpoint is created without external_model, users cannot update it to add external_model later. The task type of all external models within an endpoint must be the same. - instance_profile_arn: - description: ARN of the instance profile that the served entity uses to access - AWS resources. - max_provisioned_throughput: - description: The maximum tokens per second that the endpoint can scale up to. - min_provisioned_throughput: - description: The minimum tokens per second that the endpoint can scale down to. - name: - description: | + "instance_profile_arn": + "description": |- + ARN of the instance profile that the served entity uses to access AWS resources. + "max_provisioned_throughput": + "description": |- + The maximum tokens per second that the endpoint can scale up to. + "min_provisioned_throughput": + "description": |- + The minimum tokens per second that the endpoint can scale down to. + "name": + "description": | The name of a served entity. It must be unique across an endpoint. A served entity name can consist of alphanumeric characters, dashes, and underscores. If not specified for an external model, this field defaults to external_model.name, with '.' and ':' replaced with '-', and if not specified for other entities, it defaults to -. - scale_to_zero_enabled: - description: Whether the compute resources for the served entity should scale - down to zero. - workload_size: - description: | + "scale_to_zero_enabled": + "description": |- + Whether the compute resources for the served entity should scale down to zero. + "workload_size": + "description": | The workload size of the served entity. The workload size corresponds to a range of provisioned concurrency that the compute autoscales between. A single unit of provisioned concurrency can process one request at a time. Valid workload sizes are "Small" (4 - 4 provisioned concurrency), "Medium" (8 - 16 provisioned concurrency), and "Large" (16 - 64 provisioned concurrency). If scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size is 0. - workload_type: - description: | + "workload_type": + "description": | The workload type of the served entity. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is "CPU". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others. See the available [GPU types](https://docs.databricks.com/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types). github.com/databricks/databricks-sdk-go/service/serving.ServedModelInput: - environment_vars: - description: "An object containing a set of optional, user-specified environment - variable key-value pairs used for serving this model.\nNote: this is an experimental - feature and subject to change. \nExample model environment variables that refer - to Databricks secrets: `{\"OPENAI_API_KEY\": \"{{secrets/my_scope/my_key}}\", - \"DATABRICKS_TOKEN\": \"{{secrets/my_scope2/my_key2}}\"}`" - instance_profile_arn: - description: ARN of the instance profile that the served model will use to access - AWS resources. - max_provisioned_throughput: - description: The maximum tokens per second that the endpoint can scale up to. - min_provisioned_throughput: - description: The minimum tokens per second that the endpoint can scale down to. - model_name: - description: | + "environment_vars": + "description": "An object containing a set of optional, user-specified environment variable key-value pairs used for serving this model.\nNote: this is an experimental feature and subject to change. \nExample model environment variables that refer to Databricks secrets: `{\"OPENAI_API_KEY\": \"{{secrets/my_scope/my_key}}\", \"DATABRICKS_TOKEN\": \"{{secrets/my_scope2/my_key2}}\"}`" + "instance_profile_arn": + "description": |- + ARN of the instance profile that the served model will use to access AWS resources. + "max_provisioned_throughput": + "description": |- + The maximum tokens per second that the endpoint can scale up to. + "min_provisioned_throughput": + "description": |- + The minimum tokens per second that the endpoint can scale down to. + "model_name": + "description": | The name of the model in Databricks Model Registry to be served or if the model resides in Unity Catalog, the full name of model, in the form of __catalog_name__.__schema_name__.__model_name__. - model_version: - description: The version of the model in Databricks Model Registry or Unity Catalog - to be served. - name: - description: | + "model_version": + "description": |- + The version of the model in Databricks Model Registry or Unity Catalog to be served. + "name": + "description": | The name of a served model. It must be unique across an endpoint. If not specified, this field will default to -. A served model name can consist of alphanumeric characters, dashes, and underscores. - scale_to_zero_enabled: - description: Whether the compute resources for the served model should scale down - to zero. - workload_size: - description: | + "scale_to_zero_enabled": + "description": |- + Whether the compute resources for the served model should scale down to zero. + "workload_size": + "description": | The workload size of the served model. The workload size corresponds to a range of provisioned concurrency that the compute will autoscale between. A single unit of provisioned concurrency can process one request at a time. Valid workload sizes are "Small" (4 - 4 provisioned concurrency), "Medium" (8 - 16 provisioned concurrency), and "Large" (16 - 64 provisioned concurrency). If scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size will be 0. - workload_type: - description: | + "workload_type": + "description": | The workload type of the served model. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is "CPU". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others. See the available [GPU types](https://docs.databricks.com/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types). github.com/databricks/databricks-sdk-go/service/serving.ServedModelInputWorkloadSize: - _: - description: | + "_": + "description": | The workload size of the served model. The workload size corresponds to a range of provisioned concurrency that the compute will autoscale between. A single unit of provisioned concurrency can process one request at a time. Valid workload sizes are "Small" (4 - 4 provisioned concurrency), "Medium" (8 - 16 provisioned concurrency), and "Large" (16 - 64 provisioned concurrency). If scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size will be 0. - enum: - - Small - - Medium - - Large + "enum": + - |- + Small + - |- + Medium + - |- + Large github.com/databricks/databricks-sdk-go/service/serving.ServedModelInputWorkloadType: - _: - description: | + "_": + "description": | The workload type of the served model. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is "CPU". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others. See the available [GPU types](https://docs.databricks.com/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types). - enum: - - CPU - - GPU_SMALL - - GPU_MEDIUM - - GPU_LARGE - - MULTIGPU_MEDIUM + "enum": + - |- + CPU + - |- + GPU_SMALL + - |- + GPU_MEDIUM + - |- + GPU_LARGE + - |- + MULTIGPU_MEDIUM github.com/databricks/databricks-sdk-go/service/serving.TrafficConfig: - routes: - description: The list of routes that define traffic to each served entity. + "routes": + "description": |- + The list of routes that define traffic to each served entity. diff --git a/bundle/internal/schema/main_test.go b/bundle/internal/schema/main_test.go index 607347b6b..4eeb41d47 100644 --- a/bundle/internal/schema/main_test.go +++ b/bundle/internal/schema/main_test.go @@ -42,7 +42,8 @@ func copyFile(src, dst string) error { // Checks whether descriptions are added for new config fields in the annotations.yml file // If this test fails either manually add descriptions to the `annotations.yml` or do the following: -// 1. run `make schema` from the repository root to add placeholder descriptions +// 1. for fields described outside of CLI package fetch latest schema from the OpenAPI spec and add path to file to DATABRICKS_OPENAPI_SPEC env variable +// 2. run `make schema` from the repository root to add placeholder descriptions // 2. replace all "PLACEHOLDER" values with the actual descriptions if possible // 3. run `make schema` again to regenerate the schema with acutal descriptions func TestRequiredAnnotationsForNewFields(t *testing.T) { diff --git a/bundle/internal/schema/parser.go b/bundle/internal/schema/parser.go index 3fbec0528..e1d1a13dc 100644 --- a/bundle/internal/schema/parser.go +++ b/bundle/internal/schema/parser.go @@ -1,7 +1,6 @@ package main import ( - "bytes" "encoding/json" "fmt" "os" @@ -9,7 +8,6 @@ import ( "reflect" "strings" - "github.com/databricks/cli/libs/dyn/yamlloader" "github.com/databricks/cli/libs/jsonschema" "gopkg.in/yaml.v3" ) @@ -83,7 +81,11 @@ func (p *openapiParser) findRef(typ reflect.Type) (jsonschema.Schema, bool) { // Skip if the type is not in the openapi spec. _, ok := p.ref[k] if !ok { - continue + k = mapIncorrectTypNames(k) + _, ok = p.ref[k] + if !ok { + continue + } } // Return the first Go SDK type found in the openapi spec. @@ -93,6 +95,23 @@ func (p *openapiParser) findRef(typ reflect.Type) (jsonschema.Schema, bool) { return jsonschema.Schema{}, false } +// Fix inconsistent type names between the Go SDK and the OpenAPI spec. +// E.g. "serving.PaLmConfig" in the Go SDK is "serving.PaLMConfig" in the OpenAPI spec. +func mapIncorrectTypNames(ref string) string { + switch ref { + case "serving.PaLmConfig": + return "serving.PaLMConfig" + case "serving.OpenAiConfig": + return "serving.OpenAIConfig" + case "serving.GoogleCloudVertexAiConfig": + return "serving.GoogleCloudVertexAIConfig" + case "serving.Ai21LabsConfig": + return "serving.AI21LabsConfig" + default: + return ref + } +} + // Use the OpenAPI spec to load descriptions for the given type. func (p *openapiParser) extractAnnotations(typ reflect.Type, outputPath, overridesPath string) error { annotations := annotationFile{} @@ -142,31 +161,40 @@ func (p *openapiParser) extractAnnotations(typ reflect.Type, outputPath, overrid return err } - b, err = yaml.Marshal(overrides) + err = saveYamlWithStyle(overridesPath, overrides) if err != nil { return err } - o, err := yamlloader.LoadYAML("", bytes.NewBuffer(b)) + err = saveYamlWithStyle(outputPath, annotations) if err != nil { return err } - err = saveYamlWithStyle(overridesPath, o) + err = prependCommentToFile(outputPath, "# This file is auto-generated. DO NOT EDIT.\n") if err != nil { return err } - b, err = yaml.Marshal(annotations) - if err != nil { - return err - } - b = bytes.Join([][]byte{[]byte("# This file is auto-generated. DO NOT EDIT."), b}, []byte("\n")) - err = os.WriteFile(outputPath, b, 0o644) - if err != nil { - return err - } - return nil } +func prependCommentToFile(outputPath, comment string) error { + b, err := os.ReadFile(outputPath) + if err != nil { + return err + } + f, err := os.OpenFile(outputPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644) + if err != nil { + return err + } + defer f.Close() + + _, err = f.WriteString(comment) + if err != nil { + return err + } + _, err = f.Write(b) + return err +} + func addEmptyOverride(key, pkg string, overridesFile annotationFile) { if overridesFile[pkg] == nil { overridesFile[pkg] = map[string]annotation{} diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 8e8efa7fc..9a352ebb2 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -4772,9 +4772,11 @@ "type": "object", "properties": { "ai21labs_api_key": { + "description": "The Databricks secret key reference for an AI21 Labs API key. If you prefer to paste your API key directly, see `ai21labs_api_key_plaintext`. You must provide an API key using one of the following fields: `ai21labs_api_key` or `ai21labs_api_key_plaintext`.", "$ref": "#/$defs/string" }, "ai21labs_api_key_plaintext": { + "description": "An AI21 Labs API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `ai21labs_api_key`. You must provide an API key using one of the following fields: `ai21labs_api_key` or `ai21labs_api_key_plaintext`.", "$ref": "#/$defs/string" } }, @@ -5287,15 +5289,19 @@ "type": "object", "properties": { "private_key": { + "description": "The Databricks secret key reference for a private key for the service account which has access to the Google Cloud Vertex AI Service. See [Best practices for managing service account keys](https://cloud.google.com/iam/docs/best-practices-for-managing-service-account-keys). If you prefer to paste your API key directly, see `private_key_plaintext`. You must provide an API key using one of the following fields: `private_key` or `private_key_plaintext`", "$ref": "#/$defs/string" }, "private_key_plaintext": { + "description": "The private key for the service account which has access to the Google Cloud Vertex AI Service provided as a plaintext secret. See [Best practices for managing service account keys](https://cloud.google.com/iam/docs/best-practices-for-managing-service-account-keys). If you prefer to reference your key using Databricks Secrets, see `private_key`. You must provide an API key using one of the following fields: `private_key` or `private_key_plaintext`.", "$ref": "#/$defs/string" }, "project_id": { + "description": "This is the Google Cloud project id that the service account is associated with.", "$ref": "#/$defs/string" }, "region": { + "description": "This is the region for the Google Cloud Vertex AI Service. See [supported regions](https://cloud.google.com/vertex-ai/docs/general/locations) for more details. Some models are only available in specific regions.", "$ref": "#/$defs/string" } }, @@ -5313,36 +5319,47 @@ "type": "object", "properties": { "microsoft_entra_client_id": { + "description": "This field is only required for Azure AD OpenAI and is the Microsoft Entra Client ID.\n", "$ref": "#/$defs/string" }, "microsoft_entra_client_secret": { + "description": "The Databricks secret key reference for a client secret used for Microsoft Entra ID authentication.\nIf you prefer to paste your client secret directly, see `microsoft_entra_client_secret_plaintext`.\nYou must provide an API key using one of the following fields: `microsoft_entra_client_secret` or `microsoft_entra_client_secret_plaintext`.\n", "$ref": "#/$defs/string" }, "microsoft_entra_client_secret_plaintext": { + "description": "The client secret used for Microsoft Entra ID authentication provided as a plaintext string.\nIf you prefer to reference your key using Databricks Secrets, see `microsoft_entra_client_secret`.\nYou must provide an API key using one of the following fields: `microsoft_entra_client_secret` or `microsoft_entra_client_secret_plaintext`.\n", "$ref": "#/$defs/string" }, "microsoft_entra_tenant_id": { + "description": "This field is only required for Azure AD OpenAI and is the Microsoft Entra Tenant ID.\n", "$ref": "#/$defs/string" }, "openai_api_base": { + "description": "This is a field to provide a customized base URl for the OpenAI API.\nFor Azure OpenAI, this field is required, and is the base URL for the Azure OpenAI API service\nprovided by Azure.\nFor other OpenAI API types, this field is optional, and if left unspecified, the standard OpenAI base URL is used.\n", "$ref": "#/$defs/string" }, "openai_api_key": { + "description": "The Databricks secret key reference for an OpenAI API key using the OpenAI or Azure service. If you prefer to paste your API key directly, see `openai_api_key_plaintext`. You must provide an API key using one of the following fields: `openai_api_key` or `openai_api_key_plaintext`.", "$ref": "#/$defs/string" }, "openai_api_key_plaintext": { + "description": "The OpenAI API key using the OpenAI or Azure service provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `openai_api_key`. You must provide an API key using one of the following fields: `openai_api_key` or `openai_api_key_plaintext`.", "$ref": "#/$defs/string" }, "openai_api_type": { + "description": "This is an optional field to specify the type of OpenAI API to use.\nFor Azure OpenAI, this field is required, and adjust this parameter to represent the preferred security\naccess validation protocol. For access token validation, use azure. For authentication using Azure Active\nDirectory (Azure AD) use, azuread.\n", "$ref": "#/$defs/string" }, "openai_api_version": { + "description": "This is an optional field to specify the OpenAI API version.\nFor Azure OpenAI, this field is required, and is the version of the Azure OpenAI service to\nutilize, specified by a date.\n", "$ref": "#/$defs/string" }, "openai_deployment_name": { + "description": "This field is only required for Azure OpenAI and is the name of the deployment resource for the\nAzure OpenAI service.\n", "$ref": "#/$defs/string" }, "openai_organization": { + "description": "This is an optional field to specify the organization in OpenAI or Azure OpenAI.\n", "$ref": "#/$defs/string" } }, @@ -5360,9 +5377,11 @@ "type": "object", "properties": { "palm_api_key": { + "description": "The Databricks secret key reference for a PaLM API key. If you prefer to paste your API key directly, see `palm_api_key_plaintext`. You must provide an API key using one of the following fields: `palm_api_key` or `palm_api_key_plaintext`.", "$ref": "#/$defs/string" }, "palm_api_key_plaintext": { + "description": "The PaLM API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `palm_api_key`. You must provide an API key using one of the following fields: `palm_api_key` or `palm_api_key_plaintext`.", "$ref": "#/$defs/string" } }, From a002475a6a4175cb0fa4ef09e21ef8bcecd84899 Mon Sep 17 00:00:00 2001 From: "Lennart Kats (databricks)" Date: Fri, 27 Dec 2024 12:38:12 +0100 Subject: [PATCH 02/19] Relax checks in builtin template tests (#2042) ## Changes Relax the checks of `lib/template/builtin_test` so they don't fail for a local development copy that has uncommitted draft templates. Right now these tests fail because I have some git-ignored uncommitted templates in my local dev copy. --- libs/template/builtin_test.go | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/libs/template/builtin_test.go b/libs/template/builtin_test.go index 504e0acca..79e04cb84 100644 --- a/libs/template/builtin_test.go +++ b/libs/template/builtin_test.go @@ -11,18 +11,24 @@ import ( func TestBuiltin(t *testing.T) { out, err := Builtin() require.NoError(t, err) - assert.Len(t, out, 3) + assert.GreaterOrEqual(t, len(out), 3) - // Confirm names. - assert.Equal(t, "dbt-sql", out[0].Name) - assert.Equal(t, "default-python", out[1].Name) - assert.Equal(t, "default-sql", out[2].Name) + // Create a map of templates by name for easier lookup + templates := make(map[string]*BuiltinTemplate) + for _, tmpl := range out { + templates[tmpl.Name] = &tmpl + } - // Confirm that the filesystems work. - _, err = fs.Stat(out[0].FS, `template/{{.project_name}}/dbt_project.yml.tmpl`) + // Verify all expected templates exist + assert.Contains(t, templates, "dbt-sql") + assert.Contains(t, templates, "default-python") + assert.Contains(t, templates, "default-sql") + + // Verify the filesystems work for each template + _, err = fs.Stat(templates["dbt-sql"].FS, `template/{{.project_name}}/dbt_project.yml.tmpl`) assert.NoError(t, err) - _, err = fs.Stat(out[1].FS, `template/{{.project_name}}/tests/main_test.py.tmpl`) + _, err = fs.Stat(templates["default-python"].FS, `template/{{.project_name}}/tests/main_test.py.tmpl`) assert.NoError(t, err) - _, err = fs.Stat(out[2].FS, `template/{{.project_name}}/src/orders_daily.sql.tmpl`) + _, err = fs.Stat(templates["default-sql"].FS, `template/{{.project_name}}/src/orders_daily.sql.tmpl`) assert.NoError(t, err) } From e088d0d996979ddd56dd9ce10d3d1becbdc771f6 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 30 Dec 2024 16:18:57 +0100 Subject: [PATCH 03/19] Add lint.sh to run golanci-lint in 2 stages (#2051) First stage is to run goimports and formatter, second is full suite. This ensures that imports and formatting are fixed even in presence of other issues. Otherwise golanci-lint refuses to fix anything https://github.com/golangci/golangci-lint/issues/5257 This helpful when running aider with config like this - aider will use that to autofix what it can after every update: ``` % cat .aider.conf.yml lint-cmd: - "go: ./lint.sh" ``` --- Makefile | 2 +- lint.sh | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100755 lint.sh diff --git a/Makefile b/Makefile index f8e7834a5..7dca3b2cf 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ default: build lint: vendor @echo "✓ Linting source code with https://golangci-lint.run/ (with --fix)..." - @golangci-lint run --fix ./... + @./lint.sh ./... lintcheck: vendor @echo "✓ Linting source code with https://golangci-lint.run/ ..." diff --git a/lint.sh b/lint.sh new file mode 100755 index 000000000..c93d04c66 --- /dev/null +++ b/lint.sh @@ -0,0 +1,9 @@ +#!/bin/bash +set -euo pipefail +# With golangci-lint, if there are any compliation issues, then formatters' autofix won't be applied. +# https://github.com/golangci/golangci-lint/issues/5257 +# However, running goimports first alone will actually fix some of the compilation issues. +# Fixing formatting is also reasonable thing to do. +# For this reason, this script runs golangci-lint in two stages: +golangci-lint run --fix --no-config --disable-all --enable gofumpt,goimports $@ +exec golangci-lint run --fix $@ From 261b7f4083c4f10c0cc0e6c0ec9d95bfc2927174 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 30 Dec 2024 16:26:21 +0100 Subject: [PATCH 04/19] Move bulk of "golden tests" logic to libs/testdiff (#2054) ## Changes - Detach "golden files" assertions from testcli runner. Now any output can be compared, no matter how it is obtained. - Move those assertion to libs/testdiff package. This allows using "golden files" in non-integration tests. ## Tests Existing tests --- .../bundle/init_default_python_test.go | 7 +- internal/testcli/golden.go | 201 +---------------- libs/testdiff/golden.go | 212 ++++++++++++++++++ .../testcli => libs/testdiff}/golden_test.go | 2 +- 4 files changed, 220 insertions(+), 202 deletions(-) create mode 100644 libs/testdiff/golden.go rename {internal/testcli => libs/testdiff}/golden_test.go (92%) diff --git a/integration/bundle/init_default_python_test.go b/integration/bundle/init_default_python_test.go index 9b65636e9..c93e6b50b 100644 --- a/integration/bundle/init_default_python_test.go +++ b/integration/bundle/init_default_python_test.go @@ -11,6 +11,7 @@ import ( "github.com/databricks/cli/internal/testcli" "github.com/databricks/cli/internal/testutil" "github.com/databricks/cli/libs/python/pythontest" + "github.com/databricks/cli/libs/testdiff" "github.com/stretchr/testify/require" ) @@ -50,14 +51,14 @@ func testDefaultPython(t *testing.T, pythonVersion string) { ctx, wt := acc.WorkspaceTest(t) uniqueProjectId := testutil.RandomName("") - ctx, replacements := testcli.WithReplacementsMap(ctx) + ctx, replacements := testdiff.WithReplacementsMap(ctx) replacements.Set(uniqueProjectId, "$UNIQUE_PRJ") user, err := wt.W.CurrentUser.Me(ctx) require.NoError(t, err) require.NotNil(t, user) - testcli.PrepareReplacementsUser(t, replacements, *user) - testcli.PrepareReplacements(t, replacements, wt.W) + testdiff.PrepareReplacementsUser(t, replacements, *user) + testdiff.PrepareReplacements(t, replacements, wt.W) tmpDir := t.TempDir() testutil.Chdir(t, tmpDir) diff --git a/internal/testcli/golden.go b/internal/testcli/golden.go index 34f38f18a..a16f42c72 100644 --- a/internal/testcli/golden.go +++ b/internal/testcli/golden.go @@ -3,222 +3,27 @@ package testcli import ( "context" "fmt" - "os" - "regexp" - "slices" "strings" - "testing" "github.com/databricks/cli/internal/testutil" - "github.com/databricks/cli/libs/iamutil" "github.com/databricks/cli/libs/testdiff" - "github.com/databricks/databricks-sdk-go" - "github.com/databricks/databricks-sdk-go/service/iam" "github.com/stretchr/testify/assert" ) -var OverwriteMode = os.Getenv("TESTS_OUTPUT") == "OVERWRITE" - -func ReadFile(t testutil.TestingT, ctx context.Context, filename string) string { - data, err := os.ReadFile(filename) - if os.IsNotExist(err) { - return "" - } - assert.NoError(t, err) - // On CI, on Windows \n in the file somehow end up as \r\n - return NormalizeNewlines(string(data)) -} - func captureOutput(t testutil.TestingT, ctx context.Context, args []string) string { t.Logf("run args: [%s]", strings.Join(args, ", ")) r := NewRunner(t, ctx, args...) stdout, stderr, err := r.Run() assert.NoError(t, err) - out := stderr.String() + stdout.String() - return ReplaceOutput(t, ctx, out) -} - -func WriteFile(t testutil.TestingT, filename, data string) { - t.Logf("Overwriting %s", filename) - err := os.WriteFile(filename, []byte(data), 0o644) - assert.NoError(t, err) + return stderr.String() + stdout.String() } func AssertOutput(t testutil.TestingT, ctx context.Context, args []string, expectedPath string) { - expected := ReadFile(t, ctx, expectedPath) - out := captureOutput(t, ctx, args) - - if out != expected { - actual := fmt.Sprintf("Output from %v", args) - testdiff.AssertEqualTexts(t, expectedPath, actual, expected, out) - - if OverwriteMode { - WriteFile(t, expectedPath, out) - } - } + testdiff.AssertOutput(t, ctx, out, fmt.Sprintf("Output from %v", args), expectedPath) } func AssertOutputJQ(t testutil.TestingT, ctx context.Context, args []string, expectedPath string, ignorePaths []string) { - expected := ReadFile(t, ctx, expectedPath) - out := captureOutput(t, ctx, args) - - if out != expected { - actual := fmt.Sprintf("Output from %v", args) - testdiff.AssertEqualJQ(t.(*testing.T), expectedPath, actual, expected, out, ignorePaths) - - if OverwriteMode { - WriteFile(t, expectedPath, out) - } - } -} - -var ( - uuidRegex = regexp.MustCompile(`[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}`) - numIdRegex = regexp.MustCompile(`[0-9]{3,}`) - privatePathRegex = regexp.MustCompile(`(/tmp|/private)(/.*)/([a-zA-Z0-9]+)`) -) - -func ReplaceOutput(t testutil.TestingT, ctx context.Context, out string) string { - out = NormalizeNewlines(out) - replacements := GetReplacementsMap(ctx) - if replacements == nil { - t.Fatal("WithReplacementsMap was not called") - } - out = replacements.Replace(out) - out = uuidRegex.ReplaceAllString(out, "") - out = numIdRegex.ReplaceAllString(out, "") - out = privatePathRegex.ReplaceAllString(out, "/tmp/.../$3") - - return out -} - -type key int - -const ( - replacementsMapKey = key(1) -) - -type Replacement struct { - Old string - New string -} - -type ReplacementsContext struct { - Repls []Replacement -} - -func (r *ReplacementsContext) Replace(s string) string { - // QQQ Should probably only replace whole words - for _, repl := range r.Repls { - s = strings.ReplaceAll(s, repl.Old, repl.New) - } - return s -} - -func (r *ReplacementsContext) Set(old, new string) { - if old == "" || new == "" { - return - } - r.Repls = append(r.Repls, Replacement{Old: old, New: new}) -} - -func WithReplacementsMap(ctx context.Context) (context.Context, *ReplacementsContext) { - value := ctx.Value(replacementsMapKey) - if value != nil { - if existingMap, ok := value.(*ReplacementsContext); ok { - return ctx, existingMap - } - } - - newMap := &ReplacementsContext{} - ctx = context.WithValue(ctx, replacementsMapKey, newMap) - return ctx, newMap -} - -func GetReplacementsMap(ctx context.Context) *ReplacementsContext { - value := ctx.Value(replacementsMapKey) - if value != nil { - if existingMap, ok := value.(*ReplacementsContext); ok { - return existingMap - } - } - return nil -} - -func PrepareReplacements(t testutil.TestingT, r *ReplacementsContext, w *databricks.WorkspaceClient) { - // in some clouds (gcp) w.Config.Host includes "https://" prefix in others it's really just a host (azure) - host := strings.TrimPrefix(strings.TrimPrefix(w.Config.Host, "http://"), "https://") - r.Set(host, "$DATABRICKS_HOST") - r.Set(w.Config.ClusterID, "$DATABRICKS_CLUSTER_ID") - r.Set(w.Config.WarehouseID, "$DATABRICKS_WAREHOUSE_ID") - r.Set(w.Config.ServerlessComputeID, "$DATABRICKS_SERVERLESS_COMPUTE_ID") - r.Set(w.Config.MetadataServiceURL, "$DATABRICKS_METADATA_SERVICE_URL") - r.Set(w.Config.AccountID, "$DATABRICKS_ACCOUNT_ID") - r.Set(w.Config.Token, "$DATABRICKS_TOKEN") - r.Set(w.Config.Username, "$DATABRICKS_USERNAME") - r.Set(w.Config.Password, "$DATABRICKS_PASSWORD") - r.Set(w.Config.Profile, "$DATABRICKS_CONFIG_PROFILE") - r.Set(w.Config.ConfigFile, "$DATABRICKS_CONFIG_FILE") - r.Set(w.Config.GoogleServiceAccount, "$DATABRICKS_GOOGLE_SERVICE_ACCOUNT") - r.Set(w.Config.GoogleCredentials, "$GOOGLE_CREDENTIALS") - r.Set(w.Config.AzureResourceID, "$DATABRICKS_AZURE_RESOURCE_ID") - r.Set(w.Config.AzureClientSecret, "$ARM_CLIENT_SECRET") - // r.Set(w.Config.AzureClientID, "$ARM_CLIENT_ID") - r.Set(w.Config.AzureClientID, "$USERNAME") - r.Set(w.Config.AzureTenantID, "$ARM_TENANT_ID") - r.Set(w.Config.ActionsIDTokenRequestURL, "$ACTIONS_ID_TOKEN_REQUEST_URL") - r.Set(w.Config.ActionsIDTokenRequestToken, "$ACTIONS_ID_TOKEN_REQUEST_TOKEN") - r.Set(w.Config.AzureEnvironment, "$ARM_ENVIRONMENT") - r.Set(w.Config.ClientID, "$DATABRICKS_CLIENT_ID") - r.Set(w.Config.ClientSecret, "$DATABRICKS_CLIENT_SECRET") - r.Set(w.Config.DatabricksCliPath, "$DATABRICKS_CLI_PATH") - // This is set to words like "path" that happen too frequently - // r.Set(w.Config.AuthType, "$DATABRICKS_AUTH_TYPE") -} - -func PrepareReplacementsUser(t testutil.TestingT, r *ReplacementsContext, u iam.User) { - // There could be exact matches or overlap between different name fields, so sort them by length - // to ensure we match the largest one first and map them all to the same token - names := []string{ - u.DisplayName, - u.UserName, - iamutil.GetShortUserName(&u), - u.Name.FamilyName, - u.Name.GivenName, - } - if u.Name != nil { - names = append(names, u.Name.FamilyName) - names = append(names, u.Name.GivenName) - } - for _, val := range u.Emails { - names = append(names, val.Value) - } - stableSortReverseLength(names) - - for _, name := range names { - r.Set(name, "$USERNAME") - } - - for ind, val := range u.Groups { - r.Set(val.Value, fmt.Sprintf("$USER.Groups[%d]", ind)) - } - - r.Set(u.Id, "$USER.Id") - - for ind, val := range u.Roles { - r.Set(val.Value, fmt.Sprintf("$USER.Roles[%d]", ind)) - } -} - -func stableSortReverseLength(strs []string) { - slices.SortStableFunc(strs, func(a, b string) int { - return len(b) - len(a) - }) -} - -func NormalizeNewlines(input string) string { - output := strings.ReplaceAll(input, "\r\n", "\n") - return strings.ReplaceAll(output, "\r", "\n") + testdiff.AssertOutputJQ(t, ctx, out, fmt.Sprintf("Output from %v", args), expectedPath, ignorePaths) } diff --git a/libs/testdiff/golden.go b/libs/testdiff/golden.go new file mode 100644 index 000000000..c338ac9e5 --- /dev/null +++ b/libs/testdiff/golden.go @@ -0,0 +1,212 @@ +package testdiff + +import ( + "context" + "fmt" + "os" + "regexp" + "slices" + "strings" + "testing" + + "github.com/databricks/cli/internal/testutil" + "github.com/databricks/cli/libs/iamutil" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/service/iam" + "github.com/stretchr/testify/assert" +) + +var OverwriteMode = os.Getenv("TESTS_OUTPUT") == "OVERWRITE" + +func ReadFile(t testutil.TestingT, ctx context.Context, filename string) string { + data, err := os.ReadFile(filename) + if os.IsNotExist(err) { + return "" + } + assert.NoError(t, err) + // On CI, on Windows \n in the file somehow end up as \r\n + return NormalizeNewlines(string(data)) +} + +func WriteFile(t testutil.TestingT, filename, data string) { + t.Logf("Overwriting %s", filename) + err := os.WriteFile(filename, []byte(data), 0o644) + assert.NoError(t, err) +} + +func AssertOutput(t testutil.TestingT, ctx context.Context, out, outTitle, expectedPath string) { + expected := ReadFile(t, ctx, expectedPath) + + out = ReplaceOutput(t, ctx, out) + + if out != expected { + AssertEqualTexts(t, expectedPath, outTitle, expected, out) + + if OverwriteMode { + WriteFile(t, expectedPath, out) + } + } +} + +func AssertOutputJQ(t testutil.TestingT, ctx context.Context, out, outTitle, expectedPath string, ignorePaths []string) { + expected := ReadFile(t, ctx, expectedPath) + + out = ReplaceOutput(t, ctx, out) + + if out != expected { + AssertEqualJQ(t.(*testing.T), expectedPath, outTitle, expected, out, ignorePaths) + + if OverwriteMode { + WriteFile(t, expectedPath, out) + } + } +} + +var ( + uuidRegex = regexp.MustCompile(`[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}`) + numIdRegex = regexp.MustCompile(`[0-9]{3,}`) + privatePathRegex = regexp.MustCompile(`(/tmp|/private)(/.*)/([a-zA-Z0-9]+)`) +) + +func ReplaceOutput(t testutil.TestingT, ctx context.Context, out string) string { + out = NormalizeNewlines(out) + replacements := GetReplacementsMap(ctx) + if replacements == nil { + t.Fatal("WithReplacementsMap was not called") + } + out = replacements.Replace(out) + out = uuidRegex.ReplaceAllString(out, "") + out = numIdRegex.ReplaceAllString(out, "") + out = privatePathRegex.ReplaceAllString(out, "/tmp/.../$3") + + return out +} + +type key int + +const ( + replacementsMapKey = key(1) +) + +type Replacement struct { + Old string + New string +} + +type ReplacementsContext struct { + Repls []Replacement +} + +func (r *ReplacementsContext) Replace(s string) string { + // QQQ Should probably only replace whole words + for _, repl := range r.Repls { + s = strings.ReplaceAll(s, repl.Old, repl.New) + } + return s +} + +func (r *ReplacementsContext) Set(old, new string) { + if old == "" || new == "" { + return + } + r.Repls = append(r.Repls, Replacement{Old: old, New: new}) +} + +func WithReplacementsMap(ctx context.Context) (context.Context, *ReplacementsContext) { + value := ctx.Value(replacementsMapKey) + if value != nil { + if existingMap, ok := value.(*ReplacementsContext); ok { + return ctx, existingMap + } + } + + newMap := &ReplacementsContext{} + ctx = context.WithValue(ctx, replacementsMapKey, newMap) + return ctx, newMap +} + +func GetReplacementsMap(ctx context.Context) *ReplacementsContext { + value := ctx.Value(replacementsMapKey) + if value != nil { + if existingMap, ok := value.(*ReplacementsContext); ok { + return existingMap + } + } + return nil +} + +func PrepareReplacements(t testutil.TestingT, r *ReplacementsContext, w *databricks.WorkspaceClient) { + // in some clouds (gcp) w.Config.Host includes "https://" prefix in others it's really just a host (azure) + host := strings.TrimPrefix(strings.TrimPrefix(w.Config.Host, "http://"), "https://") + r.Set(host, "$DATABRICKS_HOST") + r.Set(w.Config.ClusterID, "$DATABRICKS_CLUSTER_ID") + r.Set(w.Config.WarehouseID, "$DATABRICKS_WAREHOUSE_ID") + r.Set(w.Config.ServerlessComputeID, "$DATABRICKS_SERVERLESS_COMPUTE_ID") + r.Set(w.Config.MetadataServiceURL, "$DATABRICKS_METADATA_SERVICE_URL") + r.Set(w.Config.AccountID, "$DATABRICKS_ACCOUNT_ID") + r.Set(w.Config.Token, "$DATABRICKS_TOKEN") + r.Set(w.Config.Username, "$DATABRICKS_USERNAME") + r.Set(w.Config.Password, "$DATABRICKS_PASSWORD") + r.Set(w.Config.Profile, "$DATABRICKS_CONFIG_PROFILE") + r.Set(w.Config.ConfigFile, "$DATABRICKS_CONFIG_FILE") + r.Set(w.Config.GoogleServiceAccount, "$DATABRICKS_GOOGLE_SERVICE_ACCOUNT") + r.Set(w.Config.GoogleCredentials, "$GOOGLE_CREDENTIALS") + r.Set(w.Config.AzureResourceID, "$DATABRICKS_AZURE_RESOURCE_ID") + r.Set(w.Config.AzureClientSecret, "$ARM_CLIENT_SECRET") + // r.Set(w.Config.AzureClientID, "$ARM_CLIENT_ID") + r.Set(w.Config.AzureClientID, "$USERNAME") + r.Set(w.Config.AzureTenantID, "$ARM_TENANT_ID") + r.Set(w.Config.ActionsIDTokenRequestURL, "$ACTIONS_ID_TOKEN_REQUEST_URL") + r.Set(w.Config.ActionsIDTokenRequestToken, "$ACTIONS_ID_TOKEN_REQUEST_TOKEN") + r.Set(w.Config.AzureEnvironment, "$ARM_ENVIRONMENT") + r.Set(w.Config.ClientID, "$DATABRICKS_CLIENT_ID") + r.Set(w.Config.ClientSecret, "$DATABRICKS_CLIENT_SECRET") + r.Set(w.Config.DatabricksCliPath, "$DATABRICKS_CLI_PATH") + // This is set to words like "path" that happen too frequently + // r.Set(w.Config.AuthType, "$DATABRICKS_AUTH_TYPE") +} + +func PrepareReplacementsUser(t testutil.TestingT, r *ReplacementsContext, u iam.User) { + // There could be exact matches or overlap between different name fields, so sort them by length + // to ensure we match the largest one first and map them all to the same token + names := []string{ + u.DisplayName, + u.UserName, + iamutil.GetShortUserName(&u), + u.Name.FamilyName, + u.Name.GivenName, + } + if u.Name != nil { + names = append(names, u.Name.FamilyName) + names = append(names, u.Name.GivenName) + } + for _, val := range u.Emails { + names = append(names, val.Value) + } + stableSortReverseLength(names) + + for _, name := range names { + r.Set(name, "$USERNAME") + } + + for ind, val := range u.Groups { + r.Set(val.Value, fmt.Sprintf("$USER.Groups[%d]", ind)) + } + + r.Set(u.Id, "$USER.Id") + + for ind, val := range u.Roles { + r.Set(val.Value, fmt.Sprintf("$USER.Roles[%d]", ind)) + } +} + +func stableSortReverseLength(strs []string) { + slices.SortStableFunc(strs, func(a, b string) int { + return len(b) - len(a) + }) +} + +func NormalizeNewlines(input string) string { + output := strings.ReplaceAll(input, "\r\n", "\n") + return strings.ReplaceAll(output, "\r", "\n") +} diff --git a/internal/testcli/golden_test.go b/libs/testdiff/golden_test.go similarity index 92% rename from internal/testcli/golden_test.go rename to libs/testdiff/golden_test.go index 215bf33d3..0fc32be21 100644 --- a/internal/testcli/golden_test.go +++ b/libs/testdiff/golden_test.go @@ -1,4 +1,4 @@ -package testcli +package testdiff import ( "testing" From 1306e5ec6745b36acd7c17c9b3b4aea1f55d6807 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 30 Dec 2024 18:41:45 +0100 Subject: [PATCH 05/19] Add CODEOWNERS (#2055) Goal is to have DABs core team automatically added as reviewers so that you don't have to click manually. Based on this example: https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners#example-of-a-codeowners-file --- .github/CODEOWNERS | 1 + 1 file changed, 1 insertion(+) create mode 100644 .github/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 000000000..76835de7d --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +* @pietern @andrewnester @shreyas-goenka @denik From 1ce20a2612031a3d5c7b663203296957671e4f1d Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 30 Dec 2024 19:39:33 +0100 Subject: [PATCH 06/19] lint.sh: read config for formatters; include gofmt (#2056) As suggested here: https://github.com/databricks/cli/pull/2051#discussion_r1899641273 --- lint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lint.sh b/lint.sh index c93d04c66..13efa9855 100755 --- a/lint.sh +++ b/lint.sh @@ -5,5 +5,5 @@ set -euo pipefail # However, running goimports first alone will actually fix some of the compilation issues. # Fixing formatting is also reasonable thing to do. # For this reason, this script runs golangci-lint in two stages: -golangci-lint run --fix --no-config --disable-all --enable gofumpt,goimports $@ +golangci-lint run --enable-only="gofmt,gofumpt,goimports" --fix $@ exec golangci-lint run --fix $@ From 511db52bb7a47270dd883bb490a8844f9b25fe76 Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Tue, 31 Dec 2024 15:21:13 +0530 Subject: [PATCH 07/19] Remove unnecessary GET call in pipeline runner (#1850) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Changes This GET API call is unnecessary and serves no purpose. Let's remove it. Noticed this when I was adding a unit test for the pipeline runner here: https://github.com/databricks/cli/pull/1849 ## Tests Manually. ### Case 1: The pipeline does not exist Before: ``` ➜ my_project git:(master) ✗ databricks bundle run my_project_pipeline -p dogfood Error: User shreyas.goenka@databricks.com does not have View permissions on pipeline 9941901a-e48b-4d04-b6ba-e0072ad126bg. ``` After: ``` ➜ my_project git:(master) ✗ cli bundle run my_project_pipeline -p dogfood Error: User shreyas.goenka@databricks.com does not have Run permissions on pipeline 9941901a-e48b-4d04-b6ba-e0072ad126bg. ``` ### Case 2: Pipeline exists Before: ``` ➜ my_project git:(master) ✗ databricks bundle run my_project_pipeline -p dogfood --restart Update URL: https://e2-dogfood.staging.cloud.databricks.com/#joblist/pipelines/9941901a-e48b-4d04-b6ba-e0072ad126bf/updates/0f988d62-9ec7-49f1-b429-5572ece3a9aa 2024-11-18T15:30:36.054Z update_progress INFO "Update 0f988d is WAITING_FOR_RESOURCES." ``` After: ``` ➜ my_project git:(master) ✗ cli bundle run my_project_pipeline -p dogfood --restart Update URL: https://e2-dogfood.staging.cloud.databricks.com/#joblist/pipelines/9941901a-e48b-4d04-b6ba-e0072ad126bf/updates/87b43350-6186-4a9b-9d0e-38da2ecf33ae 2024-11-18T15:28:27.144Z update_progress INFO "Update 87b433 is WAITING_FOR_RESOURCES." ``` --- bundle/run/pipeline.go | 5 ----- bundle/run/pipeline_test.go | 2 -- 2 files changed, 7 deletions(-) diff --git a/bundle/run/pipeline.go b/bundle/run/pipeline.go index a0e7d1e1e..d84015d76 100644 --- a/bundle/run/pipeline.go +++ b/bundle/run/pipeline.go @@ -90,11 +90,6 @@ func (r *pipelineRunner) Run(ctx context.Context, opts *Options) (output.RunOutp // Include resource key in logger. ctx = log.NewContext(ctx, log.GetLogger(ctx).With("resource", r.Key())) w := r.bundle.WorkspaceClient() - _, err := w.Pipelines.GetByPipelineId(ctx, pipelineID) - if err != nil { - log.Warnf(ctx, "Cannot get pipeline: %s", err) - return nil, err - } req, err := opts.Pipeline.toPayload(r.pipeline, pipelineID) if err != nil { diff --git a/bundle/run/pipeline_test.go b/bundle/run/pipeline_test.go index 66f9d86be..bfa0c5846 100644 --- a/bundle/run/pipeline_test.go +++ b/bundle/run/pipeline_test.go @@ -90,8 +90,6 @@ func TestPipelineRunnerRestart(t *testing.T) { PipelineId: "123", }).Return(mockWait, nil) - pipelineApi.EXPECT().GetByPipelineId(mock.Anything, "123").Return(&pipelines.GetPipelineResponse{}, nil) - // Mock runner starting a new update pipelineApi.EXPECT().StartUpdate(mock.Anything, pipelines.StartUpdate{ PipelineId: "123", From 3f523b45cc59aa2cb4fbbffdf267e8ea094b3ad6 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 31 Dec 2024 15:01:45 +0100 Subject: [PATCH 08/19] Fix lost diags across different mutators (#2057) ## Changes Fix cases where accumulated diagnostics are lost instead of being propagated further. In some cases it's not possible, add a comment there. ## Tests Existing tests --- bundle/artifacts/expand_globs.go | 2 +- bundle/config/mutator/resolve_resource_references.go | 1 + bundle/config/root.go | 3 ++- bundle/config/validate/folder_permissions.go | 3 ++- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/bundle/artifacts/expand_globs.go b/bundle/artifacts/expand_globs.go index 7d44db0be..c0af7c69e 100644 --- a/bundle/artifacts/expand_globs.go +++ b/bundle/artifacts/expand_globs.go @@ -97,7 +97,7 @@ func (m *expandGlobs) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnost return dyn.SetByPath(v, base, dyn.V(output)) }) if err != nil { - return diag.FromErr(err) + diags = diags.Extend(diag.FromErr(err)) } return diags diff --git a/bundle/config/mutator/resolve_resource_references.go b/bundle/config/mutator/resolve_resource_references.go index bf902f928..20a5b6585 100644 --- a/bundle/config/mutator/resolve_resource_references.go +++ b/bundle/config/mutator/resolve_resource_references.go @@ -40,6 +40,7 @@ func (m *resolveResourceReferences) Apply(ctx context.Context, b *bundle.Bundle) }) } + // Note, diags are lost from all goroutines except the first one to return diag return diag.FromErr(errs.Wait()) } diff --git a/bundle/config/root.go b/bundle/config/root.go index 4b1467456..91c15fd9d 100644 --- a/bundle/config/root.go +++ b/bundle/config/root.go @@ -102,7 +102,8 @@ func LoadFromBytes(path string, raw []byte) (*Root, diag.Diagnostics) { // Convert normalized configuration tree to typed configuration. err = r.updateWithDynamicValue(v) if err != nil { - return nil, diag.Errorf("failed to load %s: %v", path, err) + diags = diags.Extend(diag.Errorf("failed to load %s: %v", path, err)) + return nil, diags } return &r, diags } diff --git a/bundle/config/validate/folder_permissions.go b/bundle/config/validate/folder_permissions.go index aa89a0551..5b28d599e 100644 --- a/bundle/config/validate/folder_permissions.go +++ b/bundle/config/validate/folder_permissions.go @@ -36,7 +36,8 @@ func (f *folderPermissions) Apply(ctx context.Context, b bundle.ReadOnlyBundle) } if err := g.Wait(); err != nil { - return diag.FromErr(err) + // Note, only diag from first coroutine is captured, others are lost + diags = diags.Extend(diag.FromErr(err)) } for _, r := range results { From 3f75240a562d88a9411cdd50a2ba24139ca094b7 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 2 Jan 2025 10:49:21 +0100 Subject: [PATCH 09/19] Improve test output to include correct location (#2058) ## Changes - Add t.Helper() in testcli-related helpers, this ensures that output is attributed correctly to test case and not to the helper. - Modify testlcli.Run() to run process in foreground. This is needed for t.Helper to work. - Extend a few assertions with message to help attribute it to proper helper where needed. ## Tests Manually reviewed test output. Before: ``` + go test --timeout 3h -v -run TestDefaultPython/3.9 ./integration/bundle/ === RUN TestDefaultPython === RUN TestDefaultPython/3.9 workspace.go:26: aws golden.go:14: run args: [bundle, init, default-python, --config-file, config.json] runner.go:206: [databricks stderr]: runner.go:206: [databricks stderr]: Welcome to the default Python template for Databricks Asset Bundles! ... testdiff.go:23: Error Trace: /Users/denis.bilenko/work/cli/libs/testdiff/testdiff.go:23 /Users/denis.bilenko/work/cli/libs/testdiff/golden.go:43 /Users/denis.bilenko/work/cli/internal/testcli/golden.go:23 /Users/denis.bilenko/work/cli/integration/bundle/init_default_python_test.go:92 /Users/denis.bilenko/work/cli/integration/bundle/init_default_python_test.go:45 ... ``` After: ``` + go test --timeout 3h -v -run TestDefaultPython/3.9 ./integration/bundle/ === RUN TestDefaultPython === RUN TestDefaultPython/3.9 init_default_python_test.go:51: CLOUD_ENV=aws init_default_python_test.go:92: args: bundle, init, default-python, --config-file, config.json init_default_python_test.go:92: stderr: init_default_python_test.go:92: stderr: Welcome to the default Python template for Databricks Asset Bundles! ... init_default_python_test.go:92: Error Trace: /Users/denis.bilenko/work/cli/libs/testdiff/testdiff.go:24 /Users/denis.bilenko/work/cli/libs/testdiff/golden.go:46 /Users/denis.bilenko/work/cli/internal/testcli/golden.go:23 /Users/denis.bilenko/work/cli/integration/bundle/init_default_python_test.go:92 /Users/denis.bilenko/work/cli/integration/bundle/init_default_python_test.go:45 ... ``` --- integration/internal/acc/workspace.go | 14 +++++--- internal/testcli/golden.go | 5 +-- internal/testcli/runner.go | 47 ++++++++++++++++++++++++--- internal/testutil/interface.go | 2 ++ libs/testdiff/golden.go | 11 +++++-- libs/testdiff/testdiff.go | 4 ++- 6 files changed, 69 insertions(+), 14 deletions(-) diff --git a/integration/internal/acc/workspace.go b/integration/internal/acc/workspace.go index 2f8a5b8e7..64deda7c1 100644 --- a/integration/internal/acc/workspace.go +++ b/integration/internal/acc/workspace.go @@ -21,9 +21,10 @@ type WorkspaceT struct { } func WorkspaceTest(t testutil.TestingT) (context.Context, *WorkspaceT) { + t.Helper() loadDebugEnvIfRunFromIDE(t, "workspace") - t.Log(testutil.GetEnvOrSkipTest(t, "CLOUD_ENV")) + t.Logf("CLOUD_ENV=%s", testutil.GetEnvOrSkipTest(t, "CLOUD_ENV")) w, err := databricks.NewWorkspaceClient() require.NoError(t, err) @@ -41,9 +42,10 @@ func WorkspaceTest(t testutil.TestingT) (context.Context, *WorkspaceT) { // Run the workspace test only on UC workspaces. func UcWorkspaceTest(t testutil.TestingT) (context.Context, *WorkspaceT) { + t.Helper() loadDebugEnvIfRunFromIDE(t, "workspace") - t.Log(testutil.GetEnvOrSkipTest(t, "CLOUD_ENV")) + t.Logf("CLOUD_ENV=%s", testutil.GetEnvOrSkipTest(t, "CLOUD_ENV")) if os.Getenv("TEST_METASTORE_ID") == "" { t.Skipf("Skipping on non-UC workspaces") @@ -67,19 +69,21 @@ func UcWorkspaceTest(t testutil.TestingT) (context.Context, *WorkspaceT) { } func (t *WorkspaceT) TestClusterID() string { + t.Helper() clusterID := testutil.GetEnvOrSkipTest(t, "TEST_BRICKS_CLUSTER_ID") err := t.W.Clusters.EnsureClusterIsRunning(t.ctx, clusterID) - require.NoError(t, err) + require.NoError(t, err, "Unexpected error from EnsureClusterIsRunning for clusterID=%s", clusterID) return clusterID } func (t *WorkspaceT) RunPython(code string) (string, error) { + t.Helper() var err error // Create command executor only once per test. if t.exec == nil { t.exec, err = t.W.CommandExecution.Start(t.ctx, t.TestClusterID(), compute.LanguagePython) - require.NoError(t, err) + require.NoError(t, err, "Unexpected error from CommandExecution.Start(clusterID=%v)", t.TestClusterID()) t.Cleanup(func() { err := t.exec.Destroy(t.ctx) @@ -88,7 +92,7 @@ func (t *WorkspaceT) RunPython(code string) (string, error) { } results, err := t.exec.Execute(t.ctx, code) - require.NoError(t, err) + require.NoError(t, err, "Unexpected error from Execute(%v)", code) require.NotEqual(t, compute.ResultTypeError, results.ResultType, results.Cause) output, ok := results.Data.(string) require.True(t, ok, "unexpected type %T", results.Data) diff --git a/internal/testcli/golden.go b/internal/testcli/golden.go index a16f42c72..669cc2f9b 100644 --- a/internal/testcli/golden.go +++ b/internal/testcli/golden.go @@ -3,7 +3,6 @@ package testcli import ( "context" "fmt" - "strings" "github.com/databricks/cli/internal/testutil" "github.com/databricks/cli/libs/testdiff" @@ -11,7 +10,7 @@ import ( ) func captureOutput(t testutil.TestingT, ctx context.Context, args []string) string { - t.Logf("run args: [%s]", strings.Join(args, ", ")) + t.Helper() r := NewRunner(t, ctx, args...) stdout, stderr, err := r.Run() assert.NoError(t, err) @@ -19,11 +18,13 @@ func captureOutput(t testutil.TestingT, ctx context.Context, args []string) stri } func AssertOutput(t testutil.TestingT, ctx context.Context, args []string, expectedPath string) { + t.Helper() out := captureOutput(t, ctx, args) testdiff.AssertOutput(t, ctx, out, fmt.Sprintf("Output from %v", args), expectedPath) } func AssertOutputJQ(t testutil.TestingT, ctx context.Context, args []string, expectedPath string, ignorePaths []string) { + t.Helper() out := captureOutput(t, ctx, args) testdiff.AssertOutputJQ(t, ctx, out, fmt.Sprintf("Output from %v", args), expectedPath, ignorePaths) } diff --git a/internal/testcli/runner.go b/internal/testcli/runner.go index 95073b57c..52decad2c 100644 --- a/internal/testcli/runner.go +++ b/internal/testcli/runner.go @@ -69,6 +69,7 @@ func consumeLines(ctx context.Context, wg *sync.WaitGroup, r io.Reader) <-chan s } func (r *Runner) registerFlagCleanup(c *cobra.Command) { + r.Helper() // Find target command that will be run. Example: if the command run is `databricks fs cp`, // target command corresponds to `cp` targetCmd, _, err := c.Find(r.args) @@ -230,13 +231,48 @@ func (r *Runner) RunBackground() { } func (r *Runner) Run() (bytes.Buffer, bytes.Buffer, error) { - r.RunBackground() - err := <-r.errch - return r.stdout, r.stderr, err + r.Helper() + var stdout, stderr bytes.Buffer + ctx := cmdio.NewContext(r.ctx, &cmdio.Logger{ + Mode: flags.ModeAppend, + Reader: bufio.Reader{}, + Writer: &stderr, + }) + + cli := cmd.New(ctx) + cli.SetOut(&stdout) + cli.SetErr(&stderr) + cli.SetArgs(r.args) + + r.Logf(" args: %s", strings.Join(r.args, ", ")) + + err := root.Execute(ctx, cli) + if err != nil { + r.Logf(" error: %s", err) + } + + if stdout.Len() > 0 { + // Make a copy of the buffer such that it remains "unread". + scanner := bufio.NewScanner(bytes.NewBuffer(stdout.Bytes())) + for scanner.Scan() { + r.Logf("stdout: %s", scanner.Text()) + } + } + + if stderr.Len() > 0 { + // Make a copy of the buffer such that it remains "unread". + scanner := bufio.NewScanner(bytes.NewBuffer(stderr.Bytes())) + for scanner.Scan() { + r.Logf("stderr: %s", scanner.Text()) + } + } + + return stdout, stderr, err } // Like [require.Eventually] but errors if the underlying command has failed. func (r *Runner) Eventually(condition func() bool, waitFor, tick time.Duration, msgAndArgs ...any) { + r.Helper() ch := make(chan bool, 1) timer := time.NewTimer(waitFor) @@ -269,12 +305,14 @@ func (r *Runner) Eventually(condition func() bool, waitFor, tick time.Duration, } func (r *Runner) RunAndExpectOutput(heredoc string) { + r.Helper() stdout, _, err := r.Run() require.NoError(r, err) require.Equal(r, cmdio.Heredoc(heredoc), strings.TrimSpace(stdout.String())) } func (r *Runner) RunAndParseJSON(v any) { + r.Helper() stdout, _, err := r.Run() require.NoError(r, err) err = json.Unmarshal(stdout.Bytes(), &v) @@ -291,7 +329,7 @@ func NewRunner(t testutil.TestingT, ctx context.Context, args ...string) *Runner } func RequireSuccessfulRun(t testutil.TestingT, ctx context.Context, args ...string) (bytes.Buffer, bytes.Buffer) { - t.Logf("run args: [%s]", strings.Join(args, ", ")) + t.Helper() r := NewRunner(t, ctx, args...) stdout, stderr, err := r.Run() require.NoError(t, err) @@ -299,6 +337,7 @@ func RequireSuccessfulRun(t testutil.TestingT, ctx context.Context, args ...stri } func RequireErrorRun(t testutil.TestingT, ctx context.Context, args ...string) (bytes.Buffer, bytes.Buffer, error) { + t.Helper() r := NewRunner(t, ctx, args...) stdout, stderr, err := r.Run() require.Error(t, err) diff --git a/internal/testutil/interface.go b/internal/testutil/interface.go index 2c3004800..97441212d 100644 --- a/internal/testutil/interface.go +++ b/internal/testutil/interface.go @@ -24,4 +24,6 @@ type TestingT interface { Setenv(key, value string) TempDir() string + + Helper() } diff --git a/libs/testdiff/golden.go b/libs/testdiff/golden.go index c338ac9e5..b67eb50a9 100644 --- a/libs/testdiff/golden.go +++ b/libs/testdiff/golden.go @@ -19,22 +19,25 @@ import ( var OverwriteMode = os.Getenv("TESTS_OUTPUT") == "OVERWRITE" func ReadFile(t testutil.TestingT, ctx context.Context, filename string) string { + t.Helper() data, err := os.ReadFile(filename) if os.IsNotExist(err) { return "" } - assert.NoError(t, err) + assert.NoError(t, err, "Failed to read %s", filename) // On CI, on Windows \n in the file somehow end up as \r\n return NormalizeNewlines(string(data)) } func WriteFile(t testutil.TestingT, filename, data string) { + t.Helper() t.Logf("Overwriting %s", filename) err := os.WriteFile(filename, []byte(data), 0o644) - assert.NoError(t, err) + assert.NoError(t, err, "Failed to write %s", filename) } func AssertOutput(t testutil.TestingT, ctx context.Context, out, outTitle, expectedPath string) { + t.Helper() expected := ReadFile(t, ctx, expectedPath) out = ReplaceOutput(t, ctx, out) @@ -49,6 +52,7 @@ func AssertOutput(t testutil.TestingT, ctx context.Context, out, outTitle, expec } func AssertOutputJQ(t testutil.TestingT, ctx context.Context, out, outTitle, expectedPath string, ignorePaths []string) { + t.Helper() expected := ReadFile(t, ctx, expectedPath) out = ReplaceOutput(t, ctx, out) @@ -69,6 +73,7 @@ var ( ) func ReplaceOutput(t testutil.TestingT, ctx context.Context, out string) string { + t.Helper() out = NormalizeNewlines(out) replacements := GetReplacementsMap(ctx) if replacements == nil { @@ -136,6 +141,7 @@ func GetReplacementsMap(ctx context.Context) *ReplacementsContext { } func PrepareReplacements(t testutil.TestingT, r *ReplacementsContext, w *databricks.WorkspaceClient) { + t.Helper() // in some clouds (gcp) w.Config.Host includes "https://" prefix in others it's really just a host (azure) host := strings.TrimPrefix(strings.TrimPrefix(w.Config.Host, "http://"), "https://") r.Set(host, "$DATABRICKS_HOST") @@ -167,6 +173,7 @@ func PrepareReplacements(t testutil.TestingT, r *ReplacementsContext, w *databri } func PrepareReplacementsUser(t testutil.TestingT, r *ReplacementsContext, u iam.User) { + t.Helper() // There could be exact matches or overlap between different name fields, so sort them by length // to ensure we match the largest one first and map them all to the same token names := []string{ diff --git a/libs/testdiff/testdiff.go b/libs/testdiff/testdiff.go index 1e1df727a..71b781362 100644 --- a/libs/testdiff/testdiff.go +++ b/libs/testdiff/testdiff.go @@ -18,9 +18,10 @@ func UnifiedDiff(filename1, filename2, s1, s2 string) string { } func AssertEqualTexts(t testutil.TestingT, filename1, filename2, expected, out string) { + t.Helper() if len(out) < 1000 && len(expected) < 1000 { // This shows full strings + diff which could be useful when debugging newlines - assert.Equal(t, expected, out) + assert.Equal(t, expected, out, "%s vs %s", filename1, filename2) } else { // only show diff for large texts diff := UnifiedDiff(filename1, filename2, expected, out) @@ -29,6 +30,7 @@ func AssertEqualTexts(t testutil.TestingT, filename1, filename2, expected, out s } func AssertEqualJQ(t testutil.TestingT, expectedName, outName, expected, out string, ignorePaths []string) { + t.Helper() patch, err := jsondiff.CompareJSON([]byte(expected), []byte(out)) if err != nil { t.Logf("CompareJSON error for %s vs %s: %s (fallback to textual comparison)", outName, expectedName, err) From d7f69f6a5dc80450498119a7e2a60eecfc3f0cab Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 2 Jan 2025 11:31:35 +0100 Subject: [PATCH 10/19] Upgrade golangci-lint to v1.63.1 (#2064) Upgrade your laptops with: brew install golangci-lint This has a lot more autofixes, which makes it easier to adopt those linters. https://golangci-lint.run/product/changelog/#v1630 --- .github/workflows/push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index a51927594..c737993aa 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -75,7 +75,7 @@ jobs: - name: golangci-lint uses: golangci/golangci-lint-action@v6 with: - version: v1.62.2 + version: v1.63.1 args: --timeout=15m validate-bundle-schema: From c262b75c3d0e48839fe327697e2758d3120d780e Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 2 Jan 2025 11:33:06 +0100 Subject: [PATCH 11/19] Make lint.sh to run golangci-lint only once in the best case (#2062) Follow up to #2051 and #2056. Running golangci-lint twice always is measurably slower (1.5s vs 2.5s), so only run it twice in case it is necessary. --- lint.sh | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/lint.sh b/lint.sh index 13efa9855..1f881eaf7 100755 --- a/lint.sh +++ b/lint.sh @@ -1,9 +1,14 @@ #!/bin/bash -set -euo pipefail +set -uo pipefail # With golangci-lint, if there are any compliation issues, then formatters' autofix won't be applied. # https://github.com/golangci/golangci-lint/issues/5257 -# However, running goimports first alone will actually fix some of the compilation issues. -# Fixing formatting is also reasonable thing to do. -# For this reason, this script runs golangci-lint in two stages: -golangci-lint run --enable-only="gofmt,gofumpt,goimports" --fix $@ -exec golangci-lint run --fix $@ + +golangci-lint run --fix "$@" +lint_exit_code=$? + +if [ $lint_exit_code -ne 0 ]; then + # These linters work in presence of compilation issues when run alone, so let's get these fixes at least. + golangci-lint run --enable-only="gofmt,gofumpt,goimports" --fix "$@" +fi + +exit $lint_exit_code From b053bfb4de577d8ee2d4058672a694af69d80891 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Thu, 2 Jan 2025 11:41:55 +0100 Subject: [PATCH 12/19] Verify that the bundle schema is up to date in CI (#2061) ## Changes I noticed a diff in the schema in #2052. This check should be performed automatically. ## Tests This PR includes a commit that changes the schema to check that the workflow actually fails. --- .github/workflows/push.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index c737993aa..25bd5f4d6 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -90,6 +90,13 @@ jobs: with: go-version: 1.23.4 + - name: Verify that the schema is up to date + run: | + if ! ( make schema && git diff --exit-code ); then + echo "The schema is not up to date. Please run 'make schema' and commit the changes." + exit 1 + fi + # Github repo: https://github.com/ajv-validator/ajv-cli - name: Install ajv-cli run: npm install -g ajv-cli@5.0.0 From 0b80784df759673d0ccc3874d19d6662ff11ac54 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 2 Jan 2025 12:03:41 +0100 Subject: [PATCH 13/19] Enable testifylint and fix the issues (#2065) ## Changes - Enable new linter: testifylint. - Apply fixes with --fix. - Fix remaining issues (mostly with aider). There were 2 cases we --fix did the wrong thing - this seems to a be a bug in linter: https://github.com/Antonboom/testifylint/issues/210 Nonetheless, I kept that check enabled, it seems useful, just need to be fixed manually after autofix. ## Tests Existing tests --- .golangci.yaml | 8 +++- bundle/bundle_test.go | 3 +- .../configure_dashboard_defaults_test.go | 6 +-- .../config/mutator/default_queueing_test.go | 10 ++-- .../mutator/environments_compat_test.go | 4 +- bundle/config/mutator/merge_job_tasks_test.go | 4 +- .../mutator/process_target_mode_test.go | 12 ++--- .../resolve_variable_references_test.go | 6 +-- .../mutator/rewrite_workspace_prefix_test.go | 2 +- bundle/config/mutator/set_variables_test.go | 8 ++-- bundle/config/resources_test.go | 6 +-- bundle/config/validate/files_to_sync_test.go | 4 +- .../validate/job_cluster_key_defined_test.go | 10 ++-- bundle/context_test.go | 2 +- bundle/deploy/state_pull_test.go | 3 +- bundle/deploy/state_update_test.go | 12 ++--- bundle/deploy/terraform/convert_test.go | 10 ++-- bundle/deploy/terraform/init_test.go | 4 +- bundle/internal/schema/main_test.go | 3 +- bundle/libraries/filer_volume_test.go | 4 +- bundle/libraries/helpers_test.go | 12 ++--- bundle/permissions/filter_test.go | 28 +++++------ bundle/permissions/mutator_test.go | 2 +- .../permission_diagnostics_test.go | 2 +- bundle/run/job_test.go | 2 +- bundle/tests/complex_variables_test.go | 2 +- bundle/tests/environment_git_test.go | 5 +- bundle/tests/environment_overrides_test.go | 8 ++-- .../environments_job_and_pipeline_test.go | 6 +-- bundle/tests/git_test.go | 5 +- bundle/tests/issue_1828_test.go | 4 +- bundle/tests/job_and_pipeline_test.go | 6 +-- bundle/tests/job_cluster_key_test.go | 6 +-- bundle/tests/model_serving_endpoint_test.go | 2 +- bundle/tests/override_job_tasks_test.go | 28 +++++------ bundle/tests/presets_test.go | 4 +- bundle/tests/python_wheel_test.go | 10 ++-- bundle/tests/quality_monitor_test.go | 2 +- bundle/tests/registered_model_test.go | 2 +- .../sync_include_exclude_no_matches_test.go | 20 ++++---- bundle/tests/sync_test.go | 4 +- cmd/bundle/generate/dashboard_test.go | 2 +- cmd/bundle/generate/generate_test.go | 3 +- cmd/configure/configure_test.go | 4 +- cmd/labs/github/ref_test.go | 9 ++-- cmd/labs/github/releases_test.go | 5 +- cmd/labs/github/repositories_test.go | 5 +- cmd/labs/localcache/jsonfile_test.go | 2 +- cmd/labs/project/installer_test.go | 17 +++---- integration/bundle/artifacts_test.go | 12 ++--- integration/bundle/deploy_test.go | 3 +- integration/bundle/destroy_test.go | 5 +- integration/cmd/fs/ls_test.go | 15 +++--- integration/cmd/fs/mkdir_test.go | 8 ++-- integration/cmd/sync/sync_test.go | 2 +- integration/libs/filer/filer_test.go | 46 +++++++++---------- libs/auth/cache/file_test.go | 2 +- libs/databrickscfg/ops_test.go | 2 +- libs/databrickscfg/profile/file_test.go | 8 ++-- libs/errs/aggregate_test.go | 19 ++++---- libs/filer/completer/completer_test.go | 12 ++--- libs/filer/fs_test.go | 4 +- libs/fileset/glob_test.go | 2 +- libs/flags/json_flag_test.go | 8 ++-- libs/folders/folders_test.go | 2 +- libs/jsonschema/utils_test.go | 4 +- libs/notebook/detect_test.go | 5 +- libs/process/background_test.go | 2 +- libs/process/forwarded_test.go | 4 +- libs/process/opts_test.go | 2 +- libs/python/interpreters_unix_test.go | 2 +- libs/sync/snapshot_test.go | 24 +++++----- libs/sync/sync_test.go | 12 ++--- libs/template/config_test.go | 8 ++-- libs/template/helpers_test.go | 2 +- libs/template/renderer_test.go | 8 ++-- libs/vfs/filer_test.go | 9 ++-- 77 files changed, 277 insertions(+), 283 deletions(-) diff --git a/.golangci.yaml b/.golangci.yaml index 9e69e5146..fd83e0882 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -11,6 +11,7 @@ linters: - gofmt - gofumpt - goimports + - testifylint linters-settings: govet: enable-all: true @@ -32,7 +33,10 @@ linters-settings: gofumpt: module-path: github.com/databricks/cli extra-rules: true - #goimports: - # local-prefixes: github.com/databricks/cli + testifylint: + enable-all: true + disable: + # good check, but we have too many assert.(No)?Errorf? so excluding for now + - require-error issues: exclude-dirs-use-default: false # recommended by docs https://golangci-lint.run/usage/false-positives/ diff --git a/bundle/bundle_test.go b/bundle/bundle_test.go index 1c3102357..d52088e34 100644 --- a/bundle/bundle_test.go +++ b/bundle/bundle_test.go @@ -2,7 +2,6 @@ package bundle import ( "context" - "errors" "io/fs" "os" "path/filepath" @@ -16,7 +15,7 @@ import ( func TestLoadNotExists(t *testing.T) { b, err := Load(context.Background(), "/doesntexist") - assert.True(t, errors.Is(err, fs.ErrNotExist)) + assert.ErrorIs(t, err, fs.ErrNotExist) assert.Nil(t, b) } diff --git a/bundle/config/mutator/configure_dashboard_defaults_test.go b/bundle/config/mutator/configure_dashboard_defaults_test.go index 2234f9a73..9794d355c 100644 --- a/bundle/config/mutator/configure_dashboard_defaults_test.go +++ b/bundle/config/mutator/configure_dashboard_defaults_test.go @@ -109,19 +109,19 @@ func TestConfigureDashboardDefaultsEmbedCredentials(t *testing.T) { // Set to true; still true. v, err = dyn.Get(b.Config.Value(), "resources.dashboards.d1.embed_credentials") if assert.NoError(t, err) { - assert.Equal(t, true, v.MustBool()) + assert.True(t, v.MustBool()) } // Set to false; still false. v, err = dyn.Get(b.Config.Value(), "resources.dashboards.d2.embed_credentials") if assert.NoError(t, err) { - assert.Equal(t, false, v.MustBool()) + assert.False(t, v.MustBool()) } // Not set; now false. v, err = dyn.Get(b.Config.Value(), "resources.dashboards.d3.embed_credentials") if assert.NoError(t, err) { - assert.Equal(t, false, v.MustBool()) + assert.False(t, v.MustBool()) } // No valid dashboard; no change. diff --git a/bundle/config/mutator/default_queueing_test.go b/bundle/config/mutator/default_queueing_test.go index d3621663b..4c521812e 100644 --- a/bundle/config/mutator/default_queueing_test.go +++ b/bundle/config/mutator/default_queueing_test.go @@ -28,8 +28,8 @@ func TestDefaultQueueingApplyNoJobs(t *testing.T) { }, } d := bundle.Apply(context.Background(), b, DefaultQueueing()) - assert.Len(t, d, 0) - assert.Len(t, b.Config.Resources.Jobs, 0) + assert.Empty(t, d) + assert.Empty(t, b.Config.Resources.Jobs) } func TestDefaultQueueingApplyJobsAlreadyEnabled(t *testing.T) { @@ -47,7 +47,7 @@ func TestDefaultQueueingApplyJobsAlreadyEnabled(t *testing.T) { }, } d := bundle.Apply(context.Background(), b, DefaultQueueing()) - assert.Len(t, d, 0) + assert.Empty(t, d) assert.True(t, b.Config.Resources.Jobs["job"].Queue.Enabled) } @@ -66,7 +66,7 @@ func TestDefaultQueueingApplyEnableQueueing(t *testing.T) { }, } d := bundle.Apply(context.Background(), b, DefaultQueueing()) - assert.Len(t, d, 0) + assert.Empty(t, d) assert.NotNil(t, b.Config.Resources.Jobs["job"].Queue) assert.True(t, b.Config.Resources.Jobs["job"].Queue.Enabled) } @@ -96,7 +96,7 @@ func TestDefaultQueueingApplyWithMultipleJobs(t *testing.T) { }, } d := bundle.Apply(context.Background(), b, DefaultQueueing()) - assert.Len(t, d, 0) + assert.Empty(t, d) assert.False(t, b.Config.Resources.Jobs["job1"].Queue.Enabled) assert.True(t, b.Config.Resources.Jobs["job2"].Queue.Enabled) assert.True(t, b.Config.Resources.Jobs["job3"].Queue.Enabled) diff --git a/bundle/config/mutator/environments_compat_test.go b/bundle/config/mutator/environments_compat_test.go index 8a2129847..11facf9fb 100644 --- a/bundle/config/mutator/environments_compat_test.go +++ b/bundle/config/mutator/environments_compat_test.go @@ -44,7 +44,7 @@ func TestEnvironmentsToTargetsWithEnvironmentsDefined(t *testing.T) { diags := bundle.Apply(context.Background(), b, mutator.EnvironmentsToTargets()) require.NoError(t, diags.Error()) - assert.Len(t, b.Config.Environments, 0) + assert.Empty(t, b.Config.Environments) assert.Len(t, b.Config.Targets, 1) } @@ -61,6 +61,6 @@ func TestEnvironmentsToTargetsWithTargetsDefined(t *testing.T) { diags := bundle.Apply(context.Background(), b, mutator.EnvironmentsToTargets()) require.NoError(t, diags.Error()) - assert.Len(t, b.Config.Environments, 0) + assert.Empty(t, b.Config.Environments) assert.Len(t, b.Config.Targets, 1) } diff --git a/bundle/config/mutator/merge_job_tasks_test.go b/bundle/config/mutator/merge_job_tasks_test.go index a9dae1e10..e6675eefb 100644 --- a/bundle/config/mutator/merge_job_tasks_test.go +++ b/bundle/config/mutator/merge_job_tasks_test.go @@ -74,8 +74,8 @@ func TestMergeJobTasks(t *testing.T) { assert.Equal(t, "i3.2xlarge", cluster.NodeTypeId) assert.Equal(t, 4, cluster.NumWorkers) assert.Len(t, task0.Libraries, 2) - assert.Equal(t, task0.Libraries[0].Whl, "package1") - assert.Equal(t, task0.Libraries[1].Pypi.Package, "package2") + assert.Equal(t, "package1", task0.Libraries[0].Whl) + assert.Equal(t, "package2", task0.Libraries[1].Pypi.Package) // This task was left untouched. task1 := j.Tasks[1].NewCluster diff --git a/bundle/config/mutator/process_target_mode_test.go b/bundle/config/mutator/process_target_mode_test.go index 14d524416..e1aa9e59b 100644 --- a/bundle/config/mutator/process_target_mode_test.go +++ b/bundle/config/mutator/process_target_mode_test.go @@ -163,18 +163,18 @@ func TestProcessTargetModeDevelopment(t *testing.T) { // Job 1 assert.Equal(t, "[dev lennart] job1", b.Config.Resources.Jobs["job1"].Name) - assert.Equal(t, b.Config.Resources.Jobs["job1"].Tags["existing"], "tag") - assert.Equal(t, b.Config.Resources.Jobs["job1"].Tags["dev"], "lennart") - assert.Equal(t, b.Config.Resources.Jobs["job1"].Schedule.PauseStatus, jobs.PauseStatusPaused) + assert.Equal(t, "tag", b.Config.Resources.Jobs["job1"].Tags["existing"]) + assert.Equal(t, "lennart", b.Config.Resources.Jobs["job1"].Tags["dev"]) + assert.Equal(t, jobs.PauseStatusPaused, b.Config.Resources.Jobs["job1"].Schedule.PauseStatus) // Job 2 assert.Equal(t, "[dev lennart] job2", b.Config.Resources.Jobs["job2"].Name) - assert.Equal(t, b.Config.Resources.Jobs["job2"].Tags["dev"], "lennart") - assert.Equal(t, b.Config.Resources.Jobs["job2"].Schedule.PauseStatus, jobs.PauseStatusUnpaused) + assert.Equal(t, "lennart", b.Config.Resources.Jobs["job2"].Tags["dev"]) + assert.Equal(t, jobs.PauseStatusUnpaused, b.Config.Resources.Jobs["job2"].Schedule.PauseStatus) // Pipeline 1 assert.Equal(t, "[dev lennart] pipeline1", b.Config.Resources.Pipelines["pipeline1"].Name) - assert.Equal(t, false, b.Config.Resources.Pipelines["pipeline1"].Continuous) + assert.False(t, b.Config.Resources.Pipelines["pipeline1"].Continuous) assert.True(t, b.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Development) // Experiment 1 diff --git a/bundle/config/mutator/resolve_variable_references_test.go b/bundle/config/mutator/resolve_variable_references_test.go index 7bb6f11a0..07972ecf4 100644 --- a/bundle/config/mutator/resolve_variable_references_test.go +++ b/bundle/config/mutator/resolve_variable_references_test.go @@ -185,11 +185,11 @@ func TestResolveVariableReferencesForPrimitiveNonStringFields(t *testing.T) { // Apply for the variable prefix. This should resolve the variables to their values. diags = bundle.Apply(context.Background(), b, ResolveVariableReferences("variables")) require.NoError(t, diags.Error()) - assert.Equal(t, true, b.Config.Resources.Jobs["job1"].JobSettings.NotificationSettings.NoAlertForCanceledRuns) - assert.Equal(t, true, b.Config.Resources.Jobs["job1"].JobSettings.NotificationSettings.NoAlertForSkippedRuns) + assert.True(t, b.Config.Resources.Jobs["job1"].JobSettings.NotificationSettings.NoAlertForCanceledRuns) + assert.True(t, b.Config.Resources.Jobs["job1"].JobSettings.NotificationSettings.NoAlertForSkippedRuns) assert.Equal(t, 1, b.Config.Resources.Jobs["job1"].JobSettings.Tasks[0].NewCluster.Autoscale.MinWorkers) assert.Equal(t, 2, b.Config.Resources.Jobs["job1"].JobSettings.Tasks[0].NewCluster.Autoscale.MaxWorkers) - assert.Equal(t, 0.5, b.Config.Resources.Jobs["job1"].JobSettings.Tasks[0].NewCluster.AzureAttributes.SpotBidMaxPrice) + assert.InDelta(t, 0.5, b.Config.Resources.Jobs["job1"].JobSettings.Tasks[0].NewCluster.AzureAttributes.SpotBidMaxPrice, 0.0001) } func TestResolveComplexVariable(t *testing.T) { diff --git a/bundle/config/mutator/rewrite_workspace_prefix_test.go b/bundle/config/mutator/rewrite_workspace_prefix_test.go index 48973a4cf..099738c02 100644 --- a/bundle/config/mutator/rewrite_workspace_prefix_test.go +++ b/bundle/config/mutator/rewrite_workspace_prefix_test.go @@ -71,7 +71,7 @@ func TestNoWorkspacePrefixUsed(t *testing.T) { } for _, d := range diags { - require.Equal(t, d.Severity, diag.Warning) + require.Equal(t, diag.Warning, d.Severity) require.Contains(t, expectedErrors, d.Summary) delete(expectedErrors, d.Summary) } diff --git a/bundle/config/mutator/set_variables_test.go b/bundle/config/mutator/set_variables_test.go index d9719793f..07a5c8214 100644 --- a/bundle/config/mutator/set_variables_test.go +++ b/bundle/config/mutator/set_variables_test.go @@ -30,7 +30,7 @@ func TestSetVariableFromProcessEnvVar(t *testing.T) { err = convert.ToTyped(&variable, v) require.NoError(t, err) - assert.Equal(t, variable.Value, "process-env") + assert.Equal(t, "process-env", variable.Value) } func TestSetVariableUsingDefaultValue(t *testing.T) { @@ -48,7 +48,7 @@ func TestSetVariableUsingDefaultValue(t *testing.T) { err = convert.ToTyped(&variable, v) require.NoError(t, err) - assert.Equal(t, variable.Value, "default") + assert.Equal(t, "default", variable.Value) } func TestSetVariableWhenAlreadyAValueIsAssigned(t *testing.T) { @@ -70,7 +70,7 @@ func TestSetVariableWhenAlreadyAValueIsAssigned(t *testing.T) { err = convert.ToTyped(&variable, v) require.NoError(t, err) - assert.Equal(t, variable.Value, "assigned-value") + assert.Equal(t, "assigned-value", variable.Value) } func TestSetVariableEnvVarValueDoesNotOverridePresetValue(t *testing.T) { @@ -95,7 +95,7 @@ func TestSetVariableEnvVarValueDoesNotOverridePresetValue(t *testing.T) { err = convert.ToTyped(&variable, v) require.NoError(t, err) - assert.Equal(t, variable.Value, "assigned-value") + assert.Equal(t, "assigned-value", variable.Value) } func TestSetVariablesErrorsIfAValueCouldNotBeResolved(t *testing.T) { diff --git a/bundle/config/resources_test.go b/bundle/config/resources_test.go index 2d05acf3e..3da645585 100644 --- a/bundle/config/resources_test.go +++ b/bundle/config/resources_test.go @@ -37,11 +37,11 @@ func TestCustomMarshallerIsImplemented(t *testing.T) { field := rt.Field(i) // Fields in Resources are expected be of the form map[string]*resourceStruct - assert.Equal(t, field.Type.Kind(), reflect.Map, "Resource %s is not a map", field.Name) + assert.Equal(t, reflect.Map, field.Type.Kind(), "Resource %s is not a map", field.Name) kt := field.Type.Key() - assert.Equal(t, kt.Kind(), reflect.String, "Resource %s is not a map with string keys", field.Name) + assert.Equal(t, reflect.String, kt.Kind(), "Resource %s is not a map with string keys", field.Name) vt := field.Type.Elem() - assert.Equal(t, vt.Kind(), reflect.Ptr, "Resource %s is not a map with pointer values", field.Name) + assert.Equal(t, reflect.Ptr, vt.Kind(), "Resource %s is not a map with pointer values", field.Name) // Marshalling a resourceStruct will panic if resourceStruct does not have a custom marshaller // This is because resourceStruct embeds a Go SDK struct that implements diff --git a/bundle/config/validate/files_to_sync_test.go b/bundle/config/validate/files_to_sync_test.go index d6a1ed59a..dd40295c3 100644 --- a/bundle/config/validate/files_to_sync_test.go +++ b/bundle/config/validate/files_to_sync_test.go @@ -87,7 +87,7 @@ func TestFilesToSync_EverythingIgnored(t *testing.T) { ctx := context.Background() rb := bundle.ReadOnly(b) diags := bundle.ApplyReadOnly(ctx, rb, FilesToSync()) - require.Equal(t, 1, len(diags)) + require.Len(t, diags, 1) assert.Equal(t, diag.Warning, diags[0].Severity) assert.Equal(t, "There are no files to sync, please check your .gitignore", diags[0].Summary) } @@ -101,7 +101,7 @@ func TestFilesToSync_EverythingExcluded(t *testing.T) { ctx := context.Background() rb := bundle.ReadOnly(b) diags := bundle.ApplyReadOnly(ctx, rb, FilesToSync()) - require.Equal(t, 1, len(diags)) + require.Len(t, diags, 1) assert.Equal(t, diag.Warning, diags[0].Severity) assert.Equal(t, "There are no files to sync, please check your .gitignore and sync.exclude configuration", diags[0].Summary) } diff --git a/bundle/config/validate/job_cluster_key_defined_test.go b/bundle/config/validate/job_cluster_key_defined_test.go index 176b0fedc..2cbdb7c6a 100644 --- a/bundle/config/validate/job_cluster_key_defined_test.go +++ b/bundle/config/validate/job_cluster_key_defined_test.go @@ -34,7 +34,7 @@ func TestJobClusterKeyDefined(t *testing.T) { } diags := bundle.ApplyReadOnly(context.Background(), bundle.ReadOnly(b), JobClusterKeyDefined()) - require.Len(t, diags, 0) + require.Empty(t, diags) require.NoError(t, diags.Error()) } @@ -59,8 +59,8 @@ func TestJobClusterKeyNotDefined(t *testing.T) { diags := bundle.ApplyReadOnly(context.Background(), bundle.ReadOnly(b), JobClusterKeyDefined()) require.Len(t, diags, 1) require.NoError(t, diags.Error()) - require.Equal(t, diags[0].Severity, diag.Warning) - require.Equal(t, diags[0].Summary, "job_cluster_key do-not-exist is not defined") + require.Equal(t, diag.Warning, diags[0].Severity) + require.Equal(t, "job_cluster_key do-not-exist is not defined", diags[0].Summary) } func TestJobClusterKeyDefinedInDifferentJob(t *testing.T) { @@ -92,6 +92,6 @@ func TestJobClusterKeyDefinedInDifferentJob(t *testing.T) { diags := bundle.ApplyReadOnly(context.Background(), bundle.ReadOnly(b), JobClusterKeyDefined()) require.Len(t, diags, 1) require.NoError(t, diags.Error()) - require.Equal(t, diags[0].Severity, diag.Warning) - require.Equal(t, diags[0].Summary, "job_cluster_key do-not-exist is not defined") + require.Equal(t, diag.Warning, diags[0].Severity) + require.Equal(t, "job_cluster_key do-not-exist is not defined", diags[0].Summary) } diff --git a/bundle/context_test.go b/bundle/context_test.go index 3a0f159d9..89a6df052 100644 --- a/bundle/context_test.go +++ b/bundle/context_test.go @@ -12,7 +12,7 @@ func TestGetPanics(t *testing.T) { defer func() { r := recover() require.NotNil(t, r, "The function did not panic") - assert.Equal(t, r, "context not configured with bundle") + assert.Equal(t, "context not configured with bundle", r) }() Get(context.Background()) diff --git a/bundle/deploy/state_pull_test.go b/bundle/deploy/state_pull_test.go index 36c49fb01..f38b71f6b 100644 --- a/bundle/deploy/state_pull_test.go +++ b/bundle/deploy/state_pull_test.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "encoding/json" - "errors" "io" "io/fs" "os" @@ -279,7 +278,7 @@ func TestStatePullNoState(t *testing.T) { require.NoError(t, err) _, err = os.Stat(statePath) - require.True(t, errors.Is(err, fs.ErrNotExist)) + require.ErrorIs(t, err, fs.ErrNotExist) } func TestStatePullOlderState(t *testing.T) { diff --git a/bundle/deploy/state_update_test.go b/bundle/deploy/state_update_test.go index e561f534e..04c5579a8 100644 --- a/bundle/deploy/state_update_test.go +++ b/bundle/deploy/state_update_test.go @@ -60,7 +60,7 @@ func TestStateUpdate(t *testing.T) { require.NoError(t, err) require.Equal(t, int64(1), state.Seq) - require.Equal(t, state.Files, Filelist{ + require.Equal(t, Filelist{ { LocalPath: "test1.py", }, @@ -68,7 +68,7 @@ func TestStateUpdate(t *testing.T) { LocalPath: "test2.py", IsNotebook: true, }, - }) + }, state.Files) require.Equal(t, build.GetInfo().Version, state.CliVersion) diags = bundle.Apply(ctx, b, s) @@ -79,7 +79,7 @@ func TestStateUpdate(t *testing.T) { require.NoError(t, err) require.Equal(t, int64(2), state.Seq) - require.Equal(t, state.Files, Filelist{ + require.Equal(t, Filelist{ { LocalPath: "test1.py", }, @@ -87,7 +87,7 @@ func TestStateUpdate(t *testing.T) { LocalPath: "test2.py", IsNotebook: true, }, - }) + }, state.Files) require.Equal(t, build.GetInfo().Version, state.CliVersion) // Valid non-empty UUID is generated. @@ -130,7 +130,7 @@ func TestStateUpdateWithExistingState(t *testing.T) { require.NoError(t, err) require.Equal(t, int64(11), state.Seq) - require.Equal(t, state.Files, Filelist{ + require.Equal(t, Filelist{ { LocalPath: "test1.py", }, @@ -138,7 +138,7 @@ func TestStateUpdateWithExistingState(t *testing.T) { LocalPath: "test2.py", IsNotebook: true, }, - }) + }, state.Files) require.Equal(t, build.GetInfo().Version, state.CliVersion) // Existing UUID is not overwritten. diff --git a/bundle/deploy/terraform/convert_test.go b/bundle/deploy/terraform/convert_test.go index 61f26c088..84b3c5788 100644 --- a/bundle/deploy/terraform/convert_test.go +++ b/bundle/deploy/terraform/convert_test.go @@ -254,10 +254,10 @@ func TestBundleToTerraformPipeline(t *testing.T) { assert.Equal(t, "my pipeline", resource.Name) assert.Len(t, resource.Library, 2) assert.Len(t, resource.Notification, 2) - assert.Equal(t, resource.Notification[0].Alerts, []string{"on-update-fatal-failure"}) - assert.Equal(t, resource.Notification[0].EmailRecipients, []string{"jane@doe.com"}) - assert.Equal(t, resource.Notification[1].Alerts, []string{"on-update-failure", "on-flow-failure"}) - assert.Equal(t, resource.Notification[1].EmailRecipients, []string{"jane@doe.com", "john@doe.com"}) + assert.Equal(t, []string{"on-update-fatal-failure"}, resource.Notification[0].Alerts) + assert.Equal(t, []string{"jane@doe.com"}, resource.Notification[0].EmailRecipients) + assert.Equal(t, []string{"on-update-failure", "on-flow-failure"}, resource.Notification[1].Alerts) + assert.Equal(t, []string{"jane@doe.com", "john@doe.com"}, resource.Notification[1].EmailRecipients) assert.Nil(t, out.Data) } @@ -454,7 +454,7 @@ func TestBundleToTerraformModelServing(t *testing.T) { assert.Equal(t, "name", resource.Name) assert.Equal(t, "model_name", resource.Config.ServedModels[0].ModelName) assert.Equal(t, "1", resource.Config.ServedModels[0].ModelVersion) - assert.Equal(t, true, resource.Config.ServedModels[0].ScaleToZeroEnabled) + assert.True(t, resource.Config.ServedModels[0].ScaleToZeroEnabled) assert.Equal(t, "Small", resource.Config.ServedModels[0].WorkloadSize) assert.Equal(t, "model_name-1", resource.Config.TrafficConfig.Routes[0].ServedModelName) assert.Equal(t, 100, resource.Config.TrafficConfig.Routes[0].TrafficPercentage) diff --git a/bundle/deploy/terraform/init_test.go b/bundle/deploy/terraform/init_test.go index a1ffc5a1a..30ac9e301 100644 --- a/bundle/deploy/terraform/init_test.go +++ b/bundle/deploy/terraform/init_test.go @@ -225,7 +225,7 @@ func TestSetProxyEnvVars(t *testing.T) { env := make(map[string]string, 0) err := setProxyEnvVars(context.Background(), env, b) require.NoError(t, err) - assert.Len(t, env, 0) + assert.Empty(t, env) // Lower case set. clearEnv() @@ -293,7 +293,7 @@ func TestSetUserProfileFromInheritEnvVars(t *testing.T) { require.NoError(t, err) assert.Contains(t, env, "USERPROFILE") - assert.Equal(t, env["USERPROFILE"], "c:\\foo\\c") + assert.Equal(t, "c:\\foo\\c", env["USERPROFILE"]) } func TestInheritEnvVarsWithAbsentTFConfigFile(t *testing.T) { diff --git a/bundle/internal/schema/main_test.go b/bundle/internal/schema/main_test.go index 4eeb41d47..06e89c856 100644 --- a/bundle/internal/schema/main_test.go +++ b/bundle/internal/schema/main_test.go @@ -2,7 +2,6 @@ package main import ( "bytes" - "fmt" "io" "os" "path" @@ -80,7 +79,7 @@ func TestRequiredAnnotationsForNewFields(t *testing.T) { }, }) assert.NoError(t, err) - assert.Empty(t, updatedFieldPaths, fmt.Sprintf("Missing JSON-schema descriptions for new config fields in bundle/internal/schema/annotations.yml:\n%s", strings.Join(updatedFieldPaths, "\n"))) + assert.Empty(t, updatedFieldPaths, "Missing JSON-schema descriptions for new config fields in bundle/internal/schema/annotations.yml:\n%s", strings.Join(updatedFieldPaths, "\n")) } // Checks whether types in annotation files are still present in Config type diff --git a/bundle/libraries/filer_volume_test.go b/bundle/libraries/filer_volume_test.go index 7b2f5c5ba..2af54b0cb 100644 --- a/bundle/libraries/filer_volume_test.go +++ b/bundle/libraries/filer_volume_test.go @@ -212,12 +212,12 @@ func TestFilerForVolumeWithInvalidVolumePaths(t *testing.T) { bundletest.SetLocation(b, "workspace.artifact_path", []dyn.Location{{File: "config.yml", Line: 1, Column: 2}}) _, _, diags := GetFilerForLibraries(context.Background(), b) - require.Equal(t, diags, diag.Diagnostics{{ + require.Equal(t, diag.Diagnostics{{ Severity: diag.Error, Summary: fmt.Sprintf("expected UC volume path to be in the format /Volumes////..., got %s", p), Locations: []dyn.Location{{File: "config.yml", Line: 1, Column: 2}}, Paths: []dyn.Path{dyn.MustPathFromString("workspace.artifact_path")}, - }}) + }}, diags) } } diff --git a/bundle/libraries/helpers_test.go b/bundle/libraries/helpers_test.go index 9d7e12ee5..754aa8f95 100644 --- a/bundle/libraries/helpers_test.go +++ b/bundle/libraries/helpers_test.go @@ -12,25 +12,25 @@ func TestLibraryPath(t *testing.T) { p, err := libraryPath(&compute.Library{Whl: path}) assert.Equal(t, path, p) - assert.Nil(t, err) + assert.NoError(t, err) p, err = libraryPath(&compute.Library{Jar: path}) assert.Equal(t, path, p) - assert.Nil(t, err) + assert.NoError(t, err) p, err = libraryPath(&compute.Library{Egg: path}) assert.Equal(t, path, p) - assert.Nil(t, err) + assert.NoError(t, err) p, err = libraryPath(&compute.Library{Requirements: path}) assert.Equal(t, path, p) - assert.Nil(t, err) + assert.NoError(t, err) p, err = libraryPath(&compute.Library{}) assert.Equal(t, "", p) - assert.NotNil(t, err) + assert.Error(t, err) p, err = libraryPath(&compute.Library{Pypi: &compute.PythonPyPiLibrary{Package: "pypipackage"}}) assert.Equal(t, "", p) - assert.NotNil(t, err) + assert.Error(t, err) } diff --git a/bundle/permissions/filter_test.go b/bundle/permissions/filter_test.go index e6e5a3799..ef7167d75 100644 --- a/bundle/permissions/filter_test.go +++ b/bundle/permissions/filter_test.go @@ -99,32 +99,32 @@ func TestFilterCurrentUser(t *testing.T) { assert.NoError(t, diags.Error()) // Assert current user is filtered out. - assert.Equal(t, 2, len(b.Config.Resources.Jobs["job1"].Permissions)) + assert.Len(t, b.Config.Resources.Jobs["job1"].Permissions, 2) assert.Contains(t, b.Config.Resources.Jobs["job1"].Permissions, robot) assert.Contains(t, b.Config.Resources.Jobs["job1"].Permissions, bob) - assert.Equal(t, 2, len(b.Config.Resources.Jobs["job2"].Permissions)) + assert.Len(t, b.Config.Resources.Jobs["job2"].Permissions, 2) assert.Contains(t, b.Config.Resources.Jobs["job2"].Permissions, robot) assert.Contains(t, b.Config.Resources.Jobs["job2"].Permissions, bob) - assert.Equal(t, 2, len(b.Config.Resources.Pipelines["pipeline1"].Permissions)) + assert.Len(t, b.Config.Resources.Pipelines["pipeline1"].Permissions, 2) assert.Contains(t, b.Config.Resources.Pipelines["pipeline1"].Permissions, robot) assert.Contains(t, b.Config.Resources.Pipelines["pipeline1"].Permissions, bob) - assert.Equal(t, 2, len(b.Config.Resources.Experiments["experiment1"].Permissions)) + assert.Len(t, b.Config.Resources.Experiments["experiment1"].Permissions, 2) assert.Contains(t, b.Config.Resources.Experiments["experiment1"].Permissions, robot) assert.Contains(t, b.Config.Resources.Experiments["experiment1"].Permissions, bob) - assert.Equal(t, 2, len(b.Config.Resources.Models["model1"].Permissions)) + assert.Len(t, b.Config.Resources.Models["model1"].Permissions, 2) assert.Contains(t, b.Config.Resources.Models["model1"].Permissions, robot) assert.Contains(t, b.Config.Resources.Models["model1"].Permissions, bob) - assert.Equal(t, 2, len(b.Config.Resources.ModelServingEndpoints["endpoint1"].Permissions)) + assert.Len(t, b.Config.Resources.ModelServingEndpoints["endpoint1"].Permissions, 2) assert.Contains(t, b.Config.Resources.ModelServingEndpoints["endpoint1"].Permissions, robot) assert.Contains(t, b.Config.Resources.ModelServingEndpoints["endpoint1"].Permissions, bob) // Assert there's no change to the grant. - assert.Equal(t, 1, len(b.Config.Resources.RegisteredModels["registered_model1"].Grants)) + assert.Len(t, b.Config.Resources.RegisteredModels["registered_model1"].Grants, 1) } func TestFilterCurrentServicePrincipal(t *testing.T) { @@ -134,32 +134,32 @@ func TestFilterCurrentServicePrincipal(t *testing.T) { assert.NoError(t, diags.Error()) // Assert current user is filtered out. - assert.Equal(t, 2, len(b.Config.Resources.Jobs["job1"].Permissions)) + assert.Len(t, b.Config.Resources.Jobs["job1"].Permissions, 2) assert.Contains(t, b.Config.Resources.Jobs["job1"].Permissions, alice) assert.Contains(t, b.Config.Resources.Jobs["job1"].Permissions, bob) - assert.Equal(t, 2, len(b.Config.Resources.Jobs["job2"].Permissions)) + assert.Len(t, b.Config.Resources.Jobs["job2"].Permissions, 2) assert.Contains(t, b.Config.Resources.Jobs["job2"].Permissions, alice) assert.Contains(t, b.Config.Resources.Jobs["job2"].Permissions, bob) - assert.Equal(t, 2, len(b.Config.Resources.Pipelines["pipeline1"].Permissions)) + assert.Len(t, b.Config.Resources.Pipelines["pipeline1"].Permissions, 2) assert.Contains(t, b.Config.Resources.Pipelines["pipeline1"].Permissions, alice) assert.Contains(t, b.Config.Resources.Pipelines["pipeline1"].Permissions, bob) - assert.Equal(t, 2, len(b.Config.Resources.Experiments["experiment1"].Permissions)) + assert.Len(t, b.Config.Resources.Experiments["experiment1"].Permissions, 2) assert.Contains(t, b.Config.Resources.Experiments["experiment1"].Permissions, alice) assert.Contains(t, b.Config.Resources.Experiments["experiment1"].Permissions, bob) - assert.Equal(t, 2, len(b.Config.Resources.Models["model1"].Permissions)) + assert.Len(t, b.Config.Resources.Models["model1"].Permissions, 2) assert.Contains(t, b.Config.Resources.Models["model1"].Permissions, alice) assert.Contains(t, b.Config.Resources.Models["model1"].Permissions, bob) - assert.Equal(t, 2, len(b.Config.Resources.ModelServingEndpoints["endpoint1"].Permissions)) + assert.Len(t, b.Config.Resources.ModelServingEndpoints["endpoint1"].Permissions, 2) assert.Contains(t, b.Config.Resources.ModelServingEndpoints["endpoint1"].Permissions, alice) assert.Contains(t, b.Config.Resources.ModelServingEndpoints["endpoint1"].Permissions, bob) // Assert there's no change to the grant. - assert.Equal(t, 1, len(b.Config.Resources.RegisteredModels["registered_model1"].Grants)) + assert.Len(t, b.Config.Resources.RegisteredModels["registered_model1"].Grants, 1) } func TestFilterCurrentUserDoesNotErrorWhenNoResources(t *testing.T) { diff --git a/bundle/permissions/mutator_test.go b/bundle/permissions/mutator_test.go index 78703e90f..15586e979 100644 --- a/bundle/permissions/mutator_test.go +++ b/bundle/permissions/mutator_test.go @@ -164,7 +164,7 @@ func TestAllResourcesExplicitlyDefinedForPermissionsSupport(t *testing.T) { for _, resource := range unsupportedResources { _, ok := levelsMap[resource] - assert.False(t, ok, fmt.Sprintf("Resource %s is defined in both levelsMap and unsupportedResources", resource)) + assert.False(t, ok, "Resource %s is defined in both levelsMap and unsupportedResources", resource) } for _, resource := range r.AllResources() { diff --git a/bundle/permissions/permission_diagnostics_test.go b/bundle/permissions/permission_diagnostics_test.go index 7b0afefa0..6c55ab594 100644 --- a/bundle/permissions/permission_diagnostics_test.go +++ b/bundle/permissions/permission_diagnostics_test.go @@ -28,7 +28,7 @@ func TestPermissionDiagnosticsApplyFail(t *testing.T) { }) diags := permissions.PermissionDiagnostics().Apply(context.Background(), b) - require.Equal(t, diags[0].Severity, diag.Warning) + require.Equal(t, diag.Warning, diags[0].Severity) require.Contains(t, diags[0].Summary, "permissions section should include testuser@databricks.com or one of their groups with CAN_MANAGE permissions") } diff --git a/bundle/run/job_test.go b/bundle/run/job_test.go index 5d19ca4ff..72aecc887 100644 --- a/bundle/run/job_test.go +++ b/bundle/run/job_test.go @@ -54,7 +54,7 @@ func TestConvertPythonParams(t *testing.T) { err = runner.convertPythonParams(opts) require.NoError(t, err) require.Contains(t, opts.Job.notebookParams, "__python_params") - require.Equal(t, opts.Job.notebookParams["__python_params"], `["param1","param2","param3"]`) + require.Equal(t, `["param1","param2","param3"]`, opts.Job.notebookParams["__python_params"]) } func TestJobRunnerCancel(t *testing.T) { diff --git a/bundle/tests/complex_variables_test.go b/bundle/tests/complex_variables_test.go index e68823c33..d72b5f157 100644 --- a/bundle/tests/complex_variables_test.go +++ b/bundle/tests/complex_variables_test.go @@ -30,7 +30,7 @@ func TestComplexVariables(t *testing.T) { require.Equal(t, "true", b.Config.Resources.Jobs["my_job"].JobClusters[0].NewCluster.SparkConf["spark.speculation"]) require.Equal(t, "true", b.Config.Resources.Jobs["my_job"].JobClusters[0].NewCluster.SparkConf["spark.random"]) - require.Equal(t, 3, len(b.Config.Resources.Jobs["my_job"].Tasks[0].Libraries)) + require.Len(t, b.Config.Resources.Jobs["my_job"].Tasks[0].Libraries, 3) require.Contains(t, b.Config.Resources.Jobs["my_job"].Tasks[0].Libraries, compute.Library{ Jar: "/path/to/jar", }) diff --git a/bundle/tests/environment_git_test.go b/bundle/tests/environment_git_test.go index d4695c78d..848b972b1 100644 --- a/bundle/tests/environment_git_test.go +++ b/bundle/tests/environment_git_test.go @@ -2,7 +2,6 @@ package config_tests import ( "context" - "fmt" "strings" "testing" @@ -16,7 +15,7 @@ func TestGitAutoLoadWithEnvironment(t *testing.T) { bundle.Apply(context.Background(), b, mutator.LoadGitDetails()) assert.True(t, b.Config.Bundle.Git.Inferred) validUrl := strings.Contains(b.Config.Bundle.Git.OriginURL, "/cli") || strings.Contains(b.Config.Bundle.Git.OriginURL, "/bricks") - assert.True(t, validUrl, fmt.Sprintf("Expected URL to contain '/cli' or '/bricks', got %s", b.Config.Bundle.Git.OriginURL)) + assert.True(t, validUrl, "Expected URL to contain '/cli' or '/bricks', got %s", b.Config.Bundle.Git.OriginURL) } func TestGitManuallySetBranchWithEnvironment(t *testing.T) { @@ -25,5 +24,5 @@ func TestGitManuallySetBranchWithEnvironment(t *testing.T) { assert.False(t, b.Config.Bundle.Git.Inferred) assert.Equal(t, "main", b.Config.Bundle.Git.Branch) validUrl := strings.Contains(b.Config.Bundle.Git.OriginURL, "/cli") || strings.Contains(b.Config.Bundle.Git.OriginURL, "/bricks") - assert.True(t, validUrl, fmt.Sprintf("Expected URL to contain '/cli' or '/bricks', got %s", b.Config.Bundle.Git.OriginURL)) + assert.True(t, validUrl, "Expected URL to contain '/cli' or '/bricks', got %s", b.Config.Bundle.Git.OriginURL) } diff --git a/bundle/tests/environment_overrides_test.go b/bundle/tests/environment_overrides_test.go index 4a1115048..b68b083ff 100644 --- a/bundle/tests/environment_overrides_test.go +++ b/bundle/tests/environment_overrides_test.go @@ -21,8 +21,8 @@ func TestEnvironmentOverridesResourcesDev(t *testing.T) { assert.Equal(t, "base job", b.Config.Resources.Jobs["job1"].Name) // Base values are preserved in the development environment. - assert.Equal(t, true, b.Config.Resources.Pipelines["boolean1"].Photon) - assert.Equal(t, false, b.Config.Resources.Pipelines["boolean2"].Photon) + assert.True(t, b.Config.Resources.Pipelines["boolean1"].Photon) + assert.False(t, b.Config.Resources.Pipelines["boolean2"].Photon) } func TestEnvironmentOverridesResourcesStaging(t *testing.T) { @@ -30,6 +30,6 @@ func TestEnvironmentOverridesResourcesStaging(t *testing.T) { assert.Equal(t, "staging job", b.Config.Resources.Jobs["job1"].Name) // Override values are applied in the staging environment. - assert.Equal(t, false, b.Config.Resources.Pipelines["boolean1"].Photon) - assert.Equal(t, true, b.Config.Resources.Pipelines["boolean2"].Photon) + assert.False(t, b.Config.Resources.Pipelines["boolean1"].Photon) + assert.True(t, b.Config.Resources.Pipelines["boolean2"].Photon) } diff --git a/bundle/tests/environments_job_and_pipeline_test.go b/bundle/tests/environments_job_and_pipeline_test.go index 218d2e470..423b14c07 100644 --- a/bundle/tests/environments_job_and_pipeline_test.go +++ b/bundle/tests/environments_job_and_pipeline_test.go @@ -10,11 +10,11 @@ import ( func TestJobAndPipelineDevelopmentWithEnvironment(t *testing.T) { b := loadTarget(t, "./environments_job_and_pipeline", "development") - assert.Len(t, b.Config.Resources.Jobs, 0) + assert.Empty(t, b.Config.Resources.Jobs) assert.Len(t, b.Config.Resources.Pipelines, 1) p := b.Config.Resources.Pipelines["nyc_taxi_pipeline"] - assert.Equal(t, b.Config.Bundle.Mode, config.Development) + assert.Equal(t, config.Development, b.Config.Bundle.Mode) assert.True(t, p.Development) require.Len(t, p.Libraries, 1) assert.Equal(t, "./dlt/nyc_taxi_loader", p.Libraries[0].Notebook.Path) @@ -23,7 +23,7 @@ func TestJobAndPipelineDevelopmentWithEnvironment(t *testing.T) { func TestJobAndPipelineStagingWithEnvironment(t *testing.T) { b := loadTarget(t, "./environments_job_and_pipeline", "staging") - assert.Len(t, b.Config.Resources.Jobs, 0) + assert.Empty(t, b.Config.Resources.Jobs) assert.Len(t, b.Config.Resources.Pipelines, 1) p := b.Config.Resources.Pipelines["nyc_taxi_pipeline"] diff --git a/bundle/tests/git_test.go b/bundle/tests/git_test.go index dec6c268a..41293e450 100644 --- a/bundle/tests/git_test.go +++ b/bundle/tests/git_test.go @@ -2,7 +2,6 @@ package config_tests import ( "context" - "fmt" "strings" "testing" @@ -17,7 +16,7 @@ func TestGitAutoLoad(t *testing.T) { bundle.Apply(context.Background(), b, mutator.LoadGitDetails()) assert.True(t, b.Config.Bundle.Git.Inferred) validUrl := strings.Contains(b.Config.Bundle.Git.OriginURL, "/cli") || strings.Contains(b.Config.Bundle.Git.OriginURL, "/bricks") - assert.True(t, validUrl, fmt.Sprintf("Expected URL to contain '/cli' or '/bricks', got %s", b.Config.Bundle.Git.OriginURL)) + assert.True(t, validUrl, "Expected URL to contain '/cli' or '/bricks', got %s", b.Config.Bundle.Git.OriginURL) } func TestGitManuallySetBranch(t *testing.T) { @@ -26,7 +25,7 @@ func TestGitManuallySetBranch(t *testing.T) { assert.False(t, b.Config.Bundle.Git.Inferred) assert.Equal(t, "main", b.Config.Bundle.Git.Branch) validUrl := strings.Contains(b.Config.Bundle.Git.OriginURL, "/cli") || strings.Contains(b.Config.Bundle.Git.OriginURL, "/bricks") - assert.True(t, validUrl, fmt.Sprintf("Expected URL to contain '/cli' or '/bricks', got %s", b.Config.Bundle.Git.OriginURL)) + assert.True(t, validUrl, "Expected URL to contain '/cli' or '/bricks', got %s", b.Config.Bundle.Git.OriginURL) } func TestGitBundleBranchValidation(t *testing.T) { diff --git a/bundle/tests/issue_1828_test.go b/bundle/tests/issue_1828_test.go index 5f2becce5..31fcfeb8e 100644 --- a/bundle/tests/issue_1828_test.go +++ b/bundle/tests/issue_1828_test.go @@ -35,7 +35,7 @@ func TestIssue1828(t *testing.T) { } if assert.Contains(t, b.Config.Variables, "float") { - assert.Equal(t, 3.14, b.Config.Variables["float"].Default) + assert.InDelta(t, 3.14, b.Config.Variables["float"].Default, 0.0001) } if assert.Contains(t, b.Config.Variables, "time") { @@ -43,6 +43,6 @@ func TestIssue1828(t *testing.T) { } if assert.Contains(t, b.Config.Variables, "nil") { - assert.Equal(t, nil, b.Config.Variables["nil"].Default) + assert.Nil(t, b.Config.Variables["nil"].Default) } } diff --git a/bundle/tests/job_and_pipeline_test.go b/bundle/tests/job_and_pipeline_test.go index 65aa5bdc4..408e3e3ef 100644 --- a/bundle/tests/job_and_pipeline_test.go +++ b/bundle/tests/job_and_pipeline_test.go @@ -10,11 +10,11 @@ import ( func TestJobAndPipelineDevelopment(t *testing.T) { b := loadTarget(t, "./job_and_pipeline", "development") - assert.Len(t, b.Config.Resources.Jobs, 0) + assert.Empty(t, b.Config.Resources.Jobs) assert.Len(t, b.Config.Resources.Pipelines, 1) p := b.Config.Resources.Pipelines["nyc_taxi_pipeline"] - assert.Equal(t, b.Config.Bundle.Mode, config.Development) + assert.Equal(t, config.Development, b.Config.Bundle.Mode) assert.True(t, p.Development) require.Len(t, p.Libraries, 1) assert.Equal(t, "./dlt/nyc_taxi_loader", p.Libraries[0].Notebook.Path) @@ -23,7 +23,7 @@ func TestJobAndPipelineDevelopment(t *testing.T) { func TestJobAndPipelineStaging(t *testing.T) { b := loadTarget(t, "./job_and_pipeline", "staging") - assert.Len(t, b.Config.Resources.Jobs, 0) + assert.Empty(t, b.Config.Resources.Jobs) assert.Len(t, b.Config.Resources.Pipelines, 1) p := b.Config.Resources.Pipelines["nyc_taxi_pipeline"] diff --git a/bundle/tests/job_cluster_key_test.go b/bundle/tests/job_cluster_key_test.go index 5a8b368e5..6a08da89c 100644 --- a/bundle/tests/job_cluster_key_test.go +++ b/bundle/tests/job_cluster_key_test.go @@ -16,13 +16,13 @@ func TestJobClusterKeyNotDefinedTest(t *testing.T) { diags := bundle.ApplyReadOnly(context.Background(), bundle.ReadOnly(b), validate.JobClusterKeyDefined()) require.Len(t, diags, 1) require.NoError(t, diags.Error()) - require.Equal(t, diags[0].Severity, diag.Warning) - require.Equal(t, diags[0].Summary, "job_cluster_key key is not defined") + require.Equal(t, diag.Warning, diags[0].Severity) + require.Equal(t, "job_cluster_key key is not defined", diags[0].Summary) } func TestJobClusterKeyDefinedTest(t *testing.T) { b := loadTarget(t, "./job_cluster_key", "development") diags := bundle.ApplyReadOnly(context.Background(), bundle.ReadOnly(b), validate.JobClusterKeyDefined()) - require.Len(t, diags, 0) + require.Empty(t, diags) } diff --git a/bundle/tests/model_serving_endpoint_test.go b/bundle/tests/model_serving_endpoint_test.go index b8b800863..f779a07e6 100644 --- a/bundle/tests/model_serving_endpoint_test.go +++ b/bundle/tests/model_serving_endpoint_test.go @@ -20,7 +20,7 @@ func assertExpected(t *testing.T, p *resources.ModelServingEndpoint) { func TestModelServingEndpointDevelopment(t *testing.T) { b := loadTarget(t, "./model_serving_endpoint", "development") assert.Len(t, b.Config.Resources.ModelServingEndpoints, 1) - assert.Equal(t, b.Config.Bundle.Mode, config.Development) + assert.Equal(t, config.Development, b.Config.Bundle.Mode) p := b.Config.Resources.ModelServingEndpoints["my_model_serving_endpoint"] assert.Equal(t, "my-dev-endpoint", p.Name) diff --git a/bundle/tests/override_job_tasks_test.go b/bundle/tests/override_job_tasks_test.go index 82da04da2..85463e17c 100644 --- a/bundle/tests/override_job_tasks_test.go +++ b/bundle/tests/override_job_tasks_test.go @@ -12,14 +12,14 @@ func TestOverrideTasksDev(t *testing.T) { assert.Len(t, b.Config.Resources.Jobs["foo"].Tasks, 2) tasks := b.Config.Resources.Jobs["foo"].Tasks - assert.Equal(t, tasks[0].TaskKey, "key1") - assert.Equal(t, tasks[0].NewCluster.NodeTypeId, "i3.xlarge") - assert.Equal(t, tasks[0].NewCluster.NumWorkers, 1) - assert.Equal(t, tasks[0].SparkPythonTask.PythonFile, "./test1.py") + assert.Equal(t, "key1", tasks[0].TaskKey) + assert.Equal(t, "i3.xlarge", tasks[0].NewCluster.NodeTypeId) + assert.Equal(t, 1, tasks[0].NewCluster.NumWorkers) + assert.Equal(t, "./test1.py", tasks[0].SparkPythonTask.PythonFile) - assert.Equal(t, tasks[1].TaskKey, "key2") - assert.Equal(t, tasks[1].NewCluster.SparkVersion, "13.3.x-scala2.12") - assert.Equal(t, tasks[1].SparkPythonTask.PythonFile, "./test2.py") + assert.Equal(t, "key2", tasks[1].TaskKey) + assert.Equal(t, "13.3.x-scala2.12", tasks[1].NewCluster.SparkVersion) + assert.Equal(t, "./test2.py", tasks[1].SparkPythonTask.PythonFile) } func TestOverrideTasksStaging(t *testing.T) { @@ -28,12 +28,12 @@ func TestOverrideTasksStaging(t *testing.T) { assert.Len(t, b.Config.Resources.Jobs["foo"].Tasks, 2) tasks := b.Config.Resources.Jobs["foo"].Tasks - assert.Equal(t, tasks[0].TaskKey, "key1") - assert.Equal(t, tasks[0].NewCluster.SparkVersion, "13.3.x-scala2.12") - assert.Equal(t, tasks[0].SparkPythonTask.PythonFile, "./test1.py") + assert.Equal(t, "key1", tasks[0].TaskKey) + assert.Equal(t, "13.3.x-scala2.12", tasks[0].NewCluster.SparkVersion) + assert.Equal(t, "./test1.py", tasks[0].SparkPythonTask.PythonFile) - assert.Equal(t, tasks[1].TaskKey, "key2") - assert.Equal(t, tasks[1].NewCluster.NodeTypeId, "i3.2xlarge") - assert.Equal(t, tasks[1].NewCluster.NumWorkers, 4) - assert.Equal(t, tasks[1].SparkPythonTask.PythonFile, "./test3.py") + assert.Equal(t, "key2", tasks[1].TaskKey) + assert.Equal(t, "i3.2xlarge", tasks[1].NewCluster.NodeTypeId) + assert.Equal(t, 4, tasks[1].NewCluster.NumWorkers) + assert.Equal(t, "./test3.py", tasks[1].SparkPythonTask.PythonFile) } diff --git a/bundle/tests/presets_test.go b/bundle/tests/presets_test.go index 5fcb5d95b..c2cbe497b 100644 --- a/bundle/tests/presets_test.go +++ b/bundle/tests/presets_test.go @@ -13,7 +13,7 @@ func TestPresetsDev(t *testing.T) { assert.Equal(t, "myprefix", b.Config.Presets.NamePrefix) assert.Equal(t, config.Paused, b.Config.Presets.TriggerPauseStatus) assert.Equal(t, 10, b.Config.Presets.JobsMaxConcurrentRuns) - assert.Equal(t, true, *b.Config.Presets.PipelinesDevelopment) + assert.True(t, *b.Config.Presets.PipelinesDevelopment) assert.Equal(t, "true", b.Config.Presets.Tags["dev"]) assert.Equal(t, "finance", b.Config.Presets.Tags["team"]) assert.Equal(t, "false", b.Config.Presets.Tags["prod"]) @@ -22,7 +22,7 @@ func TestPresetsDev(t *testing.T) { func TestPresetsProd(t *testing.T) { b := loadTarget(t, "./presets", "prod") - assert.Equal(t, false, *b.Config.Presets.PipelinesDevelopment) + assert.False(t, *b.Config.Presets.PipelinesDevelopment) assert.Equal(t, "finance", b.Config.Presets.Tags["team"]) assert.Equal(t, "true", b.Config.Presets.Tags["prod"]) } diff --git a/bundle/tests/python_wheel_test.go b/bundle/tests/python_wheel_test.go index c982c09d6..06cb05270 100644 --- a/bundle/tests/python_wheel_test.go +++ b/bundle/tests/python_wheel_test.go @@ -23,7 +23,7 @@ func TestPythonWheelBuild(t *testing.T) { matches, err := filepath.Glob("./python_wheel/python_wheel/my_test_code/dist/my_test_code-*.whl") require.NoError(t, err) - require.Equal(t, 1, len(matches)) + require.Len(t, matches, 1) match := libraries.ExpandGlobReferences() diags = bundle.Apply(ctx, b, match) @@ -39,7 +39,7 @@ func TestPythonWheelBuildAutoDetect(t *testing.T) { matches, err := filepath.Glob("./python_wheel/python_wheel_no_artifact/dist/my_test_code-*.whl") require.NoError(t, err) - require.Equal(t, 1, len(matches)) + require.Len(t, matches, 1) match := libraries.ExpandGlobReferences() diags = bundle.Apply(ctx, b, match) @@ -55,7 +55,7 @@ func TestPythonWheelBuildAutoDetectWithNotebookTask(t *testing.T) { matches, err := filepath.Glob("./python_wheel/python_wheel_no_artifact_notebook/dist/my_test_code-*.whl") require.NoError(t, err) - require.Equal(t, 1, len(matches)) + require.Len(t, matches, 1) match := libraries.ExpandGlobReferences() diags = bundle.Apply(ctx, b, match) @@ -108,7 +108,7 @@ func TestPythonWheelBuildWithEnvironmentKey(t *testing.T) { matches, err := filepath.Glob("./python_wheel/environment_key/my_test_code/dist/my_test_code-*.whl") require.NoError(t, err) - require.Equal(t, 1, len(matches)) + require.Len(t, matches, 1) match := libraries.ExpandGlobReferences() diags = bundle.Apply(ctx, b, match) @@ -124,7 +124,7 @@ func TestPythonWheelBuildMultiple(t *testing.T) { matches, err := filepath.Glob("./python_wheel/python_wheel_multiple/my_test_code/dist/my_test_code*.whl") require.NoError(t, err) - require.Equal(t, 2, len(matches)) + require.Len(t, matches, 2) match := libraries.ExpandGlobReferences() diags = bundle.Apply(ctx, b, match) diff --git a/bundle/tests/quality_monitor_test.go b/bundle/tests/quality_monitor_test.go index 9b91052f5..e95c7b7c1 100644 --- a/bundle/tests/quality_monitor_test.go +++ b/bundle/tests/quality_monitor_test.go @@ -19,7 +19,7 @@ func assertExpectedMonitor(t *testing.T, p *resources.QualityMonitor) { func TestMonitorTableNames(t *testing.T) { b := loadTarget(t, "./quality_monitor", "development") assert.Len(t, b.Config.Resources.QualityMonitors, 1) - assert.Equal(t, b.Config.Bundle.Mode, config.Development) + assert.Equal(t, config.Development, b.Config.Bundle.Mode) p := b.Config.Resources.QualityMonitors["my_monitor"] assert.Equal(t, "main.test.dev", p.TableName) diff --git a/bundle/tests/registered_model_test.go b/bundle/tests/registered_model_test.go index 008db8bdd..e9d572a3a 100644 --- a/bundle/tests/registered_model_test.go +++ b/bundle/tests/registered_model_test.go @@ -19,7 +19,7 @@ func assertExpectedModel(t *testing.T, p *resources.RegisteredModel) { func TestRegisteredModelDevelopment(t *testing.T) { b := loadTarget(t, "./registered_model", "development") assert.Len(t, b.Config.Resources.RegisteredModels, 1) - assert.Equal(t, b.Config.Bundle.Mode, config.Development) + assert.Equal(t, config.Development, b.Config.Bundle.Mode) p := b.Config.Resources.RegisteredModels["my_registered_model"] assert.Equal(t, "my-dev-model", p.Name) diff --git a/bundle/tests/sync_include_exclude_no_matches_test.go b/bundle/tests/sync_include_exclude_no_matches_test.go index 0192b61e6..c206e7471 100644 --- a/bundle/tests/sync_include_exclude_no_matches_test.go +++ b/bundle/tests/sync_include_exclude_no_matches_test.go @@ -20,26 +20,26 @@ func TestSyncIncludeExcludeNoMatchesTest(t *testing.T) { require.Len(t, diags, 3) require.NoError(t, diags.Error()) - require.Equal(t, diags[0].Severity, diag.Warning) - require.Equal(t, diags[0].Summary, "Pattern dist does not match any files") + require.Equal(t, diag.Warning, diags[0].Severity) + require.Equal(t, "Pattern dist does not match any files", diags[0].Summary) require.Len(t, diags[0].Paths, 1) - require.Equal(t, diags[0].Paths[0].String(), "sync.exclude[0]") + require.Equal(t, "sync.exclude[0]", diags[0].Paths[0].String()) assert.Len(t, diags[0].Locations, 1) require.Equal(t, diags[0].Locations[0].File, filepath.Join("sync", "override", "databricks.yml")) - require.Equal(t, diags[0].Locations[0].Line, 17) - require.Equal(t, diags[0].Locations[0].Column, 11) + require.Equal(t, 17, diags[0].Locations[0].Line) + require.Equal(t, 11, diags[0].Locations[0].Column) summaries := []string{ fmt.Sprintf("Pattern %s does not match any files", filepath.Join("src", "*")), fmt.Sprintf("Pattern %s does not match any files", filepath.Join("tests", "*")), } - require.Equal(t, diags[1].Severity, diag.Warning) + require.Equal(t, diag.Warning, diags[1].Severity) require.Contains(t, summaries, diags[1].Summary) - require.Equal(t, diags[2].Severity, diag.Warning) + require.Equal(t, diag.Warning, diags[2].Severity) require.Contains(t, summaries, diags[2].Summary) } @@ -47,7 +47,7 @@ func TestSyncIncludeWithNegate(t *testing.T) { b := loadTarget(t, "./sync/negate", "default") diags := bundle.ApplyReadOnly(context.Background(), bundle.ReadOnly(b), validate.ValidateSyncPatterns()) - require.Len(t, diags, 0) + require.Empty(t, diags) require.NoError(t, diags.Error()) } @@ -58,6 +58,6 @@ func TestSyncIncludeWithNegateNoMatches(t *testing.T) { require.Len(t, diags, 1) require.NoError(t, diags.Error()) - require.Equal(t, diags[0].Severity, diag.Warning) - require.Equal(t, diags[0].Summary, "Pattern !*.txt2 does not match any files") + require.Equal(t, diag.Warning, diags[0].Severity) + require.Equal(t, "Pattern !*.txt2 does not match any files", diags[0].Summary) } diff --git a/bundle/tests/sync_test.go b/bundle/tests/sync_test.go index 15644b67e..f5a0296a9 100644 --- a/bundle/tests/sync_test.go +++ b/bundle/tests/sync_test.go @@ -115,12 +115,12 @@ func TestSyncPathsNoRoot(t *testing.T) { // If set to nil, it won't sync anything. b = loadTarget(t, "./sync/paths_no_root", "nil") assert.Equal(t, filepath.FromSlash("sync/paths_no_root"), b.SyncRootPath) - assert.Len(t, b.Config.Sync.Paths, 0) + assert.Empty(t, b.Config.Sync.Paths) // If set to an empty sequence, it won't sync anything. b = loadTarget(t, "./sync/paths_no_root", "empty") assert.Equal(t, filepath.FromSlash("sync/paths_no_root"), b.SyncRootPath) - assert.Len(t, b.Config.Sync.Paths, 0) + assert.Empty(t, b.Config.Sync.Paths) } func TestSyncSharedCode(t *testing.T) { diff --git a/cmd/bundle/generate/dashboard_test.go b/cmd/bundle/generate/dashboard_test.go index f1161950b..33a463ea0 100644 --- a/cmd/bundle/generate/dashboard_test.go +++ b/cmd/bundle/generate/dashboard_test.go @@ -44,7 +44,7 @@ func TestDashboard_ErrorOnLegacyDashboard(t *testing.T) { _, diags := d.resolveID(ctx, b) require.Len(t, diags, 1) - assert.Equal(t, diags[0].Summary, "dashboard \"legacy dashboard\" is a legacy dashboard") + assert.Equal(t, "dashboard \"legacy dashboard\" is a legacy dashboard", diags[0].Summary) } func TestDashboard_ExistingID_Nominal(t *testing.T) { diff --git a/cmd/bundle/generate/generate_test.go b/cmd/bundle/generate/generate_test.go index 896b7de51..395d4ebd4 100644 --- a/cmd/bundle/generate/generate_test.go +++ b/cmd/bundle/generate/generate_test.go @@ -3,7 +3,6 @@ package generate import ( "bytes" "context" - "errors" "fmt" "io" "io/fs" @@ -302,7 +301,7 @@ func TestGenerateJobCommandOldFileRename(t *testing.T) { // Make sure file do not exists after the run _, err = os.Stat(oldFilename) - require.True(t, errors.Is(err, fs.ErrNotExist)) + require.ErrorIs(t, err, fs.ErrNotExist) data, err := os.ReadFile(filepath.Join(configDir, "test_job.job.yml")) require.NoError(t, err) diff --git a/cmd/configure/configure_test.go b/cmd/configure/configure_test.go index e2f6c1e29..14eb0674a 100644 --- a/cmd/configure/configure_test.go +++ b/cmd/configure/configure_test.go @@ -148,9 +148,9 @@ func TestEnvVarsConfigureNoInteractive(t *testing.T) { // We should only save host and token for a profile, other env variables should not be saved _, err = defaultSection.GetKey("auth_type") - assert.NotNil(t, err) + assert.Error(t, err) _, err = defaultSection.GetKey("metadata_service_url") - assert.NotNil(t, err) + assert.Error(t, err) } func TestEnvVarsConfigureNoArgsNoInteractive(t *testing.T) { diff --git a/cmd/labs/github/ref_test.go b/cmd/labs/github/ref_test.go index cc27d1e81..9668cd7ec 100644 --- a/cmd/labs/github/ref_test.go +++ b/cmd/labs/github/ref_test.go @@ -7,14 +7,15 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) func TestFileFromRef(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/databrickslabs/ucx/main/README.md" { _, err := w.Write([]byte(`abc`)) - require.NoError(t, err) + if !assert.NoError(t, err) { + return + } return } t.Logf("Requested: %s", r.URL.Path) @@ -34,7 +35,9 @@ func TestDownloadZipball(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/repos/databrickslabs/ucx/zipball/main" { _, err := w.Write([]byte(`abc`)) - require.NoError(t, err) + if !assert.NoError(t, err) { + return + } return } t.Logf("Requested: %s", r.URL.Path) diff --git a/cmd/labs/github/releases_test.go b/cmd/labs/github/releases_test.go index 9c3d7a959..93ac33aee 100644 --- a/cmd/labs/github/releases_test.go +++ b/cmd/labs/github/releases_test.go @@ -7,14 +7,15 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) func TestLoadsReleasesForCLI(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/repos/databricks/cli/releases" { _, err := w.Write([]byte(`[{"tag_name": "v1.2.3"}, {"tag_name": "v1.2.2"}]`)) - require.NoError(t, err) + if !assert.NoError(t, err) { + return + } return } t.Logf("Requested: %s", r.URL.Path) diff --git a/cmd/labs/github/repositories_test.go b/cmd/labs/github/repositories_test.go index 412b440bc..29ec2ce03 100644 --- a/cmd/labs/github/repositories_test.go +++ b/cmd/labs/github/repositories_test.go @@ -7,14 +7,13 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) func TestRepositories(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/users/databrickslabs/repos" { _, err := w.Write([]byte(`[{"name": "x"}]`)) - require.NoError(t, err) + assert.NoError(t, err) return } t.Logf("Requested: %s", r.URL.Path) @@ -28,5 +27,5 @@ func TestRepositories(t *testing.T) { r := NewRepositoryCache("databrickslabs", t.TempDir()) all, err := r.Load(ctx) assert.NoError(t, err) - assert.True(t, len(all) > 0) + assert.NotEmpty(t, all) } diff --git a/cmd/labs/localcache/jsonfile_test.go b/cmd/labs/localcache/jsonfile_test.go index 0d852174c..9d42c6179 100644 --- a/cmd/labs/localcache/jsonfile_test.go +++ b/cmd/labs/localcache/jsonfile_test.go @@ -22,7 +22,7 @@ func TestCreatesDirectoryIfNeeded(t *testing.T) { } first, err := c.Load(ctx, tick) assert.NoError(t, err) - assert.Equal(t, first, int64(1)) + assert.Equal(t, int64(1), first) } func TestImpossibleToCreateDir(t *testing.T) { diff --git a/cmd/labs/project/installer_test.go b/cmd/labs/project/installer_test.go index a69389b31..a01ba864a 100644 --- a/cmd/labs/project/installer_test.go +++ b/cmd/labs/project/installer_test.go @@ -26,6 +26,7 @@ import ( "github.com/databricks/databricks-sdk-go/service/compute" "github.com/databricks/databricks-sdk-go/service/iam" "github.com/databricks/databricks-sdk-go/service/sql" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -169,17 +170,17 @@ func TestInstallerWorksForReleases(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/databrickslabs/blueprint/v0.3.15/labs.yml" { raw, err := os.ReadFile("testdata/installed-in-home/.databricks/labs/blueprint/lib/labs.yml") - require.NoError(t, err) + assert.NoError(t, err) _, err = w.Write(raw) - require.NoError(t, err) + assert.NoError(t, err) return } if r.URL.Path == "/repos/databrickslabs/blueprint/zipball/v0.3.15" { raw, err := zipballFromFolder("testdata/installed-in-home/.databricks/labs/blueprint/lib") - require.NoError(t, err) + assert.NoError(t, err) w.Header().Add("Content-Type", "application/octet-stream") _, err = w.Write(raw) - require.NoError(t, err) + assert.NoError(t, err) return } if r.URL.Path == "/api/2.1/clusters/get" { @@ -376,17 +377,17 @@ func TestUpgraderWorksForReleases(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/databrickslabs/blueprint/v0.4.0/labs.yml" { raw, err := os.ReadFile("testdata/installed-in-home/.databricks/labs/blueprint/lib/labs.yml") - require.NoError(t, err) + assert.NoError(t, err) _, err = w.Write(raw) - require.NoError(t, err) + assert.NoError(t, err) return } if r.URL.Path == "/repos/databrickslabs/blueprint/zipball/v0.4.0" { raw, err := zipballFromFolder("testdata/installed-in-home/.databricks/labs/blueprint/lib") - require.NoError(t, err) + assert.NoError(t, err) w.Header().Add("Content-Type", "application/octet-stream") _, err = w.Write(raw) - require.NoError(t, err) + assert.NoError(t, err) return } if r.URL.Path == "/api/2.1/clusters/get" { diff --git a/integration/bundle/artifacts_test.go b/integration/bundle/artifacts_test.go index 1b71a1c3d..9dafe2f83 100644 --- a/integration/bundle/artifacts_test.go +++ b/integration/bundle/artifacts_test.go @@ -85,13 +85,13 @@ func TestUploadArtifactFileToCorrectRemotePath(t *testing.T) { // The remote path attribute on the artifact file should have been set. require.Regexp(t, - regexp.MustCompile(path.Join(regexp.QuoteMeta(wsDir), `.internal/test\.whl`)), + path.Join(regexp.QuoteMeta(wsDir), `.internal/test\.whl`), b.Config.Artifacts["test"].Files[0].RemotePath, ) // The task library path should have been updated to the remote path. require.Regexp(t, - regexp.MustCompile(path.Join("/Workspace", regexp.QuoteMeta(wsDir), `.internal/test\.whl`)), + path.Join("/Workspace", regexp.QuoteMeta(wsDir), `.internal/test\.whl`), b.Config.Resources.Jobs["test"].JobSettings.Tasks[0].Libraries[0].Whl, ) } @@ -149,13 +149,13 @@ func TestUploadArtifactFileToCorrectRemotePathWithEnvironments(t *testing.T) { // The remote path attribute on the artifact file should have been set. require.Regexp(t, - regexp.MustCompile(path.Join(regexp.QuoteMeta(wsDir), `.internal/test\.whl`)), + path.Join(regexp.QuoteMeta(wsDir), `.internal/test\.whl`), b.Config.Artifacts["test"].Files[0].RemotePath, ) // The job environment deps path should have been updated to the remote path. require.Regexp(t, - regexp.MustCompile(path.Join("/Workspace", regexp.QuoteMeta(wsDir), `.internal/test\.whl`)), + path.Join("/Workspace", regexp.QuoteMeta(wsDir), `.internal/test\.whl`), b.Config.Resources.Jobs["test"].JobSettings.Environments[0].Spec.Dependencies[0], ) } @@ -218,13 +218,13 @@ func TestUploadArtifactFileToCorrectRemotePathForVolumes(t *testing.T) { // The remote path attribute on the artifact file should have been set. require.Regexp(t, - regexp.MustCompile(path.Join(regexp.QuoteMeta(volumePath), `.internal/test\.whl`)), + path.Join(regexp.QuoteMeta(volumePath), `.internal/test\.whl`), b.Config.Artifacts["test"].Files[0].RemotePath, ) // The task library path should have been updated to the remote path. require.Regexp(t, - regexp.MustCompile(path.Join(regexp.QuoteMeta(volumePath), `.internal/test\.whl`)), + path.Join(regexp.QuoteMeta(volumePath), `.internal/test\.whl`), b.Config.Resources.Jobs["test"].JobSettings.Tasks[0].Libraries[0].Whl, ) } diff --git a/integration/bundle/deploy_test.go b/integration/bundle/deploy_test.go index 0b37e5630..309b82917 100644 --- a/integration/bundle/deploy_test.go +++ b/integration/bundle/deploy_test.go @@ -2,7 +2,6 @@ package bundle_test import ( "context" - "errors" "fmt" "io" "os" @@ -99,7 +98,7 @@ func TestBundleDeployUcSchema(t *testing.T) { // Assert the schema is deleted _, err = w.Schemas.GetByFullName(ctx, strings.Join([]string{catalogName, schemaName}, ".")) apiErr := &apierr.APIError{} - assert.True(t, errors.As(err, &apiErr)) + assert.ErrorAs(t, err, &apiErr) assert.Equal(t, "SCHEMA_DOES_NOT_EXIST", apiErr.ErrorCode) } diff --git a/integration/bundle/destroy_test.go b/integration/bundle/destroy_test.go index f18138ce5..b69382a58 100644 --- a/integration/bundle/destroy_test.go +++ b/integration/bundle/destroy_test.go @@ -1,7 +1,6 @@ package bundle_test import ( - "errors" "os" "path/filepath" "testing" @@ -71,11 +70,11 @@ func TestBundleDestroy(t *testing.T) { // Assert snapshot file is deleted entries, err = os.ReadDir(snapshotsDir) require.NoError(t, err) - assert.Len(t, entries, 0) + assert.Empty(t, entries) // Assert bundle deployment path is deleted _, err = w.Workspace.GetStatusByPath(ctx, remoteRoot) apiErr := &apierr.APIError{} - assert.True(t, errors.As(err, &apiErr)) + assert.ErrorAs(t, err, &apiErr) assert.Equal(t, "RESOURCE_DOES_NOT_EXIST", apiErr.ErrorCode) } diff --git a/integration/cmd/fs/ls_test.go b/integration/cmd/fs/ls_test.go index 58e776d8a..25929fdf3 100644 --- a/integration/cmd/fs/ls_test.go +++ b/integration/cmd/fs/ls_test.go @@ -5,7 +5,6 @@ import ( "encoding/json" "io/fs" "path" - "regexp" "strings" "testing" @@ -65,11 +64,11 @@ func TestFsLs(t *testing.T) { assert.Equal(t, "a", parsedStdout[0]["name"]) assert.Equal(t, true, parsedStdout[0]["is_directory"]) - assert.Equal(t, float64(0), parsedStdout[0]["size"]) + assert.InDelta(t, float64(0), parsedStdout[0]["size"], 0.0001) assert.Equal(t, "bye.txt", parsedStdout[1]["name"]) assert.Equal(t, false, parsedStdout[1]["is_directory"]) - assert.Equal(t, float64(3), parsedStdout[1]["size"]) + assert.InDelta(t, float64(3), parsedStdout[1]["size"], 0.0001) }) } } @@ -99,11 +98,11 @@ func TestFsLsWithAbsolutePaths(t *testing.T) { assert.Equal(t, path.Join(tmpDir, "a"), parsedStdout[0]["name"]) assert.Equal(t, true, parsedStdout[0]["is_directory"]) - assert.Equal(t, float64(0), parsedStdout[0]["size"]) + assert.InDelta(t, float64(0), parsedStdout[0]["size"], 0.0001) assert.Equal(t, path.Join(tmpDir, "bye.txt"), parsedStdout[1]["name"]) assert.Equal(t, false, parsedStdout[1]["is_directory"]) - assert.Equal(t, float64(3), parsedStdout[1]["size"]) + assert.InDelta(t, float64(3), parsedStdout[1]["size"].(float64), 0.0001) }) } } @@ -122,7 +121,7 @@ func TestFsLsOnFile(t *testing.T) { setupLsFiles(t, f) _, _, err := testcli.RequireErrorRun(t, ctx, "fs", "ls", path.Join(tmpDir, "a", "hello.txt"), "--output=json") - assert.Regexp(t, regexp.MustCompile("not a directory: .*/a/hello.txt"), err.Error()) + assert.Regexp(t, "not a directory: .*/a/hello.txt", err.Error()) assert.ErrorAs(t, err, &filer.NotADirectory{}) }) } @@ -147,7 +146,7 @@ func TestFsLsOnEmptyDir(t *testing.T) { require.NoError(t, err) // assert on ls output - assert.Equal(t, 0, len(parsedStdout)) + assert.Empty(t, parsedStdout) }) } } @@ -166,7 +165,7 @@ func TestFsLsForNonexistingDir(t *testing.T) { _, _, err := testcli.RequireErrorRun(t, ctx, "fs", "ls", path.Join(tmpDir, "nonexistent"), "--output=json") assert.ErrorIs(t, err, fs.ErrNotExist) - assert.Regexp(t, regexp.MustCompile("no such directory: .*/nonexistent"), err.Error()) + assert.Regexp(t, "no such directory: .*/nonexistent", err.Error()) }) } } diff --git a/integration/cmd/fs/mkdir_test.go b/integration/cmd/fs/mkdir_test.go index f332bb526..eff0599a7 100644 --- a/integration/cmd/fs/mkdir_test.go +++ b/integration/cmd/fs/mkdir_test.go @@ -34,7 +34,7 @@ func TestFsMkdir(t *testing.T) { info, err := f.Stat(context.Background(), "a") require.NoError(t, err) assert.Equal(t, "a", info.Name()) - assert.Equal(t, true, info.IsDir()) + assert.True(t, info.IsDir()) }) } } @@ -60,19 +60,19 @@ func TestFsMkdirCreatesIntermediateDirectories(t *testing.T) { infoA, err := f.Stat(context.Background(), "a") require.NoError(t, err) assert.Equal(t, "a", infoA.Name()) - assert.Equal(t, true, infoA.IsDir()) + assert.True(t, infoA.IsDir()) // assert directory "b" is created infoB, err := f.Stat(context.Background(), "a/b") require.NoError(t, err) assert.Equal(t, "b", infoB.Name()) - assert.Equal(t, true, infoB.IsDir()) + assert.True(t, infoB.IsDir()) // assert directory "c" is created infoC, err := f.Stat(context.Background(), "a/b/c") require.NoError(t, err) assert.Equal(t, "c", infoC.Name()) - assert.Equal(t, true, infoC.IsDir()) + assert.True(t, infoC.IsDir()) }) } } diff --git a/integration/cmd/sync/sync_test.go b/integration/cmd/sync/sync_test.go index 984f6ac49..6f58b7e42 100644 --- a/integration/cmd/sync/sync_test.go +++ b/integration/cmd/sync/sync_test.go @@ -225,7 +225,7 @@ func (a *syncTest) snapshotContains(files []string) { assert.Equal(a.t, s.RemotePath, a.remoteRoot) for _, filePath := range files { _, ok := s.LastModifiedTimes[filePath] - assert.True(a.t, ok, fmt.Sprintf("%s not in snapshot file: %v", filePath, s.LastModifiedTimes)) + assert.True(a.t, ok, "%s not in snapshot file: %v", filePath, s.LastModifiedTimes) } assert.Equal(a.t, len(files), len(s.LastModifiedTimes)) } diff --git a/integration/libs/filer/filer_test.go b/integration/libs/filer/filer_test.go index 766f9817b..21c839e1b 100644 --- a/integration/libs/filer/filer_test.go +++ b/integration/libs/filer/filer_test.go @@ -4,11 +4,9 @@ import ( "bytes" "context" "encoding/json" - "errors" "io" "io/fs" "path" - "regexp" "strings" "testing" @@ -106,7 +104,7 @@ func commonFilerRecursiveDeleteTest(t *testing.T, ctx context.Context, f filer.F for _, e := range entriesBeforeDelete { names = append(names, e.Name()) } - assert.Equal(t, names, []string{"file1", "file2", "subdir1", "subdir2"}) + assert.Equal(t, []string{"file1", "file2", "subdir1", "subdir2"}, names) err = f.Delete(ctx, "dir") assert.ErrorAs(t, err, &filer.DirectoryNotEmptyError{}) @@ -149,13 +147,13 @@ func commonFilerReadWriteTests(t *testing.T, ctx context.Context, f filer.Filer) // Write should fail because the intermediate directory doesn't exist. err = f.Write(ctx, "/foo/bar", strings.NewReader(`hello world`)) - assert.True(t, errors.As(err, &filer.NoSuchDirectoryError{})) - assert.True(t, errors.Is(err, fs.ErrNotExist)) + assert.ErrorAs(t, err, &filer.NoSuchDirectoryError{}) + assert.ErrorIs(t, err, fs.ErrNotExist) // Read should fail because the intermediate directory doesn't yet exist. _, err = f.Read(ctx, "/foo/bar") - assert.True(t, errors.As(err, &filer.FileDoesNotExistError{})) - assert.True(t, errors.Is(err, fs.ErrNotExist)) + assert.ErrorAs(t, err, &filer.FileDoesNotExistError{}) + assert.ErrorIs(t, err, fs.ErrNotExist) // Read should fail because the path points to a directory err = f.Mkdir(ctx, "/dir") @@ -170,8 +168,8 @@ func commonFilerReadWriteTests(t *testing.T, ctx context.Context, f filer.Filer) // Write should fail because there is an existing file at the specified path. err = f.Write(ctx, "/foo/bar", strings.NewReader(`hello universe`)) - assert.True(t, errors.As(err, &filer.FileAlreadyExistsError{})) - assert.True(t, errors.Is(err, fs.ErrExist)) + assert.ErrorAs(t, err, &filer.FileAlreadyExistsError{}) + assert.ErrorIs(t, err, fs.ErrExist) // Write with OverwriteIfExists should succeed. err = f.Write(ctx, "/foo/bar", strings.NewReader(`hello universe`), filer.OverwriteIfExists) @@ -188,7 +186,7 @@ func commonFilerReadWriteTests(t *testing.T, ctx context.Context, f filer.Filer) require.NoError(t, err) assert.Equal(t, "foo", info.Name()) assert.True(t, info.Mode().IsDir()) - assert.Equal(t, true, info.IsDir()) + assert.True(t, info.IsDir()) // Stat on a file should succeed. // Note: size and modification time behave differently between backends. @@ -196,17 +194,17 @@ func commonFilerReadWriteTests(t *testing.T, ctx context.Context, f filer.Filer) require.NoError(t, err) assert.Equal(t, "bar", info.Name()) assert.True(t, info.Mode().IsRegular()) - assert.Equal(t, false, info.IsDir()) + assert.False(t, info.IsDir()) // Delete should fail if the file doesn't exist. err = f.Delete(ctx, "/doesnt_exist") assert.ErrorAs(t, err, &filer.FileDoesNotExistError{}) - assert.True(t, errors.Is(err, fs.ErrNotExist)) + assert.ErrorIs(t, err, fs.ErrNotExist) // Stat should fail if the file doesn't exist. _, err = f.Stat(ctx, "/doesnt_exist") assert.ErrorAs(t, err, &filer.FileDoesNotExistError{}) - assert.True(t, errors.Is(err, fs.ErrNotExist)) + assert.ErrorIs(t, err, fs.ErrNotExist) // Delete should succeed for file that does exist. err = f.Delete(ctx, "/foo/bar") @@ -215,7 +213,7 @@ func commonFilerReadWriteTests(t *testing.T, ctx context.Context, f filer.Filer) // Delete should fail for a non-empty directory. err = f.Delete(ctx, "/foo") assert.ErrorAs(t, err, &filer.DirectoryNotEmptyError{}) - assert.True(t, errors.Is(err, fs.ErrInvalid)) + assert.ErrorIs(t, err, fs.ErrInvalid) // Delete should succeed for a non-empty directory if the DeleteRecursively flag is set. err = f.Delete(ctx, "/foo", filer.DeleteRecursively) @@ -224,8 +222,8 @@ func commonFilerReadWriteTests(t *testing.T, ctx context.Context, f filer.Filer) // Delete of the filer root should ALWAYS fail, otherwise subsequent writes would fail. // It is not in the filer's purview to delete its root directory. err = f.Delete(ctx, "/") - assert.True(t, errors.As(err, &filer.CannotDeleteRootError{})) - assert.True(t, errors.Is(err, fs.ErrInvalid)) + assert.ErrorAs(t, err, &filer.CannotDeleteRootError{}) + assert.ErrorIs(t, err, fs.ErrInvalid) } func TestFilerReadWrite(t *testing.T) { @@ -262,7 +260,7 @@ func commonFilerReadDirTest(t *testing.T, ctx context.Context, f filer.Filer) { // We start with an empty directory. entries, err := f.ReadDir(ctx, ".") require.NoError(t, err) - assert.Len(t, entries, 0) + assert.Empty(t, entries) // Write a file. err = f.Write(ctx, "/hello.txt", strings.NewReader(`hello world`)) @@ -282,8 +280,8 @@ func commonFilerReadDirTest(t *testing.T, ctx context.Context, f filer.Filer) { // Expect an error if the path doesn't exist. _, err = f.ReadDir(ctx, "/dir/a/b/c/d/e") - assert.True(t, errors.As(err, &filer.NoSuchDirectoryError{}), err) - assert.True(t, errors.Is(err, fs.ErrNotExist)) + assert.ErrorAs(t, err, &filer.NoSuchDirectoryError{}, err) + assert.ErrorIs(t, err, fs.ErrNotExist) // Expect two entries in the root. entries, err = f.ReadDir(ctx, ".") @@ -295,7 +293,7 @@ func commonFilerReadDirTest(t *testing.T, ctx context.Context, f filer.Filer) { assert.False(t, entries[1].IsDir()) info, err = entries[1].Info() require.NoError(t, err) - assert.Greater(t, info.ModTime().Unix(), int64(0)) + assert.Positive(t, info.ModTime().Unix()) // Expect two entries in the directory. entries, err = f.ReadDir(ctx, "/dir") @@ -307,7 +305,7 @@ func commonFilerReadDirTest(t *testing.T, ctx context.Context, f filer.Filer) { assert.False(t, entries[1].IsDir()) info, err = entries[1].Info() require.NoError(t, err) - assert.Greater(t, info.ModTime().Unix(), int64(0)) + assert.Positive(t, info.ModTime().Unix()) // Expect a single entry in the nested path. entries, err = f.ReadDir(ctx, "/dir/a/b") @@ -325,7 +323,7 @@ func commonFilerReadDirTest(t *testing.T, ctx context.Context, f filer.Filer) { require.NoError(t, err) entries, err = f.ReadDir(ctx, "empty-dir") assert.NoError(t, err) - assert.Len(t, entries, 0) + assert.Empty(t, entries) // Expect one entry for a directory with a file in it err = f.Write(ctx, "dir-with-one-file/my-file.txt", strings.NewReader("abc"), filer.CreateParentDirectories) @@ -333,7 +331,7 @@ func commonFilerReadDirTest(t *testing.T, ctx context.Context, f filer.Filer) { entries, err = f.ReadDir(ctx, "dir-with-one-file") assert.NoError(t, err) assert.Len(t, entries, 1) - assert.Equal(t, entries[0].Name(), "my-file.txt") + assert.Equal(t, "my-file.txt", entries[0].Name()) assert.False(t, entries[0].IsDir()) } @@ -459,7 +457,7 @@ func TestFilerWorkspaceNotebook(t *testing.T) { // Assert uploading a second time fails due to overwrite mode missing err = f.Write(ctx, tc.name, strings.NewReader(tc.content2)) require.ErrorIs(t, err, fs.ErrExist) - assert.Regexp(t, regexp.MustCompile(`file already exists: .*/`+tc.nameWithoutExt+`$`), err.Error()) + assert.Regexp(t, `file already exists: .*/`+tc.nameWithoutExt+`$`, err.Error()) // Try uploading the notebook again with overwrite flag. This time it should succeed. err = f.Write(ctx, tc.name, strings.NewReader(tc.content2), filer.OverwriteIfExists) diff --git a/libs/auth/cache/file_test.go b/libs/auth/cache/file_test.go index 3e4aae36f..54964bed3 100644 --- a/libs/auth/cache/file_test.go +++ b/libs/auth/cache/file_test.go @@ -42,7 +42,7 @@ func TestStoreAndLookup(t *testing.T) { tok, err := l.Lookup("x") require.NoError(t, err) assert.Equal(t, "abc", tok.AccessToken) - assert.Equal(t, 2, len(l.Tokens)) + assert.Len(t, l.Tokens, 2) _, err = l.Lookup("z") assert.Equal(t, ErrNotConfigured, err) diff --git a/libs/databrickscfg/ops_test.go b/libs/databrickscfg/ops_test.go index 3ea92024c..dd8484fb7 100644 --- a/libs/databrickscfg/ops_test.go +++ b/libs/databrickscfg/ops_test.go @@ -216,7 +216,7 @@ func TestSaveToProfile_ClearingPreviousProfile(t *testing.T) { dlft, err := file.GetSection("DEFAULT") assert.NoError(t, err) - assert.Len(t, dlft.KeysHash(), 0) + assert.Empty(t, dlft.KeysHash()) abc, err := file.GetSection("abc") assert.NoError(t, err) diff --git a/libs/databrickscfg/profile/file_test.go b/libs/databrickscfg/profile/file_test.go index 8e5cfefc0..6bcaec4b7 100644 --- a/libs/databrickscfg/profile/file_test.go +++ b/libs/databrickscfg/profile/file_test.go @@ -11,10 +11,10 @@ import ( ) func TestProfileCloud(t *testing.T) { - assert.Equal(t, Profile{Host: "https://dbc-XXXXXXXX-YYYY.cloud.databricks.com"}.Cloud(), "AWS") - assert.Equal(t, Profile{Host: "https://adb-xxx.y.azuredatabricks.net/"}.Cloud(), "Azure") - assert.Equal(t, Profile{Host: "https://workspace.gcp.databricks.com/"}.Cloud(), "GCP") - assert.Equal(t, Profile{Host: "https://some.invalid.host.com/"}.Cloud(), "AWS") + assert.Equal(t, "AWS", Profile{Host: "https://dbc-XXXXXXXX-YYYY.cloud.databricks.com"}.Cloud()) + assert.Equal(t, "Azure", Profile{Host: "https://adb-xxx.y.azuredatabricks.net/"}.Cloud()) + assert.Equal(t, "GCP", Profile{Host: "https://workspace.gcp.databricks.com/"}.Cloud()) + assert.Equal(t, "AWS", Profile{Host: "https://some.invalid.host.com/"}.Cloud()) } func TestProfilesSearchCaseInsensitive(t *testing.T) { diff --git a/libs/errs/aggregate_test.go b/libs/errs/aggregate_test.go index a307e956f..1af57e099 100644 --- a/libs/errs/aggregate_test.go +++ b/libs/errs/aggregate_test.go @@ -1,7 +1,6 @@ package errs import ( - "errors" "fmt" "testing" @@ -14,13 +13,13 @@ func TestFromManyErrors(t *testing.T) { e3 := fmt.Errorf("Error 3") err := FromMany(e1, e2, e3) - assert.True(t, errors.Is(err, e1)) - assert.True(t, errors.Is(err, e2)) - assert.True(t, errors.Is(err, e3)) + assert.ErrorIs(t, err, e1) + assert.ErrorIs(t, err, e2) + assert.ErrorIs(t, err, e3) - assert.Equal(t, err.Error(), `Error 1 + assert.Equal(t, `Error 1 Error 2 -Error 3`) +Error 3`, err.Error()) } func TestFromManyErrorsWihtNil(t *testing.T) { @@ -29,9 +28,9 @@ func TestFromManyErrorsWihtNil(t *testing.T) { e3 := fmt.Errorf("Error 3") err := FromMany(e1, e2, e3) - assert.True(t, errors.Is(err, e1)) - assert.True(t, errors.Is(err, e3)) + assert.ErrorIs(t, err, e1) + assert.ErrorIs(t, err, e3) - assert.Equal(t, err.Error(), `Error 1 -Error 3`) + assert.Equal(t, `Error 1 +Error 3`, err.Error()) } diff --git a/libs/filer/completer/completer_test.go b/libs/filer/completer/completer_test.go index d284447b9..865d34c2f 100644 --- a/libs/filer/completer/completer_test.go +++ b/libs/filer/completer/completer_test.go @@ -37,7 +37,7 @@ func TestFilerCompleterSetsPrefix(t *testing.T) { assert.Equal(t, []string{"dbfs:/dir/dirA/", "dbfs:/dir/dirB/"}, completions) assert.Equal(t, cobra.ShellCompDirectiveNoSpace, directive) - assert.Nil(t, err) + assert.NoError(t, err) } func TestFilerCompleterReturnsNestedDirs(t *testing.T) { @@ -46,7 +46,7 @@ func TestFilerCompleterReturnsNestedDirs(t *testing.T) { assert.Equal(t, []string{"dir/dirA/", "dir/dirB/"}, completions) assert.Equal(t, cobra.ShellCompDirectiveNoSpace, directive) - assert.Nil(t, err) + assert.NoError(t, err) } func TestFilerCompleterReturnsAdjacentDirs(t *testing.T) { @@ -55,7 +55,7 @@ func TestFilerCompleterReturnsAdjacentDirs(t *testing.T) { assert.Equal(t, []string{"dir/dirA/", "dir/dirB/"}, completions) assert.Equal(t, cobra.ShellCompDirectiveNoSpace, directive) - assert.Nil(t, err) + assert.NoError(t, err) } func TestFilerCompleterReturnsNestedDirsAndFiles(t *testing.T) { @@ -64,7 +64,7 @@ func TestFilerCompleterReturnsNestedDirsAndFiles(t *testing.T) { assert.Equal(t, []string{"dir/dirA/", "dir/dirB/", "dir/fileA"}, completions) assert.Equal(t, cobra.ShellCompDirectiveNoSpace, directive) - assert.Nil(t, err) + assert.NoError(t, err) } func TestFilerCompleterAddsDbfsPath(t *testing.T) { @@ -78,7 +78,7 @@ func TestFilerCompleterAddsDbfsPath(t *testing.T) { assert.Equal(t, []string{"dir/dirA/", "dir/dirB/", "dbfs:/"}, completions) assert.Equal(t, cobra.ShellCompDirectiveNoSpace, directive) - assert.Nil(t, err) + assert.NoError(t, err) } func TestFilerCompleterWindowsSeparator(t *testing.T) { @@ -92,7 +92,7 @@ func TestFilerCompleterWindowsSeparator(t *testing.T) { assert.Equal(t, []string{"dir\\dirA\\", "dir\\dirB\\", "dbfs:/"}, completions) assert.Equal(t, cobra.ShellCompDirectiveNoSpace, directive) - assert.Nil(t, err) + assert.NoError(t, err) } func TestFilerCompleterNoCompletions(t *testing.T) { diff --git a/libs/filer/fs_test.go b/libs/filer/fs_test.go index 849cf6f7c..08d7a9428 100644 --- a/libs/filer/fs_test.go +++ b/libs/filer/fs_test.go @@ -63,7 +63,7 @@ func TestFsOpenFile(t *testing.T) { assert.Equal(t, "fileA", info.Name()) assert.Equal(t, int64(3), info.Size()) assert.Equal(t, fs.FileMode(0), info.Mode()) - assert.Equal(t, false, info.IsDir()) + assert.False(t, info.IsDir()) // Read until closed. b := make([]byte, 3) @@ -91,7 +91,7 @@ func TestFsOpenDir(t *testing.T) { info, err := fakeFile.Stat() require.NoError(t, err) assert.Equal(t, "root", info.Name()) - assert.Equal(t, true, info.IsDir()) + assert.True(t, info.IsDir()) de, ok := fakeFile.(fs.ReadDirFile) require.True(t, ok) diff --git a/libs/fileset/glob_test.go b/libs/fileset/glob_test.go index 9eb786db9..0224b2547 100644 --- a/libs/fileset/glob_test.go +++ b/libs/fileset/glob_test.go @@ -52,7 +52,7 @@ func TestGlobFileset(t *testing.T) { files, err = g.Files() require.NoError(t, err) - require.Equal(t, len(files), 0) + require.Empty(t, files) } func TestGlobFilesetWithRelativeRoot(t *testing.T) { diff --git a/libs/flags/json_flag_test.go b/libs/flags/json_flag_test.go index b31324011..956a3541c 100644 --- a/libs/flags/json_flag_test.go +++ b/libs/flags/json_flag_test.go @@ -123,12 +123,12 @@ func TestJsonUnmarshalForRequest(t *testing.T) { assert.Equal(t, "new job", r.NewSettings.Name) assert.Equal(t, 0, r.NewSettings.TimeoutSeconds) assert.Equal(t, 1, r.NewSettings.MaxConcurrentRuns) - assert.Equal(t, 1, len(r.NewSettings.Tasks)) + assert.Len(t, r.NewSettings.Tasks, 1) assert.Equal(t, "new task", r.NewSettings.Tasks[0].TaskKey) assert.Equal(t, 0, r.NewSettings.Tasks[0].TimeoutSeconds) assert.Equal(t, 0, r.NewSettings.Tasks[0].MaxRetries) assert.Equal(t, 0, r.NewSettings.Tasks[0].MinRetryIntervalMillis) - assert.Equal(t, true, r.NewSettings.Tasks[0].RetryOnTimeout) + assert.True(t, r.NewSettings.Tasks[0].RetryOnTimeout) } const incorrectJsonData = `{ @@ -280,8 +280,8 @@ func TestJsonUnmarshalForRequestWithForceSendFields(t *testing.T) { assert.NoError(t, diags.Error()) assert.Empty(t, diags) - assert.Equal(t, false, r.NewSettings.NotificationSettings.NoAlertForSkippedRuns) - assert.Equal(t, false, r.NewSettings.NotificationSettings.NoAlertForCanceledRuns) + assert.False(t, r.NewSettings.NotificationSettings.NoAlertForSkippedRuns) + assert.False(t, r.NewSettings.NotificationSettings.NoAlertForCanceledRuns) assert.NotContains(t, r.NewSettings.NotificationSettings.ForceSendFields, "NoAlertForSkippedRuns") assert.Contains(t, r.NewSettings.NotificationSettings.ForceSendFields, "NoAlertForCanceledRuns") } diff --git a/libs/folders/folders_test.go b/libs/folders/folders_test.go index 17afc4022..d2afc4f2d 100644 --- a/libs/folders/folders_test.go +++ b/libs/folders/folders_test.go @@ -33,6 +33,6 @@ func TestFindDirWithLeaf(t *testing.T) { { out, err := FindDirWithLeaf(root, "this-leaf-doesnt-exist-anywhere") assert.ErrorIs(t, err, os.ErrNotExist) - assert.Equal(t, out, "") + assert.Equal(t, "", out) } } diff --git a/libs/jsonschema/utils_test.go b/libs/jsonschema/utils_test.go index 89200dae3..954c723d3 100644 --- a/libs/jsonschema/utils_test.go +++ b/libs/jsonschema/utils_test.go @@ -96,7 +96,7 @@ func TestTemplateFromString(t *testing.T) { v, err = fromString("1.1", NumberType) assert.NoError(t, err) // Floating point conversions are not perfect - assert.True(t, (v.(float64)-1.1) < 0.000001) + assert.Less(t, (v.(float64) - 1.1), 0.000001) v, err = fromString("12345", IntegerType) assert.NoError(t, err) @@ -104,7 +104,7 @@ func TestTemplateFromString(t *testing.T) { v, err = fromString("123", NumberType) assert.NoError(t, err) - assert.Equal(t, float64(123), v) + assert.InDelta(t, float64(123), v.(float64), 0.0001) _, err = fromString("qrt", ArrayType) assert.EqualError(t, err, "cannot parse string as object of type array. Value of string: \"qrt\"") diff --git a/libs/notebook/detect_test.go b/libs/notebook/detect_test.go index 4ede7bf9b..49a67d2d3 100644 --- a/libs/notebook/detect_test.go +++ b/libs/notebook/detect_test.go @@ -1,7 +1,6 @@ package notebook import ( - "errors" "io/fs" "os" "path/filepath" @@ -53,7 +52,7 @@ func TestDetectCallsDetectJupyter(t *testing.T) { func TestDetectUnknownExtension(t *testing.T) { _, _, err := Detect("./testdata/doesntexist.foobar") - assert.True(t, errors.Is(err, fs.ErrNotExist)) + assert.ErrorIs(t, err, fs.ErrNotExist) nb, _, err := Detect("./testdata/unknown_extension.foobar") require.NoError(t, err) @@ -62,7 +61,7 @@ func TestDetectUnknownExtension(t *testing.T) { func TestDetectNoExtension(t *testing.T) { _, _, err := Detect("./testdata/doesntexist") - assert.True(t, errors.Is(err, fs.ErrNotExist)) + assert.ErrorIs(t, err, fs.ErrNotExist) nb, _, err := Detect("./testdata/no_extension") require.NoError(t, err) diff --git a/libs/process/background_test.go b/libs/process/background_test.go index 2e47e814b..7843375cf 100644 --- a/libs/process/background_test.go +++ b/libs/process/background_test.go @@ -95,7 +95,7 @@ func TestBackgroundNoStdin(t *testing.T) { func TestBackgroundFails(t *testing.T) { ctx := context.Background() _, err := Background(ctx, []string{"ls", "/dev/null/x"}) - assert.NotNil(t, err) + assert.Error(t, err) } func TestBackgroundFailsOnOption(t *testing.T) { diff --git a/libs/process/forwarded_test.go b/libs/process/forwarded_test.go index ddb79818f..71f0a6a63 100644 --- a/libs/process/forwarded_test.go +++ b/libs/process/forwarded_test.go @@ -27,7 +27,7 @@ func TestForwardedFails(t *testing.T) { err := Forwarded(ctx, []string{ "_non_existent_", }, strings.NewReader("abc\n"), &buf, &buf) - assert.NotNil(t, err) + assert.Error(t, err) } func TestForwardedFailsOnStdinPipe(t *testing.T) { @@ -39,5 +39,5 @@ func TestForwardedFailsOnStdinPipe(t *testing.T) { c.Stdin = strings.NewReader("x") return nil }) - assert.NotNil(t, err) + assert.Error(t, err) } diff --git a/libs/process/opts_test.go b/libs/process/opts_test.go index 3a819fbb9..8b5d51928 100644 --- a/libs/process/opts_test.go +++ b/libs/process/opts_test.go @@ -41,7 +41,7 @@ func TestWorksWithLibsEnv(t *testing.T) { vars := cmd.Environ() sort.Strings(vars) - assert.True(t, len(vars) >= 2) + assert.GreaterOrEqual(t, len(vars), 2) assert.Equal(t, "CCC=DDD", vars[0]) assert.Equal(t, "EEE=FFF", vars[1]) } diff --git a/libs/python/interpreters_unix_test.go b/libs/python/interpreters_unix_test.go index 8471644a1..57adc9279 100644 --- a/libs/python/interpreters_unix_test.go +++ b/libs/python/interpreters_unix_test.go @@ -18,7 +18,7 @@ func TestAtLeastOnePythonInstalled(t *testing.T) { assert.NoError(t, err) a := all.Latest() t.Logf("latest is: %s", a) - assert.True(t, len(all) > 0) + assert.NotEmpty(t, all) } func TestNoInterpretersFound(t *testing.T) { diff --git a/libs/sync/snapshot_test.go b/libs/sync/snapshot_test.go index eef526e58..4ba3874ae 100644 --- a/libs/sync/snapshot_test.go +++ b/libs/sync/snapshot_test.go @@ -51,7 +51,7 @@ func TestDiff(t *testing.T) { assert.NoError(t, err) change, err := state.diff(ctx, files) assert.NoError(t, err) - assert.Len(t, change.delete, 0) + assert.Empty(t, change.delete) assert.Len(t, change.put, 2) assert.Contains(t, change.put, "hello.txt") assert.Contains(t, change.put, "world.txt") @@ -67,7 +67,7 @@ func TestDiff(t *testing.T) { change, err = state.diff(ctx, files) assert.NoError(t, err) - assert.Len(t, change.delete, 0) + assert.Empty(t, change.delete) assert.Len(t, change.put, 1) assert.Contains(t, change.put, "world.txt") assertKeysOfMap(t, state.LastModifiedTimes, []string{"hello.txt", "world.txt"}) @@ -82,7 +82,7 @@ func TestDiff(t *testing.T) { change, err = state.diff(ctx, files) assert.NoError(t, err) assert.Len(t, change.delete, 1) - assert.Len(t, change.put, 0) + assert.Empty(t, change.put) assert.Contains(t, change.delete, "hello.txt") assertKeysOfMap(t, state.LastModifiedTimes, []string{"world.txt"}) assert.Equal(t, map[string]string{"world.txt": "world.txt"}, state.LocalToRemoteNames) @@ -145,8 +145,8 @@ func TestFolderDiff(t *testing.T) { assert.NoError(t, err) change, err := state.diff(ctx, files) assert.NoError(t, err) - assert.Len(t, change.delete, 0) - assert.Len(t, change.rmdir, 0) + assert.Empty(t, change.delete) + assert.Empty(t, change.rmdir) assert.Len(t, change.mkdir, 1) assert.Len(t, change.put, 1) assert.Contains(t, change.mkdir, "foo") @@ -159,8 +159,8 @@ func TestFolderDiff(t *testing.T) { assert.NoError(t, err) assert.Len(t, change.delete, 1) assert.Len(t, change.rmdir, 1) - assert.Len(t, change.mkdir, 0) - assert.Len(t, change.put, 0) + assert.Empty(t, change.mkdir) + assert.Empty(t, change.put) assert.Contains(t, change.delete, "foo/bar") assert.Contains(t, change.rmdir, "foo") } @@ -189,7 +189,7 @@ func TestPythonNotebookDiff(t *testing.T) { foo.Overwrite(t, "# Databricks notebook source\nprint(\"abc\")") change, err := state.diff(ctx, files) assert.NoError(t, err) - assert.Len(t, change.delete, 0) + assert.Empty(t, change.delete) assert.Len(t, change.put, 1) assert.Contains(t, change.put, "foo.py") assertKeysOfMap(t, state.LastModifiedTimes, []string{"foo.py"}) @@ -233,9 +233,9 @@ func TestPythonNotebookDiff(t *testing.T) { change, err = state.diff(ctx, files) assert.NoError(t, err) assert.Len(t, change.delete, 1) - assert.Len(t, change.put, 0) + assert.Empty(t, change.put) assert.Contains(t, change.delete, "foo") - assert.Len(t, state.LastModifiedTimes, 0) + assert.Empty(t, state.LastModifiedTimes) assert.Equal(t, map[string]string{}, state.LocalToRemoteNames) assert.Equal(t, map[string]string{}, state.RemoteToLocalNames) } @@ -264,7 +264,7 @@ func TestErrorWhenIdenticalRemoteName(t *testing.T) { assert.NoError(t, err) change, err := state.diff(ctx, files) assert.NoError(t, err) - assert.Len(t, change.delete, 0) + assert.Empty(t, change.delete) assert.Len(t, change.put, 2) assert.Contains(t, change.put, "foo.py") assert.Contains(t, change.put, "foo") @@ -300,7 +300,7 @@ func TestNoErrorRenameWithIdenticalRemoteName(t *testing.T) { assert.NoError(t, err) change, err := state.diff(ctx, files) assert.NoError(t, err) - assert.Len(t, change.delete, 0) + assert.Empty(t, change.delete) assert.Len(t, change.put, 1) assert.Contains(t, change.put, "foo.py") diff --git a/libs/sync/sync_test.go b/libs/sync/sync_test.go index 6168dc217..f30431770 100644 --- a/libs/sync/sync_test.go +++ b/libs/sync/sync_test.go @@ -59,7 +59,7 @@ func TestGetFileSet(t *testing.T) { fileList, err := s.GetFileList(ctx) require.NoError(t, err) - require.Equal(t, len(fileList), 10) + require.Len(t, fileList, 10) inc, err = fileset.NewGlobSet(root, []string{}) require.NoError(t, err) @@ -77,7 +77,7 @@ func TestGetFileSet(t *testing.T) { fileList, err = s.GetFileList(ctx) require.NoError(t, err) - require.Equal(t, len(fileList), 2) + require.Len(t, fileList, 2) inc, err = fileset.NewGlobSet(root, []string{"./.databricks/*.go"}) require.NoError(t, err) @@ -95,7 +95,7 @@ func TestGetFileSet(t *testing.T) { fileList, err = s.GetFileList(ctx) require.NoError(t, err) - require.Equal(t, len(fileList), 11) + require.Len(t, fileList, 11) } func TestRecursiveExclude(t *testing.T) { @@ -125,7 +125,7 @@ func TestRecursiveExclude(t *testing.T) { fileList, err := s.GetFileList(ctx) require.NoError(t, err) - require.Equal(t, len(fileList), 7) + require.Len(t, fileList, 7) } func TestNegateExclude(t *testing.T) { @@ -155,6 +155,6 @@ func TestNegateExclude(t *testing.T) { fileList, err := s.GetFileList(ctx) require.NoError(t, err) - require.Equal(t, len(fileList), 1) - require.Equal(t, fileList[0].Relative, "test/sub1/sub2/h.txt") + require.Len(t, fileList, 1) + require.Equal(t, "test/sub1/sub2/h.txt", fileList[0].Relative) } diff --git a/libs/template/config_test.go b/libs/template/config_test.go index a855019b6..515a0b9f5 100644 --- a/libs/template/config_test.go +++ b/libs/template/config_test.go @@ -24,7 +24,7 @@ func TestTemplateConfigAssignValuesFromFile(t *testing.T) { err = c.assignValuesFromFile(filepath.Join(testDir, "config.json")) if assert.NoError(t, err) { assert.Equal(t, int64(1), c.values["int_val"]) - assert.Equal(t, float64(2), c.values["float_val"]) + assert.InDelta(t, float64(2), c.values["float_val"].(float64), 0.0001) assert.Equal(t, true, c.values["bool_val"]) assert.Equal(t, "hello", c.values["string_val"]) } @@ -44,7 +44,7 @@ func TestTemplateConfigAssignValuesFromFileDoesNotOverwriteExistingConfigs(t *te err = c.assignValuesFromFile(filepath.Join(testDir, "config.json")) if assert.NoError(t, err) { assert.Equal(t, int64(1), c.values["int_val"]) - assert.Equal(t, float64(2), c.values["float_val"]) + assert.InDelta(t, float64(2), c.values["float_val"].(float64), 0.0001) assert.Equal(t, true, c.values["bool_val"]) assert.Equal(t, "this-is-not-overwritten", c.values["string_val"]) } @@ -89,7 +89,7 @@ func TestTemplateConfigAssignValuesFromDefaultValues(t *testing.T) { err = c.assignDefaultValues(r) if assert.NoError(t, err) { assert.Equal(t, int64(123), c.values["int_val"]) - assert.Equal(t, float64(123), c.values["float_val"]) + assert.InDelta(t, float64(123), c.values["float_val"].(float64), 0.0001) assert.Equal(t, true, c.values["bool_val"]) assert.Equal(t, "hello", c.values["string_val"]) } @@ -110,7 +110,7 @@ func TestTemplateConfigAssignValuesFromTemplatedDefaultValues(t *testing.T) { err = c.assignDefaultValues(r) if assert.NoError(t, err) { assert.Equal(t, int64(123), c.values["int_val"]) - assert.Equal(t, float64(123), c.values["float_val"]) + assert.InDelta(t, float64(123), c.values["float_val"].(float64), 0.0001) assert.Equal(t, true, c.values["bool_val"]) assert.Equal(t, "world", c.values["string_val"]) } diff --git a/libs/template/helpers_test.go b/libs/template/helpers_test.go index d98f40b24..f8bc1f3da 100644 --- a/libs/template/helpers_test.go +++ b/libs/template/helpers_test.go @@ -86,7 +86,7 @@ func TestTemplateRandIntFunction(t *testing.T) { assert.Len(t, r.files, 1) randInt, err := strconv.Atoi(strings.TrimSpace(string(r.files[0].(*inMemoryFile).content))) assert.Less(t, randInt, 10) - assert.Empty(t, err) + assert.NoError(t, err) } func TestTemplateUuidFunction(t *testing.T) { diff --git a/libs/template/renderer_test.go b/libs/template/renderer_test.go index eeb308732..2c14009ff 100644 --- a/libs/template/renderer_test.go +++ b/libs/template/renderer_test.go @@ -434,7 +434,7 @@ func TestRendererSkipAllFilesInCurrentDirectory(t *testing.T) { entries, err := os.ReadDir(tmpDir) require.NoError(t, err) // Assert none of the files are persisted to disk, because of {{skip "*"}} - assert.Len(t, entries, 0) + assert.Empty(t, entries) } func TestRendererSkipPatternsAreRelativeToFileDirectory(t *testing.T) { @@ -588,8 +588,8 @@ func TestRendererNonTemplatesAreCreatedAsCopyFiles(t *testing.T) { assert.NoError(t, err) assert.Len(t, r.files, 1) - assert.Equal(t, r.files[0].(*copyFile).srcPath, "not-a-template") - assert.Equal(t, r.files[0].RelPath(), "not-a-template") + assert.Equal(t, "not-a-template", r.files[0].(*copyFile).srcPath) + assert.Equal(t, "not-a-template", r.files[0].RelPath()) } func TestRendererFileTreeRendering(t *testing.T) { @@ -609,7 +609,7 @@ func TestRendererFileTreeRendering(t *testing.T) { // Assert in memory representation is created. assert.Len(t, r.files, 1) - assert.Equal(t, r.files[0].RelPath(), "my_directory/my_file") + assert.Equal(t, "my_directory/my_file", r.files[0].RelPath()) out, err := filer.NewLocalClient(tmpDir) require.NoError(t, err) diff --git a/libs/vfs/filer_test.go b/libs/vfs/filer_test.go index ee1397521..6987c288e 100644 --- a/libs/vfs/filer_test.go +++ b/libs/vfs/filer_test.go @@ -2,7 +2,6 @@ package vfs import ( "context" - "errors" "io/fs" "os" "path/filepath" @@ -42,7 +41,7 @@ func TestFilerPath(t *testing.T) { // Open non-existent file. _, err = p.Open("doesntexist_test.go") - assert.True(t, errors.Is(err, fs.ErrNotExist)) + assert.ErrorIs(t, err, fs.ErrNotExist) // Stat self. s, err = p.Stat("filer_test.go") @@ -52,7 +51,7 @@ func TestFilerPath(t *testing.T) { // Stat non-existent file. _, err = p.Stat("doesntexist_test.go") - assert.True(t, errors.Is(err, fs.ErrNotExist)) + assert.ErrorIs(t, err, fs.ErrNotExist) // ReadDir self. entries, err := p.ReadDir(".") @@ -61,7 +60,7 @@ func TestFilerPath(t *testing.T) { // ReadDir non-existent directory. _, err = p.ReadDir("doesntexist") - assert.True(t, errors.Is(err, fs.ErrNotExist)) + assert.ErrorIs(t, err, fs.ErrNotExist) // ReadFile self. buf, err = p.ReadFile("filer_test.go") @@ -70,7 +69,7 @@ func TestFilerPath(t *testing.T) { // ReadFile non-existent file. _, err = p.ReadFile("doesntexist_test.go") - assert.True(t, errors.Is(err, fs.ErrNotExist)) + assert.ErrorIs(t, err, fs.ErrNotExist) // Parent self. pp := p.Parent() From ef86d2bcaeff4d0e0f1445bf0e834299a63eadd7 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 2 Jan 2025 12:06:01 +0100 Subject: [PATCH 14/19] Speed up best case for "make test" 12x (#2060) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On main branch: ‘make test’ takes about 33s On this branch: ‘make test’ takes about 2.7s (all measurements are for hot cache) What’s done (from highest impact to lowest): - Remove -coverprofile= option - this option was disabling "go test"'s built-in cache and also it took extra time to calculate the coverage (extra 21s). - Exclude ./integration/ folder, there are no unit tests there, but having it included adds significant time. "go test"'s caching also does not work there for me, due to TestMain() presence (extra 7.2s). - Remove dependency on "make lint" - nice to have, but slow to re-check the whole repo and should already be done by IDE (extra 2.5s). - Remove dependency on "make vendor" — rarely needed; on CI it is already executed separately (extra 1.1s). The coverage option is still available under "make cover". Use "make showcover" to show it. I’ve also removed separate "make testonly". If you only want tests, run "make test". If you want lint+test run "make lint test" etc. I've also modified the test command, removed unnecessary -short, -v, --raw-command. --- .github/workflows/push.yml | 2 +- Makefile | 20 ++++++++++++-------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 25bd5f4d6..f7c4d5456 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -55,7 +55,7 @@ jobs: pip3 install wheel - name: Run tests - run: make testonly + run: make test golangci: name: lint diff --git a/Makefile b/Makefile index 7dca3b2cf..2e7e34299 100644 --- a/Makefile +++ b/Makefile @@ -1,20 +1,24 @@ default: build -lint: vendor +PACKAGES=./libs/... ./internal/... ./cmd/... ./bundle/... . + +lint: @echo "✓ Linting source code with https://golangci-lint.run/ (with --fix)..." @./lint.sh ./... -lintcheck: vendor +lintcheck: @echo "✓ Linting source code with https://golangci-lint.run/ ..." @golangci-lint run ./... -test: lint testonly - -testonly: +test: @echo "✓ Running tests ..." - @gotestsum --format pkgname-and-test-fails --no-summary=skipped --raw-command go test -v -json -short -coverprofile=coverage.txt ./... + @gotestsum --format pkgname-and-test-fails --no-summary=skipped -- ${PACKAGES} -coverage: test +cover: + @echo "✓ Running tests with coverage..." + @gotestsum --format pkgname-and-test-fails --no-summary=skipped -- -coverprofile=coverage.txt ${PACKAGES} + +showcover: @echo "✓ Opening coverage for unit tests ..." @go tool cover -html=coverage.txt @@ -42,4 +46,4 @@ integration: integration-short: $(INTEGRATION) -short -.PHONY: lint lintcheck test testonly coverage build snapshot vendor schema integration integration-short +.PHONY: lint lintcheck test cover showcover build snapshot vendor schema integration integration-short From ea8445af9eea3a6416c5c3705f664a5f8b66d085 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 2 Jan 2025 12:18:38 +0100 Subject: [PATCH 15/19] Make "make" output the commands it runs (#2066) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is useful on CI and locally for debugging and being able to copy-paste command to tweak the options. Removed redundant and imprecise messages like "✓ Running tests ...". --- Makefile | 27 +++++++++------------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/Makefile b/Makefile index 2e7e34299..b6a62765f 100644 --- a/Makefile +++ b/Makefile @@ -3,40 +3,31 @@ default: build PACKAGES=./libs/... ./internal/... ./cmd/... ./bundle/... . lint: - @echo "✓ Linting source code with https://golangci-lint.run/ (with --fix)..." - @./lint.sh ./... + ./lint.sh ./... lintcheck: - @echo "✓ Linting source code with https://golangci-lint.run/ ..." - @golangci-lint run ./... + golangci-lint run ./... test: - @echo "✓ Running tests ..." - @gotestsum --format pkgname-and-test-fails --no-summary=skipped -- ${PACKAGES} + gotestsum --format pkgname-and-test-fails --no-summary=skipped -- ${PACKAGES} cover: - @echo "✓ Running tests with coverage..." - @gotestsum --format pkgname-and-test-fails --no-summary=skipped -- -coverprofile=coverage.txt ${PACKAGES} + gotestsum --format pkgname-and-test-fails --no-summary=skipped -- -coverprofile=coverage.txt ${PACKAGES} showcover: - @echo "✓ Opening coverage for unit tests ..." - @go tool cover -html=coverage.txt + go tool cover -html=coverage.txt build: vendor - @echo "✓ Building source code with go build ..." - @go build -mod vendor + go build -mod vendor snapshot: - @echo "✓ Building dev snapshot" - @go build -o .databricks/databricks + go build -o .databricks/databricks vendor: - @echo "✓ Filling vendor folder with library code ..." - @go mod vendor + go mod vendor schema: - @echo "✓ Generating json-schema ..." - @go run ./bundle/internal/schema ./bundle/internal/schema ./bundle/schema/jsonschema.json + go run ./bundle/internal/schema ./bundle/internal/schema ./bundle/schema/jsonschema.json INTEGRATION = gotestsum --format github-actions --rerun-fails --jsonfile output.json --packages "./integration/..." -- -parallel 4 -timeout=2h From 890c57eabeb6252a98bbdfe7fe5a061a6284d48f Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Thu, 2 Jan 2025 16:52:33 +0530 Subject: [PATCH 16/19] Enable debugging integration tests in VS Code (#2053) ## Changes This PR adds back debugging functionality that was lost during migration to `internal.Main` as an entry point for integration tests. The PR that caused the regression: https://github.com/databricks/cli/pull/2009. Specifically the addition of internal.Main as the entrypoint for all integration tests. ## Tests Manually, by trying to debug a test. --- integration/internal/acc/debug.go | 4 ++-- integration/internal/main.go | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/integration/internal/acc/debug.go b/integration/internal/acc/debug.go index b4939881e..08e385b09 100644 --- a/integration/internal/acc/debug.go +++ b/integration/internal/acc/debug.go @@ -11,14 +11,14 @@ import ( ) // Detects if test is run from "debug test" feature in VS Code. -func isInDebug() bool { +func IsInDebug() bool { ex, _ := os.Executable() return strings.HasPrefix(path.Base(ex), "__debug_bin") } // Loads debug environment from ~/.databricks/debug-env.json. func loadDebugEnvIfRunFromIDE(t testutil.TestingT, key string) { - if !isInDebug() { + if !IsInDebug() { return } home, err := os.UserHomeDir() diff --git a/integration/internal/main.go b/integration/internal/main.go index 6d69dcf70..6aa2a4c93 100644 --- a/integration/internal/main.go +++ b/integration/internal/main.go @@ -4,6 +4,8 @@ import ( "fmt" "os" "testing" + + "github.com/databricks/cli/integration/internal/acc" ) // Main is the entry point for integration tests. @@ -11,7 +13,7 @@ import ( // they are not inadvertently executed when calling `go test ./...`. func Main(m *testing.M) { value := os.Getenv("CLOUD_ENV") - if value == "" { + if value == "" && !acc.IsInDebug() { fmt.Println("CLOUD_ENV is not set, skipping integration tests") return } From cae21693bbac8b74204a1f36b4c54ec66553de39 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 2 Jan 2025 12:23:48 +0100 Subject: [PATCH 17/19] lint: Raise max issues output (#2067) By default it stops after 3 issues of a given type, which gives false impression and also unhelpful if you fixing it with aider. 1000 is almost like unlimited but not unlimited in case there is a bug in a linter. --- .golangci.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.golangci.yaml b/.golangci.yaml index fd83e0882..3e9a88957 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -40,3 +40,5 @@ linters-settings: - require-error issues: exclude-dirs-use-default: false # recommended by docs https://golangci-lint.run/usage/false-positives/ + max-issues-per-linter: 1000 + max-same-issues: 1000 From 509f5aba6ad74765e97e87adf34694967da119f6 Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Thu, 2 Jan 2025 17:09:11 +0530 Subject: [PATCH 18/19] Snooze mlops-stacks integration test (#2063) ## Changes https://github.com/databricks/mlops-stacks/pull/187 broke mlops-stacks deployments for non-UC projects. Snoozing the test until upstream is fixed. ## Tests The test is skipped on my local machine. CI run will verify it's also skipped on Github Actions runners. --- integration/bundle/init_test.go | 2 ++ internal/testutil/helpers.go | 12 ++++++++++++ 2 files changed, 14 insertions(+) diff --git a/integration/bundle/init_test.go b/integration/bundle/init_test.go index f5c263ca3..bc81cd3a8 100644 --- a/integration/bundle/init_test.go +++ b/integration/bundle/init_test.go @@ -39,6 +39,8 @@ func TestBundleInitErrorOnUnknownFields(t *testing.T) { // make changes that can break the MLOps Stacks DAB. In which case we should // skip this test until the MLOps Stacks DAB is updated to work again. func TestBundleInitOnMlopsStacks(t *testing.T) { + testutil.SkipUntil(t, "2025-01-09") + ctx, wt := acc.WorkspaceTest(t) w := wt.W diff --git a/internal/testutil/helpers.go b/internal/testutil/helpers.go index 019a8e618..44c2c9375 100644 --- a/internal/testutil/helpers.go +++ b/internal/testutil/helpers.go @@ -5,6 +5,9 @@ import ( "math/rand" "os" "strings" + "time" + + "github.com/stretchr/testify/require" ) // GetEnvOrSkipTest proceeds with test only with that env variable. @@ -30,3 +33,12 @@ func RandomName(prefix ...string) string { } return string(b) } + +func SkipUntil(t TestingT, date string) { + deadline, err := time.Parse(time.DateOnly, date) + require.NoError(t, err) + + if time.Now().Before(deadline) { + t.Skipf("Skipping test until %s. Time right now: %s", deadline.Format(time.DateOnly), time.Now()) + } +} From 7beb0fb8b5bf6f5eb7f11df7b958f713356ca33d Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Thu, 2 Jan 2025 17:23:15 +0530 Subject: [PATCH 19/19] Add validation mutator for volume `artifact_path` (#2050) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Changes This PR: 1. Incrementally improves the error messages shown to the user when the volume they are referring to in `workspace.artifact_path` does not exist. 2. Performs this validation in both `bundle validate` and `bundle deploy` compared to before on just deployments. 3. It runs "fast" validations on `bundle deploy`, which earlier were only run on `bundle validate`. ## Tests Unit tests and manually. Also, existing integration tests provide coverage (`TestUploadArtifactToVolumeNotYetDeployed`, `TestUploadArtifactFileToVolumeThatDoesNotExist`) Examples: ``` .venv➜ bundle-playground git:(master) ✗ cli bundle validate Error: cannot access volume capital.whatever.my_volume: User does not have READ VOLUME on Volume 'capital.whatever.my_volume'. at workspace.artifact_path in databricks.yml:7:18 ``` and ``` .venv➜ bundle-playground git:(master) ✗ cli bundle validate Error: volume capital.whatever.foobar does not exist at workspace.artifact_path resources.volumes.foo in databricks.yml:7:18 databricks.yml:12:7 You are using a volume in your artifact_path that is managed by this bundle but which has not been deployed yet. Please first deploy the volume using 'bundle deploy' and then switch over to using it in the artifact_path. ``` --- bundle/config/validate/fast_validate.go | 51 ++++ bundle/config/validate/validate.go | 9 +- .../config/validate/validate_artifact_path.go | 129 +++++++++ .../validate/validate_artifact_path_test.go | 244 ++++++++++++++++ bundle/libraries/filer.go | 2 +- bundle/libraries/filer_test.go | 8 - bundle/libraries/filer_volume.go | 124 +-------- bundle/libraries/filer_volume_test.go | 262 +----------------- bundle/libraries/upload_test.go | 7 - cmd/bundle/deploy.go | 2 + integration/bundle/artifacts_test.go | 4 +- 11 files changed, 444 insertions(+), 398 deletions(-) create mode 100644 bundle/config/validate/fast_validate.go create mode 100644 bundle/config/validate/validate_artifact_path.go create mode 100644 bundle/config/validate/validate_artifact_path_test.go diff --git a/bundle/config/validate/fast_validate.go b/bundle/config/validate/fast_validate.go new file mode 100644 index 000000000..47d83036d --- /dev/null +++ b/bundle/config/validate/fast_validate.go @@ -0,0 +1,51 @@ +package validate + +import ( + "context" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" +) + +// FastValidate runs a subset of fast validation checks. This is a subset of the full +// suite of validation mutators that satisfy ANY ONE of the following criteria: +// +// 1. No file i/o or network requests are made in the mutator. +// 2. The validation is blocking for bundle deployments. +// +// The full suite of validation mutators is available in the [Validate] mutator. +type fastValidateReadonly struct{} + +func FastValidateReadonly() bundle.ReadOnlyMutator { + return &fastValidateReadonly{} +} + +func (f *fastValidateReadonly) Name() string { + return "fast_validate(readonly)" +} + +func (f *fastValidateReadonly) Apply(ctx context.Context, rb bundle.ReadOnlyBundle) diag.Diagnostics { + return bundle.ApplyReadOnly(ctx, rb, bundle.Parallel( + // Fast mutators with only in-memory checks + JobClusterKeyDefined(), + JobTaskClusterSpec(), + SingleNodeCluster(), + + // Blocking mutators. Deployments will fail if these checks fail. + ValidateArtifactPath(), + )) +} + +type fastValidate struct{} + +func FastValidate() bundle.Mutator { + return &fastValidate{} +} + +func (f *fastValidate) Name() string { + return "fast_validate" +} + +func (f *fastValidate) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + return bundle.ApplyReadOnly(ctx, bundle.ReadOnly(b), FastValidateReadonly()) +} diff --git a/bundle/config/validate/validate.go b/bundle/config/validate/validate.go index 131566fc9..8fdd704ab 100644 --- a/bundle/config/validate/validate.go +++ b/bundle/config/validate/validate.go @@ -30,12 +30,13 @@ func (l location) Path() dyn.Path { // Apply implements bundle.Mutator. func (v *validate) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { return bundle.ApplyReadOnly(ctx, bundle.ReadOnly(b), bundle.Parallel( - JobClusterKeyDefined(), + FastValidateReadonly(), + + // Slow mutators that require network or file i/o. These are only + // run in the `bundle validate` command. FilesToSync(), - ValidateSyncPatterns(), - JobTaskClusterSpec(), ValidateFolderPermissions(), - SingleNodeCluster(), + ValidateSyncPatterns(), )) } diff --git a/bundle/config/validate/validate_artifact_path.go b/bundle/config/validate/validate_artifact_path.go new file mode 100644 index 000000000..5bab99ccf --- /dev/null +++ b/bundle/config/validate/validate_artifact_path.go @@ -0,0 +1,129 @@ +package validate + +import ( + "context" + "errors" + "fmt" + "slices" + "strings" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/libraries" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/dynvar" + "github.com/databricks/databricks-sdk-go/apierr" +) + +type validateArtifactPath struct{} + +func ValidateArtifactPath() bundle.ReadOnlyMutator { + return &validateArtifactPath{} +} + +func (v *validateArtifactPath) Name() string { + return "validate:artifact_paths" +} + +func extractVolumeFromPath(artifactPath string) (string, string, string, error) { + if !libraries.IsVolumesPath(artifactPath) { + return "", "", "", fmt.Errorf("expected artifact_path to start with /Volumes/, got %s", artifactPath) + } + + parts := strings.Split(artifactPath, "/") + volumeFormatErr := fmt.Errorf("expected UC volume path to be in the format /Volumes////..., got %s", artifactPath) + + // Incorrect format. + if len(parts) < 5 { + return "", "", "", volumeFormatErr + } + + catalogName := parts[2] + schemaName := parts[3] + volumeName := parts[4] + + // Incorrect format. + if catalogName == "" || schemaName == "" || volumeName == "" { + return "", "", "", volumeFormatErr + } + + return catalogName, schemaName, volumeName, nil +} + +func findVolumeInBundle(r config.Root, catalogName, schemaName, volumeName string) (dyn.Path, []dyn.Location, bool) { + volumes := r.Resources.Volumes + for k, v := range volumes { + if v.CatalogName != catalogName || v.Name != volumeName { + continue + } + // UC schemas can be defined in the bundle itself, and thus might be interpolated + // at runtime via the ${resources.schemas.} syntax. Thus we match the volume + // definition if the schema name is the same as the one in the bundle, or if the + // schema name is interpolated. + // We only have to check for ${resources.schemas...} references because any + // other valid reference (like ${var.foo}) would have been interpolated by this point. + p, ok := dynvar.PureReferenceToPath(v.SchemaName) + isSchemaDefinedInBundle := ok && p.HasPrefix(dyn.Path{dyn.Key("resources"), dyn.Key("schemas")}) + if v.SchemaName != schemaName && !isSchemaDefinedInBundle { + continue + } + pathString := fmt.Sprintf("resources.volumes.%s", k) + return dyn.MustPathFromString(pathString), r.GetLocations(pathString), true + } + return nil, nil, false +} + +func (v *validateArtifactPath) Apply(ctx context.Context, rb bundle.ReadOnlyBundle) diag.Diagnostics { + // We only validate UC Volumes paths right now. + if !libraries.IsVolumesPath(rb.Config().Workspace.ArtifactPath) { + return nil + } + + wrapErrorMsg := func(s string) diag.Diagnostics { + return diag.Diagnostics{ + { + Summary: s, + Severity: diag.Error, + Locations: rb.Config().GetLocations("workspace.artifact_path"), + Paths: []dyn.Path{dyn.MustPathFromString("workspace.artifact_path")}, + }, + } + } + + catalogName, schemaName, volumeName, err := extractVolumeFromPath(rb.Config().Workspace.ArtifactPath) + if err != nil { + return wrapErrorMsg(err.Error()) + } + volumeFullName := fmt.Sprintf("%s.%s.%s", catalogName, schemaName, volumeName) + w := rb.WorkspaceClient() + _, err = w.Volumes.ReadByName(ctx, volumeFullName) + + if errors.Is(err, apierr.ErrPermissionDenied) { + return wrapErrorMsg(fmt.Sprintf("cannot access volume %s: %s", volumeFullName, err)) + } + if errors.Is(err, apierr.ErrNotFound) { + path, locations, ok := findVolumeInBundle(rb.Config(), catalogName, schemaName, volumeName) + if !ok { + return wrapErrorMsg(fmt.Sprintf("volume %s does not exist", volumeFullName)) + } + + // If the volume is defined in the bundle, provide a more helpful error diagnostic, + // with more details and location information. + return diag.Diagnostics{{ + Summary: fmt.Sprintf("volume %s does not exist", volumeFullName), + Severity: diag.Error, + Detail: `You are using a volume in your artifact_path that is managed by +this bundle but which has not been deployed yet. Please first deploy +the volume using 'bundle deploy' and then switch over to using it in +the artifact_path.`, + Locations: slices.Concat(rb.Config().GetLocations("workspace.artifact_path"), locations), + Paths: append([]dyn.Path{dyn.MustPathFromString("workspace.artifact_path")}, path), + }} + + } + if err != nil { + return wrapErrorMsg(fmt.Sprintf("cannot read volume %s: %s", volumeFullName, err)) + } + return nil +} diff --git a/bundle/config/validate/validate_artifact_path_test.go b/bundle/config/validate/validate_artifact_path_test.go new file mode 100644 index 000000000..8fb5c9618 --- /dev/null +++ b/bundle/config/validate/validate_artifact_path_test.go @@ -0,0 +1,244 @@ +package validate + +import ( + "context" + "fmt" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/internal/bundletest" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/experimental/mocks" + "github.com/databricks/databricks-sdk-go/service/catalog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +func TestValidateArtifactPathWithVolumeInBundle(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Workspace: config.Workspace{ + ArtifactPath: "/Volumes/catalogN/schemaN/volumeN/abc", + }, + Resources: config.Resources{ + Volumes: map[string]*resources.Volume{ + "foo": { + CreateVolumeRequestContent: &catalog.CreateVolumeRequestContent{ + CatalogName: "catalogN", + Name: "volumeN", + SchemaName: "schemaN", + }, + }, + }, + }, + }, + } + + bundletest.SetLocation(b, "workspace.artifact_path", []dyn.Location{{File: "file", Line: 1, Column: 1}}) + bundletest.SetLocation(b, "resources.volumes.foo", []dyn.Location{{File: "file", Line: 2, Column: 2}}) + + ctx := context.Background() + m := mocks.NewMockWorkspaceClient(t) + api := m.GetMockVolumesAPI() + api.EXPECT().ReadByName(mock.Anything, "catalogN.schemaN.volumeN").Return(nil, &apierr.APIError{ + StatusCode: 404, + }) + b.SetWorkpaceClient(m.WorkspaceClient) + + diags := bundle.ApplyReadOnly(ctx, bundle.ReadOnly(b), ValidateArtifactPath()) + assert.Equal(t, diag.Diagnostics{{ + Severity: diag.Error, + Summary: "volume catalogN.schemaN.volumeN does not exist", + Locations: []dyn.Location{ + {File: "file", Line: 1, Column: 1}, + {File: "file", Line: 2, Column: 2}, + }, + Paths: []dyn.Path{ + dyn.MustPathFromString("workspace.artifact_path"), + dyn.MustPathFromString("resources.volumes.foo"), + }, + Detail: `You are using a volume in your artifact_path that is managed by +this bundle but which has not been deployed yet. Please first deploy +the volume using 'bundle deploy' and then switch over to using it in +the artifact_path.`, + }}, diags) +} + +func TestValidateArtifactPath(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Workspace: config.Workspace{ + ArtifactPath: "/Volumes/catalogN/schemaN/volumeN/abc", + }, + }, + } + + bundletest.SetLocation(b, "workspace.artifact_path", []dyn.Location{{File: "file", Line: 1, Column: 1}}) + assertDiags := func(t *testing.T, diags diag.Diagnostics, expected string) { + assert.Len(t, diags, 1) + assert.Equal(t, diag.Diagnostics{{ + Severity: diag.Error, + Summary: expected, + Locations: []dyn.Location{{File: "file", Line: 1, Column: 1}}, + Paths: []dyn.Path{dyn.MustPathFromString("workspace.artifact_path")}, + }}, diags) + } + + rb := bundle.ReadOnly(b) + ctx := context.Background() + + tcases := []struct { + err error + expectedSummary string + }{ + { + err: &apierr.APIError{ + StatusCode: 403, + Message: "User does not have USE SCHEMA on Schema 'catalogN.schemaN'", + }, + expectedSummary: "cannot access volume catalogN.schemaN.volumeN: User does not have USE SCHEMA on Schema 'catalogN.schemaN'", + }, + { + err: &apierr.APIError{ + StatusCode: 404, + }, + expectedSummary: "volume catalogN.schemaN.volumeN does not exist", + }, + { + err: &apierr.APIError{ + StatusCode: 500, + Message: "Internal Server Error", + }, + expectedSummary: "cannot read volume catalogN.schemaN.volumeN: Internal Server Error", + }, + } + + for _, tc := range tcases { + m := mocks.NewMockWorkspaceClient(t) + api := m.GetMockVolumesAPI() + api.EXPECT().ReadByName(mock.Anything, "catalogN.schemaN.volumeN").Return(nil, tc.err) + b.SetWorkpaceClient(m.WorkspaceClient) + + diags := bundle.ApplyReadOnly(ctx, rb, ValidateArtifactPath()) + assertDiags(t, diags, tc.expectedSummary) + } +} + +func invalidVolumePaths() []string { + return []string{ + "/Volumes/", + "/Volumes/main", + "/Volumes/main/", + "/Volumes/main//", + "/Volumes/main//my_schema", + "/Volumes/main/my_schema", + "/Volumes/main/my_schema/", + "/Volumes/main/my_schema//", + "/Volumes//my_schema/my_volume", + } +} + +func TestExtractVolumeFromPath(t *testing.T) { + catalogName, schemaName, volumeName, err := extractVolumeFromPath("/Volumes/main/my_schema/my_volume") + require.NoError(t, err) + assert.Equal(t, "main", catalogName) + assert.Equal(t, "my_schema", schemaName) + assert.Equal(t, "my_volume", volumeName) + + for _, p := range invalidVolumePaths() { + _, _, _, err := extractVolumeFromPath(p) + assert.EqualError(t, err, fmt.Sprintf("expected UC volume path to be in the format /Volumes////..., got %s", p)) + } +} + +func TestValidateArtifactPathWithInvalidPaths(t *testing.T) { + for _, p := range invalidVolumePaths() { + b := &bundle.Bundle{ + Config: config.Root{ + Workspace: config.Workspace{ + ArtifactPath: p, + }, + }, + } + + bundletest.SetLocation(b, "workspace.artifact_path", []dyn.Location{{File: "config.yml", Line: 1, Column: 2}}) + + diags := bundle.ApplyReadOnly(context.Background(), bundle.ReadOnly(b), ValidateArtifactPath()) + require.Equal(t, diag.Diagnostics{{ + Severity: diag.Error, + Summary: fmt.Sprintf("expected UC volume path to be in the format /Volumes////..., got %s", p), + Locations: []dyn.Location{{File: "config.yml", Line: 1, Column: 2}}, + Paths: []dyn.Path{dyn.MustPathFromString("workspace.artifact_path")}, + }}, diags) + } +} + +func TestFindVolumeInBundle(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Volumes: map[string]*resources.Volume{ + "foo": { + CreateVolumeRequestContent: &catalog.CreateVolumeRequestContent{ + CatalogName: "main", + Name: "my_volume", + SchemaName: "my_schema", + }, + }, + }, + }, + }, + } + + bundletest.SetLocation(b, "resources.volumes.foo", []dyn.Location{ + { + File: "volume.yml", + Line: 1, + Column: 2, + }, + }) + + // volume is in DAB. + path, locations, ok := findVolumeInBundle(b.Config, "main", "my_schema", "my_volume") + assert.True(t, ok) + assert.Equal(t, []dyn.Location{{ + File: "volume.yml", + Line: 1, + Column: 2, + }}, locations) + assert.Equal(t, dyn.MustPathFromString("resources.volumes.foo"), path) + + // wrong volume name + _, _, ok = findVolumeInBundle(b.Config, "main", "my_schema", "doesnotexist") + assert.False(t, ok) + + // wrong schema name + _, _, ok = findVolumeInBundle(b.Config, "main", "doesnotexist", "my_volume") + assert.False(t, ok) + + // wrong catalog name + _, _, ok = findVolumeInBundle(b.Config, "doesnotexist", "my_schema", "my_volume") + assert.False(t, ok) + + // schema name is interpolated but does not have the right prefix. In this case + // we should not match the volume. + b.Config.Resources.Volumes["foo"].SchemaName = "${foo.bar.baz}" + _, _, ok = findVolumeInBundle(b.Config, "main", "my_schema", "my_volume") + assert.False(t, ok) + + // schema name is interpolated. + b.Config.Resources.Volumes["foo"].SchemaName = "${resources.schemas.my_schema.name}" + path, locations, ok = findVolumeInBundle(b.Config, "main", "valuedoesnotmatter", "my_volume") + assert.True(t, ok) + assert.Equal(t, []dyn.Location{{ + File: "volume.yml", + Line: 1, + Column: 2, + }}, locations) + assert.Equal(t, dyn.MustPathFromString("resources.volumes.foo"), path) +} diff --git a/bundle/libraries/filer.go b/bundle/libraries/filer.go index 4448ed325..e09c75e0e 100644 --- a/bundle/libraries/filer.go +++ b/bundle/libraries/filer.go @@ -24,7 +24,7 @@ func GetFilerForLibraries(ctx context.Context, b *bundle.Bundle) (filer.Filer, s switch { case IsVolumesPath(artifactPath): - return filerForVolume(ctx, b) + return filerForVolume(b) default: return filerForWorkspace(b) diff --git a/bundle/libraries/filer_test.go b/bundle/libraries/filer_test.go index 88ba152fc..c18da9726 100644 --- a/bundle/libraries/filer_test.go +++ b/bundle/libraries/filer_test.go @@ -7,10 +7,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/libs/filer" - sdkconfig "github.com/databricks/databricks-sdk-go/config" - "github.com/databricks/databricks-sdk-go/experimental/mocks" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) @@ -39,11 +36,6 @@ func TestGetFilerForLibrariesValidUcVolume(t *testing.T) { }, } - m := mocks.NewMockWorkspaceClient(t) - m.WorkspaceClient.Config = &sdkconfig.Config{} - m.GetMockFilesAPI().EXPECT().GetDirectoryMetadataByDirectoryPath(mock.Anything, "/Volumes/main/my_schema/my_volume").Return(nil) - b.SetWorkpaceClient(m.WorkspaceClient) - client, uploadPath, diags := GetFilerForLibraries(context.Background(), b) require.NoError(t, diags.Error()) assert.Equal(t, "/Volumes/main/my_schema/my_volume/.internal", uploadPath) diff --git a/bundle/libraries/filer_volume.go b/bundle/libraries/filer_volume.go index aecf68db1..176f475c6 100644 --- a/bundle/libraries/filer_volume.go +++ b/bundle/libraries/filer_volume.go @@ -1,132 +1,16 @@ package libraries import ( - "context" - "errors" - "fmt" "path" - "strings" "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynvar" "github.com/databricks/cli/libs/filer" - "github.com/databricks/databricks-sdk-go/apierr" ) -func extractVolumeFromPath(artifactPath string) (string, string, string, error) { - if !IsVolumesPath(artifactPath) { - return "", "", "", fmt.Errorf("expected artifact_path to start with /Volumes/, got %s", artifactPath) - } - - parts := strings.Split(artifactPath, "/") - volumeFormatErr := fmt.Errorf("expected UC volume path to be in the format /Volumes////..., got %s", artifactPath) - - // Incorrect format. - if len(parts) < 5 { - return "", "", "", volumeFormatErr - } - - catalogName := parts[2] - schemaName := parts[3] - volumeName := parts[4] - - // Incorrect format. - if catalogName == "" || schemaName == "" || volumeName == "" { - return "", "", "", volumeFormatErr - } - - return catalogName, schemaName, volumeName, nil -} - -// This function returns a filer for ".internal" folder inside the directory configured -// at `workspace.artifact_path`. -// This function also checks if the UC volume exists in the workspace and then: -// 1. If the UC volume exists in the workspace: -// Returns a filer for the UC volume. -// 2. If the UC volume does not exist in the workspace but is (with high confidence) defined in -// the bundle configuration: -// Returns an error and a warning that instructs the user to deploy the -// UC volume before using it in the artifact path. -// 3. If the UC volume does not exist in the workspace and is not defined in the bundle configuration: -// Returns an error. -func filerForVolume(ctx context.Context, b *bundle.Bundle) (filer.Filer, string, diag.Diagnostics) { - artifactPath := b.Config.Workspace.ArtifactPath +func filerForVolume(b *bundle.Bundle) (filer.Filer, string, diag.Diagnostics) { w := b.WorkspaceClient() - - catalogName, schemaName, volumeName, err := extractVolumeFromPath(artifactPath) - if err != nil { - return nil, "", diag.Diagnostics{ - { - Severity: diag.Error, - Summary: err.Error(), - Locations: b.Config.GetLocations("workspace.artifact_path"), - Paths: []dyn.Path{dyn.MustPathFromString("workspace.artifact_path")}, - }, - } - } - - // Check if the UC volume exists in the workspace. - volumePath := fmt.Sprintf("/Volumes/%s/%s/%s", catalogName, schemaName, volumeName) - err = w.Files.GetDirectoryMetadataByDirectoryPath(ctx, volumePath) - - // If the volume exists already, directly return the filer for the path to - // upload the artifacts to. - if err == nil { - uploadPath := path.Join(artifactPath, InternalDirName) - f, err := filer.NewFilesClient(w, uploadPath) - return f, uploadPath, diag.FromErr(err) - } - - baseErr := diag.Diagnostic{ - Severity: diag.Error, - Summary: fmt.Sprintf("unable to determine if volume at %s exists: %s", volumePath, err), - Locations: b.Config.GetLocations("workspace.artifact_path"), - Paths: []dyn.Path{dyn.MustPathFromString("workspace.artifact_path")}, - } - - if errors.Is(err, apierr.ErrNotFound) { - // Since the API returned a 404, the volume does not exist. - // Modify the error message to provide more context. - baseErr.Summary = fmt.Sprintf("volume %s does not exist: %s", volumePath, err) - - // If the volume is defined in the bundle, provide a more helpful error diagnostic, - // with more details and location information. - path, locations, ok := findVolumeInBundle(b, catalogName, schemaName, volumeName) - if !ok { - return nil, "", diag.Diagnostics{baseErr} - } - baseErr.Detail = `You are using a volume in your artifact_path that is managed by -this bundle but which has not been deployed yet. Please first deploy -the volume using 'bundle deploy' and then switch over to using it in -the artifact_path.` - baseErr.Paths = append(baseErr.Paths, path) - baseErr.Locations = append(baseErr.Locations, locations...) - } - - return nil, "", diag.Diagnostics{baseErr} -} - -func findVolumeInBundle(b *bundle.Bundle, catalogName, schemaName, volumeName string) (dyn.Path, []dyn.Location, bool) { - volumes := b.Config.Resources.Volumes - for k, v := range volumes { - if v.CatalogName != catalogName || v.Name != volumeName { - continue - } - // UC schemas can be defined in the bundle itself, and thus might be interpolated - // at runtime via the ${resources.schemas.} syntax. Thus we match the volume - // definition if the schema name is the same as the one in the bundle, or if the - // schema name is interpolated. - // We only have to check for ${resources.schemas...} references because any - // other valid reference (like ${var.foo}) would have been interpolated by this point. - p, ok := dynvar.PureReferenceToPath(v.SchemaName) - isSchemaDefinedInBundle := ok && p.HasPrefix(dyn.Path{dyn.Key("resources"), dyn.Key("schemas")}) - if v.SchemaName != schemaName && !isSchemaDefinedInBundle { - continue - } - pathString := fmt.Sprintf("resources.volumes.%s", k) - return dyn.MustPathFromString(pathString), b.Config.GetLocations(pathString), true - } - return nil, nil, false + uploadPath := path.Join(b.Config.Workspace.ArtifactPath, InternalDirName) + f, err := filer.NewFilesClient(w, uploadPath) + return f, uploadPath, diag.FromErr(err) } diff --git a/bundle/libraries/filer_volume_test.go b/bundle/libraries/filer_volume_test.go index 2af54b0cb..39bdc4135 100644 --- a/bundle/libraries/filer_volume_test.go +++ b/bundle/libraries/filer_volume_test.go @@ -1,277 +1,27 @@ package libraries import ( - "context" - "fmt" "path" "testing" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/bundle/internal/bundletest" - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/filer" - "github.com/databricks/databricks-sdk-go/apierr" - sdkconfig "github.com/databricks/databricks-sdk-go/config" - "github.com/databricks/databricks-sdk-go/experimental/mocks" - "github.com/databricks/databricks-sdk-go/service/catalog" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) -func TestFindVolumeInBundle(t *testing.T) { - b := &bundle.Bundle{ - Config: config.Root{ - Resources: config.Resources{ - Volumes: map[string]*resources.Volume{ - "foo": { - CreateVolumeRequestContent: &catalog.CreateVolumeRequestContent{ - CatalogName: "main", - Name: "my_volume", - SchemaName: "my_schema", - }, - }, - }, - }, - }, - } - - bundletest.SetLocation(b, "resources.volumes.foo", []dyn.Location{ - { - File: "volume.yml", - Line: 1, - Column: 2, - }, - }) - - // volume is in DAB. - path, locations, ok := findVolumeInBundle(b, "main", "my_schema", "my_volume") - assert.True(t, ok) - assert.Equal(t, []dyn.Location{{ - File: "volume.yml", - Line: 1, - Column: 2, - }}, locations) - assert.Equal(t, dyn.MustPathFromString("resources.volumes.foo"), path) - - // wrong volume name - _, _, ok = findVolumeInBundle(b, "main", "my_schema", "doesnotexist") - assert.False(t, ok) - - // wrong schema name - _, _, ok = findVolumeInBundle(b, "main", "doesnotexist", "my_volume") - assert.False(t, ok) - - // wrong catalog name - _, _, ok = findVolumeInBundle(b, "doesnotexist", "my_schema", "my_volume") - assert.False(t, ok) - - // schema name is interpolated but does not have the right prefix. In this case - // we should not match the volume. - b.Config.Resources.Volumes["foo"].SchemaName = "${foo.bar.baz}" - _, _, ok = findVolumeInBundle(b, "main", "my_schema", "my_volume") - assert.False(t, ok) - - // schema name is interpolated. - b.Config.Resources.Volumes["foo"].SchemaName = "${resources.schemas.my_schema.name}" - path, locations, ok = findVolumeInBundle(b, "main", "valuedoesnotmatter", "my_volume") - assert.True(t, ok) - assert.Equal(t, []dyn.Location{{ - File: "volume.yml", - Line: 1, - Column: 2, - }}, locations) - assert.Equal(t, dyn.MustPathFromString("resources.volumes.foo"), path) -} - -func TestFilerForVolumeForErrorFromAPI(t *testing.T) { +func TestFilerForVolume(t *testing.T) { b := &bundle.Bundle{ Config: config.Root{ Workspace: config.Workspace{ - ArtifactPath: "/Volumes/main/my_schema/my_volume", + ArtifactPath: "/Volumes/main/my_schema/my_volume/abc", }, }, } - bundletest.SetLocation(b, "workspace.artifact_path", []dyn.Location{{File: "config.yml", Line: 1, Column: 2}}) - - m := mocks.NewMockWorkspaceClient(t) - m.WorkspaceClient.Config = &sdkconfig.Config{} - m.GetMockFilesAPI().EXPECT().GetDirectoryMetadataByDirectoryPath(mock.Anything, "/Volumes/main/my_schema/my_volume").Return(fmt.Errorf("error from API")) - b.SetWorkpaceClient(m.WorkspaceClient) - - _, _, diags := filerForVolume(context.Background(), b) - assert.Equal(t, diag.Diagnostics{ - { - Severity: diag.Error, - Summary: "unable to determine if volume at /Volumes/main/my_schema/my_volume exists: error from API", - Locations: []dyn.Location{{File: "config.yml", Line: 1, Column: 2}}, - Paths: []dyn.Path{dyn.MustPathFromString("workspace.artifact_path")}, - }, - }, diags) -} - -func TestFilerForVolumeWithVolumeNotFound(t *testing.T) { - b := &bundle.Bundle{ - Config: config.Root{ - Workspace: config.Workspace{ - ArtifactPath: "/Volumes/main/my_schema/doesnotexist", - }, - }, - } - - bundletest.SetLocation(b, "workspace.artifact_path", []dyn.Location{{File: "config.yml", Line: 1, Column: 2}}) - - m := mocks.NewMockWorkspaceClient(t) - m.WorkspaceClient.Config = &sdkconfig.Config{} - m.GetMockFilesAPI().EXPECT().GetDirectoryMetadataByDirectoryPath(mock.Anything, "/Volumes/main/my_schema/doesnotexist").Return(apierr.NotFound("some error message")) - b.SetWorkpaceClient(m.WorkspaceClient) - - _, _, diags := filerForVolume(context.Background(), b) - assert.Equal(t, diag.Diagnostics{ - { - Severity: diag.Error, - Summary: "volume /Volumes/main/my_schema/doesnotexist does not exist: some error message", - Locations: []dyn.Location{{File: "config.yml", Line: 1, Column: 2}}, - Paths: []dyn.Path{dyn.MustPathFromString("workspace.artifact_path")}, - }, - }, diags) -} - -func TestFilerForVolumeNotFoundAndInBundle(t *testing.T) { - b := &bundle.Bundle{ - Config: config.Root{ - Workspace: config.Workspace{ - ArtifactPath: "/Volumes/main/my_schema/my_volume", - }, - Resources: config.Resources{ - Volumes: map[string]*resources.Volume{ - "foo": { - CreateVolumeRequestContent: &catalog.CreateVolumeRequestContent{ - CatalogName: "main", - Name: "my_volume", - VolumeType: "MANAGED", - SchemaName: "my_schema", - }, - }, - }, - }, - }, - } - - bundletest.SetLocation(b, "workspace.artifact_path", []dyn.Location{{File: "config.yml", Line: 1, Column: 2}}) - bundletest.SetLocation(b, "resources.volumes.foo", []dyn.Location{{File: "volume.yml", Line: 1, Column: 2}}) - - m := mocks.NewMockWorkspaceClient(t) - m.WorkspaceClient.Config = &sdkconfig.Config{} - m.GetMockFilesAPI().EXPECT().GetDirectoryMetadataByDirectoryPath(mock.Anything, "/Volumes/main/my_schema/my_volume").Return(apierr.NotFound("error from API")) - b.SetWorkpaceClient(m.WorkspaceClient) - - _, _, diags := GetFilerForLibraries(context.Background(), b) - assert.Equal(t, diag.Diagnostics{ - { - Severity: diag.Error, - Summary: "volume /Volumes/main/my_schema/my_volume does not exist: error from API", - Locations: []dyn.Location{{File: "config.yml", Line: 1, Column: 2}, {File: "volume.yml", Line: 1, Column: 2}}, - Paths: []dyn.Path{dyn.MustPathFromString("workspace.artifact_path"), dyn.MustPathFromString("resources.volumes.foo")}, - Detail: `You are using a volume in your artifact_path that is managed by -this bundle but which has not been deployed yet. Please first deploy -the volume using 'bundle deploy' and then switch over to using it in -the artifact_path.`, - }, - }, diags) -} - -func invalidVolumePaths() []string { - return []string{ - "/Volumes/", - "/Volumes/main", - "/Volumes/main/", - "/Volumes/main//", - "/Volumes/main//my_schema", - "/Volumes/main/my_schema", - "/Volumes/main/my_schema/", - "/Volumes/main/my_schema//", - "/Volumes//my_schema/my_volume", - } -} - -func TestFilerForVolumeWithInvalidVolumePaths(t *testing.T) { - for _, p := range invalidVolumePaths() { - b := &bundle.Bundle{ - Config: config.Root{ - Workspace: config.Workspace{ - ArtifactPath: p, - }, - }, - } - - bundletest.SetLocation(b, "workspace.artifact_path", []dyn.Location{{File: "config.yml", Line: 1, Column: 2}}) - - _, _, diags := GetFilerForLibraries(context.Background(), b) - require.Equal(t, diag.Diagnostics{{ - Severity: diag.Error, - Summary: fmt.Sprintf("expected UC volume path to be in the format /Volumes////..., got %s", p), - Locations: []dyn.Location{{File: "config.yml", Line: 1, Column: 2}}, - Paths: []dyn.Path{dyn.MustPathFromString("workspace.artifact_path")}, - }}, diags) - } -} - -func TestFilerForVolumeWithInvalidPrefix(t *testing.T) { - b := &bundle.Bundle{ - Config: config.Root{ - Workspace: config.Workspace{ - ArtifactPath: "/Volume/main/my_schema/my_volume", - }, - }, - } - - _, _, diags := filerForVolume(context.Background(), b) - require.EqualError(t, diags.Error(), "expected artifact_path to start with /Volumes/, got /Volume/main/my_schema/my_volume") -} - -func TestFilerForVolumeWithValidVolumePaths(t *testing.T) { - validPaths := []string{ - "/Volumes/main/my_schema/my_volume", - "/Volumes/main/my_schema/my_volume/", - "/Volumes/main/my_schema/my_volume/a/b/c", - "/Volumes/main/my_schema/my_volume/a/a/a", - } - - for _, p := range validPaths { - b := &bundle.Bundle{ - Config: config.Root{ - Workspace: config.Workspace{ - ArtifactPath: p, - }, - }, - } - - m := mocks.NewMockWorkspaceClient(t) - m.WorkspaceClient.Config = &sdkconfig.Config{} - m.GetMockFilesAPI().EXPECT().GetDirectoryMetadataByDirectoryPath(mock.Anything, "/Volumes/main/my_schema/my_volume").Return(nil) - b.SetWorkpaceClient(m.WorkspaceClient) - - client, uploadPath, diags := filerForVolume(context.Background(), b) - require.NoError(t, diags.Error()) - assert.Equal(t, path.Join(p, ".internal"), uploadPath) - assert.IsType(t, &filer.FilesClient{}, client) - } -} - -func TestExtractVolumeFromPath(t *testing.T) { - catalogName, schemaName, volumeName, err := extractVolumeFromPath("/Volumes/main/my_schema/my_volume") - require.NoError(t, err) - assert.Equal(t, "main", catalogName) - assert.Equal(t, "my_schema", schemaName) - assert.Equal(t, "my_volume", volumeName) - - for _, p := range invalidVolumePaths() { - _, _, _, err := extractVolumeFromPath(p) - assert.EqualError(t, err, fmt.Sprintf("expected UC volume path to be in the format /Volumes////..., got %s", p)) - } + client, uploadPath, diags := filerForVolume(b) + require.NoError(t, diags.Error()) + assert.Equal(t, path.Join("/Volumes/main/my_schema/my_volume/abc/.internal"), uploadPath) + assert.IsType(t, &filer.FilesClient{}, client) } diff --git a/bundle/libraries/upload_test.go b/bundle/libraries/upload_test.go index 493785bf5..44b194c56 100644 --- a/bundle/libraries/upload_test.go +++ b/bundle/libraries/upload_test.go @@ -11,8 +11,6 @@ import ( mockfiler "github.com/databricks/cli/internal/mocks/libs/filer" "github.com/databricks/cli/internal/testutil" "github.com/databricks/cli/libs/filer" - sdkconfig "github.com/databricks/databricks-sdk-go/config" - "github.com/databricks/databricks-sdk-go/experimental/mocks" "github.com/databricks/databricks-sdk-go/service/compute" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/stretchr/testify/mock" @@ -183,11 +181,6 @@ func TestArtifactUploadForVolumes(t *testing.T) { filer.CreateParentDirectories, ).Return(nil) - m := mocks.NewMockWorkspaceClient(t) - m.WorkspaceClient.Config = &sdkconfig.Config{} - m.GetMockFilesAPI().EXPECT().GetDirectoryMetadataByDirectoryPath(mock.Anything, "/Volumes/foo/bar/artifacts").Return(nil) - b.SetWorkpaceClient(m.WorkspaceClient) - diags := bundle.Apply(context.Background(), b, bundle.Seq(ExpandGlobReferences(), UploadWithClient(mockFiler))) require.NoError(t, diags.Error()) diff --git a/cmd/bundle/deploy.go b/cmd/bundle/deploy.go index a25e02f6c..560b07e39 100644 --- a/cmd/bundle/deploy.go +++ b/cmd/bundle/deploy.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config/validate" "github.com/databricks/cli/bundle/phases" "github.com/databricks/cli/bundle/render" "github.com/databricks/cli/cmd/bundle/utils" @@ -71,6 +72,7 @@ func newDeployCommand() *cobra.Command { diags = diags.Extend( bundle.Apply(ctx, b, bundle.Seq( phases.Initialize(), + validate.FastValidate(), phases.Build(), phases.Deploy(outputHandler), )), diff --git a/integration/bundle/artifacts_test.go b/integration/bundle/artifacts_test.go index 9dafe2f83..94b96899e 100644 --- a/integration/bundle/artifacts_test.go +++ b/integration/bundle/artifacts_test.go @@ -257,7 +257,7 @@ func TestUploadArtifactFileToVolumeThatDoesNotExist(t *testing.T) { stdout, stderr, err := testcli.RequireErrorRun(t, ctx, "bundle", "deploy") assert.Error(t, err) - assert.Equal(t, fmt.Sprintf(`Error: volume /Volumes/main/%s/doesnotexist does not exist: Not Found + assert.Equal(t, fmt.Sprintf(`Error: volume main.%s.doesnotexist does not exist at workspace.artifact_path in databricks.yml:6:18 @@ -293,7 +293,7 @@ func TestUploadArtifactToVolumeNotYetDeployed(t *testing.T) { stdout, stderr, err := testcli.RequireErrorRun(t, ctx, "bundle", "deploy") assert.Error(t, err) - assert.Equal(t, fmt.Sprintf(`Error: volume /Volumes/main/%s/my_volume does not exist: Not Found + assert.Equal(t, fmt.Sprintf(`Error: volume main.%s.my_volume does not exist at workspace.artifact_path resources.volumes.foo in databricks.yml:6:18