bundle/deploy/terraform/init.go

@@ -111,13 +111,6 @@ func inheritEnvVars(ctx context.Context, environ map[string]string) error {
 		environ["PATH"] = path
 	}
 
-	// Include $AZURE_CONFIG_FILE in set of environment variables to pass along.
-	// This is set in Azure DevOps by the AzureCLI@2 task.
-	azureConfigFile, ok := env.Lookup(ctx, "AZURE_CONFIG_FILE")
-	if ok {
-		environ["AZURE_CONFIG_FILE"] = azureConfigFile
-	}
-
 	// Include $TF_CLI_CONFIG_FILE to override terraform provider in development.
 	// See: https://developer.hashicorp.com/terraform/cli/config/config-file#explicit-installation-method-configuration
 	devConfigFile, ok := env.Lookup(ctx, "TF_CLI_CONFIG_FILE")

bundle/deploy/terraform/init_test.go

@@ -269,20 +269,19 @@ func TestSetUserAgentExtraEnvVar(t *testing.T) {
 }
 
 func TestInheritEnvVars(t *testing.T) {
+	env := map[string]string{}
+
 	t.Setenv("HOME", "/home/testuser")
 	t.Setenv("PATH", "/foo:/bar")
 	t.Setenv("TF_CLI_CONFIG_FILE", "/tmp/config.tfrc")
-	t.Setenv("AZURE_CONFIG_FILE", "/tmp/foo/bar")
 
-	ctx := context.Background()
+	err := inheritEnvVars(context.Background(), env)
-	env := map[string]string{}
+
-	err := inheritEnvVars(ctx, env)
+	require.NoError(t, err)
-	if assert.NoError(t, err) {
+
-		assert.Equal(t, "/home/testuser", env["HOME"])
+	require.Equal(t, env["HOME"], "/home/testuser")
-		assert.Equal(t, "/foo:/bar", env["PATH"])
+	require.Equal(t, env["PATH"], "/foo:/bar")
-		assert.Equal(t, "/tmp/config.tfrc", env["TF_CLI_CONFIG_FILE"])
+	require.Equal(t, env["TF_CLI_CONFIG_FILE"], "/tmp/config.tfrc")
-		assert.Equal(t, "/tmp/foo/bar", env["AZURE_CONFIG_FILE"])
-	}
 }
 
 func TestSetUserProfileFromInheritEnvVars(t *testing.T) {

bundle/python/warning.go

@@ -2,7 +2,6 @@ package python
 
 import (
 	"context"
-	"strconv"
 	"strings"
 
 	"github.com/databricks/cli/bundle"
@@ -39,7 +38,7 @@ func hasIncompatibleWheelTasks(ctx context.Context, b *bundle.Bundle) bool {
 	tasks := libraries.FindTasksWithLocalLibraries(b)
 	for _, task := range tasks {
 		if task.NewCluster != nil {
-			if lowerThanExpectedVersion(task.NewCluster.SparkVersion) {
+			if lowerThanExpectedVersion(ctx, task.NewCluster.SparkVersion) {
 				return true
 			}
 		}
@@ -48,7 +47,7 @@ func hasIncompatibleWheelTasks(ctx context.Context, b *bundle.Bundle) bool {
 			for _, job := range b.Config.Resources.Jobs {
 				for _, cluster := range job.JobClusters {
 					if task.JobClusterKey == cluster.JobClusterKey && cluster.NewCluster.SparkVersion != "" {
-						if lowerThanExpectedVersion(cluster.NewCluster.SparkVersion) {
+						if lowerThanExpectedVersion(ctx, cluster.NewCluster.SparkVersion) {
 							return true
 						}
 					}
@@ -65,7 +64,7 @@ func hasIncompatibleWheelTasks(ctx context.Context, b *bundle.Bundle) bool {
 				return false
 			}
 
-			if lowerThanExpectedVersion(version) {
+			if lowerThanExpectedVersion(ctx, version) {
 				return true
 			}
 		}
@@ -74,7 +73,7 @@ func hasIncompatibleWheelTasks(ctx context.Context, b *bundle.Bundle) bool {
 	return false
 }
 
-func lowerThanExpectedVersion(sparkVersion string) bool {
+func lowerThanExpectedVersion(ctx context.Context, sparkVersion string) bool {
 	parts := strings.Split(sparkVersion, ".")
 	if len(parts) < 2 {
 		return false
@@ -83,17 +82,6 @@ func lowerThanExpectedVersion(sparkVersion string) bool {
 	if parts[1][0] == 'x' { // treat versions like 13.x as the very latest minor (13.99)
 		parts[1] = "99"
 	}
-
-	// if any of the version parts are not numbers, we can't compare
-	// so consider it as compatible version
-	if _, err := strconv.Atoi(parts[0]); err != nil {
-		return false
-	}
-
-	if _, err := strconv.Atoi(parts[1]); err != nil {
-		return false
-	}
-
 	v := "v" + parts[0] + "." + parts[1]
 	return semver.Compare(v, "v13.1") < 0
 }

bundle/python/warning_test.go

@@ -344,8 +344,6 @@ func TestSparkVersionLowerThanExpected(t *testing.T) {
 		"14.1.x-scala2.12":                false,
 		"13.x-snapshot-scala-2.12":        false,
 		"13.x-rc-scala-2.12":              false,
-		"client.1.10-scala2.12":           false,
-		"latest-stable-gpu-scala2.11":     false,
 		"10.4.x-aarch64-photon-scala2.12": true,
 		"10.4.x-scala2.12":                true,
 		"13.0.x-scala2.12":                true,
@@ -353,7 +351,7 @@ func TestSparkVersionLowerThanExpected(t *testing.T) {
 	}
 
 	for k, v := range testCases {
-		result := lowerThanExpectedVersion(k)
+		result := lowerThanExpectedVersion(context.Background(), k)
 		require.Equal(t, v, result, k)
 	}
 }

libs/dyn/convert/normalize.go

@@ -267,8 +267,6 @@ func (n normalizeOptions) normalizeString(typ reflect.Type, src dyn.Value, path
 		out = strconv.FormatInt(src.MustInt(), 10)
 	case dyn.KindFloat:
 		out = strconv.FormatFloat(src.MustFloat(), 'f', -1, 64)
-	case dyn.KindTime:
-		out = src.MustTime().String()
 	case dyn.KindNil:
 		// Return a warning if the field is present but has a null value.
 		return dyn.InvalidValue, diags.Append(nullWarning(dyn.KindString, src, path))

libs/dyn/convert/normalize_test.go

@@ -569,14 +569,6 @@ func TestNormalizeStringFromFloat(t *testing.T) {
 	assert.Equal(t, dyn.NewValue("1.2", vin.Locations()), vout)
 }
 
-func TestNormalizeStringFromTime(t *testing.T) {
-	var typ string
-	vin := dyn.NewValue(dyn.MustTime("2024-08-29"), []dyn.Location{{File: "file", Line: 1, Column: 1}})
-	vout, err := Normalize(&typ, vin)
-	assert.Empty(t, err)
-	assert.Equal(t, dyn.NewValue("2024-08-29", vin.Locations()), vout)
-}
-
 func TestNormalizeStringError(t *testing.T) {
 	var typ string
 	vin := dyn.V(map[string]dyn.Value{"an": dyn.V("error")})

libs/dyn/kind.go

@@ -2,6 +2,7 @@ package dyn
 
 import (
 	"fmt"
+	"time"
 )
 
 type Kind int
@@ -33,7 +34,7 @@ func kindOf(v any) Kind {
 		return KindInt
 	case float32, float64:
 		return KindFloat
-	case Time:
+	case time.Time:
 		return KindTime
 	case nil:
 		return KindNil

libs/dyn/merge/override_test.go

@@ -83,16 +83,16 @@ func TestOverride_Primitive(t *testing.T) {
 		{
 			name:     "time (updated)",
 			state:    visitorState{updated: []string{"root"}},
-			left:     dyn.NewValue(dyn.FromTime(time.UnixMilli(10000)), []dyn.Location{leftLocation}),
+			left:     dyn.NewValue(time.UnixMilli(10000), []dyn.Location{leftLocation}),
-			right:    dyn.NewValue(dyn.FromTime(time.UnixMilli(10001)), []dyn.Location{rightLocation}),
+			right:    dyn.NewValue(time.UnixMilli(10001), []dyn.Location{rightLocation}),
-			expected: dyn.NewValue(dyn.FromTime(time.UnixMilli(10001)), []dyn.Location{rightLocation}),
+			expected: dyn.NewValue(time.UnixMilli(10001), []dyn.Location{rightLocation}),
 		},
 		{
 			name:     "time (not updated)",
 			state:    visitorState{},
-			left:     dyn.NewValue(dyn.FromTime(time.UnixMilli(10000)), []dyn.Location{leftLocation}),
+			left:     dyn.NewValue(time.UnixMilli(10000), []dyn.Location{leftLocation}),
-			right:    dyn.NewValue(dyn.FromTime(time.UnixMilli(10000)), []dyn.Location{rightLocation}),
+			right:    dyn.NewValue(time.UnixMilli(10000), []dyn.Location{rightLocation}),
-			expected: dyn.NewValue(dyn.FromTime(time.UnixMilli(10000)), []dyn.Location{leftLocation}),
+			expected: dyn.NewValue(time.UnixMilli(10000), []dyn.Location{leftLocation}),
 		},
 		{
 			name:     "different types (updated)",

libs/dyn/time.go

@@ -1,62 +0,0 @@
-package dyn
-
-import (
-	"fmt"
-	"time"
-)
-
-// Time represents a time-like primitive value.
-//
-// It represents a timestamp and includes the original string value
-// that was parsed to create the timestamp. This makes it possible
-// to coalesce a value that YAML interprets as a timestamp back into
-// a string without losing information.
-type Time struct {
-	t time.Time
-	s string
-}
-
-// NewTime creates a new Time from the given string.
-func NewTime(str string) (Time, error) {
-	// Try a couple of layouts
-	for _, layout := range []string{
-		"2006-1-2T15:4:5.999999999Z07:00", // RCF3339Nano with short date fields.
-		"2006-1-2t15:4:5.999999999Z07:00", // RFC3339Nano with short date fields and lower-case "t".
-		"2006-1-2 15:4:5.999999999",       // space separated with no time zone
-		"2006-1-2",                        // date only
-	} {
-		t, terr := time.Parse(layout, str)
-		if terr == nil {
-			return Time{t: t, s: str}, nil
-		}
-	}
-
-	return Time{}, fmt.Errorf("invalid time value: %q", str)
-}
-
-// MustTime creates a new Time from the given string.
-// It panics if the string cannot be parsed.
-func MustTime(str string) Time {
-	t, err := NewTime(str)
-	if err != nil {
-		panic(err)
-	}
-	return t
-}
-
-// FromTime creates a new Time from the given time.Time.
-// It uses the RFC3339Nano format for its string representation.
-// This guarantees that it can roundtrip into a string without losing information.
-func FromTime(t time.Time) Time {
-	return Time{t: t, s: t.Format(time.RFC3339Nano)}
-}
-
-// Time returns the time.Time value.
-func (t Time) Time() time.Time {
-	return t.t
-}
-
-// String returns the original string value that was parsed to create the timestamp.
-func (t Time) String() string {
-	return t.s
-}

libs/dyn/time_test.go

@@ -1,41 +0,0 @@
-package dyn_test
-
-import (
-	"testing"
-	"time"
-
-	"github.com/databricks/cli/libs/dyn"
-	assert "github.com/databricks/cli/libs/dyn/dynassert"
-)
-
-func TestTimeValid(t *testing.T) {
-	for _, tc := range []string{
-		"2024-08-29",
-		"2024-01-15T12:34:56.789012345Z",
-	} {
-		tm, err := dyn.NewTime(tc)
-		if assert.NoError(t, err) {
-			assert.NotEqual(t, time.Time{}, tm.Time())
-			assert.Equal(t, tc, tm.String())
-		}
-	}
-}
-
-func TestTimeInvalid(t *testing.T) {
-	tm, err := dyn.NewTime("invalid")
-	assert.Error(t, err)
-	assert.Equal(t, dyn.Time{}, tm)
-}
-
-func TestTimeFromTime(t *testing.T) {
-	tref := time.Now()
-	t1 := dyn.FromTime(tref)
-
-	// Verify that the underlying value is the same.
-	assert.Equal(t, tref, t1.Time())
-
-	// Verify that the string representation can be used to construct the same.
-	t2, err := dyn.NewTime(t1.String())
-	assert.NoError(t, err)
-	assert.True(t, t1.Time().Equal(t2.Time()))
-}

libs/dyn/value.go

@@ -127,8 +127,7 @@ func (v Value) AsAny() any {
 	case KindFloat:
 		return v.v
 	case KindTime:
-		t := v.v.(Time)
+		return v.v
-		return t.Time()
 	default:
 		// Panic because we only want to deal with known types.
 		panic(fmt.Sprintf("invalid kind: %d", v.k))

libs/dyn/value_underlying.go

@@ -2,6 +2,7 @@ package dyn
 
 import (
 	"fmt"
+	"time"
 )
 
 // AsMap returns the underlying mapping if this value is a map,
@@ -122,14 +123,14 @@ func (v Value) MustFloat() float64 {
 
 // AsTime returns the underlying time if this value is a time,
 // the zero value and false otherwise.
-func (v Value) AsTime() (Time, bool) {
+func (v Value) AsTime() (time.Time, bool) {
-	vv, ok := v.v.(Time)
+	vv, ok := v.v.(time.Time)
 	return vv, ok
 }
 
 // MustTime returns the underlying time if this value is a time,
 // panics otherwise.
-func (v Value) MustTime() Time {
+func (v Value) MustTime() time.Time {
 	vv, ok := v.AsTime()
 	if !ok || v.k != KindTime {
 		panic(fmt.Sprintf("expected kind %s, got %s", KindTime, v.k))

libs/dyn/value_underlying_test.go

@@ -143,7 +143,7 @@ func TestValueUnderlyingFloat(t *testing.T) {
 }
 
 func TestValueUnderlyingTime(t *testing.T) {
-	v := dyn.V(dyn.FromTime(time.Now()))
+	v := dyn.V(time.Now())
 
 	vv1, ok := v.AsTime()
 	assert.True(t, ok)

libs/dyn/yamlloader/loader.go

@@ -5,6 +5,7 @@ import (
 	"math"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/databricks/cli/libs/dyn"
 	"gopkg.in/yaml.v3"
@@ -206,10 +207,18 @@ func (d *loader) loadScalar(node *yaml.Node, loc dyn.Location) (dyn.Value, error
 	case "!!null":
 		return dyn.NewValue(nil, []dyn.Location{loc}), nil
 	case "!!timestamp":
-		t, err := dyn.NewTime(node.Value)
+		// Try a couple of layouts
-		if err == nil {
+		for _, layout := range []string{
+			"2006-1-2T15:4:5.999999999Z07:00", // RCF3339Nano with short date fields.
+			"2006-1-2t15:4:5.999999999Z07:00", // RFC3339Nano with short date fields and lower-case "t".
+			"2006-1-2 15:4:5.999999999",       // space separated with no time zone
+			"2006-1-2",                        // date only
+		} {
+			t, terr := time.Parse(layout, node.Value)
+			if terr == nil {
 				return dyn.NewValue(t, []dyn.Location{loc}), nil
 			}
+		}
 		return dyn.InvalidValue, errorf(loc, "invalid timestamp value: %v", node.Value)
 	default:
 		return dyn.InvalidValue, errorf(loc, "unknown tag: %v", st)

libs/dyn/yamlsaver/saver.go

@@ -129,7 +129,7 @@ func (s *saver) toYamlNodeWithStyle(v dyn.Value, style yaml.Style) (*yaml.Node,
 	case dyn.KindFloat:
 		return &yaml.Node{Kind: yaml.ScalarNode, Value: fmt.Sprint(v.MustFloat()), Style: style}, nil
 	case dyn.KindTime:
-		return &yaml.Node{Kind: yaml.ScalarNode, Value: v.MustTime().String(), Style: style}, nil
+		return &yaml.Node{Kind: yaml.ScalarNode, Value: v.MustTime().UTC().String(), Style: style}, nil
 	default:
 		// Panic because we only want to deal with known types.
 		panic(fmt.Sprintf("invalid kind: %d", v.Kind()))

libs/dyn/yamlsaver/saver_test.go

@@ -2,10 +2,10 @@ package yamlsaver
 
 import (
 	"testing"
+	"time"
 
 	"github.com/databricks/cli/libs/dyn"
 	assert "github.com/databricks/cli/libs/dyn/dynassert"
-	"github.com/stretchr/testify/require"
 	"gopkg.in/yaml.v3"
 )
 
@@ -45,14 +45,11 @@ func TestMarshalBoolValue(t *testing.T) {
 }
 
 func TestMarshalTimeValue(t *testing.T) {
-	tm, err := dyn.NewTime("1970-01-01")
-	require.NoError(t, err)
-
 	s := NewSaver()
-	var timeValue = dyn.V(tm)
+	var timeValue = dyn.V(time.Unix(0, 0))
 	v, err := s.toYamlNode(timeValue)
 	assert.NoError(t, err)
-	assert.Equal(t, "1970-01-01", v.Value)
+	assert.Equal(t, "1970-01-01 00:00:00 +0000 UTC", v.Value)
 	assert.Equal(t, yaml.ScalarNode, v.Kind)
 }
 

libs/template/templates/default-sql/template/{{.project_name}}/src/orders_daily.sql.tmpl

@@ -1,9 +1,10 @@
 -- This query is executed using Databricks Workflows (see resources/{{.project_name}}_sql_job.yml)
+{{- /* We can't use a materialized view here since they don't support 'create or refresh' yet.*/}}
 
 USE CATALOG {{"{{"}}catalog{{"}}"}};
 USE IDENTIFIER({{"{{"}}schema{{"}}"}});
 
-CREATE OR REPLACE MATERIALIZED VIEW
+CREATE OR REPLACE VIEW
   orders_daily
 AS SELECT
   order_date, count(*) AS number_of_orders