diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e57961..8cd81b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,18 @@ This project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +## [1.1.0] - 2026-09-16 + +### Added + +- `export --scaffold --module-version latest` resolves the latest stable public Registry version + for each selected module and writes exact version pins. Lookup failures stop before file writes. + +### Changed + +- Default scaffold pins use module version 1.0.1, with input validation and permission checks + in each resource module and unknown grant-target checks in the workspace pattern. + ## [1.0.3] - 2026-09-11 ### Security diff --git a/docs/module-layouts.md b/docs/module-layouts.md index 7879587..915fb82 100644 --- a/docs/module-layouts.md +++ b/docs/module-layouts.md @@ -6,8 +6,8 @@ Neither creates Azure workspaces or exports workloads, stored data, or secret va | `--module-layout` | Root calls | Default module version | | --- | --- | --- | -| `workspace` (default) | `536tech/workspace/databricks` | `1.0.0` | -| `resources` | Individual `536tech` Registry modules | `1.0.0` | +| `workspace` (default) | `536tech/workspace/databricks` | `1.0.1` | +| `resources` | Individual `536tech` Registry modules | `1.0.1` | ## Individual modules @@ -44,7 +44,26 @@ The modules manage the resources in the [resource matrix](../README.md#resource- ## Versions and inputs -The resource layout pins every emitted module to an exact version. Use `--module-version 1.0.0` +The default pins the latest release tested with this DataTF build. It needs no Registry lookup. +Use `--module-version latest --scaffold` to select newer stable releases: + +```sh +datatf export --profile analytics --resources warehouses --module-layout resources \ + --module-version latest --scaffold --out ./warehouse-latest +``` + +DataTF resolves each selected module independently through the public Terraform Registry. +It writes exact versions into `main.tf`; later `terraform init` calls do not select newer module releases. +Pre-release versions are excluded. A failed lookup stops the export before any output files are written. +The lookup sends module source addresses only, with no workspace credentials or metadata. + +`latest` is an explicit opt-in, not a compatibility certification. It can select a new major version. +Review the module changes and require an imports-only plan before apply. +Use the default for the tested contract. The workspace module pins its own child-module versions. +`latest` supports public Registry sources only; it rejects Git, local, and private Registry sources. + + +The resource layout pins every emitted module to an exact version. Use `--module-version 1.0.1` to select that release explicitly. A different version must exist for every selected module and keep the same inputs and resource addresses. Version ranges are not supported for this layout. diff --git a/internal/cli/export.go b/internal/cli/export.go index c876d1c..ab3fc01 100644 --- a/internal/cli/export.go +++ b/internal/cli/export.go @@ -16,17 +16,18 @@ import ( ) type exportOptions struct { - scope string - outDir string - rootModule string - allowPartial bool - scaffold bool - moduleSource string - moduleVer string - moduleLayout string - resources []string - name *string - profile string + scope string + outDir string + rootModule string + allowPartial bool + scaffold bool + moduleSource string + moduleVer string + moduleVersions map[string]string + moduleLayout string + resources []string + name *string + profile string } func newExportCommand(rc *runtime) *cobra.Command { @@ -80,7 +81,7 @@ from other groups. Use a new output directory for each export.`, flags.StringVar(&opts.moduleSource, "module-source", scaffold.DefaultModuleSource, "module source for --scaffold (registry address, Git URL, or local path)") flags.StringVar(&opts.moduleVer, "module-version", scaffold.DefaultModuleVersion, - "Registry module version for --scaffold") + "Registry module version or latest for --scaffold (default: tested release)") flags.StringSliceVar(&opts.resources, "resources", nil, "limit reads to resource groups (comma-separated)") flags.String("name", "", "select one exact name within one --resources group") @@ -122,6 +123,15 @@ func (opts *exportOptions) validateRoot() error { } func (opts *exportOptions) prepareLayout(cmd *cobra.Command) error { + if opts.moduleVer == "latest" { + if !opts.scaffold { + return fmt.Errorf("%w: --module-version latest requires --scaffold", errUsage) + } + if opts.moduleLayout == "workspace" && !scaffold.PublicRegistryModule(opts.moduleSource) { + return fmt.Errorf("%w: --module-version latest requires a public Terraform Registry module source", errUsage) + } + } + switch opts.moduleLayout { case "workspace": return nil @@ -136,8 +146,8 @@ func (opts *exportOptions) prepareLayout(cmd *cobra.Command) error { return fmt.Errorf("%w: --root-module must be empty with --module-layout resources", errUsage) } opts.rootModule = "" - if !regexp.MustCompile(`^[0-9]+\.[0-9]+\.[0-9]+$`).MatchString(opts.moduleVer) { - return fmt.Errorf("%w: --module-version needs an exact release such as 1.0.0 "+ + if opts.moduleVer != "latest" && !regexp.MustCompile(`^[0-9]+\.[0-9]+\.[0-9]+$`).MatchString(opts.moduleVer) { + return fmt.Errorf("%w: --module-version needs latest or an exact release such as 1.0.0 "+ "with --module-layout resources", errUsage) } return nil @@ -158,6 +168,9 @@ func (opts *exportOptions) run(cmd *cobra.Command, rc *runtime) error { if err := opts.checkReport(rc, rep); err != nil { return err } + if err := opts.resolveVersions(cmd.Context(), ex); err != nil { + return withHint(err, "module_version_error", "Retry the Registry lookup or use the tested default without --module-version latest.") + } files, err := opts.files(ex, rep) if err != nil { return withHint(err, "render_error", "Check scaffold options with datatf export --help.") @@ -193,7 +206,7 @@ func (opts *exportOptions) scaffoldFiles(ex *contract.Export, rep *contract.Repo ) { options := scaffold.Options{ Scope: ex.Scope, Host: rep.Host, Profile: opts.profile, RootModule: opts.rootModule, - ModuleSource: opts.moduleSource, ModuleVersion: opts.moduleVer, + ModuleSource: opts.moduleSource, ModuleVersion: opts.moduleVer, ModuleVersions: opts.moduleVersions, } if opts.moduleLayout == "resources" { return scaffold.RenderResources(ex, options, opts.scaffold) diff --git a/internal/cli/export_test.go b/internal/cli/export_test.go index 4f21944..ec72d53 100644 --- a/internal/cli/export_test.go +++ b/internal/cli/export_test.go @@ -184,7 +184,7 @@ func TestExportScaffoldRegistryDefault(t *testing.T) { } content := strings.Join(strings.Fields(string(main)), " ") for _, want := range []string{ - `source = "536tech/workspace/databricks"`, `version = "1.0.0"`, + `source = "536tech/workspace/databricks"`, `version = "1.0.1"`, } { if !strings.Contains(content, want) { t.Errorf("main.tf missing %q:\n%s", want, main) diff --git a/internal/cli/module_versions.go b/internal/cli/module_versions.go new file mode 100644 index 0000000..def853a --- /dev/null +++ b/internal/cli/module_versions.go @@ -0,0 +1,37 @@ +package cli + +import ( + "context" + + "github.com/536tech/datatf/internal/contract" + "github.com/536tech/datatf/internal/scaffold" +) + +var resolveModuleVersions = scaffold.LatestVersions + +func (opts *exportOptions) resolveVersions(ctx context.Context, ex *contract.Export) error { + if opts.moduleVer != "latest" { + return nil + } + sources := []string{opts.moduleSource} + if opts.moduleLayout == "resources" { + modules, err := contract.ResourceModules(ex.Tfvars) + if err != nil { + return err + } + sources = nil + for _, module := range modules { + sources = append(sources, module.Source) + } + } + versions, err := resolveModuleVersions(ctx, sources) + if err != nil { + return err + } + if opts.moduleLayout == "resources" { + opts.moduleVersions = versions + } else { + opts.moduleVer = versions[opts.moduleSource] + } + return nil +} diff --git a/internal/cli/module_versions_test.go b/internal/cli/module_versions_test.go new file mode 100644 index 0000000..93f7b7e --- /dev/null +++ b/internal/cli/module_versions_test.go @@ -0,0 +1,114 @@ +package cli + +import ( + "context" + "errors" + "fmt" + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/hclsyntax" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/536tech/datatf/internal/fakews" +) + +func TestLatestModuleVersionRequiresScaffold(t *testing.T) { + srv := fakews.New(t) + isolateAuth(t, srv) + code, _, stderr := run(t, "export", "--module-version", "latest", "--out", filepath.Join(t.TempDir(), "out")) + if code != exitUsage { + t.Fatalf("latest without scaffold: exit %d: %s", code, stderr) + } +} + +func TestLatestModuleVersionsPinSelectedSources(t *testing.T) { + for _, layout := range []string{"workspace", "resources"} { + t.Run(layout, func(t *testing.T) { + srv := fakews.New(t) + isolateAuth(t, srv) + old := resolveModuleVersions + t.Cleanup(func() { resolveModuleVersions = old }) + var requested []string + resolveModuleVersions = func(ctx context.Context, sources []string) (map[string]string, error) { + requested = sources + versions := map[string]string{} + for i, source := range sources { + versions[source] = fmt.Sprintf("1.2.%d", i) + } + return versions, nil + } + out := filepath.Join(t.TempDir(), "out") + code, _, stderr := run(t, "export", "--resources", "catalogs", "--module-layout", layout, "--scaffold", "--module-version", "latest", "--out", out) + if code != exitOK { + t.Fatalf("exit %d: %s", code, stderr) + } + file, diags := hclsyntax.ParseConfig(readGenerated(t, out, "main.tf"), "main.tf", hcl.InitialPos) + if diags.HasErrors() { + t.Fatal(diags) + } + blocks := file.Body.(*hclsyntax.Body).Blocks + if len(requested) != len(blocks) || len(blocks) == 0 { + t.Fatalf("lookups %v, blocks %d", requested, len(blocks)) + } + for i, block := range blocks { + value, _ := block.Body.Attributes["version"].Expr.Value(nil) + if value.AsString() != fmt.Sprintf("1.2.%d", i) { + t.Fatalf("version %v", value) + } + source, _ := block.Body.Attributes["source"].Expr.Value(nil) + if source.AsString() != requested[i] { + t.Fatalf("source %v != %s", source, requested[i]) + } + } + }) + } +} + +func TestRegistryFailureWritesNothing(t *testing.T) { + srv := fakews.New(t) + isolateAuth(t, srv) + old := resolveModuleVersions + t.Cleanup(func() { resolveModuleVersions = old }) + resolveModuleVersions = func(context.Context, []string) (map[string]string, error) { + return nil, errors.New("Registry unavailable") + } + out := filepath.Join(t.TempDir(), "out") + code, _, stderr := run(t, "export", "--json", "--scaffold", "--module-version", "latest", "--out", out) + if code != exitErr || !strings.Contains(stderr, "module_version_error") { + t.Fatalf("exit %d: %s", code, stderr) + } + if _, err := os.Stat(out); !os.IsNotExist(err) { + t.Fatal("Registry failure wrote output") + } +} + +func TestDefaultAndExplicitVersionDoNotResolve(t *testing.T) { + old := resolveModuleVersions + t.Cleanup(func() { resolveModuleVersions = old }) + resolveModuleVersions = func(context.Context, []string) (map[string]string, error) { + t.Fatal("unexpected Registry call") + return nil, nil + } + for _, version := range []string{"", "1.2.3"} { + srv := fakews.New(t) + isolateAuth(t, srv) + args := []string{"export", "--scaffold", "--out", filepath.Join(t.TempDir(), "out")} + if version != "" { + args = append(args, "--module-version", version) + } + if code, _, stderr := run(t, args...); code != exitOK { + t.Fatalf("exit %d: %s", code, stderr) + } + } +} + +func TestLatestRejectsNonPublicSource(t *testing.T) { + for _, source := range []string{"../module", "git::https://example.com/module.git", "private.example/team/module/databricks"} { + code, _, stderr := run(t, "export", "--scaffold", "--module-source", source, "--module-version", "latest") + if code != exitUsage { + t.Fatalf("%s: exit %d: %s", source, code, stderr) + } + } +} diff --git a/internal/cli/resource_modules_test.go b/internal/cli/resource_modules_test.go index d4887a6..bf3ea17 100644 --- a/internal/cli/resource_modules_test.go +++ b/internal/cli/resource_modules_test.go @@ -55,7 +55,7 @@ func readResourceModules(t *testing.T, out string) map[string]bool { } for key, want := range map[string]string{ "source": "registry.terraform.io/536tech/" + registryName + "/databricks", - "version": "1.0.0", + "version": "1.0.1", } { value, diags := block.Body.Attributes[key].Expr.Value(nil) if diags.HasErrors() || value.AsString() != want { diff --git a/internal/scaffold/resources.go b/internal/scaffold/resources.go index 61945d5..98c6b58 100644 --- a/internal/scaffold/resources.go +++ b/internal/scaffold/resources.go @@ -32,7 +32,7 @@ func RenderResources(ex *contract.Export, opts Options, root bool) (map[string][ if !root { return files, nil } - main, variables := resourceRoot(modules, opts.ModuleVersion) + main, variables := resourceRoot(modules, opts) maps.Copy(files, map[string][]byte{ "main.tf": main, "variables.tf": variables, "providers.tf": providerFile(opts), "versions.tf": []byte(versionsTF), @@ -51,11 +51,15 @@ export.json retains the canonical DataTF data structure, not the resource-module Keep this layout after import. A layout change requires a reviewed state migration. ` -func resourceRoot(modules []contract.ResourceModule, version string) ([]byte, []byte) { +func resourceRoot(modules []contract.ResourceModule, opts Options) ([]byte, []byte) { main, variables := hclwrite.NewEmptyFile(), hclwrite.NewEmptyFile() for _, module := range modules { body := main.Body().AppendNewBlock("module", []string{module.Name}).Body() body.SetAttributeValue("source", cty.StringVal(module.Source)) + version := opts.ModuleVersion + if resolved, ok := opts.ModuleVersions[module.Source]; ok { + version = resolved + } body.SetAttributeValue("version", cty.StringVal(version)) body.SetAttributeTraversal("for_each", traversal("var", module.Name)) for _, name := range resourceInputNames(module.Values) { diff --git a/internal/scaffold/scaffold.go b/internal/scaffold/scaffold.go index 5c18a55..9c7e6bc 100644 --- a/internal/scaffold/scaffold.go +++ b/internal/scaffold/scaffold.go @@ -17,16 +17,17 @@ import ( const DefaultModuleSource = "536tech/workspace/databricks" // DefaultModuleVersion pins the tested Registry release for both layouts. -const DefaultModuleVersion = "1.0.0" +const DefaultModuleVersion = "1.0.1" // Options control the generated root. type Options struct { - Scope contract.Scope - Host string - Profile string - RootModule string - ModuleSource string - ModuleVersion string + Scope contract.Scope + Host string + Profile string + RootModule string + ModuleSource string + ModuleVersion string + ModuleVersions map[string]string } var variableNames = []string{ diff --git a/internal/scaffold/versions.go b/internal/scaffold/versions.go new file mode 100644 index 0000000..4a975e5 --- /dev/null +++ b/internal/scaffold/versions.go @@ -0,0 +1,86 @@ +package scaffold + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "regexp" + "strings" + "time" + + "golang.org/x/mod/semver" +) + +var publicModule = regexp.MustCompile(`^(?:registry\.terraform\.io/)?([A-Za-z0-9_-]+/[A-Za-z0-9_-]+/[A-Za-z0-9_-]+)$`) + +// PublicRegistryModule reports whether latest can resolve this source. +func PublicRegistryModule(source string) bool { return publicModule.MatchString(source) } + +// LatestVersions resolves each public Registry source to its newest stable release. +// The default export never calls this endpoint. No workspace information is sent. +func LatestVersions(ctx context.Context, sources []string) (map[string]string, error) { + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + client := &http.Client{Timeout: 10 * time.Second} + versions := make(map[string]string, len(sources)) + for _, source := range sources { + version, err := latestVersion(ctx, client, source) + if err != nil { + return nil, err + } + versions[source] = version + } + return versions, nil +} + +func latestVersion(ctx context.Context, client *http.Client, source string) (string, error) { + parts := publicModule.FindStringSubmatch(source) + if parts == nil { + return "", fmt.Errorf("latest requires a public Terraform Registry module source") + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, + "https://registry.terraform.io/v1/modules/"+parts[1]+"/versions", nil) + if err != nil { + return "", err + } + req.Header.Set("Accept", "application/json") + res, err := client.Do(req) + if err != nil { + return "", fmt.Errorf("resolve module version for %s: %w", source, err) + } + defer res.Body.Close() + if res.StatusCode != http.StatusOK { + return "", fmt.Errorf("resolve module version for %s: Registry HTTP %d", source, res.StatusCode) + } + var listing struct { + Modules []struct { + Versions []struct { + Version string `json:"version"` + } `json:"versions"` + } `json:"modules"` + } + if err := json.NewDecoder(io.LimitReader(res.Body, 4<<20)).Decode(&listing); err != nil { + return "", fmt.Errorf("resolve module version for %s: invalid Registry response", source) + } + latest := "" + if len(listing.Modules) > 0 { + for _, release := range listing.Modules[0].Versions { + v := "v" + release.Version + // Require all three numeric components; stable releases may include build metadata. + if semver.Canonical(v)+semver.Build(v) != v || semver.Prerelease(v) != "" { + continue + } + comparison := semver.Compare(v, "v"+latest) + // Build metadata has equal SemVer precedence. Break ties independently of API order. + if latest == "" || comparison > 0 || (comparison == 0 && release.Version > latest) { + latest = strings.TrimPrefix(v, "v") + } + } + } + if latest == "" { + return "", fmt.Errorf("resolve module version for %s: no stable release", source) + } + return latest, nil +} diff --git a/internal/scaffold/versions_test.go b/internal/scaffold/versions_test.go new file mode 100644 index 0000000..33a15b3 --- /dev/null +++ b/internal/scaffold/versions_test.go @@ -0,0 +1,86 @@ +package scaffold + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" +) + +type registryTransport struct{ target *url.URL } + +func (r registryTransport) RoundTrip(req *http.Request) (*http.Response, error) { + clone := req.Clone(req.Context()) + clone.URL.Scheme, clone.URL.Host = r.target.Scheme, r.target.Host + return http.DefaultTransport.RoundTrip(clone) +} + +func TestLatestVersion(t *testing.T) { + for _, tc := range []struct { + name, body, want string + status int + }{ + {"semantic_order", `{"modules":[{"versions":[{"version":"1.9.0"},{"version":"1.10.0"},{"version":"2.0.0-rc.1"},{"version":"v9.0.0"},{"version":"8.0"},{"version":"invalid"}]}]}`, "1.10.0", 200}, + {"new_major_opt_in", `{"modules":[{"versions":[{"version":"2.0.0"},{"version":"1.0.0"}]}]}`, "2.0.0", 200}, + {"first_module_only", `{"modules":[{"versions":[{"version":"1.0.0"}]},{"versions":[{"version":"9.0.0"}]}]}`, "1.0.0", 200}, + {"build_metadata", `{"modules":[{"versions":[{"version":"1.9.0"},{"version":"1.10.0+build.1"}]}]}`, "1.10.0+build.1", 200}, + {"metadata_only", `{"modules":[{"versions":[{"version":"1.0.0+build.1"}]}]}`, "1.0.0+build.1", 200}, + {"metadata_tie", `{"modules":[{"versions":[{"version":"1.0.0+build.2"},{"version":"1.0.0+build.1"}]}]}`, "1.0.0+build.2", 200}, + {"empty", `{"modules":[]}`, "", 200}, + {"prerelease_only", `{"modules":[{"versions":[{"version":"1.0.0-beta"}]}]}`, "", 200}, + {"malformed", `{"modules":`, "", 200}, + {"missing", `private response text`, "", 404}, + {"rate_limited", `private response text`, "", 429}, + } { + t.Run(tc.name, func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v1/modules/536tech/workspace/databricks/versions" { + t.Errorf("unexpected path %s", r.URL.Path) + } + if r.Header.Get("Authorization") != "" { + t.Error("Registry request must not contain workspace credentials") + } + w.WriteHeader(tc.status) + fmt.Fprint(w, tc.body) + })) + defer srv.Close() + target, _ := url.Parse(srv.URL) + client := &http.Client{Transport: registryTransport{target}} + got, err := latestVersion(context.Background(), client, "536tech/workspace/databricks") + if tc.want == "" { + if err == nil || strings.Contains(err.Error(), "private response text") { + t.Fatalf("expected safe error: %v", err) + } + } else if err != nil || got != tc.want { + t.Fatalf("got %q, %v; want %s", got, err, tc.want) + } + }) + } +} + +func TestLatestVersionCanceled(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := latestVersion(ctx, &http.Client{}, "536tech/workspace/databricks"); err == nil { + t.Fatal("cancellation must stop resolution") + } +} + +func TestLatestSourceValidation(t *testing.T) { + for _, source := range []string{"../local", "git::https://example.com/module.git", "private.example/536tech/workspace/databricks", "https://registry.terraform.io/536tech/workspace/databricks", "536tech/workspace/databricks//subdir"} { + if PublicRegistryModule(source) { + t.Fatalf("accepted %s", source) + } + if _, err := latestVersion(context.Background(), &http.Client{}, source); err == nil { + t.Fatalf("resolved %s", source) + } + } + for _, source := range []string{"536tech/workspace/databricks", "registry.terraform.io/536tech/sql-warehouse/databricks"} { + if !PublicRegistryModule(source) { + t.Fatalf("rejected %s", source) + } + } +}