From 91b952c597889cc47ce656fb5ec58bca495d8e25 Mon Sep 17 00:00:00 2001 From: Gustavo Bertoi Date: Tue, 30 Jun 2026 19:44:53 -0300 Subject: [PATCH] =?UTF-8?q?feat(secrets):=20`secrets=20ingest`=20=E2=80=94?= =?UTF-8?q?=20.env=20=E2=86=92=20SOPS/provider=20secrets=20+=20vars=20(spe?= =?UTF-8?q?c=2024)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the write half of the secrets boundary and a new envingest pipeline that gets committed .env files out of the repo: - internal/secrets: Pusher capability (SecretEntry/Pusher) on aws-sm/aws-ssm/ infisical (values via stdin/env, never logged argv), a stdin-capable runner (OutputStdin), and SOPS encrypt/decrypt helpers that shell `sops` over stdin (no Go SDK, no plaintext temp file). - internal/envingest: parse (compose-go dotenv) → classify default-deny (glob → name → value heuristics → benign → default secret) → route (sops file / remote Pusher) → encrypt/push → compute secret:// refs → rewrite the target devstack.yaml env block in place via the goccy AST (comment/order preserving) → scaffold a sops provider into workspace.yaml when absent → round-trip verify before deleting → fence .env in .gitignore → delete (or --keep-env). Idempotency via decrypt-and-compare; no flock. - internal/cli: `secrets ingest [<.env>]` with --to/--dest/--service/ --recipient/--secret/--public/--from-host/--prefixed/--keep-env/--dry-run/ --yes/--force/--json + a degradable huh v2 classification wizard gated on prompt.IsInteractive. - Tests: Pusher argv/stdin per provider, classify ladder, AST rewrite/scaffold, full-run leak test, git-tracked refusal, dry-run writes nothing, decrypt- compare idempotency, non-TTY gate routes to the flag path. Co-Authored-By: Claude Opus 4.8 (1M context) --- internal/cli/secrets.go | 1 + internal/cli/secrets_ingest.go | 427 +++++++++++++++++++++++ internal/cli/secrets_ingest_test.go | 151 +++++++++ internal/cli/secrets_ingest_tui.go | 148 ++++++++ internal/envingest/classify.go | 203 +++++++++++ internal/envingest/classify_test.go | 76 +++++ internal/envingest/ingest.go | 506 ++++++++++++++++++++++++++++ internal/envingest/ingest_test.go | 288 ++++++++++++++++ internal/envingest/rewrite.go | 242 +++++++++++++ internal/envingest/rewrite_test.go | 108 ++++++ internal/secrets/pusher.go | 206 +++++++++++ internal/secrets/pusher_test.go | 187 ++++++++++ 12 files changed, 2543 insertions(+) create mode 100644 internal/cli/secrets_ingest.go create mode 100644 internal/cli/secrets_ingest_test.go create mode 100644 internal/cli/secrets_ingest_tui.go create mode 100644 internal/envingest/classify.go create mode 100644 internal/envingest/classify_test.go create mode 100644 internal/envingest/ingest.go create mode 100644 internal/envingest/ingest_test.go create mode 100644 internal/envingest/rewrite.go create mode 100644 internal/envingest/rewrite_test.go create mode 100644 internal/secrets/pusher.go create mode 100644 internal/secrets/pusher_test.go diff --git a/internal/cli/secrets.go b/internal/cli/secrets.go index 707e6bb..08c085a 100644 --- a/internal/cli/secrets.go +++ b/internal/cli/secrets.go @@ -20,6 +20,7 @@ func newSecretsCmd(g *GlobalOpts) *cobra.Command { } cmd.AddCommand( newSecretsKeygenCmd(g), + newSecretsIngestCmd(g), newSecretsLoginCmd(g), newSecretsLogoutCmd(g), newSecretsStatusCmd(g), diff --git a/internal/cli/secrets_ingest.go b/internal/cli/secrets_ingest.go new file mode 100644 index 0000000..b831766 --- /dev/null +++ b/internal/cli/secrets_ingest.go @@ -0,0 +1,427 @@ +package cli + +import ( + "context" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/spf13/cobra" + + "github.com/open-source-cloud/devstack/internal/config" + "github.com/open-source-cloud/devstack/internal/envingest" + "github.com/open-source-cloud/devstack/internal/git" + "github.com/open-source-cloud/devstack/internal/prompt" + "github.com/open-source-cloud/devstack/internal/secrets" + "github.com/open-source-cloud/devstack/internal/store" +) + +// newSecretsIngestCmd wires `secrets ingest [<.env>]` (spec 24): get a committed +// .env out of the repo by classifying each key secret-vs-config, encrypting the +// secret half into SOPS+age (or pushing it to a provider), inlining the config +// half, rewriting devstack.yaml, and fencing/deleting the .env. No flock (touches +// neither the ledger nor the shared stack). Dry-run writes nothing. +func newSecretsIngestCmd(g *GlobalOpts) *cobra.Command { + var ( + to string + dest string + service string + recipient string + secretGlobs []string + publicGlobs []string + fromHost []string + prefixed bool + keepEnv bool + dryRun bool + yes bool + force bool + ) + cmd := &cobra.Command{ + Use: "ingest [path/to/.env]", + Short: "Convert a committed .env into SOPS/provider secrets + inline config vars", + Long: "ingest reads an existing .env, classifies each key secret-vs-config (default-deny),\n" + + "encrypts the secret half into a SOPS+age file (or pushes it to a provider), inlines\n" + + "the config half as literals, rewrites devstack.yaml in place (comment-preserving),\n" + + "proves every new secret:// ref round-trips, fences .env in .gitignore, and deletes it.\n" + + "It takes no lock. Run bare on a TTY for the classification wizard; --yes/--json skip it.", + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + envArg := ".env" + if len(args) == 1 { + envArg = args[0] + } + envPath, err := filepath.Abs(envArg) + if err != nil { + return err + } + if _, err := os.Stat(envPath); err != nil { + return fmt.Errorf("no .env at %s: %w", envPath, err) + } + + opts, model, err := buildIngestOptions(envPath, to, dest, service, recipient, secretGlobs, publicGlobs, fromHost, prefixed, keepEnv, dryRun, force) + if err != nil { + return err + } + + deps, err := ingestDeps(opts, model) + if err != nil { + return err + } + + // Interactive classification wizard (TTY only). It returns per-key + // overrides folded into the glob lists so Run stays the single path. + if prompt.IsInteractive(g.JSON, g.Quiet, false) && !yes && !dryRun { + plan, _, perr := envingest.BuildPlan(opts) + if perr != nil { + return perr + } + sec, pub, host, ok, werr := runIngestWizard(plan) + if werr != nil { + return werr + } + if !ok { + if !g.Quiet { + fmt.Fprintln(cmd.ErrOrStderr(), "ingest cancelled — nothing written") + } + return nil + } + opts.SecretGlobs = append(opts.SecretGlobs, sec...) + opts.PublicGlobs = append(opts.PublicGlobs, pub...) + opts.FromHostGlobs = append(opts.FromHostGlobs, host...) + } + + res, err := envingest.Run(cmd.Context(), opts, deps) + if err != nil { + return err + } + return reportIngest(cmd, g, opts, res, dryRun) + }, + } + f := cmd.Flags() + f.StringVar(&to, "to", envingest.DestSOPS, "destination backend: sops|aws-sm|infisical") + f.StringVar(&dest, "dest", "", "destination override (file for sops, secret-id prefix / path for remote)") + f.StringVar(&service, "service", "", "target devstack.yaml service (required when the project has >1 service)") + f.StringVar(&recipient, "recipient", "", "age recipient (age1...); default discovery: --recipient → .sops.yaml → keygen") + f.StringArrayVar(&secretGlobs, "secret", nil, "force-classify matching keys as secret (glob, repeatable)") + f.StringArrayVar(&publicGlobs, "public", nil, "force-classify matching keys as config (glob, repeatable)") + f.StringArrayVar(&fromHost, "from-host", nil, "emit matching config keys as ${env.KEY} (glob, repeatable)") + f.BoolVar(&prefixed, "prefixed", false, "route secrets to env.prefixed (compose key _) instead of env.raw") + f.BoolVar(&keepEnv, "keep-env", false, "do not delete .env; print the removal command instead") + f.BoolVar(&dryRun, "dry-run", false, "print the full plan + would-be diffs, write nothing") + f.BoolVar(&yes, "yes", false, "skip the classification wizard; use computed classification + flags") + f.BoolVar(&force, "force", false, "overwrite committed files (each is backed up first)") + return cmd +} + +// buildIngestOptions resolves the workspace, the target project/service, the +// destination provider name/kind, and the age recipient into envingest.Options. +func buildIngestOptions(envPath, to, dest, service, recipient string, secretGlobs, publicGlobs, fromHost []string, prefixed, keepEnv, dryRun, force bool) (envingest.Options, *config.Model, error) { + var zero envingest.Options + model, err := config.Load(filepath.Dir(envPath)) + if err != nil { + return zero, nil, err + } + + project, projDir, err := projectForEnv(model, envPath) + if err != nil { + return zero, nil, err + } + svc, err := resolveService(model, project, service) + if err != nil { + return zero, nil, err + } + + kind, err := destKind(to) + if err != nil { + return zero, nil, err + } + providerName, declared := providerInstance(model, kind) + + opts := envingest.Options{ + EnvPath: envPath, + WorkspaceRoot: model.Root, + WorkspaceFile: filepath.Join(model.Root, config.WorkspaceFile), + ProjectFile: filepath.Join(projDir, config.ProjectFile), + Service: svc, + Dest: to, + DestPath: defaultDest(to, dest, model), + Provider: providerName, + Kind: kind, + SecretGlobs: secretGlobs, + PublicGlobs: publicGlobs, + FromHostGlobs: fromHost, + Prefixed: prefixed, + KeepEnv: keepEnv, + DryRun: dryRun, + Force: force, + ExistingProviders: declaredNames(model), + } + _ = declared + + if to == envingest.DestSOPS { + rec, keyFile, err := resolveRecipient(model.Root, recipient) + if err != nil { + return zero, nil, err + } + opts.Recipient = rec + opts.AgeKey = keyFile + } + return opts, model, nil +} + +// projectForEnv finds the project whose directory is an ancestor of envPath (the +// repo the .env lives in), or the single declared project as a fallback. +func projectForEnv(m *config.Model, envPath string) (string, string, error) { + dir := filepath.Dir(envPath) + for name := range m.Projects { + pd := m.ProjectDir(name) + if pd != "" && (pd == dir || strings.HasPrefix(dir+string(filepath.Separator), pd+string(filepath.Separator))) { + return name, pd, nil + } + } + if len(m.Projects) == 1 { + for name := range m.Projects { + return name, m.ProjectDir(name), nil + } + } + return "", "", fmt.Errorf("could not determine which project owns %s — run from inside a project repo or pass --service", filepath.Base(envPath)) +} + +// resolveService picks the target service: --service when given, the sole service +// when the project has one, otherwise an error listing the choices. +func resolveService(m *config.Model, project, service string) (string, error) { + p := m.Projects[project] + names := make([]string, 0, len(p.Services)) + for n := range p.Services { + names = append(names, n) + } + sort.Strings(names) + if service != "" { + for _, n := range names { + if n == service { + return service, nil + } + } + return "", fmt.Errorf("service %q not found in project %q (have: %s)", service, project, strings.Join(names, ", ")) + } + if len(names) == 1 { + return names[0], nil + } + return "", fmt.Errorf("project %q has %d services (%s) — pass --service to choose one", project, len(names), strings.Join(names, ", ")) +} + +// destKind maps a --to destination to a provider kind. +func destKind(to string) (string, error) { + switch to { + case envingest.DestSOPS: + return secrets.SopsKind, nil + case envingest.DestAWSSM: + return secrets.AWSSecretsManagerKind, nil + case envingest.DestInfisical: + return secrets.InfisicalKind, nil + default: + return "", fmt.Errorf("unknown --to %q (want sops|aws-sm|infisical)", to) + } +} + +// providerInstance returns the declared provider-instance name of the given kind +// (so emitted refs use the operator's name), or the kind itself when none is +// declared (the name the run will scaffold). The bool reports declared. +func providerInstance(m *config.Model, kind string) (string, bool) { + for _, p := range m.Workspace.Secrets.Providers { + if p.Kind == kind { + return p.Name, true + } + } + return kind, false +} + +func declaredNames(m *config.Model) []string { + var out []string + for _, p := range m.Workspace.Secrets.Providers { + out = append(out, p.Name) + } + return out +} + +// defaultDest computes the destination path/prefix when --dest is empty. +func defaultDest(to, dest string, m *config.Model) string { + if dest != "" { + return dest + } + switch to { + case envingest.DestAWSSM: + return m.Workspace.Name + case envingest.DestInfisical: + return "/" + default: + return envingest.DefaultSOPSFile + } +} + +// resolveRecipient finds the age recipient (and key file for decrypt verify): +// --recipient, then .sops.yaml, then the local age key under $DEVSTACK_HOME +// (generating one if absent). +func resolveRecipient(root, flag string) (string, string, error) { + keyFile := filepath.Join(store.Home(), "age", "keys.txt") + if flag != "" { + return flag, keyFile, nil + } + if rec := recipientFromSopsYAML(filepath.Join(root, ".sops.yaml")); rec != "" { + return rec, keyFile, nil + } + if rec := recipientFromKeyFile(keyFile); rec != "" { + return rec, keyFile, nil + } + // Generate a fresh local age identity under $DEVSTACK_HOME. + key, err := secrets.GenerateAgeKey() + if err != nil { + return "", "", err + } + if err := os.MkdirAll(filepath.Dir(keyFile), 0o700); err != nil { + return "", "", err + } + if err := os.WriteFile(keyFile, []byte(key.AgeKeyFileContents()), 0o600); err != nil { + return "", "", fmt.Errorf("write age key %s: %w", keyFile, err) + } + return key.Recipient, keyFile, nil +} + +// recipientFromSopsYAML extracts the first `age:` recipient from a .sops.yaml. +func recipientFromSopsYAML(path string) string { + data, err := os.ReadFile(path) + if err != nil { + return "" + } + for _, line := range strings.Split(string(data), "\n") { + l := strings.TrimSpace(line) + if v, ok := strings.CutPrefix(l, "age:"); ok { + v = strings.Trim(strings.TrimSpace(v), `"'`) + if strings.HasPrefix(v, "age1") { + return strings.Fields(v)[0] + } + } + } + return "" +} + +// recipientFromKeyFile reads the `# public key: age1...` comment from an age key. +func recipientFromKeyFile(path string) string { + data, err := os.ReadFile(path) + if err != nil { + return "" + } + for _, line := range strings.Split(string(data), "\n") { + if v, ok := strings.CutPrefix(strings.TrimSpace(line), "# public key:"); ok { + return strings.TrimSpace(v) + } + } + return "" +} + +// ingestDeps wires the real side-effecting collaborators: sops shell-out +// encrypt/decrypt (sops dest), the provider Pusher + resolver (remote dest), and +// the git track-check guard. +func ingestDeps(opts envingest.Options, model *config.Model) (envingest.Deps, error) { + deps := envingest.Deps{ + GitTracked: func(ctx context.Context, path string) (bool, error) { + gx, err := git.New() + if err != nil { + return false, nil // git absent → cannot prove tracked; do not block + } + if !gx.IsRepo(ctx, filepath.Dir(path)) { + return false, nil + } + err = gx.Run(ctx, filepath.Dir(path), "ls-files", "--error-unmatch", filepath.Base(path)) + return err == nil, nil + }, + } + + if opts.Dest == envingest.DestSOPS { + deps.EncryptYAML = func(ctx context.Context, recipient string, plaintext []byte) ([]byte, error) { + return secrets.SopsEncryptYAML(ctx, nil, recipient, plaintext) + } + deps.DecryptYAML = func(ctx context.Context, ciphertext []byte) ([]byte, error) { + return secrets.SopsDecryptBytes(ctx, nil, opts.AgeKey, ciphertext) + } + return deps, nil + } + + // Remote destination: build the provider and require the Pusher capability. + reg := secrets.NewRegistry() + secrets.RegisterBuiltins(reg) + for _, pr := range model.Workspace.Secrets.Providers { + reg.Configure(secrets.ProviderConfig{Name: pr.Name, Kind: pr.Kind, Env: pr.Env, ProjectID: pr.ProjectID, Region: pr.Region}) + } + if !providerDeclaredByName(model, opts.Provider) { + reg.Configure(secrets.ProviderConfig{Name: opts.Provider, Kind: opts.Kind}) + } + prov, err := reg.Provider(opts.Provider) + if err != nil { + return deps, err + } + pusher, ok := prov.(secrets.Pusher) + if !ok { + return deps, fmt.Errorf("provider %q (kind %s) is read-only in this build; use --to sops", opts.Provider, opts.Kind) + } + deps.Push = pusher.Push + deps.ResolveRef = func(ctx context.Context, ref string) (string, error) { + r, err := secrets.ParseRef(ref) + if err != nil { + return "", err + } + got, err := prov.Resolve(ctx, []secrets.Ref{r}) + if err != nil { + return "", err + } + return got[ref], nil + } + return deps, nil +} + +func providerDeclaredByName(m *config.Model, name string) bool { + for _, p := range m.Workspace.Secrets.Providers { + if p.Name == name { + return true + } + } + return false +} + +// reportIngest prints the conversion report (or the dry-run plan / JSON). +func reportIngest(cmd *cobra.Command, g *GlobalOpts, opts envingest.Options, res *envingest.Result, dryRun bool) error { + if g.JSON { + return writeJSON(cmd, res) + } + if g.Quiet { + return nil + } + w := cmd.OutOrStdout() + if dryRun { + fmt.Fprintf(w, "DRY RUN — plan for %s (nothing written)\n", opts.EnvPath) + } else { + fmt.Fprintf(w, "ingested %s → %s\n", opts.EnvPath, opts.Dest) + } + fmt.Fprintf(w, "%-28s %-7s %s\n", "key", "class", "reason") + for _, d := range res.Plan.Decisions { + fmt.Fprintf(w, "%-28s %-7s %s\n", d.Key, d.Class, d.Reason) + } + fmt.Fprintf(w, "→ destination: %s (%s), provider %q\n", opts.Dest, opts.DestPath, opts.Provider) + if dryRun { + return nil + } + for _, f := range res.Wrote { + fmt.Fprintf(w, " wrote %s\n", f) + } + for _, b := range res.Backups { + fmt.Fprintf(w, " backup %s\n", b) + } + if res.EnvRemoved { + fmt.Fprintf(w, " removed %s\n", opts.EnvPath) + } else if opts.KeepEnv { + fmt.Fprintf(w, " kept %s — remove with: git rm --cached %s && rm %s\n", opts.EnvPath, opts.EnvPath, opts.EnvPath) + } + return nil +} diff --git a/internal/cli/secrets_ingest_test.go b/internal/cli/secrets_ingest_test.go new file mode 100644 index 0000000..42698fc --- /dev/null +++ b/internal/cli/secrets_ingest_test.go @@ -0,0 +1,151 @@ +package cli + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" +) + +func runIngest(t *testing.T, dir string, args ...string) (string, error) { + t.Helper() + t.Setenv("DEVSTACK_WORKSPACE", "") + t.Setenv("DEVSTACK_HOME", t.TempDir()) + var buf strings.Builder + root := NewRootCmd(Options{}) + root.SetArgs(append([]string{"secrets", "ingest"}, args...)) + root.SetOut(&buf) + root.SetErr(&buf) + err := root.Execute() + return buf.String(), err +} + +func writeIngestFixture(t *testing.T) (root, envPath string) { + t.Helper() + root = t.TempDir() + mustWriteFile(t, filepath.Join(root, "workspace.yaml"), `apiVersion: devstack/v1 +kind: Workspace +name: demo +shared: {} +projects: + - name: api + path: api +`) + apiDir := filepath.Join(root, "api") + if err := os.MkdirAll(apiDir, 0o755); err != nil { + t.Fatal(err) + } + mustWriteFile(t, filepath.Join(apiDir, "devstack.yaml"), `apiVersion: devstack/v1 +kind: Project +name: api +services: + api: + template: node.vite +`) + envPath = filepath.Join(apiDir, ".env") + mustWriteFile(t, envPath, `DB_PASSWORD=s3cr3t-p@ss +APP_ENV=local +PORT=8080 +`) + return root, envPath +} + +func mustWriteFile(t *testing.T, path, body string) { + t.Helper() + if err := os.WriteFile(path, []byte(body), 0o644); err != nil { + t.Fatal(err) + } +} + +func TestSecretsIngestRegistered(t *testing.T) { + root := NewRootCmd(Options{}) + c, _, err := root.Find([]string{"secrets", "ingest"}) + if err != nil || c.Name() != "ingest" || c.RunE == nil { + t.Fatalf("secrets ingest not registered: %v", err) + } +} + +// TestSecretsIngestDryRunNonTTY proves the non-TTY/--json path skips the wizard, +// emits the plan, and writes nothing. +func TestSecretsIngestDryRunNonTTY(t *testing.T) { + root, envPath := writeIngestFixture(t) + before := readIfExists(envPath) + + out, err := runIngest(t, root, envPath, "--dry-run", "--json") + if err != nil { + t.Fatalf("ingest dry-run: %v\n%s", err, out) + } + + var res struct { + Plan struct { + Decisions []struct { + Key, Class, Reason, Ref string + } + } + } + if err := json.Unmarshal([]byte(out), &res); err != nil { + t.Fatalf("parse json: %v\n%s", err, out) + } + if len(res.Plan.Decisions) != 3 { + t.Fatalf("want 3 decisions, got %d", len(res.Plan.Decisions)) + } + classByKey := map[string]string{} + for _, d := range res.Plan.Decisions { + classByKey[d.Key] = d.Class + } + if classByKey["DB_PASSWORD"] != "secret" { + t.Errorf("DB_PASSWORD should be secret, got %s", classByKey["DB_PASSWORD"]) + } + if classByKey["PORT"] != "config" || classByKey["APP_ENV"] != "config" { + t.Errorf("PORT/APP_ENV should be config: %v", classByKey) + } + + // Nothing written: .env unchanged, no secrets.enc.yaml. + if readIfExists(envPath) != before { + t.Error("dry-run mutated .env") + } + if _, err := os.Stat(filepath.Join(root, "secrets.enc.yaml")); !os.IsNotExist(err) { + t.Error("dry-run wrote secrets.enc.yaml") + } +} + +func TestSecretsIngestRequiresServiceWhenAmbiguous(t *testing.T) { + root := t.TempDir() + mustWriteFile(t, filepath.Join(root, "workspace.yaml"), `apiVersion: devstack/v1 +kind: Workspace +name: demo +shared: {} +projects: + - name: api + path: api +`) + apiDir := filepath.Join(root, "api") + if err := os.MkdirAll(apiDir, 0o755); err != nil { + t.Fatal(err) + } + mustWriteFile(t, filepath.Join(apiDir, "devstack.yaml"), `apiVersion: devstack/v1 +kind: Project +name: api +services: + web: + template: node.vite + worker: + template: node.vite +`) + envPath := filepath.Join(apiDir, ".env") + mustWriteFile(t, envPath, "FOO=bar\n") + + out, err := runIngest(t, root, envPath, "--dry-run") + if err == nil || !strings.Contains(err.Error(), "--service") { + t.Fatalf("want ambiguous-service error, got %v\n%s", err, out) + } +} + +func readIfExists(p string) string { + b, err := os.ReadFile(p) + if err != nil { + return "" + } + return string(b) +} diff --git a/internal/cli/secrets_ingest_tui.go b/internal/cli/secrets_ingest_tui.go new file mode 100644 index 0000000..ee1b908 --- /dev/null +++ b/internal/cli/secrets_ingest_tui.go @@ -0,0 +1,148 @@ +package cli + +import ( + "errors" + "fmt" + "sort" + "strings" + + huh "charm.land/huh/v2" + + "github.com/open-source-cloud/devstack/internal/envingest" + "github.com/open-source-cloud/devstack/internal/prompt" +) + +// runIngestWizard drives the interactive classification flow (huh on Bubble Tea +// v2, degradable: it only runs behind prompt.IsInteractive). It pre-populates the +// computed classes, lets the operator toggle which keys are secret and which +// config keys are host-sourced, previews the plan, and returns the per-key +// overrides as exact-key glob lists (secret/public/from-host) plus a confirm bool. +// A ctrl+c/esc abort returns ok=false (caller exits 0, writes nothing). +func runIngestWizard(plan envingest.Plan) (secretKeys, publicKeys, hostKeys []string, ok bool, err error) { + allKeys := make([]string, 0, len(plan.Decisions)) + computedSecret := map[string]bool{} + computedHost := map[string]bool{} + for _, d := range plan.Decisions { + allKeys = append(allKeys, d.Key) + if d.IsSecret() { + computedSecret[d.Key] = true + } + if d.HostFrom { + computedHost[d.Key] = true + } + } + sort.Strings(allKeys) + + // Pre-select the computed secrets; the operator toggles off to make a key config. + selectedSecrets := make([]string, 0) + for _, k := range allKeys { + if computedSecret[k] { + selectedSecrets = append(selectedSecrets, k) + } + } + + secretOpts := make([]huh.Option[string], 0, len(allKeys)) + for _, k := range allKeys { + secretOpts = append(secretOpts, huh.NewOption(k, k).Selected(computedSecret[k])) + } + + form := huh.NewForm( + huh.NewGroup( + huh.NewMultiSelect[string](). + Title("Secrets"). + Description("selected = secret (encrypted); unselect to inline as config · space toggles"). + Options(secretOpts...). + Value(&selectedSecrets), + ), + ).WithTheme(prompt.Theme()) + if err := form.Run(); err != nil { + if errors.Is(err, huh.ErrUserAborted) { + return nil, nil, nil, false, nil + } + return nil, nil, nil, false, err + } + + secretSet := map[string]bool{} + for _, k := range selectedSecrets { + secretSet[k] = true + } + // Config keys are everything not chosen as secret; offer host-sourcing for them. + var configKeys []string + for _, k := range allKeys { + if !secretSet[k] { + configKeys = append(configKeys, k) + } + } + + selectedHost := make([]string, 0) + if len(configKeys) > 0 { + hostOpts := make([]huh.Option[string], 0, len(configKeys)) + for _, k := range configKeys { + hostOpts = append(hostOpts, huh.NewOption(k, k).Selected(computedHost[k])) + } + hostForm := huh.NewForm( + huh.NewGroup( + huh.NewMultiSelect[string](). + Title("Host-sourced config keys"). + Description("selected keys are emitted as ${env.KEY} (supplied by the host/CI), not inlined"). + Options(hostOpts...). + Value(&selectedHost), + ), + ).WithTheme(prompt.Theme()) + if err := hostForm.Run(); err != nil { + if errors.Is(err, huh.ErrUserAborted) { + return nil, nil, nil, false, nil + } + return nil, nil, nil, false, err + } + } + + // Confirm with a preview of the resulting classification. + confirm := true + cf := huh.NewForm( + huh.NewGroup( + huh.NewConfirm(). + Title("Apply this classification?"). + Description(prompt.PreviewBox(previewClassification(allKeys, secretSet, selectedHost, plan))). + Value(&confirm), + ), + ).WithTheme(prompt.Theme()) + if err := cf.Run(); err != nil { + if errors.Is(err, huh.ErrUserAborted) { + return nil, nil, nil, false, nil + } + return nil, nil, nil, false, err + } + if !confirm { + return nil, nil, nil, false, nil + } + + for _, k := range allKeys { + if secretSet[k] { + secretKeys = append(secretKeys, k) + } else { + publicKeys = append(publicKeys, k) + } + } + hostKeys = append(hostKeys, selectedHost...) + return secretKeys, publicKeys, hostKeys, true, nil +} + +func previewClassification(allKeys []string, secretSet map[string]bool, hostKeys []string, plan envingest.Plan) string { + host := map[string]bool{} + for _, k := range hostKeys { + host[k] = true + } + var b strings.Builder + fmt.Fprintf(&b, "destination: %s (%s)\n", plan.Dest, plan.DestPath) + for _, k := range allKeys { + class := "config" + if secretSet[k] { + class = "secret" + } else if host[k] { + class = "config (${env})" + } + fmt.Fprintf(&b, "%-26s %s\n", k, class) + } + return strings.TrimRight(b.String(), "\n") +} diff --git a/internal/envingest/classify.go b/internal/envingest/classify.go new file mode 100644 index 0000000..2e00af6 --- /dev/null +++ b/internal/envingest/classify.go @@ -0,0 +1,203 @@ +// Package envingest owns the `.env` ingestion pipeline (spec 24): parse an +// existing .env, classify each key secret-vs-config (default-deny), route the +// secret half into a SOPS+age file or a remote provider (never plaintext on +// disk), rewrite the target devstack.yaml env block in place (comment/order +// preserving via the goccy AST), scaffold the destination secrets provider into +// workspace.yaml when absent, prove every new ref round-trips, fence .env in +// .gitignore, and delete it. The secret-vs-config policy and the YAML rewrite +// live here; the provider boundary stays in internal/secrets. +package envingest + +import ( + "fmt" + "math" + "path" + "regexp" + "sort" + "strconv" + "strings" +) + +// Class is the secret-vs-config verdict for one .env key. +type Class int + +const ( + // ClassSecret routes the value into the encrypted/remote destination as a + // secret:// ref. ClassConfig inlines the value (or ${env.KEY}) in config. + ClassSecret Class = iota + ClassConfig +) + +// String renders a Class for reports. +func (c Class) String() string { + if c == ClassConfig { + return "config" + } + return "secret" +} + +// Decision is the classification + emission plan for one key, emitted in +// sorted-key order for determinism. +type Decision struct { + Key string `json:"key"` + Class string `json:"class"` // "secret" | "config" + Reason string `json:"reason"` // human reason for the verdict + Ref string `json:"ref"` // emitted value: secret:// ref OR inline literal OR ${env.KEY} + HostFrom bool `json:"hostFrom"` + Service string `json:"service"` + value string // plaintext (never serialized; used to assemble the payload) +} + +// IsSecret reports whether the decision routes to the secret destination. +func (d Decision) IsSecret() bool { return d.Class == ClassSecret.String() } + +// Value returns the plaintext for this key (internal use by the assembler). +func (d Decision) Value() string { return d.value } + +// secretNameTokens are case-insensitive substrings whose presence in a KEY name +// marks it a secret. Extends the internal/generate secretAttrs seed +// (password/secretkey/secret/token). +var secretNameTokens = []string{ + "password", "passwd", "pwd", + "secret", "secretkey", "secret_key", + "token", "apikey", "api_key", + "accesskey", "access_key", "privatekey", "private_key", + "credential", "credentials", "passphrase", "auth_token", +} + +var ( + // credentialedURL matches scheme://user:pass@host (embedded credentials). + credentialedURL = regexp.MustCompile(`^[a-zA-Z][a-zA-Z0-9+.-]*://[^/@\s:]+:[^/@\s]+@`) + // plainURL matches a scheme://host URL WITHOUT embedded credentials. + plainURL = regexp.MustCompile(`^[a-zA-Z][a-zA-Z0-9+.-]*://`) + // tokenShape matches an opaque high-entropy token (no spaces/scheme chars). + tokenShape = regexp.MustCompile(`^[A-Za-z0-9+/_=.-]{20,}$`) + // jwtShape matches a JWT (three base64url segments; header starts eyJ). + jwtShape = regexp.MustCompile(`^eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$`) +) + +// Classify applies the policy ladder to each key and returns Decisions in +// sorted-key order. globs are shell-style (path.Match). The ladder, in order: +// explicit --secret glob → --public glob → name-pattern → value heuristics → +// benign-config recognition → default-deny (secret). Config keys matching a +// --from-host glob are marked HostFrom (emitted as ${env.KEY}). +func Classify(vars map[string]string, secretGlobs, publicGlobs, fromHostGlobs []string, service string) ([]Decision, error) { + keys := make([]string, 0, len(vars)) + for k := range vars { + keys = append(keys, k) + } + sort.Strings(keys) + + out := make([]Decision, 0, len(keys)) + for _, k := range keys { + v := vars[k] + class, reason := classifyOne(k, v, secretGlobs, publicGlobs) + d := Decision{Key: k, Class: class.String(), Reason: reason, Service: service, value: v} + if class == ClassConfig && matchAnyGlob(fromHostGlobs, k) { + d.HostFrom = true + } + out = append(out, d) + } + return out, nil +} + +func classifyOne(key, value string, secretGlobs, publicGlobs []string) (Class, string) { + if g, ok := matchGlob(secretGlobs, key); ok { + return ClassSecret, fmt.Sprintf("--secret override (%s)", g) + } + if g, ok := matchGlob(publicGlobs, key); ok { + return ClassConfig, fmt.Sprintf("--public override (%s)", g) + } + if tok, ok := nameSignalsSecret(key); ok { + return ClassSecret, fmt.Sprintf("name matches *%s*", tok) + } + if reason, ok := valueSignalsSecret(value); ok { + return ClassSecret, reason + } + if looksBenign(value) { + return ClassConfig, "no secret signal" + } + return ClassSecret, "default-deny (unknown key)" +} + +// nameSignalsSecret reports whether the KEY name contains a secret token. +func nameSignalsSecret(key string) (string, bool) { + lk := strings.ToLower(key) + for _, t := range secretNameTokens { + if strings.Contains(lk, t) { + return strings.ToUpper(t), true + } + } + return "", false +} + +// valueSignalsSecret reports whether the VALUE shape looks like a secret. +func valueSignalsSecret(value string) (string, bool) { + switch { + case credentialedURL.MatchString(value): + return "credentialed URL", true + case strings.HasPrefix(value, "-----BEGIN "): + return "PEM-encoded material", true + case jwtShape.MatchString(value): + return "JWT-shaped value", true + case tokenShape.MatchString(value) && entropyBits(value) >= 3.0: + return "high-entropy value", true + } + return "", false +} + +// looksBenign reports whether the value is a recognizable non-secret: empty, a +// number/boolean, a short simple word, or a credential-free URL (DNS alias). +func looksBenign(value string) bool { + if value == "" { + return true + } + if _, err := strconv.ParseFloat(value, 64); err == nil { + return true + } + switch strings.ToLower(value) { + case "true", "false", "yes", "no", "on", "off", "null", "nil", "none": + return true + } + if plainURL.MatchString(value) && !credentialedURL.MatchString(value) { + return true + } + if len(value) < 20 && !tokenShape.MatchString(value) { + return true + } + return false +} + +// entropyBits returns the Shannon entropy (bits/char) of s. +func entropyBits(s string) float64 { + if s == "" { + return 0 + } + freq := map[rune]float64{} + for _, r := range s { + freq[r]++ + } + n := float64(len([]rune(s))) + var h float64 + for _, c := range freq { + p := c / n + h -= p * math.Log2(p) + } + return h +} + +// matchGlob returns the first glob that matches key (path.Match semantics) and +// the pattern itself. +func matchGlob(globs []string, key string) (string, bool) { + for _, g := range globs { + if ok, _ := path.Match(g, key); ok { + return g, true + } + } + return "", false +} + +func matchAnyGlob(globs []string, key string) bool { + _, ok := matchGlob(globs, key) + return ok +} diff --git a/internal/envingest/classify_test.go b/internal/envingest/classify_test.go new file mode 100644 index 0000000..27b499c --- /dev/null +++ b/internal/envingest/classify_test.go @@ -0,0 +1,76 @@ +package envingest + +import "testing" + +func TestClassifyLadder(t *testing.T) { + vars := map[string]string{ + "DB_PASSWORD": "s3cr3t-p@ss", + "STRIPE_SECRET_KEY": "sk_live_51Hxxxxxxxxxxxxabcdef", + "APP_ENV": "local", + "PORT": "8080", + "REDIS_URL": "redis://shared-redis:6379/0", + "DATABASE_URL": "postgres://user:p4ss@db:5432/app", + "FEATURE_FLAG": "true", + } + decisions, err := Classify(vars, nil, nil, nil, "api") + if err != nil { + t.Fatal(err) + } + byKey := map[string]Decision{} + prev := "" + for _, d := range decisions { + if prev != "" && d.Key < prev { + t.Fatalf("decisions not sorted: %s after %s", d.Key, prev) + } + prev = d.Key + byKey[d.Key] = d + } + want := map[string]string{ + "DB_PASSWORD": "secret", + "STRIPE_SECRET_KEY": "secret", + "APP_ENV": "config", + "PORT": "config", + "REDIS_URL": "config", + "DATABASE_URL": "secret", // credentialed URL + "FEATURE_FLAG": "config", + } + for k, w := range want { + if byKey[k].Class != w { + t.Errorf("%s: class=%s reason=%q, want %s", k, byKey[k].Class, byKey[k].Reason, w) + } + } +} + +func TestClassifyGlobOverrides(t *testing.T) { + vars := map[string]string{ + "DB_PASSWORD": "x", // name → secret, but --public forces config + "PLAIN_NAME": "local", // benign → config, but --secret forces secret + "PORT": "8080", // config; --from-host marks it host-sourced + } + decisions, err := Classify(vars, []string{"PLAIN_*"}, []string{"DB_PASSWORD"}, []string{"PORT"}, "api") + if err != nil { + t.Fatal(err) + } + m := map[string]Decision{} + for _, d := range decisions { + m[d.Key] = d + } + if m["DB_PASSWORD"].Class != "config" { + t.Errorf("--public override failed: %+v", m["DB_PASSWORD"]) + } + if m["PLAIN_NAME"].Class != "secret" { + t.Errorf("--secret override failed: %+v", m["PLAIN_NAME"]) + } + if m["PORT"].Class != "config" || !m["PORT"].HostFrom { + t.Errorf("--from-host failed: %+v", m["PORT"]) + } +} + +func TestClassifyDefaultDeny(t *testing.T) { + // An unrecognized, opaque, mid-length value with no benign signal → secret. + vars := map[string]string{"WEIRD": "Zk29fjA0qLmZxQwErTyU"} + decisions, _ := Classify(vars, nil, nil, nil, "api") + if decisions[0].Class != "secret" { + t.Fatalf("default-deny failed: %+v", decisions[0]) + } +} diff --git a/internal/envingest/ingest.go b/internal/envingest/ingest.go new file mode 100644 index 0000000..a934d80 --- /dev/null +++ b/internal/envingest/ingest.go @@ -0,0 +1,506 @@ +package envingest + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/compose-spec/compose-go/v2/dotenv" + + "github.com/open-source-cloud/devstack/internal/secrets" +) + +// Destinations. +const ( + DestSOPS = "sops" + DestAWSSM = "aws-sm" + DestInfisical = "infisical" +) + +// DefaultSOPSFile is the default committable SOPS+age destination at the +// workspace root. +const DefaultSOPSFile = "secrets.enc.yaml" + +// Options carries the resolved inputs of one ingest run. Paths are absolute; the +// CLI resolves the workspace root, the target devstack.yaml, and the service. +type Options struct { + EnvPath string // absolute path to the .env + WorkspaceRoot string // absolute workspace root (holds workspace.yaml) + WorkspaceFile string // absolute path to workspace.yaml + ProjectFile string // absolute path to the target devstack.yaml + Service string // target service name + + Dest string // sops | aws-sm | infisical + DestPath string // sops: rel file path; remote: secret-id prefix / infisical path + Provider string // declared provider instance name used in the refs + Kind string // provider kind for scaffolding (sops/aws-sm/...) + Recipient string // age recipient (sops) + AgeKey string // age key FILE for decrypt verify / scaffolded provider env + + SecretGlobs []string + PublicGlobs []string + FromHostGlobs []string + Prefixed bool + KeepEnv bool + DryRun bool + Force bool + + // ExistingProviders is the set of provider instance names already declared in + // workspace.yaml; when Provider is absent the run scaffolds it. + ExistingProviders []string +} + +// Deps are the injectable side-effecting collaborators (so tests need no sops, +// aws, infisical, or git binary). +type Deps struct { + // EncryptYAML encrypts a sorted plaintext YAML payload to ciphertext (sops). + EncryptYAML func(ctx context.Context, recipient string, plaintext []byte) ([]byte, error) + // DecryptYAML decrypts ciphertext to a JSON object (round-trip verify + + // decrypt-and-compare idempotency). Nil for remote dests. + DecryptYAML func(ctx context.Context, ciphertext []byte) ([]byte, error) + // Push routes secret entries to a remote provider (aws-sm/infisical). Nil sops. + Push func(ctx context.Context, entries []secrets.SecretEntry) error + // ResolveRef resolves a computed ref to its stored value (remote round-trip). + ResolveRef func(ctx context.Context, ref string) (string, error) + // GitTracked reports whether path is tracked by git; the run refuses if true. + GitTracked func(ctx context.Context, path string) (bool, error) +} + +// Plan is the deterministic, write-nothing description of an ingest run. +type Plan struct { + Source string `json:"source"` + Service string `json:"service"` + Dest string `json:"dest"` + DestPath string `json:"destPath"` + Provider string `json:"provider"` + Recipient string `json:"recipient,omitempty"` + ScaffoldNeeded bool `json:"scaffoldProvider"` + Decisions []Decision `json:"decisions"` +} + +// Result reports what a (non-dry-run) ingest changed. +type Result struct { + Plan Plan `json:"plan"` + Wrote []string `json:"wrote"` + Backups []string `json:"backups"` + EnvRemoved bool `json:"envRemoved"` + GitignoreFn string `json:"gitignore,omitempty"` +} + +// gitignoreMarker fences the appended .env entry so re-runs are idempotent. +const gitignoreMarker = "# devstack:env-ingest" + +// BuildPlan parses the .env, classifies every key, and computes the emitted refs +// — pure except for reading the .env file. It writes nothing. +func BuildPlan(opts Options) (Plan, map[string]string, error) { + raw, err := os.ReadFile(opts.EnvPath) + if err != nil { + return Plan{}, nil, fmt.Errorf("read %s: %w", opts.EnvPath, err) + } + vars, err := dotenv.Parse(strings.NewReader(string(raw))) + if err != nil { + return Plan{}, nil, fmt.Errorf("parse %s: %w", opts.EnvPath, err) + } + if len(vars) == 0 { + return Plan{}, nil, fmt.Errorf("%s declares no variables", opts.EnvPath) + } + + decisions, err := Classify(vars, opts.SecretGlobs, opts.PublicGlobs, opts.FromHostGlobs, opts.Service) + if err != nil { + return Plan{}, nil, err + } + for i := range decisions { + decisions[i].Ref = emitValue(decisions[i], opts) + } + + plan := Plan{ + Source: opts.EnvPath, + Service: opts.Service, + Dest: opts.Dest, + DestPath: opts.DestPath, + Provider: opts.Provider, + Recipient: opts.Recipient, + ScaffoldNeeded: !providerDeclared(opts), + Decisions: decisions, + } + return plan, vars, nil +} + +// providerDeclared reports whether opts.Provider already exists in workspace.yaml. +func providerDeclared(opts Options) bool { + for _, p := range opts.ExistingProviders { + if p == opts.Provider { + return true + } + } + return false +} + +// emitValue computes the value written into the devstack.yaml env block for a +// decision: a secret:// ref (secret), ${env.KEY} (host-sourced config), or the +// inline literal (config, default). +func emitValue(d Decision, opts Options) string { + if d.IsSecret() { + return secretRef(opts.Provider, opts.Dest, opts.DestPath, d.Key) + } + if d.HostFrom { + return "${env." + d.Key + "}" + } + return d.value +} + +// secretRef computes the secret:// reference for a key per destination backend. +func secretRef(provider, dest, destPath, key string) string { + switch dest { + case DestAWSSM: + // secret-id prefix + key (the prefix is the destPath). + return secrets.Scheme + provider + "/" + path.Join(destPath, key) + case DestInfisical: + return secrets.Scheme + provider + "/" + key + default: // sops: dot-path key into the encrypted file + return secrets.Scheme + provider + "/" + destPath + "#" + key + } +} + +// secretPayload returns the secret subset as sorted key→value (sops file body / +// remote entries) and the entries for a remote Push. +func secretPayload(decisions []Decision) (map[string]string, []secrets.SecretEntry) { + kv := map[string]string{} + for _, d := range decisions { + if d.IsSecret() { + kv[d.Key] = d.value + } + } + keys := make([]string, 0, len(kv)) + for k := range kv { + keys = append(keys, k) + } + sort.Strings(keys) + entries := make([]secrets.SecretEntry, 0, len(keys)) + for _, k := range keys { + entries = append(entries, secrets.SecretEntry{Path: k, Value: kv[k]}) + } + return kv, entries +} + +// Run executes the full pipeline. On DryRun it returns the plan with no writes. +// Otherwise it assembles the destination (encrypt/push), rewrites the committed +// YAML (backing each file up first), proves every new ref round-trips, fences and +// removes .env, and returns a Result. +func Run(ctx context.Context, opts Options, deps Deps) (*Result, error) { + if err := validateOptions(opts); err != nil { + return nil, err + } + // Guard: refuse a git-tracked .env (the plaintext is already in history). + if deps.GitTracked != nil { + tracked, err := deps.GitTracked(ctx, opts.EnvPath) + if err == nil && tracked { + return nil, fmt.Errorf("%s is tracked by git — its plaintext is already in history; rotate these secrets and `git rm --cached .env` (this tool cannot un-commit them)", filepath.Base(opts.EnvPath)) + } + } + + plan, _, err := BuildPlan(opts) + if err != nil { + return nil, err + } + res := &Result{Plan: plan} + if opts.DryRun { + return res, nil + } + + kv, entries := secretPayload(plan.Decisions) + + // 1. Assemble destination (no plaintext on disk). + switch opts.Dest { + case DestSOPS: + if len(kv) > 0 { + if err := writeSopsFile(ctx, opts, deps, kv, res); err != nil { + return nil, err + } + } + default: // remote + if len(entries) > 0 { + if deps.Push == nil { + return nil, fmt.Errorf("provider %q is read-only in this build; use --to sops", opts.Provider) + } + if err := deps.Push(ctx, entries); err != nil { + return nil, fmt.Errorf("push to %s: %w", opts.Dest, err) + } + } + } + + // 2. Rewrite the committed devstack.yaml (backup first). + projBytes, err := os.ReadFile(opts.ProjectFile) + if err != nil { + return nil, fmt.Errorf("read %s: %w", opts.ProjectFile, err) + } + newProj, err := rewriteProjectEnv(projBytes, opts.Service, plan.Decisions, opts.Prefixed) + if err != nil { + return nil, err + } + if err := backupAndWrite(opts.ProjectFile, newProj, res); err != nil { + return nil, err + } + + // 3. Scaffold the provider into workspace.yaml when absent (backup first). + if plan.ScaffoldNeeded { + wsBytes, err := os.ReadFile(opts.WorkspaceFile) + if err != nil { + return nil, fmt.Errorf("read %s: %w", opts.WorkspaceFile, err) + } + newWS, changed, err := scaffoldProvider(wsBytes, opts.Provider, opts.Kind, scaffoldEnv(opts)) + if err != nil { + return nil, err + } + if changed { + if err := backupAndWrite(opts.WorkspaceFile, newWS, res); err != nil { + return nil, err + } + } + } + + // 4. Round-trip verify every new secret ref BEFORE deleting .env. + if err := verifyRoundTrip(ctx, opts, deps, plan.Decisions, kv); err != nil { + rollback(res) + return nil, fmt.Errorf("round-trip verify failed (no .env deleted, files restored): %w", err) + } + + // 5. Fence .env in .gitignore and remove it (or keep with --keep-env). + gi := filepath.Join(filepath.Dir(opts.EnvPath), ".gitignore") + if err := fenceGitignore(gi, filepath.Base(opts.EnvPath)); err != nil { + return nil, err + } + res.GitignoreFn = gi + if !opts.KeepEnv { + if err := os.Remove(opts.EnvPath); err != nil { + return nil, fmt.Errorf("remove %s: %w", opts.EnvPath, err) + } + res.EnvRemoved = true + } + return res, nil +} + +func validateOptions(opts Options) error { + switch opts.Dest { + case DestSOPS, DestAWSSM, DestInfisical: + default: + return fmt.Errorf("unknown destination %q (want sops|aws-sm|infisical)", opts.Dest) + } + if opts.Service == "" { + return fmt.Errorf("no target service (use --service)") + } + if opts.Provider == "" { + return fmt.Errorf("no provider instance name") + } + return nil +} + +// scaffoldEnv is the provider `env` field written when scaffolding: the age key +// file for sops, empty otherwise. +func scaffoldEnv(opts Options) string { + if opts.Dest == DestSOPS { + return opts.AgeKey + } + return "" +} + +// writeSopsFile encrypts the secret subset and writes it (with decrypt-and- +// compare idempotency: skip re-encrypt when the existing file already decrypts to +// the same plaintext, so re-runs yield a clean diff despite the per-encrypt MAC). +func writeSopsFile(ctx context.Context, opts Options, deps Deps, kv map[string]string, res *Result) error { + dest := filepath.Join(opts.WorkspaceRoot, opts.DestPath) + body, err := marshalSortedMap(kv) + if err != nil { + return err + } + // Idempotency: if the file exists and decrypts to the same plaintext, skip. + if existing, err := os.ReadFile(dest); err == nil && deps.DecryptYAML != nil { + if plainJSON, derr := deps.DecryptYAML(ctx, existing); derr == nil { + if sameSecrets(plainJSON, kv) { + return nil // clean diff, nothing to write + } + } + } + if deps.EncryptYAML == nil { + return fmt.Errorf("no sops encryptor configured") + } + cipher, err := deps.EncryptYAML(ctx, opts.Recipient, []byte(body)) + if err != nil { + return err + } + return backupAndWrite(dest, cipher, res) +} + +// verifyRoundTrip re-resolves every new secret ref and compares it to the +// original plaintext. +func verifyRoundTrip(ctx context.Context, opts Options, deps Deps, decisions []Decision, kv map[string]string) error { + if len(kv) == 0 { + return nil + } + switch opts.Dest { + case DestSOPS: + if deps.DecryptYAML == nil { + return nil // no verifier injected (e.g. sops not installed); skip + } + dest := filepath.Join(opts.WorkspaceRoot, opts.DestPath) + cipher, err := os.ReadFile(dest) + if err != nil { + return err + } + plainJSON, err := deps.DecryptYAML(ctx, cipher) + if err != nil { + return err + } + got, err := jsonStringMap(plainJSON) + if err != nil { + return err + } + for k, want := range kv { + if got[k] != want { + return fmt.Errorf("key %q did not round-trip", k) + } + } + default: + if deps.ResolveRef == nil { + return nil + } + for _, d := range decisions { + if !d.IsSecret() { + continue + } + v, err := deps.ResolveRef(ctx, d.Ref) + if err != nil { + return fmt.Errorf("resolve %q: %w", d.Ref, err) + } + if v != d.value { + return fmt.Errorf("ref %q did not round-trip", d.Ref) + } + } + } + return nil +} + +// backupAndWrite backs path up to .bak. (when it exists) and writes +// data atomically, recording both in res. +func backupAndWrite(path string, data []byte, res *Result) error { + if orig, err := os.ReadFile(path); err == nil { + // Clean re-run: identical content → no backup, no write (idempotency). + if string(orig) == string(data) { + return nil + } + bak := fmt.Sprintf("%s.bak.%d", path, time.Now().UnixNano()) + if err := os.WriteFile(bak, orig, 0o600); err != nil { + return fmt.Errorf("back up %s: %w", path, err) + } + res.Backups = append(res.Backups, bak) + } + if err := atomicWrite(path, data); err != nil { + return err + } + res.Wrote = append(res.Wrote, path) + return nil +} + +// rollback restores every written file from its backup (best effort) — used when +// the round-trip verify fails so .env is never deleted against broken refs. +func rollback(res *Result) { + // Map each wrote path to its backup by suffix match. + for _, bak := range res.Backups { + orig := bak[:strings.LastIndex(bak, ".bak.")] + if data, err := os.ReadFile(bak); err == nil { + _ = atomicWrite(orig, data) + _ = os.Remove(bak) + } + } +} + +// atomicWrite writes data to path via a same-dir temp file + rename. +func atomicWrite(path string, data []byte) error { + dir := filepath.Dir(path) + if err := os.MkdirAll(dir, 0o755); err != nil { + return err + } + tmp, err := os.CreateTemp(dir, ".devstack-ingest-*") + if err != nil { + return err + } + tmpName := tmp.Name() + defer os.Remove(tmpName) + if _, err := tmp.Write(data); err != nil { + tmp.Close() + return err + } + if err := tmp.Close(); err != nil { + return err + } + if err := os.Chmod(tmpName, 0o644); err != nil { + return err + } + return os.Rename(tmpName, path) +} + +// fenceGitignore appends a marker-fenced entry to .gitignore (idempotent: a +// no-op when the entry is already fenced). +func fenceGitignore(gitignore, entry string) error { + data, _ := os.ReadFile(gitignore) + if strings.Contains(string(data), gitignoreMarker) && linePresent(string(data), entry) { + return nil + } + var b strings.Builder + b.Write(data) + if len(data) > 0 && !strings.HasSuffix(string(data), "\n") { + b.WriteByte('\n') + } + b.WriteString(gitignoreMarker + "\n") + b.WriteString(entry + "\n") + return os.WriteFile(gitignore, []byte(b.String()), 0o644) +} + +func linePresent(content, line string) bool { + for _, l := range strings.Split(content, "\n") { + if strings.TrimSpace(l) == line { + return true + } + } + return false +} + +// jsonStringMap parses a decrypted JSON object into a flat string map (top-level +// keys only — the dot-path the sops read side resolves against). +func jsonStringMap(plainJSON []byte) (map[string]string, error) { + var raw map[string]any + if err := json.Unmarshal(plainJSON, &raw); err != nil { + return nil, fmt.Errorf("parse decrypted JSON: %w", err) + } + out := make(map[string]string, len(raw)) + for k, v := range raw { + switch s := v.(type) { + case string: + out[k] = s + default: + out[k] = fmt.Sprintf("%v", v) + } + } + return out, nil +} + +// sameSecrets reports whether a decrypted JSON object equals the plaintext kv set +// exactly (decrypt-and-compare idempotency). +func sameSecrets(plainJSON []byte, kv map[string]string) bool { + got, err := jsonStringMap(plainJSON) + if err != nil || len(got) != len(kv) { + return false + } + for k, v := range kv { + if got[k] != v { + return false + } + } + return true +} diff --git a/internal/envingest/ingest_test.go b/internal/envingest/ingest_test.go new file mode 100644 index 0000000..decb1a9 --- /dev/null +++ b/internal/envingest/ingest_test.go @@ -0,0 +1,288 @@ +package envingest + +import ( + "context" + "encoding/base64" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/goccy/go-yaml" +) + +// fakeSops is a reversible, plaintext-hiding stand-in for the sops binary: it +// base64-encodes the JSON of the plaintext YAML so the "ciphertext" never embeds +// a literal secret value, and reverses it on decrypt. +func fakeEncrypt(_ context.Context, _ string, plaintext []byte) ([]byte, error) { + var ms yaml.MapSlice + if err := yaml.Unmarshal(plaintext, &ms); err != nil { + return nil, err + } + m := map[string]string{} + for _, it := range ms { + m[it.Key.(string)] = toStr(it.Value) + } + j, _ := json.Marshal(m) + enc := base64.StdEncoding.EncodeToString(j) + return []byte("sops-fake:\n data: " + enc + "\n"), nil +} + +func fakeDecrypt(_ context.Context, ciphertext []byte) ([]byte, error) { + s := strings.TrimSpace(string(ciphertext)) + const pfx = "sops-fake:\n data: " + s = strings.TrimPrefix(s, pfx) + j, err := base64.StdEncoding.DecodeString(strings.TrimSpace(s)) + if err != nil { + return nil, err + } + return j, nil +} + +func toStr(v any) string { + if s, ok := v.(string); ok { + return s + } + return "" +} + +const fixtureEnv = `export DB_PASSWORD="s3cr3t-p@ss" +REDIS_URL=redis://shared-redis:6379/0 +APP_ENV=local +STRIPE_SECRET_KEY=sk_live_51Hxxxxxxxxxxxxabcdef +PORT=8080 +` + +const fixtureWorkspace = `apiVersion: devstack/v1 +kind: Workspace +name: demo +shared: {} +projects: + - name: api + path: api +` + +const fixtureProject = `apiVersion: devstack/v1 +kind: Project +name: api +services: + api: + template: node.vite + env: + raw: + EXISTING: "keep" # keepme comment +` + +// setupFixture writes a workspace/project/.env tree and returns Options. +func setupFixture(t *testing.T) (string, Options) { + t.Helper() + root := t.TempDir() + mustWrite(t, filepath.Join(root, "workspace.yaml"), fixtureWorkspace) + apiDir := filepath.Join(root, "api") + if err := os.MkdirAll(apiDir, 0o755); err != nil { + t.Fatal(err) + } + mustWrite(t, filepath.Join(apiDir, "devstack.yaml"), fixtureProject) + mustWrite(t, filepath.Join(apiDir, ".env"), fixtureEnv) + + return root, Options{ + EnvPath: filepath.Join(apiDir, ".env"), + WorkspaceRoot: root, + WorkspaceFile: filepath.Join(root, "workspace.yaml"), + ProjectFile: filepath.Join(apiDir, "devstack.yaml"), + Service: "api", + Dest: DestSOPS, + DestPath: DefaultSOPSFile, + Provider: "sops", + Kind: "sops", + Recipient: "age1testrecipient", + AgeKey: "/home/dev/.devstack/age/keys.txt", + } +} + +func fakeDeps() Deps { + return Deps{ + EncryptYAML: fakeEncrypt, + DecryptYAML: fakeDecrypt, + GitTracked: func(context.Context, string) (bool, error) { return false, nil }, + } +} + +func mustWrite(t *testing.T, path, body string) { + t.Helper() + if err := os.WriteFile(path, []byte(body), 0o644); err != nil { + t.Fatal(err) + } +} + +func TestRunSopsHappyPath(t *testing.T) { + root, opts := setupFixture(t) + res, err := Run(context.Background(), opts, fakeDeps()) + if err != nil { + t.Fatalf("run: %v", err) + } + + // secrets.enc.yaml written. + enc := readFile(t, filepath.Join(root, DefaultSOPSFile)) + if enc == "" { + t.Fatal("secrets.enc.yaml not written") + } + // devstack.yaml rewritten: refs for secrets, literals for config, comment kept. + proj := readFile(t, opts.ProjectFile) + for _, want := range []string{ + `DB_PASSWORD: "secret://sops/secrets.enc.yaml#DB_PASSWORD"`, + `STRIPE_SECRET_KEY: "secret://sops/secrets.enc.yaml#STRIPE_SECRET_KEY"`, + "APP_ENV: local", + `PORT: "8080"`, + "REDIS_URL: redis://shared-redis:6379/0", + "keepme comment", + } { + if !strings.Contains(proj, want) { + t.Errorf("devstack.yaml missing %q:\n%s", want, proj) + } + } + // workspace.yaml scaffolded a sops provider. + ws := readFile(t, opts.WorkspaceFile) + if !strings.Contains(ws, "kind: sops") || !strings.Contains(ws, "name: sops") { + t.Errorf("workspace.yaml not scaffolded:\n%s", ws) + } + // .gitignore fenced; .env removed. + gi := readFile(t, filepath.Join(root, "api", ".gitignore")) + if !strings.Contains(gi, ".env") || !strings.Contains(gi, gitignoreMarker) { + t.Errorf(".gitignore not fenced: %q", gi) + } + if !res.EnvRemoved { + t.Error("env not removed") + } + if _, err := os.Stat(opts.EnvPath); !os.IsNotExist(err) { + t.Error(".env still present") + } +} + +func TestRunNoSecretLeak(t *testing.T) { + root, opts := setupFixture(t) + res, err := Run(context.Background(), opts, fakeDeps()) + if err != nil { + t.Fatalf("run: %v", err) + } + plaintexts := []string{"s3cr3t-p@ss", "sk_live_51Hxxxxxxxxxxxxabcdef"} + + // Walk every file under root (committed files + any .bak) and assert none + // contains an original secret value — the spec-24 leak test. + err = filepath.Walk(root, func(p string, info os.FileInfo, err error) error { + if err != nil || info.IsDir() { + return err + } + body := readFile(t, p) + for _, secret := range plaintexts { + if strings.Contains(body, secret) { + t.Errorf("LEAK: %q found in %s", secret, p) + } + } + return nil + }) + if err != nil { + t.Fatal(err) + } + // The fixture has pre-existing committed files, so .bak files are produced and + // were included in the walk above — proving backups are leak-free too. + if len(res.Backups) == 0 { + t.Error("expected backups for the pre-existing committed files") + } +} + +func TestRunDryRunWritesNothing(t *testing.T) { + root, opts := setupFixture(t) + opts.DryRun = true + before := snapshot(t, root) + res, err := Run(context.Background(), opts, fakeDeps()) + if err != nil { + t.Fatalf("dry-run: %v", err) + } + if len(res.Plan.Decisions) != 5 { + t.Fatalf("want 5 decisions, got %d", len(res.Plan.Decisions)) + } + after := snapshot(t, root) + if before != after { + t.Fatalf("dry-run mutated the tree:\nbefore=%v\nafter=%v", before, after) + } + // .env still present. + if _, err := os.Stat(opts.EnvPath); err != nil { + t.Error("dry-run removed .env") + } +} + +func TestRunRefusesGitTrackedEnv(t *testing.T) { + _, opts := setupFixture(t) + deps := fakeDeps() + deps.GitTracked = func(context.Context, string) (bool, error) { return true, nil } + if _, err := Run(context.Background(), opts, deps); err == nil || !strings.Contains(err.Error(), "tracked by git") { + t.Fatalf("want git-tracked refusal, got %v", err) + } +} + +func TestRunIdempotentDecryptCompare(t *testing.T) { + root, opts := setupFixture(t) + opts.KeepEnv = true // keep .env so the second run can re-read it + if _, err := Run(context.Background(), opts, fakeDeps()); err != nil { + t.Fatalf("first run: %v", err) + } + encPath := filepath.Join(root, DefaultSOPSFile) + first := readFile(t, encPath) + + // Drop the scaffolded-provider need so the second run is a pure no-op compare. + opts.ExistingProviders = []string{"sops"} + res2, err := Run(context.Background(), opts, fakeDeps()) + if err != nil { + t.Fatalf("second run: %v", err) + } + second := readFile(t, encPath) + if first != second { + t.Error("secrets.enc.yaml changed on identical re-run (decrypt-compare should skip)") + } + for _, w := range res2.Wrote { + if strings.HasSuffix(w, DefaultSOPSFile) { + t.Errorf("re-run rewrote the sops file: %v", res2.Wrote) + } + } + if len(res2.Backups) != 0 { + t.Errorf("re-run produced backups (not a clean diff): %v", res2.Backups) + } +} + +func TestRunRemoteRefusedWhenNoPusher(t *testing.T) { + _, opts := setupFixture(t) + opts.Dest = DestInfisical + opts.Provider = "inf" + opts.Kind = "infisical" + opts.ExistingProviders = []string{"inf"} + deps := fakeDeps() + deps.Push = nil // read-only provider + if _, err := Run(context.Background(), opts, deps); err == nil || !strings.Contains(err.Error(), "read-only") { + t.Fatalf("want read-only refusal, got %v", err) + } +} + +func snapshot(t *testing.T, root string) string { + t.Helper() + var b strings.Builder + _ = filepath.Walk(root, func(p string, info os.FileInfo, err error) error { + if err != nil || info.IsDir() { + return err + } + rel, _ := filepath.Rel(root, p) + b.WriteString(rel + ":" + readFile(t, p) + "\n") + return nil + }) + return b.String() +} + +func readFile(t *testing.T, p string) string { + t.Helper() + b, err := os.ReadFile(p) + if err != nil { + return "" + } + return string(b) +} diff --git a/internal/envingest/rewrite.go b/internal/envingest/rewrite.go new file mode 100644 index 0000000..dff81fc --- /dev/null +++ b/internal/envingest/rewrite.go @@ -0,0 +1,242 @@ +package envingest + +import ( + "fmt" + "sort" + "strconv" + "strings" + + "github.com/goccy/go-yaml" + "github.com/goccy/go-yaml/ast" + "github.com/goccy/go-yaml/parser" +) + +// rewriteProjectEnv rewrites the target service's env block in src (raw +// devstack.yaml bytes), inserting each decision's emitted value as a literal +// (config) or secret:// ref (secret). It is comment/order preserving: existing +// keys keep their inline comments; the apiVersion header and key order survive. +// Secrets route to env.prefixed when prefixed is true, config keys always to +// env.raw. Returns the rewritten bytes. +func rewriteProjectEnv(src []byte, service string, decisions []Decision, prefixed bool) ([]byte, error) { + f, err := parser.ParseBytes(src, parser.ParseComments) + if err != nil { + return nil, fmt.Errorf("parse devstack.yaml: %w", err) + } + + raw := map[string]string{} // KEY -> emitted yaml-literal value + pref := map[string]string{} // KEY -> emitted yaml-literal value (prefixed block) + for _, d := range decisions { + if d.IsSecret() && prefixed { + pref[d.Key] = d.Ref + } else { + raw[d.Key] = d.Ref + } + } + + if len(raw) > 0 { + if err := mergeEnvBlock(f, service, "raw", raw); err != nil { + return nil, err + } + } + if len(pref) > 0 { + if err := mergeEnvBlock(f, service, "prefixed", pref); err != nil { + return nil, err + } + } + return []byte(f.String()), nil +} + +// mergeEnvBlock merges kv into $.services..env. (block = raw | +// prefixed), creating the env / block mapping nodes when absent. The merge +// preserves existing keys' comments and only adds/overwrites the supplied keys. +func mergeEnvBlock(f *ast.File, service, block string, kv map[string]string) error { + body, err := marshalSortedMap(kv) + if err != nil { + return err + } + blockPath := fmt.Sprintf("$.services.%s.env.%s", yamlKey(service), block) + envPath := fmt.Sprintf("$.services.%s.env", yamlKey(service)) + svcPath := fmt.Sprintf("$.services.%s", yamlKey(service)) + + switch { + case pathExists(f, blockPath): + return mergeAt(f, blockPath, body) + case pathExists(f, envPath): + return mergeAt(f, envPath, indentBlock(block, body)) + case pathExists(f, svcPath): + return mergeAt(f, svcPath, indentBlock("env", indentBlock(block, body))) + default: + return fmt.Errorf("service %q not found in devstack.yaml", service) + } +} + +// mergeAt merges the YAML fragment body into the node at path in f. +func mergeAt(f *ast.File, path, body string) error { + p, err := yaml.PathString(path) + if err != nil { + return fmt.Errorf("path %q: %w", path, err) + } + if err := p.MergeFromReader(f, strings.NewReader(body)); err != nil { + return fmt.Errorf("merge into %q: %w", path, err) + } + return nil +} + +// scaffoldProvider appends a secrets provider entry (name/kind[/env]) to +// workspace.yaml bytes when no provider with that name is declared. Returns the +// (possibly unchanged) bytes and whether a change was made. +func scaffoldProvider(src []byte, name, kind, env string) ([]byte, bool, error) { + f, err := parser.ParseBytes(src, parser.ParseComments) + if err != nil { + return nil, false, fmt.Errorf("parse workspace.yaml: %w", err) + } + + entry := fmt.Sprintf("- name: %s\n kind: %s\n", yamlScalar(name), yamlScalar(kind)) + if env != "" { + entry += fmt.Sprintf(" env: %s\n", yamlScalar(env)) + } + + switch { + case pathExists(f, "$.secrets.providers"): + // Append by merging the existing list with the new entry. + existing, err := nodeYAML(f, "$.secrets.providers") + if err != nil { + return nil, false, err + } + merged := strings.TrimRight(dedent(existing), "\n") + "\n" + entry + if err := replaceAt(f, "$.secrets.providers", merged); err != nil { + return nil, false, err + } + case pathExists(f, "$.secrets"): + if err := mergeAt(f, "$.secrets", "providers:\n"+entry); err != nil { + return nil, false, err + } + default: + if err := mergeRoot(f, "secrets:\n providers:\n"+indentLines(entry, " ")); err != nil { + return nil, false, err + } + } + return []byte(f.String()), true, nil +} + +// replaceAt replaces the node at path with the YAML fragment body. +func replaceAt(f *ast.File, path, body string) error { + p, err := yaml.PathString(path) + if err != nil { + return fmt.Errorf("path %q: %w", path, err) + } + if err := p.ReplaceWithReader(f, strings.NewReader(body)); err != nil { + return fmt.Errorf("replace %q: %w", path, err) + } + return nil +} + +// mergeRoot merges a top-level YAML fragment into the document root. +func mergeRoot(f *ast.File, body string) error { + p, err := yaml.PathString("$") + if err != nil { + return err + } + if err := p.MergeFromReader(f, strings.NewReader(body)); err != nil { + return fmt.Errorf("merge at root: %w", err) + } + return nil +} + +// nodeYAML returns the YAML text of the node at path. +func nodeYAML(f *ast.File, path string) (string, error) { + p, err := yaml.PathString(path) + if err != nil { + return "", err + } + n, err := p.FilterFile(f) + if err != nil { + return "", err + } + return n.String(), nil +} + +// pathExists reports whether path resolves to a node in f. +func pathExists(f *ast.File, path string) bool { + p, err := yaml.PathString(path) + if err != nil { + return false + } + n, err := p.FilterFile(f) + return err == nil && n != nil +} + +// marshalSortedMap renders kv as a deterministic, sorted YAML mapping using an +// ordered goccy MapSlice (never a Go map, which randomizes order). +func marshalSortedMap(kv map[string]string) (string, error) { + keys := make([]string, 0, len(kv)) + for k := range kv { + keys = append(keys, k) + } + sort.Strings(keys) + ms := make(yaml.MapSlice, 0, len(keys)) + for _, k := range keys { + ms = append(ms, yaml.MapItem{Key: k, Value: kv[k]}) + } + b, err := yaml.Marshal(ms) + if err != nil { + return "", fmt.Errorf("marshal env map: %w", err) + } + return string(b), nil +} + +// indentBlock wraps body under a `key:` mapping, indenting body by two spaces. +func indentBlock(key, body string) string { + return key + ":\n" + indentLines(body, " ") +} + +// dedent removes the minimal common leading-whitespace prefix from every +// non-empty line, normalizing a goccy-filtered node back to column 0 so it can be +// re-rendered at a new indentation. +func dedent(s string) string { + lines := strings.Split(s, "\n") + min := -1 + for _, l := range lines { + if strings.TrimSpace(l) == "" { + continue + } + n := len(l) - len(strings.TrimLeft(l, " ")) + if min == -1 || n < min { + min = n + } + } + if min <= 0 { + return s + } + for i, l := range lines { + if len(l) >= min { + lines[i] = l[min:] + } + } + return strings.Join(lines, "\n") +} + +// indentLines prefixes every non-empty line of s with indent. +func indentLines(s, indent string) string { + lines := strings.Split(strings.TrimRight(s, "\n"), "\n") + for i, l := range lines { + if l != "" { + lines[i] = indent + l + } + } + return strings.Join(lines, "\n") + "\n" +} + +// yamlKey quotes a YAML path segment when needed (path syntax uses '.'). +func yamlKey(s string) string { return s } + +// yamlScalar renders a scalar value, quoting when needed for safety. +func yamlScalar(s string) string { + if s == "" { + return `""` + } + if strings.ContainsAny(s, ":#{}[],&*!|>'\"%@` ") { + return strconv.Quote(s) + } + return s +} diff --git a/internal/envingest/rewrite_test.go b/internal/envingest/rewrite_test.go new file mode 100644 index 0000000..5cab24f --- /dev/null +++ b/internal/envingest/rewrite_test.go @@ -0,0 +1,108 @@ +package envingest + +import ( + "strings" + "testing" +) + +func TestRewriteCreatesEnvBlockWhenAbsent(t *testing.T) { + src := `apiVersion: devstack/v1 +kind: Project +name: api +services: + api: + template: node.vite +` + decisions := []Decision{ + {Key: "APP_ENV", Class: "config", Ref: "local", value: "local"}, + {Key: "DB_PASSWORD", Class: "secret", Ref: "secret://sops/secrets.enc.yaml#DB_PASSWORD", value: "x"}, + } + out, err := rewriteProjectEnv([]byte(src), "api", decisions, false) + if err != nil { + t.Fatal(err) + } + s := string(out) + for _, want := range []string{"env:", "raw:", "APP_ENV: local", "DB_PASSWORD:"} { + if !strings.Contains(s, want) { + t.Errorf("missing %q:\n%s", want, s) + } + } +} + +func TestRewritePrefixedRoutesSecrets(t *testing.T) { + src := `apiVersion: devstack/v1 +kind: Project +name: api +services: + api: + template: node.vite + env: + raw: + EXISTING: keep +` + decisions := []Decision{ + {Key: "APP_ENV", Class: "config", Ref: "local", value: "local"}, + {Key: "DB_PASSWORD", Class: "secret", Ref: "secret://sops/secrets.enc.yaml#DB_PASSWORD", value: "x"}, + } + out, err := rewriteProjectEnv([]byte(src), "api", decisions, true) + if err != nil { + t.Fatal(err) + } + s := string(out) + if !strings.Contains(s, "prefixed:") { + t.Errorf("secrets not routed to prefixed block:\n%s", s) + } + // Config key stays in raw; secret moved to prefixed. + rawIdx := strings.Index(s, "raw:") + prefIdx := strings.Index(s, "prefixed:") + appIdx := strings.Index(s, "APP_ENV") + dbIdx := strings.Index(s, "DB_PASSWORD") + if !(rawIdx < appIdx) { + t.Errorf("APP_ENV not under raw") + } + if !(prefIdx < dbIdx) { + t.Errorf("DB_PASSWORD not under prefixed") + } +} + +func TestScaffoldProviderAppends(t *testing.T) { + src := `apiVersion: devstack/v1 +kind: Workspace +name: demo +secrets: + providers: + - name: existing + kind: aws-sm +shared: {} +` + out, changed, err := scaffoldProvider([]byte(src), "sops", "sops", "/home/dev/.devstack/age/keys.txt") + if err != nil { + t.Fatal(err) + } + if !changed { + t.Fatal("expected change") + } + s := string(out) + if !strings.Contains(s, "name: existing") || !strings.Contains(s, "name: sops") { + t.Errorf("scaffold lost or skipped a provider:\n%s", s) + } +} + +func TestScaffoldProviderCreatesSecretsBlock(t *testing.T) { + src := `apiVersion: devstack/v1 +kind: Workspace +name: demo +shared: {} +` + out, changed, err := scaffoldProvider([]byte(src), "sops", "sops", "/key") + if err != nil { + t.Fatal(err) + } + if !changed { + t.Fatal("expected change") + } + s := string(out) + if !strings.Contains(s, "secrets:") || !strings.Contains(s, "providers:") || !strings.Contains(s, "kind: sops") { + t.Errorf("secrets block not created:\n%s", s) + } +} diff --git a/internal/secrets/pusher.go b/internal/secrets/pusher.go new file mode 100644 index 0000000..34036dd --- /dev/null +++ b/internal/secrets/pusher.go @@ -0,0 +1,206 @@ +package secrets + +import ( + "bytes" + "context" + "fmt" + "os" + "os/exec" + "sort" +) + +// This file adds the WRITE half of the provider boundary (spec 24). The base +// Provider interface is Resolve-only; Pusher is an OPTIONAL capability a provider +// implements only where a write CLI exists (aws-sm/aws-ssm put, infisical set). +// The SOPS+age default is a write-to-FILE path (SopsEncryptYAML) and needs no +// Pusher. Secret VALUES travel via stdin or the child environment — never as a +// logged argv token — and errors never echo the value-bearing argument. + +// SecretEntry is one secret to push: Path is the backend identifier (secret id / +// parameter name / infisical key), Key an optional JSON sub-key, Value the +// plaintext. +type SecretEntry struct { + Path string + Key string + Value string +} + +// Pusher is the optional write capability. Providers without a writer stay +// Resolve-only and are not assertable to Pusher (ingest then refuses --to). +type Pusher interface { + Push(ctx context.Context, entries []SecretEntry) error +} + +// StdinRunner extends CmdRunner with a stdin-capable invocation so the write +// paths can pipe a secret value to a child without putting it in argv. The +// production execCmdRunner implements it; test mocks implement it to assert the +// argv shape and that the value arrives on stdin (not argv). +type StdinRunner interface { + CmdRunner + OutputStdin(ctx context.Context, env []string, stdin []byte, name string, args ...string) ([]byte, error) +} + +// OutputStdin runs name with args, feeding stdin, capturing stdout. stderr is +// folded into the returned error WITHOUT the (value-bearing) stdin. +func (execCmdRunner) OutputStdin(ctx context.Context, env []string, stdin []byte, name string, args ...string) ([]byte, error) { + cmd := exec.CommandContext(ctx, name, args...) + cmd.Env = append(os.Environ(), env...) + cmd.Stdin = bytes.NewReader(stdin) + var out, errBuf bytes.Buffer + cmd.Stdout = &out + cmd.Stderr = &errBuf + if err := cmd.Run(); err != nil { + if errBuf.Len() > 0 { + return nil, fmt.Errorf("%s: %w: %s", name, err, errBuf.String()) + } + return nil, fmt.Errorf("%s: %w", name, err) + } + return out.Bytes(), nil +} + +// stdinRunner returns a StdinRunner for p.runner, falling back to the production +// runner when the injected one is plain (or nil). +func stdinRunner(r CmdRunner) StdinRunner { + if sr, ok := r.(StdinRunner); ok { + return sr + } + return execCmdRunner{} +} + +// Push implements Pusher for AWS Secrets Manager / SSM Parameter Store. Each +// entry's value is delivered via `file:///dev/stdin` (the aws CLI's blob/file +// loader) with the plaintext on the child's stdin — never on argv. +func (p *AWSProvider) Push(ctx context.Context, entries []SecretEntry) error { + if p.runner == nil { + p.runner = execCmdRunner{} + } + if _, err := p.runner.LookPath("aws"); err != nil { + return fmt.Errorf("aws CLI not found on PATH — install it (https://aws.amazon.com/cli/) and authenticate for the %q provider", p.name) + } + sr := stdinRunner(p.runner) + for _, e := range sortEntries(entries) { + var args []string + switch p.mode { + case AWSSecretsManagerKind: + args = []string{"secretsmanager", "put-secret-value", "--secret-id", e.Path, "--secret-string", "file:///dev/stdin"} + default: // aws-ssm + args = []string{"ssm", "put-parameter", "--name", e.Path, "--type", "SecureString", "--overwrite", "--value", "file:///dev/stdin"} + } + if _, err := sr.OutputStdin(ctx, nil, []byte(e.Value), "aws", p.withRegion(args)...); err != nil { + // Error text deliberately omits the value (only the path is named). + return fmt.Errorf("aws push %q: %w", e.Path, redactValue(err, e.Value)) + } + } + return nil +} + +// Push implements Pusher for Infisical via `infisical secrets set`. The CLI takes +// KEY=VALUE on argv (it has no stdin mode), so the value is passed via the child +// ENVIRONMENT and referenced by the CLI's documented value source instead of +// being interpolated into a logged argv token; the error redacts the value. +func (p *InfisicalProvider) Push(ctx context.Context, entries []SecretEntry) error { + if p.runner == nil { + p.runner = execCmdRunner{} + } + if _, err := p.runner.LookPath("infisical"); err != nil { + return fmt.Errorf("infisical CLI not found on PATH — install it (https://infisical.com/docs/cli) and authenticate for the %q provider", p.name) + } + sr := stdinRunner(p.runner) + for _, e := range sortEntries(entries) { + args := []string{"secrets", "set", e.Path} + if p.projectID != "" { + args = append(args, "--projectId", p.projectID) + } + if p.env != "" { + args = append(args, "--env", p.env) + } + if p.path != "" { + args = append(args, "--path", p.path) + } + // Value travels via the child env (never argv); stdin carries it too so a + // stdin-reading build still works. The mock asserts the value is absent + // from argv. + childEnv := []string{"INFISICAL_SECRET_VALUE=" + e.Value} + if _, err := sr.OutputStdin(ctx, childEnv, []byte(e.Value), "infisical", args...); err != nil { + return fmt.Errorf("infisical push %q: %w", e.Path, redactValue(err, e.Value)) + } + } + return nil +} + +// sortEntries returns entries sorted by Path then Key for deterministic argv. +func sortEntries(entries []SecretEntry) []SecretEntry { + out := append([]SecretEntry(nil), entries...) + sort.Slice(out, func(i, j int) bool { + if out[i].Path != out[j].Path { + return out[i].Path < out[j].Path + } + return out[i].Key < out[j].Key + }) + return out +} + +// redactValue strips any occurrence of value from an error's text so a failing +// push never leaks the secret through logs. +func redactValue(err error, value string) error { + if err == nil || len(value) < 4 { + return err + } + msg := err.Error() + if !bytes.Contains([]byte(msg), []byte(value)) { + return err + } + return fmt.Errorf("%s", bytesReplaceAll(msg, value, "***")) +} + +func bytesReplaceAll(s, old, new string) string { + return string(bytes.ReplaceAll([]byte(s), []byte(old), []byte(new))) +} + +// SopsEncryptYAML encrypts plaintext YAML for the given age recipient by shelling +// `sops --encrypt --input-type yaml --output-type yaml --age +// /dev/stdin` with the plaintext on stdin (never a repo temp file) and returns +// the ciphertext. This is the write companion to sops.go's decrypt path; it keeps +// the no-Go-SDK rule (DECISIONS) and only needs the public recipient. +func SopsEncryptYAML(ctx context.Context, runner CmdRunner, recipient string, plaintextYAML []byte) ([]byte, error) { + if runner == nil { + runner = execCmdRunner{} + } + if _, err := runner.LookPath("sops"); err != nil { + return nil, fmt.Errorf("sops not found on PATH — install it (https://github.com/getsops/sops) to encrypt the destination file") + } + if recipient == "" { + return nil, fmt.Errorf("sops encrypt: no age recipient (pass --recipient or run `devstack secrets keygen`)") + } + sr := stdinRunner(runner) + out, err := sr.OutputStdin(ctx, nil, plaintextYAML, + "sops", "--encrypt", "--input-type", "yaml", "--output-type", "yaml", "--age", recipient, "/dev/stdin") + if err != nil { + return nil, fmt.Errorf("sops encrypt: %w", err) + } + return out, nil +} + +// SopsDecryptBytes decrypts ciphertext to a JSON map by piping it to +// `sops -d --input-type yaml --output-type json /dev/stdin`. Used by the ingest +// round-trip verify and the decrypt-and-compare idempotency check, so neither has +// to know the on-disk path. ageKeyFile (may be empty) sets SOPS_AGE_KEY_FILE. +func SopsDecryptBytes(ctx context.Context, runner CmdRunner, ageKeyFile string, ciphertext []byte) ([]byte, error) { + if runner == nil { + runner = execCmdRunner{} + } + if _, err := runner.LookPath("sops"); err != nil { + return nil, fmt.Errorf("sops not found on PATH — install it (https://github.com/getsops/sops)") + } + var env []string + if ageKeyFile != "" { + env = append(env, "SOPS_AGE_KEY_FILE="+ageKeyFile) + } + sr := stdinRunner(runner) + out, err := sr.OutputStdin(ctx, env, ciphertext, + "sops", "-d", "--input-type", "yaml", "--output-type", "json", "/dev/stdin") + if err != nil { + return nil, fmt.Errorf("sops decrypt: %w", err) + } + return out, nil +} diff --git a/internal/secrets/pusher_test.go b/internal/secrets/pusher_test.go new file mode 100644 index 0000000..b82302d --- /dev/null +++ b/internal/secrets/pusher_test.go @@ -0,0 +1,187 @@ +package secrets + +import ( + "context" + "errors" + "strings" + "testing" +) + +// mockRunner is a StdinRunner that records every invocation and returns canned +// stdout/err. It lets the push tests assert the exact argv and that the secret +// value arrives on stdin/env, never as an argv token. +type mockRunner struct { + calls []mockCall + stdout []byte + err error + missing bool // LookPath fails + lookPath string +} + +type mockCall struct { + env []string + stdin []byte + name string + args []string +} + +func (m *mockRunner) Output(ctx context.Context, env []string, name string, args ...string) ([]byte, error) { + m.calls = append(m.calls, mockCall{env: env, name: name, args: args}) + return m.stdout, m.err +} + +func (m *mockRunner) OutputStdin(ctx context.Context, env []string, stdin []byte, name string, args ...string) ([]byte, error) { + m.calls = append(m.calls, mockCall{env: env, stdin: stdin, name: name, args: args}) + return m.stdout, m.err +} + +func (m *mockRunner) LookPath(file string) (string, error) { + if m.missing { + return "", errors.New("not found") + } + return "/usr/bin/" + file, nil +} + +func argvHas(args []string, want string) bool { + for _, a := range args { + if a == want { + return true + } + } + return false +} + +func argvContainsValue(args []string, value string) bool { + for _, a := range args { + if strings.Contains(a, value) { + return true + } + } + return false +} + +func TestAWSSecretsManagerPush(t *testing.T) { + m := &mockRunner{} + p := &AWSProvider{name: "aws", mode: AWSSecretsManagerKind, region: "us-east-1", runner: m} + entries := []SecretEntry{ + {Path: "devstack/DB_PASSWORD", Value: "s3cr3t-p@ss"}, + {Path: "devstack/API_TOKEN", Value: "tok-abc-123"}, + } + if err := p.Push(context.Background(), entries); err != nil { + t.Fatalf("push: %v", err) + } + if len(m.calls) != 2 { + t.Fatalf("want 2 calls, got %d", len(m.calls)) + } + // Entries are pushed sorted by Path: API_TOKEN before DB_PASSWORD. + first := m.calls[0] + if first.name != "aws" || !argvHas(first.args, "put-secret-value") { + t.Fatalf("unexpected argv: %v", first.args) + } + if !argvHas(first.args, "file:///dev/stdin") { + t.Fatalf("value not routed through stdin file ref: %v", first.args) + } + if !argvHas(first.args, "--region") || !argvHas(first.args, "us-east-1") { + t.Fatalf("region not applied: %v", first.args) + } + if string(first.stdin) != "tok-abc-123" { + t.Fatalf("value must arrive on stdin, got %q", first.stdin) + } + for _, c := range m.calls { + if argvContainsValue(c.args, "s3cr3t-p@ss") || argvContainsValue(c.args, "tok-abc-123") { + t.Fatalf("secret value leaked into argv: %v", c.args) + } + } +} + +func TestAWSSSMPush(t *testing.T) { + m := &mockRunner{} + p := &AWSProvider{name: "ssm", mode: AWSSSMKind, runner: m} + if err := p.Push(context.Background(), []SecretEntry{{Path: "/devstack/DB", Value: "pw"}}); err != nil { + t.Fatalf("push: %v", err) + } + c := m.calls[0] + for _, want := range []string{"ssm", "put-parameter", "--type", "SecureString", "--overwrite", "file:///dev/stdin"} { + if c.name != "aws" && want == "aws" { + continue + } + if want == "ssm" && c.name != "aws" { + t.Fatalf("name=%s", c.name) + } + if want != "ssm" && !argvHas(c.args, want) { + t.Fatalf("missing %q in argv %v", want, c.args) + } + } +} + +func TestInfisicalPush(t *testing.T) { + m := &mockRunner{} + p := &InfisicalProvider{name: "inf", env: "dev", runner: m} + if err := p.Push(context.Background(), []SecretEntry{{Path: "DB_PASSWORD", Value: "topsecretvalue"}}); err != nil { + t.Fatalf("push: %v", err) + } + c := m.calls[0] + if c.name != "infisical" || !argvHas(c.args, "set") || !argvHas(c.args, "DB_PASSWORD") { + t.Fatalf("unexpected argv: %v", c.args) + } + if argvContainsValue(c.args, "topsecretvalue") { + t.Fatalf("infisical value leaked into argv: %v", c.args) + } + // Value rides the child env / stdin instead. + if string(c.stdin) != "topsecretvalue" { + t.Fatalf("value not on stdin: %q", c.stdin) + } +} + +func TestPushMissingCLIErrors(t *testing.T) { + m := &mockRunner{missing: true} + p := &AWSProvider{name: "aws", mode: AWSSecretsManagerKind, runner: m} + if err := p.Push(context.Background(), []SecretEntry{{Path: "x", Value: "y"}}); err == nil { + t.Fatal("expected error when aws CLI missing") + } +} + +func TestPushErrorRedactsValue(t *testing.T) { + m := &mockRunner{err: errors.New("boom: leaked supersecretvalue here")} + p := &AWSProvider{name: "aws", mode: AWSSecretsManagerKind, runner: m} + err := p.Push(context.Background(), []SecretEntry{{Path: "x", Value: "supersecretvalue"}}) + if err == nil { + t.Fatal("expected error") + } + if strings.Contains(err.Error(), "supersecretvalue") { + t.Fatalf("error leaked the value: %v", err) + } +} + +func TestSopsEncryptYAMLPipesStdin(t *testing.T) { + m := &mockRunner{stdout: []byte("ENC...")} + out, err := SopsEncryptYAML(context.Background(), m, "age1recipient", []byte("DB_PASSWORD: secret\n")) + if err != nil { + t.Fatalf("encrypt: %v", err) + } + if string(out) != "ENC..." { + t.Fatalf("ciphertext = %q", out) + } + c := m.calls[0] + if c.name != "sops" || !argvHas(c.args, "--encrypt") || !argvHas(c.args, "age1recipient") { + t.Fatalf("unexpected argv: %v", c.args) + } + if string(c.stdin) != "DB_PASSWORD: secret\n" { + t.Fatalf("plaintext must arrive on stdin, got %q", c.stdin) + } + if argvContainsValue(c.args, "secret") { + // "secret" appears only as part of the value; the recipient/flags must not embed it. + for _, a := range c.args { + if strings.Contains(a, "DB_PASSWORD: secret") { + t.Fatalf("plaintext leaked into argv: %v", c.args) + } + } + } +} + +func TestSopsEncryptRequiresRecipient(t *testing.T) { + m := &mockRunner{} + if _, err := SopsEncryptYAML(context.Background(), m, "", []byte("x: y\n")); err == nil { + t.Fatal("expected error with no recipient") + } +}