From 3708dd80485f4f8bae20a8933271d49be3c7b890 Mon Sep 17 00:00:00 2001 From: Gustavo Bertoi Date: Mon, 29 Jun 2026 10:56:42 -0300 Subject: [PATCH] =?UTF-8?q?feat(secrets):=20S1=20=E2=80=94=20secret://=20c?= =?UTF-8?q?ore=20(parser,=20Provider=20iface,=20registry,=20batched=20Reso?= =?UTF-8?q?lve)=20(spec=2004)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The foundation of M4, free of any concrete backend: - ParseRef for `secret:///#?opt=val` (fragment-before-query per spec 04) + IsRef. Ref keeps the Raw string — the substitution key and the Resolve result key (Ref isn't a valid map key: Opts is a map, so the spec's map[Ref]string is keyed by Raw instead). - Provider interface with a single BATCH Resolve (every backend is naturally batched); Factory + Registry mapping kind→factory and name→config, building & caching each named provider lazily with clear errors for undeclared name/unknown kind. - Resolve: group refs per provider, call each provider's Resolve exactly ONCE over its unique refs (N refs → one call), keyed by Raw; a ref a provider omits is a hard error (a missing secret never passes as empty). - Collect scans strings for distinct valid refs (the post-render pass feeds it to Resolve); a typo'd secret:// surfaces as an error. Redact scrubs resolved values from debug output (short values left alone). Concrete providers (SOPS+age S2, AWS S3, Infisical S4, keyring S5) register against this registry; the saga's secrets phase + the valueless-env coupling (S6) consume Collect→Resolve. Unit-tested with a fake provider: parse happy/error paths, collect dedupe, lazy build+cache, one-call-per-provider batching (dups collapse), missing-ref error, provider-error propagation, redaction. Co-Authored-By: Claude Opus 4.8 (1M context) --- internal/secrets/secrets.go | 233 +++++++++++++++++++++++++++++++ internal/secrets/secrets_test.go | 185 ++++++++++++++++++++++++ 2 files changed, 418 insertions(+) create mode 100644 internal/secrets/secrets.go create mode 100644 internal/secrets/secrets_test.go diff --git a/internal/secrets/secrets.go b/internal/secrets/secrets.go new file mode 100644 index 0000000..450c71d --- /dev/null +++ b/internal/secrets/secrets.go @@ -0,0 +1,233 @@ +// Package secrets resolves `secret://` references from pluggable backends and +// feeds them into containers WITHOUT writing plaintext to disk and WITHOUT +// putting secrets in committed config (spec 04, ARCHITECTURE §7.5). +// +// This file is the core (S1): the reference grammar + parser, the tiny Provider +// interface (batch Resolve), the kind→Factory registry with lazily-built named +// instances, the batched cross-provider Resolve orchestration, the ref collector +// used by the post-render pass, and a log redactor. Concrete providers (SOPS+age, +// AWS, Infisical, keyring) register against this registry (S2–S5). +package secrets + +import ( + "context" + "fmt" + "sort" + "strings" +) + +// Scheme is the reference URI scheme. +const Scheme = "secret://" + +// Ref is a parsed secret reference. Raw is the original string and is the key +// used for substitution and for Resolve results (Ref itself is not a valid map +// key — Opts is a map — so results are keyed by Raw). +// +// secret:///#?opt=val&opt2=val2 +type Ref struct { + Raw string // the exact original "secret://…" string + Provider string // named provider instance (workspace secrets.providers[].name) + Path string // backend path/identifier + Key string // optional sub-key (#key); "" if absent + Opts map[string]string // optional ?query options +} + +// IsRef reports whether s is a secret reference. +func IsRef(s string) bool { return strings.HasPrefix(s, Scheme) } + +// ParseRef parses a secret:// reference. The order is scheme, then provider, then +// path, with an optional #key and ?opts tail (fragment-before-query, per spec 04). +func ParseRef(s string) (Ref, error) { + if !IsRef(s) { + return Ref{}, fmt.Errorf("not a secret reference (must start with %q): %q", Scheme, s) + } + ref := Ref{Raw: s} + body := strings.TrimPrefix(s, Scheme) + + // Split the ?query tail first, then the #key fragment. + if before, after, found := strings.Cut(body, "?"); found { + ref.Opts = parseOpts(after) + body = before + } + if before, after, found := strings.Cut(body, "#"); found { + ref.Key = after + body = before + } + // body is now "/". + provider, path, found := strings.Cut(body, "/") + if !found { + return Ref{}, fmt.Errorf("secret ref %q is missing a provider/path separator", s) + } + ref.Provider = provider + ref.Path = path + if ref.Provider == "" { + return Ref{}, fmt.Errorf("secret ref %q has an empty provider", s) + } + if ref.Path == "" { + return Ref{}, fmt.Errorf("secret ref %q has an empty path", s) + } + return ref, nil +} + +func parseOpts(q string) map[string]string { + if q == "" { + return nil + } + out := map[string]string{} + for kv := range strings.SplitSeq(q, "&") { + if kv == "" { + continue + } + k, v, _ := strings.Cut(kv, "=") + out[k] = v + } + return out +} + +// Provider is a secrets backend. The single method is a BATCH Resolve — every +// named backend is naturally batched (Infisical List, SSM GetParameters, +// SOPS whole-file decrypt); single-key providers loop internally. The result is +// keyed by each Ref's Raw string. +type Provider interface { + Name() string + Resolve(ctx context.Context, refs []Ref) (map[string]string, error) +} + +// ProviderConfig is the typed declaration of one named provider instance +// (workspace secrets.providers[]). Kind selects the registered Factory. +type ProviderConfig struct { + Name string + Kind string // sops | aws-sm | aws-ssm | infisical | … + Env string + ProjectID string + Region string + Opts map[string]string +} + +// Factory builds a Provider from its config. Concrete backends register one per +// kind via Registry.RegisterFactory at init. +type Factory func(cfg ProviderConfig) (Provider, error) + +// Registry maps provider KINDS to factories and provider NAMES to their declared +// config, building (and caching) each named Provider lazily on first use. +type Registry struct { + factories map[string]Factory + configs map[string]ProviderConfig + built map[string]Provider +} + +// NewRegistry returns an empty registry. +func NewRegistry() *Registry { + return &Registry{ + factories: map[string]Factory{}, + configs: map[string]ProviderConfig{}, + built: map[string]Provider{}, + } +} + +// RegisterFactory registers a backend kind's factory (idempotent overwrite). +func (r *Registry) RegisterFactory(kind string, f Factory) { r.factories[kind] = f } + +// Configure declares a named provider instance (from workspace.yaml). +func (r *Registry) Configure(cfg ProviderConfig) { r.configs[cfg.Name] = cfg } + +// Provider returns the named provider, building it from its config's kind on +// first use. Errors clearly when the name is undeclared or its kind unregistered. +func (r *Registry) Provider(name string) (Provider, error) { + if p, ok := r.built[name]; ok { + return p, nil + } + cfg, ok := r.configs[name] + if !ok { + return nil, fmt.Errorf("secret provider %q is not declared in this workspace", name) + } + f, ok := r.factories[cfg.Kind] + if !ok { + return nil, fmt.Errorf("secret provider %q uses unknown kind %q (no backend registered)", name, cfg.Kind) + } + p, err := f(cfg) + if err != nil { + return nil, fmt.Errorf("build secret provider %q (kind %s): %w", name, cfg.Kind, err) + } + r.built[name] = p + return p, nil +} + +// Resolve resolves every ref by grouping them per provider and calling each +// provider's batch Resolve exactly once over its UNIQUE refs (N refs to one +// provider → one call). The result is keyed by each ref's Raw string. A provider +// that omits a requested ref yields an error (a missing secret must never pass +// silently as empty). +func Resolve(ctx context.Context, reg *Registry, refs []Ref) (map[string]string, error) { + byProvider := map[string][]Ref{} + seen := map[string]bool{} + for _, ref := range refs { + if seen[ref.Raw] { + continue + } + seen[ref.Raw] = true + byProvider[ref.Provider] = append(byProvider[ref.Provider], ref) + } + + out := map[string]string{} + for _, name := range sortedKeys(byProvider) { + p, err := reg.Provider(name) + if err != nil { + return nil, err + } + got, err := p.Resolve(ctx, byProvider[name]) + if err != nil { + return nil, fmt.Errorf("provider %q resolve: %w", name, err) + } + for _, ref := range byProvider[name] { + v, ok := got[ref.Raw] + if !ok { + return nil, fmt.Errorf("provider %q did not resolve %q", name, ref.Raw) + } + out[ref.Raw] = v + } + } + return out, nil +} + +// Collect scans the given strings and returns every distinct, valid secret +// reference found (the post-render pass feeds this to Resolve). An invalid +// secret:// string is returned as an error so a typo never silently no-ops. +func Collect(values ...string) ([]Ref, error) { + var out []Ref + seen := map[string]bool{} + for _, v := range values { + if !IsRef(v) || seen[v] { + continue + } + ref, err := ParseRef(v) + if err != nil { + return nil, err + } + seen[v] = true + out = append(out, ref) + } + return out, nil +} + +// Redact replaces every resolved secret value in s with "***" so debug output +// never leaks a secret (the redaction middleware, spec 04). Empty/short values +// are skipped to avoid mangling unrelated text. +func Redact(s string, values map[string]string) string { + for _, v := range values { + if len(v) < 4 { + continue + } + s = strings.ReplaceAll(s, v, "***") + } + return s +} + +func sortedKeys[V any](m map[string]V) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} diff --git a/internal/secrets/secrets_test.go b/internal/secrets/secrets_test.go new file mode 100644 index 0000000..f192d45 --- /dev/null +++ b/internal/secrets/secrets_test.go @@ -0,0 +1,185 @@ +package secrets + +import ( + "context" + "errors" + "testing" +) + +func TestParseRef(t *testing.T) { + cases := []struct { + in string + provider string + path string + key string + opt string // value of opt "v" if present + }{ + {"secret://infisical/prod/DB_PASSWORD", "infisical", "prod/DB_PASSWORD", "", ""}, + {"secret://aws-sm/myapp/db#password", "aws-sm", "myapp/db", "password", ""}, + {"secret://sops/secrets.enc.yaml#postgres.password", "sops", "secrets.enc.yaml", "postgres.password", ""}, + {"secret://aws-ssm/myapp/redis-url?region=eu-west-1&v=x", "aws-ssm", "myapp/redis-url", "", "x"}, + } + for _, c := range cases { + ref, err := ParseRef(c.in) + if err != nil { + t.Fatalf("ParseRef(%q): %v", c.in, err) + } + if ref.Provider != c.provider || ref.Path != c.path || ref.Key != c.key { + t.Errorf("ParseRef(%q) = %+v, want provider=%q path=%q key=%q", c.in, ref, c.provider, c.path, c.key) + } + if c.opt != "" && ref.Opts["v"] != c.opt { + t.Errorf("ParseRef(%q) opt v = %q, want %q", c.in, ref.Opts["v"], c.opt) + } + if ref.Raw != c.in { + t.Errorf("Raw = %q, want %q", ref.Raw, c.in) + } + } +} + +func TestParseRefErrors(t *testing.T) { + for _, in := range []string{ + "DB_PASSWORD", // not a ref + "secret://", // empty + "secret://onlyprovider", // no /path + "secret:///path", // empty provider + "secret://prov/", // empty path + } { + if _, err := ParseRef(in); err == nil { + t.Errorf("ParseRef(%q) should error", in) + } + } +} + +func TestIsRef(t *testing.T) { + if !IsRef("secret://a/b") || IsRef("plain") || IsRef("${ref:x}") { + t.Error("IsRef misclassified a value") + } +} + +func TestCollectDedupesAndValidates(t *testing.T) { + refs, err := Collect("plain", "secret://a/p1", "secret://a/p1", "secret://b/p2#k") + if err != nil { + t.Fatal(err) + } + if len(refs) != 2 { + t.Fatalf("Collect = %d refs, want 2 (deduped)", len(refs)) + } + if _, err := Collect("secret://bad"); err == nil { + t.Error("Collect should surface an invalid ref") + } +} + +// fakeProvider records how many times Resolve was called and with how many refs. +type fakeProvider struct { + name string + calls int + maxBatch int + vals map[string]string + err error +} + +func (f *fakeProvider) Name() string { return f.name } +func (f *fakeProvider) Resolve(_ context.Context, refs []Ref) (map[string]string, error) { + f.calls++ + if len(refs) > f.maxBatch { + f.maxBatch = len(refs) + } + if f.err != nil { + return nil, f.err + } + out := map[string]string{} + for _, r := range refs { + if v, ok := f.vals[r.Raw]; ok { + out[r.Raw] = v + } + } + return out, nil +} + +func TestRegistryLazyBuildAndCache(t *testing.T) { + reg := NewRegistry() + builds := 0 + reg.RegisterFactory("fake", func(cfg ProviderConfig) (Provider, error) { + builds++ + return &fakeProvider{name: cfg.Name}, nil + }) + reg.Configure(ProviderConfig{Name: "vault1", Kind: "fake"}) + + p1, err := reg.Provider("vault1") + if err != nil || p1.Name() != "vault1" { + t.Fatalf("Provider = %v, %v", p1, err) + } + if _, err := reg.Provider("vault1"); err != nil || builds != 1 { + t.Errorf("provider should be cached (builds=%d, want 1)", builds) + } + if _, err := reg.Provider("undeclared"); err == nil { + t.Error("undeclared provider should error") + } + reg.Configure(ProviderConfig{Name: "x", Kind: "nokind"}) + if _, err := reg.Provider("x"); err == nil { + t.Error("unknown kind should error") + } +} + +func TestResolveBatchesPerProvider(t *testing.T) { + reg := NewRegistry() + fakeA := &fakeProvider{name: "a", vals: map[string]string{ + "secret://a/p1": "v1", "secret://a/p2": "v2", + }} + fakeB := &fakeProvider{name: "b", vals: map[string]string{"secret://b/p3": "v3"}} + reg.RegisterFactory("ka", func(ProviderConfig) (Provider, error) { return fakeA, nil }) + reg.RegisterFactory("kb", func(ProviderConfig) (Provider, error) { return fakeB, nil }) + reg.Configure(ProviderConfig{Name: "a", Kind: "ka"}) + reg.Configure(ProviderConfig{Name: "b", Kind: "kb"}) + + refs, _ := Collect( + "secret://a/p1", "secret://a/p2", "secret://a/p1", // dup + "secret://b/p3", + ) + got, err := Resolve(context.Background(), reg, refs) + if err != nil { + t.Fatal(err) + } + if got["secret://a/p1"] != "v1" || got["secret://a/p2"] != "v2" || got["secret://b/p3"] != "v3" { + t.Errorf("resolved = %v", got) + } + // One call per provider; provider a saw 2 unique refs (the dup collapsed). + if fakeA.calls != 1 || fakeB.calls != 1 { + t.Errorf("calls a=%d b=%d, want 1 each", fakeA.calls, fakeB.calls) + } + if fakeA.maxBatch != 2 { + t.Errorf("provider a batch = %d, want 2", fakeA.maxBatch) + } +} + +func TestResolveMissingRefErrors(t *testing.T) { + reg := NewRegistry() + fp := &fakeProvider{name: "a", vals: map[string]string{}} // resolves nothing + reg.RegisterFactory("k", func(ProviderConfig) (Provider, error) { return fp, nil }) + reg.Configure(ProviderConfig{Name: "a", Kind: "k"}) + refs, _ := Collect("secret://a/missing") + if _, err := Resolve(context.Background(), reg, refs); err == nil { + t.Error("a ref the provider omits must error, not pass as empty") + } +} + +func TestResolvePropagatesProviderError(t *testing.T) { + reg := NewRegistry() + sentinel := errors.New("backend down") + reg.RegisterFactory("k", func(ProviderConfig) (Provider, error) { + return &fakeProvider{name: "a", err: sentinel}, nil + }) + reg.Configure(ProviderConfig{Name: "a", Kind: "k"}) + refs, _ := Collect("secret://a/p") + if _, err := Resolve(context.Background(), reg, refs); !errors.Is(err, sentinel) { + t.Errorf("provider error not propagated: %v", err) + } +} + +func TestRedact(t *testing.T) { + got := Redact("url=postgres://u:supersecret@h db=longvalue x=ab", + map[string]string{"a": "supersecret", "b": "longvalue", "c": "ab"}) + if got != "url=postgres://u:***@h db=*** x=ab" { + t.Errorf("Redact = %q (short value 'ab' must be left alone)", got) + } +}