From 8f1a6e40fc409ffb25b536b1d51c134d3752f2fd Mon Sep 17 00:00:00 2001 From: Gustavo Bertoi Date: Mon, 29 Jun 2026 15:54:23 -0300 Subject: [PATCH] =?UTF-8?q?feat(secrets):=20S3=20=E2=80=94=20AWS=20Secrets?= =?UTF-8?q?=20Manager=20+=20SSM=20provider=20via=20the=20aws=20CLI=20(spec?= =?UTF-8?q?=2004)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the `aws-sm` and `aws-ssm` provider kinds. It shells out to the `aws` CLI (NOT aws-sdk-go) — the same anti-bloat reasoning that keeps SOPS on its binary (DECISIONS): the SDK would pull a large cloud tree into the CGO_ENABLED=0 static binary. Shelling also inherits the user's AWS config / SSO / IAM-role creds the way git inherits SSH, so no AWS credential passes through devstack. - `aws-sm`: `get-secret-value` per distinct secret id (batched across refs that share an id); a keyless ref takes the raw SecretString, a `#dot.key` ref parses it as JSON and walks the path. - `aws-ssm`: ONE batched `get-parameters --with-decryption` for all referenced names; `--names` kept last (variadic) so `--region` isn't swallowed; any InvalidParameters → a clear error; optional `#key` walks a JSON-valued param. - `--region` from cfg.Region/Opts["region"]; empty lets the CLI resolve it. - Registered in RegisterBuiltins alongside sops; config passes the kind through (no allowlist), so it's declarable in workspace.yaml exactly like sops. Stdlib-only (no new deps). Unit-tested with a fake `aws` runner: SM keyless + JSON-key, per-secret batching (one call), SSM batched + region passthrough, invalid-parameter error, missing-CLI error, builtin registration. Account-gated (decision #3): logic + fakes here; real creds/localstack are a flagged human/integration step, not run in the nightly. Co-Authored-By: Claude Opus 4.8 (1M context) --- internal/secrets/aws.go | 172 ++++++++++++++++++++++++++++++++++ internal/secrets/aws_test.go | 175 +++++++++++++++++++++++++++++++++++ internal/secrets/sops.go | 2 + 3 files changed, 349 insertions(+) create mode 100644 internal/secrets/aws.go create mode 100644 internal/secrets/aws_test.go diff --git a/internal/secrets/aws.go b/internal/secrets/aws.go new file mode 100644 index 0000000..23690c9 --- /dev/null +++ b/internal/secrets/aws.go @@ -0,0 +1,172 @@ +package secrets + +import ( + "context" + "encoding/json" + "fmt" + "strings" +) + +// This file is the AWS provider (S3): Secrets Manager (`aws-sm`) and SSM +// Parameter Store (`aws-ssm`). It shells out to the `aws` CLI — NOT the +// aws-sdk-go, which would pull a large cloud SDK tree into the CGO_ENABLED=0 +// static binary (the same anti-bloat reasoning that keeps SOPS on its binary — +// DECISIONS). Shelling also inherits the user's AWS config / SSO / IAM-role +// credentials exactly as git inherits SSH, so no AWS credential ever passes +// through devstack. +// +// Reference shapes: +// secret:///# (aws-sm: SecretString, optional JSON sub-key) +// secret:/// (aws-ssm: parameter value; optional #json.key) + +// AWS provider kinds. +const ( + AWSSecretsManagerKind = "aws-sm" + AWSSSMKind = "aws-ssm" +) + +// AWSProvider resolves secrets via the `aws` CLI. mode selects the service. +type AWSProvider struct { + name string + mode string // AWSSecretsManagerKind | AWSSSMKind + region string // --region; "" inherits the CLI's resolved default + runner CmdRunner +} + +// AWSFactory builds an AWSProvider; the kind (aws-sm/aws-ssm) selects the mode. +// Region comes from cfg.Region or Opts["region"]; empty lets the CLI resolve it +// (AWS_REGION / profile / IMDS). +func AWSFactory(cfg ProviderConfig) (Provider, error) { + if cfg.Kind != AWSSecretsManagerKind && cfg.Kind != AWSSSMKind { + return nil, fmt.Errorf("aws provider %q: unsupported kind %q", cfg.Name, cfg.Kind) + } + return &AWSProvider{ + name: cfg.Name, + mode: cfg.Kind, + region: firstNonEmpty(cfg.Region, cfg.Opts["region"]), + }, nil +} + +func (p *AWSProvider) Name() string { return p.name } + +// Resolve dispatches to the Secrets Manager or SSM batch resolver. +func (p *AWSProvider) Resolve(ctx context.Context, refs []Ref) (map[string]string, error) { + if p.runner == nil { + p.runner = execCmdRunner{} + } + if _, err := p.runner.LookPath("aws"); err != nil { + return nil, fmt.Errorf("aws CLI not found on PATH — install it (https://aws.amazon.com/cli/) and authenticate for the %q provider", p.name) + } + switch p.mode { + case AWSSecretsManagerKind: + return p.resolveSM(ctx, refs) + default: + return p.resolveSSM(ctx, refs) + } +} + +// resolveSM fetches each distinct secret id once (a secret may back several refs +// via different #keys) and extracts each ref's value. A keyless ref takes the raw +// SecretString; a #key ref parses it as JSON and walks the dot-path. +func (p *AWSProvider) resolveSM(ctx context.Context, refs []Ref) (map[string]string, error) { + byID := map[string][]Ref{} + for _, r := range refs { + byID[r.Path] = append(byID[r.Path], r) + } + out := map[string]string{} + for _, id := range sortedKeys(byID) { + args := []string{"secretsmanager", "get-secret-value", "--secret-id", id, "--query", "SecretString", "--output", "text"} + raw, err := p.runner.Output(ctx, nil, "aws", p.withRegion(args)...) + if err != nil { + return nil, fmt.Errorf("aws-sm get-secret-value %q: %w", id, err) + } + secret := strings.TrimRight(string(raw), "\n") + for _, r := range byID[id] { + if r.Key == "" { + out[r.Raw] = secret + continue + } + v, ok := lookupJSONString(secret, r.Key) + if !ok { + return nil, fmt.Errorf("aws-sm: key %q not found in secret %q (is its value JSON?)", r.Key, id) + } + out[r.Raw] = v + } + } + return out, nil +} + +// ssmGetParameters mirrors the `aws ssm get-parameters` JSON envelope. +type ssmGetParameters struct { + Parameters []struct { + Name string `json:"Name"` + Value string `json:"Value"` + } `json:"Parameters"` + InvalidParameters []string `json:"InvalidParameters"` +} + +// resolveSSM fetches every referenced parameter in ONE batched GetParameters call +// (with decryption). A ref may carry a #key to walk into a JSON-valued parameter. +func (p *AWSProvider) resolveSSM(ctx context.Context, refs []Ref) (map[string]string, error) { + names := sortedKeys(groupByPath(refs)) + // --names is variadic and must come LAST, so apply --region to the base first. + args := p.withRegion([]string{"ssm", "get-parameters", "--with-decryption", "--output", "json"}) + args = append(append(args, "--names"), names...) + raw, err := p.runner.Output(ctx, nil, "aws", args...) + if err != nil { + return nil, fmt.Errorf("aws-ssm get-parameters: %w", err) + } + var resp ssmGetParameters + if err := json.Unmarshal(raw, &resp); err != nil { + return nil, fmt.Errorf("aws-ssm: parse get-parameters output: %w", err) + } + if len(resp.InvalidParameters) > 0 { + return nil, fmt.Errorf("aws-ssm: parameter(s) not found: %s", strings.Join(resp.InvalidParameters, ", ")) + } + valueByName := make(map[string]string, len(resp.Parameters)) + for _, par := range resp.Parameters { + valueByName[par.Name] = par.Value + } + out := map[string]string{} + for _, r := range refs { + v, ok := valueByName[r.Path] + if !ok { + return nil, fmt.Errorf("aws-ssm: parameter %q missing from response", r.Path) + } + if r.Key != "" { + sub, ok := lookupJSONString(v, r.Key) + if !ok { + return nil, fmt.Errorf("aws-ssm: key %q not found in parameter %q (is its value JSON?)", r.Key, r.Path) + } + v = sub + } + out[r.Raw] = v + } + return out, nil +} + +// withRegion appends --region when configured. +func (p *AWSProvider) withRegion(args []string) []string { + if p.region == "" { + return args + } + return append(args, "--region", p.region) +} + +// groupByPath buckets refs by their backend path/identifier. +func groupByPath(refs []Ref) map[string][]Ref { + out := map[string][]Ref{} + for _, r := range refs { + out[r.Path] = append(out[r.Path], r) + } + return out +} + +// lookupJSONString parses s as a JSON object and walks key's dot-path to a leaf. +func lookupJSONString(s, key string) (string, bool) { + var data map[string]any + if err := json.Unmarshal([]byte(s), &data); err != nil { + return "", false + } + return lookupPath(data, key) +} diff --git a/internal/secrets/aws_test.go b/internal/secrets/aws_test.go new file mode 100644 index 0000000..a70441e --- /dev/null +++ b/internal/secrets/aws_test.go @@ -0,0 +1,175 @@ +package secrets + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "testing" +) + +// awsFakeRunner fakes the `aws` CLI: SM returns SecretString text, SSM returns +// the get-parameters JSON envelope. +type awsFakeRunner struct { + sm map[string]string // secret-id -> SecretString + ssm map[string]string // parameter name -> value + missing bool // simulate aws not on PATH + calls [][]string +} + +func (f *awsFakeRunner) LookPath(string) (string, error) { + if f.missing { + return "", errors.New("aws: not found") + } + return "/usr/bin/aws", nil +} + +func (f *awsFakeRunner) Output(_ context.Context, _ []string, _ string, args ...string) ([]byte, error) { + f.calls = append(f.calls, args) + switch { + case len(args) >= 2 && args[0] == "secretsmanager" && args[1] == "get-secret-value": + id := argAfter(args, "--secret-id") + v, ok := f.sm[id] + if !ok { + return nil, fmt.Errorf("ResourceNotFoundException: %s", id) + } + return []byte(v + "\n"), nil // CLI text output has a trailing newline + case len(args) >= 2 && args[0] == "ssm" && args[1] == "get-parameters": + names := argsAfter(args, "--names") + var params []map[string]string + var invalid []string + for _, n := range names { + if v, ok := f.ssm[n]; ok { + params = append(params, map[string]string{"Name": n, "Value": v}) + } else { + invalid = append(invalid, n) + } + } + b, _ := json.Marshal(map[string]any{"Parameters": params, "InvalidParameters": invalid}) + return b, nil + } + return nil, fmt.Errorf("unexpected aws args: %v", args) +} + +func argAfter(args []string, flag string) string { + for i, a := range args { + if a == flag && i+1 < len(args) { + return args[i+1] + } + } + return "" +} + +func argsAfter(args []string, flag string) []string { + for i, a := range args { + if a == flag { + return args[i+1:] + } + } + return nil +} + +func mustRefs(t *testing.T, raws ...string) []Ref { + t.Helper() + refs, err := Collect(raws...) + if err != nil { + t.Fatalf("collect: %v", err) + } + return refs +} + +func TestAWSFactoryRejectsBadKind(t *testing.T) { + if _, err := AWSFactory(ProviderConfig{Name: "x", Kind: "aws-bogus"}); err == nil { + t.Fatal("AWSFactory must reject an unknown kind") + } +} + +func TestAWSSecretsManagerKeylessAndJSON(t *testing.T) { + fr := &awsFakeRunner{sm: map[string]string{ + "plain": "s3cr3t", + "app/cfg": `{"db":{"password":"pw"}}`, + }} + p := &AWSProvider{name: "aws", mode: AWSSecretsManagerKind, runner: fr} + + got, err := p.Resolve(context.Background(), mustRefs(t, + "secret://aws/plain", + "secret://aws/app/cfg#db.password", + )) + if err != nil { + t.Fatal(err) + } + if got["secret://aws/plain"] != "s3cr3t" { + t.Errorf("keyless = %q, want s3cr3t", got["secret://aws/plain"]) + } + if got["secret://aws/app/cfg#db.password"] != "pw" { + t.Errorf("json key = %q, want pw", got["secret://aws/app/cfg#db.password"]) + } +} + +func TestAWSSecretsManagerBatchesPerSecret(t *testing.T) { + fr := &awsFakeRunner{sm: map[string]string{"app/cfg": `{"a":"1","b":"2"}`}} + p := &AWSProvider{name: "aws", mode: AWSSecretsManagerKind, runner: fr} + + got, err := p.Resolve(context.Background(), mustRefs(t, + "secret://aws/app/cfg#a", + "secret://aws/app/cfg#b", + )) + if err != nil { + t.Fatal(err) + } + if got["secret://aws/app/cfg#a"] != "1" || got["secret://aws/app/cfg#b"] != "2" { + t.Errorf("batch values wrong: %v", got) + } + if len(fr.calls) != 1 { + t.Errorf("same secret fetched %d times, want 1 (batched)", len(fr.calls)) + } +} + +func TestAWSSSMBatchAndRegion(t *testing.T) { + fr := &awsFakeRunner{ssm: map[string]string{"/app/db": "url", "/app/key": "k"}} + p := &AWSProvider{name: "ssm", mode: AWSSSMKind, region: "eu-west-1", runner: fr} + + got, err := p.Resolve(context.Background(), mustRefs(t, + "secret://ssm//app/db", + "secret://ssm//app/key", + )) + if err != nil { + t.Fatal(err) + } + if got["secret://ssm//app/db"] != "url" || got["secret://ssm//app/key"] != "k" { + t.Errorf("ssm values wrong: %v", got) + } + if len(fr.calls) != 1 { + t.Errorf("ssm made %d calls, want 1 batched get-parameters", len(fr.calls)) + } + if argAfter(fr.calls[0], "--region") != "eu-west-1" { + t.Errorf("region not passed: %v", fr.calls[0]) + } +} + +func TestAWSSSMInvalidParameter(t *testing.T) { + fr := &awsFakeRunner{ssm: map[string]string{"/app/db": "url"}} + p := &AWSProvider{name: "ssm", mode: AWSSSMKind, runner: fr} + if _, err := p.Resolve(context.Background(), mustRefs(t, "secret://ssm//missing")); err == nil { + t.Fatal("a missing SSM parameter must error") + } +} + +func TestAWSMissingCLI(t *testing.T) { + p := &AWSProvider{name: "aws", mode: AWSSecretsManagerKind, runner: &awsFakeRunner{missing: true}} + if _, err := p.Resolve(context.Background(), mustRefs(t, "secret://aws/x")); err == nil { + t.Fatal("missing aws CLI must error") + } +} + +func TestAWSRegisteredAsBuiltins(t *testing.T) { + reg := NewRegistry() + RegisterBuiltins(reg) + reg.Configure(ProviderConfig{Name: "sm", Kind: AWSSecretsManagerKind}) + reg.Configure(ProviderConfig{Name: "ps", Kind: AWSSSMKind}) + for _, n := range []string{"sm", "ps"} { + if _, err := reg.Provider(n); err != nil { + t.Errorf("provider %q not built from builtins: %v", n, err) + } + } +} diff --git a/internal/secrets/sops.go b/internal/secrets/sops.go index c413d50..d6c0a02 100644 --- a/internal/secrets/sops.go +++ b/internal/secrets/sops.go @@ -52,6 +52,8 @@ func SopsFactory(cfg ProviderConfig) (Provider, error) { // SOPS+age) on a registry. AWS/Infisical register additively in S3/S4. func RegisterBuiltins(reg *Registry) { reg.RegisterFactory(SopsKind, SopsFactory) + reg.RegisterFactory(AWSSecretsManagerKind, AWSFactory) + reg.RegisterFactory(AWSSSMKind, AWSFactory) } func (p *SopsProvider) Name() string { return p.name }