feat(secrets): S3 — AWS Secrets Manager + SSM provider via the aws CLI (spec 04)

internal/secrets/aws.go

@@ -0,0 +1,172 @@
+package secrets
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strings"
+)
+
+// This file is the AWS provider (S3): Secrets Manager (`aws-sm`) and SSM
+// Parameter Store (`aws-ssm`). It shells out to the `aws` CLI — NOT the
+// aws-sdk-go, which would pull a large cloud SDK tree into the CGO_ENABLED=0
+// static binary (the same anti-bloat reasoning that keeps SOPS on its binary —
+// DECISIONS). Shelling also inherits the user's AWS config / SSO / IAM-role
+// credentials exactly as git inherits SSH, so no AWS credential ever passes
+// through devstack.
+//
+// Reference shapes:
+//   secret://<provider>/<secret-id>#<json.key>   (aws-sm: SecretString, optional JSON sub-key)
+//   secret://<provider>/<parameter-name>         (aws-ssm: parameter value; optional #json.key)
+
+// AWS provider kinds.
+const (
+	AWSSecretsManagerKind = "aws-sm"
+	AWSSSMKind            = "aws-ssm"
+)
+
+// AWSProvider resolves secrets via the `aws` CLI. mode selects the service.
+type AWSProvider struct {
+	name   string
+	mode   string // AWSSecretsManagerKind | AWSSSMKind
+	region string // --region; "" inherits the CLI's resolved default
+	runner CmdRunner
+}
+
+// AWSFactory builds an AWSProvider; the kind (aws-sm/aws-ssm) selects the mode.
+// Region comes from cfg.Region or Opts["region"]; empty lets the CLI resolve it
+// (AWS_REGION / profile / IMDS).
+func AWSFactory(cfg ProviderConfig) (Provider, error) {
+	if cfg.Kind != AWSSecretsManagerKind && cfg.Kind != AWSSSMKind {
+		return nil, fmt.Errorf("aws provider %q: unsupported kind %q", cfg.Name, cfg.Kind)
+	}
+	return &AWSProvider{
+		name:   cfg.Name,
+		mode:   cfg.Kind,
+		region: firstNonEmpty(cfg.Region, cfg.Opts["region"]),
+	}, nil
+}
+
+func (p *AWSProvider) Name() string { return p.name }
+
+// Resolve dispatches to the Secrets Manager or SSM batch resolver.
+func (p *AWSProvider) Resolve(ctx context.Context, refs []Ref) (map[string]string, error) {
+	if p.runner == nil {
+		p.runner = execCmdRunner{}
+	}
+	if _, err := p.runner.LookPath("aws"); err != nil {
+		return nil, fmt.Errorf("aws CLI not found on PATH — install it (https://aws.amazon.com/cli/) and authenticate for the %q provider", p.name)
+	}
+	switch p.mode {
+	case AWSSecretsManagerKind:
+		return p.resolveSM(ctx, refs)
+	default:
+		return p.resolveSSM(ctx, refs)
+	}
+}
+
+// resolveSM fetches each distinct secret id once (a secret may back several refs
+// via different #keys) and extracts each ref's value. A keyless ref takes the raw
+// SecretString; a #key ref parses it as JSON and walks the dot-path.
+func (p *AWSProvider) resolveSM(ctx context.Context, refs []Ref) (map[string]string, error) {
+	byID := map[string][]Ref{}
+	for _, r := range refs {
+		byID[r.Path] = append(byID[r.Path], r)
+	}
+	out := map[string]string{}
+	for _, id := range sortedKeys(byID) {
+		args := []string{"secretsmanager", "get-secret-value", "--secret-id", id, "--query", "SecretString", "--output", "text"}
+		raw, err := p.runner.Output(ctx, nil, "aws", p.withRegion(args)...)
+		if err != nil {
+			return nil, fmt.Errorf("aws-sm get-secret-value %q: %w", id, err)
+		}
+		secret := strings.TrimRight(string(raw), "\n")
+		for _, r := range byID[id] {
+			if r.Key == "" {
+				out[r.Raw] = secret
+				continue
+			}
+			v, ok := lookupJSONString(secret, r.Key)
+			if !ok {
+				return nil, fmt.Errorf("aws-sm: key %q not found in secret %q (is its value JSON?)", r.Key, id)
+			}
+			out[r.Raw] = v
+		}
+	}
+	return out, nil
+}
+
+// ssmGetParameters mirrors the `aws ssm get-parameters` JSON envelope.
+type ssmGetParameters struct {
+	Parameters []struct {
+		Name  string `json:"Name"`
+		Value string `json:"Value"`
+	} `json:"Parameters"`
+	InvalidParameters []string `json:"InvalidParameters"`
+}
+
+// resolveSSM fetches every referenced parameter in ONE batched GetParameters call
+// (with decryption). A ref may carry a #key to walk into a JSON-valued parameter.
+func (p *AWSProvider) resolveSSM(ctx context.Context, refs []Ref) (map[string]string, error) {
+	names := sortedKeys(groupByPath(refs))
+	// --names is variadic and must come LAST, so apply --region to the base first.
+	args := p.withRegion([]string{"ssm", "get-parameters", "--with-decryption", "--output", "json"})
+	args = append(append(args, "--names"), names...)
+	raw, err := p.runner.Output(ctx, nil, "aws", args...)
+	if err != nil {
+		return nil, fmt.Errorf("aws-ssm get-parameters: %w", err)
+	}
+	var resp ssmGetParameters
+	if err := json.Unmarshal(raw, &resp); err != nil {
+		return nil, fmt.Errorf("aws-ssm: parse get-parameters output: %w", err)
+	}
+	if len(resp.InvalidParameters) > 0 {
+		return nil, fmt.Errorf("aws-ssm: parameter(s) not found: %s", strings.Join(resp.InvalidParameters, ", "))
+	}
+	valueByName := make(map[string]string, len(resp.Parameters))
+	for _, par := range resp.Parameters {
+		valueByName[par.Name] = par.Value
+	}
+	out := map[string]string{}
+	for _, r := range refs {
+		v, ok := valueByName[r.Path]
+		if !ok {
+			return nil, fmt.Errorf("aws-ssm: parameter %q missing from response", r.Path)
+		}
+		if r.Key != "" {
+			sub, ok := lookupJSONString(v, r.Key)
+			if !ok {
+				return nil, fmt.Errorf("aws-ssm: key %q not found in parameter %q (is its value JSON?)", r.Key, r.Path)
+			}
+			v = sub
+		}
+		out[r.Raw] = v
+	}
+	return out, nil
+}
+
+// withRegion appends --region when configured.
+func (p *AWSProvider) withRegion(args []string) []string {
+	if p.region == "" {
+		return args
+	}
+	return append(args, "--region", p.region)
+}
+
+// groupByPath buckets refs by their backend path/identifier.
+func groupByPath(refs []Ref) map[string][]Ref {
+	out := map[string][]Ref{}
+	for _, r := range refs {
+		out[r.Path] = append(out[r.Path], r)
+	}
+	return out
+}
+
+// lookupJSONString parses s as a JSON object and walks key's dot-path to a leaf.
+func lookupJSONString(s, key string) (string, bool) {
+	var data map[string]any
+	if err := json.Unmarshal([]byte(s), &data); err != nil {
+		return "", false
+	}
+	return lookupPath(data, key)
+}

internal/secrets/aws_test.go

@@ -0,0 +1,175 @@
+package secrets
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"testing"
+)
+
+// awsFakeRunner fakes the `aws` CLI: SM returns SecretString text, SSM returns
+// the get-parameters JSON envelope.
+type awsFakeRunner struct {
+	sm      map[string]string // secret-id -> SecretString
+	ssm     map[string]string // parameter name -> value
+	missing bool              // simulate aws not on PATH
+	calls   [][]string
+}
+
+func (f *awsFakeRunner) LookPath(string) (string, error) {
+	if f.missing {
+		return "", errors.New("aws: not found")
+	}
+	return "/usr/bin/aws", nil
+}
+
+func (f *awsFakeRunner) Output(_ context.Context, _ []string, _ string, args ...string) ([]byte, error) {
+	f.calls = append(f.calls, args)
+	switch {
+	case len(args) >= 2 && args[0] == "secretsmanager" && args[1] == "get-secret-value":
+		id := argAfter(args, "--secret-id")
+		v, ok := f.sm[id]
+		if !ok {
+			return nil, fmt.Errorf("ResourceNotFoundException: %s", id)
+		}
+		return []byte(v + "\n"), nil // CLI text output has a trailing newline
+	case len(args) >= 2 && args[0] == "ssm" && args[1] == "get-parameters":
+		names := argsAfter(args, "--names")
+		var params []map[string]string
+		var invalid []string
+		for _, n := range names {
+			if v, ok := f.ssm[n]; ok {
+				params = append(params, map[string]string{"Name": n, "Value": v})
+			} else {
+				invalid = append(invalid, n)
+			}
+		}
+		b, _ := json.Marshal(map[string]any{"Parameters": params, "InvalidParameters": invalid})
+		return b, nil
+	}
+	return nil, fmt.Errorf("unexpected aws args: %v", args)
+}
+
+func argAfter(args []string, flag string) string {
+	for i, a := range args {
+		if a == flag && i+1 < len(args) {
+			return args[i+1]
+		}
+	}
+	return ""
+}
+
+func argsAfter(args []string, flag string) []string {
+	for i, a := range args {
+		if a == flag {
+			return args[i+1:]
+		}
+	}
+	return nil
+}
+
+func mustRefs(t *testing.T, raws ...string) []Ref {
+	t.Helper()
+	refs, err := Collect(raws...)
+	if err != nil {
+		t.Fatalf("collect: %v", err)
+	}
+	return refs
+}
+
+func TestAWSFactoryRejectsBadKind(t *testing.T) {
+	if _, err := AWSFactory(ProviderConfig{Name: "x", Kind: "aws-bogus"}); err == nil {
+		t.Fatal("AWSFactory must reject an unknown kind")
+	}
+}
+
+func TestAWSSecretsManagerKeylessAndJSON(t *testing.T) {
+	fr := &awsFakeRunner{sm: map[string]string{
+		"plain":   "s3cr3t",
+		"app/cfg": `{"db":{"password":"pw"}}`,
+	}}
+	p := &AWSProvider{name: "aws", mode: AWSSecretsManagerKind, runner: fr}
+
+	got, err := p.Resolve(context.Background(), mustRefs(t,
+		"secret://aws/plain",
+		"secret://aws/app/cfg#db.password",
+	))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got["secret://aws/plain"] != "s3cr3t" {
+		t.Errorf("keyless = %q, want s3cr3t", got["secret://aws/plain"])
+	}
+	if got["secret://aws/app/cfg#db.password"] != "pw" {
+		t.Errorf("json key = %q, want pw", got["secret://aws/app/cfg#db.password"])
+	}
+}
+
+func TestAWSSecretsManagerBatchesPerSecret(t *testing.T) {
+	fr := &awsFakeRunner{sm: map[string]string{"app/cfg": `{"a":"1","b":"2"}`}}
+	p := &AWSProvider{name: "aws", mode: AWSSecretsManagerKind, runner: fr}
+
+	got, err := p.Resolve(context.Background(), mustRefs(t,
+		"secret://aws/app/cfg#a",
+		"secret://aws/app/cfg#b",
+	))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got["secret://aws/app/cfg#a"] != "1" || got["secret://aws/app/cfg#b"] != "2" {
+		t.Errorf("batch values wrong: %v", got)
+	}
+	if len(fr.calls) != 1 {
+		t.Errorf("same secret fetched %d times, want 1 (batched)", len(fr.calls))
+	}
+}
+
+func TestAWSSSMBatchAndRegion(t *testing.T) {
+	fr := &awsFakeRunner{ssm: map[string]string{"/app/db": "url", "/app/key": "k"}}
+	p := &AWSProvider{name: "ssm", mode: AWSSSMKind, region: "eu-west-1", runner: fr}
+
+	got, err := p.Resolve(context.Background(), mustRefs(t,
+		"secret://ssm//app/db",
+		"secret://ssm//app/key",
+	))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got["secret://ssm//app/db"] != "url" || got["secret://ssm//app/key"] != "k" {
+		t.Errorf("ssm values wrong: %v", got)
+	}
+	if len(fr.calls) != 1 {
+		t.Errorf("ssm made %d calls, want 1 batched get-parameters", len(fr.calls))
+	}
+	if argAfter(fr.calls[0], "--region") != "eu-west-1" {
+		t.Errorf("region not passed: %v", fr.calls[0])
+	}
+}
+
+func TestAWSSSMInvalidParameter(t *testing.T) {
+	fr := &awsFakeRunner{ssm: map[string]string{"/app/db": "url"}}
+	p := &AWSProvider{name: "ssm", mode: AWSSSMKind, runner: fr}
+	if _, err := p.Resolve(context.Background(), mustRefs(t, "secret://ssm//missing")); err == nil {
+		t.Fatal("a missing SSM parameter must error")
+	}
+}
+
+func TestAWSMissingCLI(t *testing.T) {
+	p := &AWSProvider{name: "aws", mode: AWSSecretsManagerKind, runner: &awsFakeRunner{missing: true}}
+	if _, err := p.Resolve(context.Background(), mustRefs(t, "secret://aws/x")); err == nil {
+		t.Fatal("missing aws CLI must error")
+	}
+}
+
+func TestAWSRegisteredAsBuiltins(t *testing.T) {
+	reg := NewRegistry()
+	RegisterBuiltins(reg)
+	reg.Configure(ProviderConfig{Name: "sm", Kind: AWSSecretsManagerKind})
+	reg.Configure(ProviderConfig{Name: "ps", Kind: AWSSSMKind})
+	for _, n := range []string{"sm", "ps"} {
+		if _, err := reg.Provider(n); err != nil {
+			t.Errorf("provider %q not built from builtins: %v", n, err)
+		}
+	}
+}

internal/secrets/sops.go

@@ -52,6 +52,8 @@ func SopsFactory(cfg ProviderConfig) (Provider, error) {
 // SOPS+age) on a registry. AWS/Infisical register additively in S3/S4.
 func RegisterBuiltins(reg *Registry) {
 	reg.RegisterFactory(SopsKind, SopsFactory)
+	reg.RegisterFactory(AWSSecretsManagerKind, AWSFactory)
+	reg.RegisterFactory(AWSSSMKind, AWSFactory)
 }
 
 func (p *SopsProvider) Name() string { return p.name }