diff --git a/docs/external-runtime-harness.md b/docs/external-runtime-harness.md new file mode 100644 index 0000000000..411d92c627 --- /dev/null +++ b/docs/external-runtime-harness.md @@ -0,0 +1,78 @@ +# External runtime Harnesses + +An external `Harness` compiles portable agent behavior for a Codex or Claude +Code runtime connected through the reverse external gateway. kagent persists an +immutable revision, but it does not create a `WorkerPool`, `ActorTemplate`, Pod, +or other in-cluster compute for that revision. + +This foundation must be released together with profile dispatch and a strict +local-host consumer. Until those pieces are present, `ExternalRuntimePrepared` +means only that the revision was compiled and persisted; it does not mean that +a compatible runtime slot is online or that the instruction has been applied. + +## Minimal Codex example + +```yaml +apiVersion: kagent.dev/v1alpha3 +kind: Harness +metadata: + name: codex + namespace: agents +spec: + codex: {} + allowedAgentTemplates: + selector: + matchLabels: + runtime.kagent.dev/codex: "true" +--- +apiVersion: kagent.dev/v1alpha3 +kind: AgentTemplate +metadata: + name: reviewer + namespace: agents + labels: + runtime.kagent.dev/codex: "true" +spec: + # The v1alpha3 wire format still requires this field. External compilers do + # not resolve the reference or copy model credentials into the revision. + modelConfig: + name: unused-for-external-runtime + description: Review code changes + systemPrompt: Review the requested change and report concrete findings. +``` + +Use `claude: {}` instead of `codex: {}` for Claude Code. Exactly one runtime +variant is allowed. `workload`, `substrate`, and `env` are required for the +in-cluster kagent runtime and forbidden for external runtimes. + +## Portable profile boundary + +The persisted external profile contains only: + +```json +{"version":"v1","instruction":"...","tools":[]} +``` + +Model, reasoning effort, speed, filesystem access, credentials, executable +paths, and sandbox settings remain local Agent Card policy. Cluster +`ModelConfig` values, MCP URLs, headers, TLS material, and Secret values are not +copied into the profile or revision provenance. + +External v1 profiles currently reject AgentTemplate skills, plugins, shared +agent tools, MCP headers/TLS, and empty MCP allowlists. MCP entries contain only +the logical `RemoteMCPServer` name and allowed tool names; a local host must map +that name to an explicitly configured local endpoint and fail closed when the +mapping or isolation support is unavailable. + +## Rollout constraints + +- Enable the reverse gateway only with one controller replica; the Helm chart + enforces `Recreate` because sessions are currently process-local. +- Release migration 19 with the matching controller binary. Do not run mixed + migration-18 and migration-19 controller replicas: the older generated reader + uses `SELECT *` and does not understand the added profile column. +- Adding backend identity to the revision digest produces one new revision for + each existing in-cluster pair on first reconciliation. Capacity-plan that + one-time recompilation. +- Execute the migration 19 PostgreSQL round-trip tests before merging or + deploying this feature stack. diff --git a/go/api/config/crd/bases/kagent.dev_harnesses.yaml b/go/api/config/crd/bases/kagent.dev_harnesses.yaml index 0ef8b5f9be..478c88c87c 100644 --- a/go/api/config/crd/bases/kagent.dev_harnesses.yaml +++ b/go/api/config/crd/bases/kagent.dev_harnesses.yaml @@ -163,8 +163,8 @@ spec: description: KagentHarness selects the kagent runtime adapter. type: object substrate: - description: HarnessSubstratePolicy contains the Substrate policy - shared by all runtime variants. + description: Substrate is required by kagent and forbidden by external + runtimes. properties: snapshotPolicy: description: SnapshotPolicy configures runtime snapshot storage. @@ -200,8 +200,8 @@ spec: - message: workerPoolRef name must not be empty rule: self.workerPoolRef.name.size() > 0 workload: - description: HarnessWorkload identifies the immutable runtime image - used by a Harness. + description: Workload is required by kagent and forbidden by external + runtimes. properties: image: description: Image is an OCI image reference pinned by sha256 @@ -211,14 +211,16 @@ spec: required: - image type: object - required: - - substrate - - workload type: object x-kubernetes-validations: - message: exactly one of kagent, codex, or claude must be specified rule: '(has(self.kagent) ? 1 : 0) + (has(self.codex) ? 1 : 0) + (has(self.claude) ? 1 : 0) == 1' + - message: kagent requires workload and substrate + rule: '!has(self.kagent) || (has(self.workload) && has(self.substrate))' + - message: codex and claude forbid workload, substrate, and env + rule: has(self.kagent) || (!has(self.workload) && !has(self.substrate) + && !has(self.env)) status: description: HarnessStatus reports controller-derived capabilities and current health. diff --git a/go/api/database/models.go b/go/api/database/models.go index a4b1e9db31..6d07396c86 100644 --- a/go/api/database/models.go +++ b/go/api/database/models.go @@ -1,6 +1,7 @@ package database import ( + "bytes" "encoding/json" "errors" "time" @@ -287,6 +288,7 @@ type RuntimeRevision struct { EgressDestinations []string BackendKind RuntimeBackendKind ExternalRuntime ExternalRuntime + ExternalProfile json.RawMessage ActorTemplateNamespace string ActorTemplateName string ActorTemplateUID string @@ -299,13 +301,26 @@ type RuntimeRevision struct { func (r RuntimeRevision) ValidateBackendIdentity() error { switch r.BackendKind { case RuntimeBackendKindSubstrate: - if r.ExternalRuntime != "" { - return errors.New("substrate runtime revision must not select an external runtime") + if r.ExternalRuntime != "" || len(bytes.TrimSpace(r.ExternalProfile)) != 0 { + return errors.New("substrate runtime revision must not select an external runtime or profile") + } + if r.ActorTemplateNamespace == "" || r.ActorTemplateName == "" { + return errors.New("substrate runtime revision requires an actor template identity") } case RuntimeBackendKindExternal: if r.ExternalRuntime != ExternalRuntimeCodex && r.ExternalRuntime != ExternalRuntimeClaude { return errors.New("external runtime revision must select a supported runtime") } + profile := bytes.TrimSpace(r.ExternalProfile) + if len(profile) == 0 || profile[0] != '{' || !json.Valid(profile) { + return errors.New("external runtime revision requires a JSON object profile") + } + if r.ActorTemplateNamespace != "" || r.ActorTemplateName != "" || r.ActorTemplateUID != "" { + return errors.New("external runtime revision must not select an actor template") + } + if r.Phase != "Ready" || r.GoldenSnapshot != "" { + return errors.New("external runtime revision must be ready without a golden snapshot") + } default: return errors.New("runtime revision backend kind is invalid") } diff --git a/go/api/database/models_runtime_revision_test.go b/go/api/database/models_runtime_revision_test.go index 2dc581303a..0d23ec9117 100644 --- a/go/api/database/models_runtime_revision_test.go +++ b/go/api/database/models_runtime_revision_test.go @@ -8,26 +8,50 @@ import ( ) func TestRuntimeRevisionValidateBackendIdentity(t *testing.T) { + substrate := dbpkg.RuntimeRevision{ + BackendKind: dbpkg.RuntimeBackendKindSubstrate, ActorTemplateNamespace: "team-a", ActorTemplateName: "actor", + } + external := dbpkg.RuntimeRevision{ + BackendKind: dbpkg.RuntimeBackendKindExternal, ExternalRuntime: dbpkg.ExternalRuntimeCodex, + ExternalProfile: []byte(`{"version":"v1"}`), Phase: "Ready", + } tests := []struct { - name string - kind dbpkg.RuntimeBackendKind - runtime dbpkg.ExternalRuntime - wantErr string + name string + revision dbpkg.RuntimeRevision + wantErr string }{ - {name: "substrate", kind: dbpkg.RuntimeBackendKindSubstrate}, - {name: "external codex", kind: dbpkg.RuntimeBackendKindExternal, runtime: dbpkg.ExternalRuntimeCodex}, - {name: "external claude", kind: dbpkg.RuntimeBackendKindExternal, runtime: dbpkg.ExternalRuntimeClaude}, - {name: "missing kind", wantErr: "backend kind is invalid"}, - {name: "unknown kind", kind: dbpkg.RuntimeBackendKind("credential-shaped-unknown"), wantErr: "backend kind is invalid"}, - {name: "substrate with runtime", kind: dbpkg.RuntimeBackendKindSubstrate, runtime: dbpkg.ExternalRuntimeCodex, wantErr: "must not select"}, - {name: "external missing runtime", kind: dbpkg.RuntimeBackendKindExternal, wantErr: "supported runtime"}, - {name: "external unknown runtime", kind: dbpkg.RuntimeBackendKindExternal, runtime: dbpkg.ExternalRuntime("credential-shaped-unknown"), wantErr: "supported runtime"}, + {name: "substrate", revision: substrate}, + {name: "external codex", revision: external}, + {name: "external claude", revision: func() dbpkg.RuntimeRevision { + value := external + value.ExternalRuntime = dbpkg.ExternalRuntimeClaude + return value + }()}, + {name: "missing kind", revision: dbpkg.RuntimeRevision{}, wantErr: "backend kind is invalid"}, + {name: "unknown kind", revision: dbpkg.RuntimeRevision{BackendKind: dbpkg.RuntimeBackendKind("credential-shaped-unknown")}, wantErr: "backend kind is invalid"}, + {name: "substrate missing actor", revision: dbpkg.RuntimeRevision{BackendKind: dbpkg.RuntimeBackendKindSubstrate}, wantErr: "actor template identity"}, + {name: "substrate with runtime", revision: func() dbpkg.RuntimeRevision { + value := substrate + value.ExternalRuntime = dbpkg.ExternalRuntimeCodex + return value + }(), wantErr: "must not select"}, + {name: "substrate with profile", revision: func() dbpkg.RuntimeRevision { value := substrate; value.ExternalProfile = []byte(`{}`); return value }(), wantErr: "must not select"}, + {name: "external missing runtime", revision: func() dbpkg.RuntimeRevision { value := external; value.ExternalRuntime = ""; return value }(), wantErr: "supported runtime"}, + {name: "external unknown runtime", revision: func() dbpkg.RuntimeRevision { + value := external + value.ExternalRuntime = dbpkg.ExternalRuntime("credential-shaped-unknown") + return value + }(), wantErr: "supported runtime"}, + {name: "external missing profile", revision: func() dbpkg.RuntimeRevision { value := external; value.ExternalProfile = nil; return value }(), wantErr: "JSON object profile"}, + {name: "external array profile", revision: func() dbpkg.RuntimeRevision { value := external; value.ExternalProfile = []byte(`[]`); return value }(), wantErr: "JSON object profile"}, + {name: "external with actor", revision: func() dbpkg.RuntimeRevision { value := external; value.ActorTemplateName = "actor"; return value }(), wantErr: "must not select an actor"}, + {name: "external not ready", revision: func() dbpkg.RuntimeRevision { value := external; value.Phase = "Pending"; return value }(), wantErr: "must be ready"}, + {name: "external with snapshot", revision: func() dbpkg.RuntimeRevision { value := external; value.GoldenSnapshot = "snapshot"; return value }(), wantErr: "without a golden snapshot"}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - revision := dbpkg.RuntimeRevision{BackendKind: test.kind, ExternalRuntime: test.runtime} - err := revision.ValidateBackendIdentity() + err := test.revision.ValidateBackendIdentity() if test.wantErr == "" { require.NoError(t, err) return diff --git a/go/api/v1alpha3/configuration_crd_cel_test.go b/go/api/v1alpha3/configuration_crd_cel_test.go index 952537660a..fcd7d2592e 100644 --- a/go/api/v1alpha3/configuration_crd_cel_test.go +++ b/go/api/v1alpha3/configuration_crd_cel_test.go @@ -71,10 +71,56 @@ func TestConfigurationCRDValidation(t *testing.T) { name: "Harness rejects tag-only image", object: validHarness(namespace, "harness-tagged-image", HarnessSpec{ Kagent: &KagentHarness{}, - Workload: HarnessWorkload{Image: "registry.example.com/kagent:latest"}, + Workload: &HarnessWorkload{Image: "registry.example.com/kagent:latest"}, }), wantReject: "spec.workload.image", }, + { + name: "kagent Harness requires workload", + object: &Harness{ObjectMeta: metav1.ObjectMeta{Name: "kagent-no-workload", Namespace: namespace}, Spec: HarnessSpec{ + Kagent: &KagentHarness{}, + Substrate: &HarnessSubstratePolicy{ + WorkerPoolRef: corev1.LocalObjectReference{Name: "default"}, + SnapshotPolicy: HarnessSnapshotPolicy{Location: "gs://snapshots/kagent"}, + }, + }}, + wantReject: "kagent requires workload and substrate", + }, + { + name: "kagent Harness requires substrate", + object: &Harness{ObjectMeta: metav1.ObjectMeta{Name: "kagent-no-substrate", Namespace: namespace}, Spec: HarnessSpec{ + Kagent: &KagentHarness{}, + Workload: &HarnessWorkload{Image: "registry.example.com/kagent@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}, + }}, + wantReject: "kagent requires workload and substrate", + }, + { + name: "Codex Harness forbids workload", + object: &Harness{ObjectMeta: metav1.ObjectMeta{Name: "codex-workload", Namespace: namespace}, Spec: HarnessSpec{ + Codex: &CodexHarness{}, + Workload: &HarnessWorkload{Image: "registry.example.com/codex@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}, + }}, + wantReject: "codex and claude forbid workload, substrate, and env", + }, + { + name: "Claude Harness forbids substrate", + object: &Harness{ObjectMeta: metav1.ObjectMeta{Name: "claude-substrate", Namespace: namespace}, Spec: HarnessSpec{ + Claude: &ClaudeHarness{}, + Substrate: &HarnessSubstratePolicy{ + WorkerPoolRef: corev1.LocalObjectReference{Name: "default"}, + SnapshotPolicy: HarnessSnapshotPolicy{Location: "gs://snapshots/claude"}, + }, + }}, + wantReject: "codex and claude forbid workload, substrate, and env", + }, + { + name: "Codex Harness forbids env", + object: &Harness{ObjectMeta: metav1.ObjectMeta{Name: "codex-env", Namespace: namespace}, Spec: HarnessSpec{ + Codex: &CodexHarness{}, + Env: []HarnessEnvVar{{Name: "TOKEN", Value: &empty}}, + }}, + wantReject: "codex and claude forbid workload, substrate, and env", + }, { name: "Harness env requires a value source", object: validHarness(namespace, "harness-empty-env", HarnessSpec{ @@ -99,7 +145,12 @@ func TestConfigurationCRDValidation(t *testing.T) { name: "valid Harness", object: validHarness(namespace, "valid-harness", HarnessSpec{ Claude: &ClaudeHarness{}, - Env: []HarnessEnvVar{{Name: "EMPTY", Value: &empty}}, + }), + }, + { + name: "valid kagent Harness", + object: validHarness(namespace, "valid-kagent-harness", HarnessSpec{ + Kagent: &KagentHarness{}, }), }, { @@ -139,14 +190,16 @@ func TestConfigurationCRDValidation(t *testing.T) { } func validHarness(namespace, name string, overrides HarnessSpec) *Harness { - if overrides.Workload.Image == "" { - overrides.Workload.Image = "registry.example.com/kagent@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" - } - if overrides.Substrate.WorkerPoolRef.Name == "" { - overrides.Substrate.WorkerPoolRef.Name = "default" - } - if overrides.Substrate.SnapshotPolicy.Location == "" { - overrides.Substrate.SnapshotPolicy.Location = "gs://snapshots/kagent" + if overrides.Kagent != nil { + if overrides.Workload == nil { + overrides.Workload = &HarnessWorkload{Image: "registry.example.com/kagent@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"} + } + if overrides.Substrate == nil { + overrides.Substrate = &HarnessSubstratePolicy{ + WorkerPoolRef: corev1.LocalObjectReference{Name: "default"}, + SnapshotPolicy: HarnessSnapshotPolicy{Location: "gs://snapshots/kagent"}, + } + } } return &Harness{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, Spec: overrides} } diff --git a/go/api/v1alpha3/harness_types.go b/go/api/v1alpha3/harness_types.go index 9652ca4fa0..242db678ab 100644 --- a/go/api/v1alpha3/harness_types.go +++ b/go/api/v1alpha3/harness_types.go @@ -89,6 +89,8 @@ type HarnessAgentTemplateAdmission struct { // HarnessSpec defines a reusable runtime and its infrastructure policy. // // +kubebuilder:validation:XValidation:rule="(has(self.kagent) ? 1 : 0) + (has(self.codex) ? 1 : 0) + (has(self.claude) ? 1 : 0) == 1",message="exactly one of kagent, codex, or claude must be specified" +// +kubebuilder:validation:XValidation:rule="!has(self.kagent) || (has(self.workload) && has(self.substrate))",message="kagent requires workload and substrate" +// +kubebuilder:validation:XValidation:rule="has(self.kagent) || (!has(self.workload) && !has(self.substrate) && !has(self.env))",message="codex and claude forbid workload, substrate, and env" type HarnessSpec struct { // +optional Kagent *KagentHarness `json:"kagent,omitempty"` @@ -99,8 +101,9 @@ type HarnessSpec struct { // +optional Claude *ClaudeHarness `json:"claude,omitempty"` - // +required - Workload HarnessWorkload `json:"workload"` + // Workload is required by kagent and forbidden by external runtimes. + // +optional + Workload *HarnessWorkload `json:"workload,omitempty"` // +optional // +kubebuilder:validation:MaxItems=100 @@ -108,8 +111,9 @@ type HarnessSpec struct { // +listMapKey=name Env []HarnessEnvVar `json:"env,omitempty"` - // +required - Substrate HarnessSubstratePolicy `json:"substrate"` + // Substrate is required by kagent and forbidden by external runtimes. + // +optional + Substrate *HarnessSubstratePolicy `json:"substrate,omitempty"` // AllowedAgentTemplates selects AgentTemplates this Harness admits. // When omitted, the Harness admits none. diff --git a/go/api/v1alpha3/zz_generated.deepcopy.go b/go/api/v1alpha3/zz_generated.deepcopy.go index a42d4f8f64..5c4b7f2792 100644 --- a/go/api/v1alpha3/zz_generated.deepcopy.go +++ b/go/api/v1alpha3/zz_generated.deepcopy.go @@ -1422,7 +1422,11 @@ func (in *HarnessSpec) DeepCopyInto(out *HarnessSpec) { *out = new(ClaudeHarness) **out = **in } - out.Workload = in.Workload + if in.Workload != nil { + in, out := &in.Workload, &out.Workload + *out = new(HarnessWorkload) + **out = **in + } if in.Env != nil { in, out := &in.Env, &out.Env *out = make([]HarnessEnvVar, len(*in)) @@ -1430,7 +1434,11 @@ func (in *HarnessSpec) DeepCopyInto(out *HarnessSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } - out.Substrate = in.Substrate + if in.Substrate != nil { + in, out := &in.Substrate, &out.Substrate + *out = new(HarnessSubstratePolicy) + **out = **in + } if in.AllowedAgentTemplates != nil { in, out := &in.AllowedAgentTemplates, &out.AllowedAgentTemplates *out = new(HarnessAgentTemplateAdmission) diff --git a/go/core/cmd/controller-v2/runtime_router_test.go b/go/core/cmd/controller-v2/runtime_router_test.go index 813e8bb5ed..82926545fc 100644 --- a/go/core/cmd/controller-v2/runtime_router_test.go +++ b/go/core/cmd/controller-v2/runtime_router_test.go @@ -46,9 +46,11 @@ func TestDisabledExternalGatewayBuildsSubstrateOnlyRouter(t *testing.T) { store := controllerRevisionStoreFunc(func(_ context.Context, revision string) (*dbpkg.RuntimeRevision, error) { switch revision { case "existing-substrate-revision": - return &dbpkg.RuntimeRevision{BackendKind: dbpkg.RuntimeBackendKindSubstrate}, nil + value := controllerSubstrateRevision() + return &value, nil case "external-revision": - return &dbpkg.RuntimeRevision{BackendKind: dbpkg.RuntimeBackendKindExternal, ExternalRuntime: dbpkg.ExternalRuntimeCodex}, nil + value := controllerExternalRevision(dbpkg.ExternalRuntimeCodex) + return &value, nil default: return nil, nil } @@ -78,7 +80,8 @@ func TestDisabledExternalGatewayBuildsSubstrateOnlyRouter(t *testing.T) { func TestEnabledExternalGatewayAddsExplicitExternalBackend(t *testing.T) { store := controllerRevisionStoreFunc(func(context.Context, string) (*dbpkg.RuntimeRevision, error) { - return &dbpkg.RuntimeRevision{BackendKind: dbpkg.RuntimeBackendKindExternal, ExternalRuntime: dbpkg.ExternalRuntimeClaude}, nil + value := controllerExternalRevision(dbpkg.ExternalRuntimeClaude) + return &value, nil }) substrate := &controllerBackendStub{} external := &controllerBackendStub{} @@ -98,3 +101,16 @@ func TestEnabledExternalGatewayAddsExplicitExternalBackend(t *testing.T) { t.Fatalf("wrong backend selected: external=%d substrate=%d", external.createCalls, substrate.createCalls) } } + +func controllerSubstrateRevision() dbpkg.RuntimeRevision { + return dbpkg.RuntimeRevision{ + BackendKind: dbpkg.RuntimeBackendKindSubstrate, ActorTemplateNamespace: "team-a", ActorTemplateName: "actor", + } +} + +func controllerExternalRevision(runtime dbpkg.ExternalRuntime) dbpkg.RuntimeRevision { + return dbpkg.RuntimeRevision{ + BackendKind: dbpkg.RuntimeBackendKindExternal, ExternalRuntime: runtime, + ExternalProfile: []byte(`{"version":"v1","instruction":"","tools":[]}`), Phase: "Ready", + } +} diff --git a/go/core/internal/database/client_postgres.go b/go/core/internal/database/client_postgres.go index 8b062c04db..6eab7a9d19 100644 --- a/go/core/internal/database/client_postgres.go +++ b/go/core/internal/database/client_postgres.go @@ -32,6 +32,11 @@ type postgresClient struct { db *pgxpool.Pool } +// externalActorTemplateNamespace is a database-only compatibility sentinel. +// Version 18 readers require non-null actor identity columns, while external +// revisions never select or address a Kubernetes ActorTemplate. +const externalActorTemplateNamespace = "_external" + func NewClient(db *pgxpool.Pool) dbpkg.Client { return &postgresClient{ q: dbgen.New(db), @@ -373,15 +378,26 @@ func (c *postgresClient) UpsertRuntimeRevision(ctx context.Context, revision dbp if err := revision.ValidateBackendIdentity(); err != nil { return &runtimeRevisionPersistenceError{operation: "validate backend identity", cause: err} } + egressDestinations := revision.EgressDestinations + if egressDestinations == nil { + egressDestinations = []string{} + } + actorTemplateNamespace := revision.ActorTemplateNamespace + actorTemplateName := revision.ActorTemplateName + if revision.BackendKind == dbpkg.RuntimeBackendKindExternal { + actorTemplateNamespace = externalActorTemplateNamespace + actorTemplateName = revision.Revision + } rowsAffected, err := c.q.UpsertRuntimeRevision(ctx, dbgen.UpsertRuntimeRevisionParams{ Revision: revision.Revision, Namespace: revision.Namespace, AgentTemplateName: revision.AgentTemplateName, AgentTemplateUid: revision.AgentTemplateUID, HarnessName: revision.HarnessName, HarnessUid: revision.HarnessUID, SourceSnapshot: revision.SourceSnapshot, AgentCard: revision.AgentCard, - EgressDestinations: revision.EgressDestinations, + EgressDestinations: egressDestinations, BackendKind: string(revision.BackendKind), ExternalRuntime: externalRuntimeDBValue(revision.ExternalRuntime), - ActorTemplateNamespace: revision.ActorTemplateNamespace, ActorTemplateName: revision.ActorTemplateName, + ExternalProfile: revision.ExternalProfile, + ActorTemplateNamespace: actorTemplateNamespace, ActorTemplateName: actorTemplateName, ActorTemplateUid: revision.ActorTemplateUID, Phase: revision.Phase, GoldenSnapshot: revision.GoldenSnapshot, }) if err != nil { @@ -407,10 +423,17 @@ func (c *postgresClient) GetRuntimeRevision(ctx context.Context, revision string func (c *postgresClient) MarkRuntimeRevisionSuccessful(ctx context.Context, pair dbpkg.AgentTemplateHarnessPair) error { revision := pair.DesiredRevision - return c.q.MarkRuntimeRevisionSuccessful(ctx, dbgen.MarkRuntimeRevisionSuccessfulParams{ + rowsAffected, err := c.q.MarkRuntimeRevisionSuccessful(ctx, dbgen.MarkRuntimeRevisionSuccessfulParams{ Revision: &revision, Namespace: pair.Namespace, AgentTemplateUid: pair.AgentTemplateUID, HarnessUid: pair.HarnessUID, }) + if err != nil { + return err + } + if rowsAffected != 1 { + return errors.New("runtime revision is no longer the active desired revision") + } + return nil } func (c *postgresClient) RetireAgentTemplateHarnessPairs(ctx context.Context, namespace, name string) error { @@ -444,6 +467,16 @@ func (c *postgresClient) ListUnreferencedRuntimeRevisions(ctx context.Context) ( } func runtimeRevisionFromRow(row dbgen.RuntimeRevision) (dbpkg.RuntimeRevision, error) { + actorTemplateNamespace := row.ActorTemplateNamespace + actorTemplateName := row.ActorTemplateName + if dbpkg.RuntimeBackendKind(row.BackendKind) == dbpkg.RuntimeBackendKindExternal { + if actorTemplateNamespace != externalActorTemplateNamespace || actorTemplateName != row.Revision { + return dbpkg.RuntimeRevision{}, &runtimeRevisionPersistenceError{ + operation: "decode backend identity", cause: errors.New("external runtime revision has an invalid compatibility sentinel"), + } + } + actorTemplateNamespace, actorTemplateName = "", "" + } revision := dbpkg.RuntimeRevision{ Revision: row.Revision, Namespace: row.Namespace, AgentTemplateName: row.AgentTemplateName, AgentTemplateUID: row.AgentTemplateUid, @@ -452,7 +485,8 @@ func runtimeRevisionFromRow(row dbgen.RuntimeRevision) (dbpkg.RuntimeRevision, e EgressDestinations: row.EgressDestinations, BackendKind: dbpkg.RuntimeBackendKind(row.BackendKind), ExternalRuntime: externalRuntimeFromDB(row.ExternalRuntime), - ActorTemplateNamespace: row.ActorTemplateNamespace, ActorTemplateName: row.ActorTemplateName, + ExternalProfile: row.ExternalProfile, + ActorTemplateNamespace: actorTemplateNamespace, ActorTemplateName: actorTemplateName, ActorTemplateUID: row.ActorTemplateUid, Phase: row.Phase, GoldenSnapshot: row.GoldenSnapshot, } if err := revision.ValidateBackendIdentity(); err != nil { diff --git a/go/core/internal/database/gen/agent_instances.sql.go b/go/core/internal/database/gen/agent_instances.sql.go index 300967c093..df201a6ed2 100644 --- a/go/core/internal/database/gen/agent_instances.sql.go +++ b/go/core/internal/database/gen/agent_instances.sql.go @@ -199,7 +199,7 @@ func (q *Queries) GetAgentInstanceShareByTokenHash(ctx context.Context, tokenHas } const getLatestRuntimeRevisionForInstance = `-- name: GetLatestRuntimeRevisionForInstance :one -SELECT r.revision, r.namespace, r.agent_template_name, r.agent_template_uid, r.harness_name, r.harness_uid, r.source_snapshot, r.egress_destinations, r.actor_template_namespace, r.actor_template_name, r.actor_template_uid, r.phase, r.golden_snapshot, r.created_at, r.updated_at, r.agent_card, r.backend_kind, r.external_runtime, p.agent_template_labels +SELECT r.revision, r.namespace, r.agent_template_name, r.agent_template_uid, r.harness_name, r.harness_uid, r.source_snapshot, r.egress_destinations, r.actor_template_namespace, r.actor_template_name, r.actor_template_uid, r.phase, r.golden_snapshot, r.created_at, r.updated_at, r.agent_card, r.backend_kind, r.external_runtime, r.external_profile, p.agent_template_labels FROM agent_template_harness_pair p JOIN runtime_revision r ON r.revision = p.latest_successful_revision WHERE p.namespace = $1 @@ -233,6 +233,7 @@ type GetLatestRuntimeRevisionForInstanceRow struct { AgentCard []byte BackendKind string ExternalRuntime *string + ExternalProfile []byte AgentTemplateLabels []byte } @@ -258,6 +259,7 @@ func (q *Queries) GetLatestRuntimeRevisionForInstance(ctx context.Context, arg G &i.AgentCard, &i.BackendKind, &i.ExternalRuntime, + &i.ExternalProfile, &i.AgentTemplateLabels, ) return i, err diff --git a/go/core/internal/database/gen/models.go b/go/core/internal/database/gen/models.go index 0e787a9953..b9468cfc5b 100644 --- a/go/core/internal/database/gen/models.go +++ b/go/core/internal/database/gen/models.go @@ -227,6 +227,7 @@ type RuntimeRevision struct { AgentCard []byte BackendKind string ExternalRuntime *string + ExternalProfile []byte } type Session struct { diff --git a/go/core/internal/database/gen/querier.go b/go/core/internal/database/gen/querier.go index 01c332d983..bd8f9a033c 100644 --- a/go/core/internal/database/gen/querier.go +++ b/go/core/internal/database/gen/querier.go @@ -124,7 +124,7 @@ type Querier interface { LockAgentInstance(ctx context.Context, id string) (AgentInstance, error) LockReadyAgentInstanceCheckpoint(ctx context.Context, arg LockReadyAgentInstanceCheckpointParams) (AgentInstanceCheckpoint, error) MarkAgentInstanceReady(ctx context.Context, arg MarkAgentInstanceReadyParams) (AgentInstance, error) - MarkRuntimeRevisionSuccessful(ctx context.Context, arg MarkRuntimeRevisionSuccessfulParams) error + MarkRuntimeRevisionSuccessful(ctx context.Context, arg MarkRuntimeRevisionSuccessfulParams) (int64, error) RetireAgentTemplateHarnessPair(ctx context.Context, arg RetireAgentTemplateHarnessPairParams) error RetireAgentTemplateHarnessPairs(ctx context.Context, arg RetireAgentTemplateHarnessPairsParams) error RetireOtherAgentTemplateHarnessPairs(ctx context.Context, arg RetireOtherAgentTemplateHarnessPairsParams) error diff --git a/go/core/internal/database/gen/runtime_revisions.sql.go b/go/core/internal/database/gen/runtime_revisions.sql.go index 52ea272b42..81a52ae909 100644 --- a/go/core/internal/database/gen/runtime_revisions.sql.go +++ b/go/core/internal/database/gen/runtime_revisions.sql.go @@ -28,7 +28,7 @@ func (q *Queries) DeleteUnreferencedRuntimeRevision(ctx context.Context, revisio } const getRuntimeRevision = `-- name: GetRuntimeRevision :one -SELECT revision, namespace, agent_template_name, agent_template_uid, harness_name, harness_uid, source_snapshot, egress_destinations, actor_template_namespace, actor_template_name, actor_template_uid, phase, golden_snapshot, created_at, updated_at, agent_card, backend_kind, external_runtime FROM runtime_revision WHERE revision = $1 +SELECT revision, namespace, agent_template_name, agent_template_uid, harness_name, harness_uid, source_snapshot, egress_destinations, actor_template_namespace, actor_template_name, actor_template_uid, phase, golden_snapshot, created_at, updated_at, agent_card, backend_kind, external_runtime, external_profile FROM runtime_revision WHERE revision = $1 ` func (q *Queries) GetRuntimeRevision(ctx context.Context, revision string) (RuntimeRevision, error) { @@ -53,12 +53,13 @@ func (q *Queries) GetRuntimeRevision(ctx context.Context, revision string) (Runt &i.AgentCard, &i.BackendKind, &i.ExternalRuntime, + &i.ExternalProfile, ) return i, err } const listUnreferencedRuntimeRevisions = `-- name: ListUnreferencedRuntimeRevisions :many -SELECT revision, namespace, agent_template_name, agent_template_uid, harness_name, harness_uid, source_snapshot, egress_destinations, actor_template_namespace, actor_template_name, actor_template_uid, phase, golden_snapshot, created_at, updated_at, agent_card, backend_kind, external_runtime FROM runtime_revision r +SELECT revision, namespace, agent_template_name, agent_template_uid, harness_name, harness_uid, source_snapshot, egress_destinations, actor_template_namespace, actor_template_name, actor_template_uid, phase, golden_snapshot, created_at, updated_at, agent_card, backend_kind, external_runtime, external_profile FROM runtime_revision r WHERE NOT EXISTS ( SELECT 1 FROM agent_template_harness_pair p WHERE p.retired_at IS NULL @@ -97,6 +98,7 @@ func (q *Queries) ListUnreferencedRuntimeRevisions(ctx context.Context) ([]Runti &i.AgentCard, &i.BackendKind, &i.ExternalRuntime, + &i.ExternalProfile, ); err != nil { return nil, err } @@ -108,7 +110,7 @@ func (q *Queries) ListUnreferencedRuntimeRevisions(ctx context.Context) ([]Runti return items, nil } -const markRuntimeRevisionSuccessful = `-- name: MarkRuntimeRevisionSuccessful :exec +const markRuntimeRevisionSuccessful = `-- name: MarkRuntimeRevisionSuccessful :execrows UPDATE agent_template_harness_pair SET latest_successful_revision = $1, updated_at = NOW() WHERE namespace = $2 @@ -125,14 +127,17 @@ type MarkRuntimeRevisionSuccessfulParams struct { HarnessUid string } -func (q *Queries) MarkRuntimeRevisionSuccessful(ctx context.Context, arg MarkRuntimeRevisionSuccessfulParams) error { - _, err := q.db.Exec(ctx, markRuntimeRevisionSuccessful, +func (q *Queries) MarkRuntimeRevisionSuccessful(ctx context.Context, arg MarkRuntimeRevisionSuccessfulParams) (int64, error) { + result, err := q.db.Exec(ctx, markRuntimeRevisionSuccessful, arg.Revision, arg.Namespace, arg.AgentTemplateUid, arg.HarnessUid, ) - return err + if err != nil { + return 0, err + } + return result.RowsAffected(), nil } const retireAgentTemplateHarnessPair = `-- name: RetireAgentTemplateHarnessPair :exec @@ -228,21 +233,31 @@ const upsertRuntimeRevision = `-- name: UpsertRuntimeRevision :execrows INSERT INTO runtime_revision ( revision, namespace, agent_template_name, agent_template_uid, harness_name, harness_uid, source_snapshot, agent_card, egress_destinations, - backend_kind, external_runtime, + backend_kind, external_runtime, external_profile, actor_template_namespace, actor_template_name, actor_template_uid, phase, golden_snapshot ) VALUES ( $1, $2, $3, $4, $5, $6, $7, $8, - $9, $10, $11, $12, $13, $14, $15, $16 + $9, $10, $11, $12, $13, $14, $15, $16, $17 ) ON CONFLICT (revision) DO UPDATE SET - agent_card = EXCLUDED.agent_card, actor_template_uid = EXCLUDED.actor_template_uid, phase = EXCLUDED.phase, golden_snapshot = EXCLUDED.golden_snapshot, updated_at = NOW() -WHERE runtime_revision.backend_kind = EXCLUDED.backend_kind - AND COALESCE(runtime_revision.external_runtime, '') = COALESCE(EXCLUDED.external_runtime, '') +WHERE runtime_revision.namespace = EXCLUDED.namespace + AND runtime_revision.agent_template_name = EXCLUDED.agent_template_name + AND runtime_revision.agent_template_uid = EXCLUDED.agent_template_uid + AND runtime_revision.harness_name = EXCLUDED.harness_name + AND runtime_revision.harness_uid = EXCLUDED.harness_uid + AND runtime_revision.source_snapshot = EXCLUDED.source_snapshot + AND runtime_revision.agent_card = EXCLUDED.agent_card + AND runtime_revision.egress_destinations = EXCLUDED.egress_destinations + AND runtime_revision.backend_kind = EXCLUDED.backend_kind + AND runtime_revision.external_runtime IS NOT DISTINCT FROM EXCLUDED.external_runtime + AND runtime_revision.external_profile IS NOT DISTINCT FROM EXCLUDED.external_profile + AND runtime_revision.actor_template_namespace IS NOT DISTINCT FROM EXCLUDED.actor_template_namespace + AND runtime_revision.actor_template_name IS NOT DISTINCT FROM EXCLUDED.actor_template_name ` type UpsertRuntimeRevisionParams struct { @@ -257,6 +272,7 @@ type UpsertRuntimeRevisionParams struct { EgressDestinations []string BackendKind string ExternalRuntime *string + ExternalProfile []byte ActorTemplateNamespace string ActorTemplateName string ActorTemplateUid string @@ -277,6 +293,7 @@ func (q *Queries) UpsertRuntimeRevision(ctx context.Context, arg UpsertRuntimeRe arg.EgressDestinations, arg.BackendKind, arg.ExternalRuntime, + arg.ExternalProfile, arg.ActorTemplateNamespace, arg.ActorTemplateName, arg.ActorTemplateUid, diff --git a/go/core/internal/database/queries/runtime_revisions.sql b/go/core/internal/database/queries/runtime_revisions.sql index 5118c2a6d8..9ad632edd2 100644 --- a/go/core/internal/database/queries/runtime_revisions.sql +++ b/go/core/internal/database/queries/runtime_revisions.sql @@ -15,23 +15,33 @@ ON CONFLICT (namespace, agent_template_uid, harness_uid) DO UPDATE SET INSERT INTO runtime_revision ( revision, namespace, agent_template_name, agent_template_uid, harness_name, harness_uid, source_snapshot, agent_card, egress_destinations, - backend_kind, external_runtime, + backend_kind, external_runtime, external_profile, actor_template_namespace, actor_template_name, actor_template_uid, phase, golden_snapshot ) VALUES ( $1, $2, $3, $4, $5, $6, $7, $8, - $9, $10, $11, $12, $13, $14, $15, $16 + $9, $10, $11, $12, $13, $14, $15, $16, $17 ) ON CONFLICT (revision) DO UPDATE SET - agent_card = EXCLUDED.agent_card, actor_template_uid = EXCLUDED.actor_template_uid, phase = EXCLUDED.phase, golden_snapshot = EXCLUDED.golden_snapshot, updated_at = NOW() -WHERE runtime_revision.backend_kind = EXCLUDED.backend_kind - AND COALESCE(runtime_revision.external_runtime, '') = COALESCE(EXCLUDED.external_runtime, ''); +WHERE runtime_revision.namespace = EXCLUDED.namespace + AND runtime_revision.agent_template_name = EXCLUDED.agent_template_name + AND runtime_revision.agent_template_uid = EXCLUDED.agent_template_uid + AND runtime_revision.harness_name = EXCLUDED.harness_name + AND runtime_revision.harness_uid = EXCLUDED.harness_uid + AND runtime_revision.source_snapshot = EXCLUDED.source_snapshot + AND runtime_revision.agent_card = EXCLUDED.agent_card + AND runtime_revision.egress_destinations = EXCLUDED.egress_destinations + AND runtime_revision.backend_kind = EXCLUDED.backend_kind + AND runtime_revision.external_runtime IS NOT DISTINCT FROM EXCLUDED.external_runtime + AND runtime_revision.external_profile IS NOT DISTINCT FROM EXCLUDED.external_profile + AND runtime_revision.actor_template_namespace IS NOT DISTINCT FROM EXCLUDED.actor_template_namespace + AND runtime_revision.actor_template_name IS NOT DISTINCT FROM EXCLUDED.actor_template_name; --- name: MarkRuntimeRevisionSuccessful :exec +-- name: MarkRuntimeRevisionSuccessful :execrows UPDATE agent_template_harness_pair SET latest_successful_revision = sqlc.arg(revision), updated_at = NOW() WHERE namespace = sqlc.arg(namespace) diff --git a/go/core/internal/database/runtime_revision_test.go b/go/core/internal/database/runtime_revision_test.go index 9924e02293..f87f709e8d 100644 --- a/go/core/internal/database/runtime_revision_test.go +++ b/go/core/internal/database/runtime_revision_test.go @@ -25,11 +25,21 @@ func TestRuntimeRevisionBackendIdentityPersistence(t *testing.T) { external := testRuntimeRevision("external-revision", "external-actor") external.BackendKind = dbpkg.RuntimeBackendKindExternal external.ExternalRuntime = dbpkg.ExternalRuntimeCodex + external.ExternalProfile = []byte(`{"version":"v1","instruction":"help","tools":[]}`) + external.ActorTemplateNamespace = "" + external.ActorTemplateName = "" + external.ActorTemplateUID = "" + external.EgressDestinations = nil require.NoError(t, client.UpsertRuntimeRevision(t.Context(), external)) storedExternal, err := client.GetRuntimeRevision(t.Context(), external.Revision) require.NoError(t, err) assert.Equal(t, dbpkg.RuntimeBackendKindExternal, storedExternal.BackendKind) assert.Equal(t, dbpkg.ExternalRuntimeCodex, storedExternal.ExternalRuntime) + assert.JSONEq(t, string(external.ExternalProfile), string(storedExternal.ExternalProfile)) + assert.Empty(t, storedExternal.ActorTemplateNamespace) + assert.Empty(t, storedExternal.ActorTemplateName) + assert.NotNil(t, storedExternal.EgressDestinations) + assert.Empty(t, storedExternal.EgressDestinations) listed, err := client.ListUnreferencedRuntimeRevisions(t.Context()) require.NoError(t, err) @@ -78,6 +88,10 @@ func TestUpsertRuntimeRevisionDoesNotChangePersistedBackendIdentity(t *testing.T revision.BackendKind = dbpkg.RuntimeBackendKindExternal revision.ExternalRuntime = dbpkg.ExternalRuntimeClaude + revision.ExternalProfile = []byte(`{"version":"v1","instruction":"help","tools":[]}`) + revision.ActorTemplateNamespace = "" + revision.ActorTemplateName = "" + revision.ActorTemplateUID = "" err := client.UpsertRuntimeRevision(t.Context(), revision) require.ErrorContains(t, err, "failed to upsert") @@ -87,6 +101,59 @@ func TestUpsertRuntimeRevisionDoesNotChangePersistedBackendIdentity(t *testing.T assert.Empty(t, stored.ExternalRuntime) } +func TestUpsertRuntimeRevisionDoesNotChangePersistedExternalProfile(t *testing.T) { + client := NewClient(setupTestDB(t)) + revision := testRuntimeRevision("immutable-external-revision", "unused-actor") + revision.BackendKind = dbpkg.RuntimeBackendKindExternal + revision.ExternalRuntime = dbpkg.ExternalRuntimeCodex + revision.ExternalProfile = []byte(`{"version":"v1","instruction":"first","tools":[]}`) + revision.ActorTemplateNamespace = "" + revision.ActorTemplateName = "" + revision.ActorTemplateUID = "" + require.NoError(t, client.UpsertRuntimeRevision(t.Context(), revision)) + + revision.ExternalProfile = []byte(`{"version":"v1","instruction":"second","tools":[]}`) + err := client.UpsertRuntimeRevision(t.Context(), revision) + require.ErrorContains(t, err, "failed to upsert") + + stored, err := client.GetRuntimeRevision(t.Context(), revision.Revision) + require.NoError(t, err) + assert.JSONEq(t, `{"version":"v1","instruction":"first","tools":[]}`, string(stored.ExternalProfile)) +} + +func TestUpsertRuntimeRevisionDoesNotChangeDigestOwnedFields(t *testing.T) { + tests := []struct { + name string + mutate func(*dbpkg.RuntimeRevision) + }{ + {name: "agent card", mutate: func(revision *dbpkg.RuntimeRevision) { revision.AgentCard = []byte(`{"name":"changed"}`) }}, + {name: "source snapshot", mutate: func(revision *dbpkg.RuntimeRevision) { revision.SourceSnapshot = []byte(`{"changed":true}`) }}, + {name: "egress destinations", mutate: func(revision *dbpkg.RuntimeRevision) { revision.EgressDestinations = []string{"changed.example.test"} }}, + {name: "harness uid", mutate: func(revision *dbpkg.RuntimeRevision) { revision.HarnessUID = "changed-harness-uid" }}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + client := NewClient(setupTestDB(t)) + revision := testRuntimeRevision("immutable-"+test.name, "actor-"+test.name) + revision.BackendKind = dbpkg.RuntimeBackendKindSubstrate + require.NoError(t, client.UpsertRuntimeRevision(t.Context(), revision)) + + changed := revision + test.mutate(&changed) + err := client.UpsertRuntimeRevision(t.Context(), changed) + require.ErrorContains(t, err, "failed to upsert") + + stored, err := client.GetRuntimeRevision(t.Context(), revision.Revision) + require.NoError(t, err) + assert.JSONEq(t, string(revision.AgentCard), string(stored.AgentCard)) + assert.JSONEq(t, string(revision.SourceSnapshot), string(stored.SourceSnapshot)) + assert.Equal(t, revision.EgressDestinations, stored.EgressDestinations) + assert.Equal(t, revision.HarnessUID, stored.HarnessUID) + }) + } +} + func TestRuntimeRevisionMappingRejectsUnknownIdentityWithoutLeakingIt(t *testing.T) { const credential = "credential-shaped-unknown" credentialValue := credential @@ -98,6 +165,16 @@ func TestRuntimeRevisionMappingRejectsUnknownIdentityWithoutLeakingIt(t *testing assert.NotContains(t, err.Error(), credential) } +func TestRuntimeRevisionMappingRejectsInvalidExternalCompatibilitySentinel(t *testing.T) { + runtime := string(dbpkg.ExternalRuntimeCodex) + _, err := runtimeRevisionFromRow(dbgen.RuntimeRevision{ + Revision: "external-revision", BackendKind: string(dbpkg.RuntimeBackendKindExternal), ExternalRuntime: &runtime, + ExternalProfile: []byte(`{"version":"v1"}`), ActorTemplateNamespace: "team-a", ActorTemplateName: "actor", + Phase: "Ready", + }) + require.ErrorContains(t, err, "failed to decode backend identity") +} + func TestUpsertRuntimeRevisionPropagatesContextIdentity(t *testing.T) { client := NewClient(setupTestDB(t)) revision := testRuntimeRevision("cancelled-revision", "cancelled-actor") diff --git a/go/core/internal/grpcserver/agenttemplate_harness_test.go b/go/core/internal/grpcserver/agenttemplate_harness_test.go index 81f47f64ab..5867720f29 100644 --- a/go/core/internal/grpcserver/agenttemplate_harness_test.go +++ b/go/core/internal/grpcserver/agenttemplate_harness_test.go @@ -81,9 +81,18 @@ func testHarness(namespace, name, workerPool string) *v1alpha3.Harness { return &v1alpha3.Harness{ ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: name}, Spec: v1alpha3.HarnessSpec{ - Codex: &v1alpha3.CodexHarness{}, - Workload: v1alpha3.HarnessWorkload{Image: testHarnessImage}, - Substrate: v1alpha3.HarnessSubstratePolicy{ + Codex: &v1alpha3.CodexHarness{}, + }, + } +} + +func testKagentHarness(namespace, name, workerPool string) *v1alpha3.Harness { + return &v1alpha3.Harness{ + ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: name}, + Spec: v1alpha3.HarnessSpec{ + Kagent: &v1alpha3.KagentHarness{}, + Workload: &v1alpha3.HarnessWorkload{Image: testHarnessImage}, + Substrate: &v1alpha3.HarnessSubstratePolicy{ WorkerPoolRef: corev1.LocalObjectReference{Name: workerPool}, SnapshotPolicy: v1alpha3.HarnessSnapshotPolicy{Location: "s3://snapshots"}, }, @@ -218,13 +227,13 @@ func TestHarnessServiceGeneratedClient(t *testing.T) { created, err := client.CreateHarness(ctx, &apiv1alpha1.CreateHarnessRequest{ Ref: ref, - Resource: structured(t, testHarness("team", "a-created", "pool-a"), harnessKind), + Resource: structured(t, testKagentHarness("team", "a-created", "pool-a"), harnessKind), }) if err != nil { t.Fatalf("CreateHarness() error = %v", err) } - if got := created.GetHarness(); got.GetRuntime() != harnessRuntimeCodex || got.GetWorkloadImage() != testHarnessImage { - t.Fatalf("CreateHarness() = %+v, want codex runtime and pinned image", got) + if got := created.GetHarness(); got.GetRuntime() != harnessRuntimeKagent || got.GetWorkloadImage() != testHarnessImage { + t.Fatalf("CreateHarness() = %+v, want kagent runtime and pinned image", got) } if created.GetHarness().GetReady() { t.Fatal("CreateHarness() ready = true, want false before the controller observes it") @@ -232,7 +241,7 @@ func TestHarnessServiceGeneratedClient(t *testing.T) { _, err = client.CreateHarness(ctx, &apiv1alpha1.CreateHarnessRequest{ Ref: ref, - Resource: structured(t, testHarness("team", "a-created", "pool-a"), harnessKind), + Resource: structured(t, testKagentHarness("team", "a-created", "pool-a"), harnessKind), }) assertCode(t, err, codes.AlreadyExists) @@ -243,6 +252,9 @@ func TestHarnessServiceGeneratedClient(t *testing.T) { if len(listed.GetHarnesses()) != 2 { t.Fatalf("ListHarnesses() count = %d, want 2", len(listed.GetHarnesses())) } + if got := listed.GetHarnesses()[1]; got.GetRuntime() != harnessRuntimeCodex || got.GetWorkloadImage() != "" { + t.Fatalf("ListHarnesses()[1] = %+v, want external runtime without workload image", got) + } if !listed.GetHarnesses()[1].GetReady() { t.Fatal("ListHarnesses()[1].ready = false, want the Ready condition reflected") } diff --git a/go/core/internal/grpcserver/harness.go b/go/core/internal/grpcserver/harness.go index 5753d98615..611ceb1310 100644 --- a/go/core/internal/grpcserver/harness.go +++ b/go/core/internal/grpcserver/harness.go @@ -82,11 +82,15 @@ func (s *harnessServer) harness(object *v1alpha3.Harness) (*apiv1alpha1.Harness, if err != nil { return nil, serviceerrors.NewInternal("Failed to encode Harness resource", err) } + workloadImage := "" + if object.Spec.Workload != nil { + workloadImage = object.Spec.Workload.Image + } return &apiv1alpha1.Harness{ Ref: &apiv1alpha1.ResourceReference{Namespace: object.Namespace, Name: object.Name}, Resource: resource, Runtime: harnessRuntime(object), - WorkloadImage: object.Spec.Workload.Image, + WorkloadImage: workloadImage, Ready: meta.IsStatusConditionTrue(object.Status.Conditions, v1alpha3.HarnessConditionTypeReady), }, nil } diff --git a/go/core/pkg/migrations/core/000019_runtime_revision_external_profile.down.sql b/go/core/pkg/migrations/core/000019_runtime_revision_external_profile.down.sql new file mode 100644 index 0000000000..dbbf234e7a --- /dev/null +++ b/go/core/pkg/migrations/core/000019_runtime_revision_external_profile.down.sql @@ -0,0 +1,35 @@ +DO $$ +BEGIN + IF EXISTS ( + SELECT 1 + FROM information_schema.columns + WHERE table_schema = current_schema() + AND table_name = 'runtime_revision' + AND column_name = 'external_profile' + ) THEN + EXECUTE $query$ + UPDATE runtime_revision + SET source_snapshot = jsonb_build_object( + '__kagent_external_profile_compat_v1', + jsonb_build_object( + 'externalRuntime', external_runtime, + 'externalProfile', external_profile, + 'sourceSnapshot', source_snapshot + ) + ) + WHERE backend_kind = 'external' + $query$; + END IF; +END +$$; + +ALTER TABLE runtime_revision + DROP CONSTRAINT IF EXISTS runtime_revision_backend_identity_check, + DROP CONSTRAINT IF EXISTS runtime_revision_external_runtime_check, + ADD CONSTRAINT runtime_revision_external_runtime_check + CHECK ( + (backend_kind = 'substrate' AND COALESCE(external_runtime, '') = '') + OR + (backend_kind = 'external' AND external_runtime IN ('codex', 'claude')) + ), + DROP COLUMN IF EXISTS external_profile; diff --git a/go/core/pkg/migrations/core/000019_runtime_revision_external_profile.up.sql b/go/core/pkg/migrations/core/000019_runtime_revision_external_profile.up.sql new file mode 100644 index 0000000000..ea64be6bee --- /dev/null +++ b/go/core/pkg/migrations/core/000019_runtime_revision_external_profile.up.sql @@ -0,0 +1,70 @@ +DO $$ +BEGIN + IF EXISTS ( + SELECT 1 + FROM runtime_revision + WHERE ( + backend_kind = 'external' + OR ( + actor_template_namespace = '_external' + AND actor_template_name = revision + AND actor_template_uid = '' + ) + ) + AND NOT COALESCE(( + jsonb_typeof(source_snapshot -> '__kagent_external_profile_compat_v1') = 'object' + AND (source_snapshot #>> '{__kagent_external_profile_compat_v1,externalRuntime}') IN ('codex', 'claude') + AND jsonb_typeof(source_snapshot #> '{__kagent_external_profile_compat_v1,externalProfile}') = 'object' + AND source_snapshot #> '{__kagent_external_profile_compat_v1,sourceSnapshot}' IS NOT NULL + ), FALSE) + ) THEN + RAISE EXCEPTION 'cannot add external runtime profiles: version 18 contains external revisions without recoverable profiles'; + END IF; +END +$$; + +ALTER TABLE runtime_revision + ADD COLUMN IF NOT EXISTS external_profile JSONB; + +UPDATE runtime_revision +SET backend_kind = 'external', + external_runtime = source_snapshot #>> '{__kagent_external_profile_compat_v1,externalRuntime}', + external_profile = source_snapshot #> '{__kagent_external_profile_compat_v1,externalProfile}', + source_snapshot = source_snapshot #> '{__kagent_external_profile_compat_v1,sourceSnapshot}' +WHERE actor_template_namespace = '_external' + AND actor_template_name = revision + AND actor_template_uid = '' + AND jsonb_typeof(source_snapshot -> '__kagent_external_profile_compat_v1') = 'object' + AND (source_snapshot #>> '{__kagent_external_profile_compat_v1,externalRuntime}') IN ('codex', 'claude') + AND jsonb_typeof(source_snapshot #> '{__kagent_external_profile_compat_v1,externalProfile}') = 'object' + AND source_snapshot #> '{__kagent_external_profile_compat_v1,sourceSnapshot}' IS NOT NULL; + +UPDATE runtime_revision +SET external_runtime = NULL +WHERE backend_kind = 'substrate' + AND external_runtime = ''; + +ALTER TABLE runtime_revision + DROP CONSTRAINT IF EXISTS runtime_revision_external_runtime_check, + DROP CONSTRAINT IF EXISTS runtime_revision_backend_identity_check, + ADD CONSTRAINT runtime_revision_backend_identity_check + CHECK ( + ( + backend_kind = 'substrate' + AND external_runtime IS NULL + AND external_profile IS NULL + ) + OR + ( + backend_kind = 'external' + AND external_runtime IS NOT NULL + AND external_runtime IN ('codex', 'claude') + AND external_profile IS NOT NULL + AND jsonb_typeof(external_profile) = 'object' + AND actor_template_namespace = '_external' + AND actor_template_name = revision + AND actor_template_uid = '' + AND phase = 'Ready' + AND golden_snapshot = '' + ) + ); diff --git a/go/core/pkg/migrations/migration_000019_test.go b/go/core/pkg/migrations/migration_000019_test.go new file mode 100644 index 0000000000..5c7dac9a9c --- /dev/null +++ b/go/core/pkg/migrations/migration_000019_test.go @@ -0,0 +1,287 @@ +package migrations + +import ( + "context" + "database/sql" + "fmt" + "testing" +) + +func TestMigration000019ExternalRuntimeProfile(t *testing.T) { + connStr := startTestDB(t) + migrateCoreTo(t, connStr, 18) + + db, err := sql.Open("pgx", connStr) + if err != nil { + t.Fatalf("open db: %v", err) + } + defer db.Close() + ctx := context.Background() + + if _, err := db.ExecContext(ctx, ` + INSERT INTO runtime_revision ( + revision, namespace, agent_template_name, agent_template_uid, + harness_name, harness_uid, source_snapshot, agent_card, + backend_kind, external_runtime, + actor_template_namespace, actor_template_name, phase + ) VALUES ( + 'legacy-empty-runtime', 'team-a', 'assistant', 'template-uid', + 'kagent', 'harness-uid', '{}', '{}', + 'substrate', '', 'team-a', 'legacy-empty-runtime', 'Ready' + ) + `); err != nil { + t.Fatalf("insert version 18 substrate revision: %v", err) + } + + migrateCoreTo(t, connStr, 19) + + var externalRuntime *string + if err := db.QueryRowContext(ctx, ` + SELECT external_runtime + FROM runtime_revision + WHERE revision = 'legacy-empty-runtime' + `).Scan(&externalRuntime); err != nil { + t.Fatalf("read normalized substrate runtime: %v", err) + } + if externalRuntime != nil { + t.Fatalf("substrate external_runtime = %q, want NULL", *externalRuntime) + } + + tests := []struct { + name string + backendKind any + externalRuntime any + externalProfile any + actorNamespace any + actorName any + actorUID any + phase any + goldenSnapshot any + wantValid bool + }{ + {name: "substrate", backendKind: "substrate", actorNamespace: "team-a", actorName: "actor-substrate", actorUID: "actor-uid", phase: "Ready", goldenSnapshot: "snapshot", wantValid: true}, + {name: "substrate empty runtime", backendKind: "substrate", externalRuntime: "", actorNamespace: "team-a", actorName: "actor-empty", phase: "Ready"}, + {name: "substrate runtime", backendKind: "substrate", externalRuntime: "codex", actorNamespace: "team-a", actorName: "actor-runtime", phase: "Ready"}, + {name: "substrate profile", backendKind: "substrate", externalProfile: `{}`, actorNamespace: "team-a", actorName: "actor-profile", phase: "Ready"}, + {name: "external codex", backendKind: "external", externalRuntime: "codex", externalProfile: `{"version":"v1"}`, actorNamespace: "_external", actorName: "revision-4", actorUID: "", phase: "Ready", goldenSnapshot: "", wantValid: true}, + {name: "external claude", backendKind: "external", externalRuntime: "claude", externalProfile: `{"version":"v1"}`, actorNamespace: "_external", actorName: "revision-5", actorUID: "", phase: "Ready", goldenSnapshot: "", wantValid: true}, + {name: "external null runtime", backendKind: "external", externalProfile: `{}`, actorNamespace: "_external", actorName: "revision-6", actorUID: "", phase: "Ready", goldenSnapshot: ""}, + {name: "external missing profile", backendKind: "external", externalRuntime: "codex", actorNamespace: "_external", actorName: "revision-7", actorUID: "", phase: "Ready", goldenSnapshot: ""}, + {name: "external array profile", backendKind: "external", externalRuntime: "codex", externalProfile: `[]`, actorNamespace: "_external", actorName: "revision-8", actorUID: "", phase: "Ready", goldenSnapshot: ""}, + {name: "external real actor", backendKind: "external", externalRuntime: "codex", externalProfile: `{}`, actorNamespace: "team-a", actorName: "revision-9", actorUID: "", phase: "Ready", goldenSnapshot: ""}, + {name: "external wrong sentinel name", backendKind: "external", externalRuntime: "codex", externalProfile: `{}`, actorNamespace: "_external", actorName: "other", actorUID: "", phase: "Ready", goldenSnapshot: ""}, + {name: "external pending", backendKind: "external", externalRuntime: "codex", externalProfile: `{}`, actorNamespace: "_external", actorName: "revision-11", actorUID: "", phase: "Pending", goldenSnapshot: ""}, + } + for index, test := range tests { + t.Run(test.name, func(t *testing.T) { + revision := fmt.Sprintf("revision-%d", index) + _, err := db.ExecContext(ctx, ` + INSERT INTO runtime_revision ( + revision, namespace, agent_template_name, agent_template_uid, + harness_name, harness_uid, source_snapshot, agent_card, + backend_kind, external_runtime, external_profile, + actor_template_namespace, actor_template_name, actor_template_uid, + phase, golden_snapshot + ) VALUES ($1, 'team-a', 'assistant', 'template-uid', + 'kagent', 'harness-uid', '{}', '{}', $2, $3, $4, + $5, $6, $7, $8, $9) + `, revision, test.backendKind, test.externalRuntime, test.externalProfile, + test.actorNamespace, test.actorName, test.actorUID, test.phase, test.goldenSnapshot) + if test.wantValid && err != nil { + t.Fatalf("valid identity rejected: %v", err) + } + if !test.wantValid && err == nil { + t.Fatal("invalid identity was accepted") + } + }) + } + + if _, err := db.ExecContext(ctx, ` + INSERT INTO runtime_revision ( + revision, namespace, agent_template_name, agent_template_uid, + harness_name, harness_uid, source_snapshot, agent_card, + backend_kind, actor_template_namespace, actor_template_name, phase + ) VALUES ( + 'duplicate-substrate', 'team-a', 'assistant', 'template-uid', + 'kagent', 'harness-uid', '{}', '{}', + 'substrate', 'team-a', 'actor-substrate', 'Ready' + ) + `); err == nil { + t.Fatal("duplicate substrate ActorTemplate identity was accepted") + } + + migrateCoreTo(t, connStr, 18) + assertColumnExists(t, ctx, db, "runtime_revision", "external_profile", false) + + var archivedRuntime, archivedProfile string + if err := db.QueryRowContext(ctx, ` + SELECT + source_snapshot #>> '{__kagent_external_profile_compat_v1,externalRuntime}', + (source_snapshot #> '{__kagent_external_profile_compat_v1,externalProfile}')::text + FROM runtime_revision + WHERE revision = 'revision-4' + `).Scan(&archivedRuntime, &archivedProfile); err != nil { + t.Fatalf("read archived external profile: %v", err) + } + if archivedRuntime != "codex" || archivedProfile != `{"version": "v1"}` { + t.Fatalf("archived external identity = %s/%s", archivedRuntime, archivedProfile) + } + + migrateCoreTo(t, connStr, 19) + var restoredSource, restoredProfile string + if err := db.QueryRowContext(ctx, ` + SELECT source_snapshot::text, external_profile::text + FROM runtime_revision + WHERE revision = 'revision-4' + `).Scan(&restoredSource, &restoredProfile); err != nil { + t.Fatalf("read restored external profile: %v", err) + } + if restoredSource != `{}` || restoredProfile != `{"version": "v1"}` { + t.Fatalf("restored source/profile = %s/%s", restoredSource, restoredProfile) + } + + // The compatibility envelope survives the older v18 down migration as well, + // so 19 -> 17 -> 19 restores both backend identity and the profile instead of + // silently reclassifying external revisions as Substrate. + migrateCoreTo(t, connStr, 17) + migrateCoreTo(t, connStr, 19) + var backendKind, restoredRuntime, roundTripProfile string + if err := db.QueryRowContext(ctx, ` + SELECT backend_kind, external_runtime, external_profile::text + FROM runtime_revision + WHERE revision = 'revision-4' + `).Scan(&backendKind, &restoredRuntime, &roundTripProfile); err != nil { + t.Fatalf("read external revision after 19 -> 17 -> 19: %v", err) + } + if backendKind != "external" || restoredRuntime != "codex" || roundTripProfile != `{"version": "v1"}` { + t.Fatalf("restored external identity = %s/%s/%s", backendKind, restoredRuntime, roundTripProfile) + } + + // A full rollback below the migration that created runtime_revision must not + // be blocked by compatibility state. Reapplying the core track recreates a + // clean table because version 8 intentionally dropped the old revision data. + migrateCoreTo(t, connStr, 7) + assertTableExists(t, ctx, db, "runtime_revision", false) + migrateCoreTo(t, connStr, 19) + assertTableExists(t, ctx, db, "runtime_revision", true) + var count int + if err := db.QueryRowContext(ctx, `SELECT count(*) FROM runtime_revision`).Scan(&count); err != nil { + t.Fatalf("count revisions after 19 -> 7 -> 19: %v", err) + } + if count != 0 { + t.Fatalf("runtime revisions after destructive rollback = %d, want 0", count) + } +} + +func TestMigration000019RejectsUnprofiledVersion18ExternalRevision(t *testing.T) { + connStr := startTestDB(t) + migrateCoreTo(t, connStr, 18) + db, err := sql.Open("pgx", connStr) + if err != nil { + t.Fatalf("open db: %v", err) + } + defer db.Close() + ctx := context.Background() + + if _, err := db.ExecContext(ctx, ` + INSERT INTO runtime_revision ( + revision, namespace, agent_template_name, agent_template_uid, + harness_name, harness_uid, source_snapshot, agent_card, + backend_kind, external_runtime, + actor_template_namespace, actor_template_name, phase + ) VALUES ( + 'unprofiled-external', 'team-a', 'assistant', 'template-uid', + 'codex', 'harness-uid', '{}', '{}', + 'external', 'codex', 'team-a', 'legacy-external', 'Ready' + ) + `); err != nil { + t.Fatalf("insert version 18 external revision: %v", err) + } + + _, err = applySource(ctx, connStr, realCoreSource()) + if err == nil { + t.Fatal("version 19 accepted an external revision without a recoverable profile") + } + var version int + var dirty bool + if err := db.QueryRowContext(ctx, `SELECT version, dirty FROM schema_migrations`).Scan(&version, &dirty); err != nil { + t.Fatalf("read migration state after rejected upgrade: %v", err) + } + if version != 18 || dirty { + t.Fatalf("migration state = version %d dirty %t, want clean version 18", version, dirty) + } + assertColumnExists(t, ctx, db, "runtime_revision", "external_profile", false) +} + +func TestMigration000019RejectsCorruptReclassifiedCompatibilityEnvelope(t *testing.T) { + connStr := startTestDB(t) + migrateCoreTo(t, connStr, 18) + db, err := sql.Open("pgx", connStr) + if err != nil { + t.Fatalf("open db: %v", err) + } + defer db.Close() + ctx := context.Background() + + // Version 18 reclassifies archived external rows as substrate after a + // 19 -> 17 -> 18 rollback/upgrade. The reserved actor sentinel lets v19 + // distinguish those rows from real substrate revisions and fail closed when + // their compatibility envelope has been lost or corrupted. + if _, err := db.ExecContext(ctx, ` + INSERT INTO runtime_revision ( + revision, namespace, agent_template_name, agent_template_uid, + harness_name, harness_uid, source_snapshot, agent_card, + backend_kind, actor_template_namespace, actor_template_name, phase + ) VALUES ( + 'corrupt-envelope', 'team-a', 'assistant', 'template-uid', + 'codex', 'harness-uid', '{}', '{}', + 'substrate', '_external', 'corrupt-envelope', 'Ready' + ) + `); err != nil { + t.Fatalf("insert reclassified version 18 revision: %v", err) + } + + _, err = applySource(ctx, connStr, realCoreSource()) + if err == nil { + t.Fatal("version 19 accepted a corrupt reclassified compatibility envelope") + } + var version int + var dirty bool + if err := db.QueryRowContext(ctx, `SELECT version, dirty FROM schema_migrations`).Scan(&version, &dirty); err != nil { + t.Fatalf("read migration state after rejected upgrade: %v", err) + } + if version != 18 || dirty { + t.Fatalf("migration state = version %d dirty %t, want clean version 18", version, dirty) + } + assertColumnExists(t, ctx, db, "runtime_revision", "external_profile", false) +} + +func assertColumnExists(t *testing.T, ctx context.Context, db *sql.DB, table, column string, want bool) { + t.Helper() + var exists bool + if err := db.QueryRowContext(ctx, ` + SELECT EXISTS ( + SELECT 1 + FROM information_schema.columns + WHERE table_schema = current_schema() + AND table_name = $1 + AND column_name = $2 + ) + `, table, column).Scan(&exists); err != nil { + t.Fatalf("check column %s.%s: %v", table, column, err) + } + if exists != want { + t.Fatalf("column %s.%s exists = %t, want %t", table, column, exists, want) + } +} + +func assertTableExists(t *testing.T, ctx context.Context, db *sql.DB, table string, want bool) { + t.Helper() + var exists bool + if err := db.QueryRowContext(ctx, `SELECT to_regclass($1) IS NOT NULL`, table).Scan(&exists); err != nil { + t.Fatalf("check table %s: %v", table, err) + } + if exists != want { + t.Fatalf("table %s exists = %t, want %t", table, exists, want) + } +} diff --git a/go/core/v2/controller/collections_test.go b/go/core/v2/controller/collections_test.go index c2657718b7..9b919846b8 100644 --- a/go/core/v2/controller/collections_test.go +++ b/go/core/v2/controller/collections_test.go @@ -56,8 +56,8 @@ func TestReconciliationCollectionsCompileAndObserveRevision(t *testing.T) { matchingHarness := harness("team-a", "kagent", map[string]string{"runtime": "python"}) matchingHarness.UID = "harness-uid" matchingHarness.Spec.Kagent = &kagentv1alpha3.KagentHarness{} - matchingHarness.Spec.Workload.Image = "example.com/kagent@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" - matchingHarness.Spec.Substrate = kagentv1alpha3.HarnessSubstratePolicy{ + matchingHarness.Spec.Workload = &kagentv1alpha3.HarnessWorkload{Image: "example.com/kagent@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"} + matchingHarness.Spec.Substrate = &kagentv1alpha3.HarnessSubstratePolicy{ WorkerPoolRef: corev1.LocalObjectReference{Name: "default"}, SnapshotPolicy: kagentv1alpha3.HarnessSnapshotPolicy{Location: "snapshots"}, } @@ -109,7 +109,7 @@ func TestReconciliationCollectionsCompileAndObserveRevision(t *testing.T) { return false } ready := apimeta.FindStatusCondition(updates[0].Status.Harnesses[0].Conditions, kagentv1alpha3.AgentTemplateConditionReady) - return ready != nil && ready.Status == metav1.ConditionTrue && updates[0].Status.Harnesses[0].LatestSuccessfulRevision == state.RevisionID.String() + return ready != nil && ready.Status == metav1.ConditionFalse && ready.Reason == "RevisionPending" && updates[0].Status.Harnesses[0].LatestSuccessfulRevision == "" }) modelConfigs.UpdateObject(&kagentv1alpha3.ModelConfig{ObjectMeta: metav1.ObjectMeta{Namespace: "team-a", Name: "model"}, Spec: kagentv1alpha3.ModelConfigSpec{Provider: kagentv1alpha3.ModelProviderOpenAI, Model: "gpt-5.1"}}) @@ -138,8 +138,8 @@ func TestReconciliationTracksSharedAgentTemplate(t *testing.T) { } harness := harness("team-a", "kagent", map[string]string{"runtime": "python"}) harness.Spec.Kagent = &kagentv1alpha3.KagentHarness{} - harness.Spec.Workload.Image = "example.com/kagent@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" - harness.Spec.Substrate = kagentv1alpha3.HarnessSubstratePolicy{WorkerPoolRef: corev1.LocalObjectReference{Name: "default"}, SnapshotPolicy: kagentv1alpha3.HarnessSnapshotPolicy{Location: "snapshots"}} + harness.Spec.Workload = &kagentv1alpha3.HarnessWorkload{Image: "example.com/kagent@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"} + harness.Spec.Substrate = &kagentv1alpha3.HarnessSubstratePolicy{WorkerPoolRef: corev1.LocalObjectReference{Name: "default"}, SnapshotPolicy: kagentv1alpha3.HarnessSnapshotPolicy{Location: "snapshots"}} templates := krt.NewStaticCollection(nil, []*kagentv1alpha3.AgentTemplate{root, child}, opts.WithName("AgentTemplates")...) pairs := newPairCollection(templates, krt.NewStaticCollection(nil, []*kagentv1alpha3.Harness{harness}, opts.WithName("Harnesses")...), opts) reconciliations := newPairReconciliations( diff --git a/go/core/v2/controller/reconciler.go b/go/core/v2/controller/reconciler.go index 04a13b1402..87cf77bc5a 100644 --- a/go/core/v2/controller/reconciler.go +++ b/go/core/v2/controller/reconciler.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "slices" "strings" atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" @@ -13,6 +14,7 @@ import ( kagentv1alpha3 "github.com/kagent-dev/kagent/go/api/v1alpha3" "github.com/kagent-dev/kagent/go/core/v2/substrate" v2translator "github.com/kagent-dev/kagent/go/core/v2/translator" + externaltranslator "github.com/kagent-dev/kagent/go/core/v2/translator/external" kagenttranslator "github.com/kagent-dev/kagent/go/core/v2/translator/kagent" "istio.io/istio/pkg/kube/controllers" "istio.io/istio/pkg/kube/krt" @@ -46,6 +48,19 @@ type ReconciliationFailure struct { Message string } +var ( + codexHarnessCompiler = mustExternalHarnessCompiler(dbpkg.ExternalRuntimeCodex) + claudeHarnessCompiler = mustExternalHarnessCompiler(dbpkg.ExternalRuntimeClaude) +) + +func mustExternalHarnessCompiler(runtime dbpkg.ExternalRuntime) v2translator.HarnessCompiler { + compiler, err := externaltranslator.NewCompiler(runtime) + if err != nil { + panic(err) + } + return compiler +} + func newPairReconciliations( pairs krt.Collection[AgentTemplateHarnessPair], agentTemplates krt.Collection[*kagentv1alpha3.AgentTemplate], @@ -65,6 +80,8 @@ func newPairReconciliations( } revision, err := v2translator.NewCompiler(reader, map[v2translator.HarnessType]v2translator.HarnessCompiler{ v2translator.HarnessTypeKagent: kagenttranslator.NewCompiler(reader), + v2translator.HarnessTypeCodex: codexHarnessCompiler, + v2translator.HarnessTypeClaude: claudeHarnessCompiler, }).CompileAgentTemplate(context.Background(), pair.Harness, pair.AgentTemplate) if err != nil { condition, reason := kagentv1alpha3.AgentTemplateConditionResolvedRefs, "ReferenceResolutionFailed" @@ -81,6 +98,19 @@ func newPairReconciliations( state.Failure = &ReconciliationFailure{Condition: kagentv1alpha3.AgentTemplateConditionCompatible, Reason: "RevisionInvalid", Message: err.Error()} return state } + switch revision.BackendKind { + case dbpkg.RuntimeBackendKindExternal: + return state + case dbpkg.RuntimeBackendKindSubstrate: + // Continue into the Substrate materialization path below. + default: + state.Failure = &ReconciliationFailure{ + Condition: kagentv1alpha3.AgentTemplateConditionCompatible, + Reason: "RuntimeBackendUnsupported", + Message: "compiled revision selected an unsupported runtime backend", + } + return state + } workerPool := &atev1alpha1.WorkerPool{} workerKey := types.NamespacedName{Namespace: revision.Namespace, Name: revision.WorkerPoolName} @@ -237,6 +267,29 @@ func (r *Reconciler) reconcilePair(ctx context.Context, key string) error { if state.Failure != nil { return r.cleanupUnreferencedRevisions(ctx) } + if state.Revision.BackendKind == dbpkg.RuntimeBackendKindExternal { + revision := dbpkg.RuntimeRevision{ + Revision: state.RevisionID.String(), Namespace: pair.Namespace, AgentTemplateName: pair.AgentTemplateName, + AgentTemplateUID: pair.AgentTemplateUID, HarnessName: pair.HarnessName, HarnessUID: pair.HarnessUID, + SourceSnapshot: state.Revision.Provenance, AgentCard: state.Revision.AgentCardJSON, + BackendKind: dbpkg.RuntimeBackendKindExternal, ExternalRuntime: state.Revision.ExternalRuntime, + ExternalProfile: state.Revision.ExternalProfile, EgressDestinations: state.Revision.EgressDestinations, + Phase: "Ready", + } + if err := r.store.UpsertRuntimeRevision(ctx, revision); err != nil { + return fmt.Errorf("store external runtime revision %s: %w", state.RevisionID, err) + } + if err := r.store.MarkRuntimeRevisionSuccessful(ctx, pair); err != nil { + return fmt.Errorf("mark external runtime revision %s successful: %w", state.RevisionID, err) + } + if err := r.promotePairStatus(ctx, state); err != nil { + return err + } + return r.cleanupUnreferencedRevisions(ctx) + } + if state.Revision.BackendKind != dbpkg.RuntimeBackendKindSubstrate { + return fmt.Errorf("runtime revision %s selected an unsupported backend", state.RevisionID) + } if state.ObservedActorTemplate == nil { _, err := r.actors.ActorTemplates(state.DesiredActorTemplate.Namespace).Create(ctx, state.DesiredActorTemplate.DeepCopy(), metav1.CreateOptions{}) if err != nil && !apierrors.IsAlreadyExists(err) { @@ -262,17 +315,84 @@ func (r *Reconciler) reconcilePair(ctx context.Context, key string) error { if err := r.store.MarkRuntimeRevisionSuccessful(ctx, pair); err != nil { return fmt.Errorf("mark runtime revision %s successful: %w", state.RevisionID, err) } + if err := r.promotePairStatus(ctx, state); err != nil { + return err + } return r.cleanupUnreferencedRevisions(ctx) } return nil } +// promotePairStatus publishes a revision only after the database has accepted +// both the immutable revision and the latest-successful edge. Source identity +// guards reject superseded work, while reconcileStatus rejects derived values +// from an older resourceVersion so Pending cannot overwrite this acknowledgement. +func (r *Reconciler) promotePairStatus(ctx context.Context, state *PairReconciliation) error { + key := state.Pair.AgentTemplate.Namespace + "/" + state.Pair.AgentTemplate.Name + template := r.collections.AgentTemplates.GetKey(key) + if template == nil { + return fmt.Errorf("promote runtime revision %s status: AgentTemplate %s is no longer available", state.RevisionID, key) + } + // Persistence can finish after either source object has been replaced or + // after a newer generation/revision has entered the graph. The database + // write is identity-scoped, but status is name-scoped, so never publish the + // old acknowledgement onto the new object. + if (*template).UID != state.Pair.AgentTemplate.UID || (*template).Generation != state.Pair.AgentTemplate.Generation { + return nil + } + currentState := r.collections.Reconciliations.GetKey(state.ResourceName()) + if currentState == nil || + currentState.Pair.AgentTemplate.UID != state.Pair.AgentTemplate.UID || + currentState.Pair.AgentTemplate.Generation != state.Pair.AgentTemplate.Generation || + currentState.Pair.Harness.UID != state.Pair.Harness.UID || + currentState.RevisionID != state.RevisionID { + return nil + } + updated := (*template).DeepCopy() + current := (*template).Status + promoted := statusForPair(*currentState, updated.Generation, state.RevisionID.String()) + replaced := false + for index := range updated.Status.Harnesses { + if updated.Status.Harnesses[index].Harness == promoted.Harness { + updated.Status.Harnesses[index] = promoted + replaced = true + break + } + } + if !replaced { + updated.Status.Harnesses = append(updated.Status.Harnesses, promoted) + slices.SortFunc(updated.Status.Harnesses, func(left, right kagentv1alpha3.AgentTemplateHarnessStatus) int { + return strings.Compare(left.Harness, right.Harness) + }) + } + updated.Status.ObservedGeneration = updated.Generation + updated.Status = statusWithTransitionTimes(updated.Status, current) + if apiequality.Semantic.DeepEqual(updated.Status, (*template).Status) { + return nil + } + if err := r.updateStatus(ctx, updated); err != nil { + return fmt.Errorf("promote runtime revision %s status: %w", state.RevisionID, err) + } + return nil +} + func (r *Reconciler) reconcileStatus(ctx context.Context, key string) error { desired := r.collections.AgentTemplateStatuses.GetKey(key) template := r.collections.AgentTemplates.GetKey(key) if desired == nil || template == nil { return nil } + // A StatusCollection value includes the exact source object from which its + // status was derived. Do not transplant an older derived Pending status onto + // a newer resourceVersion after promotePairStatus has acknowledged Ready. + // The informer event for the newer object will either enqueue the current + // derived value or observe that the promoted status is already identical. + if desired.Obj == nil || + desired.Obj.UID != (*template).UID || + desired.Obj.Generation != (*template).Generation || + desired.Obj.ResourceVersion != (*template).ResourceVersion { + return nil + } updated := (*template).DeepCopy() updated.Status = statusWithTransitionTimes(desired.Status, updated.Status) if apiequality.Semantic.DeepEqual(updated.Status, (*template).Status) { @@ -292,6 +412,17 @@ func (r *Reconciler) cleanupUnreferencedRevisions(ctx context.Context) error { return fmt.Errorf("list unreferenced runtime revisions: %w", err) } for _, revision := range revisions { + switch revision.BackendKind { + case dbpkg.RuntimeBackendKindExternal: + if err := r.store.DeleteUnreferencedRuntimeRevision(ctx, revision.Revision); err != nil { + return fmt.Errorf("delete unreferenced external runtime revision %s: %w", revision.Revision, err) + } + continue + case dbpkg.RuntimeBackendKindSubstrate: + // Continue with the UID-guarded Kubernetes cleanup below. + default: + return fmt.Errorf("unreferenced runtime revision %s has an unsupported backend", revision.Revision) + } client := r.actors.ActorTemplates(revision.ActorTemplateNamespace) template, err := client.Get(ctx, revision.ActorTemplateName, metav1.GetOptions{}) if err != nil && !apierrors.IsNotFound(err) { diff --git a/go/core/v2/controller/reconciler_test.go b/go/core/v2/controller/reconciler_test.go index c2da62561a..5ad52ce4d9 100644 --- a/go/core/v2/controller/reconciler_test.go +++ b/go/core/v2/controller/reconciler_test.go @@ -2,6 +2,7 @@ package controller import ( "context" + "errors" "testing" atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" @@ -17,10 +18,10 @@ func TestReconcilerPersistsPairInOrder(t *testing.T) { stop := make(chan struct{}) t.Cleanup(func() { close(stop) }) opts := krt.NewOptionsBuilder(stop, "test", nil) - template := &kagentv1alpha3.AgentTemplate{ObjectMeta: metav1.ObjectMeta{Namespace: "team-a", Name: "assistant", UID: "template-uid"}} + template := &kagentv1alpha3.AgentTemplate{ObjectMeta: metav1.ObjectMeta{Namespace: "team-a", Name: "assistant", UID: "template-uid", Generation: 3}} harness := &kagentv1alpha3.Harness{ObjectMeta: metav1.ObjectMeta{Namespace: "team-a", Name: "kagent", UID: "harness-uid"}} desiredActor := &atev1alpha1.ActorTemplate{ObjectMeta: metav1.ObjectMeta{Namespace: "team-a", Name: "assistant-kagent-revision"}} - revision := &v2translator.Revision{} + revision := &v2translator.Revision{BackendKind: dbpkg.RuntimeBackendKindSubstrate} revisionID, err := revision.Digest() if err != nil { t.Fatal(err) @@ -93,11 +94,302 @@ func TestReconcilerPersistsPairInOrder(t *testing.T) { } } +func TestReconcilerPersistsExternalRevisionWithoutActorTemplate(t *testing.T) { + stop := make(chan struct{}) + t.Cleanup(func() { close(stop) }) + opts := krt.NewOptionsBuilder(stop, "test", nil) + template := &kagentv1alpha3.AgentTemplate{ObjectMeta: metav1.ObjectMeta{Namespace: "team-a", Name: "assistant", UID: "template-uid", Generation: 3}} + harness := &kagentv1alpha3.Harness{ObjectMeta: metav1.ObjectMeta{Namespace: "team-a", Name: "codex", UID: "harness-uid"}} + revision := &v2translator.Revision{ + Namespace: "team-a", AgentTemplateName: "assistant", HarnessName: "codex", + BackendKind: dbpkg.RuntimeBackendKindExternal, ExternalRuntime: dbpkg.ExternalRuntimeCodex, + ExternalProfile: []byte(`{"version":"v1","instruction":"help","tools":[]}`), + AgentCardJSON: []byte(`{"name":"assistant","version":"v1"}`), + Provenance: []byte(`{"template":"template-uid"}`), + } + revisionID, err := revision.Digest() + if err != nil { + t.Fatal(err) + } + state := PairReconciliation{ + Pair: AgentTemplateHarnessPair{AgentTemplate: template, Harness: harness}, Revision: revision, RevisionID: revisionID, + } + reconciliations := krt.NewStaticCollection(nil, []PairReconciliation{state}, opts.WithName("ExternalReconciliations")...) + store := &fakeRuntimeRevisionStore{} + actors := atefake.NewSimpleClientset().ApiV1alpha1() //nolint:staticcheck + var statusWrite *kagentv1alpha3.AgentTemplate + reconciler := &Reconciler{ + collections: Collections{ + AgentTemplates: krt.NewStaticCollection(nil, []*kagentv1alpha3.AgentTemplate{template}, opts.WithName("ExternalAgentTemplates")...), + Reconciliations: reconciliations, + }, + actors: actors, store: store, + updateStatus: func(_ context.Context, template *kagentv1alpha3.AgentTemplate) error { + statusWrite = template + return nil + }, + } + + if err := reconciler.reconcilePair(t.Context(), state.ResourceName()); err != nil { + t.Fatal(err) + } + if store.pair == nil || store.revision == nil || !store.markedSuccessful { + t.Fatalf("external revision was not persisted and marked ready: pair=%v revision=%v successful=%v", store.pair != nil, store.revision != nil, store.markedSuccessful) + } + if store.revision.BackendKind != dbpkg.RuntimeBackendKindExternal || store.revision.ExternalRuntime != dbpkg.ExternalRuntimeCodex { + t.Fatalf("unexpected external backend identity: %+v", store.revision) + } + if string(store.revision.ExternalProfile) != string(revision.ExternalProfile) || store.revision.ActorTemplateNamespace != "" || store.revision.ActorTemplateName != "" { + t.Fatalf("external revision used cluster compute identity: %+v", store.revision) + } + actorList, err := actors.ActorTemplates("team-a").List(t.Context(), metav1.ListOptions{}) + if err != nil { + t.Fatal(err) + } + if len(actorList.Items) != 0 { + t.Fatalf("external revision created %d ActorTemplates", len(actorList.Items)) + } + + pending := statusForPair(state, 3, "") + if pending.LatestSuccessfulRevision != "" { + t.Fatalf("external status promoted before persistence: %q", pending.LatestSuccessfulRevision) + } + if statusWrite == nil || len(statusWrite.Status.Harnesses) != 1 || statusWrite.Status.Harnesses[0].LatestSuccessfulRevision != revisionID.String() { + t.Fatalf("persisted external revision was not promoted in status: %+v", statusWrite) + } + ready := false + for _, condition := range statusWrite.Status.Harnesses[0].Conditions { + if condition.Type == kagentv1alpha3.AgentTemplateConditionReady { + ready = condition.Status == metav1.ConditionTrue && condition.Reason == "ExternalRuntimePrepared" + } + } + if !ready { + t.Fatalf("external revision was not reported prepared: %+v", statusWrite.Status.Harnesses[0].Conditions) + } +} + +func TestExternalRevisionStatusIsNotPromotedWhenPersistenceFails(t *testing.T) { + stop := make(chan struct{}) + t.Cleanup(func() { close(stop) }) + opts := krt.NewOptionsBuilder(stop, "test", nil) + template := &kagentv1alpha3.AgentTemplate{ + ObjectMeta: metav1.ObjectMeta{Namespace: "team-a", Name: "assistant", UID: "template-uid", Generation: 4}, + Status: kagentv1alpha3.AgentTemplateStatus{Harnesses: []kagentv1alpha3.AgentTemplateHarnessStatus{{ + Harness: "codex", LatestSuccessfulRevision: "previous-revision", + }}}, + } + harness := &kagentv1alpha3.Harness{ObjectMeta: metav1.ObjectMeta{Namespace: "team-a", Name: "codex", UID: "harness-uid"}} + revision := &v2translator.Revision{ + BackendKind: dbpkg.RuntimeBackendKindExternal, ExternalRuntime: dbpkg.ExternalRuntimeCodex, + ExternalProfile: []byte(`{"version":"v1","instruction":"help","tools":[]}`), + AgentCardJSON: []byte(`{"name":"assistant","version":"v1"}`), Provenance: []byte(`{}`), + EgressDestinations: []string{}, + } + revisionID, err := revision.Digest() + if err != nil { + t.Fatal(err) + } + state := PairReconciliation{Pair: AgentTemplateHarnessPair{AgentTemplate: template, Harness: harness}, Revision: revision, RevisionID: revisionID} + reconciliations := krt.NewStaticCollection(nil, []PairReconciliation{state}, opts.WithName("FailingExternalReconciliations")...) + store := &fakeRuntimeRevisionStore{markErr: errors.New("database unavailable")} + statusWrites := 0 + reconciler := &Reconciler{ + collections: Collections{ + AgentTemplates: krt.NewStaticCollection(nil, []*kagentv1alpha3.AgentTemplate{template}, opts.WithName("FailingExternalAgentTemplates")...), + Reconciliations: reconciliations, + }, + actors: atefake.NewSimpleClientset().ApiV1alpha1(), store: store, //nolint:staticcheck + updateStatus: func(context.Context, *kagentv1alpha3.AgentTemplate) error { statusWrites++; return nil }, + } + + err = reconciler.reconcilePair(t.Context(), state.ResourceName()) + if err == nil || !errors.Is(err, store.markErr) { + t.Fatalf("reconcile error = %v, want persistence failure", err) + } + if statusWrites != 0 { + t.Fatalf("status was promoted %d times after persistence failed", statusWrites) + } + pending := statusForPair(state, template.Generation, template.Status.Harnesses[0].LatestSuccessfulRevision) + if pending.LatestSuccessfulRevision != "previous-revision" { + t.Fatalf("pending status lost previous successful revision: %+v", pending) + } +} + +func TestReconcileStatusDoesNotOverwriteAcknowledgementFromStaleDerivation(t *testing.T) { + stop := make(chan struct{}) + t.Cleanup(func() { close(stop) }) + opts := krt.NewOptionsBuilder(stop, "test", nil) + + current := &kagentv1alpha3.AgentTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "team-a", Name: "assistant", UID: "template-uid", Generation: 4, ResourceVersion: "12", + }, + Status: kagentv1alpha3.AgentTemplateStatus{ + ObservedGeneration: 4, + Harnesses: []kagentv1alpha3.AgentTemplateHarnessStatus{{ + Harness: "codex", DesiredRevision: "revision-4", LatestSuccessfulRevision: "revision-4", + Conditions: []metav1.Condition{{ + Type: kagentv1alpha3.AgentTemplateConditionReady, Status: metav1.ConditionTrue, + Reason: "ExternalRuntimePrepared", ObservedGeneration: 4, + }}, + }}, + }, + } + staleSource := current.DeepCopy() + staleSource.ResourceVersion = "11" + staleSource.Status = kagentv1alpha3.AgentTemplateStatus{} + staleDerived := kagentv1alpha3.AgentTemplateStatus{ + ObservedGeneration: 4, + Harnesses: []kagentv1alpha3.AgentTemplateHarnessStatus{{ + Harness: "codex", DesiredRevision: "revision-4", + Conditions: []metav1.Condition{{ + Type: kagentv1alpha3.AgentTemplateConditionReady, Status: metav1.ConditionFalse, + Reason: "ExternalRevisionPending", ObservedGeneration: 4, + }}, + }}, + } + statuses := krt.NewStaticCollection(nil, []krt.ObjectWithStatus[*kagentv1alpha3.AgentTemplate, kagentv1alpha3.AgentTemplateStatus]{ + {Obj: staleSource, Status: staleDerived}, + }, opts.WithName("StaleDerivedStatuses")...) + writes := 0 + reconciler := &Reconciler{ + collections: Collections{ + AgentTemplates: krt.NewStaticCollection(nil, []*kagentv1alpha3.AgentTemplate{current}, opts.WithName("AcknowledgedAgentTemplates")...), + AgentTemplateStatuses: statuses, + }, + updateStatus: func(context.Context, *kagentv1alpha3.AgentTemplate) error { + writes++ + return nil + }, + } + + if err := reconciler.reconcileStatus(t.Context(), "team-a/assistant"); err != nil { + t.Fatal(err) + } + if writes != 0 { + t.Fatalf("stale derived status overwrote acknowledged status with %d write(s)", writes) + } +} + +func TestPromotePairStatusRejectsSupersededIdentityGenerationAndRevision(t *testing.T) { + revision := &v2translator.Revision{ + BackendKind: dbpkg.RuntimeBackendKindExternal, ExternalRuntime: dbpkg.ExternalRuntimeCodex, + ExternalProfile: []byte(`{"version":"v1","instruction":"first","tools":[]}`), + } + revisionID, err := revision.Digest() + if err != nil { + t.Fatal(err) + } + newerRevision := &v2translator.Revision{ + BackendKind: dbpkg.RuntimeBackendKindExternal, ExternalRuntime: dbpkg.ExternalRuntimeCodex, + ExternalProfile: []byte(`{"version":"v1","instruction":"second","tools":[]}`), + } + newerRevisionID, err := newerRevision.Digest() + if err != nil { + t.Fatal(err) + } + + tests := []struct { + name string + change func(*kagentv1alpha3.AgentTemplate, *kagentv1alpha3.Harness, *PairReconciliation) + }{ + { + name: "template UID", + change: func(template *kagentv1alpha3.AgentTemplate, _ *kagentv1alpha3.Harness, state *PairReconciliation) { + template.UID = "replacement-template-uid" + state.Pair.AgentTemplate = template + }, + }, + { + name: "template generation", + change: func(template *kagentv1alpha3.AgentTemplate, _ *kagentv1alpha3.Harness, state *PairReconciliation) { + template.Generation++ + state.Pair.AgentTemplate = template + }, + }, + { + name: "harness UID", + change: func(_ *kagentv1alpha3.AgentTemplate, harness *kagentv1alpha3.Harness, state *PairReconciliation) { + harness.UID = "replacement-harness-uid" + state.Pair.Harness = harness + }, + }, + { + name: "desired revision", + change: func(_ *kagentv1alpha3.AgentTemplate, _ *kagentv1alpha3.Harness, state *PairReconciliation) { + state.Revision = newerRevision + state.RevisionID = newerRevisionID + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + stop := make(chan struct{}) + t.Cleanup(func() { close(stop) }) + opts := krt.NewOptionsBuilder(stop, "test", nil) + capturedTemplate := &kagentv1alpha3.AgentTemplate{ObjectMeta: metav1.ObjectMeta{ + Namespace: "team-a", Name: "assistant", UID: "template-uid", Generation: 7, ResourceVersion: "20", + }} + capturedHarness := &kagentv1alpha3.Harness{ObjectMeta: metav1.ObjectMeta{ + Namespace: "team-a", Name: "codex", UID: "harness-uid", Generation: 2, + }} + captured := PairReconciliation{ + Pair: AgentTemplateHarnessPair{AgentTemplate: capturedTemplate, Harness: capturedHarness}, + Revision: revision, RevisionID: revisionID, + } + currentTemplate := capturedTemplate.DeepCopy() + currentHarness := capturedHarness.DeepCopy() + current := captured + current.Pair = AgentTemplateHarnessPair{AgentTemplate: currentTemplate, Harness: currentHarness} + tt.change(currentTemplate, currentHarness, ¤t) + + reconciliations := krt.NewStaticCollection(nil, []PairReconciliation{current}, opts.WithName("CurrentReconciliations")...) + writes := 0 + reconciler := &Reconciler{ + collections: Collections{ + AgentTemplates: krt.NewStaticCollection(nil, []*kagentv1alpha3.AgentTemplate{currentTemplate}, opts.WithName("CurrentAgentTemplates")...), + Reconciliations: reconciliations, + }, + updateStatus: func(context.Context, *kagentv1alpha3.AgentTemplate) error { + writes++ + return nil + }, + } + + if err := reconciler.promotePairStatus(t.Context(), &captured); err != nil { + t.Fatal(err) + } + if writes != 0 { + t.Fatalf("superseded %s produced %d status write(s)", tt.name, writes) + } + }) + } +} + +func TestCleanupExternalRevisionIsDatabaseOnly(t *testing.T) { + store := &fakeRuntimeRevisionStore{unreferenced: []dbpkg.RuntimeRevision{{ + Revision: "external-revision", BackendKind: dbpkg.RuntimeBackendKindExternal, + ExternalRuntime: dbpkg.ExternalRuntimeCodex, ExternalProfile: []byte(`{"version":"v1"}`), Phase: "Ready", + }}} + actors := atefake.NewSimpleClientset().ApiV1alpha1() //nolint:staticcheck + reconciler := &Reconciler{actors: actors, store: store} + if err := reconciler.cleanupUnreferencedRevisions(t.Context()); err != nil { + t.Fatal(err) + } + if len(store.deleted) != 1 || store.deleted[0] != "external-revision" { + t.Fatalf("external revision delete calls = %v", store.deleted) + } +} + type fakeRuntimeRevisionStore struct { pair *dbpkg.AgentTemplateHarnessPair revision *dbpkg.RuntimeRevision markedSuccessful bool retired string + unreferenced []dbpkg.RuntimeRevision + deleted []string + markErr error } func (s *fakeRuntimeRevisionStore) UpsertAgentTemplateHarnessPair(_ context.Context, pair dbpkg.AgentTemplateHarnessPair) error { @@ -111,6 +403,9 @@ func (s *fakeRuntimeRevisionStore) UpsertRuntimeRevision(_ context.Context, revi } func (s *fakeRuntimeRevisionStore) MarkRuntimeRevisionSuccessful(context.Context, dbpkg.AgentTemplateHarnessPair) error { + if s.markErr != nil { + return s.markErr + } s.markedSuccessful = true return nil } @@ -121,9 +416,10 @@ func (s *fakeRuntimeRevisionStore) RetireAgentTemplateHarnessPair(_ context.Cont } func (s *fakeRuntimeRevisionStore) ListUnreferencedRuntimeRevisions(context.Context) ([]dbpkg.RuntimeRevision, error) { - return nil, nil + return append([]dbpkg.RuntimeRevision(nil), s.unreferenced...), nil } -func (s *fakeRuntimeRevisionStore) DeleteUnreferencedRuntimeRevision(context.Context, string) error { +func (s *fakeRuntimeRevisionStore) DeleteUnreferencedRuntimeRevision(_ context.Context, revision string) error { + s.deleted = append(s.deleted, revision) return nil } diff --git a/go/core/v2/controller/status.go b/go/core/v2/controller/status.go index 5812c50eba..9f8e57eb35 100644 --- a/go/core/v2/controller/status.go +++ b/go/core/v2/controller/status.go @@ -8,6 +8,7 @@ import ( "strings" atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" + dbpkg "github.com/kagent-dev/kagent/go/api/database" kagentv1alpha3 "github.com/kagent-dev/kagent/go/api/v1alpha3" "istio.io/istio/pkg/kube/krt" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -57,11 +58,22 @@ func statusForPair(state PairReconciliation, generation int64, latestSuccessful } setPairCondition(&status, generation, kagentv1alpha3.AgentTemplateConditionResolvedRefs, metav1.ConditionTrue, "Resolved", "All runtime references resolved") setPairCondition(&status, generation, kagentv1alpha3.AgentTemplateConditionCompatible, metav1.ConditionTrue, "Compatible", "Resolved configuration is compatible with the Harness") + if state.Revision != nil && state.Revision.BackendKind == dbpkg.RuntimeBackendKindExternal { + if latestSuccessful != desired { + setPairCondition(&status, generation, kagentv1alpha3.AgentTemplateConditionReady, metav1.ConditionFalse, "ExternalRevisionPending", "waiting for the external runtime revision to be persisted") + return status + } + setPairCondition(&status, generation, kagentv1alpha3.AgentTemplateConditionReady, metav1.ConditionTrue, "ExternalRuntimePrepared", "external runtime revision is prepared; online slot availability is checked when an AgentInstance is created or resumed") + return status + } if state.ObservedActorTemplate == nil || state.ObservedActorTemplate.Status.Phase != atev1alpha1.PhaseReady { setPairCondition(&status, generation, kagentv1alpha3.AgentTemplateConditionReady, metav1.ConditionFalse, "ActorTemplatePending", "waiting for the ActorTemplate golden snapshot") return status } - status.LatestSuccessfulRevision = state.RevisionID.String() + if latestSuccessful != desired { + setPairCondition(&status, generation, kagentv1alpha3.AgentTemplateConditionReady, metav1.ConditionFalse, "RevisionPending", "waiting for the ready runtime revision to be persisted") + return status + } setPairCondition(&status, generation, kagentv1alpha3.AgentTemplateConditionReady, metav1.ConditionTrue, "Ready", "ActorTemplate golden snapshot is ready") return status } diff --git a/go/core/v2/externalruntime/lifecycle_test.go b/go/core/v2/externalruntime/lifecycle_test.go index 466d6783d2..2f23dd28aa 100644 --- a/go/core/v2/externalruntime/lifecycle_test.go +++ b/go/core/v2/externalruntime/lifecycle_test.go @@ -357,6 +357,8 @@ func externalRevision(runtime dbpkg.ExternalRuntime) *dbpkg.RuntimeRevision { Revision: lifecycleRevisionID, BackendKind: dbpkg.RuntimeBackendKindExternal, ExternalRuntime: runtime, + ExternalProfile: []byte(`{"version":"v1","instruction":"","tools":[]}`), + Phase: "Ready", } } diff --git a/go/core/v2/runtimebackend/revision_selector_test.go b/go/core/v2/runtimebackend/revision_selector_test.go index c8cea9600d..4b9a2d1a11 100644 --- a/go/core/v2/runtimebackend/revision_selector_test.go +++ b/go/core/v2/runtimebackend/revision_selector_test.go @@ -27,17 +27,17 @@ func TestRevisionSelectorUsesOnlyPreparedRevisionIdentity(t *testing.T) { }{ { name: "substrate", - revision: dbpkg.RuntimeRevision{BackendKind: dbpkg.RuntimeBackendKindSubstrate}, + revision: validSubstrateRevision(), want: runtimebackend.KindSubstrate, }, { name: "external codex", - revision: dbpkg.RuntimeRevision{BackendKind: dbpkg.RuntimeBackendKindExternal, ExternalRuntime: dbpkg.ExternalRuntimeCodex}, + revision: validExternalRevision(dbpkg.ExternalRuntimeCodex), want: runtimebackend.KindExternal, }, { name: "external claude", - revision: dbpkg.RuntimeRevision{BackendKind: dbpkg.RuntimeBackendKindExternal, ExternalRuntime: dbpkg.ExternalRuntimeClaude}, + revision: validExternalRevision(dbpkg.ExternalRuntimeClaude), want: runtimebackend.KindExternal, }, } @@ -133,7 +133,8 @@ func TestRevisionSelectorValidatesDependenciesAndInput(t *testing.T) { } selector, err := runtimebackend.NewRevisionSelector(revisionStoreFunc(func(context.Context, string) (*dbpkg.RuntimeRevision, error) { - return &dbpkg.RuntimeRevision{BackendKind: dbpkg.RuntimeBackendKindSubstrate}, nil + revision := validSubstrateRevision() + return &revision, nil })) require.NoError(t, err) _, err = selector.Select(t.Context(), nil) @@ -143,3 +144,16 @@ func TestRevisionSelectorValidatesDependenciesAndInput(t *testing.T) { _, err = selector.Select(nil, &apiv1alpha1.AgentInstance{Id: "instance-1", PreparedRevision: "revision"}) require.ErrorContains(t, err, "requires a context") } + +func validSubstrateRevision() dbpkg.RuntimeRevision { + return dbpkg.RuntimeRevision{ + BackendKind: dbpkg.RuntimeBackendKindSubstrate, ActorTemplateNamespace: "team-a", ActorTemplateName: "actor", + } +} + +func validExternalRevision(runtime dbpkg.ExternalRuntime) dbpkg.RuntimeRevision { + return dbpkg.RuntimeRevision{ + BackendKind: dbpkg.RuntimeBackendKindExternal, ExternalRuntime: runtime, + ExternalProfile: []byte(`{"version":"v1","instruction":"","tools":[]}`), Phase: "Ready", + } +} diff --git a/go/core/v2/translator/compiler.go b/go/core/v2/translator/compiler.go index 1de28ef143..a0f08b6900 100644 --- a/go/core/v2/translator/compiler.go +++ b/go/core/v2/translator/compiler.go @@ -189,12 +189,16 @@ func harnessSelector(harness *v1alpha3.Harness) (labels.Selector, error) { } func (c *Compiler) buildInputs(ctx context.Context, tree *ResolvedTree) (*HarnessInput, error) { + resolveModel := harnessType(tree.Harness) == HarnessTypeKagent var build func(*ResolvedAgent) (*AgentInput, error) build = func(agent *ResolvedAgent) (*AgentInput, error) { template := agent.Template - model := &v1alpha3.ModelConfig{} - if err := c.kube.Get(ctx, types.NamespacedName{Namespace: template.Namespace, Name: template.Spec.ModelConfig.Name}, model); err != nil { - return nil, fmt.Errorf("resolve ModelConfig %q: %w", template.Spec.ModelConfig.Name, err) + var model *v1alpha3.ModelConfig + if resolveModel { + model = &v1alpha3.ModelConfig{} + if err := c.kube.Get(ctx, types.NamespacedName{Namespace: template.Namespace, Name: template.Spec.ModelConfig.Name}, model); err != nil { + return nil, fmt.Errorf("resolve ModelConfig %q: %w", template.Spec.ModelConfig.Name, err) + } } instruction, err := c.resolveAgentTemplatePrompt(ctx, template) if err != nil { diff --git a/go/core/v2/translator/compiler_test.go b/go/core/v2/translator/compiler_test.go index 771bff0ca0..51faf88765 100644 --- a/go/core/v2/translator/compiler_test.go +++ b/go/core/v2/translator/compiler_test.go @@ -8,6 +8,7 @@ import ( "testing" "github.com/kagent-dev/kagent/go/api/adk" + dbpkg "github.com/kagent-dev/kagent/go/api/database" "github.com/kagent-dev/kagent/go/api/v1alpha3" v2translator "github.com/kagent-dev/kagent/go/core/v2/translator" kagenttranslator "github.com/kagent-dev/kagent/go/core/v2/translator/kagent" @@ -34,8 +35,8 @@ func TestCompileAgentTemplatePinsAgentPluginSources(t *testing.T) { Spec: v1alpha3.HarnessSpec{ Kagent: &v1alpha3.KagentHarness{}, AllowedAgentTemplates: &v1alpha3.HarnessAgentTemplateAdmission{Selector: metav1.LabelSelector{}}, - Workload: v1alpha3.HarnessWorkload{Image: "example.com/kagent@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}, - Substrate: v1alpha3.HarnessSubstratePolicy{ + Workload: &v1alpha3.HarnessWorkload{Image: "example.com/kagent@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}, + Substrate: &v1alpha3.HarnessSubstratePolicy{ WorkerPoolRef: corev1.LocalObjectReference{Name: "default"}, SnapshotPolicy: v1alpha3.HarnessSnapshotPolicy{Location: "snapshots"}, }, }, @@ -69,6 +70,7 @@ func TestCompileAgentTemplatePinsAgentPluginSources(t *testing.T) { if err != nil { t.Fatal(err) } + require.Equal(t, dbpkg.RuntimeBackendKindSubstrate, spec.BackendKind) var config adk.AgentConfig if err := json.Unmarshal(spec.ConfigJSON, &config); err != nil { t.Fatal(err) @@ -122,7 +124,7 @@ func (c *testHarnessCompiler) Compile(_ context.Context, input *v2translator.Har func TestCompilerAcceptsExternalHarnessCompiler(t *testing.T) { require.NoError(t, v1alpha3.AddToScheme(schemev1.Scheme)) - kube := fake.NewClientBuilder().WithScheme(schemev1.Scheme).WithObjects(modelConfig()).Build() + kube := fake.NewClientBuilder().WithScheme(schemev1.Scheme).Build() adapter := &testHarnessCompiler{} harness := &v1alpha3.Harness{ ObjectMeta: metav1.ObjectMeta{Name: "codex", Namespace: "test"}, @@ -136,6 +138,7 @@ func TestCompilerAcceptsExternalHarnessCompiler(t *testing.T) { require.NoError(t, err) require.Equal(t, "assistant", revision.AgentTemplateName) require.Equal(t, template.Name, adapter.input.Root.Template.Name) + require.Nil(t, adapter.input.Root.ModelConfig) } func TestCompileAgentTemplateResolvesCredentialsForSubstrate(t *testing.T) { @@ -166,8 +169,8 @@ func TestCompileAgentTemplateResolvesCredentialsForSubstrate(t *testing.T) { Spec: v1alpha3.HarnessSpec{ Kagent: &v1alpha3.KagentHarness{}, AllowedAgentTemplates: &v1alpha3.HarnessAgentTemplateAdmission{Selector: metav1.LabelSelector{}}, - Workload: v1alpha3.HarnessWorkload{Image: "example.com/kagent@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}, - Substrate: v1alpha3.HarnessSubstratePolicy{ + Workload: &v1alpha3.HarnessWorkload{Image: "example.com/kagent@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}, + Substrate: &v1alpha3.HarnessSubstratePolicy{ WorkerPoolRef: corev1.LocalObjectReference{Name: "default"}, SnapshotPolicy: v1alpha3.HarnessSnapshotPolicy{Location: "snapshots"}, }, @@ -222,8 +225,8 @@ func TestCompileAgentTemplateSharedAgent(t *testing.T) { ObjectMeta: metav1.ObjectMeta{Name: "kagent", Namespace: "test"}, Spec: v1alpha3.HarnessSpec{ Kagent: &v1alpha3.KagentHarness{}, AllowedAgentTemplates: selector, - Workload: v1alpha3.HarnessWorkload{Image: "example.com/kagent@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}, - Substrate: v1alpha3.HarnessSubstratePolicy{WorkerPoolRef: corev1.LocalObjectReference{Name: "default"}, SnapshotPolicy: v1alpha3.HarnessSnapshotPolicy{Location: "snapshots"}}, + Workload: &v1alpha3.HarnessWorkload{Image: "example.com/kagent@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}, + Substrate: &v1alpha3.HarnessSubstratePolicy{WorkerPoolRef: corev1.LocalObjectReference{Name: "default"}, SnapshotPolicy: v1alpha3.HarnessSnapshotPolicy{Location: "snapshots"}}, }, } child := &v1alpha3.AgentTemplate{ diff --git a/go/core/v2/translator/external/compiler.go b/go/core/v2/translator/external/compiler.go new file mode 100644 index 0000000000..f4aff4e3f8 --- /dev/null +++ b/go/core/v2/translator/external/compiler.go @@ -0,0 +1,228 @@ +package external + +import ( + "context" + "encoding/json" + "fmt" + "slices" + "strings" + + a2atype "github.com/a2aproject/a2a-go/v2/a2a" + dbpkg "github.com/kagent-dev/kagent/go/api/database" + "github.com/kagent-dev/kagent/go/api/v1alpha3" + v2translator "github.com/kagent-dev/kagent/go/core/v2/translator" +) + +const profileVersion = "v1" + +// Compiler translates one external Harness runtime into a sanitized profile. +// Model selection remains authoritative at the external runtime and therefore +// is deliberately absent from the persisted profile and revision digest. +type Compiler struct { + runtime dbpkg.ExternalRuntime +} + +var _ v2translator.HarnessCompiler = (*Compiler)(nil) + +type profile struct { + Version string `json:"version"` + Instruction string `json:"instruction"` + Tools []profileTool `json:"tools"` +} + +type profileTool struct { + Server string `json:"server"` + Allow []string `json:"allow"` +} + +type provenance struct { + Version string `json:"version"` + Harness resourceRef `json:"harness"` + AgentTemplate resourceRef `json:"agentTemplate"` + MCPServers []resourceRef `json:"mcpServers"` +} + +type resourceRef struct { + Kind string `json:"kind"` + Namespace string `json:"namespace"` + Name string `json:"name"` + UID string `json:"uid,omitempty"` +} + +// NewCompiler constructs a compiler for one supported external runtime. +func NewCompiler(runtime dbpkg.ExternalRuntime) (*Compiler, error) { + if runtime != dbpkg.ExternalRuntimeCodex && runtime != dbpkg.ExternalRuntimeClaude { + return nil, fmt.Errorf("external Harness compiler runtime is not supported") + } + return &Compiler{runtime: runtime}, nil +} + +// Compile emits only the resolved instruction and logical MCP tool allowlist. +// Local runtime capabilities, credentials, endpoints, and execution profile +// choices are owned and validated by the connected client. +func (c *Compiler) Compile(_ context.Context, input *v2translator.HarnessInput) (*v2translator.Revision, error) { + if err := c.validateInput(input); err != nil { + return nil, err + } + tools, err := compileTools(input.Root.MCPTools) + if err != nil { + return nil, err + } + externalProfile, err := json.Marshal(profile{Version: profileVersion, Instruction: input.Root.Instruction, Tools: tools}) + if err != nil { + return nil, fmt.Errorf("marshal external runtime profile: %w", err) + } + agentCard, err := json.Marshal(externalAgentCard(input.Root.Template)) + if err != nil { + return nil, fmt.Errorf("marshal external runtime Agent Card: %w", err) + } + sourceSnapshot, err := json.Marshal(externalProvenance(input)) + if err != nil { + return nil, fmt.Errorf("marshal external runtime provenance: %w", err) + } + + return &v2translator.Revision{ + Namespace: input.Root.Template.Namespace, + AgentTemplateName: input.Root.Template.Name, + HarnessName: input.Harness.Name, + BackendKind: dbpkg.RuntimeBackendKindExternal, + ExternalRuntime: c.runtime, + ExternalProfile: externalProfile, + AgentCardJSON: agentCard, + Provenance: sourceSnapshot, + EgressDestinations: []string{}, + }, nil +} + +func (c *Compiler) validateInput(input *v2translator.HarnessInput) error { + if c == nil { + return v2translator.NewValidationError("external Harness compiler is nil") + } + if input == nil || input.Harness == nil { + return v2translator.NewValidationError("external Harness compiler requires a Harness") + } + if input.Root == nil || input.Root.Template == nil { + return v2translator.NewValidationError("external Harness compiler requires a resolved root AgentTemplate") + } + if !harnessSelectsRuntime(input.Harness, c.runtime) { + return v2translator.NewValidationError("external Harness compiler runtime does not match the Harness") + } + if input.Harness.Spec.Workload != nil || input.Harness.Spec.Substrate != nil || input.Harness.Spec.Env != nil { + return v2translator.NewValidationError("external Harnesses do not support workload, substrate, or env") + } + if len(input.Root.Shared) != 0 { + return v2translator.NewValidationError("external Harness profiles do not support Shared AgentTemplate tools yet") + } + if len(input.Root.Template.Spec.Skills) != 0 { + return v2translator.NewValidationError("external Harness profiles do not support AgentTemplate skills yet") + } + if len(input.Root.Template.Spec.Plugins) != 0 { + return v2translator.NewValidationError("external Harness profiles do not support AgentTemplate plugins yet") + } + return nil +} + +func harnessSelectsRuntime(harness *v1alpha3.Harness, runtime dbpkg.ExternalRuntime) bool { + if harness.Spec.Kagent != nil { + return false + } + switch runtime { + case dbpkg.ExternalRuntimeCodex: + return harness.Spec.Codex != nil && harness.Spec.Claude == nil + case dbpkg.ExternalRuntimeClaude: + return harness.Spec.Claude != nil && harness.Spec.Codex == nil + default: + return false + } +} + +func compileTools(resolved []v2translator.ResolvedMCPTool) ([]profileTool, error) { + allowByServer := make(map[string]map[string]struct{}, len(resolved)) + for _, tool := range resolved { + server := tool.Binding.Server.Name + if tool.Binding.Server.Kind != "RemoteMCPServer" || server == "" || tool.Server == nil || tool.Server.Name != server { + return nil, v2translator.NewValidationError("external Harness profile contains an invalid logical MCP server") + } + if len(tool.Binding.Tools) == 0 { + return nil, v2translator.NewValidationError("external Harness profile requires a non-empty MCP tool allowlist") + } + if len(tool.Server.Spec.HeadersFrom) != 0 || (tool.Server.Spec.TLS != nil && !tool.Server.Spec.TLS.IsEmpty()) { + return nil, v2translator.NewValidationError("external Harness profiles cannot use RemoteMCPServer headers or TLS configuration") + } + allow := allowByServer[server] + if allow == nil { + allow = make(map[string]struct{}, len(tool.Binding.Tools)) + allowByServer[server] = allow + } + for _, name := range tool.Binding.Tools { + if name == "" { + return nil, v2translator.NewValidationError("external Harness profile contains an empty MCP tool name") + } + allow[name] = struct{}{} + } + } + + servers := make([]string, 0, len(allowByServer)) + for server := range allowByServer { + servers = append(servers, server) + } + slices.Sort(servers) + tools := make([]profileTool, 0, len(servers)) + for _, server := range servers { + allowSet := allowByServer[server] + allow := make([]string, 0, len(allowSet)) + for name := range allowSet { + allow = append(allow, name) + } + slices.Sort(allow) + tools = append(tools, profileTool{Server: server, Allow: allow}) + } + return tools, nil +} + +func externalAgentCard(template *v1alpha3.AgentTemplate) *a2atype.AgentCard { + return &a2atype.AgentCard{ + Name: strings.ReplaceAll(template.Name, "-", "_"), + Description: template.Spec.Description, + Version: profileVersion, + DefaultInputModes: []string{"text/plain"}, + DefaultOutputModes: []string{"text/plain"}, + Skills: []a2atype.AgentSkill{}, + Capabilities: a2atype.AgentCapabilities{Streaming: false}, + // The public gateway replaces this private placeholder with its own + // authenticated gRPC interface before returning the card to callers. + SupportedInterfaces: []*a2atype.AgentInterface{a2atype.NewAgentInterface( + "http://127.0.0.1", a2atype.TransportProtocolJSONRPC, + )}, + } +} + +func externalProvenance(input *v2translator.HarnessInput) provenance { + servers := make([]resourceRef, 0, len(input.Root.MCPTools)) + for _, tool := range input.Root.MCPTools { + server := tool.Server + if server == nil { + continue + } + servers = append(servers, resourceRef{ + Kind: "RemoteMCPServer", Namespace: server.Namespace, Name: server.Name, UID: string(server.UID), + }) + } + slices.SortFunc(servers, func(left, right resourceRef) int { + leftKey := left.Namespace + "\x00" + left.Name + "\x00" + left.UID + rightKey := right.Namespace + "\x00" + right.Name + "\x00" + right.UID + return strings.Compare(leftKey, rightKey) + }) + servers = slices.Compact(servers) + + template := input.Root.Template + harness := input.Harness + return provenance{ + Version: profileVersion, + Harness: resourceRef{Kind: "Harness", Namespace: harness.Namespace, Name: harness.Name, UID: string(harness.UID)}, + AgentTemplate: resourceRef{ + Kind: "AgentTemplate", Namespace: template.Namespace, Name: template.Name, UID: string(template.UID), + }, + MCPServers: servers, + } +} diff --git a/go/core/v2/translator/external/compiler_test.go b/go/core/v2/translator/external/compiler_test.go new file mode 100644 index 0000000000..ec8d58ce1f --- /dev/null +++ b/go/core/v2/translator/external/compiler_test.go @@ -0,0 +1,233 @@ +package external_test + +import ( + "context" + "encoding/json" + "testing" + + a2atype "github.com/a2aproject/a2a-go/v2/a2a" + dbpkg "github.com/kagent-dev/kagent/go/api/database" + "github.com/kagent-dev/kagent/go/api/v1alpha3" + v2translator "github.com/kagent-dev/kagent/go/core/v2/translator" + externaltranslator "github.com/kagent-dev/kagent/go/core/v2/translator/external" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestCompilerProducesCanonicalSanitizedProfile(t *testing.T) { + compiler := newCompiler(t, dbpkg.ExternalRuntimeCodex) + input := externalInput(dbpkg.ExternalRuntimeCodex) + input.Root.Instruction = "review carefully" + input.Root.MCPTools = []v2translator.ResolvedMCPTool{ + resolvedTool("z-tools", "write", "read", "write"), + resolvedTool("a-tools", "search", "fetch"), + resolvedTool("z-tools", "inspect"), + } + + revision, err := compiler.Compile(context.Background(), input) + require.NoError(t, err) + require.Equal(t, dbpkg.RuntimeBackendKindExternal, revision.BackendKind) + require.Equal(t, dbpkg.ExternalRuntimeCodex, revision.ExternalRuntime) + require.JSONEq(t, `{ + "version":"v1", + "instruction":"review carefully", + "tools":[ + {"server":"a-tools","allow":["fetch","search"]}, + {"server":"z-tools","allow":["inspect","read","write"]} + ] + }`, string(revision.ExternalProfile)) + require.Equal(t, `{"version":"v1","instruction":"review carefully","tools":[{"server":"a-tools","allow":["fetch","search"]},{"server":"z-tools","allow":["inspect","read","write"]}]}`, string(revision.ExternalProfile)) + require.Empty(t, revision.Image) + require.Nil(t, revision.Environment) + require.Nil(t, revision.ConfigJSON) + require.NotEmpty(t, revision.AgentCardJSON) + var card a2atype.AgentCard + require.NoError(t, json.Unmarshal(revision.AgentCardJSON, &card)) + require.Equal(t, "assistant", card.Name) + require.Equal(t, "v1", card.Version) + require.Len(t, card.SupportedInterfaces, 1) + require.Equal(t, a2atype.TransportProtocolJSONRPC, card.SupportedInterfaces[0].ProtocolBinding) + require.Equal(t, a2atype.Version, card.SupportedInterfaces[0].ProtocolVersion) + require.NotEmpty(t, card.DefaultInputModes) + require.NotEmpty(t, card.DefaultOutputModes) + require.NotContains(t, string(revision.AgentCardJSON), "execution-profile") + require.Empty(t, revision.WorkerPoolName) + require.Empty(t, revision.SnapshotLocation) + require.NotEmpty(t, revision.Provenance) + require.Equal(t, `{"version":"v1","harness":{"kind":"Harness","namespace":"test-namespace","name":"codex"},"agentTemplate":{"kind":"AgentTemplate","namespace":"test-namespace","name":"assistant"},"mcpServers":[{"kind":"RemoteMCPServer","namespace":"test-namespace","name":"a-tools"},{"kind":"RemoteMCPServer","namespace":"test-namespace","name":"z-tools"}]}`, string(revision.Provenance)) + require.NotNil(t, revision.EgressDestinations) + require.Empty(t, revision.EgressDestinations) + require.NotContains(t, string(revision.ExternalProfile), "mcp.example.test") + require.NotContains(t, string(revision.Provenance), "mcp.example.test") + require.NotContains(t, string(revision.ExternalProfile), "test-namespace") +} + +func TestCompilerAllowsEmptyProfile(t *testing.T) { + revision, err := newCompiler(t, dbpkg.ExternalRuntimeClaude).Compile(context.Background(), externalInput(dbpkg.ExternalRuntimeClaude)) + require.NoError(t, err) + require.Equal(t, `{"version":"v1","instruction":"","tools":[]}`, string(revision.ExternalProfile)) +} + +func TestCompilerDigestIgnoresModelConfiguration(t *testing.T) { + compiler := newCompiler(t, dbpkg.ExternalRuntimeCodex) + firstInput := externalInput(dbpkg.ExternalRuntimeCodex) + effort := v1alpha3.OpenAIReasoningEffort("low") + firstInput.Root.ModelConfig.Spec = v1alpha3.ModelConfigSpec{ + Model: "gpt-first", Provider: v1alpha3.ModelProviderOpenAI, + APIKeySecret: "cluster-model-key", APIKeySecretKey: "token", + DefaultHeaders: map[string]string{"Authorization": "credential"}, + TLS: &v1alpha3.TLSConfig{CACertSecretRef: "cluster-ca", CACertSecretKey: "ca.crt"}, + OpenAI: &v1alpha3.OpenAIConfig{ReasoningEffort: &effort, BaseURL: "https://first.example.test"}, + } + firstInput.Root.ModelConfig.Name = "first-model" + firstInput.Root.ModelConfig.UID = "first-model-uid" + firstInput.Root.ModelConfig.Generation = 7 + + secondInput := externalInput(dbpkg.ExternalRuntimeCodex) + secondInput.Root.ModelConfig = nil + + first, err := compiler.Compile(context.Background(), firstInput) + require.NoError(t, err) + second, err := compiler.Compile(context.Background(), secondInput) + require.NoError(t, err) + require.Equal(t, first.ExternalProfile, second.ExternalProfile) + require.Equal(t, first.AgentCardJSON, second.AgentCardJSON) + require.NotContains(t, string(first.ExternalProfile), "cluster-model-key") + require.NotContains(t, string(first.AgentCardJSON), "cluster-model-key") + firstDigest, err := first.Digest() + require.NoError(t, err) + secondDigest, err := second.Digest() + require.NoError(t, err) + require.Equal(t, firstDigest, secondDigest) +} + +func TestCompilerRejectsUnsupportedConfiguration(t *testing.T) { + literal := "value" + tests := []struct { + name string + mutate func(*v2translator.HarnessInput) + wantErr string + }{ + { + name: "workload", + mutate: func(input *v2translator.HarnessInput) { + input.Harness.Spec.Workload = &v1alpha3.HarnessWorkload{Image: "example.test/runtime@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"} + }, + wantErr: "workload, substrate, or env", + }, + { + name: "env", + mutate: func(input *v2translator.HarnessInput) { + input.Harness.Spec.Env = []v1alpha3.HarnessEnvVar{{Name: "TOKEN", Value: &literal}} + }, + wantErr: "workload, substrate, or env", + }, + { + name: "skills", + mutate: func(input *v2translator.HarnessInput) { + input.Root.Template.Spec.Skills = []v1alpha3.AgentTemplateSkill{{Name: "review"}} + }, + wantErr: "skills", + }, + { + name: "plugins", + mutate: func(input *v2translator.HarnessInput) { + input.Root.Template.Spec.Plugins = []v1alpha3.PluginBundle{{}} + }, + wantErr: "plugins", + }, + { + name: "shared child", + mutate: func(input *v2translator.HarnessInput) { + input.Root.Shared = []v2translator.AgentInputBinding{{Name: "child", Agent: &v2translator.AgentInput{}}} + }, + wantErr: "Shared", + }, + { + name: "MCP headers", + mutate: func(input *v2translator.HarnessInput) { + tool := resolvedTool("tools", "read") + tool.Server.Spec.HeadersFrom = []v1alpha3.ValueRef{{Name: "Authorization", Value: "secret"}} + input.Root.MCPTools = []v2translator.ResolvedMCPTool{tool} + }, + wantErr: "headers or TLS", + }, + { + name: "empty MCP allowlist", + mutate: func(input *v2translator.HarnessInput) { + input.Root.MCPTools = []v2translator.ResolvedMCPTool{resolvedTool("tools")} + }, + wantErr: "non-empty MCP tool allowlist", + }, + { + name: "MCP cluster CA", + mutate: func(input *v2translator.HarnessInput) { + tool := resolvedTool("tools", "read") + tool.Server.Spec.TLS = &v1alpha3.TLSConfig{CACertSecretRef: "mcp-ca", CACertSecretKey: "ca.crt"} + input.Root.MCPTools = []v2translator.ResolvedMCPTool{tool} + }, + wantErr: "headers or TLS", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + input := externalInput(dbpkg.ExternalRuntimeCodex) + test.mutate(input) + _, err := newCompiler(t, dbpkg.ExternalRuntimeCodex).Compile(context.Background(), input) + require.ErrorContains(t, err, test.wantErr) + }) + } +} + +func TestCompilerRejectsMismatchedAndInvalidRuntimes(t *testing.T) { + _, err := externaltranslator.NewCompiler(dbpkg.ExternalRuntime("other")) + require.ErrorContains(t, err, "not supported") + + compiler := newCompiler(t, dbpkg.ExternalRuntimeCodex) + _, err = compiler.Compile(context.Background(), externalInput(dbpkg.ExternalRuntimeClaude)) + require.ErrorContains(t, err, "does not match") +} + +func newCompiler(t *testing.T, runtime dbpkg.ExternalRuntime) *externaltranslator.Compiler { + t.Helper() + compiler, err := externaltranslator.NewCompiler(runtime) + require.NoError(t, err) + return compiler +} + +func externalInput(runtime dbpkg.ExternalRuntime) *v2translator.HarnessInput { + harness := &v1alpha3.Harness{ObjectMeta: metav1.ObjectMeta{Name: string(runtime), Namespace: "test-namespace"}} + switch runtime { + case dbpkg.ExternalRuntimeCodex: + harness.Spec.Codex = &v1alpha3.CodexHarness{} + case dbpkg.ExternalRuntimeClaude: + harness.Spec.Claude = &v1alpha3.ClaudeHarness{} + } + return &v2translator.HarnessInput{ + Harness: harness, + Root: &v2translator.AgentInput{ + Template: &v1alpha3.AgentTemplate{ObjectMeta: metav1.ObjectMeta{Name: "assistant", Namespace: "test-namespace"}}, + ModelConfig: &v1alpha3.ModelConfig{ + ObjectMeta: metav1.ObjectMeta{Name: "ignored-model", Namespace: "test-namespace"}, + Spec: v1alpha3.ModelConfigSpec{Model: "ignored", Provider: v1alpha3.ModelProviderOpenAI}, + }, + }, + } +} + +func resolvedTool(server string, allow ...string) v2translator.ResolvedMCPTool { + return v2translator.ResolvedMCPTool{ + Binding: v1alpha3.MCPToolBinding{ + Server: v1alpha3.AgentTemplateTypedLocalReference{Kind: "RemoteMCPServer", Name: server}, + Tools: allow, + }, + Server: &v1alpha3.RemoteMCPServer{ + ObjectMeta: metav1.ObjectMeta{Name: server, Namespace: "test-namespace"}, + Spec: v1alpha3.RemoteMCPServerSpec{ + URL: "https://mcp.example.test/" + server, + Protocol: v1alpha3.RemoteMCPServerProtocolStreamableHttp, + }, + }, + } +} diff --git a/go/core/v2/translator/kagent/compiler.go b/go/core/v2/translator/kagent/compiler.go index 5be6d58905..ce2036a1ce 100644 --- a/go/core/v2/translator/kagent/compiler.go +++ b/go/core/v2/translator/kagent/compiler.go @@ -11,6 +11,7 @@ import ( a2atype "github.com/a2aproject/a2a-go/v2/a2a" "github.com/kagent-dev/kagent/go/api/adk" + dbpkg "github.com/kagent-dev/kagent/go/api/database" "github.com/kagent-dev/kagent/go/api/v1alpha3" "github.com/kagent-dev/kagent/go/core/internal/utils" "github.com/kagent-dev/kagent/go/core/pkg/env" @@ -46,6 +47,9 @@ type compiledAgent struct { } func (c *Compiler) Compile(ctx context.Context, input *v2translator.HarnessInput) (*v2translator.Revision, error) { + if err := validateInput(input); err != nil { + return nil, err + } compiled, err := c.compileAgent(ctx, input.Root) if err != nil { return nil, err @@ -112,6 +116,7 @@ func (c *Compiler) Compile(ctx context.Context, input *v2translator.HarnessInput Namespace: template.Namespace, AgentTemplateName: template.Name, HarnessName: harness.Name, + BackendKind: dbpkg.RuntimeBackendKindSubstrate, Image: harness.Spec.Workload.Image, Environment: environment, ConfigJSON: configJSON, @@ -123,6 +128,23 @@ func (c *Compiler) Compile(ctx context.Context, input *v2translator.HarnessInput }, nil } +func validateInput(input *v2translator.HarnessInput) error { + if input == nil || input.Harness == nil { + return v2translator.NewValidationError("kagent compiler requires a Harness") + } + if input.Root == nil || input.Root.Template == nil || input.Root.ModelConfig == nil { + return v2translator.NewValidationError("kagent compiler requires a resolved root AgentTemplate and ModelConfig") + } + harness := input.Harness + if harness.Spec.Kagent == nil || harness.Spec.Codex != nil || harness.Spec.Claude != nil { + return v2translator.NewValidationError("kagent compiler requires the kagent Harness runtime") + } + if harness.Spec.Workload == nil || harness.Spec.Substrate == nil { + return v2translator.NewValidationError("kagent compiler requires workload and substrate") + } + return nil +} + func (c *Compiler) compileAgent(ctx context.Context, input *v2translator.AgentInput) (*compiledAgent, error) { modelRuntime, err := c.resolveModel(ctx, input.ModelConfig) if err != nil { diff --git a/go/core/v2/translator/kagent/compiler_validation_test.go b/go/core/v2/translator/kagent/compiler_validation_test.go new file mode 100644 index 0000000000..89bd3b9f05 --- /dev/null +++ b/go/core/v2/translator/kagent/compiler_validation_test.go @@ -0,0 +1,54 @@ +package kagent + +import ( + "context" + "testing" + + "github.com/kagent-dev/kagent/go/api/v1alpha3" + v2translator "github.com/kagent-dev/kagent/go/core/v2/translator" + "github.com/stretchr/testify/require" +) + +func TestCompilerValidatesRuntimeInputsBeforeDereferencingThem(t *testing.T) { + tests := []struct { + name string + input *v2translator.HarnessInput + wantErr string + }{ + {name: "nil input", wantErr: "requires a Harness"}, + { + name: "missing root", + input: &v2translator.HarnessInput{Harness: &v1alpha3.Harness{Spec: v1alpha3.HarnessSpec{ + Kagent: &v1alpha3.KagentHarness{}, Workload: &v1alpha3.HarnessWorkload{}, Substrate: &v1alpha3.HarnessSubstratePolicy{}, + }}}, + wantErr: "resolved root AgentTemplate and ModelConfig", + }, + { + name: "wrong runtime", + input: &v2translator.HarnessInput{ + Harness: &v1alpha3.Harness{Spec: v1alpha3.HarnessSpec{Codex: &v1alpha3.CodexHarness{}}}, + Root: resolvedRoot(), + }, + wantErr: "requires the kagent Harness runtime", + }, + { + name: "missing substrate policy", + input: &v2translator.HarnessInput{ + Harness: &v1alpha3.Harness{Spec: v1alpha3.HarnessSpec{Kagent: &v1alpha3.KagentHarness{}, Workload: &v1alpha3.HarnessWorkload{}}}, + Root: resolvedRoot(), + }, + wantErr: "requires workload and substrate", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, err := (&Compiler{}).Compile(context.Background(), test.input) + require.ErrorContains(t, err, test.wantErr) + }) + } +} + +func resolvedRoot() *v2translator.AgentInput { + return &v2translator.AgentInput{Template: &v1alpha3.AgentTemplate{}, ModelConfig: &v1alpha3.ModelConfig{}} +} diff --git a/go/core/v2/translator/revision.go b/go/core/v2/translator/revision.go index fa72e0a7ba..b2b1ca345a 100644 --- a/go/core/v2/translator/revision.go +++ b/go/core/v2/translator/revision.go @@ -6,6 +6,7 @@ import ( "encoding/json" "fmt" + dbpkg "github.com/kagent-dev/kagent/go/api/database" corev1 "k8s.io/api/core/v1" ) @@ -30,6 +31,12 @@ type Revision struct { Namespace string AgentTemplateName string HarnessName string + // BackendKind and ExternalRuntime select the immutable private runtime + // boundary. ExternalProfile is the sanitized runtime-neutral configuration + // dispatched to a connected external runtime. + BackendKind dbpkg.RuntimeBackendKind + ExternalRuntime dbpkg.ExternalRuntime + ExternalProfile json.RawMessage // Image and Environment describe the runtime container. Image string @@ -57,6 +64,9 @@ func (r *Revision) Digest() (RevisionID, error) { Namespace string `json:"namespace"` AgentTemplateName string `json:"agentTemplateName"` HarnessName string `json:"harnessName"` + BackendKind string `json:"backendKind"` + ExternalRuntime string `json:"externalRuntime,omitempty"` + ExternalProfile json.RawMessage `json:"externalProfile,omitempty"` Image string `json:"image"` Environment []corev1.EnvVar `json:"environment"` ConfigJSON json.RawMessage `json:"config"` @@ -67,6 +77,7 @@ func (r *Revision) Digest() (RevisionID, error) { EgressDestinations []string `json:"egressDestinations"` }{ Namespace: r.Namespace, AgentTemplateName: r.AgentTemplateName, HarnessName: r.HarnessName, + BackendKind: string(r.BackendKind), ExternalRuntime: string(r.ExternalRuntime), ExternalProfile: r.ExternalProfile, Image: r.Image, Environment: r.Environment, ConfigJSON: r.ConfigJSON, AgentCardJSON: r.AgentCardJSON, WorkerPoolName: r.WorkerPoolName, SnapshotLocation: r.SnapshotLocation, Provenance: r.Provenance, EgressDestinations: r.EgressDestinations, diff --git a/go/core/v2/translator/revision_test.go b/go/core/v2/translator/revision_test.go index b137771b9f..f3432358b9 100644 --- a/go/core/v2/translator/revision_test.go +++ b/go/core/v2/translator/revision_test.go @@ -3,6 +3,8 @@ package translator import ( "strings" "testing" + + dbpkg "github.com/kagent-dev/kagent/go/api/database" ) func TestRevisionDigestIncludesProvenance(t *testing.T) { @@ -23,3 +25,32 @@ func TestRevisionDigestIncludesProvenance(t *testing.T) { t.Fatalf("short revision %q is not a prefix of %q", first.Short(), first.String()) } } + +func TestRevisionDigestIncludesBackendIdentity(t *testing.T) { + revision := &Revision{ + Namespace: "agents", AgentTemplateName: "helper", HarnessName: "codex", + BackendKind: dbpkg.RuntimeBackendKindExternal, ExternalRuntime: dbpkg.ExternalRuntimeCodex, + ExternalProfile: []byte(`{"version":"v1"}`), + } + first, err := revision.Digest() + if err != nil { + t.Fatal(err) + } + revision.ExternalRuntime = dbpkg.ExternalRuntimeClaude + second, err := revision.Digest() + if err != nil { + t.Fatal(err) + } + if first == second { + t.Fatal("external runtime did not change runtime revision") + } + revision.ExternalRuntime = dbpkg.ExternalRuntimeCodex + revision.ExternalProfile = []byte(`{"version":"v1","instruction":"help"}`) + third, err := revision.Digest() + if err != nil { + t.Fatal(err) + } + if first == third { + t.Fatal("external profile did not change runtime revision") + } +} diff --git a/helm/kagent-crds/templates/kagent.dev_harnesses.yaml b/helm/kagent-crds/templates/kagent.dev_harnesses.yaml index 0ef8b5f9be..478c88c87c 100644 --- a/helm/kagent-crds/templates/kagent.dev_harnesses.yaml +++ b/helm/kagent-crds/templates/kagent.dev_harnesses.yaml @@ -163,8 +163,8 @@ spec: description: KagentHarness selects the kagent runtime adapter. type: object substrate: - description: HarnessSubstratePolicy contains the Substrate policy - shared by all runtime variants. + description: Substrate is required by kagent and forbidden by external + runtimes. properties: snapshotPolicy: description: SnapshotPolicy configures runtime snapshot storage. @@ -200,8 +200,8 @@ spec: - message: workerPoolRef name must not be empty rule: self.workerPoolRef.name.size() > 0 workload: - description: HarnessWorkload identifies the immutable runtime image - used by a Harness. + description: Workload is required by kagent and forbidden by external + runtimes. properties: image: description: Image is an OCI image reference pinned by sha256 @@ -211,14 +211,16 @@ spec: required: - image type: object - required: - - substrate - - workload type: object x-kubernetes-validations: - message: exactly one of kagent, codex, or claude must be specified rule: '(has(self.kagent) ? 1 : 0) + (has(self.codex) ? 1 : 0) + (has(self.claude) ? 1 : 0) == 1' + - message: kagent requires workload and substrate + rule: '!has(self.kagent) || (has(self.workload) && has(self.substrate))' + - message: codex and claude forbid workload, substrate, and env + rule: has(self.kagent) || (!has(self.workload) && !has(self.substrate) + && !has(self.env)) status: description: HarnessStatus reports controller-derived capabilities and current health. diff --git a/helm/kagent/README.md b/helm/kagent/README.md new file mode 100644 index 0000000000..14b6293b61 --- /dev/null +++ b/helm/kagent/README.md @@ -0,0 +1,52 @@ +# kagent chart + +See the [Helm installation guide](../README.md) for general installation and +upgrade instructions. This page documents chart-specific external runtime +gateway settings. See [External runtime Harnesses](../../docs/external-runtime-harness.md) +for the portable profile boundary and rollout constraints. + +## External runtime gateway + +The external runtime gateway lets a local Codex or Claude Code connector make +an outbound connection to the in-cluster kagent controller. It is disabled by +default and has no inline token value: create the token Secret in the release +namespace before enabling the gateway. + +```bash +kubectl -n kagent create secret generic kagent-external-gateway-token \ + --from-file=token=/secure/path/to/device-token +``` + +The token must contain 32-4096 visible ASCII bytes. A minimal values file is: + +```yaml +controller: + replicas: 1 + externalGateway: + enabled: true + deviceId: workstation-1 + codexSlotId: codex-1 + claudeSlotId: "" + existingSecret: + name: kagent-external-gateway-token + key: token +``` + +`deviceId` and configured slot IDs must be lowercase DNS labels. Configure at +least one slot. The chart mounts only the selected Secret key, read-only, and +passes its file path to the controller. It does not create a Secret or accept a +token, encoded token, or `values_base64` field in Helm values. + +Enabling the gateway creates a dedicated +`-kagent-external-gateway` `ClusterIP` Service on port 8085. It never +inherits `controller.service.type`, so an existing controller `LoadBalancer` or +`NodePort` cannot publish the bearer-token transport. The chart does not add an +Ingress, external load balancer, TLS termination, Cloudflare resource, or +NetworkPolicy. Configure external exposure separately and terminate TLS before +traffic reaches this internal Service. + +Gateway sessions currently live in controller memory. The chart therefore +requires `controller.replicas: 1` and switches the Deployment to `Recreate` so +an upgrade cannot briefly run two brokers. An upgrade or restart disconnects +clients, which must reconnect. Changing the Secret contents also requires a +controller rollout because the token is read when the process starts. diff --git a/helm/kagent/templates/_helpers.tpl b/helm/kagent/templates/_helpers.tpl index ce184a2f8b..e29f52dcbd 100644 --- a/helm/kagent/templates/_helpers.tpl +++ b/helm/kagent/templates/_helpers.tpl @@ -181,6 +181,88 @@ Check if leader election should be enabled (more than 1 replica) {{- gt (.Values.controller.replicas | int) 1 -}} {{- end -}} +{{/* +Validate the external runtime gateway configuration. The broker owns sessions +in memory, so both the steady-state replica count and rollout strategy must +prevent two controller pods from accepting the same device token. +*/}} +{{- define "kagent.controller.externalGateway.validate" -}} +{{- $externalGateway := .Values.controller.externalGateway | default dict -}} +{{- range $entry := .Values.controller.env | default list -}} + {{- $name := get $entry "name" | default "" -}} + {{- if hasPrefix "EXTERNAL_GATEWAY_" $name -}} + {{- fail (printf "controller.env[%s] is reserved; configure the external gateway through controller.externalGateway" $name) -}} + {{- end -}} +{{- end -}} +{{- if (get $externalGateway "enabled") -}} + {{- if ne (.Values.controller.replicas | int) 1 -}} + {{- fail "controller.externalGateway requires controller.replicas=1 because gateway sessions are process-local" -}} + {{- end -}} + {{- $gatewayPort := include "kagent.controller.externalGateway.port" . -}} + {{- $grpcBindPort := regexFind "[0-9]+$" (.Values.controller.grpc.bindAddress | toString) -}} + {{- if eq $grpcBindPort $gatewayPort -}} + {{- fail (printf "controller.grpc.bindAddress must not use external gateway port %s" $gatewayPort) -}} + {{- end -}} + {{- if eq (.Values.controller.service.ports.grpc | toString) $gatewayPort -}} + {{- fail (printf "controller.service.ports.grpc must not use external gateway port %s" $gatewayPort) -}} + {{- end -}} + {{- if eq (.Values.controller.service.ports.targetPort | toString) $gatewayPort -}} + {{- fail (printf "controller.service.ports.targetPort must not use external gateway port %s" $gatewayPort) -}} + {{- end -}} + {{- if and (include "kagent.controller.metricsEnabled" .) (eq (include "kagent.controller.metricsPort" .) $gatewayPort) -}} + {{- fail (printf "controller.metrics.bindAddress must not use external gateway port %s" $gatewayPort) -}} + {{- end -}} + {{- $dnsLabelPattern := "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$" -}} + {{- $deviceID := get $externalGateway "deviceId" | default "" -}} + {{- if or (not $deviceID) (gt (len $deviceID) 63) (not (regexMatch $dnsLabelPattern $deviceID)) -}} + {{- fail "controller.externalGateway.deviceId must be a lowercase DNS label with at most 63 characters" -}} + {{- end -}} + {{- $codexSlotID := get $externalGateway "codexSlotId" | default "" -}} + {{- $claudeSlotID := get $externalGateway "claudeSlotId" | default "" -}} + {{- if and (not $codexSlotID) (not $claudeSlotID) -}} + {{- fail "controller.externalGateway requires at least one of codexSlotId or claudeSlotId" -}} + {{- end -}} + {{- if and $codexSlotID (or (gt (len $codexSlotID) 63) (not (regexMatch $dnsLabelPattern $codexSlotID))) -}} + {{- fail "controller.externalGateway.codexSlotId must be a lowercase DNS label with at most 63 characters" -}} + {{- end -}} + {{- if and $claudeSlotID (or (gt (len $claudeSlotID) 63) (not (regexMatch $dnsLabelPattern $claudeSlotID))) -}} + {{- fail "controller.externalGateway.claudeSlotId must be a lowercase DNS label with at most 63 characters" -}} + {{- end -}} + {{- $existingSecret := get $externalGateway "existingSecret" | default dict -}} + {{- $secretName := get $existingSecret "name" | default "" -}} + {{- if not $secretName -}} + {{- fail "controller.externalGateway.existingSecret.name is required" -}} + {{- end -}} + {{- $dnsSubdomainPattern := "^[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?)*$" -}} + {{- if or (gt (len $secretName) 253) (not (regexMatch $dnsSubdomainPattern $secretName)) -}} + {{- fail "controller.externalGateway.existingSecret.name must be a valid DNS subdomain with labels of at most 63 characters" -}} + {{- end -}} + {{- if not (get $existingSecret "key") -}} + {{- fail "controller.externalGateway.existingSecret.key is required" -}} + {{- end -}} + {{- range $volume := .Values.controller.volumes | default list -}} + {{- if eq (get $volume "name" | default "") "external-gateway-token" -}} + {{- fail "controller.volumes[external-gateway-token] is reserved while controller.externalGateway is enabled" -}} + {{- end -}} + {{- end -}} + {{- $reservedMountPath := include "kagent.controller.externalGateway.tokenMountPath" $ -}} + {{- range $volumeMount := .Values.controller.volumeMounts | default list -}} + {{- $mountPath := trimSuffix "/" (get $volumeMount "mountPath" | default "") -}} + {{- $overlapsReservedPath := and $mountPath (or + (eq $mountPath $reservedMountPath) + (hasPrefix (printf "%s/" $mountPath) $reservedMountPath) + (hasPrefix (printf "%s/" $reservedMountPath) $mountPath)) -}} + {{- if or (eq (get $volumeMount "name" | default "") "external-gateway-token") $overlapsReservedPath -}} + {{- fail "the external gateway token volume and mount path are reserved while controller.externalGateway is enabled" -}} + {{- end -}} + {{- end -}} +{{- end -}} +{{- end -}} + +{{- define "kagent.controller.externalGateway.port" -}}8085{{- end -}} +{{- define "kagent.controller.externalGateway.tokenMountPath" -}}/var/run/secrets/kagent/external-gateway{{- end -}} +{{- define "kagent.controller.externalGateway.tokenFile" -}}{{ include "kagent.controller.externalGateway.tokenMountPath" . }}/token{{- end -}} + {{/* Extract the TCP port from controller.metrics.bindAddress. diff --git a/helm/kagent/templates/controller-deployment.yaml b/helm/kagent/templates/controller-deployment.yaml index 7e74c94378..b9f1c26045 100644 --- a/helm/kagent/templates/controller-deployment.yaml +++ b/helm/kagent/templates/controller-deployment.yaml @@ -1,3 +1,5 @@ +{{- include "kagent.controller.externalGateway.validate" . }} +{{- $externalGateway := .Values.controller.externalGateway | default dict }} apiVersion: apps/v1 kind: Deployment metadata: @@ -12,6 +14,12 @@ metadata: {{- include "kagent.controller.labels" . | nindent 4 }} spec: replicas: {{ .Values.controller.replicas }} + {{- if $externalGateway.enabled }} + # Gateway sessions are process-local. Recreate avoids a rolling-update surge + # briefly running two independently authenticated brokers. + strategy: + type: Recreate + {{- end }} selector: matchLabels: {{- include "kagent.controller.selectorLabels" . | nindent 6 }} @@ -38,7 +46,7 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} serviceAccountName: {{ include "kagent.fullname" . }}-controller - {{- if or (gt (len .Values.controller.volumes) 0) (and .Values.controller.substrate .Values.controller.substrate.enabled) }} + {{- if or (gt (len .Values.controller.volumes) 0) (and .Values.controller.substrate .Values.controller.substrate.enabled) $externalGateway.enabled }} volumes: {{- if and .Values.controller.substrate .Values.controller.substrate.enabled }} - name: substrate-servicedns @@ -58,6 +66,15 @@ spec: keyType: ECDSAP256 credentialBundlePath: credential-bundle.pem {{- end }} + {{- if $externalGateway.enabled }} + - name: external-gateway-token + secret: + secretName: {{ $externalGateway.existingSecret.name | quote }} + defaultMode: 0444 + items: + - key: {{ $externalGateway.existingSecret.key | quote }} + path: token + {{- end }} {{- with .Values.controller.volumes }} {{- toYaml . | nindent 6 }} {{- end }} @@ -138,6 +155,27 @@ spec: - name: GRPC_TLS_KEY_FILE value: {{ . | quote }} {{- end }} + # Keep the disabled state explicit so controller.envFrom cannot + # activate the gateway without the corresponding Secret mount, + # Service port, placement validation, and single-replica policy. + - name: EXTERNAL_GATEWAY_ENABLED + value: {{ ($externalGateway.enabled | default false) | quote }} + {{- if $externalGateway.enabled }} + - name: EXTERNAL_GATEWAY_BIND_ADDRESS + value: ":{{ include "kagent.controller.externalGateway.port" . }}" + - name: EXTERNAL_GATEWAY_TOKEN_FILE + value: {{ include "kagent.controller.externalGateway.tokenFile" . | quote }} + - name: EXTERNAL_GATEWAY_DEVICE_ID + value: {{ $externalGateway.deviceId | quote }} + {{- with $externalGateway.codexSlotId }} + - name: EXTERNAL_GATEWAY_CODEX_SLOT_ID + value: {{ . | quote }} + {{- end }} + {{- with $externalGateway.claudeSlotId }} + - name: EXTERNAL_GATEWAY_CLAUDE_SLOT_ID + value: {{ . | quote }} + {{- end }} + {{- end }} {{- with .Values.controller.env }} {{- toYaml . | nindent 12 }} {{- end }} @@ -170,6 +208,13 @@ spec: - name: grpc containerPort: {{ .Values.controller.service.ports.grpc }} protocol: TCP + {{- if $externalGateway.enabled }} + # Container port names use IANA service-name rules (15 characters); + # the Service port retains the requested external-gateway name. + - name: ext-gateway + containerPort: {{ include "kagent.controller.externalGateway.port" . | int }} + protocol: TCP + {{- end }} {{- if .Values.controller.metrics.enabled }} - name: metrics containerPort: {{ include "kagent.controller.metricsPort" . | int }} @@ -202,7 +247,7 @@ spec: port: http periodSeconds: 30 {{- end }} - {{- if or (gt (len .Values.controller.volumeMounts) 0) (and .Values.controller.substrate .Values.controller.substrate.enabled) }} + {{- if or (gt (len .Values.controller.volumeMounts) 0) (and .Values.controller.substrate .Values.controller.substrate.enabled) $externalGateway.enabled }} volumeMounts: {{- if and .Values.controller.substrate .Values.controller.substrate.enabled }} - name: substrate-servicedns @@ -212,6 +257,11 @@ spec: mountPath: /run/substrate-podidentity readOnly: true {{- end }} + {{- if $externalGateway.enabled }} + - name: external-gateway-token + mountPath: {{ include "kagent.controller.externalGateway.tokenMountPath" . }} + readOnly: true + {{- end }} {{- with .Values.controller.volumeMounts }} {{- toYaml . | nindent 12 }} {{- end }} diff --git a/helm/kagent/templates/controller-external-gateway-service.yaml b/helm/kagent/templates/controller-external-gateway-service.yaml new file mode 100644 index 0000000000..aecc27ed69 --- /dev/null +++ b/helm/kagent/templates/controller-external-gateway-service.yaml @@ -0,0 +1,22 @@ +{{- $externalGateway := .Values.controller.externalGateway | default dict -}} +{{- if $externalGateway.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "kagent.fullname" . }}-external-gateway + namespace: {{ include "kagent.namespace" . }} + labels: + {{- include "kagent.controller.labels" . | nindent 4 }} +spec: + # The bearer-token transport is intentionally reachable only inside the + # cluster. TLS/Cloudflare exposure must terminate in a separately managed + # ingress or tunnel and cannot inherit controller.service.type. + type: ClusterIP + ports: + - port: {{ include "kagent.controller.externalGateway.port" . | int }} + targetPort: ext-gateway + protocol: TCP + name: external-gateway + selector: + {{- include "kagent.controller.selectorLabels" . | nindent 4 }} +{{- end }} diff --git a/helm/kagent/tests/external-gateway_test.yaml b/helm/kagent/tests/external-gateway_test.yaml new file mode 100644 index 0000000000..19cae34e09 --- /dev/null +++ b/helm/kagent/tests/external-gateway_test.yaml @@ -0,0 +1,340 @@ +suite: test external runtime gateway +templates: + - controller-deployment.yaml + - controller-service.yaml + - controller-external-gateway-service.yaml + - controller-configmap.yaml + - postgresql-secret.yaml +tests: + - it: should keep the gateway disabled by default + template: controller-deployment.yaml + asserts: + - notExists: + path: spec.strategy + - contains: + path: spec.template.spec.containers[0].env + content: + name: EXTERNAL_GATEWAY_ENABLED + value: "false" + - notContains: + path: spec.template.spec.containers[0].env + content: + name: EXTERNAL_GATEWAY_TOKEN_FILE + - notContains: + path: spec.template.spec.containers[0].ports + content: + name: ext-gateway + - notExists: + path: spec.template.spec.volumes + + - it: should not create the gateway service by default + template: controller-external-gateway-service.yaml + asserts: + - hasDocuments: + count: 0 + + - it: should configure a Codex gateway from an existing Secret + template: controller-deployment.yaml + set: + controller: + externalGateway: + enabled: true + deviceId: workstation-1 + codexSlotId: codex-1 + existingSecret: + name: external-device-token + key: bearer + asserts: + - equal: + path: spec.replicas + value: 1 + - equal: + path: spec.strategy.type + value: Recreate + - equal: + path: spec.template.spec.securityContext.runAsNonRoot + value: true + - equal: + path: spec.template.spec.securityContext.seccompProfile.type + value: RuntimeDefault + - equal: + path: spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem + value: true + - equal: + path: spec.template.spec.containers[0].securityContext.allowPrivilegeEscalation + value: false + - contains: + path: spec.template.spec.containers[0].env + content: + name: EXTERNAL_GATEWAY_ENABLED + value: "true" + - contains: + path: spec.template.spec.containers[0].env + content: + name: EXTERNAL_GATEWAY_BIND_ADDRESS + value: ":8085" + - contains: + path: spec.template.spec.containers[0].env + content: + name: EXTERNAL_GATEWAY_TOKEN_FILE + value: /var/run/secrets/kagent/external-gateway/token + - contains: + path: spec.template.spec.containers[0].env + content: + name: EXTERNAL_GATEWAY_DEVICE_ID + value: workstation-1 + - contains: + path: spec.template.spec.containers[0].env + content: + name: EXTERNAL_GATEWAY_CODEX_SLOT_ID + value: codex-1 + - notContains: + path: spec.template.spec.containers[0].env + content: + name: EXTERNAL_GATEWAY_CLAUDE_SLOT_ID + - contains: + path: spec.template.spec.containers[0].ports + content: + name: ext-gateway + containerPort: 8085 + protocol: TCP + - contains: + path: spec.template.spec.volumes + content: + name: external-gateway-token + secret: + secretName: external-device-token + defaultMode: 292 + items: + - key: bearer + path: token + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: external-gateway-token + mountPath: /var/run/secrets/kagent/external-gateway + readOnly: true + + - it: should configure a Claude-only gateway without a Codex placement + template: controller-deployment.yaml + set: + controller: + externalGateway: + enabled: true + deviceId: workstation-1 + claudeSlotId: claude-1 + existingSecret: + name: external-device-token + key: token + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: EXTERNAL_GATEWAY_CLAUDE_SLOT_ID + value: claude-1 + - notContains: + path: spec.template.spec.containers[0].env + content: + name: EXTERNAL_GATEWAY_CODEX_SLOT_ID + + - it: should configure both runtime slots and expose a dedicated internal service + template: controller-external-gateway-service.yaml + set: + controller: + externalGateway: + enabled: true + deviceId: workstation-1 + codexSlotId: codex-1 + claudeSlotId: claude-1 + existingSecret: + name: external-device-token + key: token + asserts: + - equal: + path: spec.type + value: ClusterIP + - contains: + path: spec.ports + content: + port: 8085 + targetPort: ext-gateway + protocol: TCP + name: external-gateway + + - it: should keep the gateway internal when the controller service is public + template: controller-external-gateway-service.yaml + set: + controller: + service: + type: LoadBalancer + externalGateway: + enabled: true + deviceId: workstation-1 + codexSlotId: codex-1 + existingSecret: + name: external-device-token + key: token + asserts: + - equal: + path: spec.type + value: ClusterIP + + - it: should reject multiple controller replicas + template: controller-deployment.yaml + set: + controller: + replicas: 2 + externalGateway: + enabled: true + deviceId: workstation-1 + codexSlotId: codex-1 + existingSecret: + name: external-device-token + key: token + asserts: + - failedTemplate: + errorMessage: "controller.externalGateway requires controller.replicas=1 because gateway sessions are process-local" + + - it: should reject a colliding gRPC bind address + template: controller-deployment.yaml + set: + controller: + grpc: + bindAddress: ":8085" + externalGateway: + enabled: true + deviceId: workstation-1 + codexSlotId: codex-1 + existingSecret: + name: external-device-token + key: token + asserts: + - failedTemplate: + errorMessage: "controller.grpc.bindAddress must not use external gateway port 8085" + + - it: should reject a controller gRPC service port that exposes the gateway + template: controller-deployment.yaml + set: + controller: + service: + ports: + grpc: 8085 + externalGateway: + enabled: true + deviceId: workstation-1 + codexSlotId: codex-1 + existingSecret: + name: external-device-token + key: token + asserts: + - failedTemplate: + errorMessage: "controller.service.ports.grpc must not use external gateway port 8085" + + - it: should reject a controller HTTP target port that exposes the gateway + template: controller-deployment.yaml + set: + controller: + service: + ports: + targetPort: 8085 + externalGateway: + enabled: true + deviceId: workstation-1 + codexSlotId: codex-1 + existingSecret: + name: external-device-token + key: token + asserts: + - failedTemplate: + errorMessage: "controller.service.ports.targetPort must not use external gateway port 8085" + + - it: should reject a colliding metrics bind address + template: controller-deployment.yaml + set: + controller: + metrics: + enabled: true + bindAddress: ":8085" + externalGateway: + enabled: true + deviceId: workstation-1 + codexSlotId: codex-1 + existingSecret: + name: external-device-token + key: token + asserts: + - failedTemplate: + errorMessage: "controller.metrics.bindAddress must not use external gateway port 8085" + + - it: should require a configured runtime slot + template: controller-deployment.yaml + set: + controller: + externalGateway: + enabled: true + deviceId: workstation-1 + existingSecret: + name: external-device-token + key: token + asserts: + - failedTemplate: + errorMessage: "controller.externalGateway requires at least one of codexSlotId or claudeSlotId" + + - it: should require an existing Secret name + template: controller-deployment.yaml + set: + controller: + externalGateway: + enabled: true + deviceId: workstation-1 + codexSlotId: codex-1 + asserts: + - failedTemplate: + errorMessage: "controller.externalGateway.existingSecret.name is required" + + - it: should reject direct gateway environment overrides + template: controller-deployment.yaml + set: + controller: + env: + - name: EXTERNAL_GATEWAY_ENABLED + value: "true" + asserts: + - failedTemplate: + errorMessage: "controller.env[EXTERNAL_GATEWAY_ENABLED] is reserved; configure the external gateway through controller.externalGateway" + + - it: should reject a conflicting custom token volume + template: controller-deployment.yaml + set: + controller: + externalGateway: + enabled: true + deviceId: workstation-1 + codexSlotId: codex-1 + existingSecret: + name: external-device-token + key: token + volumes: + - name: external-gateway-token + emptyDir: {} + asserts: + - failedTemplate: + errorMessage: "controller.volumes[external-gateway-token] is reserved while controller.externalGateway is enabled" + + - it: should reject a parent mount that shadows the token path + template: controller-deployment.yaml + set: + controller: + externalGateway: + enabled: true + deviceId: workstation-1 + codexSlotId: codex-1 + existingSecret: + name: external-device-token + key: token + volumeMounts: + - name: shadow + mountPath: /var/run/secrets/kagent + asserts: + - failedTemplate: + errorMessage: "the external gateway token volume and mount path are reserved while controller.externalGateway is enabled" diff --git a/helm/kagent/values.schema.json b/helm/kagent/values.schema.json new file mode 100644 index 0000000000..fee99f07e8 --- /dev/null +++ b/helm/kagent/values.schema.json @@ -0,0 +1,63 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "controller": { + "type": "object", + "properties": { + "replicas": { + "type": "integer", + "minimum": 0, + "description": "Controller replica count; must be 1 while the external gateway is enabled." + }, + "externalGateway": { + "type": "object", + "description": "Reverse gateway for locally running Codex and Claude Code agents.", + "properties": { + "enabled": { + "type": "boolean", + "default": false, + "description": "Enable the external runtime gateway." + }, + "deviceId": { + "type": "string", + "maxLength": 63, + "pattern": "^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?$", + "description": "Stable lowercase DNS label for the external device." + }, + "codexSlotId": { + "type": "string", + "maxLength": 63, + "pattern": "^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?$", + "description": "Stable lowercase DNS label for the Codex runtime slot, or empty." + }, + "claudeSlotId": { + "type": "string", + "maxLength": 63, + "pattern": "^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?$", + "description": "Stable lowercase DNS label for the Claude Code runtime slot, or empty." + }, + "existingSecret": { + "type": "object", + "description": "Reference to an existing Secret; the chart never renders token material.", + "properties": { + "name": { + "type": "string", + "maxLength": 253, + "pattern": "^$|^[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?(?:\\.[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?)*$", + "description": "Secret name in the kagent release namespace." + }, + "key": { + "type": "string", + "maxLength": 253, + "pattern": "^$|^[A-Za-z0-9._-]+$", + "description": "Secret data key containing the bearer token." + } + } + } + } + } + } + } + } +} diff --git a/helm/kagent/values.yaml b/helm/kagent/values.yaml index 2e3875e99f..15fcaa45d9 100644 --- a/helm/kagent/values.yaml +++ b/helm/kagent/values.yaml @@ -177,6 +177,9 @@ controller: # Useful for GCP Workload Identity, AWS IRSA, or Azure Workload Identity. # @default -- {} (no extra annotations) annotations: {} + # -- Number of controller replicas. Must remain `1` while + # `controller.externalGateway.enabled` is true because external runtime + # sessions are currently held in controller memory. replicas: 1 loglevel: "info" # Authentication mode: "unsecure" (default) or "trusted-proxy" @@ -321,7 +324,33 @@ controller: tlsCertFile: "" tlsKeyFile: "" + # -- Reverse HTTP gateway for locally running Codex and Claude Code agents. + # Disabled by default. When enabled, the chart creates a dedicated ClusterIP + # Service on port 8085 and mounts a token from an existing Kubernetes Secret. + # The chart never creates or accepts the token as an inline Helm value. + externalGateway: + # -- Enable the external runtime gateway. Requires exactly one controller + # replica and at least one of `codexSlotId` or `claudeSlotId`. + enabled: false + # -- Stable external device identity. Use a lowercase DNS label (max 63 + # characters). Both runtime slots, when configured, belong to this device. + deviceId: "" + # -- Stable Codex runtime slot identity, or empty to disable Codex for this + # device. Must be a lowercase DNS label when set. + codexSlotId: "" + # -- Stable Claude Code runtime slot identity, or empty to disable Claude + # Code for this device. Must be a lowercase DNS label when set. + claudeSlotId: "" + existingSecret: + # -- Name of an existing Secret in the kagent release namespace. The + # Secret is mounted read-only; it is never copied into a rendered value. + name: "" + # -- Secret data key containing the external device bearer token. + key: token + # Extra controller env (mapped to flags via SUBSTRATE_* env names). + # EXTERNAL_GATEWAY_* names are reserved; configure them through + # controller.externalGateway so the chart can enforce its security invariants. env: [] # Agent Substrate (OpenClaw harness runtime=substrate). Requires ate-system installed.