feat: compile and persist external runtime harnesses

docs/external-runtime-harness.md

@@ -0,0 +1,78 @@
+# External runtime Harnesses
+
+An external `Harness` compiles portable agent behavior for a Codex or Claude
+Code runtime connected through the reverse external gateway. kagent persists an
+immutable revision, but it does not create a `WorkerPool`, `ActorTemplate`, Pod,
+or other in-cluster compute for that revision.
+
+This foundation must be released together with profile dispatch and a strict
+local-host consumer. Until those pieces are present, `ExternalRuntimePrepared`
+means only that the revision was compiled and persisted; it does not mean that
+a compatible runtime slot is online or that the instruction has been applied.
+
+## Minimal Codex example
+
+```yaml
+apiVersion: kagent.dev/v1alpha3
+kind: Harness
+metadata:
+  name: codex
+  namespace: agents
+spec:
+  codex: {}
+  allowedAgentTemplates:
+    selector:
+      matchLabels:
+        runtime.kagent.dev/codex: "true"
+---
+apiVersion: kagent.dev/v1alpha3
+kind: AgentTemplate
+metadata:
+  name: reviewer
+  namespace: agents
+  labels:
+    runtime.kagent.dev/codex: "true"
+spec:
+  # The v1alpha3 wire format still requires this field. External compilers do
+  # not resolve the reference or copy model credentials into the revision.
+  modelConfig:
+    name: unused-for-external-runtime
+  description: Review code changes
+  systemPrompt: Review the requested change and report concrete findings.
+```
+
+Use `claude: {}` instead of `codex: {}` for Claude Code. Exactly one runtime
+variant is allowed. `workload`, `substrate`, and `env` are required for the
+in-cluster kagent runtime and forbidden for external runtimes.
+
+## Portable profile boundary
+
+The persisted external profile contains only:
+
+```json
+{"version":"v1","instruction":"...","tools":[]}
+```
+
+Model, reasoning effort, speed, filesystem access, credentials, executable
+paths, and sandbox settings remain local Agent Card policy. Cluster
+`ModelConfig` values, MCP URLs, headers, TLS material, and Secret values are not
+copied into the profile or revision provenance.
+
+External v1 profiles currently reject AgentTemplate skills, plugins, shared
+agent tools, MCP headers/TLS, and empty MCP allowlists. MCP entries contain only
+the logical `RemoteMCPServer` name and allowed tool names; a local host must map
+that name to an explicitly configured local endpoint and fail closed when the
+mapping or isolation support is unavailable.
+
+## Rollout constraints
+
+- Enable the reverse gateway only with one controller replica; the Helm chart
+  enforces `Recreate` because sessions are currently process-local.
+- Release migration 19 with the matching controller binary. Do not run mixed
+  migration-18 and migration-19 controller replicas: the older generated reader
+  uses `SELECT *` and does not understand the added profile column.
+- Adding backend identity to the revision digest produces one new revision for
+  each existing in-cluster pair on first reconciliation. Capacity-plan that
+  one-time recompilation.
+- Execute the migration 19 PostgreSQL round-trip tests before merging or
+  deploying this feature stack.

go/api/config/crd/bases/kagent.dev_harnesses.yaml

@@ -163,8 +163,8 @@ spec:
                 description: KagentHarness selects the kagent runtime adapter.
                 type: object
               substrate:
-                description: HarnessSubstratePolicy contains the Substrate policy
-                  shared by all runtime variants.
+                description: Substrate is required by kagent and forbidden by external
+                  runtimes.
                 properties:
                   snapshotPolicy:
                     description: SnapshotPolicy configures runtime snapshot storage.
@@ -200,8 +200,8 @@ spec:
                 - message: workerPoolRef name must not be empty
                   rule: self.workerPoolRef.name.size() > 0
               workload:
-                description: HarnessWorkload identifies the immutable runtime image
-                  used by a Harness.
+                description: Workload is required by kagent and forbidden by external
+                  runtimes.
                 properties:
                   image:
                     description: Image is an OCI image reference pinned by sha256
@@ -211,14 +211,16 @@ spec:
                 required:
                 - image
                 type: object
-            required:
-            - substrate
-            - workload
             type: object
             x-kubernetes-validations:
             - message: exactly one of kagent, codex, or claude must be specified
               rule: '(has(self.kagent) ? 1 : 0) + (has(self.codex) ? 1 : 0) + (has(self.claude)
                 ? 1 : 0) == 1'
+            - message: kagent requires workload and substrate
+              rule: '!has(self.kagent) || (has(self.workload) && has(self.substrate))'
+            - message: codex and claude forbid workload, substrate, and env
+              rule: has(self.kagent) || (!has(self.workload) && !has(self.substrate)
+                && !has(self.env))
           status:
             description: HarnessStatus reports controller-derived capabilities and
               current health.

go/api/database/models.go

@@ -1,6 +1,7 @@
 package database
 
 import (
+	"bytes"
 	"encoding/json"
 	"errors"
 	"time"
@@ -287,6 +288,7 @@ type RuntimeRevision struct {
 	EgressDestinations     []string
 	BackendKind            RuntimeBackendKind
 	ExternalRuntime        ExternalRuntime
+	ExternalProfile        json.RawMessage
 	ActorTemplateNamespace string
 	ActorTemplateName      string
 	ActorTemplateUID       string
@@ -299,13 +301,26 @@ type RuntimeRevision struct {
 func (r RuntimeRevision) ValidateBackendIdentity() error {
 	switch r.BackendKind {
 	case RuntimeBackendKindSubstrate:
-		if r.ExternalRuntime != "" {
-			return errors.New("substrate runtime revision must not select an external runtime")
+		if r.ExternalRuntime != "" || len(bytes.TrimSpace(r.ExternalProfile)) != 0 {
+			return errors.New("substrate runtime revision must not select an external runtime or profile")
+		}
+		if r.ActorTemplateNamespace == "" || r.ActorTemplateName == "" {
+			return errors.New("substrate runtime revision requires an actor template identity")
 		}
 	case RuntimeBackendKindExternal:
 		if r.ExternalRuntime != ExternalRuntimeCodex && r.ExternalRuntime != ExternalRuntimeClaude {
 			return errors.New("external runtime revision must select a supported runtime")
 		}
+		profile := bytes.TrimSpace(r.ExternalProfile)
+		if len(profile) == 0 || profile[0] != '{' || !json.Valid(profile) {
+			return errors.New("external runtime revision requires a JSON object profile")
+		}
+		if r.ActorTemplateNamespace != "" || r.ActorTemplateName != "" || r.ActorTemplateUID != "" {
+			return errors.New("external runtime revision must not select an actor template")
+		}
+		if r.Phase != "Ready" || r.GoldenSnapshot != "" {
+			return errors.New("external runtime revision must be ready without a golden snapshot")
+		}
 	default:
 		return errors.New("runtime revision backend kind is invalid")
 	}

go/api/database/models_runtime_revision_test.go

@@ -8,26 +8,50 @@ import (
 )
 
 func TestRuntimeRevisionValidateBackendIdentity(t *testing.T) {
+	substrate := dbpkg.RuntimeRevision{
+		BackendKind: dbpkg.RuntimeBackendKindSubstrate, ActorTemplateNamespace: "team-a", ActorTemplateName: "actor",
+	}
+	external := dbpkg.RuntimeRevision{
+		BackendKind: dbpkg.RuntimeBackendKindExternal, ExternalRuntime: dbpkg.ExternalRuntimeCodex,
+		ExternalProfile: []byte(`{"version":"v1"}`), Phase: "Ready",
+	}
 	tests := []struct {
-		name    string
-		kind    dbpkg.RuntimeBackendKind
-		runtime dbpkg.ExternalRuntime
-		wantErr string
+		name     string
+		revision dbpkg.RuntimeRevision
+		wantErr  string
 	}{
-		{name: "substrate", kind: dbpkg.RuntimeBackendKindSubstrate},
-		{name: "external codex", kind: dbpkg.RuntimeBackendKindExternal, runtime: dbpkg.ExternalRuntimeCodex},
-		{name: "external claude", kind: dbpkg.RuntimeBackendKindExternal, runtime: dbpkg.ExternalRuntimeClaude},
-		{name: "missing kind", wantErr: "backend kind is invalid"},
-		{name: "unknown kind", kind: dbpkg.RuntimeBackendKind("credential-shaped-unknown"), wantErr: "backend kind is invalid"},
-		{name: "substrate with runtime", kind: dbpkg.RuntimeBackendKindSubstrate, runtime: dbpkg.ExternalRuntimeCodex, wantErr: "must not select"},
-		{name: "external missing runtime", kind: dbpkg.RuntimeBackendKindExternal, wantErr: "supported runtime"},
-		{name: "external unknown runtime", kind: dbpkg.RuntimeBackendKindExternal, runtime: dbpkg.ExternalRuntime("credential-shaped-unknown"), wantErr: "supported runtime"},
+		{name: "substrate", revision: substrate},
+		{name: "external codex", revision: external},
+		{name: "external claude", revision: func() dbpkg.RuntimeRevision {
+			value := external
+			value.ExternalRuntime = dbpkg.ExternalRuntimeClaude
+			return value
+		}()},
+		{name: "missing kind", revision: dbpkg.RuntimeRevision{}, wantErr: "backend kind is invalid"},
+		{name: "unknown kind", revision: dbpkg.RuntimeRevision{BackendKind: dbpkg.RuntimeBackendKind("credential-shaped-unknown")}, wantErr: "backend kind is invalid"},
+		{name: "substrate missing actor", revision: dbpkg.RuntimeRevision{BackendKind: dbpkg.RuntimeBackendKindSubstrate}, wantErr: "actor template identity"},
+		{name: "substrate with runtime", revision: func() dbpkg.RuntimeRevision {
+			value := substrate
+			value.ExternalRuntime = dbpkg.ExternalRuntimeCodex
+			return value
+		}(), wantErr: "must not select"},
+		{name: "substrate with profile", revision: func() dbpkg.RuntimeRevision { value := substrate; value.ExternalProfile = []byte(`{}`); return value }(), wantErr: "must not select"},
+		{name: "external missing runtime", revision: func() dbpkg.RuntimeRevision { value := external; value.ExternalRuntime = ""; return value }(), wantErr: "supported runtime"},
+		{name: "external unknown runtime", revision: func() dbpkg.RuntimeRevision {
+			value := external
+			value.ExternalRuntime = dbpkg.ExternalRuntime("credential-shaped-unknown")
+			return value
+		}(), wantErr: "supported runtime"},
+		{name: "external missing profile", revision: func() dbpkg.RuntimeRevision { value := external; value.ExternalProfile = nil; return value }(), wantErr: "JSON object profile"},
+		{name: "external array profile", revision: func() dbpkg.RuntimeRevision { value := external; value.ExternalProfile = []byte(`[]`); return value }(), wantErr: "JSON object profile"},
+		{name: "external with actor", revision: func() dbpkg.RuntimeRevision { value := external; value.ActorTemplateName = "actor"; return value }(), wantErr: "must not select an actor"},
+		{name: "external not ready", revision: func() dbpkg.RuntimeRevision { value := external; value.Phase = "Pending"; return value }(), wantErr: "must be ready"},
+		{name: "external with snapshot", revision: func() dbpkg.RuntimeRevision { value := external; value.GoldenSnapshot = "snapshot"; return value }(), wantErr: "without a golden snapshot"},
 	}
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			revision := dbpkg.RuntimeRevision{BackendKind: test.kind, ExternalRuntime: test.runtime}
-			err := revision.ValidateBackendIdentity()
+			err := test.revision.ValidateBackendIdentity()
 			if test.wantErr == "" {
 				require.NoError(t, err)
 				return

go/api/v1alpha3/configuration_crd_cel_test.go

@@ -71,10 +71,56 @@ func TestConfigurationCRDValidation(t *testing.T) {
 			name: "Harness rejects tag-only image",
 			object: validHarness(namespace, "harness-tagged-image", HarnessSpec{
 				Kagent:   &KagentHarness{},
-				Workload: HarnessWorkload{Image: "registry.example.com/kagent:latest"},
+				Workload: &HarnessWorkload{Image: "registry.example.com/kagent:latest"},
 			}),
 			wantReject: "spec.workload.image",
 		},
+		{
+			name: "kagent Harness requires workload",
+			object: &Harness{ObjectMeta: metav1.ObjectMeta{Name: "kagent-no-workload", Namespace: namespace}, Spec: HarnessSpec{
+				Kagent: &KagentHarness{},
+				Substrate: &HarnessSubstratePolicy{
+					WorkerPoolRef:  corev1.LocalObjectReference{Name: "default"},
+					SnapshotPolicy: HarnessSnapshotPolicy{Location: "gs://snapshots/kagent"},
+				},
+			}},
+			wantReject: "kagent requires workload and substrate",
+		},
+		{
+			name: "kagent Harness requires substrate",
+			object: &Harness{ObjectMeta: metav1.ObjectMeta{Name: "kagent-no-substrate", Namespace: namespace}, Spec: HarnessSpec{
+				Kagent:   &KagentHarness{},
+				Workload: &HarnessWorkload{Image: "registry.example.com/kagent@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},
+			}},
+			wantReject: "kagent requires workload and substrate",
+		},
+		{
+			name: "Codex Harness forbids workload",
+			object: &Harness{ObjectMeta: metav1.ObjectMeta{Name: "codex-workload", Namespace: namespace}, Spec: HarnessSpec{
+				Codex:    &CodexHarness{},
+				Workload: &HarnessWorkload{Image: "registry.example.com/codex@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},
+			}},
+			wantReject: "codex and claude forbid workload, substrate, and env",
+		},
+		{
+			name: "Claude Harness forbids substrate",
+			object: &Harness{ObjectMeta: metav1.ObjectMeta{Name: "claude-substrate", Namespace: namespace}, Spec: HarnessSpec{
+				Claude: &ClaudeHarness{},
+				Substrate: &HarnessSubstratePolicy{
+					WorkerPoolRef:  corev1.LocalObjectReference{Name: "default"},
+					SnapshotPolicy: HarnessSnapshotPolicy{Location: "gs://snapshots/claude"},
+				},
+			}},
+			wantReject: "codex and claude forbid workload, substrate, and env",
+		},
+		{
+			name: "Codex Harness forbids env",
+			object: &Harness{ObjectMeta: metav1.ObjectMeta{Name: "codex-env", Namespace: namespace}, Spec: HarnessSpec{
+				Codex: &CodexHarness{},
+				Env:   []HarnessEnvVar{{Name: "TOKEN", Value: &empty}},
+			}},
+			wantReject: "codex and claude forbid workload, substrate, and env",
+		},
 		{
 			name: "Harness env requires a value source",
 			object: validHarness(namespace, "harness-empty-env", HarnessSpec{
@@ -99,7 +145,12 @@ func TestConfigurationCRDValidation(t *testing.T) {
 			name: "valid Harness",
 			object: validHarness(namespace, "valid-harness", HarnessSpec{
 				Claude: &ClaudeHarness{},
-				Env:    []HarnessEnvVar{{Name: "EMPTY", Value: &empty}},
+			}),
+		},
+		{
+			name: "valid kagent Harness",
+			object: validHarness(namespace, "valid-kagent-harness", HarnessSpec{
+				Kagent: &KagentHarness{},
 			}),
 		},
 		{
@@ -139,14 +190,16 @@ func TestConfigurationCRDValidation(t *testing.T) {
 }
 
 func validHarness(namespace, name string, overrides HarnessSpec) *Harness {
-	if overrides.Workload.Image == "" {
-		overrides.Workload.Image = "registry.example.com/kagent@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
-	}
-	if overrides.Substrate.WorkerPoolRef.Name == "" {
-		overrides.Substrate.WorkerPoolRef.Name = "default"
-	}
-	if overrides.Substrate.SnapshotPolicy.Location == "" {
-		overrides.Substrate.SnapshotPolicy.Location = "gs://snapshots/kagent"
+	if overrides.Kagent != nil {
+		if overrides.Workload == nil {
+			overrides.Workload = &HarnessWorkload{Image: "registry.example.com/kagent@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}
+		}
+		if overrides.Substrate == nil {
+			overrides.Substrate = &HarnessSubstratePolicy{
+				WorkerPoolRef:  corev1.LocalObjectReference{Name: "default"},
+				SnapshotPolicy: HarnessSnapshotPolicy{Location: "gs://snapshots/kagent"},
+			}
+		}
 	}
 	return &Harness{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, Spec: overrides}
 }

go/api/v1alpha3/harness_types.go

@@ -89,6 +89,8 @@ type HarnessAgentTemplateAdmission struct {
 // HarnessSpec defines a reusable runtime and its infrastructure policy.
 //
 // +kubebuilder:validation:XValidation:rule="(has(self.kagent) ? 1 : 0) + (has(self.codex) ? 1 : 0) + (has(self.claude) ? 1 : 0) == 1",message="exactly one of kagent, codex, or claude must be specified"
+// +kubebuilder:validation:XValidation:rule="!has(self.kagent) || (has(self.workload) && has(self.substrate))",message="kagent requires workload and substrate"
+// +kubebuilder:validation:XValidation:rule="has(self.kagent) || (!has(self.workload) && !has(self.substrate) && !has(self.env))",message="codex and claude forbid workload, substrate, and env"
 type HarnessSpec struct {
 	// +optional
 	Kagent *KagentHarness `json:"kagent,omitempty"`
@@ -99,17 +101,19 @@ type HarnessSpec struct {
 	// +optional
 	Claude *ClaudeHarness `json:"claude,omitempty"`
 
-	// +required
-	Workload HarnessWorkload `json:"workload"`
+	// Workload is required by kagent and forbidden by external runtimes.
+	// +optional
+	Workload *HarnessWorkload `json:"workload,omitempty"`
 
 	// +optional
 	// +kubebuilder:validation:MaxItems=100
 	// +listType=map
 	// +listMapKey=name
 	Env []HarnessEnvVar `json:"env,omitempty"`
 
-	// +required
-	Substrate HarnessSubstratePolicy `json:"substrate"`
+	// Substrate is required by kagent and forbidden by external runtimes.
+	// +optional
+	Substrate *HarnessSubstratePolicy `json:"substrate,omitempty"`
 
 	// AllowedAgentTemplates selects AgentTemplates this Harness admits.
 	// When omitted, the Harness admits none.