diff --git a/src/claim-evidence.test.ts b/src/claim-evidence.test.ts index 1bdab1c..554e6e7 100644 --- a/src/claim-evidence.test.ts +++ b/src/claim-evidence.test.ts @@ -248,6 +248,37 @@ describe('verdictFor — a check that cannot fail is refused at the checkable ru }) }) +describe('gradeFor — a nonzero exit does not outrank the observation the claim promised', () => { + it('verifies a refusal control that exits nonzero and prints the refusal it promised', () => { + // A gate that must REFUSE when an artifact is absent exits 3 and prints REFUSED=1; the exit + // status alone read that correct refusal as a contradiction (agent-knowledge#192). + expect( + verdictFor( + { rung: 3, check: 'python3 check_c1_locate.py', expect: 'REFUSED=1' }, + { exitCode: 3, stdout: 'REFUSAL_JSON REFUSED=1 reason=ARTIFACT_ABSENT', stderr: '' }, + ), + ).toBe('verified') + }) + + it('does not rescue a run that never reached its input with a coincidental substring', () => { + expect( + verdictFor( + { rung: 3, check: 'python3 check_c1_locate.py', expect: 'REFUSED=1' }, + { exitCode: 1, stdout: 'REFUSED=1', stderr: 'FileNotFoundError: k3.json' }, + ), + ).toBe('unrunnable') + }) + + it('still contradicts a nonzero exit whose output lacks the expectation', () => { + expect( + verdictFor( + { rung: 3, check: 'python3 check_c1_locate.py', expect: 'REFUSED=1' }, + { exitCode: 3, stdout: 'REFUSED=0 reason=ARTIFACT_PRESENT', stderr: '' }, + ), + ).toBe('contradicted') + }) +}) + describe('gradeFor — the refusal says which shape it refused', () => { it('names the missing expectation', () => { const grade = gradeFor({ rung: 4, check: 'pnpm test' }, { ...PASSED, stdout: 'ok' }) diff --git a/src/claim-evidence.ts b/src/claim-evidence.ts index fe8c3a5..101b14f 100644 --- a/src/claim-evidence.ts +++ b/src/claim-evidence.ts @@ -297,7 +297,27 @@ export function gradeFor( // the condition it caused must not have to spell it in words a regex happens to know. if (execution.killedBySignal) return { verdict: 'unrunnable', note: ABORTED_NOTE } if (execution.outputTruncated) return { verdict: 'unrunnable', note: TRUNCATED_OUTPUT_NOTE } - if (execution.exitCode !== 0) return refutation(output, evidence.expect) + if (execution.exitCode !== 0) { + // The expectation is the claim's own statement of what settles it, so a nonzero exit does + // not refute a claim whose promised observation is in the output: a negative control that + // correctly refuses exits nonzero and prints the refusal it promised, and refuting it for + // the status made `expect` dead code on every such claim. An unreached-input signature in + // the output still outranks the substring, and an expectation that itself names a + // signature keeps the guard's own rule below, so a run that never happened is not rescued. + if ( + evidence.expect && + output.includes(evidence.expect) && + !UNRUNNABLE_SIGNATURES.test(evidence.expect) && + !UNRUNNABLE_SIGNATURES.test(output) + ) { + if (mustBeCheckable) { + const note = expectationRefusalNote(evidence.expect) + if (note) return { verdict: 'uncheckable', note } + } + return { verdict: 'verified' } + } + return refutation(output, evidence.expect) + } if (mustBeCheckable) { const note = expectationRefusalNote(evidence.expect) if (note) return { verdict: 'uncheckable', note }