Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 31 additions & 0 deletions src/claim-evidence.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -248,6 +248,37 @@ describe('verdictFor — a check that cannot fail is refused at the checkable ru
})
})

describe('gradeFor — a nonzero exit does not outrank the observation the claim promised', () => {
it('verifies a refusal control that exits nonzero and prints the refusal it promised', () => {
// A gate that must REFUSE when an artifact is absent exits 3 and prints REFUSED=1; the exit
// status alone read that correct refusal as a contradiction (agent-knowledge#192).
expect(
verdictFor(
{ rung: 3, check: 'python3 check_c1_locate.py', expect: 'REFUSED=1' },
{ exitCode: 3, stdout: 'REFUSAL_JSON REFUSED=1 reason=ARTIFACT_ABSENT', stderr: '' },
),
).toBe('verified')
})

it('does not rescue a run that never reached its input with a coincidental substring', () => {
expect(
verdictFor(
{ rung: 3, check: 'python3 check_c1_locate.py', expect: 'REFUSED=1' },
{ exitCode: 1, stdout: 'REFUSED=1', stderr: 'FileNotFoundError: k3.json' },
),
).toBe('unrunnable')
})

it('still contradicts a nonzero exit whose output lacks the expectation', () => {
expect(
verdictFor(
{ rung: 3, check: 'python3 check_c1_locate.py', expect: 'REFUSED=1' },
{ exitCode: 3, stdout: 'REFUSED=0 reason=ARTIFACT_PRESENT', stderr: '' },
),
).toBe('contradicted')
})
})

describe('gradeFor — the refusal says which shape it refused', () => {
it('names the missing expectation', () => {
const grade = gradeFor({ rung: 4, check: 'pnpm test' }, { ...PASSED, stdout: 'ok' })
Expand Down
22 changes: 21 additions & 1 deletion src/claim-evidence.ts
Original file line number Diff line number Diff line change
Expand Up @@ -297,7 +297,27 @@ export function gradeFor(
// the condition it caused must not have to spell it in words a regex happens to know.
if (execution.killedBySignal) return { verdict: 'unrunnable', note: ABORTED_NOTE }
if (execution.outputTruncated) return { verdict: 'unrunnable', note: TRUNCATED_OUTPUT_NOTE }
if (execution.exitCode !== 0) return refutation(output, evidence.expect)
if (execution.exitCode !== 0) {
// The expectation is the claim's own statement of what settles it, so a nonzero exit does
// not refute a claim whose promised observation is in the output: a negative control that
// correctly refuses exits nonzero and prints the refusal it promised, and refuting it for
// the status made `expect` dead code on every such claim. An unreached-input signature in
// the output still outranks the substring, and an expectation that itself names a
// signature keeps the guard's own rule below, so a run that never happened is not rescued.
if (
evidence.expect &&
output.includes(evidence.expect) &&
!UNRUNNABLE_SIGNATURES.test(evidence.expect) &&
!UNRUNNABLE_SIGNATURES.test(output)
) {
if (mustBeCheckable) {
const note = expectationRefusalNote(evidence.expect)
if (note) return { verdict: 'uncheckable', note }
}
return { verdict: 'verified' }
}
return refutation(output, evidence.expect)
}
if (mustBeCheckable) {
const note = expectationRefusalNote(evidence.expect)
if (note) return { verdict: 'uncheckable', note }
Expand Down