Extract graders model

problemtools/checks/__init__.py

@@ -1,9 +1,11 @@
+from .graders import check_graders
 from .includes import check_includes
 from .submissions import check_submissions
 from .testdata import check_testdata
 from .validators import check_input_validators, check_output_validators
 
 __all__ = [
+    'check_graders',
     'check_includes',
     'check_input_validators',
     'check_output_validators',

problemtools/checks/graders.py

@@ -0,0 +1,24 @@
+"""Checks for a problem package's custom graders."""
+
+from __future__ import annotations
+
+from ..diagnostics import Diagnostics
+from ..metadata import Metadata
+from ..model import Graders
+
+
+def check_graders(graders: Graders, metadata: Metadata, diag: Diagnostics) -> None:
+    """Run all checks on a problem's custom graders."""
+    if len(graders.graders) > 1:
+        diag.fatal('There is more than one custom grader')
+
+    grader = graders.grader
+    if grader is None:
+        return
+
+    if metadata.is_pass_fail():
+        diag.fatal('There is a grader but the problem is pass-fail')
+
+    success, msg = grader.compile()
+    if not success:
+        diag.fatal(f'Compile error for {grader}', msg)

problemtools/checks/submissions.py

@@ -11,7 +11,7 @@
 from ..diagnostics import Diagnostics
 from ..judge import SubmissionJudge, SubmissionResult
 from ..metadata import Metadata
-from ..model import LegacyPolicy, Submission, Submissions, TestCase, TestDataGroup
+from ..model import Graders, LegacyPolicy, Submission, Submissions, TestCase, TestDataGroup
 from ..run import Program
 
 # Temporary consts to keep code structure as similar as possible to old code from
@@ -31,7 +31,7 @@ def check_submissions(
     metadata: Metadata,
     testdata: TestDataGroup,
     output_validator: Program,
-    custom_grader: Program | None,
+    graders: Graders,
     tmpdir: str,
     probdir: Path,
     context: Context,
@@ -93,7 +93,7 @@ def check_submissions(
                     metadata,
                     testdata,
                     output_validator,
-                    custom_grader,
+                    graders,
                     tmpdir,
                     probdir,
                     seen_oob_score_groups,
@@ -162,7 +162,7 @@ def _check_submission(
     metadata: Metadata,
     testdata: TestDataGroup,
     output_validator: Program,
-    custom_grader: Program | None,
+    graders: Graders,
     tmpdir: str,
     probdir: Path,
     seen_oob_score_groups: set[int],
@@ -182,8 +182,8 @@ def _check_submission(
         root=testdata,
         base_dir=Path(tmpdir),
         context=context,
+        graders=graders,
         diag=diag,
-        custom_grader=custom_grader,
     )
     if context.executor is not None:
         judge.precompute(timelim_high)

problemtools/checks/testdata.py

@@ -13,7 +13,16 @@
 from ..formatversion import FormatVersion
 from ..judge import validate_output
 from ..metadata import Metadata
-from ..model import DEFAULT_CONFIG, SCORING_ONLY_KEYS, InputValidators, OutputValidators, TestCase, TestDataGroup
+from ..model import (
+    DEFAULT_CONFIG,
+    DEFAULT_GRADER,
+    SCORING_ONLY_KEYS,
+    Graders,
+    InputValidators,
+    OutputValidators,
+    TestCase,
+    TestDataGroup,
+)
 from ..run import Program
 from .validators import check_testcase_input
 
@@ -23,8 +32,7 @@ def check_testdata(
     context: Context,
     metadata: Metadata,
     probdir: Path,
-    has_custom_grader: bool,
-    has_default_grader: bool,
+    graders: Graders,
     input_validators: InputValidators,
     output_validators: OutputValidators,
     format: FormatVersion,
@@ -36,6 +44,9 @@ def check_testdata(
     if output_validator is None:
         diag.fatal('Unable to locate default validator')
 
+    has_custom_grader = graders.grader is not None
+    has_default_grader = DEFAULT_GRADER is not None
+
     _check_group(
         testdata,
         context,

problemtools/judge/submission_judge.py

@@ -9,8 +9,8 @@
 from ..context import Context
 from ..diagnostics import Diagnostics
 from ..metadata import Metadata
-from ..model import TestCase, TestDataGroup
-from ..run import Program, get_tool
+from ..model import DEFAULT_GRADER, Graders, TestCase, TestDataGroup
+from ..run import Program
 from .cache import ResultStore
 from .execute import execute_testcase
 from .grade import grade_group
@@ -53,8 +53,6 @@ class SubmissionJudge:
     jobs complete normally; their results are simply not consumed by judge().
     """
 
-    _default_grader: Program | None = get_tool('default_grader')
-
     def __init__(
         self,
         sub: Program,
@@ -63,16 +61,16 @@ def __init__(
         root: TestDataGroup,
         base_dir: Path,
         context: Context,
+        graders: Graders,
         diag: Diagnostics,
-        custom_grader: Program | None = None,
     ) -> None:
         self._sub = sub
         self._output_validator = output_validator
         self._metadata = metadata
         self._base_dir = base_dir
         self._context = context
         self._diag = diag
-        self._custom_grader = custom_grader
+        self._graders = graders
         self._store = ResultStore()
         self._root = root
         self._cancelled = _Cancelled()
@@ -150,8 +148,8 @@ def _cancel_subtree(self, group: TestDataGroup) -> None:
 
     def _grader_for(self, group: TestDataGroup) -> Program | None:
         if group.config.get('grading') == 'custom':
-            return self._custom_grader
-        return self._default_grader
+            return self._graders.grader
+        return DEFAULT_GRADER
 
     def _judge_group(self, group: TestDataGroup, timelim: float) -> list[SubmissionResult]:
         all_results: list[SubmissionResult] = []  # Results of all children, groups and test cases, in DFS order. Our return value

problemtools/model/__init__.py

@@ -3,16 +3,19 @@
 #: A submission's (or testcase's) verdict, e.g. as expected by policy or produced by judging.
 Verdict = Literal['AC', 'TLE', 'OLE', 'MLE', 'RTE', 'WA', 'PAC', 'JE']
 
+from .graders import DEFAULT_GRADER, Graders, load_graders
 from .includes import DEFAULT_LANGUAGE, IncludeFile, Includes, LanguageIncludes, load_includes
 from .submissions import LegacyPolicy, Submission, Submissions, load_submissions
 from .testdata import DEFAULT_CONFIG, SCORING_ONLY_KEYS, TestCase, TestDataGroup, load_testdata
 from .validators import DEFAULT_VALIDATOR, InputValidators, OutputValidators, load_input_validators, load_output_validators
 
 __all__ = [
     'DEFAULT_CONFIG',
+    'DEFAULT_GRADER',
     'DEFAULT_LANGUAGE',
     'DEFAULT_VALIDATOR',
     'SCORING_ONLY_KEYS',
+    'Graders',
     'IncludeFile',
     'Includes',
     'InputValidators',
@@ -24,6 +27,7 @@
     'TestCase',
     'TestDataGroup',
     'Verdict',
+    'load_graders',
     'load_includes',
     'load_input_validators',
     'load_output_validators',

problemtools/model/graders.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from ..languages import Languages
+from ..run import Program, find_programs, get_tool
+
+DEFAULT_GRADER = get_tool('default_grader')
+
+
+@dataclass(frozen=True)
+class Graders:
+    """A problem's graders: custom grader programs found on disk, if any."""
+
+    graders: list[Program] = field(default_factory=list)
+
+    @property
+    def grader(self) -> Program | None:
+        """The custom grader, if there's exactly one; None if there are zero (default grading is used).
+
+        Does not validate that there's at most one grader; callers that care about the invalid
+        case of more than one custom grader must check `len(graders)` themselves and report it."""
+        return self.graders[0] if len(self.graders) == 1 else None
+
+
+def load_graders(probdir: Path, language_config: Languages, work_dir: str) -> Graders:
+    graders = find_programs(str(probdir / 'graders'), language_config=language_config, work_dir=work_dir)
+    return Graders(graders=graders)

problemtools/verifyproblem.py

@@ -371,19 +371,12 @@ def check(self, context: Context) -> bool:
 
 
 class Graders(ProblemPart):
-    _default_grader = run.get_tool('default_grader')
+    """Seam to integrate a model + checks setup into verifyproblem in a somewhat clean way"""
 
     PART_NAME = 'grader'
 
     def setup(self) -> None:
-        graders: list = run.find_programs(
-            os.path.join(self.problem.probdir, 'graders'),
-            language_config=self.problem.language_config,
-            work_dir=self.problem.tmpdir,
-        )
-        if len(graders) > 1:
-            self.fatal('There is more than one custom grader')
-        self._grader = graders[0] if graders else None
+        self.graders = model.load_graders(Path(self.problem.probdir), self.problem.language_config, self.problem.tmpdir)
 
     def __str__(self) -> str:
         return 'graders'
@@ -393,13 +386,11 @@ def check(self, context: Context) -> bool:
             return self._check_res
         self._check_res = True
 
-        if self._grader:
-            if self.problem.is_pass_fail() and self._grader:
-                self.fatal('There is a grader but the problem is pass-fail')
+        errors_before = self.errors
+        checks.check_graders(self.graders, self.problem.metadata, self._diag)
+        if self.errors > errors_before:
+            self._check_res = False
 
-            success, msg = self._grader.compile()
-            if not success:
-                self.fatal(f'Compile error for {self._grader}', msg)
         return self._check_res
 
 
@@ -507,7 +498,7 @@ def check(self, context: Context) -> bool:
             self.problem.metadata,
             self.problem.testdata.testdata,
             self.problem.output_validators.output_validator,
-            self.problem.graders._grader,
+            self.problem.graders.graders,
             self.problem.tmpdir,
             Path(self.problem.probdir),
             context,
@@ -543,8 +534,7 @@ def check(self, context: Context) -> bool:
             context,
             self.problem.metadata,
             Path(self.problem.probdir),
-            self.problem.graders._grader is not None,
-            Graders._default_grader is not None,
+            self.problem.graders.graders,
             self.problem.input_validators.input_validators,
             self.problem.output_validators.output_validators,
             self.problem.format,