UiPath · akshaylive · Aug 10, 2026 · Jul 29, 2026 · Aug 10, 2026 · Aug 10, 2026
@@ -21,6 +21,7 @@ Complete reference for defining evaluation tasks in Coder Eval.
 - [Template Sources](#template-sources)
 - [Success Criteria](#success-criteria)
   - [Continuous Scoring](#continuous-scoring)
+  - [Glob patterns in path](#glob-patterns-in-path)
   - [file_exists](#file_exists)
   - [file_contains](#file_contains)
   - [file_check](#file_check)
@@ -670,6 +671,30 @@ score mattered.
 
 **Weighted score:** `weighted_score = sum(score * weight) / sum(weight)` — calculated regardless for quality assessment.
 
+### Glob patterns in `path`
+
+Every sandbox-relative path field accepts a glob — `path` on `file_exists`, `file_contains`, `file_matches_regex`, `file_check`, `json_check` and `classification_match`, `json_schema` on `json_check`, and `agent_file` on `reference_comparison`. Use one when the prompt does not pin where the file lands — a scaffolding tool that creates a wrapper directory the agent names itself, for example.
+
+```yaml
+- type: "file_contains"
+  path: "**/*.flow"                     # matches any depth under the sandbox root
+  includes: ['"core.logic.decision"']
+  description: "flow wires a Decision node"
+```
+
+Rules:
+
+- **A path that exists is never treated as a pattern.** A literal `path` behaves exactly as before, including one containing `*`, `?`, or `[` — a real file named `report[2024].json` is graded as itself, not as a character class that would match `report2.json`. Globbing only kicks in when the literal path does not exist.
+- **Glob matches skip ignored directories.** Expansion runs over the live sandbox root, which also holds harness-created content the agent never wrote (`.venv` for any task with a `python:` block, `node_modules`, `dist`, `build`, `__pycache__`, …), so matches are filtered through the same [`ignore_patterns`](#sandbox-configuration) set used for template copying. A segment your pattern names *literally* is an opt-in and survives, so `dist/**/*.js` still grades `dist`; to un-ignore a directory a wildcard has to discover, use the negation escape hatch — `ignore_patterns: ["!dist"]`.
+- Matches are sorted, and directories are skipped.
+- `file_exists` passes when the glob matches **at least one** file.
+- Content checks require the glob to match **exactly one** file. An ambiguous glob scores 0.0 and reports the matches (first 10, then `+N more`) rather than silently grading one of them — narrow the pattern.
+- When a glob resolves, the file that was actually graded is echoed in the criterion's `details` as `resolved: <path>`.
+
+Prefer a glob over a hardcoded path whose leading directory the task prompt never specifies: a correct artifact in an unexpected directory otherwise scores 0.0 on the path alone. Glob away only the segment the prompt leaves free, though — if the free part is an unknown wrapper directory, `**/<Name>.flow` stays unique where a blanket `**/*.flow` turns exactly-one into a hard 0.0 the moment a second flow file exists.
+
+> **Dataset note:** `${row.<field>}` substitution runs over `success_criteria` string leaves, so a row value containing `*`, `?`, or `[` lands inside `path`. Literal-first resolution means such a path still grades the real file when it exists; it falls back to glob expansion only when it does not.
+
 ### `file_exists`
 
 Checks if a file exists. **Binary scoring.**

@@ -47,21 +47,27 @@ def _check_impl(
         has_includes = len(criterion.includes) > 0
         has_excludes = len(criterion.excludes) > 0
         has_patterns = len(criterion.patterns) > 0
+        resolved = sandbox.resolved_path_label(criterion.path)
 
         # 2. Pure existence check (no sub-checks specified)
         if not has_includes and not has_excludes and not has_patterns:
+            details = f"File '{criterion.path}' exists"
+            if resolved:
+                details += f" (resolved: {resolved})"
             return CriterionResult(
                 criterion_type=criterion.type,
                 description=criterion.description,
                 score=1.0,
-                details=f"File '{criterion.path}' exists",
+                details=details,
             )
 
         # 3. Read file content
         content = sandbox.get_file_content(criterion.path)
 
         scores: list[float] = []
         details_parts: list[str] = []
+        if resolved:
+            details_parts.append(f"Resolved: {resolved}")
 
         # 4a. Includes score
         if has_includes:

@@ -71,6 +71,9 @@ def _check_impl(
 
         # Build details
         details_parts = []
+        resolved = sandbox.resolved_path_label(criterion.path)
+        if resolved:
+            details_parts.append(f"Resolved: {resolved}")
         details_parts.append(f"Includes: {includes_found}/{includes_total} found")
         if criterion.excludes:
             excludes_absent = len(criterion.excludes) - sum(1 for exc in criterion.excludes if exc in content)

@@ -37,9 +37,14 @@ def _check_impl(
         exists = sandbox.file_exists(criterion.path)
         score = 1.0 if exists else 0.0
 
+        details = f"File '{criterion.path}' {'exists' if exists else 'does not exist'}"
+        resolved = sandbox.resolved_path_label(criterion.path)
+        if resolved:
+            details += f" (resolved: {resolved})"
+
         return CriterionResult(
             criterion_type=criterion.type,
             description=criterion.description,
             score=score,
-            details=f"File '{criterion.path}' {'exists' if exists else 'does not exist'}",
+            details=details,
         )
@@ -78,6 +78,10 @@ def _check_impl(
                 matched_text = match.group()[:100]
                 details = f"Pattern '{criterion.pattern}' found but should not be present (matched: '{matched_text}')"
 
+        resolved = sandbox.resolved_path_label(criterion.path)
+        if resolved:
+            details += f" (resolved: {resolved})"
+
         return CriterionResult(
             criterion_type=criterion.type,
             description=criterion.description,

@@ -91,18 +91,24 @@ def _check_impl(
 
         has_schema = criterion.json_schema is not None
         has_assertions = len(criterion.assertions) > 0
+        resolved = sandbox.resolved_path_label(criterion.path)
 
         # 3. Pure validity check
         if not has_schema and not has_assertions:
+            details = f"'{criterion.path}' is valid JSON"
+            if resolved:
+                details += f" (resolved: {resolved})"
             return CriterionResult(
                 criterion_type=criterion.type,
                 description=criterion.description,
                 score=1.0,
-                details=f"'{criterion.path}' is valid JSON",
+                details=details,
             )
 
         scores: list[float] = []
         details_parts: list[str] = []
+        if resolved:
+            details_parts.append(f"Resolved: {resolved}")
 
         # 4. Schema validation (gates assertions — if schema fails, skip assertions)
         if has_schema:

@@ -62,18 +62,18 @@ def _check_impl(
                 error="Sandbox not initialized",
             )
 
-        # Load agent code
-        agent_path = sandbox.sandbox_dir / criterion.agent_file
-        if not agent_path.exists():
+        # Load agent code through the shared path seam, so `agent_file` resolves
+        # (glob expansion, ignore filtering, exactly-one) like every other
+        # sandbox-relative criterion path.
+        try:
+            agent_code = sandbox.get_file_content(criterion.agent_file)
+        except FileNotFoundError:
             return CriterionResult(
                 criterion_type="reference_comparison",
                 description=criterion.description,
                 score=0.0,
                 error=f"Agent file not found: {criterion.agent_file}",
             )
-
-        try:
-            agent_code = agent_path.read_text(encoding="utf-8")
         except Exception as e:
             return CriterionResult(
                 criterion_type="reference_comparison",

@@ -321,7 +321,9 @@ class FileExistsCriterion(BaseSuccessCriterion):
     """
 
     type: Literal["file_exists"] = "file_exists"
-    path: str = Field(description="Path to the file that must exist")
+    path: str = Field(
+        description="Path to the file that must exist; a glob pattern passes when it matches at least one file"
+    )
 
 
 class FileContainsCriterion(BaseSuccessCriterion):
@@ -331,7 +333,7 @@ class FileContainsCriterion(BaseSuccessCriterion):
     """
 
     type: Literal["file_contains"] = "file_contains"
-    path: str = Field(description="Path to the file to check")
+    path: str = Field(description="Path to the file to check; may be a glob matching exactly one file")
     includes: list[str] = Field(description="List of strings that must be present in the file")
     excludes: list[str] | None = Field(default=None, description="List of strings that must NOT be present in the file")
 
@@ -404,7 +406,7 @@ class FileMatchesRegexCriterion(BaseSuccessCriterion):
     """
 
     type: Literal["file_matches_regex"] = "file_matches_regex"
-    path: str = Field(description="Path to the file to check")
+    path: str = Field(description="Path to the file to check; may be a glob matching exactly one file")
     pattern: str = Field(description="Regex pattern that must match somewhere in the file")
     must_match: bool = Field(default=True, description="If True, pattern must match; if False, pattern must NOT match")
     flags: int = Field(default=0, description="Regex flags (e.g., re.IGNORECASE=2, re.MULTILINE=8, re.DOTALL=16)")
@@ -777,8 +779,13 @@ class JsonCheckCriterion(BaseSuccessCriterion):
     """
 
     type: Literal["json_check"] = "json_check"
-    path: str = Field(description="Path to the JSON file (relative to sandbox root)")
-    json_schema: str | None = Field(default=None, description="Path to JSON Schema file (relative to sandbox root)")
+    path: str = Field(
+        description="Path to the JSON file (relative to sandbox root); may be a glob matching exactly one file"
+    )
+    json_schema: str | None = Field(
+        default=None,
+        description="Path to JSON Schema file (relative to sandbox root); may be a glob matching exactly one file",
+    )
     assertions: list[JMESPathAssertion] = Field(
         default_factory=list, description="JMESPath assertions to evaluate against the parsed JSON"
     )
@@ -809,7 +816,9 @@ class FileCheckCriterion(BaseSuccessCriterion):
     """
 
     type: Literal["file_check"] = "file_check"
-    path: str = Field(description="Path to the file to check (relative to sandbox root)")
+    path: str = Field(
+        description="Path to the file to check (relative to sandbox root); may be a glob matching exactly one file"
+    )
     includes: list[str] = Field(default_factory=list, description="Strings that must be present in the file")
     excludes: list[str] = Field(default_factory=list, description="Strings that must NOT be present in the file")
     patterns: list[RegexPattern] = Field(
@@ -841,7 +850,9 @@ class ReferenceComparisonCriterion(BaseSuccessCriterion):
     type: Literal["reference_comparison"] = "reference_comparison"
 
     # Required fields
-    agent_file: str = Field(description="Path to agent's generated file (relative to sandbox root)")
+    agent_file: str = Field(
+        description="Path to agent's generated file (relative to sandbox root); may be a glob matching exactly one file"
+    )
 
     comparison_method: Literal["ast", "token", "complexity"] = Field(
         default="ast",
@@ -1033,7 +1044,12 @@ class ClassificationMatchCriterion(BaseSuccessCriterion):
     """
 
     type: Literal["classification_match"] = "classification_match"
-    path: str = Field(description="Path to the file (relative to sandbox) containing the agent's predicted label")
+    path: str = Field(
+        description=(
+            "Path to the file (relative to sandbox) containing the agent's predicted label; "
+            "may be a glob matching exactly one file"
+        )
+    )
     expected_label: str = Field(description="Ground-truth label for this row")
     allowed_labels: list[str] = Field(
         min_length=1,

@@ -66,6 +66,27 @@
     ".wget-hsts",
 )
 
+# Characters that make a criterion `path` eligible for glob expansion. Eligible,
+# not automatic: `Sandbox.resolve_files` tries the literal path first.
+_GLOB_METACHARACTERS = "*?["
+
+# Cap on how many matches an ambiguity error enumerates. The message is
+# persisted to task.json and injected into judge prompts, so an unbounded
+# listing over a wide pattern is a real payload.
+_MAX_LISTED_MATCHES = 10
+
+
+def _is_glob(path: str) -> bool:
+    """Return whether ``path`` contains a glob metacharacter."""
+    return any(c in path for c in _GLOB_METACHARACTERS)
+
+
+def _format_matches(matches: list[Path], root: Path) -> str:
+    """Render matches as sandbox-relative paths, truncated to a bounded list."""
+    listed = ", ".join(str(p.relative_to(root)) for p in matches[:_MAX_LISTED_MATCHES])
+    remaining = len(matches) - _MAX_LISTED_MATCHES
+    return f"{listed}, +{remaining} more" if remaining > 0 else listed
+
 
 def _grant_read_traverse(root: Path) -> None:
     """Recursively apply ``chmod a+rX`` semantics under ``root``.
@@ -1093,38 +1114,121 @@ def run_command(self, command: str, timeout: float | int | None = None) -> tuple
     # needs filesystem access beyond the sandbox root (e.g., reading installed packages,
     # system headers). Path traversal protection is handled at the agent permission level.
 
+    def resolve_files(self, path: str) -> list[Path]:
+        """Resolve a criterion ``path`` to the sandbox files it addresses.
+
+        A path that names an existing file or directory resolves to itself,
+        **even when it contains a glob metacharacter** — a real file called
+        ``report[2024].json`` is graded as itself rather than reinterpreted as
+        a character class that would silently match ``report2.json``. Only when
+        the literal does not exist is a path containing ``*``, ``?`` or ``[``
+        expanded against the sandbox root, so a criterion can address a file
+        whose exact location the task prompt does not pin — e.g. ``**/*.flow``
+        matches a scaffolded wrapper directory the agent was free to name.
+
+        Glob matches are filtered through the sandbox's ignore patterns
+        (``.venv``, ``node_modules``, ``dist``, … — see
+        :func:`~coder_eval.resources.get_ignore_patterns`), because the sandbox
+        root holds harness-created content the agent never authored and
+        grading off it is neither fair nor deterministic. Only path segments
+        the glob *discovered* are filtered: a segment the pattern names
+        literally (``dist/**/*.js``) is an explicit opt-in and survives.
+        Matches are sorted so grading is deterministic, and directories are
+        dropped so a glob cannot resolve to something unreadable.
+
+        Args:
+            path: Relative path or glob pattern
+
+        Returns:
+            Sorted matching files; empty when nothing matches
+        """
+        if not self.sandbox_dir:
+            return []
+
+        # Literal first: an existing path is never reinterpreted as a pattern.
+        candidate = self.sandbox_dir / path
+        if candidate.exists():
+            return [candidate]
+
+        if not _is_glob(path):
+            return []
+
+        patterns = get_ignore_patterns(self.config.ignore_patterns)
+        pinned = {segment for segment in path.split("/") if segment and not _is_glob(segment)}
+
+        matches: list[Path] = []
+        for match in self.sandbox_dir.glob(path):
+            if not match.is_file():
+                continue
+            discovered = [part for part in match.relative_to(self.sandbox_dir).parts if part not in pinned]
+            if discovered and should_ignore_path(Path(*discovered), patterns):
+                continue
+            matches.append(match)
+
+        return sorted(matches)
+
+    def resolved_path_label(self, path: str) -> str | None:
+        """Sandbox-relative path a glob resolved to, for grading transparency.
+
+        With exactly-one-match semantics on content reads, *which* file was
+        graded is most of the signal. Returns ``None`` for a literal path
+        (nothing was inferred) and for a pattern that did not resolve to
+        exactly one file.
+
+        Args:
+            path: Relative path or glob pattern
+
+        Returns:
+            Sandbox-relative path of the single match, or ``None``
+        """
+        if not self.sandbox_dir or not _is_glob(path):
+            return None
+
+        matches = self.resolve_files(path)
+        if len(matches) != 1:
+            return None
+
+        return str(matches[0].relative_to(self.sandbox_dir))
+
     def get_file_content(self, path: str) -> str:
         """Read the content of a file in the sandbox.
 
         Args:
-            path: Relative path to the file
+            path: Relative path to the file, or a glob pattern matching exactly
+                one file
 
         Returns:
             File content as string
 
         Raises:
             RuntimeError: If sandbox is not set up
-            FileNotFoundError: If file doesn't exist
+            FileNotFoundError: If nothing matches ``path``
+            ValueError: If a glob matches more than one file
         """
         if not self.sandbox_dir:
             raise RuntimeError("Sandbox not set up")
 
-        file_path = self.sandbox_dir / path
-        return file_path.read_text(encoding="utf-8")
+        matches = self.resolve_files(path)
+        if not matches:
+            raise FileNotFoundError(f"No file matches '{path}' in the sandbox")
+        if len(matches) > 1:
+            raise ValueError(
+                f"Pattern '{path}' matches {len(matches)} files — refusing to guess which to grade: "
+                + _format_matches(matches, self.sandbox_dir)
+            )
+
+        return matches[0].read_text(encoding="utf-8")
 
     def file_exists(self, path: str) -> bool:
         """Check if a file exists in the sandbox.
 
         Args:
-            path: Relative path to the file
+            path: Relative path to the file, or a glob pattern
 
         Returns:
-            True if file exists, False otherwise
+            True if at least one file matches, False otherwise
         """
-        if not self.sandbox_dir:
-            return False
-
-        return (self.sandbox_dir / path).exists()
+        return bool(self.resolve_files(path))
 
     def list_files(self, path: str = ".") -> list[str]:
         """List files in a directory within the sandbox.