Merge pull request #25 from KadenMc/fix/validator-batch-ledger-resolution

fix: batch citations resolve against the universal ledger

Author: KadenMc

CHANGELOG.md

@@ -7,6 +7,40 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.6.1] - 2026-05-26
+
+### Fixed
+
+- **Validator batch citations now resolve against the universal ledger.**
+  In 0.6.0 the job-id citation check correctly consulted both the local
+  signac project AND `.aexp/ledger/<id>.json`, but the batch-citation
+  check still went through `aexp.linking.list_batches()` — which walks
+  the local signac project only. The result: a finding citing a batch
+  selector like `{type: batch, experiment_id: E005, selector: {condition:
+  X}}` would spuriously emit `finding.empty_batch` even when the
+  universal ledger held matching terminal runs from another machine.
+  Now the batch-resolution path also builds an `(experiment_id,
+  condition) -> [job_ids]` index from ledger entries and counts those
+  matches before falling through to the empty/elsewhere paths. Closes
+  the gap surfaced during the electricrag laptop validate at 0.6.0:
+  after the cluster backfilled, F005 batch citations correctly resolve.
+  Regression test: `tests/test_validate_cross_machine.py::test_validator_batch_citation_resolves_via_ledger_only`.
+
+### Notes
+
+- `registered_machine` in ledger entries remains *informational
+  provenance only* — the validator does not consult it for resolution.
+  A run is in the ledger or it isn't; which machine registered it
+  doesn't affect whether it satisfies a citation. (This is intentional
+  per the Phase 2 universal-ledger design and is documented in the new
+  regression test's comment.)
+
+### Note about PyPI
+
+0.6.0 was tagged in source but never published to PyPI; this 0.6.1
+release is the first publish of the cross-machine-ledger surface. PyPI
+history therefore goes 0.5.0 → 0.6.1.
+
 ## [0.6.0] - 2026-05-25
 
 ### Added

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "agentic-experiments"
-version = "0.6.0"
+version = "0.6.1"
 description = "Git-first, hypothesis-forcing experiment tracking for agent-driven ML research. Bundles a research harness for the H->E->F artifact model, uses signac for local execution/run state, and bridges to W&B for remote observability."
 authors = [
     {name = "Kaden McKeen", email = "mckeenkaden@gmail.com"}

src/aexp/validate.py

@@ -290,7 +290,7 @@ def _check_finding_citations(
     """
     # Lazy imports to keep validate.py importable when runs_index/ledger
     # have issues, and to defer the (small) import cost.
-    from aexp.ledger import list_ledger_job_ids
+    from aexp.ledger import list_ledger_job_ids, load_ledger_entry
     from aexp.runs_index import collect_known_elsewhere
 
     issues: list[Issue] = []
@@ -353,6 +353,25 @@ def _check_finding_citations(
         if isinstance(exp, str) and isinstance(cond, str):
             elsewhere_batches.setdefault((exp, cond), []).append(entry)
 
+    # Universal ledger entries also need to participate in batch resolution.
+    # The job-id existence check above already counts ledger entries via
+    # `known_job_ids = ledger_ids | local_ids`, but the batch-citation check
+    # uses `list_batches()` which walks the local signac project only and
+    # therefore can't see ledger-only entries. Without this lookup, a batch
+    # citation like `{type: batch, experiment_id: E005, selector: {condition: X}}`
+    # spuriously emits `finding.empty_batch` even though the universal ledger
+    # has matching terminal runs from another machine.
+    ledger_batches: dict[tuple[str, str], list[str]] = {}
+    for jid in ledger_ids:
+        entry = load_ledger_entry(repo_root, jid)
+        if not entry:
+            continue
+        sp = entry.get("statepoint", {}) or {}
+        exp = sp.get("experiment_id")
+        cond = sp.get("condition")
+        if isinstance(exp, str) and isinstance(cond, str):
+            ledger_batches.setdefault((exp, cond), []).append(jid)
+
     # finding.no_run_store: a single warning per validate run when no
     # source-of-truth for run identity is available — no ledger entries,
     # no local store, no cross-machine indexes. Without this, the validator
@@ -494,10 +513,28 @@ def _check_finding_citations(
                 local_count = sum(b.count for b in local_matches) if local_matches else 0
 
                 if local_count > 0:
-                    continue  # here: clean
+                    continue  # here: clean (local matches)
+
+                # Then check the universal ledger. `list_batches()` only walks
+                # the local signac project, so ledger-only entries (no local
+                # workspace) need a separate check via the (exp, cond) index
+                # built above. Without this, batch citations spuriously emit
+                # `finding.empty_batch` even when the ledger has matching
+                # terminal runs from another machine.
+                cond = selector.get("condition") if isinstance(selector, dict) else None
+                ledger_match_count = 0
+                if isinstance(cond, str):
+                    ledger_match_count = len(ledger_batches.get((exp_id, cond), []))
+                elif isinstance(selector, dict) and not selector:
+                    # Empty selector: any ledger entry for this experiment counts.
+                    ledger_match_count = sum(
+                        1 for (e, _c) in ledger_batches.keys() if e == exp_id
+                    )
+
+                if ledger_match_count > 0:
+                    continue  # here: clean (ledger matches)
 
                 # Check the elsewhere index. Match by experiment_id + condition.
-                cond = selector.get("condition") if isinstance(selector, dict) else None
                 elsewhere_matches: list[dict[str, Any]] = []
                 if isinstance(cond, str):
                     elsewhere_matches = elsewhere_batches.get((exp_id, cond), [])

tests/test_validate_cross_machine.py

@@ -687,6 +687,96 @@ def test_validator_treats_ledger_entry_as_here(installed_repo: Path) -> None:
     assert result.ok
 
 
+def test_validator_batch_citation_resolves_via_ledger_only(
+    installed_repo: Path,
+) -> None:
+    """Regression: batch citations must resolve against ledger entries
+    that have NO local signac workspace.
+
+    Before the fix, the batch-citation check only walked the local signac
+    project (via `list_batches`), so a finding citing
+    `{type: batch, experiment_id: E001, selector: {condition: X}}` whose
+    only matching runs lived in the ledger from another machine would
+    spuriously emit `finding.empty_batch` even though the universal ledger
+    had matching terminal runs.
+    """
+    kb = installed_repo / "kb"
+    _seed_h_e_artifacts(kb)
+
+    # Synthesize a ledger entry that lives ONLY in .aexp/ledger/
+    # (no corresponding .runs/workspace/<id>/ on this machine).
+    #
+    # NOTE on `registered_machine`: this field is *informational
+    # provenance* — the validator does NOT consult it during batch
+    # resolution (or any other resolution path). The string `"cluster"`
+    # below is purely to mirror the real-world scenario this test
+    # regresses (cluster-registered runs satisfying a citation read on
+    # the laptop); the test would pass identically with any other value
+    # in `registered_machine`. The Phase 2 design choice was that the
+    # ledger is universal: a run is in the ledger or it isn't, and
+    # machine identity does not gate satisfaction.
+    jid = "9" * 32
+    target = installed_repo / LEDGER_DIR_REL / f"{jid}.json"
+    target.parent.mkdir(parents=True, exist_ok=True)
+    target.write_text(
+        _json.dumps(
+            {
+                "schema_version": 1,
+                "job_id": jid,
+                "status": "complete",
+                "statepoint": {
+                    "experiment_id": "E001",
+                    "condition": "remote_only",
+                },
+                "registered_machine": "cluster",
+                "promoted_at": "2026-05-26T00:00:00Z",
+            },
+            indent=2,
+            sort_keys=True,
+        )
+        + "\n",
+        encoding="utf-8",
+    )
+
+    # F001 cites a BATCH (not a job id) by experiment + condition selector.
+    _write_artifact(
+        kb,
+        "research/findings",
+        "F001-a.md",
+        {
+            "id": "F001",
+            "type": "finding",
+            "hypothesis": "H001",
+            "experiment": "E001",
+            "impact": "moderate",
+            "created": "2026-04-20",
+            "supporting_runs": [
+                {
+                    "type": "batch",
+                    "experiment_id": "E001",
+                    "selector": {"condition": "remote_only"},
+                }
+            ],
+        },
+        "# F001\n> **Hypothesis**: [[H001]]\n> **Experiment**: [[E001]]\n> **Impact**: moderate\n> **Created**: 2026-04-20\n",
+    )
+
+    result = validate_repo(installed_repo, mode="runs-only")
+    citation_codes = [
+        i.code
+        for i in result.issues
+        if i.code
+        in (
+            "finding.broken_run_citation",
+            "finding.absent_run_citation",
+            "finding.empty_batch",
+            "finding.absent_batch_runs",
+        )
+    ]
+    assert citation_codes == [], [i.message for i in result.issues]
+    assert result.ok
+
+
 def test_validator_ledger_supersedes_runs_index_for_same_job(
     installed_repo: Path,
 ) -> None: