Build/cache native modules (#2470)

Author: Dreamsorcerer

.github/workflows/ci.yml

@@ -315,11 +315,109 @@ jobs:
           use_pypi: true
           version: "11.2.8"
 
+  cmu-nav-natives-marker:
+    # Check whether the modules-for-this-config marker cache
+    # already exists. Used to skip cmu-nav-natives without using the cachix environment.
+    if: |
+      github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read  # For checkout
+    outputs:
+      cache-hit: ${{ steps.marker.outputs.cache-hit }}
+    steps:
+      - name: Checkout files used for cache key
+        uses: actions/checkout@v6
+        with:
+          sparse-checkout: |
+            bin/build-cmu-nav-natives
+            dimos/navigation/cmu_nav/modules
+          sparse-checkout-cone-mode: false
+      - name: Look up marker cache
+        id: marker
+        uses: actions/cache/restore@v4
+        with:
+          path: /tmp/cmu-nav-natives-marker
+          key: cmu-nav-natives-${{ hashFiles('bin/build-cmu-nav-natives', 'dimos/navigation/cmu_nav/modules/**') }}
+          lookup-only: true
+
+  cmu-nav-natives:
+    # ⚠️ MUST NEVER RUN ON EXTERNAL PRs ⚠️
+    # This job has write access to CACHIX_AUTH_TOKEN. Any workflow change that
+    # reaches a `run:` step here could exfiltrate the token, letting an attacker
+    # push poisoned binaries to the Cachix cache — which subsequent CI runs and
+    # every dev with the substituter configured would unwittingly install. The
+    # `if:` guard below is the only thing keeping fork PRs out — DO NOT REMOVE.
+    if: |
+      !cancelled() &&
+      (github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository) &&
+      needs.cmu-nav-natives-marker.outputs.cache-hit != 'true'
+    needs: cmu-nav-natives-marker
+    timeout-minutes: 300
+    environment: cachix
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read  # For checkout
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+      - name: Free runner disk space
+        uses: BRAINSia/free-disk-space@v2.1.3
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: true
+      - name: Install minimal dimos for module discovery
+        run: uv sync --no-default-groups --frozen
+      - name: Install Nix (with Cachix substituter)
+        env:
+          INPUT_EXTRA_NIX_CONFIG: |
+            extra-substituters = https://dimensionalos.cachix.org
+            extra-trusted-public-keys = dimensionalos.cachix.org-1:20ynj6TjpoD3qTxkdNoeHtgs2G2pNvgAq1EQYLTHJXI=
+          INPUT_SET_AS_TRUSTED_USER: "true"
+        run: bash docker/ros/install-nix.sh
+      - name: Cache Nix fetcher + eval cache
+        # ~/.cache/nix holds the flake fetcher cache (the github tarballs of
+        # each `nix build github:…` ref) and the eval cache (compiled flake
+        # outputs). ~/.local/state/nix holds profile/gcroot bookkeeping.
+        # Restoring these skips the ~20s/module evaluation cost, which is
+        # paid every cold run otherwise. /nix/store is intentionally NOT
+        # cached here — that's the slow heavy part that Cachix handles.
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/nix
+            ~/.local/state/nix
+          key: nix-fetcher-build-${{ runner.os }}-${{ hashFiles('bin/build-cmu-nav-natives', 'dimos/navigation/cmu_nav/modules/**/*.py') }}
+          restore-keys: nix-fetcher-build-${{ runner.os }}-
+      - name: Authenticate Cachix
+        uses: cachix/cachix-action@v15
+        with:
+          name: dimensionalos
+          authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
+      - name: Build cmu_nav native modules
+        env:
+          BUILD_WORKERS: "2"
+        run: uv run bin/build-cmu-nav-natives
+      - name: Write marker file
+        run: install -D /dev/null /tmp/cmu-nav-natives-marker/ok
+      - name: Save marker cache
+        # Done as a final step (rather than via actions/cache's post-step) so
+        # a build failure doesn't poison the cache with a marker for a config
+        # we never actually pushed.
+        uses: actions/cache/save@v4
+        with:
+          path: /tmp/cmu-nav-natives-marker
+          key: cmu-nav-natives-${{ hashFiles('bin/build-cmu-nav-natives', 'dimos/navigation/cmu_nav/modules/**') }}
+
   self-hosted-tests:
     # Skip on PRs from forks which would expose the self-hosted runner to untrusted code from external contributors.
     if: |
-      github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository
-    needs: compute-ros-pin
+      !cancelled() &&
+      (github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository) &&
+      contains(fromJSON('["success", "skipped"]'), needs.cmu-nav-natives.result)
+    needs: [compute-ros-pin, cmu-nav-natives]
     env:
       OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
       ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
@@ -333,6 +431,11 @@ jobs:
             # GitHub Actions only honours `container:` on Linux runners.
             container:
               image: ghcr.io/dimensionalos/ros-dev@${{ needs.compute-ros-pin.outputs.digest }}
+              # Persist Nix state on the self-hosted runner's filesystem so
+              # the ~3 GB module closures survive between jobs.
+              volumes:
+                - /var/cache/dimos-nix:/nix
+                - /var/cache/dimos-root-cache:/root/.cache
             markers: "self_hosted or skipif_no_ros"
             experimental: false
           - os: macOS
@@ -396,6 +499,35 @@ jobs:
             echo "ROS_VERSION=$ROS_VERSION"
             echo "ROS_PYTHON_VERSION=$ROS_PYTHON_VERSION"
           } >> "$GITHUB_ENV"
+      - name: Install Nix (with Cachix substituter)
+        if: contains(matrix.markers, 'skipif_no_ros')
+        # /nix is bind-mounted from the self-hosted runner's host filesystem
+        # (see this job's matrix container.volumes). On warm runs Nix is
+        # already installed; install-nix.sh's `type -p nix` check will exit
+        # early, but only if the profile bin is on PATH — prepend it before
+        # calling. If /nix is empty (first job on a fresh runner), nix isn't
+        # on PATH and the install proceeds normally. After install-nix.sh,
+        # we unconditionally (re-)write /etc/nix/nix.conf: on warm runs
+        # install-nix.sh exited before it would have written the config,
+        # which leaves `experimental-features = nix-command flakes` absent
+        # and the build dying with "experimental Nix feature 'nix-command'
+        # is disabled". Writing it here keeps cold and warm runs identical.
+        run: |
+          export PATH="/nix/var/nix/profiles/default/bin:$PATH"
+          bash docker/ros/install-nix.sh
+          sudo mkdir -p /etc/nix
+          sudo tee /etc/nix/nix.conf > /dev/null <<'EOF'
+          experimental-features = nix-command flakes
+          extra-substituters = https://dimensionalos.cachix.org
+          extra-trusted-public-keys = dimensionalos.cachix.org-1:20ynj6TjpoD3qTxkdNoeHtgs2G2pNvgAq1EQYLTHJXI=
+          always-allow-substitutes = true
+          build-users-group =
+          max-jobs = auto
+          EOF
+          echo "/nix/var/nix/profiles/default/bin" >> "$GITHUB_PATH"
+      - name: Fetch cmu_nav native modules from Cachix
+        if: contains(matrix.markers, 'skipif_no_ros')
+        run: uv run bin/build-cmu-nav-natives
       - name: Run tests
         run: uv run pytest --cov=dimos/ --junitxml=junit.xml -m '(${{ matrix.markers }}) and not mujoco'
       - name: Re-run the failing tests with maximum verbosity
@@ -559,6 +691,7 @@ jobs:
     - md-babel
     - docs-validate
     - tests
+    - cmu-nav-natives
     - self-hosted-tests
     #- self-hosted-large-tests
 
@@ -570,9 +703,7 @@ jobs:
     - name: Decide whether the needed jobs succeeded or failed
       uses: re-actors/alls-green@release/v1
       with:
-        allowed-skips: |
-          self-hosted-tests
-          self-hosted-large-tests
+        allowed-skips: cmu-nav-natives, self-hosted-tests, self-hosted-large-tests
         jobs: ${{ toJSON(needs) }}
     - name: Trigger Codecov notifications
       uses: codecov/codecov-action@v6

bin/build-cmu-nav-natives

@@ -0,0 +1,111 @@
+#!/usr/bin/env python3
+"""Pre-build native modules ready for tests.
+
+Discovers every cmu_nav module that defines a NativeModuleConfig with a
+`build_command` set, then runs that command. CI invokes this after installing
+Nix + the Cachix substituter so warm-cache builds are near-instant.
+
+Discovery imports each module and reads the config defaults off an
+instantiated `NativeModuleConfig` — robust against any refactor of how those
+fields are declared (literal vs computed vs factory), at the cost of needing
+dimos importable (run via `uv run`).
+"""
+
+from concurrent.futures import ProcessPoolExecutor, as_completed
+import importlib
+import os
+from pathlib import Path
+import re
+import subprocess
+import sys
+import time
+
+from dimos.core.native_module import NativeModuleConfig
+import dimos.navigation.cmu_nav.modules as modules_pkg
+
+MODULES_DIR = Path(modules_pkg.__path__[0])
+# Captured at import time; forked workers inherit it
+_T0 = time.monotonic()
+
+
+def _log(msg: str) -> None:
+    print(f"[{time.monotonic() - _T0:6.1f}s] {msg}", flush=True)
+
+
+# Pre-filter source files for a NativeModuleConfig subclass before importing
+# them. Modules that don't subclass NativeModuleConfig (e.g. nav_record) may
+# pull in heavy deps like torch at top-level import, and we don't want to
+# require those just to discover.
+_HAS_NATIVE_CONFIG = re.compile(r"^class\s+\w+\s*\(\s*NativeModuleConfig\b", re.MULTILINE)
+
+
+def discover() -> tuple[type[NativeModuleConfig], ...]:
+    found: list[type[NativeModuleConfig]] = []
+    for pkg_dir in sorted(MODULES_DIR.iterdir()):
+        if not pkg_dir.is_dir():
+            continue
+        leaf = pkg_dir.name
+        src = pkg_dir / f"{leaf}.py"
+        if not src.exists() or not _HAS_NATIVE_CONFIG.search(src.read_text()):
+            continue
+        mod = importlib.import_module(f"{modules_pkg.__name__}.{leaf}.{leaf}")
+        for attr in vars(mod).values():
+            if (
+                isinstance(attr, type)
+                and attr is not NativeModuleConfig
+                and issubclass(attr, NativeModuleConfig)
+                and attr.__module__ == mod.__name__
+                and attr.model_fields["build_command"].default is not None
+            ):
+                found.append(attr)
+    return tuple(found)
+
+
+def build_one(cfg_cls: type[NativeModuleConfig]) -> str:
+    name = cfg_cls.__name__
+    cfg = cfg_cls()
+    assert cfg.build_command is not None
+    mod = importlib.import_module(cfg_cls.__module__)
+    base_dir = Path(mod.__file__).resolve().parent  # type: ignore[arg-type]
+    cwd = Path(cfg.cwd) if cfg.cwd is not None else base_dir
+    if not cwd.is_absolute():
+        cwd = base_dir / cwd
+    exe = Path(cfg.executable)
+    if not exe.is_absolute():
+        exe = cwd / exe
+    if exe.exists():
+        return f"OK:    {name} already built at {exe}"
+    _log(f"START: {name} in {cwd}: {cfg.build_command}")
+    proc = subprocess.run(
+        cfg.build_command,
+        shell=True,
+        cwd=str(cwd),
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    body = (proc.stdout or "") + (proc.stderr or "")
+    if proc.returncode != 0:
+        raise RuntimeError(f"{name} exited {proc.returncode}\n{body}")
+    return f"DONE:  {name}\n{body}".rstrip()
+
+
+def main() -> None:
+    cfgs = discover()
+    failed = 0
+    workers = int(os.environ.get("BUILD_WORKERS") or os.cpu_count() or 2)
+    _log(f"Building {len(cfgs)} module(s) with up to {workers} workers")
+    with ProcessPoolExecutor(max_workers=workers) as ex:
+        futs = {ex.submit(build_one, c): c for c in cfgs}
+        for fut in as_completed(futs):
+            try:
+                _log(fut.result())
+            except Exception as e:
+                _log(f"FAIL:  {futs[fut].__name__}: {e}")
+                failed += 1
+    if failed:
+        sys.exit(f"{failed} build(s) failed")
+
+
+if __name__ == "__main__":
+    main()

dimos/navigation/cmu_nav/modules/far_planner/far_planner.py

@@ -34,7 +34,7 @@ class FarPlannerConfig(NativeModuleConfig):
     cwd: str | None = str(Path(__file__).resolve().parent)
     executable: str = "result/bin/far_planner_native"
     build_command: str | None = (
-        "nix build github:dimensionalOS/dimos-module-far-planner/v0.5.0 --no-write-lock-file"
+        "nix build github:dimensionalOS/dimos-module-far-planner/v0.7.1 --no-write-lock-file"
     )
 
     # C++ binary uses snake_case CLI args.

dimos/navigation/cmu_nav/modules/path_follower/path_follower.py

@@ -34,7 +34,7 @@ class PathFollowerConfig(NativeModuleConfig):
     cwd: str | None = str(Path(__file__).resolve().parent)
     executable: str = "result/bin/path_follower"
     build_command: str | None = (
-        "nix build github:dimensionalOS/dimos-module-path-follower/v0.2.0 --no-write-lock-file"
+        "nix build github:dimensionalOS/dimos-module-path-follower/v0.2.1 --no-write-lock-file"
     )
 
     cli_name_override: dict[str, str] = {

dimos/navigation/cmu_nav/modules/terrain_analysis/terrain_analysis.py

@@ -25,7 +25,7 @@ class TerrainAnalysisConfig(NativeModuleConfig):
     cwd: str | None = "."
     executable: str = "result/bin/terrain_analysis"
     build_command: str | None = (
-        "nix build github:dimensionalOS/dimos-module-terrain-analysis/v0.1.1 --no-write-lock-file"
+        "nix build github:dimensionalOS/dimos-module-terrain-analysis/v0.1.2 --no-write-lock-file"
     )
     cli_name_override: dict[str, str] = {
         "sensor_range": "sensorRange",

docker/ros/install-nix.sh

@@ -1,6 +1,13 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
+# Defaults so this script can be invoked from a plain workflow `run:` step,
+# not only via the composite action that exports these INPUT_* env vars.
+: "${INPUT_ENABLE_KVM:=false}"
+: "${INPUT_SET_AS_TRUSTED_USER:=false}"
+: "${INPUT_EXTRA_NIX_CONFIG:=}"
+: "${INPUT_INSTALL_OPTIONS:=}"
+
 if nix_path="$(type -p nix)" ; then
   echo "Aborting: Nix is already installed at ${nix_path}"
   exit

docs/usage/native_modules.md

@@ -268,3 +268,13 @@ class MyLidarConfig(NativeModuleConfig):
 `cwd` is used for both the build command and the runtime subprocess. Relative paths are resolved against the directory of the Python file that defines the module
 
 If the executable already exists, the build step is skipped entirely.
+
+### Faster builds via the Cachix substituter
+
+CI pre-builds the `cmu_nav` native modules and pushes the Nix store paths to the `dimensionalos` Cachix cache. Opt in locally to skip cold compiles when the cache has them:
+
+```
+# ~/.config/nix/nix.conf  (single-user)  or  /etc/nix/nix.conf  (multi-user)
+extra-substituters = https://dimensionalos.cachix.org
+extra-trusted-public-keys = dimensionalos.cachix.org-1:20ynj6TjpoD3qTxkdNoeHtgs2G2pNvgAq1EQYLTHJXI=
+```