From 85cd1d2d9b542f764998ee248dc1b1ddd3151f41 Mon Sep 17 00:00:00 2001 From: naidenovaleksei Date: Wed, 1 Jul 2026 11:10:54 +0200 Subject: [PATCH] feat: add onnx benchmark --- .gitignore | 2 +- config/benchmark_onnx.yaml | 20 + justfile | 7 + prek.toml | 2 +- pyproject.toml | 13 + src/rmind/scripts/benchmark_onnx.py | 963 ++++++++++++++++++++++++++++ uv.lock | 61 ++ 7 files changed, 1066 insertions(+), 2 deletions(-) create mode 100644 config/benchmark_onnx.yaml create mode 100644 src/rmind/scripts/benchmark_onnx.py diff --git a/.gitignore b/.gitignore index 0c5dfe50..d050eb1b 100644 --- a/.gitignore +++ b/.gitignore @@ -156,7 +156,7 @@ CLAUDE.md # artifacts inference_results/ -.rbyte_cache +.rbyte_cache* .dprint.json diff --git a/config/benchmark_onnx.yaml b/config/benchmark_onnx.yaml new file mode 100644 index 00000000..53d43c17 --- /dev/null +++ b/config/benchmark_onnx.yaml @@ -0,0 +1,20 @@ +# @package _global_ + +--- +defaults: + - export: null + - _self_ + +data_dir: ${oc.env:HOME}/data/Niro122-HQ/2023-05-25--09-34-14 +start_frame: 2910 +num_episodes: 50 +frame_step: 10 +output: /tmp/rmind.csv +warmup: 1 +image_size: null + +# override per run, e.g.: +# onnx=/path/to/model.onnx wandb_model=yaak/rmind/model-XXXXXXXX:vN +# export=yaak/control_transformer/finetuned # apples-to-apples vs the ONNX export +onnx: null +wandb_model: null diff --git a/justfile b/justfile index 17334d87..cdaca3d4 100644 --- a/justfile +++ b/justfile @@ -87,6 +87,13 @@ test *ARGS: generate-config update-snapshots: uv run python -m tests.scripts.update_snapshots +benchmark-onnx *ARGS: generate-config + LD_LIBRARY_PATH="$(find /nix/store -maxdepth 1 -name '*gcc-15*-lib' -type d -print -quit 2>/dev/null)/lib:${LD_LIBRARY_PATH:-}" \ + uv run --group benchmark rmind-benchmark-onnx \ + --config-path {{ justfile_directory() }}/config \ + --config-name benchmark_onnx.yaml \ + {{ ARGS }} + export-onnx *ARGS: generate-config uv run \ --extra export \ diff --git a/prek.toml b/prek.toml index 1266f692..1be640a1 100644 --- a/prek.toml +++ b/prek.toml @@ -57,7 +57,7 @@ hooks = [ repo = "https://github.com/codespell-project/codespell" rev = "v2.4.2" hooks = [ - { id = "codespell" }, + { id = "codespell", args = ["--ignore-words-list=drivr"] }, ] [[repos]] diff --git a/pyproject.toml b/pyproject.toml index c481e93e..b9fa86e6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,6 +41,7 @@ dependencies = [ repo = "https://github.com/yaak-ai/rmind" [project.scripts] +rmind-benchmark-onnx = "rmind.scripts.benchmark_onnx:main" rmind-check-git = "rmind.scripts.check_git:main" rmind-export-onnx = "rmind.scripts.export_onnx:main" rmind-predict = "rmind.scripts.predict:main" @@ -76,6 +77,13 @@ dev = [ "pudb>=2025.1.5", "wat-inspector", ] +benchmark = [ + "onnxruntime>=1.23.1", + "opencv-python-headless", + "pyproj>=3.6.0", + "tabulate>=0.9.0", + "wandb>=0.24.2", +] check = [ "ruff>=0.15.18", "ty>=0.0.52", @@ -104,13 +112,18 @@ DEP002 = [ ] DEP003 = ["rmind"] DEP004 = [ + "cv2", "onnxruntime", + "pyproj", + "tabulate", "tensorrt", ] [tool.deptry.package_module_name_map] hydra-core = "hydra" +opencv-python-headless = "cv2" tensordict-nightly = "tensordict" +tensorrt-cu13 = "tensorrt" pytorch-lightning = [ "pytorch_lightning", "lightning_fabric", diff --git a/src/rmind/scripts/benchmark_onnx.py b/src/rmind/scripts/benchmark_onnx.py new file mode 100644 index 00000000..a8a388b3 --- /dev/null +++ b/src/rmind/scripts/benchmark_onnx.py @@ -0,0 +1,963 @@ +"""Benchmark ControlTransformer on raw ride data — ONNX and/or wandb model. + +Both backends can be active at once so their predictions are compared side by side. +Fixed run parameters (data_dir, start_frame, num_episodes, ...) live in +config/benchmark_onnx.yaml — override just `onnx=` and/or `wandb_model=` per run. + +Usage: + # ONNX only (compare with drivr's benchmark_all_models.py) + just benchmark-onnx onnx=~/rmind/outputs/.../model.onnx + + # wandb PyTorch only + just benchmark-onnx wandb_model=yaak/rmind/model-XXXXXXXX:vN + + # Both side by side (ONNX vs torch, same batches) + just benchmark-onnx \\ + onnx=~/rmind/outputs/.../model.onnx \\ + wandb_model=yaak/rmind/model-XXXXXXXX:vN +""" + +from __future__ import annotations + +import bisect +import csv +import json +import mmap +import operator +import time +from collections.abc import Sequence # noqa: TC003 — needed at runtime by pydantic +from dataclasses import dataclass, field, replace +from pathlib import Path +from typing import Any, ClassVar, cast + +import hydra +import numpy as np +import structlog +import torch +from omegaconf import DictConfig, OmegaConf +from pydantic import BaseModel, ConfigDict, field_validator + +logger = structlog.get_logger(__name__) + +EMBED_DIM = 384 +NUM_TIMESTEPS = 6 +NUM_WAYPOINTS = 10 +DEFAULT_IMAGE_SIZE = (324, 576) # (H, W) +_ONNX_IMAGE_INPUT_NDIM = 5 # [B, T, C, H, W] +_TARGET_LATENCY_MS = 100 +_MIN_BACKENDS_FOR_COMPARISON = 2 +_VALIDATION_TOLERANCE = 1e-3 + +# Lowercase batch_data_* keys — match drivr's naming for case-insensitive ONNX input matching +_K_CAM = "batch_data_cam_front_left" +_K_SPEED = "batch_data_meta_vehiclemotion_speed" +_K_GAS = "batch_data_meta_vehiclemotion_gas_pedal_normalized" +_K_BRAKE = "batch_data_meta_vehiclemotion_brake_pedal_normalized" +_K_STEER = "batch_data_meta_vehiclemotion_steering_angle_normalized" +_K_TURN = "batch_data_meta_vehiclestate_turn_signal" +_K_WP = "batch_data_waypoints_xy_normalized" + +# Nested dict keys for the PyTorch model (under the "data" key) +_PT_CAM = "cam_front_left" +_PT_SPEED = "meta/VehicleMotion/speed" +_PT_GAS = "meta/VehicleMotion/gas_pedal_normalized" +_PT_BRAKE = "meta/VehicleMotion/brake_pedal_normalized" +_PT_STEER = "meta/VehicleMotion/steering_angle_normalized" +_PT_TURN = "meta/VehicleState/turn_signal" +_PT_WP = "waypoints/xy_normalized" + + +# ── Lightweight data containers ─────────────────────────────────────────────── + + +@dataclass +class Predictions: + gas: float + brake: float + steer: float + turn: int + time_ms: float = 0.0 + + +@dataclass +class GroundTruth: + gas: float + brake: float + steer: float + turn: int + + +@dataclass +class _VehicleState: + timestamp: float + speed: float = 0.0 + gas_pedal: float = 0.0 + brake_pedal: float = 0.0 + steering_angle: float = 0.0 + turn_signal: int = 0 + + +@dataclass +class _GnssPosition: + timestamp: float + latitude: float = 0.0 + longitude: float = 0.0 + heading: float = 0.0 + + +# ── Metadata reader (mirrors drivr's RbyteMetadataReader) ──────────────────── + + +@dataclass +class _MetadataReader: + metadata_path: Path + camera_name: str = "cam_front_left" + _motion: list[_VehicleState] = field(init=False, default_factory=list) + _gnss: list[_GnssPosition] = field(init=False, default_factory=list) + _frame_ts: dict[int, float] = field(init=False, default_factory=dict) + _turn_entries: list[tuple[float, int]] = field(init=False, default_factory=list) + + def load(self) -> None: + from rbyte.io.yaak.metadata.message_iterator import ( # noqa: PLC0415 + YaakMetadataMessageIterator, + ) + + with Path(self.metadata_path).open("rb") as f: + mm = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) + for msg_cls, msg_bytes in YaakMetadataMessageIterator(mm): + msg = msg_cls() + msg.ParseFromString(msg_bytes) + ts = msg.time_stamp.ToMicroseconds() / 1_000_000.0 # ty:ignore[unresolved-attribute] + name = msg_cls.__name__ + if name == "VehicleMotion": + self._motion.append( + _VehicleState( + timestamp=ts, + speed=getattr(msg, "speed", 0.0), + gas_pedal=getattr(msg, "gas_pedal_normalized", 0.0), + brake_pedal=getattr(msg, "brake_pedal_normalized", 0.0), + steering_angle=getattr( + msg, "steering_angle_normalized", 0.0 + ), + ) + ) + elif ( + name == "ImageMetadata" + and getattr(msg, "camera_name", None) == self.camera_name + ): + self._frame_ts[getattr(msg, "frame_idx", 0)] = ts + elif name == "VehicleState": + self._turn_entries.append((ts, int(getattr(msg, "turn_signal", 0)))) + elif name == "Gnss": + self._gnss.append( + _GnssPosition( + timestamp=ts, + latitude=getattr(msg, "latitude", 0.0), + longitude=getattr(msg, "longitude", 0.0), + heading=getattr(msg, "heading", 0.0), + ) + ) + mm.close() + + self._gnss.sort(key=lambda x: x.timestamp) + self._turn_entries.sort(key=operator.itemgetter(0)) + logger.info( + "Metadata loaded", + motion=len(self._motion), + gnss=len(self._gnss), + frames=len(self._frame_ts), + ) + + @staticmethod + def _nearest_ts(entries: Sequence[_VehicleState | _GnssPosition], ts: float) -> int: + times = [e.timestamp for e in entries] + idx = bisect.bisect_left(times, ts) + if idx >= len(times): + return len(times) - 1 + if idx > 0 and abs(times[idx - 1] - ts) < abs(times[idx] - ts): + return idx - 1 + return idx + + def _turn_at(self, ts: float) -> int: + if not self._turn_entries: + return 0 + times = [t for t, _ in self._turn_entries] + idx = bisect.bisect_left(times, ts) + if idx >= len(times): + idx = len(times) - 1 + elif idx > 0 and abs(times[idx - 1] - ts) < abs(times[idx] - ts): + idx -= 1 + return self._turn_entries[idx][1] + + def _state_at(self, ts: float) -> _VehicleState: + if not self._motion: + return _VehicleState(timestamp=0.0) + idx = self._nearest_ts(self._motion, ts) + e = self._motion[idx] + return _VehicleState( + timestamp=e.timestamp, + speed=e.speed, + gas_pedal=e.gas_pedal, + brake_pedal=e.brake_pedal, + steering_angle=e.steering_angle, + turn_signal=self._turn_at(e.timestamp), + ) + + def _gnss_at(self, ts: float) -> _GnssPosition: + if not self._gnss: + return _GnssPosition(timestamp=0.0) + return self._gnss[self._nearest_ts(self._gnss, ts)] + + def _frame_lookup(self, frame_idx: int) -> float: + if frame_idx in self._frame_ts: + return self._frame_ts[frame_idx] + if self._frame_ts: + closest = min(self._frame_ts, key=lambda f: abs(f - frame_idx)) + return self._frame_ts[closest] + return 0.0 + + def get_state_for_frame(self, frame_idx: int) -> _VehicleState: + return self._state_at(self._frame_lookup(frame_idx)) + + def get_gnss_for_frame(self, frame_idx: int) -> _GnssPosition: + return self._gnss_at(self._frame_lookup(frame_idx)) + + +# ── Waypoint loader (mirrors drivr's WaypointLoader) ───────────────────────── + + +@dataclass +class _WaypointLoader: + waypoints_path: Path + _wps: list[dict] = field(init=False, default_factory=list) + _times: list[float] = field(init=False, default_factory=list) + + def load(self) -> None: + from pyproj import Transformer # noqa: PLC0415 + + t = Transformer.from_crs("EPSG:4326", "EPSG:25832", always_xy=True) + with Path(self.waypoints_path).open(encoding="utf-8") as f: + data = json.load(f) + for feat in data.get("features", []): + geom = feat.get("geometry", {}) + props = feat.get("properties", {}) + if geom.get("type") == "Point": + lon, lat = geom["coordinates"][:2] + x, y = t.transform(lon, lat) + self._wps.append({ + "timestamp": props.get("timestamp", 0.0), + "heading": props.get("heading", 0.0), + "lon": lon, + "lat": lat, + "x": x, + "y": y, + }) + self._wps.sort(key=operator.itemgetter("timestamp")) + self._times = [w["timestamp"] for w in self._wps] + logger.info("Waypoints loaded", count=len(self._wps)) + + def get_for_gnss(self, gnss: _GnssPosition, n: int = NUM_WAYPOINTS) -> np.ndarray: + from pyproj import Transformer # noqa: PLC0415 + + if not self._wps: + return np.zeros((n, 2), dtype=np.float32) + + idx = bisect.bisect_left(self._times, gnss.timestamp) + wps = self._wps[idx : idx + n] + if not wps: + wps = self._wps[-n:] + + coords = np.array([[w["x"], w["y"]] for w in wps], dtype=np.float64) + if len(coords) < n: + last = coords[-1] if len(coords) else np.zeros(2) + coords = np.vstack([coords, np.tile(last, (n - len(coords), 1))]) + + t = Transformer.from_crs("EPSG:4326", "EPSG:25832", always_xy=True) + ego_x, ego_y = t.transform(gnss.longitude, gnss.latitude) + coords[:, 0] -= ego_x + coords[:, 1] -= ego_y + + heading_rad = np.radians(gnss.heading) + cos_h, sin_h = np.cos(heading_rad), np.sin(heading_rad) + x_rot = coords[:, 0] * cos_h - coords[:, 1] * sin_h + y_rot = coords[:, 0] * sin_h + coords[:, 1] * cos_h + return np.stack([x_rot, y_rot], axis=1).astype(np.float32) + + +# ── Batch loading ───────────────────────────────────────────────────────────── + + +def _preprocess_image(image: np.ndarray, image_size: tuple[int, int]) -> np.ndarray: + """HWC uint8 → resize → CHW float32 [0, 1]. Mirrors drivr's preprocess_image.""" + from torchvision.transforms import functional as TF # noqa: PLC0415 + + tensor = torch.from_numpy(image).permute(2, 0, 1).float().unsqueeze(0) + resized = TF.resize( + tensor, + list(image_size), + interpolation=TF.InterpolationMode.BILINEAR, + antialias=True, + ) + return (resized / 255.0)[0].numpy() + + +@dataclass +class _BatchRequest: + video_path: Path + metadata: _MetadataReader + waypoints: _WaypointLoader + start_frame: int + frame_step: int + image_size: tuple[int, int] + + +def _read_timestep( + cap: Any, frame_idx: int, request: _BatchRequest +) -> tuple[np.ndarray, _VehicleState, np.ndarray]: + import cv2 # noqa: PLC0415 + + cap.set(cv2.CAP_PROP_POS_FRAMES, frame_idx) + ret, frame_bgr = cap.read() + if not ret: + msg = f"Cannot read frame {frame_idx}" + raise ValueError(msg) + + frame_rgb = cv2.cvtColor(frame_bgr, cv2.COLOR_BGR2RGB) + state = request.metadata.get_state_for_frame(frame_idx) + gnss = request.metadata.get_gnss_for_frame(frame_idx) + image = _preprocess_image(frame_rgb, request.image_size) + wp = request.waypoints.get_for_gnss(gnss) + return image, state, wp + + +def _read_ground_truth(cap: Any, gt_idx: int, metadata: _MetadataReader) -> GroundTruth: + import cv2 # noqa: PLC0415 + + cap.set(cv2.CAP_PROP_POS_FRAMES, gt_idx) + ret, _ = cap.read() + if not ret: + msg = f"Cannot read GT frame {gt_idx}" + raise ValueError(msg) + gt_state = metadata.get_state_for_frame(gt_idx) + return GroundTruth( + gas=gt_state.gas_pedal, + brake=gt_state.brake_pedal, + steer=gt_state.steering_angle, + turn=gt_state.turn_signal, + ) + + +def _load_batch(request: _BatchRequest) -> tuple[dict[str, np.ndarray], GroundTruth]: + import cv2 # noqa: PLC0415 + + cap = cv2.VideoCapture(str(request.video_path)) + if not cap.isOpened(): + msg = f"Cannot open video: {request.video_path}" + raise RuntimeError(msg) + + try: + images, states, wps_list = [], [], [] + + for t in range(NUM_TIMESTEPS): + frame_idx = request.start_frame + t * request.frame_step + image, state, wp = _read_timestep(cap, frame_idx, request) + images.append(image) + states.append(state) + wps_list.append(wp) + + # Ground truth: frame AFTER the episode (matches drivr) + gt_idx = request.start_frame + NUM_TIMESTEPS * request.frame_step + gt = _read_ground_truth(cap, gt_idx, request.metadata) + finally: + cap.release() + + wp_array = np.clip(np.stack(wps_list) / 100.0, -1.0, 1.0) # [T, N, 2], 100m horizon + + batch = { + _K_CAM: np.stack(images)[np.newaxis].astype(np.float32), # [1, T, 3, H, W] + _K_SPEED: np.array([s.speed for s in states], dtype=np.float32).reshape( + 1, -1, 1 + ), + _K_GAS: np.array([s.gas_pedal for s in states], dtype=np.float32).reshape( + 1, -1, 1 + ), + _K_BRAKE: np.array([s.brake_pedal for s in states], dtype=np.float32).reshape( + 1, -1, 1 + ), + _K_STEER: np.array( + [s.steering_angle for s in states], dtype=np.float32 + ).reshape(1, -1, 1), + _K_TURN: np.array([s.turn_signal for s in states], dtype=np.int32).reshape( + 1, -1, 1 + ), + _K_WP: wp_array[np.newaxis].astype(np.float32), # [1, T, N, 2] + } + return batch, gt + + +# ── Inference backends ──────────────────────────────────────────────────────── + + +class _ONNXBackend: + def __init__(self, model_path: Path) -> None: + import onnxruntime as ort # noqa: PLC0415 + + providers = ( + ["CUDAExecutionProvider", "CPUExecutionProvider"] + if torch.cuda.is_available() + else ["CPUExecutionProvider"] + ) + self.session = ort.InferenceSession(str(model_path), providers=providers) + self.input_names = {inp.name for inp in self.session.get_inputs()} + self.output_map: dict[str, int] = { + o.name: i for i, o in enumerate(self.session.get_outputs()) + } + self.image_size = self._read_image_size() + logger.info( + "ONNX model loaded", + path=str(model_path), + image_size=self.image_size, + providers=providers, + ) + + def _read_image_size(self) -> tuple[int, int]: + for inp in self.session.get_inputs(): + if ( + "cam_front_left" in inp.name.lower() + and len(inp.shape or []) >= _ONNX_IMAGE_INPUT_NDIM + ): + h, w = inp.shape[3], inp.shape[4] + if isinstance(h, int) and isinstance(w, int) and h > 0 and w > 0: + return (h, w) + return DEFAULT_IMAGE_SIZE + + def run( + self, batch: dict[str, np.ndarray], cache: np.ndarray | None + ) -> tuple[Predictions, np.ndarray | None]: + inputs = dict(batch) + inputs["cached_projected_embeddings"] = ( + cache + if cache is not None + else np.zeros((1, 0, EMBED_DIM), dtype=np.float32) + ) + # ONNX input names are the short canonical names (e.g. "cam_front_left"), + # while batch keys carry a "batch_data_..." prefix — match by suffix, + # case-insensitively (identical to drivr's ONNXModel.run). + matched = { + n: v + for n in self.input_names + for k, v in inputs.items() + if k.lower().endswith(n.lower()) + } + + if torch.cuda.is_available(): + torch.cuda.synchronize() + t0 = time.perf_counter() + outputs = self.session.run(None, matched) + if torch.cuda.is_available(): + torch.cuda.synchronize() + elapsed_ms = (time.perf_counter() - t0) * 1000 + + def _get(key: str, fallback_idx: int) -> np.ndarray: + return cast("np.ndarray", outputs[self.output_map.get(key, fallback_idx)]) + + preds = Predictions( + gas=float(_get("policy.continuous.gas_pedal", 0).squeeze()), + brake=float(_get("policy.continuous.brake_pedal", 1).squeeze()), + steer=float(_get("policy.continuous.steering_angle", 2).squeeze()), + turn=int(_get("policy.discrete.turn_signal", 3).squeeze()), + time_ms=elapsed_ms, + ) + new_cache = ( + cast("np.ndarray", outputs[self.output_map["cached_projected_embeddings"]]) + if "cached_projected_embeddings" in self.output_map + else None + ) + return preds, new_cache + + +class _WandbBackend: + def __init__( + self, + artifact: str, + *, + hparams_jq: str | None = None, + strict: bool | None = None, + ) -> None: + from rmind.models.control_transformer import ControlTransformer # noqa: PLC0415 + + # map_location="cpu": we .to(self.device) right below anyway, and it avoids + # torch.load trying to restore the checkpoint's original CUDA tensors when + # hparams_jq is set (that path doesn't default map_location like Lightning's + # own load_from_checkpoint does), which errors out on a CPU-only machine. + kwargs: dict[str, Any] = {"map_location": "cpu"} + if hparams_jq is not None: + kwargs["hparams_jq"] = hparams_jq + if strict is not None: + kwargs["strict"] = strict + self.model = ControlTransformer.load_from_wandb_artifact( + artifact, **kwargs + ).eval() + self.device = "cuda" if torch.cuda.is_available() else "cpu" + self.model = self.model.to(self.device) + self.image_size = DEFAULT_IMAGE_SIZE + # input_transform is stripped when hparams_jq is set (matching an ONNX export + # config), so the model then expects raw CHW input straight through, same as + # the ONNX backend — see the permute skip in run() below. + self.transform_stripped = hparams_jq is not None + logger.info( + "Wandb model loaded", + artifact=artifact, + device=self.device, + transform_stripped=self.transform_stripped, + ) + + def run(self, onnx_batch: dict[str, np.ndarray]) -> Predictions: + dev = self.device + + def _t(arr: np.ndarray) -> torch.Tensor: + return torch.from_numpy(arr).to(dev) + + cam = _t(onnx_batch[_K_CAM]) + if not self.transform_stripped: + # The model's image encoder starts with Rearrange('... h w c -> ... c h w'), + # so it expects HWC input [B, T, H, W, C]. Our batch has CHW [B, T, C, H, W]. + cam = cam.permute(0, 1, 3, 4, 2) # [B, T, H, W, C] + + # Reconstruct the nested {"data": {...}} batch that ControlTransformer.forward expects + batch: dict = { + "data": { + _PT_CAM: cam, + _PT_SPEED: _t(onnx_batch[_K_SPEED]), + _PT_GAS: _t(onnx_batch[_K_GAS]), + _PT_BRAKE: _t(onnx_batch[_K_BRAKE]), + _PT_STEER: _t(onnx_batch[_K_STEER]), + _PT_TURN: _t(onnx_batch[_K_TURN]), + _PT_WP: _t(onnx_batch[_K_WP]), + } + } + + if dev == "cuda": + torch.cuda.synchronize() + t0 = time.perf_counter() + with torch.inference_mode(): + out = self.model(batch) + if dev == "cuda": + torch.cuda.synchronize() + elapsed_ms = (time.perf_counter() - t0) * 1000 + + policy = out["policy"] + return Predictions( + gas=float(policy["continuous"]["gas_pedal"].squeeze()), + brake=float(policy["continuous"]["brake_pedal"].squeeze()), + steer=float(policy["continuous"]["steering_angle"].squeeze()), + turn=int(policy["discrete"]["turn_signal"].squeeze()), + time_ms=elapsed_ms, + ) + + +# ── Output helpers ──────────────────────────────────────────────────────────── + + +def _print_timing_table( + backends: dict, all_preds: dict[str, list[Predictions]], n_episodes: int +) -> None: + from tabulate import tabulate # noqa: PLC0415 + + rows = [] + has_cpu_star = False + for label, backend in backends.items(): + times = np.array([p.time_ms for p in all_preds[label]]) + if isinstance(backend, _ONNXBackend): + providers = backend.session.get_providers() + on_gpu = "CUDAExecutionProvider" in providers + device = "CUDA" if on_gpu else "CPU*" + if not on_gpu: + has_cpu_star = True + else: + device = backend.device.upper() + rows.append([ + label, + device, + "full", + n_episodes, + f"{times.mean():.1f}", + f"{times.min():.1f}", + f"{times.max():.1f}", + f"{times.std():.1f}", + "✓" if times.mean() < _TARGET_LATENCY_MS else "✗", + ]) + + sep = "=" * 100 + print(f"\n{sep}") # noqa: T201 + print(f"TIMING RESULTS (GPU) - {n_episodes} episodes") # noqa: T201 + print(sep) # noqa: T201 + print( # noqa: T201 + tabulate( + rows, + headers=[ + "Model", + "Device", + "Type", + "Episodes", + "Mean (ms)", + "Min (ms)", + "Max (ms)", + "Std (ms)", + "10 Hz OK?", + ], + tablefmt="grid", + colalign=( + "left", + "left", + "left", + "right", + "right", + "right", + "right", + "right", + "right", + ), + ) + ) + if has_cpu_star: + print(" * ONNX runs on CPU (no CUDAExecutionProvider). GPU not available.") # noqa: T201 + + +def _print_error_table( + backends: dict, all_preds: dict[str, list[Predictions]], all_gt: list[GroundTruth] +) -> None: + from tabulate import tabulate # noqa: PLC0415 + + rows = [] + for label in backends: + preds = all_preds[label] + gas_e = [abs(p.gas - g.gas) for p, g in zip(preds, all_gt, strict=False)] + brake_e = [abs(p.brake - g.brake) for p, g in zip(preds, all_gt, strict=False)] + steer_e = [abs(p.steer - g.steer) for p, g in zip(preds, all_gt, strict=False)] + turn_match = ( + np.mean([p.turn == g.turn for p, g in zip(preds, all_gt, strict=False)]) + * 100 + ) + rows.append([ + label, + f"{np.mean(gas_e):.6f}", + f"{np.mean(brake_e):.6f}", + f"{np.mean(steer_e):.6f}", + f"{np.max(gas_e):.6f}", + f"{np.max(brake_e):.6f}", + f"{np.max(steer_e):.6f}", + f"{turn_match:.1f}%", + ]) + + print("\nERROR VS GROUND TRUTH:") # noqa: T201 + print( # noqa: T201 + tabulate( + rows, + headers=[ + "Model", + "Gas MAE", + "Brake MAE", + "Steer MAE", + "Gas Max", + "Brake Max", + "Steer Max", + "Turn Match %", + ], + tablefmt="grid", + ) + ) + + +def _print_validation(backends: dict, all_preds: dict[str, list[Predictions]]) -> None: + labels = list(backends) + sep = "=" * 100 + print("\nVALIDATION CHECKS") # noqa: T201 + print(sep) # noqa: T201 + if len(labels) < _MIN_BACKENDS_FOR_COMPARISON: + print(" Only one backend — no cross-model comparison.") # noqa: T201 + return + + for i in range(len(labels)): + for j in range(i + 1, len(labels)): + la, lb = labels[i], labels[j] + pa, pb = all_preds[la], all_preds[lb] + gas_max = max(abs(a.gas - b.gas) for a, b in zip(pa, pb, strict=False)) + brake_max = max( + abs(a.brake - b.brake) for a, b in zip(pa, pb, strict=False) + ) + steer_max = max( + abs(a.steer - b.steer) for a, b in zip(pa, pb, strict=False) + ) + ok = ( + gas_max <= _VALIDATION_TOLERANCE + and brake_max <= _VALIDATION_TOLERANCE + and steer_max <= _VALIDATION_TOLERANCE + ) + icon = "✓" if ok else "⚠" + verb = "agree within" if ok else "diffs exceed" + print( # noqa: T201 + f"{icon} {la} vs {lb} {verb} tolerance " + f"(max: gas={gas_max:.8f}, brake={brake_max:.8f}, steer={steer_max:.8f})" + ) + + +def _print_per_episode( + backends: dict, rows: list[dict], all_gt: list[GroundTruth] +) -> None: + from tabulate import tabulate # noqa: PLC0415 + + labels = list(backends) + headers = ( + ["Ep", "Field"] + [f"T={i}" for i in range(NUM_TIMESTEPS)] + ["GT"] + labels + ) + sep = "=" * 150 + print("\nPER-EPISODE PREDICTIONS") # noqa: T201 + print(sep) # noqa: T201 + + for row, gt in zip(rows, all_gt, strict=False): + ep = row["episode"] + 1 + table_rows = [] + for field_key, field_label, gt_val in [ + ("brake", "brake", gt.brake), + ("gas", "gas", gt.gas), + ("steer", "steer", gt.steer), + ]: + hist = row[f"history_{field_key}"] + t_vals = [f"{v:.6f}" for v in hist] + pred_vals = [f"{row[f'{lbl}_{field_key}']:.6f}" for lbl in labels] + table_rows.append([ep, field_label, *t_vals, f"{gt_val:.6f}", *pred_vals]) + print(tabulate(table_rows, headers=headers, tablefmt="grid")) # noqa: T201 + + +def _print_summary_footer( + backends: dict, all_preds: dict[str, list[Predictions]], all_gt: list[GroundTruth] +) -> None: + sep = "=" * 100 + print(f"\n{sep}") # noqa: T201 + print("SUMMARY") # noqa: T201 + print(sep) # noqa: T201 + for label in backends: + times = np.array([p.time_ms for p in all_preds[label]]) + hz = 1000.0 / times.mean() if times.mean() > 0 else 0.0 + print(f"{label}: {times.mean():.1f} ms ({hz:.1f} Hz)") # noqa: T201 + + all_gas_max = max( + max(abs(p.gas - g.gas) for p, g in zip(all_preds[lbl], all_gt, strict=False)) + for lbl in backends + ) + all_steer_max = max( + max( + abs(p.steer - g.steer) for p, g in zip(all_preds[lbl], all_gt, strict=False) + ) + for lbl in backends + ) + print(f"Max error vs GT: Gas={all_gas_max:.6f}, Steer={all_steer_max:.6f}") # noqa: T201 + + labels = list(backends) + if len(labels) >= _MIN_BACKENDS_FOR_COMPARISON: + la, lb = labels[0], labels[1] + pa, pb = all_preds[la], all_preds[lb] + max(abs(a.gas - b.gas) for a, b in zip(pa, pb, strict=False)) + max(abs(a.brake - b.brake) for a, b in zip(pa, pb, strict=False)) + max(abs(a.steer - b.steer) for a, b in zip(pa, pb, strict=False)) + + +# ── Main ────────────────────────────────────────────────────────────────────── + + +class Config(BaseModel): + model_config: ClassVar[ConfigDict] = ConfigDict(extra="ignore") + + data_dir: Path + start_frame: int = 0 + num_episodes: int = 10 + frame_step: int = 10 + onnx: Sequence[Path] | None = None + # not `model` — that name collides with Hydra's config/model/ group, which makes + # `model=...` a defaults-list override attempt instead of a plain value override. + wandb_model: Sequence[str] | None = None + image_size: tuple[int, int] | None = None + output: Path | None = None + warmup: int = 1 + + @field_validator("onnx", "wandb_model", mode="before") + @classmethod + def _coerce_to_list(cls, v: Any) -> Any: + # lets a single `onnx=path`/`wandb_model=artifact` override stand in for a list + return v if v is None or isinstance(v, list | tuple) else [v] + + +def _validate_config(config: Config) -> None: + if not config.onnx and not config.wandb_model: + msg = "At least one of onnx= or wandb_model= is required" + raise ValueError(msg) + + for fname in ("cam_front_left.pii.mp4", "metadata.log", "waypoints.json"): + if not (config.data_dir / fname).exists(): + msg = f"Missing: {config.data_dir / fname}" + raise FileNotFoundError(msg) + + +def _build_backends( + config: Config, *, hparams_jq: str | None = None, strict: bool | None = None +) -> tuple[dict[str, _ONNXBackend | _WandbBackend], tuple[int, int]]: + # ordered dict: label → backend + backends: dict[str, _ONNXBackend | _WandbBackend] = {} + image_size: tuple[int, int] = DEFAULT_IMAGE_SIZE + + onnx_paths = config.onnx or [] + for idx, path in enumerate(onnx_paths): + label = "ONNX Full" if len(onnx_paths) == 1 else f"ONNX Full {idx + 1}" + backend = _ONNXBackend(path) + backends[label] = backend + image_size = backend.image_size # last ONNX wins if multiple + + wandb_artifacts = config.wandb_model or [] + for idx, artifact in enumerate(wandb_artifacts): + label = ( + "PyTorch Native" + if len(wandb_artifacts) == 1 + else f"PyTorch Native {idx + 1}" + ) + backends[label] = _WandbBackend(artifact, hparams_jq=hparams_jq, strict=strict) + + if config.image_size: + image_size = config.image_size + + logger.info("Backends", labels=list(backends), image_size=image_size) + return backends, image_size + + +def _warmup_backends( + onnx_backends: dict[str, _ONNXBackend], request: _BatchRequest, num_warmup: int +) -> None: + if not onnx_backends or num_warmup <= 0: + return + + logger.info("Warming up", runs=num_warmup) + try: + wb, _ = _load_batch(request) + for _ in range(num_warmup): + for backend in onnx_backends.values(): + backend.run(wb, None) + except Exception as e: # noqa: BLE001 — best-effort warmup, must not abort the run + logger.warning("Warmup failed", error=str(e)) + + +def _run_benchmark( + backends: dict[str, _ONNXBackend | _WandbBackend], + request: _BatchRequest, + num_episodes: int, +) -> tuple[dict[str, list[Predictions]], list[GroundTruth], list[dict]]: + all_preds: dict[str, list[Predictions]] = {label: [] for label in backends} + all_gt: list[GroundTruth] = [] + rows: list[dict] = [] + + for i in range(num_episodes): + ep_request = replace( + request, start_frame=request.start_frame + i * request.frame_step + ) + try: + batch, gt = _load_batch(ep_request) + except (ValueError, RuntimeError) as e: + logger.warning("Skipping episode", episode=i, error=str(e)) + break + + row: dict = { + "episode": i, + "frame": ep_request.start_frame, + "history_gas": batch[_K_GAS][0, :, 0].tolist(), + "history_brake": batch[_K_BRAKE][0, :, 0].tolist(), + "history_steer": batch[_K_STEER][0, :, 0].tolist(), + } + + for label, backend in backends.items(): + if isinstance(backend, _ONNXBackend): + # Always pass None (zeros) — matches drivr's full-forward behavior + preds, _ = backend.run(batch, None) + else: + preds = backend.run(batch) # type: ignore[assignment] + + all_preds[label].append(preds) + row[f"{label}_gas"] = round(preds.gas, 6) + row[f"{label}_brake"] = round(preds.brake, 6) + row[f"{label}_steer"] = round(preds.steer, 6) + row[f"{label}_turn"] = preds.turn + row[f"{label}_time_ms"] = round(preds.time_ms, 3) + + all_gt.append(gt) + rows.append(row) + + return all_preds, all_gt, rows + + +def _resolve_export_hparams(cfg: DictConfig) -> tuple[str | None, bool | None]: + # `export=yaak/control_transformer/finetuned` (same group export_onnx.py uses) + # injects a top-level `model:` block (it's `@package _global_`) with the exact + # hparams_jq/strict that export applies — select it to load PyTorch with an + # identically-stripped input_transform, for an apples-to-apples ONNX comparison. + export_model_cfg = OmegaConf.select(cfg, "model", default=None) + if export_model_cfg is None: + return None, None + return export_model_cfg.hparams_jq, export_model_cfg.strict + + +@hydra.main(version_base=None) +def main(cfg: DictConfig) -> None: + hparams_jq, strict = _resolve_export_hparams(cfg) + + config = Config(**OmegaConf.to_container(cfg, resolve=True)) # ty:ignore[invalid-argument-type] + _validate_config(config) + + backends, image_size = _build_backends(config, hparams_jq=hparams_jq, strict=strict) + + meta = _MetadataReader(config.data_dir / "metadata.log") + meta.load() + wps = _WaypointLoader(config.data_dir / "waypoints.json") + wps.load() + + request = _BatchRequest( + video_path=config.data_dir / "cam_front_left.pii.mp4", + metadata=meta, + waypoints=wps, + start_frame=config.start_frame, + frame_step=config.frame_step, + image_size=image_size, + ) + + onnx_backends = { + label: b for label, b in backends.items() if isinstance(b, _ONNXBackend) + } + _warmup_backends(onnx_backends, request, config.warmup) + + logger.info("=" * 70) + logger.info( + "Benchmark: %d episodes, start_frame=%d, frame_step=%d", + config.num_episodes, + config.start_frame, + config.frame_step, + ) + logger.info("=" * 70) + + all_preds, all_gt, rows = _run_benchmark(backends, request, config.num_episodes) + + n = len(all_gt) + if n == 0: + logger.error("No episodes completed") + return + + _print_timing_table(backends, all_preds, n) + _print_error_table(backends, all_preds, all_gt) + _print_validation(backends, all_preds) + _print_per_episode(backends, rows, all_gt) + _print_summary_footer(backends, all_preds, all_gt) + + if config.output and rows: + config.output.parent.mkdir(parents=True, exist_ok=True) + fieldnames = list(rows[0].keys()) + with Path(config.output).open("w", encoding="utf-8", newline="") as f: + w = csv.DictWriter(f, fieldnames=fieldnames) + w.writeheader() + w.writerows(rows) + logger.info("Predictions saved", path=str(config.output)) diff --git a/uv.lock b/uv.lock index 42406cca..858b3d36 100644 --- a/uv.lock +++ b/uv.lock @@ -1537,6 +1537,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b9/ce/2ed92575cc3be4ea1db5f38f16f20765f9b20b69b14d6c1d9972658a8ee9/onnxscript-0.7.0-py3-none-any.whl", hash = "sha256:5b356907d4501e9919f8599c91d8da967406a37b1fac2b40caa55a49acf242ea", size = 714842, upload-time = "2026-04-20T17:09:22.089Z" }, ] +[[package]] +name = "opencv-python-headless" +version = "4.13.0.92" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/79/42/2310883be3b8826ac58c3f2787b9358a2d46923d61f88fedf930bc59c60c/opencv_python_headless-4.13.0.92-cp37-abi3-macosx_13_0_arm64.whl", hash = "sha256:1a7d040ac656c11b8c38677cc8cccdc149f98535089dbe5b081e80a4e5903209", size = 46247192, upload-time = "2026-02-05T07:01:35.187Z" }, + { url = "https://files.pythonhosted.org/packages/2d/1e/6f9e38005a6f7f22af785df42a43139d0e20f169eb5787ce8be37ee7fcc9/opencv_python_headless-4.13.0.92-cp37-abi3-macosx_14_0_x86_64.whl", hash = "sha256:3e0a6f0a37994ec6ce5f59e936be21d5d6384a4556f2d2da9c2f9c5dc948394c", size = 32568914, upload-time = "2026-02-05T07:01:51.989Z" }, + { url = "https://files.pythonhosted.org/packages/21/76/9417a6aef9def70e467a5bf560579f816148a4c658b7d525581b356eda9e/opencv_python_headless-4.13.0.92-cp37-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5c8cfc8e87ed452b5cecb9419473ee5560a989859fe1d10d1ce11ae87b09a2cb", size = 33703709, upload-time = "2026-02-05T10:24:46.469Z" }, + { url = "https://files.pythonhosted.org/packages/92/ce/bd17ff5772938267fd49716e94ca24f616ff4cb1ff4c6be13085108037be/opencv_python_headless-4.13.0.92-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0525a3d2c0b46c611e2130b5fdebc94cf404845d8fa64d2f3a3b679572a5bd22", size = 56016764, upload-time = "2026-02-05T10:26:48.904Z" }, + { url = "https://files.pythonhosted.org/packages/8f/b4/b7bcbf7c874665825a8c8e1097e93ea25d1f1d210a3e20d4451d01da30aa/opencv_python_headless-4.13.0.92-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:eb60e36b237b1ebd40a912da5384b348df8ed534f6f644d8e0b4f103e272ba7d", size = 35010236, upload-time = "2026-02-05T10:28:11.031Z" }, + { url = "https://files.pythonhosted.org/packages/4b/33/b5db29a6c00eb8f50708110d8d453747ca125c8b805bc437b289dbdcc057/opencv_python_headless-4.13.0.92-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:0bd48544f77c68b2941392fcdf9bcd2b9cdf00e98cb8c29b2455d194763cf99e", size = 60391106, upload-time = "2026-02-05T10:30:14.236Z" }, + { url = "https://files.pythonhosted.org/packages/fb/c3/52cfea47cd33e53e8c0fbd6e7c800b457245c1fda7d61660b4ffe9596a7f/opencv_python_headless-4.13.0.92-cp37-abi3-win32.whl", hash = "sha256:a7cf08e5b191f4ebb530791acc0825a7986e0d0dee2a3c491184bd8599848a4b", size = 30812232, upload-time = "2026-02-05T07:02:29.594Z" }, + { url = "https://files.pythonhosted.org/packages/4a/90/b338326131ccb2aaa3c2c85d00f41822c0050139a4bfe723cfd95455bd2d/opencv_python_headless-4.13.0.92-cp37-abi3-win_amd64.whl", hash = "sha256:77a82fe35ddcec0f62c15f2ba8a12ecc2ed4207c17b0902c7a3151ae29f37fb6", size = 40070414, upload-time = "2026-02-05T07:02:26.448Z" }, +] + [[package]] name = "optree" version = "0.19.1" @@ -2007,6 +2025,26 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/10/bd/c038d7cc38edc1aa5bf91ab8068b63d4308c66c4c8bb3cbba7dfbc049f9c/pyparsing-3.3.2-py3-none-any.whl", hash = "sha256:850ba148bd908d7e2411587e247a1e4f0327839c40e2e5e6d05a007ecc69911d", size = 122781, upload-time = "2026-01-21T03:57:55.912Z" }, ] +[[package]] +name = "pyproj" +version = "3.7.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/04/90/67bd7260b4ea9b8b20b4f58afef6c223ecb3abf368eb4ec5bc2cdef81b49/pyproj-3.7.2.tar.gz", hash = "sha256:39a0cf1ecc7e282d1d30f36594ebd55c9fae1fda8a2622cee5d100430628f88c", size = 226279, upload-time = "2025-08-14T12:05:42.18Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8d/ab/9893ea9fb066be70ed9074ae543914a618c131ed8dff2da1e08b3a4df4db/pyproj-3.7.2-cp312-cp312-macosx_13_0_x86_64.whl", hash = "sha256:0a9bb26a6356fb5b033433a6d1b4542158fb71e3c51de49b4c318a1dff3aeaab", size = 6219832, upload-time = "2025-08-14T12:04:10.264Z" }, + { url = "https://files.pythonhosted.org/packages/53/78/4c64199146eed7184eb0e85bedec60a4aa8853b6ffe1ab1f3a8b962e70a0/pyproj-3.7.2-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:567caa03021178861fad27fabde87500ec6d2ee173dd32f3e2d9871e40eebd68", size = 4620650, upload-time = "2025-08-14T12:04:11.978Z" }, + { url = "https://files.pythonhosted.org/packages/b6/ac/14a78d17943898a93ef4f8c6a9d4169911c994e3161e54a7cedeba9d8dde/pyproj-3.7.2-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:c203101d1dc3c038a56cff0447acc515dd29d6e14811406ac539c21eed422b2a", size = 9667087, upload-time = "2025-08-14T12:04:13.964Z" }, + { url = "https://files.pythonhosted.org/packages/b8/be/212882c450bba74fc8d7d35cbd57e4af84792f0a56194819d98106b075af/pyproj-3.7.2-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:1edc34266c0c23ced85f95a1ee8b47c9035eae6aca5b6b340327250e8e281630", size = 9552797, upload-time = "2025-08-14T12:04:16.624Z" }, + { url = "https://files.pythonhosted.org/packages/ba/c0/c0f25c87b5d2a8686341c53c1792a222a480d6c9caf60311fec12c99ec26/pyproj-3.7.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:aa9f26c21bc0e2dc3d224cb1eb4020cf23e76af179a7c66fea49b828611e4260", size = 10837036, upload-time = "2025-08-14T12:04:18.733Z" }, + { url = "https://files.pythonhosted.org/packages/5d/37/5cbd6772addde2090c91113332623a86e8c7d583eccb2ad02ea634c4a89f/pyproj-3.7.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f9428b318530625cb389b9ddc9c51251e172808a4af79b82809376daaeabe5e9", size = 10775952, upload-time = "2025-08-14T12:04:20.709Z" }, + { url = "https://files.pythonhosted.org/packages/69/a1/dc250e3cf83eb4b3b9a2cf86fdb5e25288bd40037ae449695550f9e96b2f/pyproj-3.7.2-cp312-cp312-win32.whl", hash = "sha256:b3d99ed57d319da042f175f4554fc7038aa4bcecc4ac89e217e350346b742c9d", size = 5898872, upload-time = "2025-08-14T12:04:22.485Z" }, + { url = "https://files.pythonhosted.org/packages/4a/a6/6fe724b72b70f2b00152d77282e14964d60ab092ec225e67c196c9b463e5/pyproj-3.7.2-cp312-cp312-win_amd64.whl", hash = "sha256:11614a054cd86a2ed968a657d00987a86eeb91fdcbd9ad3310478685dc14a128", size = 6312176, upload-time = "2025-08-14T12:04:24.736Z" }, + { url = "https://files.pythonhosted.org/packages/5d/68/915cc32c02a91e76d02c8f55d5a138d6ef9e47a0d96d259df98f4842e558/pyproj-3.7.2-cp312-cp312-win_arm64.whl", hash = "sha256:509a146d1398bafe4f53273398c3bb0b4732535065fa995270e52a9d3676bca3", size = 6233452, upload-time = "2025-08-14T12:04:27.287Z" }, +] + [[package]] name = "pytest" version = "9.1.1" @@ -2322,6 +2360,13 @@ train = [ ] [package.dev-dependencies] +benchmark = [ + { name = "onnxruntime" }, + { name = "opencv-python-headless" }, + { name = "pyproj" }, + { name = "tabulate" }, + { name = "wandb" }, +] check = [ { name = "ruff" }, { name = "ty" }, @@ -2381,6 +2426,13 @@ requires-dist = [ provides-extras = ["export", "predict", "train"] [package.metadata.requires-dev] +benchmark = [ + { name = "onnxruntime", specifier = ">=1.23.1" }, + { name = "opencv-python-headless" }, + { name = "pyproj", specifier = ">=3.6.0" }, + { name = "tabulate", specifier = ">=0.9.0" }, + { name = "wandb", specifier = ">=0.24.2" }, +] check = [ { name = "ruff", specifier = ">=0.15.18" }, { name = "ty", specifier = ">=0.0.52" }, @@ -2558,6 +2610,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl", hash = "sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5", size = 6299353, upload-time = "2025-04-27T18:04:59.103Z" }, ] +[[package]] +name = "tabulate" +version = "0.10.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/46/58/8c37dea7bbf769b20d58e7ace7e5edfe65b849442b00ffcdd56be88697c6/tabulate-0.10.0.tar.gz", hash = "sha256:e2cfde8f79420f6deeffdeda9aaec3b6bc5abce947655d17ac662b126e48a60d", size = 91754, upload-time = "2026-03-04T18:55:34.402Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl", hash = "sha256:f0b0622e567335c8fabaaa659f1b33bcb6ddfe2e496071b743aa113f8774f2d3", size = 39814, upload-time = "2026-03-04T18:55:31.284Z" }, +] + [[package]] name = "tensordict" version = "0.13.0"