Decouple hardware components into standalone services (#1413)

.gitignore

@@ -57,3 +57,6 @@ temp*
 src/panoptes/pocs/_version.py
 json_store
 .junie
+
+# AI Agent Plans
+plans/

AGENTS.md

@@ -208,10 +208,10 @@ manage configuration. The server provides centralized configuration access acros
 
 ```bash
 # For normal development
-panoptes-config-server --host 0.0.0.0 --port 6563 run --config-file conf_files/pocs.yaml
+panoptes-utils config run --host 0.0.0.0 --port 6563 --config-file conf_files/pocs.yaml
 
 # For testing (use testing config)
-panoptes-config-server --host 0.0.0.0 --port 6563 run --config-file tests/testing.yaml
+panoptes-utils config run --host 0.0.0.0 --port 6563 --config-file tests/testing.yaml
 ```
 
 **Notes:**
@@ -677,10 +677,10 @@ When making changes, update:
 
 ```bash
 # Start config server (required before running POCS)
-panoptes-config-server --host 0.0.0.0 --port 6563 run --config-file conf_files/pocs.yaml
+panoptes-utils config run --host 0.0.0.0 --port 6563 --config-file conf_files/pocs.yaml
 
 # Start config server for testing
-panoptes-config-server --host 0.0.0.0 --port 6563 run --config-file tests/testing.yaml
+panoptes-utils config run --host 0.0.0.0 --port 6563 --config-file tests/testing.yaml
 
 # Install dependencies
 uv sync

CHANGELOG.md

@@ -10,6 +10,7 @@
 
 ### Added
 
+- Added ability to decouple POCS hardware (Mount, Camera, Scheduler) into standalone FastAPI services. Hardware objects can now run as remote servers using `panoptes.pocs.services.*_api` and be accessed by the Observatory using remote proxies (`panoptes.pocs.mount.remote`, `panoptes.pocs.camera.remote`, `panoptes.pocs.scheduler.remote`) by specifying their endpoints in `pocs.yaml`.
 - Calibration frame commands for taking bias frames and flat fields
     - `pocs camera take-bias`: Takes bias frames (zero exposure), stacks them, and reports statistics (default: 10 frames)
     - `pocs run take-flats`: Takes flat field images with automatic exposure adjustment and mount positioning
@@ -19,6 +20,9 @@
 
 ### Changed
 
+- Updated `conf_files/pocs-supervisord.conf`, `GEMINI.md`, `AGENTS.md`, and `README.md` to use the new `panoptes-utils config` subcommand instead of the removed `panoptes-config-server`.
+- Updated `conftest.py` to use the default POCS config server port `6563` (previously `8765`) for test setup.
+- Upgraded `panoptes-utils` dependency to `0.3.2`.
 - Upgraded `fastapi` from `<0.106.0` to the latest stable release (v0.135.1). #1408
 - Observation exposure time defaults: when `exptime` is not specified in field files, observations now use the 
   `cameras.defaults.exptime` configuration value instead of a hardcoded 120 seconds. This allows setting 

GEMINI.md

@@ -14,7 +14,7 @@ Gemini CLI must rigorously adhere to the standards, workflows, and architectural
 - **Testing:** Every change MUST include corresponding `pytest` tests. Maintain high coverage.
 - **Package Management:** Use `uv` for all dependency and environment management.
 - **Logging:** Use `loguru` via `self.logger` (from `PanBase`) or direct import for standalone utilities.
-- **Configuration:** The `panoptes-config-server` MUST be running for POCS or its tests to function correctly.
+- **Configuration:** The `panoptes-utils config` server MUST be running for POCS or its tests to function correctly.
 
 ## Project-Specific Workflow Mandates
 

README.md

@@ -103,9 +103,9 @@ from panoptes.pocs.core import POCS
 
 os.environ['PANDIR'] = '/var/panoptes'
 conf_server = config_server('conf_files/pocs.yaml')
-I 01-20 01:01:10.886 Starting panoptes-config-server with  config_file='conf_files/pocs.yaml'
-S 01-20 01:01:10.926 Config server Loaded 17 top-level items
-I 01-20 01:01:10.928 Config items saved to flask config-server
+I 01-20 01:01:10.886 Starting panoptes-utils config server with config_file='conf_files/pocs.yaml'
+S 01-20 01:01:10.926 Config server loaded 17 top-level items
+I 01-20 01:01:10.928 Config items saved to server state
 I 01-20 01:01:10.934 Starting panoptes config server with localhost:6563
 
 pocs = POCS.from_config(simulators=['all'])

conf_files/pocs-supervisord.conf

@@ -12,7 +12,7 @@ environment=
 priority=1
 user=panoptes
 directory=/home/panoptes
-command=panoptes-config-server --host 0.0.0.0 --port 6563 run --config-file conf_files/pocs.yaml
+command=panoptes-utils config run --host 0.0.0.0 --port 6563 --config-file conf_files/pocs.yaml
 redirect_stderr=true
 stdout_logfile=/home/panoptes/logs/config-server.log
 autorestart=true
@@ -72,3 +72,39 @@ stdout_logfile=/home/panoptes/logs/metadata-uploader.log
 autostart=true
 stopasgroup=true
 killasgroup=true
+
+[program:pocs-mount-service]
+priority=10
+user=panoptes
+directory=/home/panoptes
+command=uvicorn --host 0.0.0.0 --port 8001 panoptes.pocs.services.mount_api:app
+redirect_stderr=true
+stdout_logfile=/home/panoptes/logs/mount-service.log
+autostart=true
+startsecs=5
+stopasgroup=true
+killasgroup=true
+
+[program:pocs-camera-service]
+priority=11
+user=panoptes
+directory=/home/panoptes
+command=uvicorn --host 0.0.0.0 --port 8002 panoptes.pocs.services.camera_api:app
+redirect_stderr=true
+stdout_logfile=/home/panoptes/logs/camera-service.log
+autostart=true
+startsecs=5
+stopasgroup=true
+killasgroup=true
+
+[program:pocs-scheduler-service]
+priority=12
+user=panoptes
+directory=/home/panoptes
+command=uvicorn --host 0.0.0.0 --port 8003 panoptes.pocs.services.scheduler_api:app
+redirect_stderr=true
+stdout_logfile=/home/panoptes/logs/scheduler-service.log
+autostart=true
+startsecs=5
+stopasgroup=true
+killasgroup=true

conftest.py

@@ -56,12 +56,13 @@
 def pytest_configure(config):
     """Set up the testing."""
     from astropy.utils.iers import conf
+
     conf.auto_max_age = None
     logger.info("Setting up the config server.")
     config_file = "tests/testing.yaml"
 
     host = "localhost"
-    port = "8765"
+    port = "6563"
 
     os.environ["PANOPTES_CONFIG_HOST"] = host
     os.environ["PANOPTES_CONFIG_PORT"] = port

pyproject.toml

@@ -36,9 +36,10 @@ dependencies = [
     "click>=8.2.1",
     "fastapi[standard]",
     "GitPython",
+    "httpx",
     "human-readable",
     "numpy>=2",
-    "panoptes-utils[config,images]>=0.2.53",
+    "panoptes-utils[config,images]>=0.3.2",
     "pandas",
     "pick",
     "Pillow>=10.0.1",
@@ -92,6 +93,7 @@ testing = [
     "pytest-loguru",
     "pytest-remotedata>=0.3.1",
     "responses",
+    "respx>=0.22.0",
 ]
 lint = [
     "ruff",

src/panoptes/pocs/camera/__init__.py

@@ -63,7 +63,7 @@ def list_connected_gphoto2_cameras(endpoint: AnyHttpUrl | None = None):
 
 
 def create_cameras_from_config(
-    config=None, cameras=None, auto_primary=True, recreate_existing=False, *args, **kwargs
+    config=None, cameras=None, auto_primary=True, recreate_existing=False, *args, client_mode=True, **kwargs
 ):
     """Create camera object(s) based on the config.
 
@@ -81,6 +81,9 @@ def create_cameras_from_config(
             existing camera with the same `uid` is already assigned. Should currently
             only affect cameras that use the `sdk` (i.g. not DSLRs). Default False
             raises an exception if camera is already assigned.
+        client_mode: If True (default), intercept configs with an `endpoint_url` and
+            return a proxy object (`RemoteCamera`) instead of the physical driver.
+            Set to False in the hardware service API to instantiate the physical device.
         *args (list): Passed to `get_config`.
         **kwargs (dict): Can pass a `cameras` object that overrides the info in
             the configuration file. Can also pass `auto_detect`(bool) to try and
@@ -134,6 +137,11 @@ def create_cameras_from_config(
         device_config.update(cfg)
 
         cam_name = device_config.setdefault("name", f"Cam{cam_num:02d}")
+
+        # Intercept client mode with an endpoint URL
+        if client_mode and "endpoint_url" in device_config:
+            logger.debug(f"Client mode enabled and endpoint_url found, using remote proxy for {cam_name}")
+            device_config["model"] = "remote"
 
         # Check for proper connection method.
         model = device_config["model"]
@@ -151,7 +159,10 @@ def create_cameras_from_config(
         logger.debug(f"Creating camera: {model}")
 
         try:
-            module = load_module(model)
+            module_name = model
+            if model == "remote":
+                module_name = f"panoptes.pocs.camera.{model}"
+            module = load_module(module_name)
             logger.debug(f"Camera module: module={module!r}")
 
             if recreate_existing:

src/panoptes/pocs/camera/camera.py

@@ -627,10 +627,7 @@ def log_thread_error(exc_info):
             name=f"{self.name}PollExposureThread",
             target=self._poll_exposure,
             args=(readout_args, seconds),
-            kwargs=dict(
-                timeout=timeout_duration,
-                interval=get_quantity_value(self.readout_time, u.second)
-            ),
+            kwargs=dict(timeout=timeout_duration, interval=get_quantity_value(self.readout_time, u.second)),
         )
         readout_thread.start()
 
@@ -883,9 +880,9 @@ def _poll_exposure(self, readout_args, exposure_time, timeout=None, interval=0.0
         """
         if timeout is None:
             timer_duration = (
-                get_quantity_value(self.timeout, u.second) +
-                get_quantity_value(self.readout_time, u.second) +
-                get_quantity_value(exposure_time, u.second)
+                get_quantity_value(self.timeout, u.second)
+                + get_quantity_value(self.readout_time, u.second)
+                + get_quantity_value(exposure_time, u.second)
             )
         else:
             timer_duration = get_quantity_value(timeout, u.second)

src/panoptes/pocs/camera/remote.py

@@ -0,0 +1,157 @@
+import threading
+from typing import Any
+from urllib.parse import quote
+
+import httpx
+from panoptes.utils import error
+from panoptes.utils.serializers import from_json
+
+from panoptes.pocs.camera.camera import AbstractCamera
+
+
+class Camera(AbstractCamera):
+    """A proxy Camera class that delegates commands to a remote FastAPI service."""
+
+    def __init__(
+        self,
+        name="Remote Camera",
+        model="remote",
+        port=None,
+        primary=False,
+        endpoint_url="http://localhost:8002",
+        *args,
+        **kwargs,
+    ):
+        super().__init__(name=name, model=model, port=port, primary=primary, *args, **kwargs)
+        self.endpoint_url = endpoint_url
+        self.client = httpx.Client(timeout=300.0)
+        # URL-safe camera name for routing
+        self.safe_name = quote(self.name)
+        self.logger.info(f"Initialized RemoteCamera {name} pointing to {self.endpoint_url}")
+
+    def _get(self, path):
+        try:
+            response = self.client.get(f"{self.endpoint_url}/{self.safe_name}/{path}")
+            response.raise_for_status()
+            data = from_json(response.text)
+            if isinstance(data, dict) and "result" in data:
+                return data["result"]
+            return data
+        except Exception as e:
+            self.logger.error(f"Remote camera GET {path} failed: {e}")
+            return None
+
+    def _post(self, path, json=None, params=None):
+        try:
+            response = self.client.post(
+                f"{self.endpoint_url}/{self.safe_name}/{path}", json=json, params=params
+            )
+            response.raise_for_status()
+            data = from_json(response.text)
+            if isinstance(data, dict) and "result" in data:
+                return data["result"]
+            return data.get("result", True) if isinstance(data, dict) else data
+        except Exception as e:
+            self.logger.error(f"Remote camera POST {path} failed: {e}")
+            return None
+
+    def connect(self):
+        self._connected = self._post("connect")
+        return self._connected
+
+    @property
+    def is_connected(self):
+        status = self._get("status")
+        return status.get("is_connected", False) if status else False
+
+    @property
+    def temperature(self):
+        status = self._get("status")
+        return status.get("temperature") if status else None
+
+    @property
+    def target_temperature(self):
+        status = self._get("status")
+        return status.get("target_temperature") if status else None
+
+    @target_temperature.setter
+    def target_temperature(self, target):
+        self._post("set_target_temperature", params={"target": target})
+
+    @property
+    def cooling_enabled(self):
+        status = self._get("status")
+        return status.get("cooling_enabled", False) if status else False
+
+    @cooling_enabled.setter
+    def cooling_enabled(self, enabled):
+        self._post("set_cooling_enabled", params={"enabled": enabled})
+
+    @property
+    def cooling_power(self):
+        status = self._get("status")
+        return status.get("cooling_power", 0.0) if status else 0.0
+
+    @property
+    def is_exposing(self):
+        status = self._get("status")
+        return status.get("is_exposing", False) if status else False
+
+    @property
+    def is_ready(self):
+        status = self._get("status")
+        return status.get("is_ready", False) if status else False
+
+    def take_exposure(
+        self,
+        seconds=1.0,
+        filename=None,
+        metadata=None,
+        dark=False,
+        blocking=False,
+        timeout=10.0,
+        *args,
+        **kwargs,
+    ):
+        """Proxy exposure logic to remote node."""
+        if not filename:
+            raise error.PanError("Must pass filename for take_exposure")
+
+        seconds_val = seconds.value if hasattr(seconds, "value") else seconds
+        timeout_val = timeout.value if hasattr(timeout, "value") else timeout
+
+        params = {
+            "seconds": seconds_val,
+            "filename": str(filename),
+            "metadata": metadata or {},
+            "dark": dark,
+            "blocking": blocking,
+            "timeout": timeout_val,
+        }
+
+        self._post("take_exposure", json=params)
+
+        # The remote handles its own threading. We can just return a dummy thread
+        # if blocking is false, since AbstractCamera expects a thread.
+        dummy_thread = threading.Thread(target=lambda: None)
+        dummy_thread.start()
+        return dummy_thread
+
+    def process_exposure(self, metadata, **kwargs):
+        """Proxy processing logic to remote node."""
+        self._post("process_exposure", json={"metadata": metadata})
+
+    def _set_target_temperature(self, target):
+        pass
+
+    def _set_cooling_enabled(self, enable):
+        pass
+
+    def _start_exposure(self, seconds=None, filename=None, dark=False, header=None, *args, **kwargs):
+        pass
+
+    def _readout(self, *args, **kwargs):
+        pass
+
+    def _process_fits(self, file_path, metadata):
+        return file_path