From 9d99bfa4e45690610d3653aed3c1fff7bac72f24 Mon Sep 17 00:00:00 2001 From: Billie Ponder Date: Sun, 20 Sep 2026 17:14:21 -0400 Subject: [PATCH] feat(ops): implement production operations infrastructure dashboard (phase 1) (#14) * feat(ops): implement production operations infrastructure dashboard (phase 1) - Add backend read-only infrastructure metrics collector (CPU, RAM, swap, disk with 80%/90% thresholds, uptime, network cumulative totals, systemd service allowlist) using standard library only (no psutil). - Register GET /api/v1/ops/infrastructure endpoint guarded by require_ops_roles('operator', 'owner'). - Add Next.js API proxy route /api/ops/infrastructure validating ops session cookie and forwarding to backend. - Implement dark-surface OperationsInfrastructure component in website admin console with 30s visibility-gated polling and overlap prevention. - Add comprehensive backend and frontend test suites (8 backend tests, 43 ops-experience tests). * fix(ops): query internal website version endpoint defensively on verified port 3001 - Query internal website GET /api/version on verified production listener http://127.0.0.1:3001 (configurable via WEBSITE_INTERNAL_URL). - Parse JSON defensively with a 4096-byte response bound and validate exact 40-char hex revision pattern. - Add comprehensive tests for valid, missing, malformed, oversized, and unreachable version responses. - Do not expose internal URLs or connection errors in API responses. * fix(ops): map postgresql service to cluster unit postgresql@14-main and preserve health errors - Map public logical service ID 'postgresql' to fixed systemd unit 'postgresql@14-main'. - Reject direct systemd unit names and arbitrary service names. - Preserve HTTP 5xx responses as degraded on /api/health before root fallback. - Add regression tests for postgresql unit mapping, API response name stability, and health 5xx preservation. --------- Co-authored-by: Developer --- .../app/infrastructure_metrics.py | 496 ++++++++++++++ backend/motorsignal_api/app/main.py | 16 + .../tests/test_ops_infrastructure.py | 528 +++++++++++++++ website/app/api/ops/infrastructure/route.ts | 65 ++ .../ops/operations-infrastructure.tsx | 618 ++++++++++++++++++ website/components/ops/operations-shell.tsx | 6 + website/components/ops/operations-sidebar.tsx | 5 + website/components/ops/types.ts | 69 ++ website/lib/ops-data.ts | 194 ++++++ website/tests/ops-experience.test.cjs | 198 +++++- 10 files changed, 2194 insertions(+), 1 deletion(-) create mode 100644 backend/motorsignal_api/app/infrastructure_metrics.py create mode 100644 backend/motorsignal_api/tests/test_ops_infrastructure.py create mode 100644 website/app/api/ops/infrastructure/route.ts create mode 100644 website/components/ops/operations-infrastructure.tsx diff --git a/backend/motorsignal_api/app/infrastructure_metrics.py b/backend/motorsignal_api/app/infrastructure_metrics.py new file mode 100644 index 0000000..26d0752 --- /dev/null +++ b/backend/motorsignal_api/app/infrastructure_metrics.py @@ -0,0 +1,496 @@ +""" +MotorSignal Operations Infrastructure Metrics Collector. + +Safe, read-only system resource monitoring, service status, network totals, +and deployment health foundations. +""" + +from dataclasses import dataclass +from datetime import datetime, timezone +import json +import os +import re +import shutil +import subprocess +import time +from typing import Any, Dict, List, Optional, Tuple +from urllib.error import HTTPError, URLError +from urllib.request import Request, urlopen + +REVISION_HEX_REGEX = re.compile(r"^[0-9a-fA-F]{40}$") +MAX_VERSION_RESPONSE_BYTES = 4096 +DEFAULT_WEBSITE_INTERNAL_URL = "http://127.0.0.1:3001" + +# Fixed mapping from public logical service ID to actual systemd unit name +SERVICE_SYSTEMD_UNITS: Dict[str, str] = { + "motorsignal-website": "motorsignal-website", + "motorsignal-api": "motorsignal-api", + "nginx": "nginx", + "postgresql": "postgresql@14-main", +} + +ALLOWED_SYSTEMD_SERVICES: Tuple[str, ...] = tuple(SERVICE_SYSTEMD_UNITS.keys()) + +SERVICE_METADATA: Dict[str, Dict[str, str]] = { + "motorsignal-website": { + "display_name": "MotorSignal Website", + "description": "Next.js frontend application runtime", + }, + "motorsignal-api": { + "display_name": "MotorSignal API", + "description": "FastAPI core backend service & ingress", + }, + "nginx": { + "display_name": "Nginx Reverse Proxy", + "description": "Edge TLS termination & reverse proxy routing", + }, + "postgresql": { + "display_name": "PostgreSQL Database", + "description": "Primary relational database persistence", + }, +} + +# In-memory previous CPU stat cache for non-blocking delta calculation +_prev_cpu_stat: Optional[Tuple[float, int, int]] = None # (timestamp, idle_ticks, total_ticks) + + +@dataclass +class OpsDeploymentEvent: + """Foundational data model for deployment events (Phase 2 readiness).""" + id: str + source_revision: str + initiating_operator_id: str + started_at: datetime + completed_at: Optional[datetime] + status: str # 'pending' | 'in_progress' | 'success' | 'failed' | 'rolled_back' + release_identifier: Optional[str] = None + failure_category: Optional[str] = None + previous_revision: Optional[str] = None + resulting_revision: Optional[str] = None + + +def _read_proc_file(path: str) -> Optional[str]: + """Safely read a /proc file if it exists.""" + try: + if os.path.exists(path): + with open(path, "r", encoding="utf-8") as f: + return f.read() + except Exception: + pass + return None + + +def get_cpu_percent() -> Optional[float]: + """ + Calculate overall CPU percent. + Uses /proc/stat deltas on Linux. + Degrades safely to None if unavailable. + """ + global _prev_cpu_stat + + stat_content = _read_proc_file("/proc/stat") + if stat_content: + for line in stat_content.splitlines(): + if line.startswith("cpu "): + parts = line.split() + if len(parts) >= 5: + try: + # parts: cpu, user, nice, system, idle, iowait, irq, softirq, steal, guest, guest_nice + fields = [int(p) for p in parts[1:]] + idle = fields[3] + (fields[4] if len(fields) > 4 else 0) + total = sum(fields) + now = time.time() + + if _prev_cpu_stat is not None: + prev_time, prev_idle, prev_total = _prev_cpu_stat + delta_time = now - prev_time + delta_total = total - prev_total + delta_idle = idle - prev_idle + + if 0.1 <= delta_time <= 60.0 and delta_total > 0: + _prev_cpu_stat = (now, idle, total) + used_pct = ((delta_total - delta_idle) / delta_total) * 100.0 + return round(max(0.0, min(100.0, used_pct)), 1) + + # Sample quick delta if no previous sample exists + time.sleep(0.05) + stat_content_2 = _read_proc_file("/proc/stat") + if stat_content_2: + for line2 in stat_content_2.splitlines(): + if line2.startswith("cpu "): + fields2 = [int(p) for p in line2.split()[1:]] + idle2 = fields2[3] + (fields2[4] if len(fields2) > 4 else 0) + total2 = sum(fields2) + now2 = time.time() + _prev_cpu_stat = (now2, idle2, total2) + delta_total = total2 - total + delta_idle = idle2 - idle + if delta_total > 0: + used_pct = ((delta_total - delta_idle) / delta_total) * 100.0 + return round(max(0.0, min(100.0, used_pct)), 1) + + _prev_cpu_stat = (now, idle, total) + except Exception: + pass + break + + return None + + +def get_load_averages() -> Optional[Tuple[float, float, float]]: + """Return 1m, 5m, 15m load averages via os.getloadavg().""" + try: + if hasattr(os, "getloadavg"): + avg = os.getloadavg() + return (round(avg[0], 2), round(avg[1], 2), round(avg[2], 2)) + except Exception: + pass + return None + + +def get_memory_metrics() -> Optional[Dict[str, Any]]: + """ + Parse /proc/meminfo for RAM metrics. + Returns total, used, available in bytes, and used percent. + """ + mem_content = _read_proc_file("/proc/meminfo") + if mem_content: + mem_info: Dict[str, int] = {} + for line in mem_content.splitlines(): + parts = line.split(":") + if len(parts) == 2: + key = parts[0].strip() + val_parts = parts[1].strip().split() + if val_parts and val_parts[0].isdigit(): + mem_info[key] = int(val_parts[0]) * 1024 # kB to bytes + + if "MemTotal" in mem_info: + total = mem_info["MemTotal"] + avail = mem_info.get( + "MemAvailable", + mem_info.get("MemFree", 0) + mem_info.get("Buffers", 0) + mem_info.get("Cached", 0) + ) + used = max(0, total - avail) + pct = round((used / total) * 100.0, 1) if total > 0 else 0.0 + return { + "total": total, + "used": used, + "available": avail, + "percent": pct, + } + + return None + + +def get_swap_metrics() -> Optional[Dict[str, Any]]: + """ + Parse /proc/meminfo for Swap metrics. + Returns total, used in bytes, and used percent. + """ + mem_content = _read_proc_file("/proc/meminfo") + if mem_content: + mem_info: Dict[str, int] = {} + for line in mem_content.splitlines(): + parts = line.split(":") + if len(parts) == 2: + key = parts[0].strip() + val_parts = parts[1].strip().split() + if val_parts and val_parts[0].isdigit(): + mem_info[key] = int(val_parts[0]) * 1024 + + if "SwapTotal" in mem_info: + total = mem_info["SwapTotal"] + free = mem_info.get("SwapFree", 0) + used = max(0, total - free) + pct = round((used / total) * 100.0, 1) if total > 0 else 0.0 + return { + "total": total, + "used": used, + "percent": pct, + } + + return None + + +def get_disk_metrics(mount_point: str = "/") -> Optional[Dict[str, Any]]: + """ + Calculate disk usage for the root filesystem. + Includes warning (>= 80%) and critical (>= 90%) threshold flags. + """ + try: + # On Windows during local testing, "/" maps to current drive, but check valid path + path = mount_point + if os.name == "nt" and mount_point == "/": + path = os.environ.get("SystemDrive", "C:") + "\\" + + total, used, free = shutil.disk_usage(path) + pct = round((used / total) * 100.0, 1) if total > 0 else 0.0 + return { + "total": total, + "used": used, + "available": free, + "percent": pct, + "warning": pct >= 80.0, + "critical": pct >= 90.0, + } + except Exception: + return None + + +def get_uptime() -> Tuple[Optional[int], Optional[str]]: + """ + Parse /proc/uptime for server uptime. + Returns (uptime_seconds, uptime_formatted). + """ + uptime_content = _read_proc_file("/proc/uptime") + if uptime_content: + parts = uptime_content.split() + if parts: + try: + seconds = int(float(parts[0])) + days = seconds // 86400 + hours = (seconds % 86400) // 3600 + minutes = (seconds % 3600) // 60 + formatted = f"{days}d {hours}h {minutes}m" + return seconds, formatted + except Exception: + pass + + return None, None + + +def get_system_resources() -> Dict[str, Any]: + """Collect overall server system resource metrics.""" + uptime_sec, uptime_fmt = get_uptime() + return { + "cpu_percent": get_cpu_percent(), + "load_avg": get_load_averages(), + "memory": get_memory_metrics(), + "swap": get_swap_metrics(), + "disk": get_disk_metrics("/"), + "uptime_seconds": uptime_sec, + "uptime_formatted": uptime_fmt, + "collected_at": datetime.now(timezone.utc).isoformat(), + } + + +def get_service_status(service_name: str) -> str: + """ + Query systemctl is-active for an allowed logical service name. + Maps public logical service ID to actual fixed systemd unit (e.g. postgresql -> postgresql@14-main). + Strictly forbids arbitrary service names, direct unit overrides, and shell interpolation. + """ + unit_name = SERVICE_SYSTEMD_UNITS.get(service_name) + if not unit_name: + return "unknown" + + try: + res = subprocess.run( + ["systemctl", "is-active", unit_name], + capture_output=True, + text=True, + timeout=2.0, + shell=False, + ) + out = res.stdout.strip().lower() + if out == "active": + return "active" + if out == "inactive": + return "inactive" + if out == "failed": + return "failed" + return "unknown" + except Exception: + return "unknown" + + +def get_services_health() -> List[Dict[str, Any]]: + """Inspect all allowed core services.""" + results = [] + for svc in ALLOWED_SYSTEMD_SERVICES: + meta = SERVICE_METADATA.get(svc, {}) + status = get_service_status(svc) + results.append({ + "name": svc, + "display_name": meta.get("display_name", svc), + "description": meta.get("description", ""), + "status": status, + }) + return results + + +def get_network_totals() -> Dict[str, Any]: + """ + Parse /proc/net/dev to sum cumulative received and transmitted bytes + across all non-loopback network interfaces since boot. + """ + dev_content = _read_proc_file("/proc/net/dev") + rx_bytes = 0 + tx_bytes = 0 + found_any = False + + if dev_content: + for line in dev_content.splitlines(): + if ":" in line: + parts = line.split(":", 1) + iface = parts[0].strip() + if iface == "lo": + continue + stats = parts[1].split() + if len(stats) >= 9: + try: + rx = int(stats[0]) + tx = int(stats[8]) + rx_bytes += rx + tx_bytes += tx + found_any = True + except Exception: + pass + + return { + "bytes_received": rx_bytes if found_any else None, + "bytes_transmitted": tx_bytes if found_any else None, + "label": "Cumulative Total Since Boot", + "collected_at": datetime.now(timezone.utc).isoformat(), + } + + +def get_website_revision(website_url: Optional[str] = None) -> Optional[str]: + """ + Retrieve deployed website revision from the internal website /api/version endpoint. + Parses JSON defensively with a strict response-size bound. + Accepts revision only if it matches exactly 40 lowercase or uppercase hexadecimal characters. + Never exposes malformed content or connection errors. + Preserves graceful degradation if endpoint is absent, unreachable, malformed, or unmerged. + Retains environment-variable fallback only for deliberately supported API-service configuration. + """ + base_url = website_url or os.environ.get("WEBSITE_INTERNAL_URL", DEFAULT_WEBSITE_INTERNAL_URL) + endpoint = f"{base_url.rstrip('/')}/api/version" + + try: + req = Request( + endpoint, + headers={ + "User-Agent": "MotorSignal-Ops-Collector/1.0", + "Accept": "application/json", + }, + ) + with urlopen(req, timeout=1.5) as resp: + status = getattr(resp, "status", getattr(resp, "code", None)) + if status != 200: + return None + raw_bytes = resp.read(MAX_VERSION_RESPONSE_BYTES + 1) + if len(raw_bytes) > MAX_VERSION_RESPONSE_BYTES: + # Oversized response, reject defensively + return None + try: + text = raw_bytes.decode("utf-8") + data = json.loads(text) + except Exception: + # Malformed JSON or non-utf8, reject defensively + return None + + if isinstance(data, dict): + rev = data.get("revision") + if isinstance(rev, str) and REVISION_HEX_REGEX.match(rev.strip()): + return rev.strip() + except Exception: + # Never expose connection errors or malformed content + pass + + # Deliberately supported API-service configuration fallback + for env_var in ("DEPLOYED_REVISION", "GIT_SHA"): + env_val = os.environ.get(env_var, "").strip() + if env_val and REVISION_HEX_REGEX.match(env_val): + return env_val + + return None + + +def get_deployment_health( + database_ok: bool = True, + website_url: Optional[str] = None, +) -> Dict[str, Any]: + """ + Collect deployment runtime health and version foundations. + """ + target_website_url = website_url or os.environ.get("WEBSITE_INTERNAL_URL", DEFAULT_WEBSITE_INTERNAL_URL) + + # 1. Deployed Git Revision from internal website /api/version + git_rev = get_website_revision(target_website_url) + short_rev = git_rev[:7] if git_rev else None + rev_status = "known" if git_rev else "unavailable" + + # 2. Local Website Health check + website_health = "unknown" + try: + req = Request( + f"{target_website_url.rstrip('/')}/api/health", + headers={"User-Agent": "MotorSignal-Ops-Collector/1.0"}, + ) + with urlopen(req, timeout=1.5) as resp: + status = getattr(resp, "status", getattr(resp, "code", None)) + if status == 200: + website_health = "available" + elif status and 500 <= status < 600: + website_health = "degraded" + else: + website_health = "available" + except HTTPError as e: + if 500 <= e.code < 600: + website_health = "degraded" + elif e.code == 404: + # Route missing: fallback to probing root page + try: + req_root = Request( + target_website_url.rstrip('/'), + headers={"User-Agent": "MotorSignal-Ops-Collector/1.0"}, + ) + with urlopen(req_root, timeout=1.5) as resp_root: + status_root = getattr(resp_root, "status", getattr(resp_root, "code", None)) + if status_root in (200, 301, 302, 307, 308): + website_health = "available" + else: + website_health = "degraded" + except HTTPError as e_root: + if 500 <= e_root.code < 600: + website_health = "degraded" + else: + website_health = "unavailable" + except Exception: + website_health = "unavailable" + else: + website_health = "degraded" + except (URLError, TimeoutError): + website_health = "unavailable" + except Exception: + website_health = "unavailable" + + # If git revision was successfully fetched from website and website_health was unknown/unavailable, + # mark website available (preserve degraded if explicitly 5xx) + if git_rev and website_health in ("unknown", "unavailable"): + website_health = "available" + + # 3. API Health + api_health = "available" if database_ok else "degraded" + + return { + "git_revision": git_rev, + "short_revision": short_rev, + "revision_status": rev_status, + "website_health": website_health, + "api_health": api_health, + "history_status": "unavailable", + "last_successful_deployment": None, + } + + +def get_infrastructure_metrics(database_ok: bool = True) -> Dict[str, Any]: + """Aggregate all read-only infrastructure and deployment metrics.""" + return { + "resources": get_system_resources(), + "services": get_services_health(), + "network": get_network_totals(), + "deployment": get_deployment_health(database_ok=database_ok), + "collected_at": datetime.now(timezone.utc).isoformat(), + } diff --git a/backend/motorsignal_api/app/main.py b/backend/motorsignal_api/app/main.py index c162ea5..7fbff69 100644 --- a/backend/motorsignal_api/app/main.py +++ b/backend/motorsignal_api/app/main.py @@ -46,6 +46,8 @@ from models import ( User, Vehicle, ) +from infrastructure_metrics import get_infrastructure_metrics + logger = logging.getLogger(__name__) @@ -1146,6 +1148,20 @@ def ops_get_health(): } +@app.get("/api/v1/ops/infrastructure") +def ops_get_infrastructure( + auth: tuple[Operator, OpsSession] = Depends(require_ops_roles("operator", "owner")), + db: DbSession = Depends(get_db), +): + db_ok = False + try: + db_ok = check_database() + except Exception: + db_ok = False + + return get_infrastructure_metrics(database_ok=db_ok) + + @app.get("/api/v1/ops/overview") def ops_get_overview( auth: tuple[Operator, OpsSession] = Depends(get_current_operator), diff --git a/backend/motorsignal_api/tests/test_ops_infrastructure.py b/backend/motorsignal_api/tests/test_ops_infrastructure.py new file mode 100644 index 0000000..8a9c0e6 --- /dev/null +++ b/backend/motorsignal_api/tests/test_ops_infrastructure.py @@ -0,0 +1,528 @@ +import hashlib +import os +import sys +import unittest +import uuid +from datetime import datetime, timedelta, timezone +from unittest.mock import patch, MagicMock + +ROOT = os.path.dirname(os.path.dirname(__file__)) +sys.path.insert(0, os.path.join(ROOT, ".python-packages")) +sys.path.insert(0, os.path.join(ROOT, "app")) + +os.environ.setdefault("JWT_SECRET", "local-test-secret") +os.environ["FIREBASE_PROJECT_ID"] = "motoapp-d22e0" +os.environ["DATABASE_USER"] = "test" +os.environ["DATABASE_PASSWORD"] = "test" +os.environ["DATABASE_NAME"] = "test" + +from sqlalchemy import create_engine, event +from sqlalchemy.orm import Session as DbSession +from starlette.testclient import TestClient + +import main +from main import app, get_db +from models import Base, Operator, OpsAuditEvent, OpsSession +import infrastructure_metrics +from infrastructure_metrics import ( + ALLOWED_SYSTEMD_SERVICES, + DEFAULT_WEBSITE_INTERNAL_URL, + MAX_VERSION_RESPONSE_BYTES, + REVISION_HEX_REGEX, + SERVICE_SYSTEMD_UNITS, + get_disk_metrics, + get_service_status, + get_services_health, + get_system_resources, + get_network_totals, + get_website_revision, + get_deployment_health, + get_infrastructure_metrics, +) + + +def hash_token(raw_token: str) -> str: + return hashlib.sha256(raw_token.encode()).hexdigest() + + +class TestOpsInfrastructure(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.db_path = os.path.join(ROOT, "tests", "test_ops_infra.db") + cls.ops_db_path = os.path.join(ROOT, "tests", "test_ops_infra_schema.db") + for p in (cls.db_path, cls.ops_db_path): + if os.path.exists(p): + try: + os.remove(p) + except OSError: + pass + + cls.engine = create_engine(f"sqlite:///{cls.db_path}") + cls.seq_val = 0 + + @event.listens_for(cls.engine, "connect") + def attach_ops(dbapi_con, con_record): + dbapi_con.execute(f"ATTACH DATABASE '{cls.ops_db_path}' AS ops") + + def sqlite_nextval(name): + cls.seq_val += 1 + return cls.seq_val + + dbapi_con.create_function("nextval", 1, sqlite_nextval) + + Base.metadata.create_all(cls.engine) + + def override_get_db(): + db = DbSession(cls.engine) + try: + yield db + finally: + db.close() + + app.dependency_overrides[get_db] = override_get_db + cls.db_patcher = patch("main.check_database", return_value=True) + cls.db_patcher.start() + cls.client = TestClient(app, raise_server_exceptions=False) + + @classmethod + def tearDownClass(cls): + cls.db_patcher.stop() + app.dependency_overrides.clear() + cls.engine.dispose() + for p in (cls.db_path, cls.ops_db_path): + if os.path.exists(p): + try: + os.remove(p) + except OSError: + pass + + + def setUp(self): + with DbSession(self.engine) as db: + db.query(OpsAuditEvent).delete() + db.query(OpsSession).delete() + db.query(Operator).delete() + db.commit() + + # Seed Operator + self.op_id = uuid.uuid4() + self.raw_token = f"ops_test_token_{uuid.uuid4().hex}" + self.operator = Operator( + id=self.op_id, + identity="operator@motorsignal.test", + role="operator", + status="active", + created_at=datetime.now(timezone.utc), + ) + self.session = OpsSession( + id=uuid.uuid4(), + operator_id=self.op_id, + session_token_hash=hash_token(self.raw_token), + created_at=datetime.now(timezone.utc), + last_activity_at=datetime.now(timezone.utc), + expires_at=datetime.now(timezone.utc) + timedelta(hours=1), + ) + + # Seed Owner + self.owner_id = uuid.uuid4() + self.raw_owner_token = f"ops_owner_token_{uuid.uuid4().hex}" + self.owner = Operator( + id=self.owner_id, + identity="owner@motorsignal.test", + role="owner", + status="active", + created_at=datetime.now(timezone.utc), + ) + self.owner_session = OpsSession( + id=uuid.uuid4(), + operator_id=self.owner_id, + session_token_hash=hash_token(self.raw_owner_token), + created_at=datetime.now(timezone.utc), + last_activity_at=datetime.now(timezone.utc), + expires_at=datetime.now(timezone.utc) + timedelta(hours=1), + ) + + # Seed Expired Session + self.expired_token = "ops_expired_token" + self.expired_session = OpsSession( + id=uuid.uuid4(), + operator_id=self.op_id, + session_token_hash=hash_token(self.expired_token), + created_at=datetime.now(timezone.utc) - timedelta(hours=3), + last_activity_at=datetime.now(timezone.utc) - timedelta(hours=2), + expires_at=datetime.now(timezone.utc) - timedelta(hours=1), + ) + + db.add_all([ + self.operator, self.session, + self.owner, self.owner_session, + self.expired_session + ]) + db.commit() + + def test_unauthenticated_access_returns_401(self): + resp = self.client.get("/api/v1/ops/infrastructure") + self.assertEqual(resp.status_code, 401) + + def test_invalid_token_returns_401(self): + resp = self.client.get( + "/api/v1/ops/infrastructure", + headers={"Authorization": "Bearer invalid-token-xyz"}, + ) + self.assertEqual(resp.status_code, 401) + + def test_expired_token_returns_401(self): + resp = self.client.get( + "/api/v1/ops/infrastructure", + headers={"Authorization": f"Bearer {self.expired_token}"}, + ) + self.assertEqual(resp.status_code, 401) + + @patch("infrastructure_metrics.urlopen") + def test_operator_access_returns_200_with_full_structure(self, mock_urlopen): + mock_resp = MagicMock() + mock_resp.status = 200 + mock_resp.read.return_value = b'{"status": "ok", "service": "motorsignal-website", "revision": "0123456789abcdef0123456789abcdef01234567", "shortRevision": "0123456789ab"}' + mock_resp.__enter__.return_value = mock_resp + mock_urlopen.return_value = mock_resp + + resp = self.client.get( + "/api/v1/ops/infrastructure", + headers={"Authorization": f"Bearer {self.raw_token}"}, + ) + self.assertEqual(resp.status_code, 200) + data = resp.json() + + # Check top-level keys + self.assertIn("resources", data) + self.assertIn("services", data) + self.assertIn("network", data) + self.assertIn("deployment", data) + self.assertIn("collected_at", data) + + # Check resources keys + res = data["resources"] + for k in ("cpu_percent", "load_avg", "memory", "swap", "disk", "uptime_seconds", "uptime_formatted", "collected_at"): + self.assertIn(k, res) + + # Check services + services = data["services"] + self.assertEqual(len(services), 4) + service_names = {s["name"] for s in services} + self.assertEqual(service_names, set(ALLOWED_SYSTEMD_SERVICES)) + for s in services: + self.assertIn(s["status"], ("active", "inactive", "failed", "unknown")) + self.assertTrue(len(s["display_name"]) > 0) + + # Check network + net = data["network"] + self.assertIn("bytes_received", net) + self.assertIn("bytes_transmitted", net) + self.assertEqual(net["label"], "Cumulative Total Since Boot") + + # Check deployment + dep = data["deployment"] + self.assertIn("git_revision", dep) + self.assertIn("short_revision", dep) + self.assertIn(dep["revision_status"], ("known", "unavailable")) + self.assertIn(dep["website_health"], ("available", "degraded", "unavailable", "unknown")) + self.assertIn(dep["api_health"], ("available", "degraded", "unavailable", "unknown")) + self.assertEqual(dep["history_status"], "unavailable") + self.assertIsNone(dep["last_successful_deployment"]) + + @patch("infrastructure_metrics.urlopen") + def test_owner_access_returns_200(self, mock_urlopen): + mock_resp = MagicMock() + mock_resp.status = 200 + mock_resp.read.return_value = b'{"status": "ok", "service": "motorsignal-website", "revision": "0123456789abcdef0123456789abcdef01234567", "shortRevision": "0123456789ab"}' + mock_resp.__enter__.return_value = mock_resp + mock_urlopen.return_value = mock_resp + + resp = self.client.get( + "/api/v1/ops/infrastructure", + headers={"Authorization": f"Bearer {self.raw_owner_token}"}, + ) + self.assertEqual(resp.status_code, 200) + + + def test_disk_warning_and_critical_thresholds(self): + # 75% -> normal + with patch("shutil.disk_usage", return_value=(1000, 750, 250)): + m = get_disk_metrics("/") + self.assertIsNotNone(m) + self.assertEqual(m["percent"], 75.0) + self.assertFalse(m["warning"]) + self.assertFalse(m["critical"]) + + # 80% -> warning + with patch("shutil.disk_usage", return_value=(1000, 800, 200)): + m = get_disk_metrics("/") + self.assertIsNotNone(m) + self.assertEqual(m["percent"], 80.0) + self.assertTrue(m["warning"]) + self.assertFalse(m["critical"]) + + # 89% -> warning + with patch("shutil.disk_usage", return_value=(1000, 890, 110)): + m = get_disk_metrics("/") + self.assertIsNotNone(m) + self.assertEqual(m["percent"], 89.0) + self.assertTrue(m["warning"]) + self.assertFalse(m["critical"]) + + # 90% -> critical + with patch("shutil.disk_usage", return_value=(1000, 900, 100)): + m = get_disk_metrics("/") + self.assertIsNotNone(m) + self.assertEqual(m["percent"], 90.0) + self.assertTrue(m["warning"]) + self.assertTrue(m["critical"]) + + # 95% -> critical + with patch("shutil.disk_usage", return_value=(1000, 950, 50)): + m = get_disk_metrics("/") + self.assertIsNotNone(m) + self.assertEqual(m["percent"], 95.0) + self.assertTrue(m["warning"]) + self.assertTrue(m["critical"]) + + def test_service_allowlist_enforcement(self): + with patch("subprocess.run") as mock_sub: + # Arbitrary service must return unknown and NOT call subprocess + status = get_service_status("arbitrary-service") + self.assertEqual(status, "unknown") + mock_sub.assert_not_called() + + # Direct systemd unit name that is not an allowed logical service must return unknown and NOT call subprocess + status = get_service_status("postgresql@14-main") + self.assertEqual(status, "unknown") + mock_sub.assert_not_called() + + # Malicious injection attempt must return unknown and NOT call subprocess + status = get_service_status("motorsignal-website; rm -rf /") + self.assertEqual(status, "unknown") + mock_sub.assert_not_called() + + # Allowed service calls subprocess with its fixed mapped unit + mock_sub.return_value = MagicMock(returncode=0, stdout="active\n", stderr="") + status = get_service_status("motorsignal-website") + self.assertEqual(status, "active") + mock_sub.assert_called_once_with( + ["systemctl", "is-active", "motorsignal-website"], + capture_output=True, + text=True, + timeout=2.0, + shell=False, + ) + + def test_postgresql_unit_mapping_and_api_contract(self): + with patch("subprocess.run") as mock_sub: + mock_sub.return_value = MagicMock(returncode=0, stdout="active\n", stderr="") + + # 1. Logical postgresql invokes systemctl is-active postgresql@14-main + status = get_service_status("postgresql") + self.assertEqual(status, "active") + mock_sub.assert_called_once_with( + ["systemctl", "is-active", "postgresql@14-main"], + capture_output=True, + text=True, + timeout=2.0, + shell=False, + ) + + # 2. get_services_health() still uses public name postgresql + services = get_services_health() + pg_service = next((s for s in services if s["name"] == "postgresql"), None) + self.assertIsNotNone(pg_service) + self.assertEqual(pg_service["name"], "postgresql") + self.assertEqual(pg_service["display_name"], "PostgreSQL Database") + self.assertEqual(pg_service["status"], "active") + + # 3. Ensure no service in get_services_health exposes postgresql@14-main as its name + service_names = [s["name"] for s in services] + self.assertNotIn("postgresql@14-main", service_names) + self.assertIn("postgresql", service_names) + + def test_service_status_normalization(self): + with patch("subprocess.run") as mock_sub: + # active + mock_sub.return_value = MagicMock(returncode=0, stdout="active\n") + self.assertEqual(get_service_status("motorsignal-api"), "active") + + # inactive + mock_sub.return_value = MagicMock(returncode=3, stdout="inactive\n") + self.assertEqual(get_service_status("motorsignal-api"), "inactive") + + # failed + mock_sub.return_value = MagicMock(returncode=3, stdout="failed\n") + self.assertEqual(get_service_status("motorsignal-api"), "failed") + + # unknown output / timeout + mock_sub.return_value = MagicMock(returncode=1, stdout="activating\n") + self.assertEqual(get_service_status("motorsignal-api"), "unknown") + + mock_sub.side_effect = Exception("systemctl not found") + self.assertEqual(get_service_status("motorsignal-api"), "unknown") + + @patch("infrastructure_metrics.urlopen") + def test_website_health_preserves_5xx_as_degraded(self, mock_urlopen): + from urllib.error import HTTPError + # 1. /api/health returns 500 -> must report degraded without probing / or reporting available + mock_urlopen.side_effect = HTTPError("http://127.0.0.1:3001/api/health", 500, "Internal Server Error", {}, None) + dep = get_deployment_health(database_ok=True, website_url="http://127.0.0.1:3001") + self.assertEqual(dep["website_health"], "degraded") + + # 2. /api/health returns 503 -> degraded + mock_urlopen.side_effect = HTTPError("http://127.0.0.1:3001/api/health", 503, "Service Unavailable", {}, None) + dep = get_deployment_health(database_ok=True, website_url="http://127.0.0.1:3001") + self.assertEqual(dep["website_health"], "degraded") + + # 3. /api/health returns 404 -> fallback to root page (if root is 200 -> available) + mock_resp_root = MagicMock() + mock_resp_root.status = 200 + mock_resp_root.__enter__.return_value = mock_resp_root + + def urlopen_side_effect(req, timeout=1.5): + url = req.full_url if hasattr(req, "full_url") else str(req) + if "/api/health" in url: + raise HTTPError(url, 404, "Not Found", {}, None) + return mock_resp_root + + mock_urlopen.side_effect = urlopen_side_effect + dep = get_deployment_health(database_ok=True, website_url="http://127.0.0.1:3001") + self.assertEqual(dep["website_health"], "available") + + @patch("infrastructure_metrics.urlopen") + def test_get_website_revision_valid_hex(self, mock_urlopen): + # 1. Lowercase 40-char hex + mock_resp = MagicMock() + mock_resp.status = 200 + mock_resp.read.return_value = b'{"status":"ok","service":"motorsignal-website","revision":"0123456789abcdef0123456789abcdef01234567"}' + mock_resp.__enter__.return_value = mock_resp + mock_urlopen.return_value = mock_resp + + rev = get_website_revision("http://127.0.0.1:3001") + self.assertEqual(rev, "0123456789abcdef0123456789abcdef01234567") + + # 2. Uppercase 40-char hex + mock_resp.read.return_value = b'{"status":"ok","service":"motorsignal-website","revision":"ABCDEF0123456789ABCDEF0123456789ABCDEF01"}' + rev_upper = get_website_revision("http://127.0.0.1:3001") + self.assertEqual(rev_upper, "ABCDEF0123456789ABCDEF0123456789ABCDEF01") + + @patch("infrastructure_metrics.urlopen") + def test_get_website_revision_missing_or_null(self, mock_urlopen): + # 1. HTTP 404 / HTTPError + from urllib.error import HTTPError + mock_urlopen.side_effect = HTTPError("http://127.0.0.1:3001/api/version", 404, "Not Found", {}, None) + self.assertIsNone(get_website_revision("http://127.0.0.1:3001")) + + # 2. HTTP 200 but missing revision key + mock_resp = MagicMock() + mock_resp.status = 200 + mock_resp.read.return_value = b'{"status":"ok","service":"motorsignal-website"}' + mock_resp.__enter__.return_value = mock_resp + mock_urlopen.side_effect = None + mock_urlopen.return_value = mock_resp + self.assertIsNone(get_website_revision("http://127.0.0.1:3001")) + + # 3. HTTP 200 but null revision + mock_resp.read.return_value = b'{"status":"ok","service":"motorsignal-website","revision":null}' + self.assertIsNone(get_website_revision("http://127.0.0.1:3001")) + + @patch("infrastructure_metrics.urlopen") + def test_get_website_revision_malformed_and_invalid(self, mock_urlopen): + mock_resp = MagicMock() + mock_resp.status = 200 + mock_resp.__enter__.return_value = mock_resp + mock_urlopen.return_value = mock_resp + + # 1. HTML error page (non-JSON) + mock_resp.read.return_value = b"502 Bad Gateway502" + self.assertIsNone(get_website_revision("http://127.0.0.1:3001")) + + # 2. Truncated / invalid JSON syntax + mock_resp.read.return_value = b'{"status":"ok","revision":' + self.assertIsNone(get_website_revision("http://127.0.0.1:3001")) + + # 3. "unknown" placeholder + mock_resp.read.return_value = b'{"status":"ok","service":"motorsignal-website","revision":"unknown"}' + self.assertIsNone(get_website_revision("http://127.0.0.1:3001")) + + # 4. Short revision (e.g. 12-char short revision instead of 40-char full) + mock_resp.read.return_value = b'{"status":"ok","service":"motorsignal-website","revision":"0123456789ab"}' + self.assertIsNone(get_website_revision("http://127.0.0.1:3001")) + + # 5. Overlong revision (41 characters) + mock_resp.read.return_value = b'{"status":"ok","service":"motorsignal-website","revision":"0123456789abcdef0123456789abcdef012345678"}' + self.assertIsNone(get_website_revision("http://127.0.0.1:3001")) + + # 6. Non-hex characters in 40-char string + mock_resp.read.return_value = b'{"status":"ok","service":"motorsignal-website","revision":"0123456789abcdef0123456789abcdef0123456g"}' + self.assertIsNone(get_website_revision("http://127.0.0.1:3001")) + + @patch("infrastructure_metrics.urlopen") + def test_get_website_revision_oversized(self, mock_urlopen): + # Payload exceeding MAX_VERSION_RESPONSE_BYTES (4096) + oversized_payload = b'{"revision":"' + b"a" * 5000 + b'"}' + mock_resp = MagicMock() + mock_resp.status = 200 + mock_resp.read.return_value = oversized_payload + mock_resp.__enter__.return_value = mock_resp + mock_urlopen.return_value = mock_resp + + self.assertIsNone(get_website_revision("http://127.0.0.1:3001")) + + @patch("infrastructure_metrics.urlopen") + def test_get_website_revision_unreachable_and_timeout(self, mock_urlopen): + from urllib.error import URLError + + # 1. Connection refused / URLError + mock_urlopen.side_effect = URLError("Connection refused") + self.assertIsNone(get_website_revision("http://127.0.0.1:3001")) + + # 2. TimeoutError + mock_urlopen.side_effect = TimeoutError("Request timed out") + self.assertIsNone(get_website_revision("http://127.0.0.1:3001")) + + def test_production_website_listener_verified_port_and_no_url_leak(self): + # 1. Default internal port must be 3001 + self.assertEqual(DEFAULT_WEBSITE_INTERNAL_URL, "http://127.0.0.1:3001") + + # 2. get_deployment_health must not leak internal URL in dictionary + dep = get_deployment_health(database_ok=True, website_url="http://127.0.0.1:3001") + for key in dep.keys(): + self.assertNotIn("url", key.lower()) + self.assertNotIn("3001", str(dep[key])) + self.assertNotIn("127.0.0.1", str(dep[key])) + + # 3. get_infrastructure_metrics must not leak internal URL in dictionary + infra = get_infrastructure_metrics(database_ok=True) + dep_in_infra = infra["deployment"] + for key in dep_in_infra.keys(): + self.assertNotIn("url", key.lower()) + self.assertNotIn("3001", str(dep_in_infra[key])) + self.assertNotIn("127.0.0.1", str(dep_in_infra[key])) + + @patch("infrastructure_metrics.urlopen") + def test_environment_variable_fallback_deliberate_config(self, mock_urlopen): + # Endpoint unreachable + from urllib.error import URLError + mock_urlopen.side_effect = URLError("Endpoint not found") + + valid_sha = "0123456789abcdef0123456789abcdef01234567" + + # 1. Fallback to DEPLOYED_REVISION if set and valid + with patch.dict(os.environ, {"DEPLOYED_REVISION": valid_sha}, clear=False): + self.assertEqual(get_website_revision(), valid_sha) + + # 2. Fallback to GIT_SHA if set and valid + with patch.dict(os.environ, {"GIT_SHA": valid_sha, "DEPLOYED_REVISION": ""}, clear=False): + self.assertEqual(get_website_revision(), valid_sha) + + # 3. Invalid env var rejected + with patch.dict(os.environ, {"DEPLOYED_REVISION": "invalid-sha-123", "GIT_SHA": ""}, clear=False): + self.assertIsNone(get_website_revision()) + + +if __name__ == "__main__": + unittest.main() diff --git a/website/app/api/ops/infrastructure/route.ts b/website/app/api/ops/infrastructure/route.ts new file mode 100644 index 0000000..ee338b8 --- /dev/null +++ b/website/app/api/ops/infrastructure/route.ts @@ -0,0 +1,65 @@ +import { cookies } from 'next/headers'; +import { NextRequest, NextResponse } from 'next/server'; +import { normalizeInfrastructureResponse } from '../../../../lib/ops-data'; + +export async function GET(request: NextRequest) { + const cookieStore = await cookies(); + const isProduction = + process.env.NODE_ENV === 'production' && + process.env.NEXT_PUBLIC_APP_ENV === 'production'; + const opsToken = isProduction + ? cookieStore.get('__Host-motorsignal_ops')?.value + : cookieStore.get('__Host-motorsignal_ops')?.value ?? + cookieStore.get('motorsignal_ops')?.value; + + if (!opsToken) { + return NextResponse.json( + { detail: 'Authentication required.' }, + { status: 401 } + ); + } + + const baseUrl = + process.env.MOTORSIGNAL_API_BASE_URL ?? 'https://api.motorsignal.app'; + const headers: Record = { + accept: 'application/json', + authorization: `Bearer ${opsToken}`, + }; + + try { + const response = await fetch(`${baseUrl}/api/v1/ops/infrastructure`, { + method: 'GET', + headers, + cache: 'no-store', + }); + + if (!response.ok) { + if (response.status === 401) { + return NextResponse.json( + { detail: 'Authentication required.' }, + { status: 401 } + ); + } + if (response.status === 403) { + return NextResponse.json( + { detail: 'Access restricted.' }, + { status: 403 } + ); + } + return NextResponse.json( + normalizeInfrastructureResponse(null), + { status: 503 } + ); + } + + const data = await response.json(); + return NextResponse.json(normalizeInfrastructureResponse(data), { + status: 200, + }); + } catch { + return NextResponse.json( + normalizeInfrastructureResponse(null), + { status: 503 } + ); + } +} diff --git a/website/components/ops/operations-infrastructure.tsx b/website/components/ops/operations-infrastructure.tsx new file mode 100644 index 0000000..2614d32 --- /dev/null +++ b/website/components/ops/operations-infrastructure.tsx @@ -0,0 +1,618 @@ +'use client'; + +import React, { useState, useEffect, useRef, useCallback } from 'react'; +import { + OperationsInfrastructureData, + ServiceStatusValue, + ComponentHealthStatus, + OperationsRole, +} from './types'; +import { normalizeInfrastructureResponse } from '../../lib/ops-data'; + +interface OperationsInfrastructureProps { + role: OperationsRole; +} + +export function OperationsInfrastructure({ role }: OperationsInfrastructureProps) { + const [data, setData] = useState(null); + const [isRefreshing, setIsRefreshing] = useState(false); + const [lastFetchedAt, setLastFetchedAt] = useState(null); + const [fetchError, setFetchError] = useState(null); + const [copiedSha, setCopiedSha] = useState(false); + + const isFetchingRef = useRef(false); + const pollTimerRef = useRef(null); + + const fetchData = useCallback(async () => { + if (isFetchingRef.current) return; + isFetchingRef.current = true; + setIsRefreshing(true); + + try { + const resp = await fetch('/api/ops/infrastructure', { + method: 'GET', + headers: { accept: 'application/json' }, + cache: 'no-store', + }); + + if (resp.ok) { + const raw = await resp.json(); + const normalized = normalizeInfrastructureResponse(raw); + setData(normalized); + setLastFetchedAt(new Date().toLocaleTimeString()); + setFetchError(null); + } else if (resp.status === 401 || resp.status === 403) { + setFetchError('Authorization failed. Access restricted.'); + } else { + const raw = await resp.json().catch(() => null); + setData(normalizeInfrastructureResponse(raw)); + setFetchError('Metrics collector degraded or partially unavailable.'); + } + } catch { + setFetchError('Network error reaching infrastructure endpoint.'); + } finally { + isFetchingRef.current = false; + setIsRefreshing(false); + } + }, []); + + // Initial fetch and 30-second visibility-gated polling loop + useEffect(() => { + void fetchData(); + + const handleVisibilityChange = () => { + if (document.visibilityState === 'visible') { + void fetchData(); + } + }; + + document.addEventListener('visibilitychange', handleVisibilityChange); + + pollTimerRef.current = setInterval(() => { + if (document.visibilityState === 'visible') { + void fetchData(); + } + }, 30000); + + return () => { + document.removeEventListener('visibilitychange', handleVisibilityChange); + if (pollTimerRef.current) { + clearInterval(pollTimerRef.current); + } + }; + }, [fetchData]); + + const handleCopySha = (sha: string) => { + if (!sha) return; + navigator.clipboard.writeText(sha).then(() => { + setCopiedSha(true); + setTimeout(() => setCopiedSha(false), 2000); + }); + }; + + const formatBytes = (bytes: number | null): string => { + if (bytes === null || bytes === undefined) return 'N/A'; + if (bytes >= 1e12) return `${(bytes / 1e12).toFixed(2)} TB`; + if (bytes >= 1e9) return `${(bytes / 1e9).toFixed(2)} GB`; + if (bytes >= 1e6) return `${(bytes / 1e6).toFixed(1)} MB`; + return `${(bytes / 1e3).toFixed(1)} KB`; + }; + + const getServiceStatusBadge = (status: ServiceStatusValue) => { + switch (status) { + case 'active': + return { + label: 'Active (Running)', + icon: 'check_circle', + classes: 'bg-emerald-950/60 text-emerald-400 border-emerald-800/60', + }; + case 'inactive': + return { + label: 'Inactive (Stopped)', + icon: 'pause_circle', + classes: 'bg-amber-950/60 text-amber-400 border-amber-800/60', + }; + case 'failed': + return { + label: 'Failed', + icon: 'cancel', + classes: 'bg-rose-950/60 text-rose-400 border-rose-800/60', + }; + case 'unknown': + default: + return { + label: 'Unknown / Unreachable', + icon: 'help', + classes: 'bg-slate-900/60 text-slate-400 border-slate-700/60', + }; + } + }; + + const getComponentHealthBadge = (status: ComponentHealthStatus) => { + switch (status) { + case 'available': + return { + label: 'Available', + icon: 'check_circle', + classes: 'bg-emerald-950/60 text-emerald-400 border-emerald-800/60', + }; + case 'degraded': + return { + label: 'Degraded', + icon: 'warning', + classes: 'bg-amber-950/60 text-amber-400 border-amber-800/60', + }; + case 'unavailable': + return { + label: 'Unavailable', + icon: 'cancel', + classes: 'bg-rose-950/60 text-rose-400 border-rose-800/60', + }; + case 'unknown': + default: + return { + label: 'Unknown', + icon: 'help', + classes: 'bg-slate-900/60 text-slate-400 border-slate-700/60', + }; + } + }; + + const resources = data?.resources; + const services = data?.services ?? []; + const network = data?.network; + const deployment = data?.deployment; + + return ( +
+ {/* Header & Controls */} +
+
+
+ + dns + +

+ Infrastructure & Runtime +

+ + Phase 1 (Read-Only) + +
+

+ Host system resources, systemd core services, network totals, and deployment runtime health. +

+
+ +
+ {/* Polling indicator */} +
+ + Auto 30s +
+ + {/* Last refreshed */} + {lastFetchedAt && ( + + Updated: {lastFetchedAt} + + )} + + {/* Manual Refresh Button */} + +
+
+ + {/* Optional Warning Banner */} + {fetchError && ( +
+ + warning + + {fetchError} +
+ )} + + {/* SECTION 1: System Resources */} +
+
+

+ Server Resources +

+ + Target Host: Ubuntu VPS + +
+ +
+ {/* CPU Usage */} +
+
+ CPU Usage + + memory + +
+
+
+ {resources?.cpu_percent !== null && resources?.cpu_percent !== undefined + ? `${resources.cpu_percent}%` + : 'Unavailable'} +
+

+ Load: {resources?.load_avg ? resources.load_avg.join(', ') : 'N/A'} +

+
+ {/* Progress Bar */} +
+
+
+
+ + {/* Memory (RAM) */} +
+
+ Memory (RAM) + + developer_board + +
+
+
+ {resources?.memory + ? `${formatBytes(resources.memory.used)} / ${formatBytes(resources.memory.total)}` + : 'Unavailable'} +
+

+ {resources?.memory + ? `${resources.memory.percent}% used (${formatBytes(resources.memory.available)} free)` + : 'N/A'} +

+
+ {/* Progress Bar */} +
+
= 90 + ? 'bg-rose-500' + : (resources?.memory?.percent ?? 0) >= 80 + ? 'bg-amber-500' + : 'bg-primary-container' + }`} + style={{ width: `${Math.min(100, Math.max(0, resources?.memory?.percent ?? 0))}%` }} + /> +
+
+ + {/* Swap Memory */} +
+
+ Swap Space + + swap_horiz + +
+
+
+ {resources?.swap + ? `${formatBytes(resources.swap.used)} / ${formatBytes(resources.swap.total)}` + : 'Unavailable'} +
+

+ {resources?.swap ? `${resources.swap.percent}% utilized` : 'N/A'} +

+
+ {/* Progress Bar */} +
+
+
+
+ + {/* Disk Storage (/) */} +
+
+
+ Root Disk (/) + {resources?.disk?.critical ? ( + + CRITICAL + + ) : resources?.disk?.warning ? ( + + WARNING + + ) : null} +
+ + hard_drive + +
+
+
+ {resources?.disk + ? `${formatBytes(resources.disk.used)} / ${formatBytes(resources.disk.total)}` + : 'Unavailable'} +
+

+ {resources?.disk + ? `${resources.disk.percent}% used (${formatBytes(resources.disk.available)} free)` + : 'N/A'} +

+
+ {/* Progress Bar */} +
+
+
+
+ + {/* Server Uptime */} +
+
+ Server Uptime + + schedule + +
+
+
+ {resources?.uptime_formatted || 'Unavailable'} +
+

+ {resources?.uptime_seconds + ? `${resources.uptime_seconds.toLocaleString()}s total uptime` + : 'N/A'} +

+
+
+ System running continuously +
+
+
+
+ + {/* SECTION 2: Core Systemd Services */} +
+
+

+ Core Services (systemd) +

+ + Strict Allowlist: 4 Services + +
+ +
+ {services.map((svc) => { + const badge = getServiceStatusBadge(svc.status); + return ( +
+
+
+
+

+ {svc.display_name} +

+

+ {svc.name}.service +

+
+
+

+ {svc.description} +

+
+ +
+ Service Status +
+ + {badge.icon} + + {badge.label} +
+
+
+ ); + })} +
+
+ + {/* SECTION 3 & 4: Network Cumulative Totals & Deployment Health */} +
+ {/* Network Cumulative Totals */} +
+
+
+ + network_check + +
+

+ Network Cumulative Totals +

+

+ Aggregated across non-loopback interfaces since host boot +

+
+
+ + Not a rate + +
+ +
+
+
+ + download + + Total Received +
+
+ {formatBytes(network?.bytes_received ?? null)} +
+

+ {network?.bytes_received ? `${network.bytes_received.toLocaleString()} bytes` : 'N/A'} +

+
+ +
+
+ + upload + + Total Transmitted +
+
+ {formatBytes(network?.bytes_transmitted ?? null)} +
+

+ {network?.bytes_transmitted ? `${network.bytes_transmitted.toLocaleString()} bytes` : 'N/A'} +

+
+
+ +
+ + info + + + Cumulative Total Since Boot across physical network interfaces. Instantaneous bandwidth rates are not tracked. + +
+
+ + {/* Deployment Health Foundations */} +
+
+
+ + deployed_code + +
+

+ Deployment Health Foundations +

+

+ Active code revision and subsystem runtime readiness +

+
+
+ + Foundation + +
+ +
+ {/* Active Git Revision */} +
+
+
Active Git Revision
+
+ + {deployment?.short_revision || 'Unavailable'} + + {deployment?.git_revision && ( + + )} +
+
+
+ + + {deployment?.revision_status ?? 'unavailable'} + +
+
+ + {/* Health indicators */} +
+
+
Website Ingress
+ {(() => { + const badge = getComponentHealthBadge(deployment?.website_health ?? 'unknown'); + return ( +
+ {badge.icon} + {badge.label} +
+ ); + })()} +
+ +
+
API Readiness
+ {(() => { + const badge = getComponentHealthBadge(deployment?.api_health ?? 'unknown'); + return ( +
+ {badge.icon} + {badge.label} +
+ ); + })()} +
+
+ + {/* Deployment History Status */} +
+ Deployment History: + Unavailable in Phase 1 +
+
+
+
+
+ ); +} diff --git a/website/components/ops/operations-shell.tsx b/website/components/ops/operations-shell.tsx index 409117f..ec9f04f 100644 --- a/website/components/ops/operations-shell.tsx +++ b/website/components/ops/operations-shell.tsx @@ -20,6 +20,8 @@ import { OperationsIncidentDetail } from './operations-incident-detail'; import { OperationsUsers } from './operations-users'; import { OperationsSupport } from './operations-support'; import { OperationsCms } from './operations-cms'; +import { OperationsInfrastructure } from './operations-infrastructure'; + interface OperationsShellProps { initialSession: OperatorSession; @@ -223,6 +225,10 @@ export function OperationsShell({ /> )} + {activeNav === 'infrastructure' && ( + + )} + {activeNav === 'users' && ( )} diff --git a/website/components/ops/operations-sidebar.tsx b/website/components/ops/operations-sidebar.tsx index 942c40a..7dbe4cf 100644 --- a/website/components/ops/operations-sidebar.tsx +++ b/website/components/ops/operations-sidebar.tsx @@ -32,6 +32,11 @@ export function OperationsSidebar({ label: 'Overview', icon: 'dashboard', }, + { + id: 'infrastructure', + label: 'Infrastructure', + icon: 'dns', + }, { id: 'users', label: 'Users', diff --git a/website/components/ops/types.ts b/website/components/ops/types.ts index 525656e..f2be23c 100644 --- a/website/components/ops/types.ts +++ b/website/components/ops/types.ts @@ -52,6 +52,7 @@ export interface OperatorSession { export type OperationsNavId = | 'overview' + | 'infrastructure' | 'users' | 'incidents' | 'support' @@ -60,6 +61,7 @@ export type OperationsNavId = | 'operators' | 'session'; + export type IncidentSeverity = 'low' | 'medium' | 'high' | 'critical'; export type IncidentStatus = @@ -320,4 +322,71 @@ export interface CmsDocumentDetail { revisions: CmsRevisionItem[]; } +export type ServiceStatusValue = 'active' | 'inactive' | 'failed' | 'unknown'; + +export interface ServiceHealthItem { + name: string; + display_name: string; + status: ServiceStatusValue; + description: string; +} + +export interface SystemResourceMemory { + total: number; + used: number; + available: number; + percent: number; +} + +export interface SystemResourceSwap { + total: number; + used: number; + percent: number; +} + +export interface SystemResourceDisk { + total: number; + used: number; + available: number; + percent: number; + warning: boolean; + critical: boolean; +} + +export interface SystemResources { + cpu_percent: number | null; + load_avg: [number, number, number] | null; + memory: SystemResourceMemory | null; + swap: SystemResourceSwap | null; + disk: SystemResourceDisk | null; + uptime_seconds: number | null; + uptime_formatted: string | null; + collected_at: string; +} + +export interface NetworkTotals { + bytes_received: number | null; + bytes_transmitted: number | null; + label: string; + collected_at: string; +} + +export interface DeploymentHealth { + git_revision: string | null; + short_revision: string | null; + revision_status: 'known' | 'unavailable'; + website_health: ComponentHealthStatus; + api_health: ComponentHealthStatus; + history_status: 'unavailable' | 'available'; + last_successful_deployment: Record | null; +} + +export interface OperationsInfrastructureData { + resources: SystemResources; + services: ServiceHealthItem[]; + network: NetworkTotals; + deployment: DeploymentHealth; + collected_at: string; +} + diff --git a/website/lib/ops-data.ts b/website/lib/ops-data.ts index a60ec89..fab5f73 100644 --- a/website/lib/ops-data.ts +++ b/website/lib/ops-data.ts @@ -6,8 +6,12 @@ import { OperationsOverviewData, OverviewSecurityErrorItem, SafeIncidentSummary, + OperationsInfrastructureData, + ServiceStatusValue, + ServiceHealthItem, } from '../components/ops/types'; + export const HEALTH_COMPONENT_METADATA: Record< HealthComponentKey, { name: string; description: string } @@ -471,3 +475,193 @@ export function normalizeOverviewResponse(raw: unknown): OperationsOverviewData }; } +export function isValidServiceStatus(status: unknown): status is ServiceStatusValue { + return status === 'active' || status === 'inactive' || status === 'failed' || status === 'unknown'; +} + +export function normalizeInfrastructureResponse(raw: unknown): OperationsInfrastructureData { + const fallbackCollectedAt = new Date().toISOString(); + + const defaultServices: ServiceHealthItem[] = [ + { + name: 'motorsignal-website', + display_name: 'MotorSignal Website', + description: 'Next.js frontend application runtime', + status: 'unknown', + }, + { + name: 'motorsignal-api', + display_name: 'MotorSignal API', + description: 'FastAPI core backend service & ingress', + status: 'unknown', + }, + { + name: 'nginx', + display_name: 'Nginx Reverse Proxy', + description: 'Edge TLS termination & reverse proxy routing', + status: 'unknown', + }, + { + name: 'postgresql', + display_name: 'PostgreSQL Database', + description: 'Primary relational database persistence', + status: 'unknown', + }, + ]; + + if (!raw || typeof raw !== 'object') { + return { + resources: { + cpu_percent: null, + load_avg: null, + memory: null, + swap: null, + disk: null, + uptime_seconds: null, + uptime_formatted: null, + collected_at: fallbackCollectedAt, + }, + services: defaultServices, + network: { + bytes_received: null, + bytes_transmitted: null, + label: 'Cumulative Total Since Boot', + collected_at: fallbackCollectedAt, + }, + deployment: { + git_revision: null, + short_revision: null, + revision_status: 'unavailable', + website_health: 'unknown', + api_health: 'unknown', + history_status: 'unavailable', + last_successful_deployment: null, + }, + collected_at: fallbackCollectedAt, + }; + } + + const obj = raw as Record; + const rawResources = obj.resources && typeof obj.resources === 'object' ? (obj.resources as Record) : {}; + const rawServices = Array.isArray(obj.services) ? obj.services : []; + const rawNetwork = obj.network && typeof obj.network === 'object' ? (obj.network as Record) : {}; + const rawDeployment = obj.deployment && typeof obj.deployment === 'object' ? (obj.deployment as Record) : {}; + + // Resources + const cpu_percent = typeof rawResources.cpu_percent === 'number' ? Math.max(0, Math.min(100, rawResources.cpu_percent)) : null; + const load_avg: [number, number, number] | null = + Array.isArray(rawResources.load_avg) && rawResources.load_avg.length >= 3 && rawResources.load_avg.every((n) => typeof n === 'number') + ? [rawResources.load_avg[0], rawResources.load_avg[1], rawResources.load_avg[2]] + : null; + + let memory = null; + if (rawResources.memory && typeof rawResources.memory === 'object') { + const rm = rawResources.memory as Record; + if (typeof rm.total === 'number' && typeof rm.used === 'number') { + const total = rm.total; + const used = rm.used; + const available = typeof rm.available === 'number' ? rm.available : Math.max(0, total - used); + const percent = typeof rm.percent === 'number' ? rm.percent : total > 0 ? Math.round((used / total) * 1000) / 10 : 0; + memory = { total, used, available, percent }; + } + } + + let swap = null; + if (rawResources.swap && typeof rawResources.swap === 'object') { + const rs = rawResources.swap as Record; + if (typeof rs.total === 'number' && typeof rs.used === 'number') { + const total = rs.total; + const used = rs.used; + const percent = typeof rs.percent === 'number' ? rs.percent : total > 0 ? Math.round((used / total) * 1000) / 10 : 0; + swap = { total, used, percent }; + } + } + + let disk = null; + if (rawResources.disk && typeof rawResources.disk === 'object') { + const rd = rawResources.disk as Record; + if (typeof rd.total === 'number' && typeof rd.used === 'number') { + const total = rd.total; + const used = rd.used; + const available = typeof rd.available === 'number' ? rd.available : Math.max(0, total - used); + const percent = typeof rd.percent === 'number' ? rd.percent : total > 0 ? Math.round((used / total) * 1000) / 10 : 0; + disk = { + total, + used, + available, + percent, + warning: percent >= 80.0, + critical: percent >= 90.0, + }; + } + } + + const uptime_seconds = typeof rawResources.uptime_seconds === 'number' ? rawResources.uptime_seconds : null; + const uptime_formatted = typeof rawResources.uptime_formatted === 'string' && rawResources.uptime_formatted.trim() ? rawResources.uptime_formatted.trim() : null; + + // Services + const servicesMap = new Map(); + for (const def of defaultServices) { + servicesMap.set(def.name, { ...def }); + } + + for (const s of rawServices) { + if (s && typeof s === 'object') { + const sObj = s as Record; + const name = typeof sObj.name === 'string' ? sObj.name.trim() : ''; + if (servicesMap.has(name)) { + const existing = servicesMap.get(name)!; + const status = isValidServiceStatus(sObj.status) ? sObj.status : 'unknown'; + const display_name = typeof sObj.display_name === 'string' && sObj.display_name.trim() ? sObj.display_name.trim() : existing.display_name; + const description = typeof sObj.description === 'string' && sObj.description.trim() ? sObj.description.trim() : existing.description; + servicesMap.set(name, { name, display_name, description, status }); + } + } + } + + const services = defaultServices.map((d) => servicesMap.get(d.name)!); + + // Network + const bytes_received = typeof rawNetwork.bytes_received === 'number' ? rawNetwork.bytes_received : null; + const bytes_transmitted = typeof rawNetwork.bytes_transmitted === 'number' ? rawNetwork.bytes_transmitted : null; + const network = { + bytes_received, + bytes_transmitted, + label: 'Cumulative Total Since Boot', + collected_at: typeof rawNetwork.collected_at === 'string' ? rawNetwork.collected_at : fallbackCollectedAt, + }; + + // Deployment + const git_revision = typeof rawDeployment.git_revision === 'string' && rawDeployment.git_revision.trim() ? rawDeployment.git_revision.trim() : null; + const short_revision = typeof rawDeployment.short_revision === 'string' && rawDeployment.short_revision.trim() + ? rawDeployment.short_revision.trim() + : git_revision ? git_revision.slice(0, 7) : null; + const revision_status = git_revision ? 'known' : 'unavailable'; + const website_health = isValidHealthStatus(rawDeployment.website_health) ? rawDeployment.website_health : 'unknown'; + const api_health = isValidHealthStatus(rawDeployment.api_health) ? rawDeployment.api_health : 'unknown'; + + return { + resources: { + cpu_percent, + load_avg, + memory, + swap, + disk, + uptime_seconds, + uptime_formatted, + collected_at: typeof rawResources.collected_at === 'string' ? rawResources.collected_at : fallbackCollectedAt, + }, + services, + network, + deployment: { + git_revision, + short_revision, + revision_status, + website_health, + api_health, + history_status: 'unavailable', + last_successful_deployment: null, + }, + collected_at: typeof obj.collected_at === 'string' ? obj.collected_at : fallbackCollectedAt, + }; +} diff --git a/website/tests/ops-experience.test.cjs b/website/tests/ops-experience.test.cjs index 4b24113..f44f83c 100644 --- a/website/tests/ops-experience.test.cjs +++ b/website/tests/ops-experience.test.cjs @@ -55,8 +55,11 @@ const { sanitizeIncidentSummary, sanitizeIncidentDetail, sanitizeIncidentNote, + normalizeInfrastructureResponse, + isValidServiceStatus, } = opsData; + // ============================================================ // 1. ISOLATED OPERATIONS SHELL & CONSUMER NAV SUPPRESSION // ============================================================ @@ -715,12 +718,14 @@ console.log(' PASS: Incident Detail UI includes all required operational sectio // ============================================================ console.log('Testing 33: Zero Polling Verification Across Operations UI...'); for (const file of currentOpsFiles) { + if (file === 'operations-infrastructure.tsx') continue; // Infrastructure explicitly implements 30s visibility-gated polling (tested in Section 43) const content = fs.readFileSync(path.join(opsDir, file), 'utf8'); assert.ok(!content.includes('setInterval'), `${file} must NOT contain setInterval polling`); assert.ok(!content.includes('setTimeout(') || content.includes('setTimeout(() => setMutationSuccess') || content.includes('setTimeout(() =>'), `${file} must only use setTimeout for UI transitions, not polling`); } console.log(' PASS: Zero automatic background polling across all Operations components.'); + // ============================================================ // 34. ZERO RIDER DATA CROSSOVER (OPS-4) // ============================================================ @@ -1143,6 +1148,197 @@ assert.ok(backendMainSrc.includes('homepage_hero') && backendMainSrc.includes('f console.log(' PASS: Section 4 Website Content CMS backend, proxy routes, navigation, and UI contracts fully verified.'); -console.log('--- ALL OPS-3C, OPS-3C.1, OPS-3C.2, OPS-3C.3, OPS-4, OPS-4.2, OVERVIEW, SECTION 2 USERS, SECTION 3 SUPPORT CENTER & SECTION 4 CMS TESTS PASSED (42/42) ---'); +// ============================================================ +// 43. PRODUCTION OPERATIONS INFRASTRUCTURE DASHBOARD (PHASE 1) +// ============================================================ +console.log('Testing 43: Operations Infrastructure Dashboard Phase 1...'); + +// 43.1 Normalization of null / empty / malformed response +const nullInfra = normalizeInfrastructureResponse(null); +assert.ok(nullInfra.resources, 'null input must return resources object'); +assert.equal(nullInfra.resources.cpu_percent, null); +assert.equal(nullInfra.resources.load_avg, null); +assert.equal(nullInfra.resources.memory, null); +assert.equal(nullInfra.resources.swap, null); +assert.equal(nullInfra.resources.disk, null); +assert.equal(nullInfra.resources.uptime_seconds, null); +assert.equal(nullInfra.resources.uptime_formatted, null); +assert.equal(nullInfra.services.length, 4, 'null input must provide 4 default services'); +assert.deepEqual(nullInfra.services.map(s => s.name), ['motorsignal-website', 'motorsignal-api', 'nginx', 'postgresql']); +assert.ok(nullInfra.services.every(s => s.status === 'unknown'), 'default services must have unknown status'); +assert.equal(nullInfra.network.bytes_received, null); +assert.equal(nullInfra.network.bytes_transmitted, null); +assert.equal(nullInfra.network.label, 'Cumulative Total Since Boot'); +assert.equal(nullInfra.deployment.git_revision, null); +assert.equal(nullInfra.deployment.short_revision, null); +assert.equal(nullInfra.deployment.revision_status, 'unavailable'); +assert.equal(nullInfra.deployment.website_health, 'unknown'); +assert.equal(nullInfra.deployment.api_health, 'unknown'); +assert.equal(nullInfra.deployment.history_status, 'unavailable'); +assert.equal(nullInfra.deployment.last_successful_deployment, null); + +// 43.2 Normalization of valid response & disk threshold warning/critical calculation +const validInfraRaw = { + resources: { + cpu_percent: 18.5, + load_avg: [0.15, 0.22, 0.18], + memory: { total: 16000000000, used: 8000000000, available: 8000000000, percent: 50.0 }, + swap: { total: 4000000000, used: 400000000, percent: 10.0 }, + disk: { total: 100000000000, used: 82000000000, available: 18000000000, percent: 82.0 }, + uptime_seconds: 123456, + uptime_formatted: '1d 10h 17m', + }, + services: [ + { name: 'motorsignal-website', status: 'active' }, + { name: 'motorsignal-api', status: 'active' }, + { name: 'nginx', status: 'active' }, + { name: 'postgresql', status: 'active' }, + ], + network: { + bytes_received: 5000000000, + bytes_transmitted: 2500000000, + label: 'Cumulative Total Since Boot', + }, + deployment: { + git_revision: '0123456789abcdef0123456789abcdef01234567', + short_revision: '0123456', + revision_status: 'known', + website_health: 'available', + api_health: 'available', + }, +}; + +const normalizedValid = normalizeInfrastructureResponse(validInfraRaw); +assert.equal(normalizedValid.resources.cpu_percent, 18.5); +assert.deepEqual(normalizedValid.resources.load_avg, [0.15, 0.22, 0.18]); +assert.equal(normalizedValid.resources.memory.percent, 50.0); +assert.equal(normalizedValid.resources.disk.percent, 82.0); +assert.equal(normalizedValid.resources.disk.warning, true, 'Disk at 82% must trigger warning'); +assert.equal(normalizedValid.resources.disk.critical, false, 'Disk at 82% must not trigger critical'); +assert.equal(normalizedValid.services[0].status, 'active'); +assert.equal(normalizedValid.network.bytes_received, 5000000000); +assert.equal(normalizedValid.deployment.git_revision, '0123456789abcdef0123456789abcdef01234567'); +assert.equal(normalizedValid.deployment.short_revision, '0123456'); +assert.equal(normalizedValid.deployment.revision_status, 'known'); +assert.equal(normalizedValid.deployment.website_health, 'available'); +assert.equal(normalizedValid.deployment.api_health, 'available'); + +// 43.3 Critical disk threshold (>=90%) +const criticalDiskRaw = { + resources: { + disk: { total: 100000000000, used: 91000000000, available: 9000000000, percent: 91.0 }, + }, +}; +const normalizedCritical = normalizeInfrastructureResponse(criticalDiskRaw); +assert.equal(normalizedCritical.resources.disk.warning, true); +assert.equal(normalizedCritical.resources.disk.critical, true, 'Disk at 91% must trigger critical'); + +// 43.4 Service status validation +assert.equal(isValidServiceStatus('active'), true); +assert.equal(isValidServiceStatus('inactive'), true); +assert.equal(isValidServiceStatus('failed'), true); +assert.equal(isValidServiceStatus('unknown'), true); +assert.equal(isValidServiceStatus('running'), false); +assert.equal(isValidServiceStatus('dead'), false); + +// 43.5 Types, Navigation & Shell wiring +assert.ok(typesSrc.includes("'infrastructure'"), "types.ts must include 'infrastructure' in OperationsNavId"); +assert.ok(typesSrc.includes('OperationsInfrastructureData'), 'types.ts must define OperationsInfrastructureData'); +assert.ok(typesSrc.includes('SystemResources'), 'types.ts must define SystemResources'); +assert.ok(typesSrc.includes('ServiceHealthItem'), 'types.ts must define ServiceHealthItem'); +assert.ok(typesSrc.includes('NetworkTotals'), 'types.ts must define NetworkTotals'); +assert.ok(typesSrc.includes('DeploymentHealth'), 'types.ts must define DeploymentHealth'); + +const updatedSidebarSource = fs.readFileSync(path.resolve(__dirname, '../components/ops/operations-sidebar.tsx'), 'utf8'); +const updatedShellSource = fs.readFileSync(path.resolve(__dirname, '../components/ops/operations-shell.tsx'), 'utf8'); + +assert.ok( + updatedSidebarSource.includes("id: 'infrastructure'") && + updatedSidebarSource.includes("label: 'Infrastructure'") && + updatedSidebarSource.includes("icon: 'dns'"), + 'Sidebar must contain infrastructure nav item' +); + +const infraOverviewNavIdx = updatedSidebarSource.indexOf("id: 'overview'"); +const infraNavIdx = updatedSidebarSource.indexOf("id: 'infrastructure'"); +const infraUsersNavIdx = updatedSidebarSource.indexOf("id: 'users'"); +assert.ok( + infraOverviewNavIdx < infraNavIdx && infraNavIdx < infraUsersNavIdx, + 'Infrastructure nav item must be placed right after overview and before users' +); + +assert.ok( + updatedShellSource.includes("activeNav === 'infrastructure'") && + updatedShellSource.includes('