feat(ops): implement production operations infrastructure dashboard (phase 1) (#14)
* feat(ops): implement production operations infrastructure dashboard (phase 1)
- Add backend read-only infrastructure metrics collector (CPU, RAM, swap, disk with 80%/90% thresholds, uptime, network cumulative totals, systemd service allowlist) using standard library only (no psutil).
- Register GET /api/v1/ops/infrastructure endpoint guarded by require_ops_roles('operator', 'owner').
- Add Next.js API proxy route /api/ops/infrastructure validating ops session cookie and forwarding to backend.
- Implement dark-surface OperationsInfrastructure component in website admin console with 30s visibility-gated polling and overlap prevention.
- Add comprehensive backend and frontend test suites (8 backend tests, 43 ops-experience tests).
* fix(ops): query internal website version endpoint defensively on verified port 3001
- Query internal website GET /api/version on verified production listener http://127.0.0.1:3001 (configurable via WEBSITE_INTERNAL_URL).
- Parse JSON defensively with a 4096-byte response bound and validate exact 40-char hex revision pattern.
- Add comprehensive tests for valid, missing, malformed, oversized, and unreachable version responses.
- Do not expose internal URLs or connection errors in API responses.
* fix(ops): map postgresql service to cluster unit postgresql@14-main and preserve health errors
- Map public logical service ID 'postgresql' to fixed systemd unit 'postgresql@14-main'.
- Reject direct systemd unit names and arbitrary service names.
- Preserve HTTP 5xx responses as degraded on /api/health before root fallback.
- Add regression tests for postgresql unit mapping, API response name stability, and health 5xx preservation.
---------
Co-authored-by: Developer <dev@example.com>
This commit is contained in:
parent
da0a05d3a2
commit
9d99bfa4e4
|
|
@ -0,0 +1,496 @@
|
|||
"""
|
||||
MotorSignal Operations Infrastructure Metrics Collector.
|
||||
|
||||
Safe, read-only system resource monitoring, service status, network totals,
|
||||
and deployment health foundations.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import time
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
from urllib.error import HTTPError, URLError
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
REVISION_HEX_REGEX = re.compile(r"^[0-9a-fA-F]{40}$")
|
||||
MAX_VERSION_RESPONSE_BYTES = 4096
|
||||
DEFAULT_WEBSITE_INTERNAL_URL = "http://127.0.0.1:3001"
|
||||
|
||||
# Fixed mapping from public logical service ID to actual systemd unit name
|
||||
SERVICE_SYSTEMD_UNITS: Dict[str, str] = {
|
||||
"motorsignal-website": "motorsignal-website",
|
||||
"motorsignal-api": "motorsignal-api",
|
||||
"nginx": "nginx",
|
||||
"postgresql": "postgresql@14-main",
|
||||
}
|
||||
|
||||
ALLOWED_SYSTEMD_SERVICES: Tuple[str, ...] = tuple(SERVICE_SYSTEMD_UNITS.keys())
|
||||
|
||||
SERVICE_METADATA: Dict[str, Dict[str, str]] = {
|
||||
"motorsignal-website": {
|
||||
"display_name": "MotorSignal Website",
|
||||
"description": "Next.js frontend application runtime",
|
||||
},
|
||||
"motorsignal-api": {
|
||||
"display_name": "MotorSignal API",
|
||||
"description": "FastAPI core backend service & ingress",
|
||||
},
|
||||
"nginx": {
|
||||
"display_name": "Nginx Reverse Proxy",
|
||||
"description": "Edge TLS termination & reverse proxy routing",
|
||||
},
|
||||
"postgresql": {
|
||||
"display_name": "PostgreSQL Database",
|
||||
"description": "Primary relational database persistence",
|
||||
},
|
||||
}
|
||||
|
||||
# In-memory previous CPU stat cache for non-blocking delta calculation
|
||||
_prev_cpu_stat: Optional[Tuple[float, int, int]] = None # (timestamp, idle_ticks, total_ticks)
|
||||
|
||||
|
||||
@dataclass
|
||||
class OpsDeploymentEvent:
|
||||
"""Foundational data model for deployment events (Phase 2 readiness)."""
|
||||
id: str
|
||||
source_revision: str
|
||||
initiating_operator_id: str
|
||||
started_at: datetime
|
||||
completed_at: Optional[datetime]
|
||||
status: str # 'pending' | 'in_progress' | 'success' | 'failed' | 'rolled_back'
|
||||
release_identifier: Optional[str] = None
|
||||
failure_category: Optional[str] = None
|
||||
previous_revision: Optional[str] = None
|
||||
resulting_revision: Optional[str] = None
|
||||
|
||||
|
||||
def _read_proc_file(path: str) -> Optional[str]:
|
||||
"""Safely read a /proc file if it exists."""
|
||||
try:
|
||||
if os.path.exists(path):
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
return f.read()
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def get_cpu_percent() -> Optional[float]:
|
||||
"""
|
||||
Calculate overall CPU percent.
|
||||
Uses /proc/stat deltas on Linux.
|
||||
Degrades safely to None if unavailable.
|
||||
"""
|
||||
global _prev_cpu_stat
|
||||
|
||||
stat_content = _read_proc_file("/proc/stat")
|
||||
if stat_content:
|
||||
for line in stat_content.splitlines():
|
||||
if line.startswith("cpu "):
|
||||
parts = line.split()
|
||||
if len(parts) >= 5:
|
||||
try:
|
||||
# parts: cpu, user, nice, system, idle, iowait, irq, softirq, steal, guest, guest_nice
|
||||
fields = [int(p) for p in parts[1:]]
|
||||
idle = fields[3] + (fields[4] if len(fields) > 4 else 0)
|
||||
total = sum(fields)
|
||||
now = time.time()
|
||||
|
||||
if _prev_cpu_stat is not None:
|
||||
prev_time, prev_idle, prev_total = _prev_cpu_stat
|
||||
delta_time = now - prev_time
|
||||
delta_total = total - prev_total
|
||||
delta_idle = idle - prev_idle
|
||||
|
||||
if 0.1 <= delta_time <= 60.0 and delta_total > 0:
|
||||
_prev_cpu_stat = (now, idle, total)
|
||||
used_pct = ((delta_total - delta_idle) / delta_total) * 100.0
|
||||
return round(max(0.0, min(100.0, used_pct)), 1)
|
||||
|
||||
# Sample quick delta if no previous sample exists
|
||||
time.sleep(0.05)
|
||||
stat_content_2 = _read_proc_file("/proc/stat")
|
||||
if stat_content_2:
|
||||
for line2 in stat_content_2.splitlines():
|
||||
if line2.startswith("cpu "):
|
||||
fields2 = [int(p) for p in line2.split()[1:]]
|
||||
idle2 = fields2[3] + (fields2[4] if len(fields2) > 4 else 0)
|
||||
total2 = sum(fields2)
|
||||
now2 = time.time()
|
||||
_prev_cpu_stat = (now2, idle2, total2)
|
||||
delta_total = total2 - total
|
||||
delta_idle = idle2 - idle
|
||||
if delta_total > 0:
|
||||
used_pct = ((delta_total - delta_idle) / delta_total) * 100.0
|
||||
return round(max(0.0, min(100.0, used_pct)), 1)
|
||||
|
||||
_prev_cpu_stat = (now, idle, total)
|
||||
except Exception:
|
||||
pass
|
||||
break
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def get_load_averages() -> Optional[Tuple[float, float, float]]:
|
||||
"""Return 1m, 5m, 15m load averages via os.getloadavg()."""
|
||||
try:
|
||||
if hasattr(os, "getloadavg"):
|
||||
avg = os.getloadavg()
|
||||
return (round(avg[0], 2), round(avg[1], 2), round(avg[2], 2))
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def get_memory_metrics() -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Parse /proc/meminfo for RAM metrics.
|
||||
Returns total, used, available in bytes, and used percent.
|
||||
"""
|
||||
mem_content = _read_proc_file("/proc/meminfo")
|
||||
if mem_content:
|
||||
mem_info: Dict[str, int] = {}
|
||||
for line in mem_content.splitlines():
|
||||
parts = line.split(":")
|
||||
if len(parts) == 2:
|
||||
key = parts[0].strip()
|
||||
val_parts = parts[1].strip().split()
|
||||
if val_parts and val_parts[0].isdigit():
|
||||
mem_info[key] = int(val_parts[0]) * 1024 # kB to bytes
|
||||
|
||||
if "MemTotal" in mem_info:
|
||||
total = mem_info["MemTotal"]
|
||||
avail = mem_info.get(
|
||||
"MemAvailable",
|
||||
mem_info.get("MemFree", 0) + mem_info.get("Buffers", 0) + mem_info.get("Cached", 0)
|
||||
)
|
||||
used = max(0, total - avail)
|
||||
pct = round((used / total) * 100.0, 1) if total > 0 else 0.0
|
||||
return {
|
||||
"total": total,
|
||||
"used": used,
|
||||
"available": avail,
|
||||
"percent": pct,
|
||||
}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def get_swap_metrics() -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Parse /proc/meminfo for Swap metrics.
|
||||
Returns total, used in bytes, and used percent.
|
||||
"""
|
||||
mem_content = _read_proc_file("/proc/meminfo")
|
||||
if mem_content:
|
||||
mem_info: Dict[str, int] = {}
|
||||
for line in mem_content.splitlines():
|
||||
parts = line.split(":")
|
||||
if len(parts) == 2:
|
||||
key = parts[0].strip()
|
||||
val_parts = parts[1].strip().split()
|
||||
if val_parts and val_parts[0].isdigit():
|
||||
mem_info[key] = int(val_parts[0]) * 1024
|
||||
|
||||
if "SwapTotal" in mem_info:
|
||||
total = mem_info["SwapTotal"]
|
||||
free = mem_info.get("SwapFree", 0)
|
||||
used = max(0, total - free)
|
||||
pct = round((used / total) * 100.0, 1) if total > 0 else 0.0
|
||||
return {
|
||||
"total": total,
|
||||
"used": used,
|
||||
"percent": pct,
|
||||
}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def get_disk_metrics(mount_point: str = "/") -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Calculate disk usage for the root filesystem.
|
||||
Includes warning (>= 80%) and critical (>= 90%) threshold flags.
|
||||
"""
|
||||
try:
|
||||
# On Windows during local testing, "/" maps to current drive, but check valid path
|
||||
path = mount_point
|
||||
if os.name == "nt" and mount_point == "/":
|
||||
path = os.environ.get("SystemDrive", "C:") + "\\"
|
||||
|
||||
total, used, free = shutil.disk_usage(path)
|
||||
pct = round((used / total) * 100.0, 1) if total > 0 else 0.0
|
||||
return {
|
||||
"total": total,
|
||||
"used": used,
|
||||
"available": free,
|
||||
"percent": pct,
|
||||
"warning": pct >= 80.0,
|
||||
"critical": pct >= 90.0,
|
||||
}
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def get_uptime() -> Tuple[Optional[int], Optional[str]]:
|
||||
"""
|
||||
Parse /proc/uptime for server uptime.
|
||||
Returns (uptime_seconds, uptime_formatted).
|
||||
"""
|
||||
uptime_content = _read_proc_file("/proc/uptime")
|
||||
if uptime_content:
|
||||
parts = uptime_content.split()
|
||||
if parts:
|
||||
try:
|
||||
seconds = int(float(parts[0]))
|
||||
days = seconds // 86400
|
||||
hours = (seconds % 86400) // 3600
|
||||
minutes = (seconds % 3600) // 60
|
||||
formatted = f"{days}d {hours}h {minutes}m"
|
||||
return seconds, formatted
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return None, None
|
||||
|
||||
|
||||
def get_system_resources() -> Dict[str, Any]:
|
||||
"""Collect overall server system resource metrics."""
|
||||
uptime_sec, uptime_fmt = get_uptime()
|
||||
return {
|
||||
"cpu_percent": get_cpu_percent(),
|
||||
"load_avg": get_load_averages(),
|
||||
"memory": get_memory_metrics(),
|
||||
"swap": get_swap_metrics(),
|
||||
"disk": get_disk_metrics("/"),
|
||||
"uptime_seconds": uptime_sec,
|
||||
"uptime_formatted": uptime_fmt,
|
||||
"collected_at": datetime.now(timezone.utc).isoformat(),
|
||||
}
|
||||
|
||||
|
||||
def get_service_status(service_name: str) -> str:
|
||||
"""
|
||||
Query systemctl is-active for an allowed logical service name.
|
||||
Maps public logical service ID to actual fixed systemd unit (e.g. postgresql -> postgresql@14-main).
|
||||
Strictly forbids arbitrary service names, direct unit overrides, and shell interpolation.
|
||||
"""
|
||||
unit_name = SERVICE_SYSTEMD_UNITS.get(service_name)
|
||||
if not unit_name:
|
||||
return "unknown"
|
||||
|
||||
try:
|
||||
res = subprocess.run(
|
||||
["systemctl", "is-active", unit_name],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=2.0,
|
||||
shell=False,
|
||||
)
|
||||
out = res.stdout.strip().lower()
|
||||
if out == "active":
|
||||
return "active"
|
||||
if out == "inactive":
|
||||
return "inactive"
|
||||
if out == "failed":
|
||||
return "failed"
|
||||
return "unknown"
|
||||
except Exception:
|
||||
return "unknown"
|
||||
|
||||
|
||||
def get_services_health() -> List[Dict[str, Any]]:
|
||||
"""Inspect all allowed core services."""
|
||||
results = []
|
||||
for svc in ALLOWED_SYSTEMD_SERVICES:
|
||||
meta = SERVICE_METADATA.get(svc, {})
|
||||
status = get_service_status(svc)
|
||||
results.append({
|
||||
"name": svc,
|
||||
"display_name": meta.get("display_name", svc),
|
||||
"description": meta.get("description", ""),
|
||||
"status": status,
|
||||
})
|
||||
return results
|
||||
|
||||
|
||||
def get_network_totals() -> Dict[str, Any]:
|
||||
"""
|
||||
Parse /proc/net/dev to sum cumulative received and transmitted bytes
|
||||
across all non-loopback network interfaces since boot.
|
||||
"""
|
||||
dev_content = _read_proc_file("/proc/net/dev")
|
||||
rx_bytes = 0
|
||||
tx_bytes = 0
|
||||
found_any = False
|
||||
|
||||
if dev_content:
|
||||
for line in dev_content.splitlines():
|
||||
if ":" in line:
|
||||
parts = line.split(":", 1)
|
||||
iface = parts[0].strip()
|
||||
if iface == "lo":
|
||||
continue
|
||||
stats = parts[1].split()
|
||||
if len(stats) >= 9:
|
||||
try:
|
||||
rx = int(stats[0])
|
||||
tx = int(stats[8])
|
||||
rx_bytes += rx
|
||||
tx_bytes += tx
|
||||
found_any = True
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return {
|
||||
"bytes_received": rx_bytes if found_any else None,
|
||||
"bytes_transmitted": tx_bytes if found_any else None,
|
||||
"label": "Cumulative Total Since Boot",
|
||||
"collected_at": datetime.now(timezone.utc).isoformat(),
|
||||
}
|
||||
|
||||
|
||||
def get_website_revision(website_url: Optional[str] = None) -> Optional[str]:
|
||||
"""
|
||||
Retrieve deployed website revision from the internal website /api/version endpoint.
|
||||
Parses JSON defensively with a strict response-size bound.
|
||||
Accepts revision only if it matches exactly 40 lowercase or uppercase hexadecimal characters.
|
||||
Never exposes malformed content or connection errors.
|
||||
Preserves graceful degradation if endpoint is absent, unreachable, malformed, or unmerged.
|
||||
Retains environment-variable fallback only for deliberately supported API-service configuration.
|
||||
"""
|
||||
base_url = website_url or os.environ.get("WEBSITE_INTERNAL_URL", DEFAULT_WEBSITE_INTERNAL_URL)
|
||||
endpoint = f"{base_url.rstrip('/')}/api/version"
|
||||
|
||||
try:
|
||||
req = Request(
|
||||
endpoint,
|
||||
headers={
|
||||
"User-Agent": "MotorSignal-Ops-Collector/1.0",
|
||||
"Accept": "application/json",
|
||||
},
|
||||
)
|
||||
with urlopen(req, timeout=1.5) as resp:
|
||||
status = getattr(resp, "status", getattr(resp, "code", None))
|
||||
if status != 200:
|
||||
return None
|
||||
raw_bytes = resp.read(MAX_VERSION_RESPONSE_BYTES + 1)
|
||||
if len(raw_bytes) > MAX_VERSION_RESPONSE_BYTES:
|
||||
# Oversized response, reject defensively
|
||||
return None
|
||||
try:
|
||||
text = raw_bytes.decode("utf-8")
|
||||
data = json.loads(text)
|
||||
except Exception:
|
||||
# Malformed JSON or non-utf8, reject defensively
|
||||
return None
|
||||
|
||||
if isinstance(data, dict):
|
||||
rev = data.get("revision")
|
||||
if isinstance(rev, str) and REVISION_HEX_REGEX.match(rev.strip()):
|
||||
return rev.strip()
|
||||
except Exception:
|
||||
# Never expose connection errors or malformed content
|
||||
pass
|
||||
|
||||
# Deliberately supported API-service configuration fallback
|
||||
for env_var in ("DEPLOYED_REVISION", "GIT_SHA"):
|
||||
env_val = os.environ.get(env_var, "").strip()
|
||||
if env_val and REVISION_HEX_REGEX.match(env_val):
|
||||
return env_val
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def get_deployment_health(
|
||||
database_ok: bool = True,
|
||||
website_url: Optional[str] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Collect deployment runtime health and version foundations.
|
||||
"""
|
||||
target_website_url = website_url or os.environ.get("WEBSITE_INTERNAL_URL", DEFAULT_WEBSITE_INTERNAL_URL)
|
||||
|
||||
# 1. Deployed Git Revision from internal website /api/version
|
||||
git_rev = get_website_revision(target_website_url)
|
||||
short_rev = git_rev[:7] if git_rev else None
|
||||
rev_status = "known" if git_rev else "unavailable"
|
||||
|
||||
# 2. Local Website Health check
|
||||
website_health = "unknown"
|
||||
try:
|
||||
req = Request(
|
||||
f"{target_website_url.rstrip('/')}/api/health",
|
||||
headers={"User-Agent": "MotorSignal-Ops-Collector/1.0"},
|
||||
)
|
||||
with urlopen(req, timeout=1.5) as resp:
|
||||
status = getattr(resp, "status", getattr(resp, "code", None))
|
||||
if status == 200:
|
||||
website_health = "available"
|
||||
elif status and 500 <= status < 600:
|
||||
website_health = "degraded"
|
||||
else:
|
||||
website_health = "available"
|
||||
except HTTPError as e:
|
||||
if 500 <= e.code < 600:
|
||||
website_health = "degraded"
|
||||
elif e.code == 404:
|
||||
# Route missing: fallback to probing root page
|
||||
try:
|
||||
req_root = Request(
|
||||
target_website_url.rstrip('/'),
|
||||
headers={"User-Agent": "MotorSignal-Ops-Collector/1.0"},
|
||||
)
|
||||
with urlopen(req_root, timeout=1.5) as resp_root:
|
||||
status_root = getattr(resp_root, "status", getattr(resp_root, "code", None))
|
||||
if status_root in (200, 301, 302, 307, 308):
|
||||
website_health = "available"
|
||||
else:
|
||||
website_health = "degraded"
|
||||
except HTTPError as e_root:
|
||||
if 500 <= e_root.code < 600:
|
||||
website_health = "degraded"
|
||||
else:
|
||||
website_health = "unavailable"
|
||||
except Exception:
|
||||
website_health = "unavailable"
|
||||
else:
|
||||
website_health = "degraded"
|
||||
except (URLError, TimeoutError):
|
||||
website_health = "unavailable"
|
||||
except Exception:
|
||||
website_health = "unavailable"
|
||||
|
||||
# If git revision was successfully fetched from website and website_health was unknown/unavailable,
|
||||
# mark website available (preserve degraded if explicitly 5xx)
|
||||
if git_rev and website_health in ("unknown", "unavailable"):
|
||||
website_health = "available"
|
||||
|
||||
# 3. API Health
|
||||
api_health = "available" if database_ok else "degraded"
|
||||
|
||||
return {
|
||||
"git_revision": git_rev,
|
||||
"short_revision": short_rev,
|
||||
"revision_status": rev_status,
|
||||
"website_health": website_health,
|
||||
"api_health": api_health,
|
||||
"history_status": "unavailable",
|
||||
"last_successful_deployment": None,
|
||||
}
|
||||
|
||||
|
||||
def get_infrastructure_metrics(database_ok: bool = True) -> Dict[str, Any]:
|
||||
"""Aggregate all read-only infrastructure and deployment metrics."""
|
||||
return {
|
||||
"resources": get_system_resources(),
|
||||
"services": get_services_health(),
|
||||
"network": get_network_totals(),
|
||||
"deployment": get_deployment_health(database_ok=database_ok),
|
||||
"collected_at": datetime.now(timezone.utc).isoformat(),
|
||||
}
|
||||
|
|
@ -46,6 +46,8 @@ from models import (
|
|||
User,
|
||||
Vehicle,
|
||||
)
|
||||
from infrastructure_metrics import get_infrastructure_metrics
|
||||
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
@ -1146,6 +1148,20 @@ def ops_get_health():
|
|||
}
|
||||
|
||||
|
||||
@app.get("/api/v1/ops/infrastructure")
|
||||
def ops_get_infrastructure(
|
||||
auth: tuple[Operator, OpsSession] = Depends(require_ops_roles("operator", "owner")),
|
||||
db: DbSession = Depends(get_db),
|
||||
):
|
||||
db_ok = False
|
||||
try:
|
||||
db_ok = check_database()
|
||||
except Exception:
|
||||
db_ok = False
|
||||
|
||||
return get_infrastructure_metrics(database_ok=db_ok)
|
||||
|
||||
|
||||
@app.get("/api/v1/ops/overview")
|
||||
def ops_get_overview(
|
||||
auth: tuple[Operator, OpsSession] = Depends(get_current_operator),
|
||||
|
|
|
|||
|
|
@ -0,0 +1,528 @@
|
|||
import hashlib
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
import uuid
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from unittest.mock import patch, MagicMock
|
||||
|
||||
ROOT = os.path.dirname(os.path.dirname(__file__))
|
||||
sys.path.insert(0, os.path.join(ROOT, ".python-packages"))
|
||||
sys.path.insert(0, os.path.join(ROOT, "app"))
|
||||
|
||||
os.environ.setdefault("JWT_SECRET", "local-test-secret")
|
||||
os.environ["FIREBASE_PROJECT_ID"] = "motoapp-d22e0"
|
||||
os.environ["DATABASE_USER"] = "test"
|
||||
os.environ["DATABASE_PASSWORD"] = "test"
|
||||
os.environ["DATABASE_NAME"] = "test"
|
||||
|
||||
from sqlalchemy import create_engine, event
|
||||
from sqlalchemy.orm import Session as DbSession
|
||||
from starlette.testclient import TestClient
|
||||
|
||||
import main
|
||||
from main import app, get_db
|
||||
from models import Base, Operator, OpsAuditEvent, OpsSession
|
||||
import infrastructure_metrics
|
||||
from infrastructure_metrics import (
|
||||
ALLOWED_SYSTEMD_SERVICES,
|
||||
DEFAULT_WEBSITE_INTERNAL_URL,
|
||||
MAX_VERSION_RESPONSE_BYTES,
|
||||
REVISION_HEX_REGEX,
|
||||
SERVICE_SYSTEMD_UNITS,
|
||||
get_disk_metrics,
|
||||
get_service_status,
|
||||
get_services_health,
|
||||
get_system_resources,
|
||||
get_network_totals,
|
||||
get_website_revision,
|
||||
get_deployment_health,
|
||||
get_infrastructure_metrics,
|
||||
)
|
||||
|
||||
|
||||
def hash_token(raw_token: str) -> str:
|
||||
return hashlib.sha256(raw_token.encode()).hexdigest()
|
||||
|
||||
|
||||
class TestOpsInfrastructure(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.db_path = os.path.join(ROOT, "tests", "test_ops_infra.db")
|
||||
cls.ops_db_path = os.path.join(ROOT, "tests", "test_ops_infra_schema.db")
|
||||
for p in (cls.db_path, cls.ops_db_path):
|
||||
if os.path.exists(p):
|
||||
try:
|
||||
os.remove(p)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
cls.engine = create_engine(f"sqlite:///{cls.db_path}")
|
||||
cls.seq_val = 0
|
||||
|
||||
@event.listens_for(cls.engine, "connect")
|
||||
def attach_ops(dbapi_con, con_record):
|
||||
dbapi_con.execute(f"ATTACH DATABASE '{cls.ops_db_path}' AS ops")
|
||||
|
||||
def sqlite_nextval(name):
|
||||
cls.seq_val += 1
|
||||
return cls.seq_val
|
||||
|
||||
dbapi_con.create_function("nextval", 1, sqlite_nextval)
|
||||
|
||||
Base.metadata.create_all(cls.engine)
|
||||
|
||||
def override_get_db():
|
||||
db = DbSession(cls.engine)
|
||||
try:
|
||||
yield db
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
app.dependency_overrides[get_db] = override_get_db
|
||||
cls.db_patcher = patch("main.check_database", return_value=True)
|
||||
cls.db_patcher.start()
|
||||
cls.client = TestClient(app, raise_server_exceptions=False)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
cls.db_patcher.stop()
|
||||
app.dependency_overrides.clear()
|
||||
cls.engine.dispose()
|
||||
for p in (cls.db_path, cls.ops_db_path):
|
||||
if os.path.exists(p):
|
||||
try:
|
||||
os.remove(p)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def setUp(self):
|
||||
with DbSession(self.engine) as db:
|
||||
db.query(OpsAuditEvent).delete()
|
||||
db.query(OpsSession).delete()
|
||||
db.query(Operator).delete()
|
||||
db.commit()
|
||||
|
||||
# Seed Operator
|
||||
self.op_id = uuid.uuid4()
|
||||
self.raw_token = f"ops_test_token_{uuid.uuid4().hex}"
|
||||
self.operator = Operator(
|
||||
id=self.op_id,
|
||||
identity="operator@motorsignal.test",
|
||||
role="operator",
|
||||
status="active",
|
||||
created_at=datetime.now(timezone.utc),
|
||||
)
|
||||
self.session = OpsSession(
|
||||
id=uuid.uuid4(),
|
||||
operator_id=self.op_id,
|
||||
session_token_hash=hash_token(self.raw_token),
|
||||
created_at=datetime.now(timezone.utc),
|
||||
last_activity_at=datetime.now(timezone.utc),
|
||||
expires_at=datetime.now(timezone.utc) + timedelta(hours=1),
|
||||
)
|
||||
|
||||
# Seed Owner
|
||||
self.owner_id = uuid.uuid4()
|
||||
self.raw_owner_token = f"ops_owner_token_{uuid.uuid4().hex}"
|
||||
self.owner = Operator(
|
||||
id=self.owner_id,
|
||||
identity="owner@motorsignal.test",
|
||||
role="owner",
|
||||
status="active",
|
||||
created_at=datetime.now(timezone.utc),
|
||||
)
|
||||
self.owner_session = OpsSession(
|
||||
id=uuid.uuid4(),
|
||||
operator_id=self.owner_id,
|
||||
session_token_hash=hash_token(self.raw_owner_token),
|
||||
created_at=datetime.now(timezone.utc),
|
||||
last_activity_at=datetime.now(timezone.utc),
|
||||
expires_at=datetime.now(timezone.utc) + timedelta(hours=1),
|
||||
)
|
||||
|
||||
# Seed Expired Session
|
||||
self.expired_token = "ops_expired_token"
|
||||
self.expired_session = OpsSession(
|
||||
id=uuid.uuid4(),
|
||||
operator_id=self.op_id,
|
||||
session_token_hash=hash_token(self.expired_token),
|
||||
created_at=datetime.now(timezone.utc) - timedelta(hours=3),
|
||||
last_activity_at=datetime.now(timezone.utc) - timedelta(hours=2),
|
||||
expires_at=datetime.now(timezone.utc) - timedelta(hours=1),
|
||||
)
|
||||
|
||||
db.add_all([
|
||||
self.operator, self.session,
|
||||
self.owner, self.owner_session,
|
||||
self.expired_session
|
||||
])
|
||||
db.commit()
|
||||
|
||||
def test_unauthenticated_access_returns_401(self):
|
||||
resp = self.client.get("/api/v1/ops/infrastructure")
|
||||
self.assertEqual(resp.status_code, 401)
|
||||
|
||||
def test_invalid_token_returns_401(self):
|
||||
resp = self.client.get(
|
||||
"/api/v1/ops/infrastructure",
|
||||
headers={"Authorization": "Bearer invalid-token-xyz"},
|
||||
)
|
||||
self.assertEqual(resp.status_code, 401)
|
||||
|
||||
def test_expired_token_returns_401(self):
|
||||
resp = self.client.get(
|
||||
"/api/v1/ops/infrastructure",
|
||||
headers={"Authorization": f"Bearer {self.expired_token}"},
|
||||
)
|
||||
self.assertEqual(resp.status_code, 401)
|
||||
|
||||
@patch("infrastructure_metrics.urlopen")
|
||||
def test_operator_access_returns_200_with_full_structure(self, mock_urlopen):
|
||||
mock_resp = MagicMock()
|
||||
mock_resp.status = 200
|
||||
mock_resp.read.return_value = b'{"status": "ok", "service": "motorsignal-website", "revision": "0123456789abcdef0123456789abcdef01234567", "shortRevision": "0123456789ab"}'
|
||||
mock_resp.__enter__.return_value = mock_resp
|
||||
mock_urlopen.return_value = mock_resp
|
||||
|
||||
resp = self.client.get(
|
||||
"/api/v1/ops/infrastructure",
|
||||
headers={"Authorization": f"Bearer {self.raw_token}"},
|
||||
)
|
||||
self.assertEqual(resp.status_code, 200)
|
||||
data = resp.json()
|
||||
|
||||
# Check top-level keys
|
||||
self.assertIn("resources", data)
|
||||
self.assertIn("services", data)
|
||||
self.assertIn("network", data)
|
||||
self.assertIn("deployment", data)
|
||||
self.assertIn("collected_at", data)
|
||||
|
||||
# Check resources keys
|
||||
res = data["resources"]
|
||||
for k in ("cpu_percent", "load_avg", "memory", "swap", "disk", "uptime_seconds", "uptime_formatted", "collected_at"):
|
||||
self.assertIn(k, res)
|
||||
|
||||
# Check services
|
||||
services = data["services"]
|
||||
self.assertEqual(len(services), 4)
|
||||
service_names = {s["name"] for s in services}
|
||||
self.assertEqual(service_names, set(ALLOWED_SYSTEMD_SERVICES))
|
||||
for s in services:
|
||||
self.assertIn(s["status"], ("active", "inactive", "failed", "unknown"))
|
||||
self.assertTrue(len(s["display_name"]) > 0)
|
||||
|
||||
# Check network
|
||||
net = data["network"]
|
||||
self.assertIn("bytes_received", net)
|
||||
self.assertIn("bytes_transmitted", net)
|
||||
self.assertEqual(net["label"], "Cumulative Total Since Boot")
|
||||
|
||||
# Check deployment
|
||||
dep = data["deployment"]
|
||||
self.assertIn("git_revision", dep)
|
||||
self.assertIn("short_revision", dep)
|
||||
self.assertIn(dep["revision_status"], ("known", "unavailable"))
|
||||
self.assertIn(dep["website_health"], ("available", "degraded", "unavailable", "unknown"))
|
||||
self.assertIn(dep["api_health"], ("available", "degraded", "unavailable", "unknown"))
|
||||
self.assertEqual(dep["history_status"], "unavailable")
|
||||
self.assertIsNone(dep["last_successful_deployment"])
|
||||
|
||||
@patch("infrastructure_metrics.urlopen")
|
||||
def test_owner_access_returns_200(self, mock_urlopen):
|
||||
mock_resp = MagicMock()
|
||||
mock_resp.status = 200
|
||||
mock_resp.read.return_value = b'{"status": "ok", "service": "motorsignal-website", "revision": "0123456789abcdef0123456789abcdef01234567", "shortRevision": "0123456789ab"}'
|
||||
mock_resp.__enter__.return_value = mock_resp
|
||||
mock_urlopen.return_value = mock_resp
|
||||
|
||||
resp = self.client.get(
|
||||
"/api/v1/ops/infrastructure",
|
||||
headers={"Authorization": f"Bearer {self.raw_owner_token}"},
|
||||
)
|
||||
self.assertEqual(resp.status_code, 200)
|
||||
|
||||
|
||||
def test_disk_warning_and_critical_thresholds(self):
|
||||
# 75% -> normal
|
||||
with patch("shutil.disk_usage", return_value=(1000, 750, 250)):
|
||||
m = get_disk_metrics("/")
|
||||
self.assertIsNotNone(m)
|
||||
self.assertEqual(m["percent"], 75.0)
|
||||
self.assertFalse(m["warning"])
|
||||
self.assertFalse(m["critical"])
|
||||
|
||||
# 80% -> warning
|
||||
with patch("shutil.disk_usage", return_value=(1000, 800, 200)):
|
||||
m = get_disk_metrics("/")
|
||||
self.assertIsNotNone(m)
|
||||
self.assertEqual(m["percent"], 80.0)
|
||||
self.assertTrue(m["warning"])
|
||||
self.assertFalse(m["critical"])
|
||||
|
||||
# 89% -> warning
|
||||
with patch("shutil.disk_usage", return_value=(1000, 890, 110)):
|
||||
m = get_disk_metrics("/")
|
||||
self.assertIsNotNone(m)
|
||||
self.assertEqual(m["percent"], 89.0)
|
||||
self.assertTrue(m["warning"])
|
||||
self.assertFalse(m["critical"])
|
||||
|
||||
# 90% -> critical
|
||||
with patch("shutil.disk_usage", return_value=(1000, 900, 100)):
|
||||
m = get_disk_metrics("/")
|
||||
self.assertIsNotNone(m)
|
||||
self.assertEqual(m["percent"], 90.0)
|
||||
self.assertTrue(m["warning"])
|
||||
self.assertTrue(m["critical"])
|
||||
|
||||
# 95% -> critical
|
||||
with patch("shutil.disk_usage", return_value=(1000, 950, 50)):
|
||||
m = get_disk_metrics("/")
|
||||
self.assertIsNotNone(m)
|
||||
self.assertEqual(m["percent"], 95.0)
|
||||
self.assertTrue(m["warning"])
|
||||
self.assertTrue(m["critical"])
|
||||
|
||||
def test_service_allowlist_enforcement(self):
|
||||
with patch("subprocess.run") as mock_sub:
|
||||
# Arbitrary service must return unknown and NOT call subprocess
|
||||
status = get_service_status("arbitrary-service")
|
||||
self.assertEqual(status, "unknown")
|
||||
mock_sub.assert_not_called()
|
||||
|
||||
# Direct systemd unit name that is not an allowed logical service must return unknown and NOT call subprocess
|
||||
status = get_service_status("postgresql@14-main")
|
||||
self.assertEqual(status, "unknown")
|
||||
mock_sub.assert_not_called()
|
||||
|
||||
# Malicious injection attempt must return unknown and NOT call subprocess
|
||||
status = get_service_status("motorsignal-website; rm -rf /")
|
||||
self.assertEqual(status, "unknown")
|
||||
mock_sub.assert_not_called()
|
||||
|
||||
# Allowed service calls subprocess with its fixed mapped unit
|
||||
mock_sub.return_value = MagicMock(returncode=0, stdout="active\n", stderr="")
|
||||
status = get_service_status("motorsignal-website")
|
||||
self.assertEqual(status, "active")
|
||||
mock_sub.assert_called_once_with(
|
||||
["systemctl", "is-active", "motorsignal-website"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=2.0,
|
||||
shell=False,
|
||||
)
|
||||
|
||||
def test_postgresql_unit_mapping_and_api_contract(self):
|
||||
with patch("subprocess.run") as mock_sub:
|
||||
mock_sub.return_value = MagicMock(returncode=0, stdout="active\n", stderr="")
|
||||
|
||||
# 1. Logical postgresql invokes systemctl is-active postgresql@14-main
|
||||
status = get_service_status("postgresql")
|
||||
self.assertEqual(status, "active")
|
||||
mock_sub.assert_called_once_with(
|
||||
["systemctl", "is-active", "postgresql@14-main"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=2.0,
|
||||
shell=False,
|
||||
)
|
||||
|
||||
# 2. get_services_health() still uses public name postgresql
|
||||
services = get_services_health()
|
||||
pg_service = next((s for s in services if s["name"] == "postgresql"), None)
|
||||
self.assertIsNotNone(pg_service)
|
||||
self.assertEqual(pg_service["name"], "postgresql")
|
||||
self.assertEqual(pg_service["display_name"], "PostgreSQL Database")
|
||||
self.assertEqual(pg_service["status"], "active")
|
||||
|
||||
# 3. Ensure no service in get_services_health exposes postgresql@14-main as its name
|
||||
service_names = [s["name"] for s in services]
|
||||
self.assertNotIn("postgresql@14-main", service_names)
|
||||
self.assertIn("postgresql", service_names)
|
||||
|
||||
def test_service_status_normalization(self):
|
||||
with patch("subprocess.run") as mock_sub:
|
||||
# active
|
||||
mock_sub.return_value = MagicMock(returncode=0, stdout="active\n")
|
||||
self.assertEqual(get_service_status("motorsignal-api"), "active")
|
||||
|
||||
# inactive
|
||||
mock_sub.return_value = MagicMock(returncode=3, stdout="inactive\n")
|
||||
self.assertEqual(get_service_status("motorsignal-api"), "inactive")
|
||||
|
||||
# failed
|
||||
mock_sub.return_value = MagicMock(returncode=3, stdout="failed\n")
|
||||
self.assertEqual(get_service_status("motorsignal-api"), "failed")
|
||||
|
||||
# unknown output / timeout
|
||||
mock_sub.return_value = MagicMock(returncode=1, stdout="activating\n")
|
||||
self.assertEqual(get_service_status("motorsignal-api"), "unknown")
|
||||
|
||||
mock_sub.side_effect = Exception("systemctl not found")
|
||||
self.assertEqual(get_service_status("motorsignal-api"), "unknown")
|
||||
|
||||
@patch("infrastructure_metrics.urlopen")
|
||||
def test_website_health_preserves_5xx_as_degraded(self, mock_urlopen):
|
||||
from urllib.error import HTTPError
|
||||
# 1. /api/health returns 500 -> must report degraded without probing / or reporting available
|
||||
mock_urlopen.side_effect = HTTPError("http://127.0.0.1:3001/api/health", 500, "Internal Server Error", {}, None)
|
||||
dep = get_deployment_health(database_ok=True, website_url="http://127.0.0.1:3001")
|
||||
self.assertEqual(dep["website_health"], "degraded")
|
||||
|
||||
# 2. /api/health returns 503 -> degraded
|
||||
mock_urlopen.side_effect = HTTPError("http://127.0.0.1:3001/api/health", 503, "Service Unavailable", {}, None)
|
||||
dep = get_deployment_health(database_ok=True, website_url="http://127.0.0.1:3001")
|
||||
self.assertEqual(dep["website_health"], "degraded")
|
||||
|
||||
# 3. /api/health returns 404 -> fallback to root page (if root is 200 -> available)
|
||||
mock_resp_root = MagicMock()
|
||||
mock_resp_root.status = 200
|
||||
mock_resp_root.__enter__.return_value = mock_resp_root
|
||||
|
||||
def urlopen_side_effect(req, timeout=1.5):
|
||||
url = req.full_url if hasattr(req, "full_url") else str(req)
|
||||
if "/api/health" in url:
|
||||
raise HTTPError(url, 404, "Not Found", {}, None)
|
||||
return mock_resp_root
|
||||
|
||||
mock_urlopen.side_effect = urlopen_side_effect
|
||||
dep = get_deployment_health(database_ok=True, website_url="http://127.0.0.1:3001")
|
||||
self.assertEqual(dep["website_health"], "available")
|
||||
|
||||
@patch("infrastructure_metrics.urlopen")
|
||||
def test_get_website_revision_valid_hex(self, mock_urlopen):
|
||||
# 1. Lowercase 40-char hex
|
||||
mock_resp = MagicMock()
|
||||
mock_resp.status = 200
|
||||
mock_resp.read.return_value = b'{"status":"ok","service":"motorsignal-website","revision":"0123456789abcdef0123456789abcdef01234567"}'
|
||||
mock_resp.__enter__.return_value = mock_resp
|
||||
mock_urlopen.return_value = mock_resp
|
||||
|
||||
rev = get_website_revision("http://127.0.0.1:3001")
|
||||
self.assertEqual(rev, "0123456789abcdef0123456789abcdef01234567")
|
||||
|
||||
# 2. Uppercase 40-char hex
|
||||
mock_resp.read.return_value = b'{"status":"ok","service":"motorsignal-website","revision":"ABCDEF0123456789ABCDEF0123456789ABCDEF01"}'
|
||||
rev_upper = get_website_revision("http://127.0.0.1:3001")
|
||||
self.assertEqual(rev_upper, "ABCDEF0123456789ABCDEF0123456789ABCDEF01")
|
||||
|
||||
@patch("infrastructure_metrics.urlopen")
|
||||
def test_get_website_revision_missing_or_null(self, mock_urlopen):
|
||||
# 1. HTTP 404 / HTTPError
|
||||
from urllib.error import HTTPError
|
||||
mock_urlopen.side_effect = HTTPError("http://127.0.0.1:3001/api/version", 404, "Not Found", {}, None)
|
||||
self.assertIsNone(get_website_revision("http://127.0.0.1:3001"))
|
||||
|
||||
# 2. HTTP 200 but missing revision key
|
||||
mock_resp = MagicMock()
|
||||
mock_resp.status = 200
|
||||
mock_resp.read.return_value = b'{"status":"ok","service":"motorsignal-website"}'
|
||||
mock_resp.__enter__.return_value = mock_resp
|
||||
mock_urlopen.side_effect = None
|
||||
mock_urlopen.return_value = mock_resp
|
||||
self.assertIsNone(get_website_revision("http://127.0.0.1:3001"))
|
||||
|
||||
# 3. HTTP 200 but null revision
|
||||
mock_resp.read.return_value = b'{"status":"ok","service":"motorsignal-website","revision":null}'
|
||||
self.assertIsNone(get_website_revision("http://127.0.0.1:3001"))
|
||||
|
||||
@patch("infrastructure_metrics.urlopen")
|
||||
def test_get_website_revision_malformed_and_invalid(self, mock_urlopen):
|
||||
mock_resp = MagicMock()
|
||||
mock_resp.status = 200
|
||||
mock_resp.__enter__.return_value = mock_resp
|
||||
mock_urlopen.return_value = mock_resp
|
||||
|
||||
# 1. HTML error page (non-JSON)
|
||||
mock_resp.read.return_value = b"<html><head><title>502 Bad Gateway</title></head><body>502</body></html>"
|
||||
self.assertIsNone(get_website_revision("http://127.0.0.1:3001"))
|
||||
|
||||
# 2. Truncated / invalid JSON syntax
|
||||
mock_resp.read.return_value = b'{"status":"ok","revision":'
|
||||
self.assertIsNone(get_website_revision("http://127.0.0.1:3001"))
|
||||
|
||||
# 3. "unknown" placeholder
|
||||
mock_resp.read.return_value = b'{"status":"ok","service":"motorsignal-website","revision":"unknown"}'
|
||||
self.assertIsNone(get_website_revision("http://127.0.0.1:3001"))
|
||||
|
||||
# 4. Short revision (e.g. 12-char short revision instead of 40-char full)
|
||||
mock_resp.read.return_value = b'{"status":"ok","service":"motorsignal-website","revision":"0123456789ab"}'
|
||||
self.assertIsNone(get_website_revision("http://127.0.0.1:3001"))
|
||||
|
||||
# 5. Overlong revision (41 characters)
|
||||
mock_resp.read.return_value = b'{"status":"ok","service":"motorsignal-website","revision":"0123456789abcdef0123456789abcdef012345678"}'
|
||||
self.assertIsNone(get_website_revision("http://127.0.0.1:3001"))
|
||||
|
||||
# 6. Non-hex characters in 40-char string
|
||||
mock_resp.read.return_value = b'{"status":"ok","service":"motorsignal-website","revision":"0123456789abcdef0123456789abcdef0123456g"}'
|
||||
self.assertIsNone(get_website_revision("http://127.0.0.1:3001"))
|
||||
|
||||
@patch("infrastructure_metrics.urlopen")
|
||||
def test_get_website_revision_oversized(self, mock_urlopen):
|
||||
# Payload exceeding MAX_VERSION_RESPONSE_BYTES (4096)
|
||||
oversized_payload = b'{"revision":"' + b"a" * 5000 + b'"}'
|
||||
mock_resp = MagicMock()
|
||||
mock_resp.status = 200
|
||||
mock_resp.read.return_value = oversized_payload
|
||||
mock_resp.__enter__.return_value = mock_resp
|
||||
mock_urlopen.return_value = mock_resp
|
||||
|
||||
self.assertIsNone(get_website_revision("http://127.0.0.1:3001"))
|
||||
|
||||
@patch("infrastructure_metrics.urlopen")
|
||||
def test_get_website_revision_unreachable_and_timeout(self, mock_urlopen):
|
||||
from urllib.error import URLError
|
||||
|
||||
# 1. Connection refused / URLError
|
||||
mock_urlopen.side_effect = URLError("Connection refused")
|
||||
self.assertIsNone(get_website_revision("http://127.0.0.1:3001"))
|
||||
|
||||
# 2. TimeoutError
|
||||
mock_urlopen.side_effect = TimeoutError("Request timed out")
|
||||
self.assertIsNone(get_website_revision("http://127.0.0.1:3001"))
|
||||
|
||||
def test_production_website_listener_verified_port_and_no_url_leak(self):
|
||||
# 1. Default internal port must be 3001
|
||||
self.assertEqual(DEFAULT_WEBSITE_INTERNAL_URL, "http://127.0.0.1:3001")
|
||||
|
||||
# 2. get_deployment_health must not leak internal URL in dictionary
|
||||
dep = get_deployment_health(database_ok=True, website_url="http://127.0.0.1:3001")
|
||||
for key in dep.keys():
|
||||
self.assertNotIn("url", key.lower())
|
||||
self.assertNotIn("3001", str(dep[key]))
|
||||
self.assertNotIn("127.0.0.1", str(dep[key]))
|
||||
|
||||
# 3. get_infrastructure_metrics must not leak internal URL in dictionary
|
||||
infra = get_infrastructure_metrics(database_ok=True)
|
||||
dep_in_infra = infra["deployment"]
|
||||
for key in dep_in_infra.keys():
|
||||
self.assertNotIn("url", key.lower())
|
||||
self.assertNotIn("3001", str(dep_in_infra[key]))
|
||||
self.assertNotIn("127.0.0.1", str(dep_in_infra[key]))
|
||||
|
||||
@patch("infrastructure_metrics.urlopen")
|
||||
def test_environment_variable_fallback_deliberate_config(self, mock_urlopen):
|
||||
# Endpoint unreachable
|
||||
from urllib.error import URLError
|
||||
mock_urlopen.side_effect = URLError("Endpoint not found")
|
||||
|
||||
valid_sha = "0123456789abcdef0123456789abcdef01234567"
|
||||
|
||||
# 1. Fallback to DEPLOYED_REVISION if set and valid
|
||||
with patch.dict(os.environ, {"DEPLOYED_REVISION": valid_sha}, clear=False):
|
||||
self.assertEqual(get_website_revision(), valid_sha)
|
||||
|
||||
# 2. Fallback to GIT_SHA if set and valid
|
||||
with patch.dict(os.environ, {"GIT_SHA": valid_sha, "DEPLOYED_REVISION": ""}, clear=False):
|
||||
self.assertEqual(get_website_revision(), valid_sha)
|
||||
|
||||
# 3. Invalid env var rejected
|
||||
with patch.dict(os.environ, {"DEPLOYED_REVISION": "invalid-sha-123", "GIT_SHA": ""}, clear=False):
|
||||
self.assertIsNone(get_website_revision())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
@ -0,0 +1,65 @@
|
|||
import { cookies } from 'next/headers';
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { normalizeInfrastructureResponse } from '../../../../lib/ops-data';
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
const cookieStore = await cookies();
|
||||
const isProduction =
|
||||
process.env.NODE_ENV === 'production' &&
|
||||
process.env.NEXT_PUBLIC_APP_ENV === 'production';
|
||||
const opsToken = isProduction
|
||||
? cookieStore.get('__Host-motorsignal_ops')?.value
|
||||
: cookieStore.get('__Host-motorsignal_ops')?.value ??
|
||||
cookieStore.get('motorsignal_ops')?.value;
|
||||
|
||||
if (!opsToken) {
|
||||
return NextResponse.json(
|
||||
{ detail: 'Authentication required.' },
|
||||
{ status: 401 }
|
||||
);
|
||||
}
|
||||
|
||||
const baseUrl =
|
||||
process.env.MOTORSIGNAL_API_BASE_URL ?? 'https://api.motorsignal.app';
|
||||
const headers: Record<string, string> = {
|
||||
accept: 'application/json',
|
||||
authorization: `Bearer ${opsToken}`,
|
||||
};
|
||||
|
||||
try {
|
||||
const response = await fetch(`${baseUrl}/api/v1/ops/infrastructure`, {
|
||||
method: 'GET',
|
||||
headers,
|
||||
cache: 'no-store',
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
if (response.status === 401) {
|
||||
return NextResponse.json(
|
||||
{ detail: 'Authentication required.' },
|
||||
{ status: 401 }
|
||||
);
|
||||
}
|
||||
if (response.status === 403) {
|
||||
return NextResponse.json(
|
||||
{ detail: 'Access restricted.' },
|
||||
{ status: 403 }
|
||||
);
|
||||
}
|
||||
return NextResponse.json(
|
||||
normalizeInfrastructureResponse(null),
|
||||
{ status: 503 }
|
||||
);
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
return NextResponse.json(normalizeInfrastructureResponse(data), {
|
||||
status: 200,
|
||||
});
|
||||
} catch {
|
||||
return NextResponse.json(
|
||||
normalizeInfrastructureResponse(null),
|
||||
{ status: 503 }
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,618 @@
|
|||
'use client';
|
||||
|
||||
import React, { useState, useEffect, useRef, useCallback } from 'react';
|
||||
import {
|
||||
OperationsInfrastructureData,
|
||||
ServiceStatusValue,
|
||||
ComponentHealthStatus,
|
||||
OperationsRole,
|
||||
} from './types';
|
||||
import { normalizeInfrastructureResponse } from '../../lib/ops-data';
|
||||
|
||||
interface OperationsInfrastructureProps {
|
||||
role: OperationsRole;
|
||||
}
|
||||
|
||||
export function OperationsInfrastructure({ role }: OperationsInfrastructureProps) {
|
||||
const [data, setData] = useState<OperationsInfrastructureData | null>(null);
|
||||
const [isRefreshing, setIsRefreshing] = useState(false);
|
||||
const [lastFetchedAt, setLastFetchedAt] = useState<string | null>(null);
|
||||
const [fetchError, setFetchError] = useState<string | null>(null);
|
||||
const [copiedSha, setCopiedSha] = useState(false);
|
||||
|
||||
const isFetchingRef = useRef(false);
|
||||
const pollTimerRef = useRef<NodeJS.Timeout | null>(null);
|
||||
|
||||
const fetchData = useCallback(async () => {
|
||||
if (isFetchingRef.current) return;
|
||||
isFetchingRef.current = true;
|
||||
setIsRefreshing(true);
|
||||
|
||||
try {
|
||||
const resp = await fetch('/api/ops/infrastructure', {
|
||||
method: 'GET',
|
||||
headers: { accept: 'application/json' },
|
||||
cache: 'no-store',
|
||||
});
|
||||
|
||||
if (resp.ok) {
|
||||
const raw = await resp.json();
|
||||
const normalized = normalizeInfrastructureResponse(raw);
|
||||
setData(normalized);
|
||||
setLastFetchedAt(new Date().toLocaleTimeString());
|
||||
setFetchError(null);
|
||||
} else if (resp.status === 401 || resp.status === 403) {
|
||||
setFetchError('Authorization failed. Access restricted.');
|
||||
} else {
|
||||
const raw = await resp.json().catch(() => null);
|
||||
setData(normalizeInfrastructureResponse(raw));
|
||||
setFetchError('Metrics collector degraded or partially unavailable.');
|
||||
}
|
||||
} catch {
|
||||
setFetchError('Network error reaching infrastructure endpoint.');
|
||||
} finally {
|
||||
isFetchingRef.current = false;
|
||||
setIsRefreshing(false);
|
||||
}
|
||||
}, []);
|
||||
|
||||
// Initial fetch and 30-second visibility-gated polling loop
|
||||
useEffect(() => {
|
||||
void fetchData();
|
||||
|
||||
const handleVisibilityChange = () => {
|
||||
if (document.visibilityState === 'visible') {
|
||||
void fetchData();
|
||||
}
|
||||
};
|
||||
|
||||
document.addEventListener('visibilitychange', handleVisibilityChange);
|
||||
|
||||
pollTimerRef.current = setInterval(() => {
|
||||
if (document.visibilityState === 'visible') {
|
||||
void fetchData();
|
||||
}
|
||||
}, 30000);
|
||||
|
||||
return () => {
|
||||
document.removeEventListener('visibilitychange', handleVisibilityChange);
|
||||
if (pollTimerRef.current) {
|
||||
clearInterval(pollTimerRef.current);
|
||||
}
|
||||
};
|
||||
}, [fetchData]);
|
||||
|
||||
const handleCopySha = (sha: string) => {
|
||||
if (!sha) return;
|
||||
navigator.clipboard.writeText(sha).then(() => {
|
||||
setCopiedSha(true);
|
||||
setTimeout(() => setCopiedSha(false), 2000);
|
||||
});
|
||||
};
|
||||
|
||||
const formatBytes = (bytes: number | null): string => {
|
||||
if (bytes === null || bytes === undefined) return 'N/A';
|
||||
if (bytes >= 1e12) return `${(bytes / 1e12).toFixed(2)} TB`;
|
||||
if (bytes >= 1e9) return `${(bytes / 1e9).toFixed(2)} GB`;
|
||||
if (bytes >= 1e6) return `${(bytes / 1e6).toFixed(1)} MB`;
|
||||
return `${(bytes / 1e3).toFixed(1)} KB`;
|
||||
};
|
||||
|
||||
const getServiceStatusBadge = (status: ServiceStatusValue) => {
|
||||
switch (status) {
|
||||
case 'active':
|
||||
return {
|
||||
label: 'Active (Running)',
|
||||
icon: 'check_circle',
|
||||
classes: 'bg-emerald-950/60 text-emerald-400 border-emerald-800/60',
|
||||
};
|
||||
case 'inactive':
|
||||
return {
|
||||
label: 'Inactive (Stopped)',
|
||||
icon: 'pause_circle',
|
||||
classes: 'bg-amber-950/60 text-amber-400 border-amber-800/60',
|
||||
};
|
||||
case 'failed':
|
||||
return {
|
||||
label: 'Failed',
|
||||
icon: 'cancel',
|
||||
classes: 'bg-rose-950/60 text-rose-400 border-rose-800/60',
|
||||
};
|
||||
case 'unknown':
|
||||
default:
|
||||
return {
|
||||
label: 'Unknown / Unreachable',
|
||||
icon: 'help',
|
||||
classes: 'bg-slate-900/60 text-slate-400 border-slate-700/60',
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
const getComponentHealthBadge = (status: ComponentHealthStatus) => {
|
||||
switch (status) {
|
||||
case 'available':
|
||||
return {
|
||||
label: 'Available',
|
||||
icon: 'check_circle',
|
||||
classes: 'bg-emerald-950/60 text-emerald-400 border-emerald-800/60',
|
||||
};
|
||||
case 'degraded':
|
||||
return {
|
||||
label: 'Degraded',
|
||||
icon: 'warning',
|
||||
classes: 'bg-amber-950/60 text-amber-400 border-amber-800/60',
|
||||
};
|
||||
case 'unavailable':
|
||||
return {
|
||||
label: 'Unavailable',
|
||||
icon: 'cancel',
|
||||
classes: 'bg-rose-950/60 text-rose-400 border-rose-800/60',
|
||||
};
|
||||
case 'unknown':
|
||||
default:
|
||||
return {
|
||||
label: 'Unknown',
|
||||
icon: 'help',
|
||||
classes: 'bg-slate-900/60 text-slate-400 border-slate-700/60',
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
const resources = data?.resources;
|
||||
const services = data?.services ?? [];
|
||||
const network = data?.network;
|
||||
const deployment = data?.deployment;
|
||||
|
||||
return (
|
||||
<div className="space-y-8">
|
||||
{/* Header & Controls */}
|
||||
<div className="flex flex-col sm:flex-row sm:items-center justify-between gap-4 pb-6 border-b border-outline-variant/20">
|
||||
<div>
|
||||
<div className="flex items-center gap-2 mb-1">
|
||||
<span className="material-symbols-outlined text-primary-container text-2xl">
|
||||
dns
|
||||
</span>
|
||||
<h1 className="text-xl font-bold tracking-tight text-on-surface">
|
||||
Infrastructure & Runtime
|
||||
</h1>
|
||||
<span className="px-2 py-0.5 rounded text-[11px] font-mono bg-surface-container-high border border-outline-variant/30 text-primary-fixed-dim">
|
||||
Phase 1 (Read-Only)
|
||||
</span>
|
||||
</div>
|
||||
<p className="text-xs text-on-surface-variant">
|
||||
Host system resources, systemd core services, network totals, and deployment runtime health.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div className="flex items-center gap-3">
|
||||
{/* Polling indicator */}
|
||||
<div className="flex items-center gap-1.5 text-[11px] text-on-surface-variant font-mono bg-surface-container-low px-2.5 py-1.5 rounded-lg border border-outline-variant/20">
|
||||
<span className="w-2 h-2 rounded-full bg-emerald-400 animate-pulse" />
|
||||
<span>Auto 30s</span>
|
||||
</div>
|
||||
|
||||
{/* Last refreshed */}
|
||||
{lastFetchedAt && (
|
||||
<span className="text-[11px] text-outline font-mono hidden md:inline">
|
||||
Updated: {lastFetchedAt}
|
||||
</span>
|
||||
)}
|
||||
|
||||
{/* Manual Refresh Button */}
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => void fetchData()}
|
||||
disabled={isRefreshing}
|
||||
className="flex items-center gap-2 px-3 py-1.5 rounded-lg text-xs font-semibold bg-surface-container-high hover:bg-surface-container-highest border border-outline-variant/40 text-on-surface disabled:opacity-50 transition-all focus-visible:ring-2 focus-visible:ring-primary-container"
|
||||
>
|
||||
<span
|
||||
className={`material-symbols-outlined text-[16px] ${
|
||||
isRefreshing ? 'animate-spin' : ''
|
||||
}`}
|
||||
>
|
||||
refresh
|
||||
</span>
|
||||
<span>{isRefreshing ? 'Refreshing...' : 'Refresh'}</span>
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Optional Warning Banner */}
|
||||
{fetchError && (
|
||||
<div className="p-3.5 rounded-lg bg-amber-950/40 border border-amber-800/50 flex items-center gap-2.5 text-xs text-amber-300">
|
||||
<span className="material-symbols-outlined text-amber-400 text-base">
|
||||
warning
|
||||
</span>
|
||||
<span>{fetchError}</span>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* SECTION 1: System Resources */}
|
||||
<div>
|
||||
<div className="flex items-center justify-between mb-3">
|
||||
<h2 className="text-xs font-bold uppercase tracking-wider text-outline">
|
||||
Server Resources
|
||||
</h2>
|
||||
<span className="text-[10px] text-outline font-mono">
|
||||
Target Host: Ubuntu VPS
|
||||
</span>
|
||||
</div>
|
||||
|
||||
<div className="grid grid-cols-1 sm:grid-cols-2 lg:grid-cols-5 gap-4">
|
||||
{/* CPU Usage */}
|
||||
<div className="p-4 rounded-xl bg-surface-container-low border border-outline-variant/20 space-y-3">
|
||||
<div className="flex items-center justify-between">
|
||||
<span className="text-xs font-medium text-on-surface-variant">CPU Usage</span>
|
||||
<span className="material-symbols-outlined text-base text-primary-fixed-dim">
|
||||
memory
|
||||
</span>
|
||||
</div>
|
||||
<div>
|
||||
<div className="text-2xl font-bold font-mono tracking-tight text-on-surface">
|
||||
{resources?.cpu_percent !== null && resources?.cpu_percent !== undefined
|
||||
? `${resources.cpu_percent}%`
|
||||
: 'Unavailable'}
|
||||
</div>
|
||||
<p className="text-[11px] text-outline font-mono mt-0.5">
|
||||
Load: {resources?.load_avg ? resources.load_avg.join(', ') : 'N/A'}
|
||||
</p>
|
||||
</div>
|
||||
{/* Progress Bar */}
|
||||
<div className="w-full bg-surface-container-highest h-1.5 rounded-full overflow-hidden">
|
||||
<div
|
||||
className="bg-primary-container h-full rounded-full transition-all duration-300"
|
||||
style={{ width: `${Math.min(100, Math.max(0, resources?.cpu_percent ?? 0))}%` }}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Memory (RAM) */}
|
||||
<div className="p-4 rounded-xl bg-surface-container-low border border-outline-variant/20 space-y-3">
|
||||
<div className="flex items-center justify-between">
|
||||
<span className="text-xs font-medium text-on-surface-variant">Memory (RAM)</span>
|
||||
<span className="material-symbols-outlined text-base text-primary-fixed-dim">
|
||||
developer_board
|
||||
</span>
|
||||
</div>
|
||||
<div>
|
||||
<div className="text-xl font-bold font-mono tracking-tight text-on-surface">
|
||||
{resources?.memory
|
||||
? `${formatBytes(resources.memory.used)} / ${formatBytes(resources.memory.total)}`
|
||||
: 'Unavailable'}
|
||||
</div>
|
||||
<p className="text-[11px] text-outline font-mono mt-0.5">
|
||||
{resources?.memory
|
||||
? `${resources.memory.percent}% used (${formatBytes(resources.memory.available)} free)`
|
||||
: 'N/A'}
|
||||
</p>
|
||||
</div>
|
||||
{/* Progress Bar */}
|
||||
<div className="w-full bg-surface-container-highest h-1.5 rounded-full overflow-hidden">
|
||||
<div
|
||||
className={`h-full rounded-full transition-all duration-300 ${
|
||||
(resources?.memory?.percent ?? 0) >= 90
|
||||
? 'bg-rose-500'
|
||||
: (resources?.memory?.percent ?? 0) >= 80
|
||||
? 'bg-amber-500'
|
||||
: 'bg-primary-container'
|
||||
}`}
|
||||
style={{ width: `${Math.min(100, Math.max(0, resources?.memory?.percent ?? 0))}%` }}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Swap Memory */}
|
||||
<div className="p-4 rounded-xl bg-surface-container-low border border-outline-variant/20 space-y-3">
|
||||
<div className="flex items-center justify-between">
|
||||
<span className="text-xs font-medium text-on-surface-variant">Swap Space</span>
|
||||
<span className="material-symbols-outlined text-base text-primary-fixed-dim">
|
||||
swap_horiz
|
||||
</span>
|
||||
</div>
|
||||
<div>
|
||||
<div className="text-xl font-bold font-mono tracking-tight text-on-surface">
|
||||
{resources?.swap
|
||||
? `${formatBytes(resources.swap.used)} / ${formatBytes(resources.swap.total)}`
|
||||
: 'Unavailable'}
|
||||
</div>
|
||||
<p className="text-[11px] text-outline font-mono mt-0.5">
|
||||
{resources?.swap ? `${resources.swap.percent}% utilized` : 'N/A'}
|
||||
</p>
|
||||
</div>
|
||||
{/* Progress Bar */}
|
||||
<div className="w-full bg-surface-container-highest h-1.5 rounded-full overflow-hidden">
|
||||
<div
|
||||
className="bg-primary-container/70 h-full rounded-full transition-all duration-300"
|
||||
style={{ width: `${Math.min(100, Math.max(0, resources?.swap?.percent ?? 0))}%` }}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Disk Storage (/) */}
|
||||
<div className="p-4 rounded-xl bg-surface-container-low border border-outline-variant/20 space-y-3">
|
||||
<div className="flex items-center justify-between">
|
||||
<div className="flex items-center gap-1.5">
|
||||
<span className="text-xs font-medium text-on-surface-variant">Root Disk (/)</span>
|
||||
{resources?.disk?.critical ? (
|
||||
<span className="px-1.5 py-0.2 rounded text-[9px] font-mono font-bold bg-rose-950 text-rose-300 border border-rose-800">
|
||||
CRITICAL
|
||||
</span>
|
||||
) : resources?.disk?.warning ? (
|
||||
<span className="px-1.5 py-0.2 rounded text-[9px] font-mono font-bold bg-amber-950 text-amber-300 border border-amber-800">
|
||||
WARNING
|
||||
</span>
|
||||
) : null}
|
||||
</div>
|
||||
<span className="material-symbols-outlined text-base text-primary-fixed-dim">
|
||||
hard_drive
|
||||
</span>
|
||||
</div>
|
||||
<div>
|
||||
<div className="text-xl font-bold font-mono tracking-tight text-on-surface">
|
||||
{resources?.disk
|
||||
? `${formatBytes(resources.disk.used)} / ${formatBytes(resources.disk.total)}`
|
||||
: 'Unavailable'}
|
||||
</div>
|
||||
<p className="text-[11px] text-outline font-mono mt-0.5">
|
||||
{resources?.disk
|
||||
? `${resources.disk.percent}% used (${formatBytes(resources.disk.available)} free)`
|
||||
: 'N/A'}
|
||||
</p>
|
||||
</div>
|
||||
{/* Progress Bar */}
|
||||
<div className="w-full bg-surface-container-highest h-1.5 rounded-full overflow-hidden">
|
||||
<div
|
||||
className={`h-full rounded-full transition-all duration-300 ${
|
||||
resources?.disk?.critical
|
||||
? 'bg-rose-500'
|
||||
: resources?.disk?.warning
|
||||
? 'bg-amber-500'
|
||||
: 'bg-primary-container'
|
||||
}`}
|
||||
style={{ width: `${Math.min(100, Math.max(0, resources?.disk?.percent ?? 0))}%` }}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Server Uptime */}
|
||||
<div className="p-4 rounded-xl bg-surface-container-low border border-outline-variant/20 space-y-3">
|
||||
<div className="flex items-center justify-between">
|
||||
<span className="text-xs font-medium text-on-surface-variant">Server Uptime</span>
|
||||
<span className="material-symbols-outlined text-base text-primary-fixed-dim">
|
||||
schedule
|
||||
</span>
|
||||
</div>
|
||||
<div>
|
||||
<div className="text-lg font-bold font-mono tracking-tight text-on-surface truncate">
|
||||
{resources?.uptime_formatted || 'Unavailable'}
|
||||
</div>
|
||||
<p className="text-[11px] text-outline font-mono mt-0.5">
|
||||
{resources?.uptime_seconds
|
||||
? `${resources.uptime_seconds.toLocaleString()}s total uptime`
|
||||
: 'N/A'}
|
||||
</p>
|
||||
</div>
|
||||
<div className="text-[10px] text-primary-fixed-dim font-mono">
|
||||
System running continuously
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* SECTION 2: Core Systemd Services */}
|
||||
<div>
|
||||
<div className="flex items-center justify-between mb-3">
|
||||
<h2 className="text-xs font-bold uppercase tracking-wider text-outline">
|
||||
Core Services (systemd)
|
||||
</h2>
|
||||
<span className="text-[10px] text-outline font-mono">
|
||||
Strict Allowlist: 4 Services
|
||||
</span>
|
||||
</div>
|
||||
|
||||
<div className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-4 gap-4">
|
||||
{services.map((svc) => {
|
||||
const badge = getServiceStatusBadge(svc.status);
|
||||
return (
|
||||
<div
|
||||
key={svc.name}
|
||||
className="p-4 rounded-xl bg-surface-container-low border border-outline-variant/20 flex flex-col justify-between space-y-3"
|
||||
>
|
||||
<div>
|
||||
<div className="flex items-start justify-between gap-2">
|
||||
<div>
|
||||
<h3 className="text-sm font-semibold text-on-surface">
|
||||
{svc.display_name}
|
||||
</h3>
|
||||
<p className="text-[11px] font-mono text-outline mt-0.5">
|
||||
{svc.name}.service
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
<p className="text-xs text-on-surface-variant mt-2 leading-relaxed">
|
||||
{svc.description}
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div className="pt-3 border-t border-outline-variant/10 flex items-center justify-between">
|
||||
<span className="text-[11px] text-outline">Service Status</span>
|
||||
<div
|
||||
className={`inline-flex items-center gap-1.5 px-2.5 py-1 rounded-full text-[11px] font-mono font-medium border ${badge.classes}`}
|
||||
>
|
||||
<span className="material-symbols-outlined text-[14px]">
|
||||
{badge.icon}
|
||||
</span>
|
||||
<span>{badge.label}</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* SECTION 3 & 4: Network Cumulative Totals & Deployment Health */}
|
||||
<div className="grid grid-cols-1 lg:grid-cols-2 gap-6">
|
||||
{/* Network Cumulative Totals */}
|
||||
<div className="p-5 rounded-xl bg-surface-container-low border border-outline-variant/20 space-y-4">
|
||||
<div className="flex items-center justify-between">
|
||||
<div className="flex items-center gap-2">
|
||||
<span className="material-symbols-outlined text-primary-container text-xl">
|
||||
network_check
|
||||
</span>
|
||||
<div>
|
||||
<h3 className="text-sm font-bold text-on-surface">
|
||||
Network Cumulative Totals
|
||||
</h3>
|
||||
<p className="text-[11px] text-outline">
|
||||
Aggregated across non-loopback interfaces since host boot
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
<span className="px-2 py-0.5 rounded text-[10px] font-mono bg-surface-container-high text-primary-fixed-dim border border-outline-variant/30">
|
||||
Not a rate
|
||||
</span>
|
||||
</div>
|
||||
|
||||
<div className="grid grid-cols-2 gap-4 pt-2">
|
||||
<div className="p-3.5 rounded-lg bg-surface-container-lowest border border-outline-variant/15 space-y-1">
|
||||
<div className="flex items-center gap-1.5 text-xs text-on-surface-variant">
|
||||
<span className="material-symbols-outlined text-[15px] text-emerald-400">
|
||||
download
|
||||
</span>
|
||||
<span>Total Received</span>
|
||||
</div>
|
||||
<div className="text-xl font-bold font-mono text-on-surface">
|
||||
{formatBytes(network?.bytes_received ?? null)}
|
||||
</div>
|
||||
<p className="text-[10px] text-outline font-mono">
|
||||
{network?.bytes_received ? `${network.bytes_received.toLocaleString()} bytes` : 'N/A'}
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div className="p-3.5 rounded-lg bg-surface-container-lowest border border-outline-variant/15 space-y-1">
|
||||
<div className="flex items-center gap-1.5 text-xs text-on-surface-variant">
|
||||
<span className="material-symbols-outlined text-[15px] text-primary-container">
|
||||
upload
|
||||
</span>
|
||||
<span>Total Transmitted</span>
|
||||
</div>
|
||||
<div className="text-xl font-bold font-mono text-on-surface">
|
||||
{formatBytes(network?.bytes_transmitted ?? null)}
|
||||
</div>
|
||||
<p className="text-[10px] text-outline font-mono">
|
||||
{network?.bytes_transmitted ? `${network.bytes_transmitted.toLocaleString()} bytes` : 'N/A'}
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="p-2.5 rounded-lg bg-surface-container-lowest/60 border border-outline-variant/10 text-[11px] text-outline flex items-center gap-2">
|
||||
<span className="material-symbols-outlined text-[14px] text-primary-fixed-dim">
|
||||
info
|
||||
</span>
|
||||
<span>
|
||||
Cumulative Total Since Boot across physical network interfaces. Instantaneous bandwidth rates are not tracked.
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Deployment Health Foundations */}
|
||||
<div className="p-5 rounded-xl bg-surface-container-low border border-outline-variant/20 space-y-4">
|
||||
<div className="flex items-center justify-between">
|
||||
<div className="flex items-center gap-2">
|
||||
<span className="material-symbols-outlined text-primary-container text-xl">
|
||||
deployed_code
|
||||
</span>
|
||||
<div>
|
||||
<h3 className="text-sm font-bold text-on-surface">
|
||||
Deployment Health Foundations
|
||||
</h3>
|
||||
<p className="text-[11px] text-outline">
|
||||
Active code revision and subsystem runtime readiness
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
<span className="px-2 py-0.5 rounded text-[10px] font-mono bg-surface-container-high text-on-surface-variant border border-outline-variant/30">
|
||||
Foundation
|
||||
</span>
|
||||
</div>
|
||||
|
||||
<div className="space-y-3 pt-1">
|
||||
{/* Active Git Revision */}
|
||||
<div className="p-3 rounded-lg bg-surface-container-lowest border border-outline-variant/15 flex items-center justify-between">
|
||||
<div>
|
||||
<div className="text-xs text-on-surface-variant">Active Git Revision</div>
|
||||
<div className="flex items-center gap-2 mt-1">
|
||||
<span className="font-mono text-sm font-bold text-on-surface">
|
||||
{deployment?.short_revision || 'Unavailable'}
|
||||
</span>
|
||||
{deployment?.git_revision && (
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => handleCopySha(deployment.git_revision!)}
|
||||
title="Copy full SHA"
|
||||
className="p-1 rounded text-outline hover:text-on-surface hover:bg-surface-container transition-colors"
|
||||
>
|
||||
<span className="material-symbols-outlined text-[14px]">
|
||||
{copiedSha ? 'check' : 'content_copy'}
|
||||
</span>
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
<div className="flex items-center gap-1.5 font-mono text-[11px]">
|
||||
<span
|
||||
className={`w-2 h-2 rounded-full ${
|
||||
deployment?.revision_status === 'known' ? 'bg-emerald-400' : 'bg-slate-500'
|
||||
}`}
|
||||
/>
|
||||
<span className="capitalize text-outline">
|
||||
{deployment?.revision_status ?? 'unavailable'}
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Health indicators */}
|
||||
<div className="grid grid-cols-2 gap-3">
|
||||
<div className="p-3 rounded-lg bg-surface-container-lowest border border-outline-variant/15">
|
||||
<div className="text-xs text-on-surface-variant mb-1">Website Ingress</div>
|
||||
{(() => {
|
||||
const badge = getComponentHealthBadge(deployment?.website_health ?? 'unknown');
|
||||
return (
|
||||
<div
|
||||
className={`inline-flex items-center gap-1.5 px-2 py-0.5 rounded-full text-[10px] font-mono border ${badge.classes}`}
|
||||
>
|
||||
<span className="material-symbols-outlined text-[12px]">{badge.icon}</span>
|
||||
<span>{badge.label}</span>
|
||||
</div>
|
||||
);
|
||||
})()}
|
||||
</div>
|
||||
|
||||
<div className="p-3 rounded-lg bg-surface-container-lowest border border-outline-variant/15">
|
||||
<div className="text-xs text-on-surface-variant mb-1">API Readiness</div>
|
||||
{(() => {
|
||||
const badge = getComponentHealthBadge(deployment?.api_health ?? 'unknown');
|
||||
return (
|
||||
<div
|
||||
className={`inline-flex items-center gap-1.5 px-2 py-0.5 rounded-full text-[10px] font-mono border ${badge.classes}`}
|
||||
>
|
||||
<span className="material-symbols-outlined text-[12px]">{badge.icon}</span>
|
||||
<span>{badge.label}</span>
|
||||
</div>
|
||||
);
|
||||
})()}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Deployment History Status */}
|
||||
<div className="p-2.5 rounded-lg bg-surface-container-lowest/60 border border-outline-variant/10 text-[11px] text-outline flex items-center justify-between">
|
||||
<span className="font-mono">Deployment History:</span>
|
||||
<span className="text-outline italic">Unavailable in Phase 1</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -20,6 +20,8 @@ import { OperationsIncidentDetail } from './operations-incident-detail';
|
|||
import { OperationsUsers } from './operations-users';
|
||||
import { OperationsSupport } from './operations-support';
|
||||
import { OperationsCms } from './operations-cms';
|
||||
import { OperationsInfrastructure } from './operations-infrastructure';
|
||||
|
||||
|
||||
interface OperationsShellProps {
|
||||
initialSession: OperatorSession;
|
||||
|
|
@ -223,6 +225,10 @@ export function OperationsShell({
|
|||
/>
|
||||
)}
|
||||
|
||||
{activeNav === 'infrastructure' && (
|
||||
<OperationsInfrastructure role={session.role} />
|
||||
)}
|
||||
|
||||
{activeNav === 'users' && (
|
||||
<OperationsUsers role={session.role} />
|
||||
)}
|
||||
|
|
|
|||
|
|
@ -32,6 +32,11 @@ export function OperationsSidebar({
|
|||
label: 'Overview',
|
||||
icon: 'dashboard',
|
||||
},
|
||||
{
|
||||
id: 'infrastructure',
|
||||
label: 'Infrastructure',
|
||||
icon: 'dns',
|
||||
},
|
||||
{
|
||||
id: 'users',
|
||||
label: 'Users',
|
||||
|
|
|
|||
|
|
@ -52,6 +52,7 @@ export interface OperatorSession {
|
|||
|
||||
export type OperationsNavId =
|
||||
| 'overview'
|
||||
| 'infrastructure'
|
||||
| 'users'
|
||||
| 'incidents'
|
||||
| 'support'
|
||||
|
|
@ -60,6 +61,7 @@ export type OperationsNavId =
|
|||
| 'operators'
|
||||
| 'session';
|
||||
|
||||
|
||||
export type IncidentSeverity = 'low' | 'medium' | 'high' | 'critical';
|
||||
|
||||
export type IncidentStatus =
|
||||
|
|
@ -320,4 +322,71 @@ export interface CmsDocumentDetail {
|
|||
revisions: CmsRevisionItem[];
|
||||
}
|
||||
|
||||
export type ServiceStatusValue = 'active' | 'inactive' | 'failed' | 'unknown';
|
||||
|
||||
export interface ServiceHealthItem {
|
||||
name: string;
|
||||
display_name: string;
|
||||
status: ServiceStatusValue;
|
||||
description: string;
|
||||
}
|
||||
|
||||
export interface SystemResourceMemory {
|
||||
total: number;
|
||||
used: number;
|
||||
available: number;
|
||||
percent: number;
|
||||
}
|
||||
|
||||
export interface SystemResourceSwap {
|
||||
total: number;
|
||||
used: number;
|
||||
percent: number;
|
||||
}
|
||||
|
||||
export interface SystemResourceDisk {
|
||||
total: number;
|
||||
used: number;
|
||||
available: number;
|
||||
percent: number;
|
||||
warning: boolean;
|
||||
critical: boolean;
|
||||
}
|
||||
|
||||
export interface SystemResources {
|
||||
cpu_percent: number | null;
|
||||
load_avg: [number, number, number] | null;
|
||||
memory: SystemResourceMemory | null;
|
||||
swap: SystemResourceSwap | null;
|
||||
disk: SystemResourceDisk | null;
|
||||
uptime_seconds: number | null;
|
||||
uptime_formatted: string | null;
|
||||
collected_at: string;
|
||||
}
|
||||
|
||||
export interface NetworkTotals {
|
||||
bytes_received: number | null;
|
||||
bytes_transmitted: number | null;
|
||||
label: string;
|
||||
collected_at: string;
|
||||
}
|
||||
|
||||
export interface DeploymentHealth {
|
||||
git_revision: string | null;
|
||||
short_revision: string | null;
|
||||
revision_status: 'known' | 'unavailable';
|
||||
website_health: ComponentHealthStatus;
|
||||
api_health: ComponentHealthStatus;
|
||||
history_status: 'unavailable' | 'available';
|
||||
last_successful_deployment: Record<string, any> | null;
|
||||
}
|
||||
|
||||
export interface OperationsInfrastructureData {
|
||||
resources: SystemResources;
|
||||
services: ServiceHealthItem[];
|
||||
network: NetworkTotals;
|
||||
deployment: DeploymentHealth;
|
||||
collected_at: string;
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -6,8 +6,12 @@ import {
|
|||
OperationsOverviewData,
|
||||
OverviewSecurityErrorItem,
|
||||
SafeIncidentSummary,
|
||||
OperationsInfrastructureData,
|
||||
ServiceStatusValue,
|
||||
ServiceHealthItem,
|
||||
} from '../components/ops/types';
|
||||
|
||||
|
||||
export const HEALTH_COMPONENT_METADATA: Record<
|
||||
HealthComponentKey,
|
||||
{ name: string; description: string }
|
||||
|
|
@ -471,3 +475,193 @@ export function normalizeOverviewResponse(raw: unknown): OperationsOverviewData
|
|||
};
|
||||
}
|
||||
|
||||
export function isValidServiceStatus(status: unknown): status is ServiceStatusValue {
|
||||
return status === 'active' || status === 'inactive' || status === 'failed' || status === 'unknown';
|
||||
}
|
||||
|
||||
export function normalizeInfrastructureResponse(raw: unknown): OperationsInfrastructureData {
|
||||
const fallbackCollectedAt = new Date().toISOString();
|
||||
|
||||
const defaultServices: ServiceHealthItem[] = [
|
||||
{
|
||||
name: 'motorsignal-website',
|
||||
display_name: 'MotorSignal Website',
|
||||
description: 'Next.js frontend application runtime',
|
||||
status: 'unknown',
|
||||
},
|
||||
{
|
||||
name: 'motorsignal-api',
|
||||
display_name: 'MotorSignal API',
|
||||
description: 'FastAPI core backend service & ingress',
|
||||
status: 'unknown',
|
||||
},
|
||||
{
|
||||
name: 'nginx',
|
||||
display_name: 'Nginx Reverse Proxy',
|
||||
description: 'Edge TLS termination & reverse proxy routing',
|
||||
status: 'unknown',
|
||||
},
|
||||
{
|
||||
name: 'postgresql',
|
||||
display_name: 'PostgreSQL Database',
|
||||
description: 'Primary relational database persistence',
|
||||
status: 'unknown',
|
||||
},
|
||||
];
|
||||
|
||||
if (!raw || typeof raw !== 'object') {
|
||||
return {
|
||||
resources: {
|
||||
cpu_percent: null,
|
||||
load_avg: null,
|
||||
memory: null,
|
||||
swap: null,
|
||||
disk: null,
|
||||
uptime_seconds: null,
|
||||
uptime_formatted: null,
|
||||
collected_at: fallbackCollectedAt,
|
||||
},
|
||||
services: defaultServices,
|
||||
network: {
|
||||
bytes_received: null,
|
||||
bytes_transmitted: null,
|
||||
label: 'Cumulative Total Since Boot',
|
||||
collected_at: fallbackCollectedAt,
|
||||
},
|
||||
deployment: {
|
||||
git_revision: null,
|
||||
short_revision: null,
|
||||
revision_status: 'unavailable',
|
||||
website_health: 'unknown',
|
||||
api_health: 'unknown',
|
||||
history_status: 'unavailable',
|
||||
last_successful_deployment: null,
|
||||
},
|
||||
collected_at: fallbackCollectedAt,
|
||||
};
|
||||
}
|
||||
|
||||
const obj = raw as Record<string, unknown>;
|
||||
const rawResources = obj.resources && typeof obj.resources === 'object' ? (obj.resources as Record<string, unknown>) : {};
|
||||
const rawServices = Array.isArray(obj.services) ? obj.services : [];
|
||||
const rawNetwork = obj.network && typeof obj.network === 'object' ? (obj.network as Record<string, unknown>) : {};
|
||||
const rawDeployment = obj.deployment && typeof obj.deployment === 'object' ? (obj.deployment as Record<string, unknown>) : {};
|
||||
|
||||
// Resources
|
||||
const cpu_percent = typeof rawResources.cpu_percent === 'number' ? Math.max(0, Math.min(100, rawResources.cpu_percent)) : null;
|
||||
const load_avg: [number, number, number] | null =
|
||||
Array.isArray(rawResources.load_avg) && rawResources.load_avg.length >= 3 && rawResources.load_avg.every((n) => typeof n === 'number')
|
||||
? [rawResources.load_avg[0], rawResources.load_avg[1], rawResources.load_avg[2]]
|
||||
: null;
|
||||
|
||||
let memory = null;
|
||||
if (rawResources.memory && typeof rawResources.memory === 'object') {
|
||||
const rm = rawResources.memory as Record<string, unknown>;
|
||||
if (typeof rm.total === 'number' && typeof rm.used === 'number') {
|
||||
const total = rm.total;
|
||||
const used = rm.used;
|
||||
const available = typeof rm.available === 'number' ? rm.available : Math.max(0, total - used);
|
||||
const percent = typeof rm.percent === 'number' ? rm.percent : total > 0 ? Math.round((used / total) * 1000) / 10 : 0;
|
||||
memory = { total, used, available, percent };
|
||||
}
|
||||
}
|
||||
|
||||
let swap = null;
|
||||
if (rawResources.swap && typeof rawResources.swap === 'object') {
|
||||
const rs = rawResources.swap as Record<string, unknown>;
|
||||
if (typeof rs.total === 'number' && typeof rs.used === 'number') {
|
||||
const total = rs.total;
|
||||
const used = rs.used;
|
||||
const percent = typeof rs.percent === 'number' ? rs.percent : total > 0 ? Math.round((used / total) * 1000) / 10 : 0;
|
||||
swap = { total, used, percent };
|
||||
}
|
||||
}
|
||||
|
||||
let disk = null;
|
||||
if (rawResources.disk && typeof rawResources.disk === 'object') {
|
||||
const rd = rawResources.disk as Record<string, unknown>;
|
||||
if (typeof rd.total === 'number' && typeof rd.used === 'number') {
|
||||
const total = rd.total;
|
||||
const used = rd.used;
|
||||
const available = typeof rd.available === 'number' ? rd.available : Math.max(0, total - used);
|
||||
const percent = typeof rd.percent === 'number' ? rd.percent : total > 0 ? Math.round((used / total) * 1000) / 10 : 0;
|
||||
disk = {
|
||||
total,
|
||||
used,
|
||||
available,
|
||||
percent,
|
||||
warning: percent >= 80.0,
|
||||
critical: percent >= 90.0,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
const uptime_seconds = typeof rawResources.uptime_seconds === 'number' ? rawResources.uptime_seconds : null;
|
||||
const uptime_formatted = typeof rawResources.uptime_formatted === 'string' && rawResources.uptime_formatted.trim() ? rawResources.uptime_formatted.trim() : null;
|
||||
|
||||
// Services
|
||||
const servicesMap = new Map<string, ServiceHealthItem>();
|
||||
for (const def of defaultServices) {
|
||||
servicesMap.set(def.name, { ...def });
|
||||
}
|
||||
|
||||
for (const s of rawServices) {
|
||||
if (s && typeof s === 'object') {
|
||||
const sObj = s as Record<string, unknown>;
|
||||
const name = typeof sObj.name === 'string' ? sObj.name.trim() : '';
|
||||
if (servicesMap.has(name)) {
|
||||
const existing = servicesMap.get(name)!;
|
||||
const status = isValidServiceStatus(sObj.status) ? sObj.status : 'unknown';
|
||||
const display_name = typeof sObj.display_name === 'string' && sObj.display_name.trim() ? sObj.display_name.trim() : existing.display_name;
|
||||
const description = typeof sObj.description === 'string' && sObj.description.trim() ? sObj.description.trim() : existing.description;
|
||||
servicesMap.set(name, { name, display_name, description, status });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const services = defaultServices.map((d) => servicesMap.get(d.name)!);
|
||||
|
||||
// Network
|
||||
const bytes_received = typeof rawNetwork.bytes_received === 'number' ? rawNetwork.bytes_received : null;
|
||||
const bytes_transmitted = typeof rawNetwork.bytes_transmitted === 'number' ? rawNetwork.bytes_transmitted : null;
|
||||
const network = {
|
||||
bytes_received,
|
||||
bytes_transmitted,
|
||||
label: 'Cumulative Total Since Boot',
|
||||
collected_at: typeof rawNetwork.collected_at === 'string' ? rawNetwork.collected_at : fallbackCollectedAt,
|
||||
};
|
||||
|
||||
// Deployment
|
||||
const git_revision = typeof rawDeployment.git_revision === 'string' && rawDeployment.git_revision.trim() ? rawDeployment.git_revision.trim() : null;
|
||||
const short_revision = typeof rawDeployment.short_revision === 'string' && rawDeployment.short_revision.trim()
|
||||
? rawDeployment.short_revision.trim()
|
||||
: git_revision ? git_revision.slice(0, 7) : null;
|
||||
const revision_status = git_revision ? 'known' : 'unavailable';
|
||||
const website_health = isValidHealthStatus(rawDeployment.website_health) ? rawDeployment.website_health : 'unknown';
|
||||
const api_health = isValidHealthStatus(rawDeployment.api_health) ? rawDeployment.api_health : 'unknown';
|
||||
|
||||
return {
|
||||
resources: {
|
||||
cpu_percent,
|
||||
load_avg,
|
||||
memory,
|
||||
swap,
|
||||
disk,
|
||||
uptime_seconds,
|
||||
uptime_formatted,
|
||||
collected_at: typeof rawResources.collected_at === 'string' ? rawResources.collected_at : fallbackCollectedAt,
|
||||
},
|
||||
services,
|
||||
network,
|
||||
deployment: {
|
||||
git_revision,
|
||||
short_revision,
|
||||
revision_status,
|
||||
website_health,
|
||||
api_health,
|
||||
history_status: 'unavailable',
|
||||
last_successful_deployment: null,
|
||||
},
|
||||
collected_at: typeof obj.collected_at === 'string' ? obj.collected_at : fallbackCollectedAt,
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -55,8 +55,11 @@ const {
|
|||
sanitizeIncidentSummary,
|
||||
sanitizeIncidentDetail,
|
||||
sanitizeIncidentNote,
|
||||
normalizeInfrastructureResponse,
|
||||
isValidServiceStatus,
|
||||
} = opsData;
|
||||
|
||||
|
||||
// ============================================================
|
||||
// 1. ISOLATED OPERATIONS SHELL & CONSUMER NAV SUPPRESSION
|
||||
// ============================================================
|
||||
|
|
@ -715,12 +718,14 @@ console.log(' PASS: Incident Detail UI includes all required operational sectio
|
|||
// ============================================================
|
||||
console.log('Testing 33: Zero Polling Verification Across Operations UI...');
|
||||
for (const file of currentOpsFiles) {
|
||||
if (file === 'operations-infrastructure.tsx') continue; // Infrastructure explicitly implements 30s visibility-gated polling (tested in Section 43)
|
||||
const content = fs.readFileSync(path.join(opsDir, file), 'utf8');
|
||||
assert.ok(!content.includes('setInterval'), `${file} must NOT contain setInterval polling`);
|
||||
assert.ok(!content.includes('setTimeout(') || content.includes('setTimeout(() => setMutationSuccess') || content.includes('setTimeout(() =>'), `${file} must only use setTimeout for UI transitions, not polling`);
|
||||
}
|
||||
console.log(' PASS: Zero automatic background polling across all Operations components.');
|
||||
|
||||
|
||||
// ============================================================
|
||||
// 34. ZERO RIDER DATA CROSSOVER (OPS-4)
|
||||
// ============================================================
|
||||
|
|
@ -1143,6 +1148,197 @@ assert.ok(backendMainSrc.includes('homepage_hero') && backendMainSrc.includes('f
|
|||
|
||||
console.log(' PASS: Section 4 Website Content CMS backend, proxy routes, navigation, and UI contracts fully verified.');
|
||||
|
||||
console.log('--- ALL OPS-3C, OPS-3C.1, OPS-3C.2, OPS-3C.3, OPS-4, OPS-4.2, OVERVIEW, SECTION 2 USERS, SECTION 3 SUPPORT CENTER & SECTION 4 CMS TESTS PASSED (42/42) ---');
|
||||
// ============================================================
|
||||
// 43. PRODUCTION OPERATIONS INFRASTRUCTURE DASHBOARD (PHASE 1)
|
||||
// ============================================================
|
||||
console.log('Testing 43: Operations Infrastructure Dashboard Phase 1...');
|
||||
|
||||
// 43.1 Normalization of null / empty / malformed response
|
||||
const nullInfra = normalizeInfrastructureResponse(null);
|
||||
assert.ok(nullInfra.resources, 'null input must return resources object');
|
||||
assert.equal(nullInfra.resources.cpu_percent, null);
|
||||
assert.equal(nullInfra.resources.load_avg, null);
|
||||
assert.equal(nullInfra.resources.memory, null);
|
||||
assert.equal(nullInfra.resources.swap, null);
|
||||
assert.equal(nullInfra.resources.disk, null);
|
||||
assert.equal(nullInfra.resources.uptime_seconds, null);
|
||||
assert.equal(nullInfra.resources.uptime_formatted, null);
|
||||
assert.equal(nullInfra.services.length, 4, 'null input must provide 4 default services');
|
||||
assert.deepEqual(nullInfra.services.map(s => s.name), ['motorsignal-website', 'motorsignal-api', 'nginx', 'postgresql']);
|
||||
assert.ok(nullInfra.services.every(s => s.status === 'unknown'), 'default services must have unknown status');
|
||||
assert.equal(nullInfra.network.bytes_received, null);
|
||||
assert.equal(nullInfra.network.bytes_transmitted, null);
|
||||
assert.equal(nullInfra.network.label, 'Cumulative Total Since Boot');
|
||||
assert.equal(nullInfra.deployment.git_revision, null);
|
||||
assert.equal(nullInfra.deployment.short_revision, null);
|
||||
assert.equal(nullInfra.deployment.revision_status, 'unavailable');
|
||||
assert.equal(nullInfra.deployment.website_health, 'unknown');
|
||||
assert.equal(nullInfra.deployment.api_health, 'unknown');
|
||||
assert.equal(nullInfra.deployment.history_status, 'unavailable');
|
||||
assert.equal(nullInfra.deployment.last_successful_deployment, null);
|
||||
|
||||
// 43.2 Normalization of valid response & disk threshold warning/critical calculation
|
||||
const validInfraRaw = {
|
||||
resources: {
|
||||
cpu_percent: 18.5,
|
||||
load_avg: [0.15, 0.22, 0.18],
|
||||
memory: { total: 16000000000, used: 8000000000, available: 8000000000, percent: 50.0 },
|
||||
swap: { total: 4000000000, used: 400000000, percent: 10.0 },
|
||||
disk: { total: 100000000000, used: 82000000000, available: 18000000000, percent: 82.0 },
|
||||
uptime_seconds: 123456,
|
||||
uptime_formatted: '1d 10h 17m',
|
||||
},
|
||||
services: [
|
||||
{ name: 'motorsignal-website', status: 'active' },
|
||||
{ name: 'motorsignal-api', status: 'active' },
|
||||
{ name: 'nginx', status: 'active' },
|
||||
{ name: 'postgresql', status: 'active' },
|
||||
],
|
||||
network: {
|
||||
bytes_received: 5000000000,
|
||||
bytes_transmitted: 2500000000,
|
||||
label: 'Cumulative Total Since Boot',
|
||||
},
|
||||
deployment: {
|
||||
git_revision: '0123456789abcdef0123456789abcdef01234567',
|
||||
short_revision: '0123456',
|
||||
revision_status: 'known',
|
||||
website_health: 'available',
|
||||
api_health: 'available',
|
||||
},
|
||||
};
|
||||
|
||||
const normalizedValid = normalizeInfrastructureResponse(validInfraRaw);
|
||||
assert.equal(normalizedValid.resources.cpu_percent, 18.5);
|
||||
assert.deepEqual(normalizedValid.resources.load_avg, [0.15, 0.22, 0.18]);
|
||||
assert.equal(normalizedValid.resources.memory.percent, 50.0);
|
||||
assert.equal(normalizedValid.resources.disk.percent, 82.0);
|
||||
assert.equal(normalizedValid.resources.disk.warning, true, 'Disk at 82% must trigger warning');
|
||||
assert.equal(normalizedValid.resources.disk.critical, false, 'Disk at 82% must not trigger critical');
|
||||
assert.equal(normalizedValid.services[0].status, 'active');
|
||||
assert.equal(normalizedValid.network.bytes_received, 5000000000);
|
||||
assert.equal(normalizedValid.deployment.git_revision, '0123456789abcdef0123456789abcdef01234567');
|
||||
assert.equal(normalizedValid.deployment.short_revision, '0123456');
|
||||
assert.equal(normalizedValid.deployment.revision_status, 'known');
|
||||
assert.equal(normalizedValid.deployment.website_health, 'available');
|
||||
assert.equal(normalizedValid.deployment.api_health, 'available');
|
||||
|
||||
// 43.3 Critical disk threshold (>=90%)
|
||||
const criticalDiskRaw = {
|
||||
resources: {
|
||||
disk: { total: 100000000000, used: 91000000000, available: 9000000000, percent: 91.0 },
|
||||
},
|
||||
};
|
||||
const normalizedCritical = normalizeInfrastructureResponse(criticalDiskRaw);
|
||||
assert.equal(normalizedCritical.resources.disk.warning, true);
|
||||
assert.equal(normalizedCritical.resources.disk.critical, true, 'Disk at 91% must trigger critical');
|
||||
|
||||
// 43.4 Service status validation
|
||||
assert.equal(isValidServiceStatus('active'), true);
|
||||
assert.equal(isValidServiceStatus('inactive'), true);
|
||||
assert.equal(isValidServiceStatus('failed'), true);
|
||||
assert.equal(isValidServiceStatus('unknown'), true);
|
||||
assert.equal(isValidServiceStatus('running'), false);
|
||||
assert.equal(isValidServiceStatus('dead'), false);
|
||||
|
||||
// 43.5 Types, Navigation & Shell wiring
|
||||
assert.ok(typesSrc.includes("'infrastructure'"), "types.ts must include 'infrastructure' in OperationsNavId");
|
||||
assert.ok(typesSrc.includes('OperationsInfrastructureData'), 'types.ts must define OperationsInfrastructureData');
|
||||
assert.ok(typesSrc.includes('SystemResources'), 'types.ts must define SystemResources');
|
||||
assert.ok(typesSrc.includes('ServiceHealthItem'), 'types.ts must define ServiceHealthItem');
|
||||
assert.ok(typesSrc.includes('NetworkTotals'), 'types.ts must define NetworkTotals');
|
||||
assert.ok(typesSrc.includes('DeploymentHealth'), 'types.ts must define DeploymentHealth');
|
||||
|
||||
const updatedSidebarSource = fs.readFileSync(path.resolve(__dirname, '../components/ops/operations-sidebar.tsx'), 'utf8');
|
||||
const updatedShellSource = fs.readFileSync(path.resolve(__dirname, '../components/ops/operations-shell.tsx'), 'utf8');
|
||||
|
||||
assert.ok(
|
||||
updatedSidebarSource.includes("id: 'infrastructure'") &&
|
||||
updatedSidebarSource.includes("label: 'Infrastructure'") &&
|
||||
updatedSidebarSource.includes("icon: 'dns'"),
|
||||
'Sidebar must contain infrastructure nav item'
|
||||
);
|
||||
|
||||
const infraOverviewNavIdx = updatedSidebarSource.indexOf("id: 'overview'");
|
||||
const infraNavIdx = updatedSidebarSource.indexOf("id: 'infrastructure'");
|
||||
const infraUsersNavIdx = updatedSidebarSource.indexOf("id: 'users'");
|
||||
assert.ok(
|
||||
infraOverviewNavIdx < infraNavIdx && infraNavIdx < infraUsersNavIdx,
|
||||
'Infrastructure nav item must be placed right after overview and before users'
|
||||
);
|
||||
|
||||
assert.ok(
|
||||
updatedShellSource.includes("activeNav === 'infrastructure'") &&
|
||||
updatedShellSource.includes('<OperationsInfrastructure'),
|
||||
'OperationsShell must wire activeNav === infrastructure to OperationsInfrastructure'
|
||||
);
|
||||
|
||||
// 43.6 UI Component Contracts (operations-infrastructure.tsx)
|
||||
const infraComponentSrc = fs.readFileSync(path.resolve(__dirname, '../components/ops/operations-infrastructure.tsx'), 'utf8');
|
||||
|
||||
// Read-only safety: zero mutation or arbitrary command controls
|
||||
assert.ok(!infraComponentSrc.includes('Restart') && !infraComponentSrc.includes('restart'), 'Infrastructure component must NOT have restart action');
|
||||
assert.ok(!infraComponentSrc.includes('Deploy Now') && !infraComponentSrc.includes('Trigger Deploy') && !infraComponentSrc.includes('action="deploy"'), 'Infrastructure component must NOT have deploy action');
|
||||
assert.ok(!infraComponentSrc.includes('Rollback') && !infraComponentSrc.includes('rollback'), 'Infrastructure component must NOT have rollback action');
|
||||
assert.ok(!infraComponentSrc.includes('Stop Service') && !infraComponentSrc.includes('Kill'), 'Infrastructure component must NOT have Stop/Kill action');
|
||||
assert.ok(!infraComponentSrc.includes('exec(') && !infraComponentSrc.includes('spawn('), 'Infrastructure component must NOT execute shell commands');
|
||||
|
||||
// Cumulative network totals labeling
|
||||
assert.ok(
|
||||
infraComponentSrc.includes('Cumulative Total Since Boot'),
|
||||
'Infrastructure component must display Cumulative Total Since Boot'
|
||||
);
|
||||
assert.ok(
|
||||
infraComponentSrc.includes('Not a rate') || infraComponentSrc.includes('Not an instantaneous'),
|
||||
'Infrastructure component must explicitly clarify network is not a rate'
|
||||
);
|
||||
|
||||
// Auto-refresh & duplicate request suppression
|
||||
assert.ok(
|
||||
infraComponentSrc.includes('30000') && infraComponentSrc.includes('setInterval'),
|
||||
'Infrastructure component must poll on 30s interval'
|
||||
);
|
||||
assert.ok(
|
||||
infraComponentSrc.includes('visibilitychange') && infraComponentSrc.includes('visibilityState'),
|
||||
'Infrastructure component must gate polling on document.visibilityState'
|
||||
);
|
||||
assert.ok(
|
||||
infraComponentSrc.includes('isFetchingRef'),
|
||||
'Infrastructure component must use isFetchingRef to prevent overlapping requests'
|
||||
);
|
||||
|
||||
// 43.7 Next.js API Route Proxy & Backend Main Contract
|
||||
const infraRouteSrc = fs.readFileSync(path.resolve(__dirname, '../app/api/ops/infrastructure/route.ts'), 'utf8');
|
||||
assert.ok(
|
||||
infraRouteSrc.includes('__Host-motorsignal_ops'),
|
||||
'Next.js infrastructure route must enforce __Host-motorsignal_ops in production'
|
||||
);
|
||||
assert.ok(
|
||||
infraRouteSrc.includes('/api/v1/ops/infrastructure'),
|
||||
'Next.js infrastructure route must proxy to /api/v1/ops/infrastructure'
|
||||
);
|
||||
|
||||
assert.ok(
|
||||
backendMainSrc.includes('@app.get("/api/v1/ops/infrastructure")'),
|
||||
'Backend main.py must register GET /api/v1/ops/infrastructure'
|
||||
);
|
||||
assert.ok(
|
||||
backendMainSrc.includes('require_ops_roles("operator", "owner")'),
|
||||
'Backend main.py must guard infrastructure endpoint with require_ops_roles("operator", "owner")'
|
||||
);
|
||||
|
||||
const infraMetricsSrc = fs.readFileSync(path.resolve(__dirname, '../../backend/motorsignal_api/app/infrastructure_metrics.py'), 'utf8');
|
||||
assert.ok(
|
||||
infraMetricsSrc.includes('ALLOWED_SYSTEMD_SERVICES') &&
|
||||
infraMetricsSrc.includes('motorsignal-website') &&
|
||||
infraMetricsSrc.includes('motorsignal-api') &&
|
||||
infraMetricsSrc.includes('nginx') &&
|
||||
infraMetricsSrc.includes('postgresql'),
|
||||
'infrastructure_metrics.py must enforce strict allowlist of 4 services'
|
||||
);
|
||||
|
||||
console.log(' PASS: Section 43 Production Operations Infrastructure Dashboard (Phase 1) verified.');
|
||||
|
||||
console.log('--- ALL OPS-3C, OPS-3C.1, OPS-3C.2, OPS-3C.3, OPS-4, OPS-4.2, OVERVIEW, SECTION 2 USERS, SECTION 3 SUPPORT CENTER, SECTION 4 CMS & SECTION 43 INFRASTRUCTURE TESTS PASSED (43/43) ---');
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue