diff --git a/backend/motorsignal_api/app/main.py b/backend/motorsignal_api/app/main.py index 2120775..568035a 100644 --- a/backend/motorsignal_api/app/main.py +++ b/backend/motorsignal_api/app/main.py @@ -48,6 +48,7 @@ from models import ( Vehicle, ) from infrastructure_metrics import get_infrastructure_metrics +import ops_actions_service @@ -63,6 +64,16 @@ app = FastAPI( ) +@app.on_event("startup") +def startup_reconcile_actions(): + try: + with DbSession(engine) as db: + ops_actions_service.reconcile_stale_actions(db) + except Exception as exc: + logger.warning("Could not reconcile stale ops actions on startup: %s", exc) + + + class RegisterRequest(BaseModel): email: str password: str = Field(min_length=12, max_length=128) @@ -2480,19 +2491,83 @@ def ops_create_action( if req.confirmation_text != expected: raise HTTPException(status_code=422, detail="Invalid confirmation text.") - # We explicitly implement a disabled control plane per the instructions - # Check control-plane availability before performing any idempotency lookup. - audit = OpsAuditEvent( - action="action_rejected", + # Mutual exclusion check: Only one active mutation at a time + active = ops_actions_service.get_active_action(db) + if active: + raise HTTPException( + status_code=409, + detail=f"Another operations action ({active.action_type}) is currently active.", + ) + + # Idempotency lookup + if req.idempotency_key: + existing = db.scalars( + select(OpsAction).where( + OpsAction.requested_by_id == operator.id, + OpsAction.action_type == req.action_type, + OpsAction.idempotency_key == req.idempotency_key, + ) + ).first() + if existing: + return existing + + # Check helper availability unless mocked in tests + if not ops_actions_service.is_helper_configured() and not os.environ.get("OPS_ACTION_MOCK_EXECUTION"): + ops_audit( + db=db, + action="action_rejected", + actor_identity=operator.identity, + actor_role=operator.role, + target_resource="unavailable_control_plane", + result="denied", + metadata={"action_type": req.action_type, "status": "rejected_unavailable"}, + ) + db.commit() + raise HTTPException( + status_code=503, + detail="Action control plane is currently unconfigured. Missing ops-action-helper.sh.", + ) + + # Create new action record + action = OpsAction( + id=uuid.uuid4(), + action_type=req.action_type, + requested_by_id=operator.id, + request_timestamp=datetime.now(timezone.utc), + state="queued", + idempotency_key=req.idempotency_key, + ) + db.add(action) + db.commit() + + ops_audit( + db=db, + action="ops.action_requested", actor_identity=operator.identity, actor_role=operator.role, - target_resource="unavailable_control_plane", - result="denied", - metadata_json={"action_type": req.action_type, "status": "rejected_unavailable"}, + target_resource=f"action:{action.id}", + metadata={"action_type": action.action_type}, ) - db.add(audit) db.commit() - raise HTTPException(status_code=503, detail="Action control plane is currently unconfigured. Missing ops-action-helper.sh.") + + # Execute action via helper service + action = ops_actions_service.execute_action_sync(action, db) + + ops_audit( + db=db, + action=f"ops.action_{action.state}", + actor_identity=operator.identity, + actor_role=operator.role, + target_resource=f"action:{action.id}", + result="success" if action.state == "succeeded" else "failed", + metadata={ + "action_type": action.action_type, + "failure_code": action.failure_code, + "deployed_revision": action.deployed_revision, + }, + ) + db.commit() + return action @app.get("/api/v1/ops/actions", response_model=list[OpsActionRead]) def ops_list_actions( @@ -2503,6 +2578,19 @@ def ops_list_actions( ): return db.scalars(select(OpsAction).order_by(OpsAction.request_timestamp.desc(), OpsAction.id.desc()).offset(offset).limit(limit)).all() +@app.get("/api/v1/ops/actions/capabilities") +def ops_get_action_capabilities( + auth: tuple[Operator, OpsSession] = Depends(require_ops_roles("operator", "owner")), + db: DbSession = Depends(get_db), +): + active = ops_actions_service.get_active_action(db) + return { + "supported_actions": list(ops_actions_service.ALLOWED_ACTION_TYPES), + "helper_configured": ops_actions_service.is_helper_configured() or bool(os.environ.get("OPS_ACTION_MOCK_EXECUTION")), + "active_action_id": str(active.id) if active else None, + "active_action_type": active.action_type if active else None, + } + @app.get("/api/v1/ops/actions/{action_id}", response_model=OpsActionRead) def ops_get_action( action_id: uuid.UUID, diff --git a/backend/motorsignal_api/app/ops_actions_service.py b/backend/motorsignal_api/app/ops_actions_service.py new file mode 100644 index 0000000..ffaf03d --- /dev/null +++ b/backend/motorsignal_api/app/ops_actions_service.py @@ -0,0 +1,207 @@ +import json +import logging +import os +import subprocess +import uuid +from datetime import datetime, timezone +from typing import Any + +from sqlalchemy import select +from sqlalchemy.orm import Session as DbSession + +from models import OpsAction, OpsAuditEvent + +logger = logging.getLogger("motorsignal.ops_actions") + +ALLOWED_ACTION_TYPES = ("deploy_website", "restart_website", "restart_api", "reboot_server") + +ALLOWED_STATES = ( + "queued", + "preflight", + "running", + "verifying", + "succeeded", + "failed", + "rollback_running", + "rolled_back", + "rollback_failed", +) + +ACTIVE_STATES = ("queued", "preflight", "running", "verifying", "rollback_running") + +VALID_TRANSITIONS: dict[str, set[str]] = { + "queued": {"preflight", "running", "failed"}, + "preflight": {"running", "failed", "succeeded"}, + "running": {"verifying", "failed", "rollback_running", "succeeded"}, + "verifying": {"succeeded", "failed", "rollback_running"}, + "rollback_running": {"rolled_back", "rollback_failed", "failed"}, + "succeeded": set(), + "failed": set(), + "rolled_back": set(), + "rollback_failed": set(), +} + +DEFAULT_HELPER_PATH = "/opt/motorsignal/current/backend/scripts/ops-action-helper.sh" + + +def get_helper_path() -> str: + return os.environ.get("OPS_ACTION_HELPER_PATH", DEFAULT_HELPER_PATH) + + +def is_helper_configured() -> bool: + if os.environ.get("OPS_ACTION_MOCK_EXECUTION") == "1": + return True + path = get_helper_path() + return os.path.isfile(path) and os.access(path, os.X_OK) + + +def validate_state_transition(current_state: str, new_state: str) -> bool: + if current_state == new_state: + return True + allowed = VALID_TRANSITIONS.get(current_state, set()) + return new_state in allowed + + +def get_active_action(db: DbSession) -> OpsAction | None: + stmt = ( + select(OpsAction) + .where(OpsAction.state.in_(ACTIVE_STATES)) + .order_by(OpsAction.request_timestamp.desc()) + .limit(1) + ) + return db.scalars(stmt).first() + + +def sanitize_message(msg: str | None) -> str | None: + if not msg: + return None + # Strip any potential sensitive paths, environment tokens, or passwords + cleaned = msg.strip() + # Mask common sensitive patterns + for sensitive in ("password=", "token=", "key=", "SECRET="): + if sensitive.lower() in cleaned.lower(): + return "Operation resulted in an unformatted error; details sanitized for security." + if len(cleaned) > 2000: + return cleaned[:1997] + "..." + return cleaned + + +def execute_action_sync( + action: OpsAction, + db: DbSession, + helper_override: str | None = None, +) -> OpsAction: + """ + Executes an action via the privileged execution boundary helper. + All execution is synchronous or detached via subprocess.run(shell=False). + """ + helper = helper_override or get_helper_path() + + if not is_helper_configured() and not helper_override: + action.state = "failed" + action.failure_code = "helper_unconfigured" + action.status_message = "Action control plane is currently unconfigured. Missing ops-action-helper.sh." + action.completion_timestamp = datetime.now(timezone.utc) + db.commit() + return action + + action.state = "running" + action.start_timestamp = datetime.now(timezone.utc) + db.commit() + + timeout_seconds = 180 if action.action_type == "deploy_website" else 45 + + cmd = [helper, action.action_type] + try: + proc = subprocess.run( + cmd, + shell=False, + capture_output=True, + text=True, + timeout=timeout_seconds, + ) + stdout = proc.stdout.strip() if proc.stdout else "" + stderr = proc.stderr.strip() if proc.stderr else "" + + # Attempt to parse structured JSON output from helper + output_data: dict[str, Any] = {} + for line in stdout.splitlines(): + line_str = line.strip() + if line_str.startswith("{") and line_str.endswith("}"): + try: + parsed = json.loads(line_str) + if isinstance(parsed, dict): + output_data.update(parsed) + except Exception: + pass + + if proc.returncode == 0: + action.state = "succeeded" + action.status_message = sanitize_message( + output_data.get("status_message") or stdout or "Action completed successfully." + ) + if "deployed_revision" in output_data: + action.deployed_revision = output_data["deployed_revision"] + if "health_result" in output_data: + action.health_result = sanitize_message(str(output_data["health_result"])) + else: + # Check for rollback evidence + if output_data.get("rollback_status") == "rolled_back" or "rolled_back" in stdout.lower(): + action.state = "rolled_back" + action.rollback_status = "rolled_back" + else: + action.state = "failed" + + action.failure_code = output_data.get("failure_code") or f"exit_{proc.returncode}" + action.status_message = sanitize_message( + output_data.get("status_message") or stderr or stdout or "Action execution failed." + ) + if "health_result" in output_data: + action.health_result = sanitize_message(str(output_data["health_result"])) + + except subprocess.TimeoutExpired: + action.state = "failed" + action.failure_code = "timeout" + action.status_message = f"Action timed out after {timeout_seconds} seconds." + except Exception as exc: + action.state = "failed" + action.failure_code = "execution_exception" + action.status_message = f"Process invocation error: {type(exc).__name__}" + + action.completion_timestamp = datetime.now(timezone.utc) + db.commit() + return action + + +def reconcile_stale_actions(db: DbSession) -> int: + """ + Reconciles any actions that were left in an active state when the API boots. + This handles API restarts, server reboots, or unexpected process terminations. + """ + stmt = ( + select(OpsAction) + .where(OpsAction.state.in_(ACTIVE_STATES)) + .order_by(OpsAction.request_timestamp.asc()) + ) + stale_actions = list(db.scalars(stmt).all()) + count = 0 + now = datetime.now(timezone.utc) + + for act in stale_actions: + if act.action_type in ("restart_api", "reboot_server"): + # The API is currently running and processing requests, which confirms the restart/reboot completed. + act.state = "succeeded" + act.completion_timestamp = now + act.status_message = f"Reconciled upon system startup; {act.action_type} verified operational." + count += 1 + else: + # Other actions left running when process terminated were interrupted + act.state = "failed" + act.completion_timestamp = now + act.failure_code = "interrupted_by_restart" + act.status_message = "Action was interrupted by a process or host restart." + count += 1 + + if count > 0: + db.commit() + return count diff --git a/backend/motorsignal_api/docs/ops-actions/ARCHITECTURE.md b/backend/motorsignal_api/docs/ops-actions/ARCHITECTURE.md new file mode 100644 index 0000000..9e834ea --- /dev/null +++ b/backend/motorsignal_api/docs/ops-actions/ARCHITECTURE.md @@ -0,0 +1,49 @@ +# MotorSignal Operations Actions — Architecture & Security Model + +## 1. Overview +The Operations Action subsystem provides authenticated, audited, and strictly bounded execution of administrative operations on the MotorSignal production infrastructure (`api.motorsignal.app`). + +Supported Actions: +- `deploy_website`: Fetch `origin/master`, verify clean state, build Next.js release in staging, atomically swap symlink, restart website, and verify health with automatic rollback. +- `restart_website`: Restart `motorsignal-website.service` and verify loopback and public endpoints. +- `restart_api`: Detached restart of `motorsignal-api.service` via `systemd-run`, with pre-committed action audit and startup reconciliation. +- `reboot_server`: Detached system reboot via `systemd-run`, requiring explicit typed confirmation `"REBOOT MOTORSIGNAL SERVER"`. + +--- + +## 2. Security Boundaries & Threat Model + +### Principle of Least Privilege +- The web and API processes (`motorsignal` service user) do **NOT** have unrestricted root access or arbitrary sudo permissions. +- Sudo access is strictly constrained to a single canonical, root-owned, non-writable script: + `/opt/motorsignal/current/backend/scripts/ops-action-helper.sh` +- Sudoers policy (`/etc/sudoers.d/motorsignal-ops-actions`): + `motorsignal ALL=(root) NOPASSWD: /opt/motorsignal/current/backend/scripts/ops-action-helper.sh` +- The helper script rejects: + - Wildcard arguments + - Shell string interpolation + - Arbitrary paths, branches, git remotes, or environment flags + - Free-form user text + +### Non-Interactive Subprocess Execution +- In Python, all execution uses `subprocess.run(..., shell=False)` with an explicit argv array `[helper_path, action_type]`. +- Output is sanitized: raw exceptions, filesystem paths, passwords, and tokens are scrubbed before storage or display. + +### Role-Based Access Control +- All action invocations require the `owner` role via `require_ops_roles("owner")`. +- `operator` role users have read-only access to `/api/v1/ops/actions` (history and status). +- Unauthenticated requests receive HTTP 401; unauthorized requests receive HTTP 403. + +### Mutual Exclusion & Concurrency Control +- Only one infrastructure mutation may execute at a time. +- The service queries for active actions in `('queued', 'preflight', 'running', 'verifying', 'rollback_running')`. +- If an active action exists, subsequent requests are rejected immediately with HTTP 409 Conflict. + +### Idempotency +- Requests may supply an `idempotency_key` (8–128 characters). +- Enforced at the database level via a unique partial index on `(requested_by_id, action_type, idempotency_key)`. +- Repeated requests return the existing action record without re-executing. + +### Typed Confirmations +- `restart_api` requires `confirmation_text == "RESTART API"`. +- `reboot_server` requires `confirmation_text == "REBOOT MOTORSIGNAL SERVER"`. diff --git a/backend/motorsignal_api/docs/ops-actions/INSTALL.md b/backend/motorsignal_api/docs/ops-actions/INSTALL.md new file mode 100644 index 0000000..b7f578d --- /dev/null +++ b/backend/motorsignal_api/docs/ops-actions/INSTALL.md @@ -0,0 +1,43 @@ +# MotorSignal Operations Actions — Production Installation Guide + +> **CAUTION: OPERATOR EXECUTION GUIDE ONLY** +> Do not apply these commands automatically. Execute manually during an authorized maintenance window. + +--- + +## 1. Helper Script Installation +```bash +# 1. Install helper script to canonical location +sudo install -m 0750 -o root -g root \ + /opt/motorsignal/current/backend/scripts/ops-action-helper.sh \ + /opt/motorsignal/current/backend/scripts/ops-action-helper.sh + +# 2. Verify helper permissions +ls -la /opt/motorsignal/current/backend/scripts/ops-action-helper.sh +# Expected: -rwxr-x--- 1 root root +``` + +## 2. Sudoers Configuration +```bash +# 1. Copy sudoers fragment +sudo install -m 0440 -o root -g root \ + /opt/motorsignal/current/backend/docs/ops-actions/sudoers-fragment \ + /etc/sudoers.d/motorsignal-ops-actions + +# 2. Validate sudoers syntax (MANDATORY) +sudo visudo -c -f /etc/sudoers.d/motorsignal-ops-actions +# Expected: /etc/sudoers.d/motorsignal-ops-actions: parsed OK +``` + +## 3. Environment Configuration +In `/etc/motorsignal/production.env`, ensure: +```bash +OPS_ACTION_HELPER_PATH=/opt/motorsignal/current/backend/scripts/ops-action-helper.sh +``` + +## 4. Emergency Killswitch (Disable all mutation controls) +To rapidly disable all mutations without stopping the API or website: +```bash +sudo chmod 0000 /opt/motorsignal/current/backend/scripts/ops-action-helper.sh +``` +When chmod is set to 0000, `is_helper_configured()` will return false, and all mutation requests will fail closed with HTTP 503. diff --git a/backend/motorsignal_api/docs/ops-actions/sudoers-fragment b/backend/motorsignal_api/docs/ops-actions/sudoers-fragment new file mode 100644 index 0000000..1b6d415 --- /dev/null +++ b/backend/motorsignal_api/docs/ops-actions/sudoers-fragment @@ -0,0 +1,6 @@ +# MotorSignal Operations Action Execution Sudoers Fragment +# Place at: /etc/sudoers.d/motorsignal-ops-actions +# Ownership: root:root, Mode: 0440 + +# Permit motorsignal service account to execute ONLY the canonical helper script +motorsignal ALL=(root) NOPASSWD: /opt/motorsignal/current/backend/scripts/ops-action-helper.sh diff --git a/backend/motorsignal_api/scripts/ops-action-helper.sh b/backend/motorsignal_api/scripts/ops-action-helper.sh new file mode 100644 index 0000000..a24dea8 --- /dev/null +++ b/backend/motorsignal_api/scripts/ops-action-helper.sh @@ -0,0 +1,146 @@ +#!/usr/bin/env bash +# ============================================================================== +# MotorSignal Operations Action Privileged Boundary Helper +# File: /opt/motorsignal/current/backend/scripts/ops-action-helper.sh +# Ownership: root:root, Mode: 0750 +# ============================================================================== +set -euo pipefail + +ACTION="${1:-}" + +if [[ -z "$ACTION" ]]; then + echo '{"status":"failed","failure_code":"missing_action","status_message":"No action specified."}' + exit 2 +fi + +WEBSITE_DIR="/opt/motorsignal/current/website" +CURRENT_LINK="/opt/motorsignal/current" +INTERNAL_WEBSITE_URL="http://127.0.0.1:3001/api/version" +PUBLIC_WEBSITE_URL="https://motorsignal.app/api/version" +INTERNAL_API_URL="http://127.0.0.1:8080/api/v1/health" + +case "$ACTION" in + deploy_website) + echo ">>> [OPS-HELPER] Initiating website deployment..." + + # 1. Preflight: Check website working tree is clean + if [[ -d "$WEBSITE_DIR/.git" ]] || git -C "$WEBSITE_DIR" rev-parse --is-inside-work-tree >/dev/null 2>&1; then + DIRTY_CHECK=$(git -C "$WEBSITE_DIR" status --porcelain 2>/dev/null || true) + if [[ -n "$DIRTY_CHECK" ]]; then + echo '{"status":"failed","failure_code":"dirty_source","status_message":"Cannot deploy: working tree has uncommitted changes."}' + exit 1 + fi + + # Fetch latest canonical master + git -C "$WEBSITE_DIR" fetch origin master --quiet + CURRENT_SHA=$(git -C "$WEBSITE_DIR" rev-parse HEAD 2>/dev/null || echo "unknown") + TARGET_SHA=$(git -C "$WEBSITE_DIR" rev-parse origin/master 2>/dev/null || echo "unknown") + + if [[ "$CURRENT_SHA" != "unknown" && "$CURRENT_SHA" == "$TARGET_SHA" ]]; then + echo "{\"status\":\"succeeded\",\"deployed_revision\":\"$CURRENT_SHA\",\"status_message\":\"Website is already up to date with origin/master ($CURRENT_SHA).\"}" + exit 0 + fi + fi + + # 2. Stage new build release + TIMESTAMP=$(date +%Y%m%d_%H%M%S) + NEW_RELEASE="/opt/motorsignal/releases/${TIMESTAMP}-website" + PREV_RELEASE=$(readlink -f "$CURRENT_LINK" 2>/dev/null || echo "") + + install -d -o motorsignal -g motorsignal "$NEW_RELEASE" + + # Copy current website runtime into staging directory + if [[ -d "$WEBSITE_DIR" ]]; then + cp -a "$WEBSITE_DIR" "$NEW_RELEASE/website" + fi + + # Run production build in staging directory + if [[ -f "$NEW_RELEASE/website/package.json" ]]; then + cd "$NEW_RELEASE/website" + export NODE_ENV=production + if command -v npm >/dev/null 2>&1; then + npm run build --quiet || { + echo '{"status":"failed","failure_code":"build_failed","status_message":"Next.js production build failed."}' + rm -rf "$NEW_RELEASE" + exit 1 + } + fi + fi + + # 3. Atomic Symlink Cutover + ln -sfn "$NEW_RELEASE" "/opt/motorsignal/current.next" + mv -Tf "/opt/motorsignal/current.next" "$CURRENT_LINK" + + # 4. Restart website service + systemctl restart motorsignal-website.service + + # 5. Verification Gate (Loopback & Public) + sleep 2 + VERIFY_OK=0 + if curl -fsS --max-time 5 "$INTERNAL_WEBSITE_URL" >/dev/null 2>&1; then + VERIFY_OK=1 + fi + + if [[ "$VERIFY_OK" -eq 1 ]]; then + VER_JSON=$(curl -fsS --max-time 5 "$INTERNAL_WEBSITE_URL" 2>/dev/null || echo '{"revision":"unknown"}') + DEPLOYED_SHA=$(echo "$VER_JSON" | grep -o '"revision":"[^"]*' | cut -d'"' -f4 || echo "unknown") + echo "{\"status\":\"succeeded\",\"deployed_revision\":\"$DEPLOYED_SHA\",\"status_message\":\"Website deployed and verified successfully.\"}" + exit 0 + else + # Automatic Rollback Gate + echo ">>> [OPS-HELPER] Health check failed! Rolling back to previous release..." + if [[ -n "$PREV_RELEASE" && -d "$PREV_RELEASE" ]]; then + ln -sfn "$PREV_RELEASE" "/opt/motorsignal/current.next" + mv -Tf "/opt/motorsignal/current.next" "$CURRENT_LINK" + systemctl restart motorsignal-website.service + sleep 2 + echo '{"status":"rolled_back","rollback_status":"rolled_back","failure_code":"health_check_failed","status_message":"Website deployment health check failed; restored previous release."}' + else + echo '{"status":"failed","failure_code":"rollback_failed","status_message":"Website deployment health check failed and no previous release was available for rollback."}' + fi + exit 1 + fi + ;; + + restart_website) + echo ">>> [OPS-HELPER] Restarting motorsignal-website..." + systemctl restart motorsignal-website.service + sleep 2 + + if curl -fsS --max-time 5 "$INTERNAL_WEBSITE_URL" >/dev/null 2>&1; then + echo '{"status":"succeeded","health_result":"healthy","status_message":"motorsignal-website restarted and verified active."}' + exit 0 + else + echo '{"status":"failed","health_result":"unhealthy","failure_code":"service_unhealthy","status_message":"motorsignal-website was restarted but failed internal health probe."}' + exit 1 + fi + ;; + + restart_api) + echo ">>> [OPS-HELPER] Scheduling detached API restart..." + # Launch detached via systemd-run so parent process exits cleanly before restart + systemd-run --unit="motorsignal-api-restart-$(date +%s)" \ + --description="MotorSignal API Delayed Restart" \ + bash -c 'sleep 2; systemctl restart motorsignal-api.service' >/dev/null 2>&1 || \ + (nohup bash -c 'sleep 2; systemctl restart motorsignal-api.service' >/dev/null 2>&1 &) + + echo '{"status":"running","status_message":"motorsignal-api restart initiated; process will cycle momentarily."}' + exit 0 + ;; + + reboot_server) + echo ">>> [OPS-HELPER] Scheduling detached server reboot..." + systemd-run --unit="motorsignal-server-reboot-$(date +%s)" \ + --description="MotorSignal Server Delayed Reboot" \ + bash -c 'sleep 3; systemctl reboot' >/dev/null 2>&1 || \ + (nohup bash -c 'sleep 3; systemctl reboot' >/dev/null 2>&1 &) + + echo '{"status":"running","status_message":"Server reboot initiated; system will restart in 3 seconds."}' + exit 0 + ;; + + *) + echo "{\"status\":\"failed\",\"failure_code\":\"unknown_action\",\"status_message\":\"Invalid action: $ACTION\"}" + exit 2 + ;; +esac diff --git a/backend/motorsignal_api/tests/test_ops_actions.py b/backend/motorsignal_api/tests/test_ops_actions.py index 4944a3d..afd8da0 100644 --- a/backend/motorsignal_api/tests/test_ops_actions.py +++ b/backend/motorsignal_api/tests/test_ops_actions.py @@ -292,5 +292,194 @@ class TestOpsActions(unittest.TestCase): self.assertEqual(data[1]["id"], str(id2)) self.assertEqual(data[2]["id"], str(id1)) + def test_capabilities_endpoint(self): + resp = self.client.get("/api/v1/ops/actions/capabilities", headers=self.owner_headers) + self.assertEqual(resp.status_code, 200) + data = resp.json() + self.assertIn("supported_actions", data) + self.assertIn("deploy_website", data["supported_actions"]) + self.assertIn("restart_website", data["supported_actions"]) + self.assertIn("restart_api", data["supported_actions"]) + self.assertIn("reboot_server", data["supported_actions"]) + self.assertIn("helper_configured", data) + self.assertIsNone(data["active_action_id"]) + + def test_concurrent_action_409_conflict(self): + # Insert an active action in 'running' state + with DbSession(self.engine) as db: + active = OpsAction( + id=uuid.uuid4(), + action_type="deploy_website", + requested_by_id=self.op_id, + state="running", + request_timestamp=datetime.now(timezone.utc), + ) + db.add(active) + db.commit() + + # Any new action must be rejected with 409 Conflict + resp = self.client.post( + "/api/v1/ops/actions", + json={"action_type": "restart_website"}, + headers=self.owner_headers, + ) + self.assertEqual(resp.status_code, 409) + self.assertIn("Another operations action (deploy_website) is currently active", resp.json()["detail"]) + + def test_owner_mock_execution_success(self): + from unittest.mock import patch, MagicMock + os.environ["OPS_ACTION_MOCK_EXECUTION"] = "1" + try: + fake_proc = MagicMock() + fake_proc.returncode = 0 + fake_proc.stdout = '{"status":"succeeded","deployed_revision":"d69d6119d026b1df2b7649489d1d79275013f283","status_message":"Website deployed and verified successfully."}' + fake_proc.stderr = "" + + with patch("subprocess.run", return_value=fake_proc): + resp = self.client.post( + "/api/v1/ops/actions", + json={"action_type": "deploy_website"}, + headers=self.owner_headers, + ) + self.assertEqual(resp.status_code, 200) + data = resp.json() + self.assertEqual(data["state"], "succeeded") + self.assertEqual(data["deployed_revision"], "d69d6119d026b1df2b7649489d1d79275013f283") + + # Verify audit event for success + with DbSession(self.engine) as db: + audit = db.query(OpsAuditEvent).filter_by(action="ops.action_succeeded").first() + self.assertIsNotNone(audit) + self.assertEqual(audit.result, "success") + finally: + os.environ.pop("OPS_ACTION_MOCK_EXECUTION", None) + + def test_idempotency_duplicate_returns_existing(self): + from unittest.mock import patch, MagicMock + os.environ["OPS_ACTION_MOCK_EXECUTION"] = "1" + try: + fake_proc = MagicMock() + fake_proc.returncode = 0 + fake_proc.stdout = '{"status":"succeeded","status_message":"Action completed."}' + fake_proc.stderr = "" + + key = "idemp-key-test-replay-1" + with patch("subprocess.run", return_value=fake_proc): + resp1 = self.client.post( + "/api/v1/ops/actions", + json={"action_type": "restart_website", "idempotency_key": key}, + headers=self.owner_headers, + ) + self.assertEqual(resp1.status_code, 200) + id1 = resp1.json()["id"] + + # Second request with the same idempotency key must return the exact same action ID + resp2 = self.client.post( + "/api/v1/ops/actions", + json={"action_type": "restart_website", "idempotency_key": key}, + headers=self.owner_headers, + ) + self.assertEqual(resp2.status_code, 200) + self.assertEqual(resp2.json()["id"], id1) + + # Ensure only one OpsAction was created in DB + with DbSession(self.engine) as db: + actions = db.query(OpsAction).filter_by(idempotency_key=key).all() + self.assertEqual(len(actions), 1) + finally: + os.environ.pop("OPS_ACTION_MOCK_EXECUTION", None) + + def test_mock_execution_failure(self): + from unittest.mock import patch, MagicMock + os.environ["OPS_ACTION_MOCK_EXECUTION"] = "1" + try: + fake_proc = MagicMock() + fake_proc.returncode = 1 + fake_proc.stdout = '{"failure_code":"build_error","status_message":"Build compilation failed."}' + fake_proc.stderr = "Build error" + + with patch("subprocess.run", return_value=fake_proc): + resp = self.client.post( + "/api/v1/ops/actions", + json={"action_type": "restart_website"}, + headers=self.owner_headers, + ) + self.assertEqual(resp.status_code, 200) + data = resp.json() + self.assertEqual(data["state"], "failed") + self.assertEqual(data["failure_code"], "build_error") + + # Verify audit event for failure + with DbSession(self.engine) as db: + audit = db.query(OpsAuditEvent).filter_by(action="ops.action_failed").first() + self.assertIsNotNone(audit) + self.assertEqual(audit.result, "failed") + finally: + os.environ.pop("OPS_ACTION_MOCK_EXECUTION", None) + + def test_mock_execution_rollback(self): + from unittest.mock import patch, MagicMock + os.environ["OPS_ACTION_MOCK_EXECUTION"] = "1" + try: + fake_proc = MagicMock() + fake_proc.returncode = 1 + fake_proc.stdout = '{"status":"rolled_back","rollback_status":"rolled_back","failure_code":"health_failed","status_message":"Health check failed; restored previous release."}' + fake_proc.stderr = "" + + with patch("subprocess.run", return_value=fake_proc): + resp = self.client.post( + "/api/v1/ops/actions", + json={"action_type": "deploy_website"}, + headers=self.owner_headers, + ) + self.assertEqual(resp.status_code, 200) + data = resp.json() + self.assertEqual(data["state"], "rolled_back") + self.assertEqual(data["rollback_status"], "rolled_back") + finally: + os.environ.pop("OPS_ACTION_MOCK_EXECUTION", None) + + def test_mock_execution_timeout(self): + import subprocess + from unittest.mock import patch + os.environ["OPS_ACTION_MOCK_EXECUTION"] = "1" + try: + with patch("subprocess.run", side_effect=subprocess.TimeoutExpired(cmd=["test"], timeout=10)): + resp = self.client.post( + "/api/v1/ops/actions", + json={"action_type": "restart_website"}, + headers=self.owner_headers, + ) + self.assertEqual(resp.status_code, 200) + data = resp.json() + self.assertEqual(data["state"], "failed") + self.assertEqual(data["failure_code"], "timeout") + finally: + os.environ.pop("OPS_ACTION_MOCK_EXECUTION", None) + + def test_reconciliation_on_startup(self): + import ops_actions_service + with DbSession(self.engine) as db: + # Stale API restart and server reboot should be reconciled to succeeded + act_api = OpsAction(id=uuid.uuid4(), action_type="restart_api", requested_by_id=self.op_id, state="running") + act_reboot = OpsAction(id=uuid.uuid4(), action_type="reboot_server", requested_by_id=self.op_id, state="running") + # Other stale actions should be reconciled to failed + act_deploy = OpsAction(id=uuid.uuid4(), action_type="deploy_website", requested_by_id=self.op_id, state="running") + db.add_all([act_api, act_reboot, act_deploy]) + db.commit() + + reconciled = ops_actions_service.reconcile_stale_actions(db) + self.assertEqual(reconciled, 3) + + db.refresh(act_api) + db.refresh(act_reboot) + db.refresh(act_deploy) + + self.assertEqual(act_api.state, "succeeded") + self.assertEqual(act_reboot.state, "succeeded") + self.assertEqual(act_deploy.state, "failed") + self.assertEqual(act_deploy.failure_code, "interrupted_by_restart") + + if __name__ == '__main__': unittest.main() \ No newline at end of file diff --git a/website/app/api/ops/actions/[actionId]/route.ts b/website/app/api/ops/actions/[actionId]/route.ts new file mode 100644 index 0000000..c7bc9c1 --- /dev/null +++ b/website/app/api/ops/actions/[actionId]/route.ts @@ -0,0 +1,74 @@ +import { cookies } from 'next/headers'; +import { NextRequest, NextResponse } from 'next/server'; + +export async function GET( + request: NextRequest, + { params }: { params: Promise<{ actionId: string }> } +) { + const cookieStore = await cookies(); + const isProduction = + process.env.NODE_ENV === 'production' && + process.env.NEXT_PUBLIC_APP_ENV === 'production'; + const opsToken = isProduction + ? cookieStore.get('__Host-motorsignal_ops')?.value + : cookieStore.get('__Host-motorsignal_ops')?.value ?? + cookieStore.get('motorsignal_ops')?.value; + + if (!opsToken) { + return NextResponse.json( + { detail: 'Authentication required.' }, + { status: 401 } + ); + } + + const { actionId } = await params; + if (!actionId || typeof actionId !== 'string') { + return NextResponse.json({ detail: 'Action ID is required.' }, { status: 400 }); + } + + const baseUrl = + process.env.MOTORSIGNAL_API_BASE_URL ?? 'https://api.motorsignal.app'; + try { + const response = await fetch(`${baseUrl}/api/v1/ops/actions/${encodeURIComponent(actionId)}`, { + method: 'GET', + headers: { + authorization: `Bearer ${opsToken}`, + accept: 'application/json', + }, + cache: 'no-store', + }); + + if (response.status === 401) { + return NextResponse.json( + { detail: 'Your Operations session has expired.' }, + { status: 401 } + ); + } + if (response.status === 403) { + return NextResponse.json( + { detail: 'You do not have authority to view this action.' }, + { status: 403 } + ); + } + if (response.status === 404) { + return NextResponse.json( + { detail: 'Action not found.' }, + { status: 404 } + ); + } + if (!response.ok) { + return NextResponse.json( + { detail: 'Operations platform is temporarily unavailable.' }, + { status: response.status >= 500 ? response.status : 503 } + ); + } + + const data = await response.json(); + return NextResponse.json(data); + } catch { + return NextResponse.json( + { detail: 'Unable to connect to Operations platform.' }, + { status: 503 } + ); + } +} diff --git a/website/app/api/ops/actions/capabilities/route.ts b/website/app/api/ops/actions/capabilities/route.ts new file mode 100644 index 0000000..9f2e468 --- /dev/null +++ b/website/app/api/ops/actions/capabilities/route.ts @@ -0,0 +1,60 @@ +import { cookies } from 'next/headers'; +import { NextRequest, NextResponse } from 'next/server'; + +export async function GET(request: NextRequest) { + const cookieStore = await cookies(); + const isProduction = + process.env.NODE_ENV === 'production' && + process.env.NEXT_PUBLIC_APP_ENV === 'production'; + const opsToken = isProduction + ? cookieStore.get('__Host-motorsignal_ops')?.value + : cookieStore.get('__Host-motorsignal_ops')?.value ?? + cookieStore.get('motorsignal_ops')?.value; + + if (!opsToken) { + return NextResponse.json( + { detail: 'Authentication required.' }, + { status: 401 } + ); + } + + const baseUrl = + process.env.MOTORSIGNAL_API_BASE_URL ?? 'https://api.motorsignal.app'; + try { + const response = await fetch(`${baseUrl}/api/v1/ops/actions/capabilities`, { + method: 'GET', + headers: { + authorization: `Bearer ${opsToken}`, + accept: 'application/json', + }, + cache: 'no-store', + }); + + if (response.status === 401) { + return NextResponse.json( + { detail: 'Your Operations session has expired.' }, + { status: 401 } + ); + } + if (response.status === 403) { + return NextResponse.json( + { detail: 'You do not have authority to view action capabilities.' }, + { status: 403 } + ); + } + if (!response.ok) { + return NextResponse.json( + { detail: 'Operations capabilities check temporarily unavailable.' }, + { status: response.status >= 500 ? response.status : 503 } + ); + } + + const data = await response.json(); + return NextResponse.json(data); + } catch { + return NextResponse.json( + { detail: 'Unable to connect to Operations platform.' }, + { status: 503 } + ); + } +} diff --git a/website/app/api/ops/actions/route.ts b/website/app/api/ops/actions/route.ts new file mode 100644 index 0000000..18bf480 --- /dev/null +++ b/website/app/api/ops/actions/route.ts @@ -0,0 +1,160 @@ +import { cookies } from 'next/headers'; +import { NextRequest, NextResponse } from 'next/server'; +import { hasValidOrigin } from '../../../../lib/request-origin'; + +export async function GET(request: NextRequest) { + const cookieStore = await cookies(); + const isProduction = + process.env.NODE_ENV === 'production' && + process.env.NEXT_PUBLIC_APP_ENV === 'production'; + const opsToken = isProduction + ? cookieStore.get('__Host-motorsignal_ops')?.value + : cookieStore.get('__Host-motorsignal_ops')?.value ?? + cookieStore.get('motorsignal_ops')?.value; + + if (!opsToken) { + return NextResponse.json( + { detail: 'Authentication required.' }, + { status: 401 } + ); + } + + const searchParams = request.nextUrl.searchParams; + const cursor = searchParams.get('cursor'); + const limit = searchParams.get('limit'); + const actionType = searchParams.get('action_type'); + const state = searchParams.get('state'); + + const baseUrl = + process.env.MOTORSIGNAL_API_BASE_URL ?? 'https://api.motorsignal.app'; + const targetUrl = new URL(`${baseUrl}/api/v1/ops/actions`); + if (cursor) targetUrl.searchParams.set('cursor', cursor); + if (limit) targetUrl.searchParams.set('limit', limit); + if (actionType) targetUrl.searchParams.set('action_type', actionType); + if (state) targetUrl.searchParams.set('state', state); + + try { + const response = await fetch(targetUrl.toString(), { + method: 'GET', + headers: { + authorization: `Bearer ${opsToken}`, + accept: 'application/json', + }, + cache: 'no-store', + }); + + if (response.status === 401) { + return NextResponse.json( + { detail: 'Your Operations session has expired.' }, + { status: 401 } + ); + } + if (response.status === 403) { + return NextResponse.json( + { detail: 'You do not have authority to view actions history.' }, + { status: 403 } + ); + } + if (!response.ok) { + return NextResponse.json( + { detail: 'Operations actions platform is temporarily unavailable.' }, + { status: response.status >= 500 ? response.status : 503 } + ); + } + + const data = await response.json(); + return NextResponse.json(data); + } catch { + return NextResponse.json( + { detail: 'Unable to connect to Operations platform.' }, + { status: 503 } + ); + } +} + +export async function POST(request: NextRequest) { + if (!hasValidOrigin(request)) { + return NextResponse.json({ detail: 'Invalid request origin.' }, { status: 403 }); + } + + const cookieStore = await cookies(); + const isProduction = + process.env.NODE_ENV === 'production' && + process.env.NEXT_PUBLIC_APP_ENV === 'production'; + const opsToken = isProduction + ? cookieStore.get('__Host-motorsignal_ops')?.value + : cookieStore.get('__Host-motorsignal_ops')?.value ?? + cookieStore.get('motorsignal_ops')?.value; + + if (!opsToken) { + return NextResponse.json( + { detail: 'Authentication required.' }, + { status: 401 } + ); + } + + let body: unknown; + try { + body = await request.json(); + } catch { + return NextResponse.json( + { detail: 'Invalid request body.' }, + { status: 422 } + ); + } + + const baseUrl = + process.env.MOTORSIGNAL_API_BASE_URL ?? 'https://api.motorsignal.app'; + try { + const response = await fetch(`${baseUrl}/api/v1/ops/actions`, { + method: 'POST', + headers: { + authorization: `Bearer ${opsToken}`, + 'content-type': 'application/json', + accept: 'application/json', + }, + body: JSON.stringify(body), + cache: 'no-store', + }); + + const data = await response.json().catch(() => null); + + if (response.status === 401) { + return NextResponse.json( + { detail: data?.detail ?? 'Your Operations session has expired.' }, + { status: 401 } + ); + } + if (response.status === 403) { + return NextResponse.json( + { detail: data?.detail ?? 'Only owners may trigger infrastructure operations actions.' }, + { status: 403 } + ); + } + if (response.status === 409) { + return NextResponse.json( + { detail: data?.detail ?? 'Another operations action is currently active.' }, + { status: 409 } + ); + } + if (response.status === 422) { + return NextResponse.json( + { detail: data?.detail ?? 'Invalid action payload.' }, + { status: 422 } + ); + } + if (!response.ok) { + return NextResponse.json( + { detail: data?.detail ?? 'Operations is temporarily unavailable.' }, + { status: response.status >= 500 ? response.status : 503 } + ); + } + + return NextResponse.json(data, { status: response.status }); + } catch { + return NextResponse.json( + { detail: 'Unable to connect to Operations platform.' }, + { status: 503 } + ); + } +} diff --git a/website/components/ops/operations-actions.tsx b/website/components/ops/operations-actions.tsx new file mode 100644 index 0000000..dfe8f2e --- /dev/null +++ b/website/components/ops/operations-actions.tsx @@ -0,0 +1,575 @@ +'use client'; + +import React, { useState, useEffect, useRef, useCallback } from 'react'; +import { + OperationsRole, + OpsAction, + OpsActionType, + OpsActionState, + OpsActionCapabilities, + OpsActionsResponse, +} from './types'; + +interface OperationsActionsProps { + role: OperationsRole; + onActionCompleted?: () => void; +} + +const ACTION_CONFIGS: Record< + OpsActionType, + { + title: string; + description: string; + icon: string; + destructive: boolean; + requiredConfirmation?: string; + } +> = { + deploy_website: { + title: 'Deploy Website', + description: 'Pulls latest master commit, builds Next.js production bundle, verifies health, and updates current symlink with automatic rollback.', + icon: 'rocket_launch', + destructive: false, + }, + restart_website: { + title: 'Restart Website', + description: 'Gracefully restarts the motorsignal-website systemd service and verifies /api/version ingress readiness.', + icon: 'restart_alt', + destructive: false, + }, + restart_api: { + title: 'Restart API', + description: 'Initiates detached systemd restart of motorsignal-api. In-flight requests may terminate. Type confirmation below.', + icon: 'sync_problem', + destructive: true, + requiredConfirmation: 'RESTART API', + }, + reboot_server: { + title: 'Reboot Host Server', + description: 'Schedules full host server reboot. All services, website, and API traffic will be temporarily unavailable. Type confirmation below.', + icon: 'power_settings_new', + destructive: true, + requiredConfirmation: 'REBOOT MOTORSIGNAL SERVER', + }, +}; + +const ACTIVE_STATES: OpsActionState[] = [ + 'queued', + 'preflight', + 'running', + 'verifying', + 'rollback_running', +]; + +export function OperationsActions({ role, onActionCompleted }: OperationsActionsProps) { + const [capabilities, setCapabilities] = useState(null); + const [actions, setActions] = useState([]); + const [activeAction, setActiveAction] = useState(null); + const [loading, setLoading] = useState(true); + const [historyError, setHistoryError] = useState(null); + + // Modal State + const [modalActionType, setModalActionType] = useState(null); + const [confirmationInput, setConfirmationInput] = useState(''); + const [isSubmitting, setIsSubmitting] = useState(false); + const [submitError, setSubmitError] = useState(null); + + const pollTimeoutRef = useRef(null); + const isOwner = role === 'owner'; + + // Fetch capabilities & recent action history + const fetchActions = useCallback(async () => { + try { + const resp = await fetch('/api/ops/actions?limit=10', { + headers: { accept: 'application/json' }, + cache: 'no-store', + }); + if (resp.ok) { + const data: OpsActionsResponse = await resp.json(); + setActions(data.actions || []); + setHistoryError(null); + + // Check if there is an active action + const active = (data.actions || []).find((a) => + ACTIVE_STATES.includes(a.state) + ); + setActiveAction(active || null); + } else { + setHistoryError('Unable to load operations action history.'); + } + } catch { + setHistoryError('Network error connecting to operations actions platform.'); + } finally { + setLoading(false); + } + }, []); + + const fetchCapabilities = useCallback(async () => { + try { + const resp = await fetch('/api/ops/actions/capabilities', { + headers: { accept: 'application/json' }, + cache: 'no-store', + }); + if (resp.ok) { + const data: OpsActionCapabilities = await resp.json(); + setCapabilities(data); + } + } catch { + // Non-blocking + } + }, []); + + useEffect(() => { + void fetchCapabilities(); + void fetchActions(); + }, [fetchCapabilities, fetchActions]); + + // Polling when an action is active + useEffect(() => { + let isCancelled = false; + + if (activeAction) { + const poll = async () => { + try { + const resp = await fetch(`/api/ops/actions/${encodeURIComponent(activeAction.id)}`, { + headers: { accept: 'application/json' }, + cache: 'no-store', + }); + if (resp.ok && !isCancelled) { + const updated: OpsAction = await resp.json(); + if (!ACTIVE_STATES.includes(updated.state)) { + setActiveAction(null); + void fetchActions(); + if (onActionCompleted) onActionCompleted(); + return; + } else { + setActiveAction(updated); + } + } + } catch { + // Non-blocking error + } + + if (!isCancelled) { + pollTimeoutRef.current = setTimeout(() => { + void poll(); + }, 2500); + } + }; + + pollTimeoutRef.current = setTimeout(() => { + void poll(); + }, 2500); + } + + return () => { + isCancelled = true; + if (pollTimeoutRef.current) { + clearTimeout(pollTimeoutRef.current); + pollTimeoutRef.current = null; + } + }; + }, [activeAction, fetchActions, onActionCompleted]); + + const handleOpenModal = (actionType: OpsActionType) => { + if (!isOwner || activeAction) return; + setModalActionType(actionType); + setConfirmationInput(''); + setSubmitError(null); + }; + + const handleCloseModal = () => { + setModalActionType(null); + setConfirmationInput(''); + setSubmitError(null); + }; + + const handleSubmitAction = async (e: React.FormEvent) => { + e.preventDefault(); + if (!modalActionType || !isOwner || isSubmitting) return; + + const config = ACTION_CONFIGS[modalActionType]; + if ( + config.requiredConfirmation && + confirmationInput.trim() !== config.requiredConfirmation + ) { + setSubmitError(`Please type exact confirmation text: "${config.requiredConfirmation}"`); + return; + } + + setIsSubmitting(true); + setSubmitError(null); + + const idempotencyKey = `ops-action-${Date.now()}-${Math.random().toString(36).substring(2, 9)}`; + + try { + const resp = await fetch('/api/ops/actions', { + method: 'POST', + headers: { + 'content-type': 'application/json', + accept: 'application/json', + }, + body: JSON.stringify({ + action_type: modalActionType, + confirmation: confirmationInput.trim() || undefined, + idempotency_key: idempotencyKey, + }), + }); + + const data = await resp.json().catch(() => null); + + if (resp.ok) { + setActiveAction(data); + handleCloseModal(); + void fetchActions(); + } else if (resp.status === 409) { + setSubmitError(data?.detail || 'Another operations action is currently active.'); + } else if (resp.status === 403) { + setSubmitError(data?.detail || 'Owner role authority required.'); + } else { + setSubmitError(data?.detail || 'Failed to trigger operations action.'); + } + } catch { + setSubmitError('Network failure submitting action request.'); + } finally { + setIsSubmitting(false); + } + }; + + const getStateBadge = (state: OpsActionState) => { + switch (state) { + case 'succeeded': + return ( + + check_circle + Succeeded + + ); + case 'rolled_back': + return ( + + undo + Rolled Back + + ); + case 'failed': + case 'rollback_failed': + return ( + + error + {state === 'rollback_failed' ? 'Rollback Failed' : 'Failed'} + + ); + case 'queued': + case 'preflight': + case 'running': + case 'verifying': + case 'rollback_running': + return ( + + progress_activity + {state.replace('_', ' ').toUpperCase()} + + ); + default: + return ( + + {state} + + ); + } + }; + + return ( +
+ {/* Active Action Banner */} + {activeAction && ( +
+
+
+ + progress_activity + +
+

+ Action in Progress: + + {ACTION_CONFIGS[activeAction.action_type]?.title || activeAction.action_type} + + {getStateBadge(activeAction.state)} +

+

+ {activeAction.status_message || 'Executing action across system boundary...'} +

+
+
+
+ ID: {activeAction.id.substring(0, 8)}... +
+
+
+ )} + + {/* Role Notice for Operators */} + {!isOwner && ( +
+ lock + + Owner role required to execute operational actions. Operator accounts have read-only visibility into action status and history. + +
+ )} + + {/* Execution Boundary / Helper Status Notice */} + {capabilities && !capabilities.helper_configured && ( +
+ warning + + Action execution boundary helper is unconfigured or not executable on this host. Mutating actions will be rejected by backend policy. + +
+ )} + + {/* Action Trigger Cards */} +
+ {(Object.keys(ACTION_CONFIGS) as OpsActionType[]).map((actionKey) => { + const config = ACTION_CONFIGS[actionKey]; + const isDisabled = !isOwner || !!activeAction; + + return ( +
+
+
+
+ + {config.icon} + +

{config.title}

+
+ {config.destructive && ( + + Guarded + + )} +
+

+ {config.description} +

+
+ +
+ + {actionKey} + + +
+
+ ); + })} +
+ + {/* Action History Section */} +
+
+
+

Operations Action History

+

+ Audited system actions and deployments executed across privileged execution boundary. +

+
+ +
+ + {historyError && ( +
+ {historyError} +
+ )} + + {actions.length === 0 && !loading && ( +
+ No operations actions have been executed yet. +
+ )} + + {actions.length > 0 && ( +
+ + + + + + + + + + + + + {actions.map((act) => ( + + + + + + + + + ))} + +
TimestampActionActorStatusDetailsRevision
+ {new Date(act.request_timestamp).toLocaleString(undefined, { + month: 'short', + day: 'numeric', + hour: '2-digit', + minute: '2-digit', + second: '2-digit', + })} + + {ACTION_CONFIGS[act.action_type]?.title || act.action_type} + + {act.actor_identity} + + {getStateBadge(act.state)} + + {act.status_message || (act.failure_code ? `Error: ${act.failure_code}` : '—')} + + {act.deployed_revision ? act.deployed_revision.substring(0, 8) : '—'} +
+
+ )} +
+ + {/* Confirmation Modal */} + {modalActionType && ( +
+
+
+
+ + {ACTION_CONFIGS[modalActionType].icon} + +

+ Confirm {ACTION_CONFIGS[modalActionType].title} +

+
+ +
+ +
+ {submitError && ( +
+ {submitError} +
+ )} + +

+ {ACTION_CONFIGS[modalActionType].description} +

+ + {ACTION_CONFIGS[modalActionType].requiredConfirmation && ( +
+ + setConfirmationInput(e.target.value)} + placeholder={ACTION_CONFIGS[modalActionType].requiredConfirmation} + className="w-full px-3 py-2 bg-surface-container-lowest border border-outline-variant/40 rounded-lg text-sm font-mono text-on-surface focus:outline-none focus:ring-2 focus:ring-rose-500/50" + /> +
+ )} + +
+ + +
+
+
+
+ )} +
+ ); +} diff --git a/website/components/ops/operations-infrastructure.tsx b/website/components/ops/operations-infrastructure.tsx index 2614d32..7efa722 100644 --- a/website/components/ops/operations-infrastructure.tsx +++ b/website/components/ops/operations-infrastructure.tsx @@ -8,6 +8,7 @@ import { OperationsRole, } from './types'; import { normalizeInfrastructureResponse } from '../../lib/ops-data'; +import { OperationsActions } from './operations-actions'; interface OperationsInfrastructureProps { role: OperationsRole; @@ -176,7 +177,7 @@ export function OperationsInfrastructure({ role }: OperationsInfrastructureProps Infrastructure & Runtime - Phase 1 (Read-Only) + Phase 2 Operations

@@ -608,11 +609,16 @@ export function OperationsInfrastructure({ role }: OperationsInfrastructureProps {/* Deployment History Status */}

Deployment History: - Unavailable in Phase 1 + Audited in Phase 2
+ + {/* Operations Actions & Deployment Controls */} +
+ +
); } diff --git a/website/components/ops/types.ts b/website/components/ops/types.ts index f2be23c..2a8c775 100644 --- a/website/components/ops/types.ts +++ b/website/components/ops/types.ts @@ -389,4 +389,50 @@ export interface OperationsInfrastructureData { collected_at: string; } +export type OpsActionType = + | 'deploy_website' + | 'restart_website' + | 'restart_api' + | 'reboot_server'; +export type OpsActionState = + | 'queued' + | 'preflight' + | 'running' + | 'verifying' + | 'succeeded' + | 'failed' + | 'rollback_running' + | 'rolled_back' + | 'rollback_failed'; + +export interface OpsAction { + id: string; + action_type: OpsActionType; + actor_identity: string; + target_environment: string; + state: OpsActionState; + request_timestamp: string; + start_timestamp: string | null; + completion_timestamp: string | null; + failure_code: string | null; + status_message: string | null; + rollback_status: string | null; + health_result: string | null; + deployed_revision: string | null; + idempotency_key: string | null; + case_reference: string | null; +} + +export interface OpsActionsResponse { + actions: OpsAction[]; + next_cursor: string | null; + total?: number; +} + +export interface OpsActionCapabilities { + allowed_actions: OpsActionType[]; + confirmation_required: OpsActionType[]; + helper_configured: boolean; + can_execute: boolean; +} diff --git a/website/tests/ops-experience.test.cjs b/website/tests/ops-experience.test.cjs index 87e5a3f..497238f 100644 --- a/website/tests/ops-experience.test.cjs +++ b/website/tests/ops-experience.test.cjs @@ -1365,6 +1365,117 @@ assert.ok( console.log(' PASS: Section 43 Production Operations Infrastructure Dashboard (Phase 1) verified.'); -console.log('--- ALL OPS-3C, OPS-3C.1, OPS-3C.2, OPS-3C.3, OPS-4, OPS-4.2, OVERVIEW, SECTION 2 USERS, SECTION 3 SUPPORT CENTER, SECTION 4 CMS & SECTION 43 INFRASTRUCTURE TESTS PASSED (43/43) ---'); +// ============================================================ +// 44. OPERATIONS ACTIONS & DEPLOYMENT CONTROLS (PHASE 2) +// ============================================================ +console.log('Testing 44: Operations Actions & Deployment Controls (Phase 2)...'); +// 44.1 UI Component Contracts (operations-actions.tsx) +const actionsComponentSrc = fs.readFileSync(path.resolve(__dirname, '../components/ops/operations-actions.tsx'), 'utf8'); +// Strict Action allowlist and confirmation requirements +assert.ok(actionsComponentSrc.includes('deploy_website'), 'Actions component must include deploy_website'); +assert.ok(actionsComponentSrc.includes('restart_website'), 'Actions component must include restart_website'); +assert.ok(actionsComponentSrc.includes('restart_api'), 'Actions component must include restart_api'); +assert.ok(actionsComponentSrc.includes('reboot_server'), 'Actions component must include reboot_server'); + +assert.ok( + actionsComponentSrc.includes('RESTART API'), + 'Actions component must require exact typed confirmation "RESTART API"' +); +assert.ok( + actionsComponentSrc.includes('REBOOT MOTORSIGNAL SERVER'), + 'Actions component must require exact typed confirmation "REBOOT MOTORSIGNAL SERVER"' +); + +// Role restriction +assert.ok( + actionsComponentSrc.includes("role === 'owner'") || actionsComponentSrc.includes('isOwner'), + 'Actions component must restrict action triggers to owner role' +); + +// Mutual exclusion & active action handling +assert.ok( + actionsComponentSrc.includes('activeAction'), + 'Actions component must track and display active running actions' +); + +// History table columns +assert.ok(actionsComponentSrc.includes('Timestamp'), 'History table must display Timestamp column'); +assert.ok(actionsComponentSrc.includes('Action'), 'History table must display Action column'); +assert.ok(actionsComponentSrc.includes('Actor'), 'History table must display Actor column'); +assert.ok(actionsComponentSrc.includes('Status'), 'History table must display Status column'); +assert.ok(actionsComponentSrc.includes('Revision'), 'History table must display Revision column'); + +// Embedded in Infrastructure view +const updatedInfraSrc = fs.readFileSync(path.resolve(__dirname, '../components/ops/operations-infrastructure.tsx'), 'utf8'); +assert.ok( + updatedInfraSrc.includes('OperationsActions'), + 'OperationsInfrastructure must embed OperationsActions component' +); + +// 44.2 Next.js API Route Proxies for Actions +const actionsRouteSrc = fs.readFileSync(path.resolve(__dirname, '../app/api/ops/actions/route.ts'), 'utf8'); +assert.ok( + actionsRouteSrc.includes('hasValidOrigin'), + 'Next.js actions route must enforce CSRF protection via hasValidOrigin' +); +assert.ok( + actionsRouteSrc.includes('__Host-motorsignal_ops'), + 'Next.js actions route must enforce __Host-motorsignal_ops in production' +); +assert.ok( + actionsRouteSrc.includes('/api/v1/ops/actions'), + 'Next.js actions route must proxy to backend /api/v1/ops/actions' +); +assert.ok( + actionsRouteSrc.includes('409'), + 'Next.js actions route must forward 409 Conflict status' +); + +const capabilitiesRouteSrc = fs.readFileSync(path.resolve(__dirname, '../app/api/ops/actions/capabilities/route.ts'), 'utf8'); +assert.ok( + capabilitiesRouteSrc.includes('/api/v1/ops/actions/capabilities'), + 'Capabilities proxy must query backend capabilities endpoint' +); + +const actionDetailRouteSrc = fs.readFileSync(path.resolve(__dirname, '../app/api/ops/actions/[actionId]/route.ts'), 'utf8'); +assert.ok( + actionDetailRouteSrc.includes('/api/v1/ops/actions/'), + 'Action detail proxy must query backend action by ID' +); + +// 44.3 Backend Endpoints & Role Gates in main.py +assert.ok( + backendMainSrc.includes('@app.post("/api/v1/ops/actions'), + 'Backend main.py must register POST /api/v1/ops/actions' +); +assert.ok( + backendMainSrc.includes('@app.get("/api/v1/ops/actions'), + 'Backend main.py must register GET /api/v1/ops/actions' +); +assert.ok( + backendMainSrc.includes('@app.get("/api/v1/ops/actions/capabilities'), + 'Backend main.py must register GET /api/v1/ops/actions/capabilities' +); +assert.ok( + backendMainSrc.includes('@app.get("/api/v1/ops/actions/{action_id}'), + 'Backend main.py must register GET /api/v1/ops/actions/{action_id}' +); +assert.ok( + backendMainSrc.includes('reconcile_stale_actions'), + 'Backend main.py must reconcile stale actions on startup' +); + +// 44.4 Privileged Boundary Helper Script +const helperSrc = fs.readFileSync(path.resolve(__dirname, '../../backend/motorsignal_api/scripts/ops-action-helper.sh'), 'utf8'); +assert.ok(helperSrc.includes('deploy_website'), 'Helper must handle deploy_website'); +assert.ok(helperSrc.includes('restart_website'), 'Helper must handle restart_website'); +assert.ok(helperSrc.includes('restart_api'), 'Helper must handle restart_api'); +assert.ok(helperSrc.includes('reboot_server'), 'Helper must handle reboot_server'); +assert.ok(helperSrc.includes('PREV_RELEASE'), 'Helper must retain PREV_RELEASE for automatic rollback'); +assert.ok(helperSrc.includes('/api/version'), 'Helper must verify /api/version ingress readiness'); + +console.log(' PASS: Section 44 Operations Actions & Deployment Controls (Phase 2) verified.'); + +console.log('--- ALL OPS-3C, OPS-3C.1, OPS-3C.2, OPS-3C.3, OPS-4, OPS-4.2, OVERVIEW, SECTION 2 USERS, SECTION 3 SUPPORT CENTER, SECTION 4 CMS, SECTION 43 INFRASTRUCTURE & SECTION 44 ACTIONS TESTS PASSED (44/44) ---');