diff --git a/.gitignore b/.gitignore index 2e544a0..9fd5254 100644 --- a/.gitignore +++ b/.gitignore @@ -61,3 +61,16 @@ gemini-mcp-config-search.txt *.keystore android/key.properties android/local.properties +# Playwright CLI output (may contain credentials) +.playwright-cli/ +.playwright/ +k3-mcp-toolbox-public/ + +# Targeted generated and transient artifacts +*.tar.gz +website/tsconfig.tsbuildinfo +website/tests/screenshots/ +website/build_failures.log +scratch_*.py +*.scratch.* +1.29 diff --git a/1.29 b/1.29 deleted file mode 100644 index e69de29..0000000 diff --git a/docs/CODEX_HANDOFF_2026-09-08.md b/docs/CODEX_HANDOFF_2026-09-08.md new file mode 100644 index 0000000..3774716 --- /dev/null +++ b/docs/CODEX_HANDOFF_2026-09-08.md @@ -0,0 +1,239 @@ +# MotorSignal Agent Handoff + +Last updated: 2026-09-08 + +## Mission + +MotorSignal is a Flutter motorcycle telemetry app with a Next.js companion website and a FastAPI/PostgreSQL production API. Current work covers the public website, Firebase-linked app data, RBAC/admin operations, and VPS deployment. + +Workspace: + + C:\Users\Bill\Desktop\flutter-app + +Production: + + Website: https://motorsignal.app + API: https://api.motorsignal.app + VPS: motorsignal-admin@89.31.239.10 + +Never put SSH keys, passwords, Firebase private credentials, JWT secrets, SMTP credentials, or database passwords in this file or Git. + +## Read First + +1. Root AGENTS.md and its referenced C:\Users\Bill\.codex\RTK.md. +2. docs/PRODUCTION_DEPLOYMENT_CHECKLIST.md. +3. docs/WEBSITE_ADMIN_CURRENT_STATE.md. +4. Relevant docs under docs/agents/. + +Rules that matter: + +- Investigate source, callers, schemas, and tests before changing behavior. +- Do not casually change Flutter telemetry cadence, SensorFusion, RideStateMachine, or Drift schema 7. +- Preserve the offline-first ride sync pipeline and stable cloud ride IDs. +- Behavioral changes need focused automated coverage. +- Preserve unrelated dirty-worktree changes. +- Do not commit or push without explicit authorization. + +## Current Local State + +The worktree is intentionally dirty. Preserve all existing changes. + +The website currently includes: + +- Firebase account linking and Firestore app-data display. +- Password reset, email verification, and session management. +- RBAC-protected admin console with user, vehicle, ride, and audit views. +- Stitch-inspired landing page. +- Completed public Features, Group Rides, Clubs, and Pricing pages. +- Public menu links for Features, Group rides, Clubs, Pricing, Download, and Support. +- Hero image at website/public/images/motorsignal-hero.jpg. + +Important files: + + website/app/page.tsx + website/app/features/page.tsx + website/app/group-rides/page.tsx + website/app/clubs/page.tsx + website/app/pricing/page.tsx + website/app/account/page.tsx + website/app/admin/page.tsx + website/app/globals.css + website/components/site-header.tsx + website/components/companion-list.tsx + website/components/app-cloud-data.tsx + website/components/firebase-link.tsx + website/components/session-manager.tsx + website/lib/firebase-client.ts + +The latest public-page source is newer than the deployed bundles. Build and package it before deploying. + +## Stitch Reference Assets + +Downloaded under stitch/admin-dashboard-revamp/: + +- flagship-landing/ +- features-telemetry-architecture/ +- group-rides-pack-radar/ +- clubs-fleet-intelligence/ +- pricing-hardware-plans/ +- rider-account-portal/ +- Admin console screenshot and generated HTML + +These are design references. Do not copy fake Stitch KPI values into production data displays unless labeled illustrative. Admin KPIs should come from real API totals. + +## Firebase Data Contract + +The Flutter app writes: + + users/{firebaseUid}/motorcycles/{motorcycleId} + users/{firebaseUid}/rides/{cloudRideId} + +Motorcycle fields include year, makeName, modelName, variantName, nickname, and isDefault. + +Ride fields include startTime, distanceMiles, topSpeedMph, motorcycleYear, motorcycleMake, motorcycleModel, qualityRating, and telemetry bundle metadata. + +Website AppCloudData reads these owner-scoped collections after Google/Firebase account linking. Firestore rules enforce the Firebase owner UID. Do not merge records using email alone. + +Firebase project: + + motoapp-d22e0 + +Backend Firebase linking validates the identity and stores users.firebase_uid. X.509 certificate parsing is fixed and covered by a regression test. + +## Backend and Production + +Backend source: + + backend/motorsignal_api/ + +Important files: + + backend/motorsignal_api/alembic/versions/e0f5a7123b46_link_firebase_identity.py + backend/motorsignal_api/tests/test_firebase_auth.py + backend/motorsignal_api/tests/test_api_smoke.py + +Services: + + motorsignal-api.service + motorsignal-website.service + nginx.service + +Release layout: + + /opt/motorsignal/current + /opt/motorsignal/releases/ + +The API expects /opt/motorsignal/current/app and /opt/motorsignal/current/venv. + +## Deployment History + + 20260908-06 Account Firestore app-data display + 20260908-07 Stitch-inspired admin console + 20260908-08 First public landing/features/rides/clubs implementation + 20260908-09 Expanded landing page + +Checksums: + + 20260908-08 website: + 5ab316a7889b76185bee3af86798bf51d22c4dac41907afe27104c0d4d2ea91c + + 20260908-09 website: + ab561d8974ab10f9f9755c7c54d2df71ea64b4a6e3417d60c0ff112ea10d5f94 + +The latest full Features, Group Rides, Clubs, and Pricing implementation was made after release 09 and has not been packaged. Build a new release, preferably 20260908-10, from the current source. + +The active VPS release is not confirmed. Check it: + + readlink -f /opt/motorsignal/current + +## Deployment Procedure + +Build: + + cd website + npm run build + +Package the current standalone website: + + $release='20260908-10' + $stage="scratch/deploy/$release" + New-Item -ItemType Directory -Path "$stage/website" -Force | Out-Null + Copy-Item website/.next/standalone/* "$stage/website" -Recurse -Force + New-Item -ItemType Directory -Path "$stage/website/.next/static" -Force | Out-Null + Copy-Item website/.next/static/* "$stage/website/.next/static" -Recurse -Force + if (Test-Path website/public) { Copy-Item website/public "$stage/website/public" -Recurse -Force } + Compress-Archive -Path "$stage/website" -DestinationPath "$stage/motorsignal-website-$release.zip" -Force + +Upload with the existing SSH key: + + scp -i C:\Users\Bill\.ssh\motorsignal_prod scratch/deploy/20260908-10/motorsignal-backend-20260908-10.zip scratch/deploy/20260908-10/motorsignal-website-20260908-10.zip motorsignal-admin@89.31.239.10:/tmp/ + +Verify SHA-256 on both local and remote files before staging. + +Root/provider-console cutover: + + set -euo pipefail + RELEASE=20260908-10 + DIR=/opt/motorsignal/releases/$RELEASE + printf 'DIR=%s\n' "$DIR" + test "$DIR" = "/opt/motorsignal/releases/$RELEASE" + install -d -o motorsignal -g motorsignal "$DIR" + unzip -q /tmp/motorsignal-backend-$RELEASE.zip -d "$DIR" + unzip -q /tmp/motorsignal-website-$RELEASE.zip -d "$DIR" + ln -sfn backend/app "$DIR/app" + ln -sfn backend/alembic "$DIR/alembic" + ln -sfn backend/alembic.ini "$DIR/alembic.ini" + ln -sfn backend/requirements.lock "$DIR/requirements.lock" + cp -a /opt/motorsignal/current/venv "$DIR/venv" + chown -R motorsignal:motorsignal "$DIR" + ln -sfn "$DIR" /opt/motorsignal/current + systemctl restart motorsignal-api + systemctl restart motorsignal-website + sleep 3 + curl -fsS https://api.motorsignal.app/api/v1/health + curl -fsS -I https://motorsignal.app/ + curl -fsS -I https://motorsignal.app/features + curl -fsS -I https://motorsignal.app/group-rides + curl -fsS -I https://motorsignal.app/clubs + curl -fsS -I https://motorsignal.app/pricing + +## Current Deployment Blocker + +The user reported: + + ln: failed to create symbolic link '/opt/motorsignal/releases/20260908-08/app/app': File exists + +This means DIR was likely set one level too deep, ending in /app, or a command was run with an incorrect working directory. The correct target is: + + /opt/motorsignal/releases/20260908-08/app + +not: + + /opt/motorsignal/releases/20260908-08/app/app + +Always reset and print DIR before symlink commands. Use ln -sfn for known release-local symlinks. Do not recursively delete a release without inspecting it using find, stat, and readlink. + +## Validation Remaining + +The latest local website build passed: + + npm run build + +Still needed: + +- Package the latest source as release 20260908-10. +- Deploy and verify the VPS release. +- Browser-check the public routes after deployment. +- Test signed-in Firebase-backed Group Rides, Clubs, and Account data with the real account. +- Confirm the hero image loads through the standalone bundle. +- Run the complete Flutter gates before declaring the broader project complete: + + flutter test + flutter analyze + flutter build apk --debug + +Do not claim Firebase rides exist unless the app has completed cloud sync. Local-only rides will not appear in Firestore. + +## Git Safety + +No commit or push was authorized for this handoff. Do not stage the entire repository because it contains unrelated ongoing work. Stage only explicitly requested files if a later user authorizes a commit. diff --git a/docs/OPS4_PRODUCTION_COMMAND_CARD.md b/docs/OPS4_PRODUCTION_COMMAND_CARD.md new file mode 100644 index 0000000..ebac3c5 --- /dev/null +++ b/docs/OPS4_PRODUCTION_COMMAND_CARD.md @@ -0,0 +1,215 @@ +# MotorSignal OPS-4 Production Command Card +*(Hardened Operator Companion Card)* + +> **OPERATOR COMPANION CARD — DOCUMENTATION & PLANNING ONLY** +> Checkpoint Commit: `8681e6f25972cea768106e534b11418b58374d28` +> PostgreSQL Engine: `PostgreSQL 14.24` | Database: `motorsignal` (User: `motorsignal_app`) +> Baseline Alembic Revision: `e0f5a7123b46` $\to$ Target Alembic Revision: `f1a2b3c4d5e6` +> Authoritative Runbook: `docs/OPS4_PRODUCTION_DEPLOYMENT_RUNBOOK.md` + +--- + +## 1. Preflight & Release Artifact Verification + +```bash +# 1. SSH into host & record current active release +ssh -i ~/.ssh/motorsignal_prod motorsignal-admin@89.31.239.10 +PREV_RELEASE=$(readlink -f /opt/motorsignal/current) +echo "PREVIOUS_RELEASE=${PREV_RELEASE}" + +# 2. Stage new release directory & unpack artifacts +RELEASE_ID="20260910-ops4" +NEW_RELEASE="/opt/motorsignal/releases/${RELEASE_ID}" +sudo install -d -o motorsignal -g motorsignal "${NEW_RELEASE}/backend" "${NEW_RELEASE}/website" +sudo -u motorsignal unzip -q /tmp/motorsignal-backend-${RELEASE_ID}.zip -d "${NEW_RELEASE}/backend" +sudo -u motorsignal unzip -q /tmp/motorsignal-website-${RELEASE_ID}.zip -d "${NEW_RELEASE}/website" + +# 3. Create compatibility symlinks & dedicated venv +sudo -u motorsignal ln -sfn backend/app "${NEW_RELEASE}/app" +sudo -u motorsignal ln -sfn backend/alembic "${NEW_RELEASE}/alembic" +sudo -u motorsignal ln -sfn backend/alembic.ini "${NEW_RELEASE}/alembic.ini" +sudo -u motorsignal ln -sfn backend/requirements.lock "${NEW_RELEASE}/requirements.lock" +sudo -u motorsignal python3 -m venv "${NEW_RELEASE}/venv" +sudo -u motorsignal "${NEW_RELEASE}/venv/bin/pip" install --no-cache-dir -r "${NEW_RELEASE}/requirements.lock" + +# 4. Mandatory Artifact Preflight Gate +"${NEW_RELEASE}/venv/bin/alembic" -c "${NEW_RELEASE}/alembic.ini" heads +# MUST output exactly: f1a2b3c4d5e6 (head) +sudo -u postgres psql -d motorsignal -t -A -c "SELECT version_num FROM alembic_version;" +# MUST output exactly: e0f5a7123b46 +``` + +--- + +## 2. Safe Pre-Migration Backup & Restore Drill + +```bash +# 1. Execute safe custom-format backup (No passwords in argv) +sudo install -d -m 0750 -o postgres -g postgres /var/backups/motorsignal +BACKUP_TS=$(date +%Y%m%d_%H%M%S) +BACKUP_FILE="/var/backups/motorsignal/motorsignal_pre_ops4_${BACKUP_TS}.dump" +export BACKUP_FILE + +sudo -u motorsignal BACKUP_FILE="${BACKUP_FILE}" "${NEW_RELEASE}/venv/bin/python" - <<'PY' +import os, subprocess, sys +from dotenv import dotenv_values +env_vars = dotenv_values("/etc/motorsignal/production.env") +pg_env = os.environ.copy() +pg_env["PGHOST"] = env_vars.get("DATABASE_HOST", "127.0.0.1") +pg_env["PGPORT"] = str(env_vars.get("DATABASE_PORT", "5432")) +pg_env["PGDATABASE"] = env_vars.get("DATABASE_NAME", "motorsignal") +pg_env["PGUSER"] = env_vars.get("DATABASE_USER", "motorsignal_app") +if "DATABASE_PASSWORD" in env_vars: + pg_env["PGPASSWORD"] = env_vars["DATABASE_PASSWORD"] +res = subprocess.run(["pg_dump", "-Fc", "-v", "-f", os.environ["BACKUP_FILE"], pg_env["PGDATABASE"]], env=pg_env) +sys.exit(res.returncode) +PY + +test -s "${BACKUP_FILE}" && sha256sum "${BACKUP_FILE}" | sudo tee "${BACKUP_FILE}.sha256" +sudo -u postgres pg_restore --list "${BACKUP_FILE}" | head -n 20 + +# 2. Disposable Restore Drill (Mandatory Gate) +DRILL_DB="motorsignal_restore_drill_${BACKUP_TS}" +sudo -u postgres psql -c "CREATE DATABASE ${DRILL_DB};" +sudo -u postgres pg_restore -v --no-owner --no-privileges -d "${DRILL_DB}" "${BACKUP_FILE}" +sudo -u postgres psql -d "${DRILL_DB}" -t -A -c "SELECT version_num FROM alembic_version;" +# MUST output: e0f5a7123b46 +sudo -u postgres psql -c "DROP DATABASE ${DRILL_DB};" +``` + +--- + +## 3. Quiesce & Final Backup + +```bash +# Quiesce application writers (Nginx remains active) +sudo systemctl stop motorsignal-api.service motorsignal-website.service +sudo systemctl is-active motorsignal-api.service || echo "API_STOPPED" +sudo systemctl is-active postgresql@14-main.service + +# Final quiesced snapshot +FINAL_BACKUP_FILE="/var/backups/motorsignal/motorsignal_final_quiesced_${BACKUP_TS}.dump" +sudo -u motorsignal BACKUP_FILE="${FINAL_BACKUP_FILE}" "${NEW_RELEASE}/venv/bin/python" - <<'PY' +import os, subprocess, sys +from dotenv import dotenv_values +env_vars = dotenv_values("/etc/motorsignal/production.env") +pg_env = os.environ.copy() +pg_env["PGHOST"] = env_vars.get("DATABASE_HOST", "127.0.0.1") +pg_env["PGPORT"] = str(env_vars.get("DATABASE_PORT", "5432")) +pg_env["PGDATABASE"] = env_vars.get("DATABASE_NAME", "motorsignal") +pg_env["PGUSER"] = env_vars.get("DATABASE_USER", "motorsignal_app") +if "DATABASE_PASSWORD" in env_vars: + pg_env["PGPASSWORD"] = env_vars["DATABASE_PASSWORD"] +res = subprocess.run(["pg_dump", "-Fc", "-v", "-f", os.environ["BACKUP_FILE"], pg_env["PGDATABASE"]], env=pg_env) +sys.exit(res.returncode) +PY +test -s "${FINAL_BACKUP_FILE}" && sha256sum "${FINAL_BACKUP_FILE}" | sudo tee "${FINAL_BACKUP_FILE}.sha256" +``` + +--- + +## 4. Single-Execution Database Migration + +```bash +# Execute Alembic upgrade safely via Python runner (STOP IMMEDIATELY ON ERROR) +sudo -u motorsignal NEW_RELEASE="${NEW_RELEASE}" "${NEW_RELEASE}/venv/bin/python" - <<'PY' +import os, subprocess, sys +from dotenv import dotenv_values +release = os.environ["NEW_RELEASE"] +env_vars = dotenv_values("/etc/motorsignal/production.env") +run_env = os.environ.copy() +for k, v in env_vars.items(): + if v is not None: + run_env[k] = v +res = subprocess.run( + [f"{release}/venv/bin/alembic", "-c", f"{release}/alembic.ini", "upgrade", "f1a2b3c4d5e6"], + cwd=release, + env=run_env +) +sys.exit(res.returncode) +PY +# STOP IMMEDIATELY if process exits non-zero! +``` + +--- + +## 5. Post-Migration DB Verification + +```bash +# Verify Alembic version is f1a2b3c4d5e6 +sudo -u postgres psql -d motorsignal -t -A -c "SELECT version_num FROM alembic_version;" + +# Verify ops schema & 5 tables +sudo -u postgres psql -d motorsignal -c " +SELECT table_name FROM information_schema.tables WHERE table_schema = 'ops' ORDER BY table_name;" + +# Verify sequence, function, and trigger +sudo -u postgres psql -d motorsignal -c " +SELECT sequence_name FROM information_schema.sequences WHERE sequence_schema = 'ops'; +SELECT routine_name FROM information_schema.routines WHERE routine_schema = 'ops'; +SELECT trigger_name, event_object_table FROM information_schema.triggers WHERE trigger_schema = 'ops';" +``` + +--- + +## 6. Atomic Symlink Cutover & Service Start + +```bash +# 1. Pre-check: current.next must not exist +test ! -e /opt/motorsignal/current.next || { echo "STOP: current.next exists!"; exit 1; } + +# 2. Create temporary symlink & verify +sudo -u motorsignal ln -s "${NEW_RELEASE}" /opt/motorsignal/current.next +test "$(readlink -f /opt/motorsignal/current.next)" = "${NEW_RELEASE}" || { echo "STOP: target mismatch!"; exit 1; } + +# 3. Truly atomic cutover via Linux rename +sudo mv -Tf /opt/motorsignal/current.next /opt/motorsignal/current +test "$(readlink -f /opt/motorsignal/current)" = "${NEW_RELEASE}" && echo "CUTOVER_OK" + +# 4. Start services +sudo systemctl start motorsignal-api.service motorsignal-website.service +sudo systemctl status motorsignal-api.service motorsignal-website.service --no-pager +``` + +--- + +## 7. Local & Public Health Checks + +```bash +# Local loopback probes +curl -fsS http://127.0.0.1:8080/api/v1/health +curl -fsS http://127.0.0.1:8080/api/v1/ready +curl -fsS http://127.0.0.1:8080/api/v1/ops/health +curl -fsS -I http://127.0.0.1:3001/ + +# Public HTTPS probes +curl -fsS https://api.motorsignal.app/api/v1/health +curl -fsS https://api.motorsignal.app/api/v1/ready +curl -fsS -I https://motorsignal.app/ + +# Safe unauthenticated security probes (Must return HTTP 401, NOT 404) +curl -s -o /dev/null -w "%{http_code}\n" https://api.motorsignal.app/api/v1/ops/me +curl -s -o /dev/null -w "%{http_code}\n" https://api.motorsignal.app/api/v1/ops/incidents +``` + +--- + +## 8. Rollback Triggers & Fast Action + +### Case A: Application Failure (No New Writes) +```bash +# 1. Stop services +sudo systemctl stop motorsignal-api.service motorsignal-website.service +# 2. Atomically repoint symlink to previous release +sudo -u motorsignal ln -s "${PREV_RELEASE}" /opt/motorsignal/current.prev +sudo mv -Tf /opt/motorsignal/current.prev /opt/motorsignal/current +# 3. Restart previous services & verify +sudo systemctl start motorsignal-api.service motorsignal-website.service +curl -fsS http://127.0.0.1:8080/api/v1/health && curl -fsS http://127.0.0.1:8080/api/v1/ready +``` + +### Case B: Migration Non-Zero Exit +- **DO NOT** restart application services. +- **DO NOT** run `alembic stamp`. +- **DO NOT** retry repeatedly. +- Keep writers stopped; inspect `journalctl` and PostgreSQL logs. diff --git a/docs/OPS4_PRODUCTION_DEPLOYMENT_RUNBOOK.md b/docs/OPS4_PRODUCTION_DEPLOYMENT_RUNBOOK.md new file mode 100644 index 0000000..ef61b00 --- /dev/null +++ b/docs/OPS4_PRODUCTION_DEPLOYMENT_RUNBOOK.md @@ -0,0 +1,720 @@ +# MotorSignal OPS-4 Production Deployment & Database Migration Runbook +*(Hardened Final Execution Guide)* + +> **NOTICE: DOCUMENTATION & PLANNING ONLY** +> This runbook is a procedural guide for human execution during an authorized maintenance window on the MotorSignal production VPS. +> **DO NOT EXECUTE AUTOMATICALLY.** Every destructive or state-changing step requires human review, validation, and confirmation. + +--- + +## Document Metadata & Verified Baseline + +- **Target Checkpoint Commit:** `8681e6f25972cea768106e534b11418b58374d28` +- **Target Branch:** `fix/restore-clean-build` +- **Production Host:** `api.motorsignal.app` (`89.31.239.10`) +- **Operating System:** Ubuntu 22.04.5 LTS (Jammy Jellyfish) +- **Database Engine:** PostgreSQL 14.24 (`postgresql@14-main.service` on `127.0.0.1:5432`) +- **Database Name / User:** `database=motorsignal`, `user=motorsignal_app` (Size: ~9.2 MB) +- **Pre-Migration Alembic Revision:** `e0f5a7123b46` (`link_firebase_identity`) +- **Target Alembic Revision:** `f1a2b3c4d5e6` (`add_ops_schema_and_incidents`) +- **Authoritative Migration Path:** + `e0f5a7123b46` $\to$ `f1a6b8234c57` $\to$ `f1b7c8345d68` $\to$ `f1a2b3c4d5e6` +- **Deployment User / Service User:** `motorsignal-admin` (deployer) / `motorsignal:motorsignal` (service account) +- **Service Ingress & Ports:** + - API: `motorsignal-api.service` (`127.0.0.1:8080`, `WorkingDirectory=/opt/motorsignal/current/app`) + - Website: `motorsignal-website.service` (`127.0.0.1:3001`, Next.js standalone `server.js`) + - Ingress: `nginx.service` (proxies `api.motorsignal.app` $\to$ `8080`, `motorsignal.app` $\to$ `3001`) +- **Authoritative Backup Directory:** `/var/backups/motorsignal` (DO NOT use `/opt/motorsignal/backups`) +- **Host Security Baseline:** Fail2ban active; UFW inactive (DO NOT alter firewall during deployment). + +--- + +## Core Operating Principles + +1. **Investigate First & Stop on Error:** If any step produces an unexpected result, error, or revision mismatch, **STOP IMMEDIATELY**. Never "continue anyway" or retry failing commands blindly. +2. **Zero-Drift & Safe Secret Handling:** Follow the exact commands specified. Never expand secrets into command-line arguments or echo passwords. Pass environment variables in-memory using dedicated Python runners. +3. **Controlled Quiescence:** Application writers (`motorsignal-api.service` and `motorsignal-website.service`) must be stopped before database migration to eliminate version-skew writes. +4. **Immutable Releases & Truly Atomic Cutover:** Deploy via timestamped release directories (`/opt/motorsignal/releases/`) and atomic filesystem renames (`current.next` $\to$ `current`). +5. **Rollback Discipline:** Never run blind database downgrades on production after writes have occurred. Emergency disaster recovery uses the verified pre-migration backup. + +--- + +# PHASE 0 — Scope & Safety + +### 0.1 Deployment Scope Summary +- **Backend API:** FastAPI application update supporting OPS-4 operational incidents workflow, CSRF origin verification on mutations, audit-event rollback safety, and rate-limiting enhancements. +- **Operations Frontend:** Next.js Operations console (`/admin`, `/admin/incidents`, `/admin/incidents/[id]`, `/admin/incidents/create`), session management, and CSRF-protected proxy routes. +- **Database Schema:** PostgreSQL `ops` schema creation and evolution: + - `ops.operators`: Operations staff accounts with `authority_version` and immutable identity trigger. + - `ops.sessions`: Operator sessions tied to `__Host-motorsignal_ops` cookies. + - `ops.audit_events`: Tamper-evident operational audit trail. + - `ops.incidents`: Incident management records with case references (`OPS-YYYY-NNNNNN`). + - `ops.incident_notes`: Internal operator notes. + - `ops.incident_case_seq`: Sequence for case reference generation. + - `ops.guard_operator_identity()`: Trigger function enforcing identity immutability. + +--- + +# PHASE 1 — Preflight & Host Baseline Checks + +### 1.1 VPS Connection & Identity Verification +- **Precondition:** SSH access configured with deployment key `C:\Users\Bill\.ssh\motorsignal_prod`. +- **Command:** + ```bash + ssh -i ~/.ssh/motorsignal_prod motorsignal-admin@89.31.239.10 + hostname + whoami + id + ``` +- **Expected Result:** Hostname is `api.motorsignal.app`; user is `motorsignal-admin` (`uid=1000`, member of `sudo` and `motorsignal` groups). +- **Failure Action:** **STOP**. Do not proceed on an unverified host. + +### 1.2 Host Storage & Resources Inspection +- **Command:** + ```bash + # Storage check (require >= 10 GB free on root /) + df -h / + + # Memory and load check (require >= 1 GB available RAM) + free -m + uptime + ``` +- **Expected Result:** Free disk $\ge$ 10 GB (discovery showed ~272 GB free); available RAM $\ge$ 1 GB; system load normal. +- **Failure Action:** **STOP**. Resolve resource constraints before proceeding. + +### 1.3 Service Baseline Status +- **Command:** + ```bash + systemctl status motorsignal-api.service --no-pager + systemctl status motorsignal-website.service --no-pager + systemctl status nginx.service --no-pager + systemctl status postgresql@14-main.service --no-pager + ``` +- **Expected Result:** All services reporting `active (running)`. +- **Failure Action:** **STOP**. Investigate pre-existing service failures. + +### 1.4 PostgreSQL 14.24 Engine & Exact Alembic Revision Gate +- **Command:** + ```bash + # Check PostgreSQL version + sudo -u postgres psql -d motorsignal -c "SELECT version();" + + # Query exact current Alembic revision + sudo -u postgres psql -d motorsignal -t -A -c "SELECT version_num FROM alembic_version;" + ``` +- **Expected Result:** Version string indicates PostgreSQL 14.24; revision is EXACTLY `e0f5a7123b46`. +- **Failure Action:** **STOP IMMEDIATELY**. If the revision is not `e0f5a7123b46`, abort deployment. + +### 1.5 Baseline Health Probes +- **Command:** + ```bash + # Record currently active release + PREV_RELEASE=$(readlink -f /opt/motorsignal/current) + echo "CURRENT_ACTIVE_RELEASE=${PREV_RELEASE}" + + # Probe API and website baseline + curl -fsS http://127.0.0.1:8080/api/v1/health + curl -fsS http://127.0.0.1:8080/api/v1/ready + curl -fsS -I http://127.0.0.1:3001/ + ``` +- **Expected Result:** API returns `{"status":"ok","service":"motorsignal-api"}` and `{"status":"ready","database":"ok"}`; website returns HTTP 200. +- **Failure Action:** **STOP**. Do not deploy on a degraded baseline. + +### 1.6 Security & Firewall Notice +- **Command:** + ```bash + systemctl is-active fail2ban + ``` +- **Expected Result:** `active`. +- **Policy Notice:** Discovery confirmed UFW is inactive on this host. **DO NOT modify UFW or iptables during OPS-4 deployment.** + +--- + +# PHASE 2 — Release Staging & Artifact Preflight + +*Execute this phase while production is STILL ONLINE and serving traffic from `${PREV_RELEASE}`.* + +### 2.1 Release Directory Setup & Namespace Isolation +To satisfy `motorsignal-api.service` (`WorkingDirectory=/opt/motorsignal/current/app`) and `motorsignal-website.service` (`ExecStart=/usr/bin/node /opt/motorsignal/current/website/server.js`) without directory collision: + +- **Command:** + ```bash + RELEASE_ID="20260910-ops4" + NEW_RELEASE="/opt/motorsignal/releases/${RELEASE_ID}" + + # Ensure release directory structure exists + sudo install -d -o motorsignal -g motorsignal "${NEW_RELEASE}" + sudo install -d -o motorsignal -g motorsignal "${NEW_RELEASE}/backend" + sudo install -d -o motorsignal -g motorsignal "${NEW_RELEASE}/website" + ``` + +### 2.2 Unpack Release Artifacts & Compatibility Symlinks +- **Command:** + ```bash + # Extract uploaded artifacts + sudo -u motorsignal unzip -q /tmp/motorsignal-backend-${RELEASE_ID}.zip -d "${NEW_RELEASE}/backend" + sudo -u motorsignal unzip -q /tmp/motorsignal-website-${RELEASE_ID}.zip -d "${NEW_RELEASE}/website" + + # Establish release-internal compatibility symlinks for systemd and Alembic + sudo -u motorsignal ln -sfn backend/app "${NEW_RELEASE}/app" + sudo -u motorsignal ln -sfn backend/alembic "${NEW_RELEASE}/alembic" + sudo -u motorsignal ln -sfn backend/alembic.ini "${NEW_RELEASE}/alembic.ini" + sudo -u motorsignal ln -sfn backend/requirements.lock "${NEW_RELEASE}/requirements.lock" + + sudo chown -R motorsignal:motorsignal "${NEW_RELEASE}" + ``` + +### 2.3 Virtual Environment Setup +- **Command:** + ```bash + # Create dedicated virtual environment for this release + sudo -u motorsignal python3 -m venv "${NEW_RELEASE}/venv" + + # Install locked dependencies strictly from requirements.lock + sudo -u motorsignal "${NEW_RELEASE}/venv/bin/pip" install --no-cache-dir -r "${NEW_RELEASE}/requirements.lock" + ``` +- **Expected Result:** Virtualenv created cleanly with all locked dependencies installed. + +### 2.4 Artifact Preflight & Manifest Verification (MANDATORY GATE) +- **Command:** + ```bash + # 1. Verify release manifest source commit + grep "source_commit=8681e6f25972cea768106e534b11418b58374d28" "${NEW_RELEASE}/backend/release_manifest.json" || \ + grep "8681e6f25972cea768106e534b11418b58374d28" /tmp/release_manifest_${RELEASE_ID}.txt + + # 2. Verify all runtime files and migration files exist and resolve + test -f "${NEW_RELEASE}/app/main.py" && echo "BACKEND_MAIN_OK" + test -f "${NEW_RELEASE}/alembic.ini" && echo "ALEMBIC_INI_OK" + test -f "${NEW_RELEASE}/alembic/env.py" && echo "ALEMBIC_ENV_OK" + test -f "${NEW_RELEASE}/alembic/versions/e0f5a7123b46_link_firebase_identity.py" && echo "MIGRATION_e0f5_OK" + test -f "${NEW_RELEASE}/alembic/versions/f1a6b8234c57_add_ops_foundation.py" && echo "MIGRATION_f1a6_OK" + test -f "${NEW_RELEASE}/alembic/versions/f1b7c8345d68_evolve_ops_runtime_schema.py" && echo "MIGRATION_f1b7_OK" + test -f "${NEW_RELEASE}/alembic/versions/f1a2b3c4d5e6_add_ops_schema_and_incidents.py" && echo "MIGRATION_f1a2_OK" + test -f "${NEW_RELEASE}/requirements.lock" && echo "LOCK_OK" + test -x "${NEW_RELEASE}/venv/bin/python" && echo "VENV_PYTHON_OK" + test -x "${NEW_RELEASE}/venv/bin/alembic" && echo "VENV_ALEMBIC_OK" + test -f "${NEW_RELEASE}/website/server.js" && echo "WEBSITE_SERVER_OK" + + # 3. Check Python engine version + "${NEW_RELEASE}/venv/bin/python" --version + + # 4. Verify Alembic heads in new release (Must be exactly f1a2b3c4d5e6, single head) + "${NEW_RELEASE}/venv/bin/alembic" -c "${NEW_RELEASE}/alembic.ini" heads + ``` +- **Expected Result:** All artifact checks print `OK`; Python version is 3.10.x; Alembic heads shows `f1a2b3c4d5e6 (head)`. +- **Failure Action:** **STOP BEFORE MAINTENANCE**. If any artifact or migration file is missing, abort. + +### 2.5 Pre-Maintenance Authorization Gate +- [ ] `NEW_RELEASE` directory created at `/opt/motorsignal/releases/20260910-ops4` +- [ ] Source commit confirmed as `8681e6f25972cea768106e534b11418b58374d28` +- [ ] Backend runtime files verified (`app/main.py`, `alembic.ini`, `alembic/env.py`) +- [ ] Website runtime file verified (`website/server.js`) +- [ ] Dedicated `venv` created and populated from locked requirements +- [ ] All 4 migration files present in chain (`e0f5`, `f1a6`, `f1b7`, `f1a2`) +- [ ] Alembic head confirmed as `f1a2b3c4d5e6` (single head) +- [ ] Active release path recorded in `PREV_RELEASE` +- [ ] Production currently healthy (`/health` == 200, `/ready` == 200) + +> **GATE DECISION:** If ALL items checked, proceed to Phase 3. Otherwise, **DO NOT STOP SERVICES**. + +--- + +# PHASE 3 — Pre-Maintenance Backup & Disposable Restore Drill + +### 3.1 Safe Pre-Migration Logical Backup +- **Command:** + ```bash + # Ensure backup directory exists + sudo install -d -m 0750 -o postgres -g postgres /var/backups/motorsignal + BACKUP_TS=$(date +%Y%m%d_%H%M%S) + BACKUP_FILE="/var/backups/motorsignal/motorsignal_pre_ops4_${BACKUP_TS}.dump" + export BACKUP_FILE + + # Execute pg_dump securely via Python runner (No passwords in argv or stdout) + sudo -u motorsignal BACKUP_FILE="${BACKUP_FILE}" "${NEW_RELEASE}/venv/bin/python" - <<'PY' + import os, subprocess, sys + from dotenv import dotenv_values + + env_vars = dotenv_values("/etc/motorsignal/production.env") + pg_env = os.environ.copy() + pg_env["PGHOST"] = env_vars.get("DATABASE_HOST", "127.0.0.1") + pg_env["PGPORT"] = str(env_vars.get("DATABASE_PORT", "5432")) + pg_env["PGDATABASE"] = env_vars.get("DATABASE_NAME", "motorsignal") + pg_env["PGUSER"] = env_vars.get("DATABASE_USER", "motorsignal_app") + if "DATABASE_PASSWORD" in env_vars: + pg_env["PGPASSWORD"] = env_vars["DATABASE_PASSWORD"] + + backup_file = os.environ["BACKUP_FILE"] + res = subprocess.run(["pg_dump", "-Fc", "-v", "-f", backup_file, pg_env["PGDATABASE"]], env=pg_env) + sys.exit(res.returncode) + PY + + # Verify backup file size, checksum, and table-of-contents readability + test -s "${BACKUP_FILE}" && echo "BACKUP_SIZE_OK" + sha256sum "${BACKUP_FILE}" | sudo tee "${BACKUP_FILE}.sha256" + sudo -u postgres pg_restore --list "${BACKUP_FILE}" | head -n 25 + ``` +- **Expected Result:** Backup created with non-zero size (~9 MB); checksum recorded; `pg_restore --list` displays valid TOC. + +### 3.2 Disposable Restore Drill (Mandatory Gate) +- **Command:** + ```bash + DRILL_DB="motorsignal_restore_drill_${BACKUP_TS}" + + # 1. Create temporary drill database + sudo -u postgres psql -c "CREATE DATABASE ${DRILL_DB};" + + # 2. Restore backup into drill database + sudo -u postgres pg_restore -v --no-owner --no-privileges -d "${DRILL_DB}" "${BACKUP_FILE}" + + # 3. Verify Alembic revision in drill database + sudo -u postgres psql -d "${DRILL_DB}" -t -A -c "SELECT version_num FROM alembic_version;" + # Expected: e0f5a7123b46 + + # 4. Verify core table counts match + sudo -u postgres psql -d "${DRILL_DB}" -c "SELECT 'users' AS tbl, count(*) FROM users UNION ALL SELECT 'rides', count(*) FROM rides;" + + # 5. Clean up drill database + sudo -u postgres psql -c "DROP DATABASE ${DRILL_DB};" + ``` +- **Expected Result:** Drill DB restores cleanly; `alembic_version` is `e0f5a7123b46`; counts match; drill DB dropped. +- **Failure Action:** **STOP DEPLOYMENT**. If the drill fails, do not proceed with migration. + +--- + +# PHASE 4 — Maintenance Quiesce + +### 4.1 Quiesce Application Writers +- **Command:** + ```bash + # Record pre-quiesce service states + sudo systemctl status motorsignal-api.service motorsignal-website.service --no-pager + + # Stop application services + sudo systemctl stop motorsignal-api.service + sudo systemctl stop motorsignal-website.service + + # Confirm services are inactive + sudo systemctl is-active motorsignal-api.service || echo "API_STOPPED" + sudo systemctl is-active motorsignal-website.service || echo "WEBSITE_STOPPED" + + # Confirm PostgreSQL remains active + sudo systemctl is-active postgresql@14-main.service + ``` +- **Expected Result:** Application services report `inactive`; PostgreSQL remains `active (running)`. +- **Note:** Nginx remains running. Public traffic temporarily receives `502 Bad Gateway` during the ~1–2 minute migration window. + +--- + +# PHASE 5 — Final Quiesced Database Backup + +### 5.1 Final Quiesced Snapshot +To guarantee an absolute zero-skew recovery point after writers are confirmed stopped: +- **Command:** + ```bash + FINAL_BACKUP_FILE="/var/backups/motorsignal/motorsignal_final_quiesced_${BACKUP_TS}.dump" + export FINAL_BACKUP_FILE + + sudo -u motorsignal BACKUP_FILE="${FINAL_BACKUP_FILE}" "${NEW_RELEASE}/venv/bin/python" - <<'PY' + import os, subprocess, sys + from dotenv import dotenv_values + + env_vars = dotenv_values("/etc/motorsignal/production.env") + pg_env = os.environ.copy() + pg_env["PGHOST"] = env_vars.get("DATABASE_HOST", "127.0.0.1") + pg_env["PGPORT"] = str(env_vars.get("DATABASE_PORT", "5432")) + pg_env["PGDATABASE"] = env_vars.get("DATABASE_NAME", "motorsignal") + pg_env["PGUSER"] = env_vars.get("DATABASE_USER", "motorsignal_app") + if "DATABASE_PASSWORD" in env_vars: + pg_env["PGPASSWORD"] = env_vars["DATABASE_PASSWORD"] + + backup_file = os.environ["BACKUP_FILE"] + res = subprocess.run(["pg_dump", "-Fc", "-v", "-f", backup_file, pg_env["PGDATABASE"]], env=pg_env) + sys.exit(res.returncode) + PY + + test -s "${FINAL_BACKUP_FILE}" && sha256sum "${FINAL_BACKUP_FILE}" | sudo tee "${FINAL_BACKUP_FILE}.sha256" + ``` +- **Expected Result:** Final quiesced backup created and checksum recorded. + +--- + +# PHASE 6 — Database Migration + +> **CRITICAL EXECUTION GATE:** +> Executes migration against production database. Run once. DO NOT retry blindly on error. + +### 6.1 Pre-Migration State Gate +- **Command:** + ```bash + # Verify current database revision is exactly e0f5a7123b46 + sudo -u postgres psql -d motorsignal -t -A -c "SELECT version_num FROM alembic_version;" + # Must output: e0f5a7123b46 + ``` + +### 6.2 Execute Migration via Secure Python Runner +- **Command:** + ```bash + sudo -u motorsignal NEW_RELEASE="${NEW_RELEASE}" "${NEW_RELEASE}/venv/bin/python" - <<'PY' + import os, subprocess, sys + from dotenv import dotenv_values + + release = os.environ["NEW_RELEASE"] + env_vars = dotenv_values("/etc/motorsignal/production.env") + run_env = os.environ.copy() + for k, v in env_vars.items(): + if v is not None: + run_env[k] = v + + res = subprocess.run( + [ + f"{release}/venv/bin/alembic", + "-c", + f"{release}/alembic.ini", + "upgrade", + "f1a2b3c4d5e6", + ], + cwd=release, + env=run_env, + ) + sys.exit(res.returncode) + PY + ``` +- **Expected Result:** Alembic logs execution through: + - `e0f5a7123b46 -> f1a6b8234c57` (`add_ops_foundation`) + - `f1a6b8234c57 -> f1b7c8345d68` (`evolve_ops_runtime_schema`) + - `f1b7c8345d68 -> f1a2b3c4d5e6` (`add_ops_schema_and_incidents`) + Python process exits with code `0`. +- **Failure Action:** If non-zero exit code: **STOP IMMEDIATELY**. Do not retry. Proceed to **PHASE 12 (Rollback - Case C)**. + +--- + +# PHASE 7 — Post-Migration Database Verification + +*Perform read-only PostgreSQL catalog inspection before cutover.* + +### 7.1 Verify Alembic Version +- **Command:** + ```bash + sudo -u postgres psql -d motorsignal -t -A -c "SELECT version_num FROM alembic_version;" + ``` +- **Expected Result:** EXACTLY `f1a2b3c4d5e6`. + +### 7.2 Verify `ops` Schema & Tables +- **Command:** + ```bash + sudo -u postgres psql -d motorsignal -c " + SELECT table_schema, table_name + FROM information_schema.tables + WHERE table_schema = 'ops' + ORDER BY table_name;" + ``` +- **Expected Result:** 5 tables listed: + - `ops.audit_events` + - `ops.incident_notes` + - `ops.incidents` + - `ops.operators` + - `ops.sessions` + +### 7.3 Verify Sequence, Functions & Triggers +- **Command:** + ```bash + # Sequence check + sudo -u postgres psql -d motorsignal -c " + SELECT sequence_name FROM information_schema.sequences WHERE sequence_schema = 'ops';" + + # Function check + sudo -u postgres psql -d motorsignal -c " + SELECT routine_name FROM information_schema.routines WHERE routine_schema = 'ops';" + + # Trigger check + sudo -u postgres psql -d motorsignal -c " + SELECT trigger_name, event_manipulation, event_object_table + FROM information_schema.triggers + WHERE trigger_schema = 'ops';" + ``` +- **Expected Result:** `ops.incident_case_seq` exists; `guard_operator_identity` function and trigger on `ops.operators` exist. + +### 7.4 Verify Permissions & Constraints +- **Command:** + ```bash + # Check PUBLIC schema usage revoked + sudo -u postgres psql -d motorsignal -t -A -c "SELECT has_schema_privilege('public', 'ops', 'usage');" + # Expected: f + ``` +- **Expected Result:** Returns `f` (false). + +--- + +# PHASE 8 — Atomic Symlink Cutover & Service Startup + +### 8.1 Truly Atomic Symlink Cutover +- **Command:** + ```bash + # 1. Verify temporary target does NOT already exist + if [ -e /opt/motorsignal/current.next ] || [ -L /opt/motorsignal/current.next ]; then + echo "STOP: /opt/motorsignal/current.next already exists! Investigate before proceeding." + exit 1 + fi + + # 2. Create temporary symlink on the same filesystem + sudo -u motorsignal ln -s "${NEW_RELEASE}" /opt/motorsignal/current.next + + # 3. Verify temporary symlink resolves to NEW_RELEASE + test "$(readlink -f /opt/motorsignal/current.next)" = "${NEW_RELEASE}" || { echo "STOP: current.next target mismatch!"; exit 1; } + + # 4. Atomically replace current symlink via Linux rename semantics + sudo mv -Tf /opt/motorsignal/current.next /opt/motorsignal/current + + # 5. Verify active release + test "$(readlink -f /opt/motorsignal/current)" = "${NEW_RELEASE}" && echo "CUTOVER_SUCCESSFUL" + ``` +- **Expected Result:** `CUTOVER_SUCCESSFUL` printed; `/opt/motorsignal/current` resolves to `${NEW_RELEASE}`. + +### 8.2 Start Services +- **Command:** + ```bash + sudo systemctl start motorsignal-api.service + sudo systemctl start motorsignal-website.service + + # Check service statuses + sudo systemctl status motorsignal-api.service --no-pager + sudo systemctl status motorsignal-website.service --no-pager + ``` +- **Expected Result:** Both services report `active (running)`. + +### 8.3 Inspect Startup Journals +- **Command:** + ```bash + sudo journalctl -u motorsignal-api.service -n 50 --no-pager + sudo journalctl -u motorsignal-website.service -n 50 --no-pager + ``` +- **Expected Result:** Uvicorn listening on `127.0.0.1:8080`, Next.js standalone listening on `127.0.0.1:3001`, zero startup errors. + +--- + +# PHASE 9 — Local Service Health Verification + +### 9.1 Local Loopback Health Probes +- **Command:** + ```bash + # Core API liveness probe + curl -fsS -v http://127.0.0.1:8080/api/v1/health + + # Database readiness probe + curl -fsS -v http://127.0.0.1:8080/api/v1/ready + + # Operations subsystem health probe + curl -fsS -v http://127.0.0.1:8080/api/v1/ops/health + + # Next.js website loopback probe + curl -fsS -I http://127.0.0.1:3001/ + ``` +- **Expected Results:** + - `/api/v1/health` $\to$ HTTP 200 `{"status":"ok","service":"motorsignal-api"}` + - `/api/v1/ready` $\to$ HTTP 200 `{"status":"ready","database":"ok"}` + - `/api/v1/ops/health` $\to$ HTTP 200 `{"status":"available",...}` + - `http://127.0.0.1:3001/` $\to$ HTTP 200 OK +- **Failure Action:** **STOP**. Do not consider deployment successful if local probes fail. + +--- + +# PHASE 10 — Public HTTPS Health Verification + +### 10.1 Public Ingress Probes +- **Command:** + ```bash + # Public API health + curl -fsS -i https://api.motorsignal.app/api/v1/health + curl -fsS -i https://api.motorsignal.app/api/v1/ready + + # Public Website routes + curl -fsS -I https://motorsignal.app/ + curl -fsS -I https://motorsignal.app/features + curl -fsS -I https://motorsignal.app/pricing + ``` +- **Expected Result:** HTTP 200 OK across public endpoints. + +### 10.2 Operations Security Probes (Safe Unauthenticated Checks) +- **Command:** + ```bash + # Probe /ops/me unauthenticated (Must return 401, NOT 404) + curl -s -o /dev/null -w "%{http_code}\n" https://api.motorsignal.app/api/v1/ops/me + # Expected: 401 + + # Probe /ops/incidents unauthenticated (Must return 401, NOT 404) + curl -s -o /dev/null -w "%{http_code}\n" https://api.motorsignal.app/api/v1/ops/incidents + # Expected: 401 + + # Check Operations web route redirect / response + curl -s -I https://motorsignal.app/admin | head -n 5 + ``` +- **Expected Result:** `/ops/me` and `/ops/incidents` return `HTTP 401 Unauthorized` (confirming OPS-4 routes are active and secured); no 404s. + +--- + +# PHASE 11 — Observation Window + +### 11.1 30-Minute Intensive Monitoring Checklist +- [ ] **Service Restart Counts:** + ```bash + systemctl show motorsignal-api.service -p NRestarts + systemctl show motorsignal-website.service -p NRestarts + # Must remain 0 + ``` +- [ ] **Nginx Error Logs:** + ```bash + sudo tail -n 100 /var/log/nginx/error.log + ``` +- [ ] **PostgreSQL Connection & Activity:** + ```bash + sudo -u postgres psql -c "SELECT count(*), state FROM pg_stat_activity GROUP BY state;" + ``` +- [ ] **Live Journal Monitoring:** + ```bash + sudo journalctl -u motorsignal-api.service -f --since "10 minutes ago" + ``` + +### 11.2 Rollback Criteria During Observation +- Sustained HTTP 5xx error rate > 1%. +- Database deadlocks, lock contention, or connection exhaustion. +- Uncaught exceptions or crash loops in API service. +- Auth or session corruption affecting consumer riders or operators. + +--- + +# PHASE 12 — Rollback & Recovery Procedures + +### 12.1 Rollback Philosophy & Decision Matrix + +| Scenario | Nature of Failure | Action | +| :--- | :--- | :--- | +| **CASE A** | Application fails immediately post-cutover; **ZERO** new production writes. | 1. Stop services.
2. Atomically repoint `/opt/motorsignal/current` to `${PREV_RELEASE}`.
3. Restart previous services.
4. Re-verify health.
5. Do NOT downgrade database unless old code is incompatible with expanded schema. | +| **CASE B** | Failure occurs after production traffic and **NEW WRITES** have occurred. | 1. **DO NOT run blind alembic downgrade** (risks data loss).
2. Deploy forward-fix patch, **OR**
3. If catastrophic, execute Disaster Recovery Restore from pre-migration backup. | +| **CASE C** | Alembic migration command fails during Phase 6. | 1. Keep application stopped.
2. Inspect PostgreSQL transaction state (Alembic runs in transaction).
3. Fix migration issue or restart old application against unchanged `e0f5a7123b46` schema. | + +--- + +### 12.2 Procedure: Application Rollback (Case A) +- **Precondition:** Stored `${PREV_RELEASE}` path (e.g. `/opt/motorsignal/releases/20260908-15`). +- **Command:** + ```bash + # 1. Stop application services + sudo systemctl stop motorsignal-api.service + sudo systemctl stop motorsignal-website.service + + # 2. Atomically repoint current symlink to previous release + sudo -u motorsignal ln -s "${PREV_RELEASE}" /opt/motorsignal/current.prev + sudo mv -Tf /opt/motorsignal/current.prev /opt/motorsignal/current + + # 3. Restart previous services + sudo systemctl start motorsignal-api.service + sudo systemctl start motorsignal-website.service + + # 4. Verify local health + curl -fsS http://127.0.0.1:8080/api/v1/health + curl -fsS http://127.0.0.1:8080/api/v1/ready + curl -fsS -I http://127.0.0.1:3001/ + ``` + +--- + +### 12.3 Procedure: Disaster Recovery Database Restore (Emergency Only) +> **DANGER: RESTORATIVE DISASTER RECOVERY.** +> Requires explicit operator authorization. Replaces database contents with pre-migration backup. + +- **Command:** + ```bash + # 1. Stop application services + sudo systemctl stop motorsignal-api.service + sudo systemctl stop motorsignal-website.service + + # 2. Terminate active database connections + sudo -u postgres psql -c " + SELECT pg_terminate_backend(pid) + FROM pg_stat_activity + WHERE datname = 'motorsignal' AND pid <> pg_backend_pid();" + + # 3. Rename degraded database for forensic preservation (DO NOT DROP) + TS=$(date +%Y%m%d_%H%M%S) + sudo -u postgres psql -c "ALTER DATABASE motorsignal RENAME TO motorsignal_failed_${TS};" + + # 4. Create fresh database + sudo -u postgres psql -c "CREATE DATABASE motorsignal OWNER motorsignal_app;" + + # 5. Restore from verified pre-migration backup + sudo -u postgres pg_restore -v --no-owner --no-privileges -d motorsignal "${FINAL_BACKUP_FILE}" + + # 6. Verify restored database revision + sudo -u postgres psql -d motorsignal -t -A -c "SELECT version_num FROM alembic_version;" + # Expected: e0f5a7123b46 + + # 7. Start previous release + sudo -u motorsignal ln -s "${PREV_RELEASE}" /opt/motorsignal/current.prev + sudo mv -Tf /opt/motorsignal/current.prev /opt/motorsignal/current + sudo systemctl start motorsignal-api.service + sudo systemctl start motorsignal-website.service + + # 8. Re-verify health + curl -fsS http://127.0.0.1:8080/api/v1/health + curl -fsS http://127.0.0.1:8080/api/v1/ready + ``` + +--- + +# PHASE 13 — Post-Deployment Execution Record + +*Complete and archive this record upon deployment execution.* + +```text +================================================================================ +MOTORSIGNAL PRODUCTION DEPLOYMENT EXECUTION RECORD +================================================================================ +Deployment Date/Time (UTC) : ________________________________________________ +Operator Name / ID : ________________________________________________ +Approved Checkpoint Commit : 8681e6f25972cea768106e534b11418b58374d28 +Deployed Release Directory : /opt/motorsignal/releases/______________________ +Previous Release Directory : /opt/motorsignal/releases/______________________ + +DATABASE STATE +-------------------------------------------------------------------------------- +Pre-Migration Alembic Rev : e0f5a7123b46 +Target Alembic Revision : f1a2b3c4d5e6 +Post-Migration Alembic Rev : ________________________________________________ +PostgreSQL Engine Version : PostgreSQL 14.24 +Backup Filename : /var/backups/motorsignal/_______________________ +Backup SHA-256 Checksum : ________________________________________________ +Restore Drill Result : [ ] PASS (Verified e0f5a7123b46 in drill DB) + +CHANGE WINDOW TIMINGS +-------------------------------------------------------------------------------- +Application Quiesced : _____:_____:_____ UTC +Migration Start Time : _____:_____:_____ UTC +Migration End Time : _____:_____:_____ UTC +Migration Exit Code : ________________ +Services Restarted : _____:_____:_____ UTC +Total Change Window Duration : _____ minutes + +VERIFICATION & HEALTH +-------------------------------------------------------------------------------- +Local API Health (200 OK) : [ ] PASS [ ] FAIL +Local API Ready (200 OK) : [ ] PASS [ ] FAIL +Local Ops Health (200 OK) : [ ] PASS [ ] FAIL +Local Website (200 OK) : [ ] PASS [ ] FAIL +Public HTTPS API Health : [ ] PASS [ ] FAIL +Public HTTPS Ready Check : [ ] PASS [ ] FAIL +Public Website Routes Check : [ ] PASS [ ] FAIL +Ops Unauth Probe /ops/me : [ ] PASS (401) [ ] FAIL +Ops Unauth Probe /ops/incidents: [ ] PASS (401) [ ] FAIL +Observation Window (30 min) : [ ] PASS [ ] FAIL + +INCIDENTS & ROLLBACK +-------------------------------------------------------------------------------- +Rollback Required? : [ ] YES [ ] NO +Rollback Scenario (if any) : [ ] CASE A [ ] CASE B [ ] CASE C +Incidents Encountered : ________________________________________________ +Corrective Notes : ________________________________________________ + +Sign-off Operator Signature : ________________________________________________ +================================================================================ +``` diff --git a/docs/PRODUCTION_DEPLOYMENT_CHECKLIST.md b/docs/PRODUCTION_DEPLOYMENT_CHECKLIST.md index f7dae69..1bfa360 100644 --- a/docs/PRODUCTION_DEPLOYMENT_CHECKLIST.md +++ b/docs/PRODUCTION_DEPLOYMENT_CHECKLIST.md @@ -27,6 +27,33 @@ Prepared 2026-09-08 after read-only VPS preflight. 8. Restart services in order, then run health, auth, account, admin, and rollback checks. 9. Keep `/opt/motorsignal/releases/20260906-01` available until acceptance completes. +## Prepared local artifacts + +- `scratch/deploy/20260908-02/motorsignal-backend-20260908-02.zip` +- `scratch/deploy/20260908-02/motorsignal-website-20260908-02.zip` + +Website bundle is Next standalone output. It includes `server.js` and required +runtime dependencies; run it behind Nginx on loopback only. + +## After sudo access returns + +From Windows, upload artifacts to the deployment user's home directory with the +configured SSH key. As root, create a new release directory, extract backend +and website bundles, set ownership to the service users, and preserve the old +release. Then run: + +```bash +sudo -u motorsignal python3 -m venv /opt/motorsignal/releases/20260908-02/venv +sudo -u motorsignal /opt/motorsignal/releases/20260908-02/venv/bin/pip install \ + -r /opt/motorsignal/releases/20260908-02/requirements.lock +sudo -u motorsignal /opt/motorsignal/releases/20260908-02/venv/bin/alembic \ + -c /opt/motorsignal/releases/20260908-02/alembic.ini upgrade head +``` + +Verify `d9e4f6012a35` before restarting the API. Configure a separate Node +systemd unit for website `server.js`, update Nginx upstream routing, run +`nginx -t`, reload Nginx, and execute live smoke tests. + ## Current blockers - Deployment user cannot run passwordless `sudo`; privileged Nginx/systemd/database steps need operator execution or temporary reviewed access. diff --git a/docs/WEBSITE_ADMIN_CURRENT_STATE.md b/docs/WEBSITE_ADMIN_CURRENT_STATE.md index 3737efa..d65e834 100644 --- a/docs/WEBSITE_ADMIN_CURRENT_STATE.md +++ b/docs/WEBSITE_ADMIN_CURRENT_STATE.md @@ -52,10 +52,17 @@ Ride-linked telemetry persistence is available through bounded batch ingestion at `/api/v1/rides/{ride_id}/telemetry`, owner-scoped reads, and support/admin reads. Points are indexed by ride and timestamp. +Owner and support/admin telemetry summary endpoints expose point count, duration, +maximum speed, maximum lean angle, and maximum G-force without requiring raw +bundle transfer. + Subscription and entitlement baselines are available through authenticated account reads and admin-controlled updates. Default subscription is `free`; no payment provider or billing state is fabricated. +The account portal now exposes active sessions and supports revoking individual +non-current sessions or all sessions through secure server-side proxy routes. + Database-backed authentication rate limits now protect login, registration, verification, and password-reset paths. Limit is 10 attempts per client IP per 15-minute window, with `Retry-After` responses. diff --git a/scripts/deploy-website.ps1 b/scripts/deploy-website.ps1 new file mode 100644 index 0000000..5dbf9e2 --- /dev/null +++ b/scripts/deploy-website.ps1 @@ -0,0 +1,98 @@ +param( + [string]$HostIp = $env:MOTORSIGNAL_PROD_HOST, + [string]$User = $(if ($env:MOTORSIGNAL_PROD_USER) { $env:MOTORSIGNAL_PROD_USER } else { "motorsignal-admin" }), + [string]$Key = $env:MOTORSIGNAL_PROD_SSH_KEY, + [switch]$BuildRemote +) + +$ErrorActionPreference = "Stop" + +if (-not $HostIp) { + Write-Error "HostIp parameter or MOTORSIGNAL_PROD_HOST environment variable is required." + exit 1 +} + +$SshKeyArgs = @() +if ($Key) { + if (-not (Test-Path $Key)) { + Write-Error "Specified SSH key file not found: $Key" + exit 1 + } + $SshKeyArgs = @("-i", $Key) +} + +$Timestamp = Get-Date -Format "yyyyMMdd-HHmmss" + +if ($BuildRemote) { + # Ultra-lean mode: Upload only source files (~100 KB total) + $TarName = "website-src-$Timestamp.tar.gz" + $TempTar = "$PSScriptRoot\$TarName" + + Write-Host ">>> 1. Packaging source code (~100 KB)..." -ForegroundColor Cyan + tar -czf "$TempTar" -C website app components lib public package.json tsconfig.json next.config.ts + + Write-Host ">>> 2. Uploading source code to server..." -ForegroundColor Cyan + scp @SshKeyArgs "$TempTar" "${User}@${HostIp}:/tmp/$TarName" + Remove-Item "$TempTar" -Force + + Write-Host ">>> 3. Building and restarting on server..." -ForegroundColor Cyan + $RemoteCmd = @" +bash -c ' +set -euo pipefail +TAR=/tmp/$TarName +TARGET=/opt/motorsignal/current/website + +echo "Updating source files in `$TARGET..." +sudo tar -xzf "`$TAR" -C "`$TARGET" +rm -f "`$TAR" + +cd "`$TARGET" +echo "Running Next.js production build on server..." +npm run build + +echo "Restarting motorsignal-website service..." +sudo systemctl restart motorsignal-website +echo "Done!" +' +"@ + ssh -t @SshKeyArgs "${User}@${HostIp}" $RemoteCmd +} else { + # Fast local build: Strip Next.js cache & dev artifacts (~2 MB total instead of 500 MB) + Write-Host ">>> 1. Building Next.js application locally..." -ForegroundColor Cyan + Set-Location -Path "$PSScriptRoot\..\website" + npm run build + Set-Location -Path "$PSScriptRoot\.." + + $TarName = "website-update-$Timestamp.tar.gz" + $TempTar = "$PSScriptRoot\$TarName" + + tar -czf "$TempTar" --exclude="cache" --exclude="dev" --exclude="standalone" -C website .next/server .next/static .next/types .next/BUILD_ID .next/*.json public package.json app components lib + + $SizeMb = [math]::Round(((Get-Item "$TempTar").Length / 1MB), 2) + Write-Host ">>> 3. Uploading $SizeMb MB to server..." -ForegroundColor Cyan + scp @SshKeyArgs "$TempTar" "${User}@${HostIp}:/tmp/$TarName" + Remove-Item "$TempTar" -Force + + Write-Host ">>> 4. Activating update on server..." -ForegroundColor Cyan + $RemoteCmd = @" +bash -c ' +set -euo pipefail +TAR=/tmp/$TarName +TARGET=/opt/motorsignal/current/website + +echo "Extracting runtime to `$TARGET..." +sudo tar -xzf "`$TAR" -C "`$TARGET" +rm -f "`$TAR" + +echo "Restarting motorsignal-website service..." +sudo systemctl restart motorsignal-website +echo "Service restarted." +' +"@ + ssh -t @SshKeyArgs "${User}@${HostIp}" $RemoteCmd +} + +Write-Host ">>> 5. Checking website response..." -ForegroundColor Green +Start-Sleep -Seconds 2 +$StatusCode = (Invoke-WebRequest -Uri "https://motorsignal.app/account" -Method Head -UseBasicParsing).StatusCode +Write-Host ">>> Deployed successfully! HTTP Status: $StatusCode" -ForegroundColor Green