motorsignal
/
MotorSignal-App
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
MotorSignal-App/docs/OPS4_PRODUCTION_COMMAND_CAR...

8.4 KiB
Raw Permalink Blame History

MotorSignal OPS-4 Production Command Card

(Hardened Operator Companion Card)

OPERATOR COMPANION CARD — DOCUMENTATION & PLANNING ONLY
Checkpoint Commit: 8681e6f25972cea768106e534b11418b58374d28
PostgreSQL Engine: PostgreSQL 14.24 | Database: motorsignal (User: motorsignal_app)
Baseline Alembic Revision: e0f5a7123b46 \to Target Alembic Revision: f1a2b3c4d5e6
Authoritative Runbook: docs/OPS4_PRODUCTION_DEPLOYMENT_RUNBOOK.md

1. Preflight & Release Artifact Verification

# 1. SSH into host & record current active release
ssh -i ~/.ssh/motorsignal_prod motorsignal-admin@89.31.239.10
PREV_RELEASE=$(readlink -f /opt/motorsignal/current)
echo "PREVIOUS_RELEASE=${PREV_RELEASE}"

# 2. Stage new release directory & unpack artifacts
RELEASE_ID="20260910-ops4"
NEW_RELEASE="/opt/motorsignal/releases/${RELEASE_ID}"
sudo install -d -o motorsignal -g motorsignal "${NEW_RELEASE}/backend" "${NEW_RELEASE}/website"
sudo -u motorsignal unzip -q /tmp/motorsignal-backend-${RELEASE_ID}.zip -d "${NEW_RELEASE}/backend"
sudo -u motorsignal unzip -q /tmp/motorsignal-website-${RELEASE_ID}.zip -d "${NEW_RELEASE}/website"

# 3. Create compatibility symlinks & dedicated venv
sudo -u motorsignal ln -sfn backend/app "${NEW_RELEASE}/app"
sudo -u motorsignal ln -sfn backend/alembic "${NEW_RELEASE}/alembic"
sudo -u motorsignal ln -sfn backend/alembic.ini "${NEW_RELEASE}/alembic.ini"
sudo -u motorsignal ln -sfn backend/requirements.lock "${NEW_RELEASE}/requirements.lock"
sudo -u motorsignal python3 -m venv "${NEW_RELEASE}/venv"
sudo -u motorsignal "${NEW_RELEASE}/venv/bin/pip" install --no-cache-dir -r "${NEW_RELEASE}/requirements.lock"

# 4. Mandatory Artifact Preflight Gate
"${NEW_RELEASE}/venv/bin/alembic" -c "${NEW_RELEASE}/alembic.ini" heads
# MUST output exactly: f1a2b3c4d5e6 (head)
sudo -u postgres psql -d motorsignal -t -A -c "SELECT version_num FROM alembic_version;"
# MUST output exactly: e0f5a7123b46

2. Safe Pre-Migration Backup & Restore Drill

# 1. Execute safe custom-format backup (No passwords in argv)
sudo install -d -m 0750 -o postgres -g postgres /var/backups/motorsignal
BACKUP_TS=$(date +%Y%m%d_%H%M%S)
BACKUP_FILE="/var/backups/motorsignal/motorsignal_pre_ops4_${BACKUP_TS}.dump"
export BACKUP_FILE

sudo -u motorsignal BACKUP_FILE="${BACKUP_FILE}" "${NEW_RELEASE}/venv/bin/python" - <<'PY'
import os, subprocess, sys
from dotenv import dotenv_values
env_vars = dotenv_values("/etc/motorsignal/production.env")
pg_env = os.environ.copy()
pg_env["PGHOST"] = env_vars.get("DATABASE_HOST", "127.0.0.1")
pg_env["PGPORT"] = str(env_vars.get("DATABASE_PORT", "5432"))
pg_env["PGDATABASE"] = env_vars.get("DATABASE_NAME", "motorsignal")
pg_env["PGUSER"] = env_vars.get("DATABASE_USER", "motorsignal_app")
if "DATABASE_PASSWORD" in env_vars:
    pg_env["PGPASSWORD"] = env_vars["DATABASE_PASSWORD"]
res = subprocess.run(["pg_dump", "-Fc", "-v", "-f", os.environ["BACKUP_FILE"], pg_env["PGDATABASE"]], env=pg_env)
sys.exit(res.returncode)
PY

test -s "${BACKUP_FILE}" && sha256sum "${BACKUP_FILE}" | sudo tee "${BACKUP_FILE}.sha256"
sudo -u postgres pg_restore --list "${BACKUP_FILE}" | head -n 20

# 2. Disposable Restore Drill (Mandatory Gate)
DRILL_DB="motorsignal_restore_drill_${BACKUP_TS}"
sudo -u postgres psql -c "CREATE DATABASE ${DRILL_DB};"
sudo -u postgres pg_restore -v --no-owner --no-privileges -d "${DRILL_DB}" "${BACKUP_FILE}"
sudo -u postgres psql -d "${DRILL_DB}" -t -A -c "SELECT version_num FROM alembic_version;"
# MUST output: e0f5a7123b46
sudo -u postgres psql -c "DROP DATABASE ${DRILL_DB};"

3. Quiesce & Final Backup

# Quiesce application writers (Nginx remains active)
sudo systemctl stop motorsignal-api.service motorsignal-website.service
sudo systemctl is-active motorsignal-api.service || echo "API_STOPPED"
sudo systemctl is-active postgresql@14-main.service

# Final quiesced snapshot
FINAL_BACKUP_FILE="/var/backups/motorsignal/motorsignal_final_quiesced_${BACKUP_TS}.dump"
sudo -u motorsignal BACKUP_FILE="${FINAL_BACKUP_FILE}" "${NEW_RELEASE}/venv/bin/python" - <<'PY'
import os, subprocess, sys
from dotenv import dotenv_values
env_vars = dotenv_values("/etc/motorsignal/production.env")
pg_env = os.environ.copy()
pg_env["PGHOST"] = env_vars.get("DATABASE_HOST", "127.0.0.1")
pg_env["PGPORT"] = str(env_vars.get("DATABASE_PORT", "5432"))
pg_env["PGDATABASE"] = env_vars.get("DATABASE_NAME", "motorsignal")
pg_env["PGUSER"] = env_vars.get("DATABASE_USER", "motorsignal_app")
if "DATABASE_PASSWORD" in env_vars:
    pg_env["PGPASSWORD"] = env_vars["DATABASE_PASSWORD"]
res = subprocess.run(["pg_dump", "-Fc", "-v", "-f", os.environ["BACKUP_FILE"], pg_env["PGDATABASE"]], env=pg_env)
sys.exit(res.returncode)
PY
test -s "${FINAL_BACKUP_FILE}" && sha256sum "${FINAL_BACKUP_FILE}" | sudo tee "${FINAL_BACKUP_FILE}.sha256"

4. Single-Execution Database Migration

# Execute Alembic upgrade safely via Python runner (STOP IMMEDIATELY ON ERROR)
sudo -u motorsignal NEW_RELEASE="${NEW_RELEASE}" "${NEW_RELEASE}/venv/bin/python" - <<'PY'
import os, subprocess, sys
from dotenv import dotenv_values
release = os.environ["NEW_RELEASE"]
env_vars = dotenv_values("/etc/motorsignal/production.env")
run_env = os.environ.copy()
for k, v in env_vars.items():
    if v is not None:
        run_env[k] = v
res = subprocess.run(
    [f"{release}/venv/bin/alembic", "-c", f"{release}/alembic.ini", "upgrade", "f1a2b3c4d5e6"],
    cwd=release,
    env=run_env
)
sys.exit(res.returncode)
PY
# STOP IMMEDIATELY if process exits non-zero!

5. Post-Migration DB Verification

# Verify Alembic version is f1a2b3c4d5e6
sudo -u postgres psql -d motorsignal -t -A -c "SELECT version_num FROM alembic_version;"

# Verify ops schema & 5 tables
sudo -u postgres psql -d motorsignal -c "
SELECT table_name FROM information_schema.tables WHERE table_schema = 'ops' ORDER BY table_name;"

# Verify sequence, function, and trigger
sudo -u postgres psql -d motorsignal -c "
SELECT sequence_name FROM information_schema.sequences WHERE sequence_schema = 'ops';
SELECT routine_name FROM information_schema.routines WHERE routine_schema = 'ops';
SELECT trigger_name, event_object_table FROM information_schema.triggers WHERE trigger_schema = 'ops';"

6. Atomic Symlink Cutover & Service Start

# 1. Pre-check: current.next must not exist
test ! -e /opt/motorsignal/current.next || { echo "STOP: current.next exists!"; exit 1; }

# 2. Create temporary symlink & verify
sudo -u motorsignal ln -s "${NEW_RELEASE}" /opt/motorsignal/current.next
test "$(readlink -f /opt/motorsignal/current.next)" = "${NEW_RELEASE}" || { echo "STOP: target mismatch!"; exit 1; }

# 3. Truly atomic cutover via Linux rename
sudo mv -Tf /opt/motorsignal/current.next /opt/motorsignal/current
test "$(readlink -f /opt/motorsignal/current)" = "${NEW_RELEASE}" && echo "CUTOVER_OK"

# 4. Start services
sudo systemctl start motorsignal-api.service motorsignal-website.service
sudo systemctl status motorsignal-api.service motorsignal-website.service --no-pager

7. Local & Public Health Checks

# Local loopback probes
curl -fsS http://127.0.0.1:8080/api/v1/health
curl -fsS http://127.0.0.1:8080/api/v1/ready
curl -fsS http://127.0.0.1:8080/api/v1/ops/health
curl -fsS -I http://127.0.0.1:3001/

# Public HTTPS probes
curl -fsS https://api.motorsignal.app/api/v1/health
curl -fsS https://api.motorsignal.app/api/v1/ready
curl -fsS -I https://motorsignal.app/

# Safe unauthenticated security probes (Must return HTTP 401, NOT 404)
curl -s -o /dev/null -w "%{http_code}\n" https://api.motorsignal.app/api/v1/ops/me
curl -s -o /dev/null -w "%{http_code}\n" https://api.motorsignal.app/api/v1/ops/incidents

8. Rollback Triggers & Fast Action

Case A: Application Failure (No New Writes)

# 1. Stop services
sudo systemctl stop motorsignal-api.service motorsignal-website.service
# 2. Atomically repoint symlink to previous release
sudo -u motorsignal ln -s "${PREV_RELEASE}" /opt/motorsignal/current.prev
sudo mv -Tf /opt/motorsignal/current.prev /opt/motorsignal/current
# 3. Restart previous services & verify
sudo systemctl start motorsignal-api.service motorsignal-website.service
curl -fsS http://127.0.0.1:8080/api/v1/health && curl -fsS http://127.0.0.1:8080/api/v1/ready

Case B: Migration Non-Zero Exit

  • DO NOT restart application services.
  • DO NOT run alembic stamp.
  • DO NOT retry repeatedly.
  • Keep writers stopped; inspect journalctl and PostgreSQL logs.