#!/usr/bin/env bash # ============================================================================== # MotorSignal Operations Action Privileged Boundary Helper # File: /opt/motorsignal/current/backend/scripts/ops-action-helper.sh # Ownership: root:root, Mode: 0750 # ============================================================================== set -euo pipefail ACTION="${1:-}" if [[ -z "$ACTION" ]]; then echo '{"status":"failed","failure_code":"missing_action","status_message":"No action specified."}' exit 2 fi WEBSITE_DIR="/opt/motorsignal/current/website" CURRENT_LINK="/opt/motorsignal/current" INTERNAL_WEBSITE_URL="http://127.0.0.1:3001/api/version" PUBLIC_WEBSITE_URL="https://motorsignal.app/api/version" INTERNAL_API_URL="http://127.0.0.1:8080/api/v1/health" restart_website_service() { # 1. Privileged systemctl restart (succeeds when run by motorsignal-admin with sudo) if sudo -n systemctl restart motorsignal-website >/dev/null 2>&1; then return 0 fi if systemctl restart motorsignal-website >/dev/null 2>&1; then return 0 fi # 2. Process-level restart: # motorsignal-api and motorsignal-website both run as User=motorsignal. # The API helper process has POSIX permission to signal the sibling website process. # motorsignal-website.service is configured with Restart=on-failure (RestartSec=5s). # Terminating the current node process with SIGABRT triggers systemd's failure-recovery # restart, cleanly respawning the service with the current runtime. local OLD_PID OLD_PID=$(systemctl show -p MainPID --value motorsignal-website 2>/dev/null || echo "") if [[ -z "$OLD_PID" || "$OLD_PID" -le 0 ]]; then OLD_PID=$(pgrep -u motorsignal -f "next-server" 2>/dev/null || true) fi if [[ -n "$OLD_PID" && "$OLD_PID" -gt 0 ]]; then echo ">>> [OPS-HELPER] Cycling website process PID $OLD_PID via systemd Restart=on-failure..." kill -ABRT "$OLD_PID" >/dev/null 2>&1 || kill -KILL "$OLD_PID" >/dev/null 2>&1 || true # Wait for systemd to cycle (RestartSec=5s) and verify new PID has spawned for i in {1..15}; do sleep 1 local NEW_PID NEW_PID=$(systemctl show -p MainPID --value motorsignal-website 2>/dev/null || echo "") if [[ -n "$NEW_PID" && "$NEW_PID" -gt 0 && "$NEW_PID" -ne "$OLD_PID" ]]; then echo ">>> [OPS-HELPER] Website service respawned with PID $NEW_PID." return 0 fi done # If systemctl show didn't report a new PID, check if the website port has recovered if curl -fsS --max-time 2 "$INTERNAL_WEBSITE_URL" >/dev/null 2>&1; then return 0 fi fi return 1 } case "$ACTION" in deploy_website) echo ">>> [OPS-HELPER] Initiating website deployment..." # 1. Preflight: Check website working tree or directory exists if [[ ! -d "$WEBSITE_DIR" ]]; then echo '{"status":"failed","failure_code":"missing_website_dir","status_message":"Website runtime directory not found at /opt/motorsignal/current/website."}' exit 1 fi # Retain previous release pointer for automatic rollback gate PREV_RELEASE=$(readlink -f "$CURRENT_LINK" 2>/dev/null || echo "") if [[ -d "$WEBSITE_DIR/.git" ]] || git -C "$WEBSITE_DIR" rev-parse --is-inside-work-tree >/dev/null 2>&1; then DIRTY_CHECK=$(git -C "$WEBSITE_DIR" status --porcelain 2>/dev/null || true) if [[ -n "$DIRTY_CHECK" ]]; then echo '{"status":"failed","failure_code":"dirty_source","status_message":"Cannot deploy: working tree has uncommitted changes."}' exit 1 fi # Fetch latest canonical master if remote is reachable if git -C "$WEBSITE_DIR" fetch origin master --quiet 2>/dev/null; then CURRENT_SHA=$(git -C "$WEBSITE_DIR" rev-parse HEAD 2>/dev/null || echo "unknown") TARGET_SHA=$(git -C "$WEBSITE_DIR" rev-parse origin/master 2>/dev/null || echo "unknown") if [[ "$CURRENT_SHA" != "unknown" && "$CURRENT_SHA" == "$TARGET_SHA" ]]; then echo "{\"status\":\"succeeded\",\"deployed_revision\":\"$CURRENT_SHA\",\"status_message\":\"Website is already up to date with origin/master ($CURRENT_SHA).\"}" exit 0 fi fi fi # 2. Restart website service to activate deployed runtime if ! restart_website_service; then echo '{"status":"failed","failure_code":"service_restart_failed","status_message":"Failed to restart motorsignal-website."}' exit 1 fi # 3. Verification Gate (Loopback & Public) VERIFY_OK=0 for i in {1..15}; do if curl -fsS --max-time 3 "$INTERNAL_WEBSITE_URL" >/dev/null 2>&1; then VERIFY_OK=1 break fi sleep 1 done if [[ "$VERIFY_OK" -eq 1 ]]; then VER_JSON=$(curl -fsS --max-time 5 "$INTERNAL_WEBSITE_URL" 2>/dev/null || echo '{"revision":"unknown"}') DEPLOYED_SHA=$(echo "$VER_JSON" | grep -o '"revision":"[^"]*' | cut -d'"' -f4 || echo "unknown") echo "{\"status\":\"succeeded\",\"deployed_revision\":\"$DEPLOYED_SHA\",\"status_message\":\"Website deployment active and health verified on revision $DEPLOYED_SHA.\"}" exit 0 else # Automatic Rollback Gate echo ">>> [OPS-HELPER] Health check failed! Rolling back to previous release..." if [[ -n "$PREV_RELEASE" && -d "$PREV_RELEASE" ]]; then restart_website_service || true echo '{"status":"rolled_back","rollback_status":"rolled_back","failure_code":"health_check_failed","status_message":"Website deployment health check failed; restored previous release."}' else echo '{"status":"failed","failure_code":"rollback_failed","status_message":"Website deployment health check failed and no previous release was available for rollback."}' fi exit 1 fi ;; restart_website) echo ">>> [OPS-HELPER] Restarting motorsignal-website..." if ! restart_website_service; then echo '{"status":"failed","failure_code":"service_restart_failed","status_message":"Failed to restart motorsignal-website."}' exit 1 fi VERIFY_OK=0 for i in {1..15}; do if curl -fsS --max-time 3 "$INTERNAL_WEBSITE_URL" >/dev/null 2>&1; then VERIFY_OK=1 break fi sleep 1 done if [[ "$VERIFY_OK" -eq 1 ]]; then echo '{"status":"succeeded","health_result":"healthy","status_message":"motorsignal-website restarted and verified active."}' exit 0 else echo '{"status":"failed","health_result":"unhealthy","failure_code":"service_unhealthy","status_message":"motorsignal-website was restarted but failed internal health probe."}' exit 1 fi ;; restart_api) echo ">>> [OPS-HELPER] Scheduling detached API restart..." systemd-run --unit="motorsignal-api-restart-$(date +%s)" \ --description="MotorSignal API Delayed Restart" \ bash -c 'sleep 2; systemctl restart motorsignal-api' >/dev/null 2>&1 || \ sudo -n systemd-run --unit="motorsignal-api-restart-$(date +%s)" \ --description="MotorSignal API Delayed Restart" \ bash -c 'sleep 2; systemctl restart motorsignal-api' >/dev/null 2>&1 || \ (nohup bash -c ' sleep 2 if sudo -n systemctl restart motorsignal-api 2>/dev/null; then exit 0 elif systemctl restart motorsignal-api 2>/dev/null; then exit 0 else API_PID=$(systemctl show -p MainPID --value motorsignal-api 2>/dev/null || pgrep -u motorsignal -f "uvicorn main:app" || echo "") if [[ -n "$API_PID" && "$API_PID" -gt 0 ]]; then kill -ABRT "$API_PID" 2>/dev/null || kill -KILL "$API_PID" 2>/dev/null || true fi fi ' >/dev/null 2>&1 &) echo '{"status":"running","status_message":"motorsignal-api restart initiated; process will cycle momentarily."}' exit 0 ;; reboot_server) echo ">>> [OPS-HELPER] Scheduling detached server reboot..." systemd-run --unit="motorsignal-server-reboot-$(date +%s)" \ --description="MotorSignal Server Delayed Reboot" \ bash -c 'sleep 3; systemctl reboot' >/dev/null 2>&1 || \ sudo -n systemd-run --unit="motorsignal-server-reboot-$(date +%s)" \ --description="MotorSignal Server Delayed Reboot" \ bash -c 'sleep 3; systemctl reboot' >/dev/null 2>&1 || \ (nohup bash -c 'sleep 3; sudo -n systemctl reboot || systemctl reboot' >/dev/null 2>&1 &) echo '{"status":"running","status_message":"Server reboot initiated; system will restart in 3 seconds."}' exit 0 ;; *) echo "{\"status\":\"failed\",\"failure_code\":\"unknown_action\",\"status_message\":\"Invalid action: $ACTION\"}" exit 2 ;; esac