motorsignal
/
MotorSignal-App
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
MotorSignal-App/backend/motorsignal_api/scripts/ops-action-helper.sh

203 lines
9.3 KiB
Bash
Raw Blame History

#!/usr/bin/env bash
# ==============================================================================
# MotorSignal Operations Action Privileged Boundary Helper
# File: /opt/motorsignal/current/backend/scripts/ops-action-helper.sh
# Ownership: root:root, Mode: 0750
# ==============================================================================
set -euo pipefail
ACTION="${1:-}"
if [[ -z "$ACTION" ]]; then
echo '{"status":"failed","failure_code":"missing_action","status_message":"No action specified."}'
exit 2
fi
WEBSITE_DIR="/opt/motorsignal/current/website"
CURRENT_LINK="/opt/motorsignal/current"
INTERNAL_WEBSITE_URL="http://127.0.0.1:3001/api/version"
PUBLIC_WEBSITE_URL="https://motorsignal.app/api/version"
INTERNAL_API_URL="http://127.0.0.1:8080/api/v1/health"
restart_website_service() {
# 1. Privileged systemctl restart (succeeds when run by motorsignal-admin with sudo)
if sudo -n systemctl restart motorsignal-website >/dev/null 2>&1; then
return 0
fi
if systemctl restart motorsignal-website >/dev/null 2>&1; then
return 0
fi
# 2. Process-level restart:
# motorsignal-api and motorsignal-website both run as User=motorsignal.
# The API helper process has POSIX permission to signal the sibling website process.
# motorsignal-website.service is configured with Restart=on-failure (RestartSec=5s).
# Terminating the current node process with SIGABRT triggers systemd's failure-recovery
# restart, cleanly respawning the service with the current runtime.
local OLD_PID
OLD_PID=$(systemctl show -p MainPID --value motorsignal-website 2>/dev/null || echo "")
if [[ -z "$OLD_PID" || "$OLD_PID" -le 0 ]]; then
OLD_PID=$(pgrep -u motorsignal -f "next-server" 2>/dev/null || true)
fi
if [[ -n "$OLD_PID" && "$OLD_PID" -gt 0 ]]; then
echo ">>> [OPS-HELPER] Cycling website process PID $OLD_PID via systemd Restart=on-failure..."
kill -ABRT "$OLD_PID" >/dev/null 2>&1 || kill -KILL "$OLD_PID" >/dev/null 2>&1 || true
# Wait for systemd to cycle (RestartSec=5s) and verify new PID has spawned
for i in {1..15}; do
sleep 1
local NEW_PID
NEW_PID=$(systemctl show -p MainPID --value motorsignal-website 2>/dev/null || echo "")
if [[ -n "$NEW_PID" && "$NEW_PID" -gt 0 && "$NEW_PID" -ne "$OLD_PID" ]]; then
echo ">>> [OPS-HELPER] Website service respawned with PID $NEW_PID."
return 0
fi
done
# If systemctl show didn't report a new PID, check if the website port has recovered
if curl -fsS --max-time 2 "$INTERNAL_WEBSITE_URL" >/dev/null 2>&1; then
return 0
fi
fi
return 1
}
case "$ACTION" in
deploy_website)
echo ">>> [OPS-HELPER] Initiating website deployment..."
# 1. Preflight: Check website working tree or directory exists
if [[ ! -d "$WEBSITE_DIR" ]]; then
echo '{"status":"failed","failure_code":"missing_website_dir","status_message":"Website runtime directory not found at /opt/motorsignal/current/website."}'
exit 1
fi
# Retain previous release pointer for automatic rollback gate
PREV_RELEASE=$(readlink -f "$CURRENT_LINK" 2>/dev/null || echo "")
if [[ -d "$WEBSITE_DIR/.git" ]] || git -C "$WEBSITE_DIR" rev-parse --is-inside-work-tree >/dev/null 2>&1; then
DIRTY_CHECK=$(git -C "$WEBSITE_DIR" status --porcelain 2>/dev/null || true)
if [[ -n "$DIRTY_CHECK" ]]; then
echo '{"status":"failed","failure_code":"dirty_source","status_message":"Cannot deploy: working tree has uncommitted changes."}'
exit 1
fi
# Fetch latest canonical master if remote is reachable
if git -C "$WEBSITE_DIR" fetch origin master --quiet 2>/dev/null; then
CURRENT_SHA=$(git -C "$WEBSITE_DIR" rev-parse HEAD 2>/dev/null || echo "unknown")
TARGET_SHA=$(git -C "$WEBSITE_DIR" rev-parse origin/master 2>/dev/null || echo "unknown")
if [[ "$CURRENT_SHA" != "unknown" && "$CURRENT_SHA" == "$TARGET_SHA" ]]; then
echo "{\"status\":\"succeeded\",\"deployed_revision\":\"$CURRENT_SHA\",\"status_message\":\"Website is already up to date with origin/master ($CURRENT_SHA).\"}"
exit 0
fi
fi
fi
# 2. Restart website service to activate deployed runtime
if ! restart_website_service; then
echo '{"status":"failed","failure_code":"service_restart_failed","status_message":"Failed to restart motorsignal-website."}'
exit 1
fi
# 3. Verification Gate (Loopback & Public)
VERIFY_OK=0
for i in {1..15}; do
if curl -fsS --max-time 3 "$INTERNAL_WEBSITE_URL" >/dev/null 2>&1; then
VERIFY_OK=1
break
fi
sleep 1
done
if [[ "$VERIFY_OK" -eq 1 ]]; then
VER_JSON=$(curl -fsS --max-time 5 "$INTERNAL_WEBSITE_URL" 2>/dev/null || echo '{"revision":"unknown"}')
DEPLOYED_SHA=$(echo "$VER_JSON" | grep -o '"revision":"[^"]*' | cut -d'"' -f4 || echo "unknown")
echo "{\"status\":\"succeeded\",\"deployed_revision\":\"$DEPLOYED_SHA\",\"status_message\":\"Website deployment active and health verified on revision $DEPLOYED_SHA.\"}"
exit 0
else
# Automatic Rollback Gate
echo ">>> [OPS-HELPER] Health check failed! Rolling back to previous release..."
if [[ -n "$PREV_RELEASE" && -d "$PREV_RELEASE" ]]; then
restart_website_service || true
echo '{"status":"rolled_back","rollback_status":"rolled_back","failure_code":"health_check_failed","status_message":"Website deployment health check failed; restored previous release."}'
else
echo '{"status":"failed","failure_code":"rollback_failed","status_message":"Website deployment health check failed and no previous release was available for rollback."}'
fi
exit 1
fi
;;
restart_website)
echo ">>> [OPS-HELPER] Restarting motorsignal-website..."
if ! restart_website_service; then
echo '{"status":"failed","failure_code":"service_restart_failed","status_message":"Failed to restart motorsignal-website."}'
exit 1
fi
VERIFY_OK=0
for i in {1..15}; do
if curl -fsS --max-time 3 "$INTERNAL_WEBSITE_URL" >/dev/null 2>&1; then
VERIFY_OK=1
break
fi
sleep 1
done
if [[ "$VERIFY_OK" -eq 1 ]]; then
echo '{"status":"succeeded","health_result":"healthy","status_message":"motorsignal-website restarted and verified active."}'
exit 0
else
echo '{"status":"failed","health_result":"unhealthy","failure_code":"service_unhealthy","status_message":"motorsignal-website was restarted but failed internal health probe."}'
exit 1
fi
;;
restart_api)
echo ">>> [OPS-HELPER] Scheduling detached API restart..."
systemd-run --unit="motorsignal-api-restart-$(date +%s)" \
--description="MotorSignal API Delayed Restart" \
bash -c 'sleep 2; systemctl restart motorsignal-api' >/dev/null 2>&1 || \
sudo -n systemd-run --unit="motorsignal-api-restart-$(date +%s)" \
--description="MotorSignal API Delayed Restart" \
bash -c 'sleep 2; systemctl restart motorsignal-api' >/dev/null 2>&1 || \
(nohup bash -c '
sleep 2
if sudo -n systemctl restart motorsignal-api 2>/dev/null; then
exit 0
elif systemctl restart motorsignal-api 2>/dev/null; then
exit 0
else
API_PID=$(systemctl show -p MainPID --value motorsignal-api 2>/dev/null || pgrep -u motorsignal -f "uvicorn main:app" || echo "")
if [[ -n "$API_PID" && "$API_PID" -gt 0 ]]; then
kill -ABRT "$API_PID" 2>/dev/null || kill -KILL "$API_PID" 2>/dev/null || true
fi
fi
' >/dev/null 2>&1 &)
echo '{"status":"running","status_message":"motorsignal-api restart initiated; process will cycle momentarily."}'
exit 0
;;
reboot_server)
echo ">>> [OPS-HELPER] Scheduling detached server reboot..."
systemd-run --unit="motorsignal-server-reboot-$(date +%s)" \
--description="MotorSignal Server Delayed Reboot" \
bash -c 'sleep 3; systemctl reboot' >/dev/null 2>&1 || \
sudo -n systemd-run --unit="motorsignal-server-reboot-$(date +%s)" \
--description="MotorSignal Server Delayed Reboot" \
bash -c 'sleep 3; systemctl reboot' >/dev/null 2>&1 || \
(nohup bash -c 'sleep 3; sudo -n systemctl reboot || systemctl reboot' >/dev/null 2>&1 &)
echo '{"status":"running","status_message":"Server reboot initiated; system will restart in 3 seconds."}'
exit 0
;;
*)
echo "{\"status\":\"failed\",\"failure_code\":\"unknown_action\",\"status_message\":\"Invalid action: $ACTION\"}"
exit 2
;;
esac
Reference in New Issue View Git Blame Copy Permalink