From e8cde3a9a1003db538b21e7a69e00b912555d683 Mon Sep 17 00:00:00 2001 From: Billie Ponder Date: Tue, 22 Sep 2026 01:18:39 -0400 Subject: [PATCH] fix(ops): add sibling process cycling fallback and health probe loop to helper script (#22) Co-authored-by: Developer --- .../scripts/ops-action-helper.sh | 84 ++++++++++++++++--- 1 file changed, 71 insertions(+), 13 deletions(-) diff --git a/backend/motorsignal_api/scripts/ops-action-helper.sh b/backend/motorsignal_api/scripts/ops-action-helper.sh index f2204ab..74d6908 100644 --- a/backend/motorsignal_api/scripts/ops-action-helper.sh +++ b/backend/motorsignal_api/scripts/ops-action-helper.sh @@ -20,13 +20,48 @@ PUBLIC_WEBSITE_URL="https://motorsignal.app/api/version" INTERNAL_API_URL="http://127.0.0.1:8080/api/v1/health" restart_website_service() { + # 1. Privileged systemctl restart (succeeds when run by motorsignal-admin with sudo) if sudo -n systemctl restart motorsignal-website >/dev/null 2>&1; then return 0 - elif systemctl restart motorsignal-website >/dev/null 2>&1; then - return 0 - else - return 1 fi + if systemctl restart motorsignal-website >/dev/null 2>&1; then + return 0 + fi + + # 2. Process-level restart: + # motorsignal-api and motorsignal-website both run as User=motorsignal. + # The API helper process has POSIX permission to signal the sibling website process. + # motorsignal-website.service is configured with Restart=on-failure (RestartSec=5s). + # Terminating the current node process with SIGABRT triggers systemd's failure-recovery + # restart, cleanly respawning the service with the current runtime. + local OLD_PID + OLD_PID=$(systemctl show -p MainPID --value motorsignal-website 2>/dev/null || echo "") + if [[ -z "$OLD_PID" || "$OLD_PID" -le 0 ]]; then + OLD_PID=$(pgrep -u motorsignal -f "next-server" 2>/dev/null || true) + fi + + if [[ -n "$OLD_PID" && "$OLD_PID" -gt 0 ]]; then + echo ">>> [OPS-HELPER] Cycling website process PID $OLD_PID via systemd Restart=on-failure..." + kill -ABRT "$OLD_PID" >/dev/null 2>&1 || kill -KILL "$OLD_PID" >/dev/null 2>&1 || true + + # Wait for systemd to cycle (RestartSec=5s) and verify new PID has spawned + for i in {1..15}; do + sleep 1 + local NEW_PID + NEW_PID=$(systemctl show -p MainPID --value motorsignal-website 2>/dev/null || echo "") + if [[ -n "$NEW_PID" && "$NEW_PID" -gt 0 && "$NEW_PID" -ne "$OLD_PID" ]]; then + echo ">>> [OPS-HELPER] Website service respawned with PID $NEW_PID." + return 0 + fi + done + + # If systemctl show didn't report a new PID, check if the website port has recovered + if curl -fsS --max-time 2 "$INTERNAL_WEBSITE_URL" >/dev/null 2>&1; then + return 0 + fi + fi + + return 1 } case "$ACTION" in @@ -63,16 +98,19 @@ case "$ACTION" in # 2. Restart website service to activate deployed runtime if ! restart_website_service; then - echo '{"status":"failed","failure_code":"service_restart_failed","status_message":"Failed to restart motorsignal-website.service."}' + echo '{"status":"failed","failure_code":"service_restart_failed","status_message":"Failed to restart motorsignal-website."}' exit 1 fi # 3. Verification Gate (Loopback & Public) - sleep 2 VERIFY_OK=0 - if curl -fsS --max-time 5 "$INTERNAL_WEBSITE_URL" >/dev/null 2>&1; then - VERIFY_OK=1 - fi + for i in {1..15}; do + if curl -fsS --max-time 3 "$INTERNAL_WEBSITE_URL" >/dev/null 2>&1; then + VERIFY_OK=1 + break + fi + sleep 1 + done if [[ "$VERIFY_OK" -eq 1 ]]; then VER_JSON=$(curl -fsS --max-time 5 "$INTERNAL_WEBSITE_URL" 2>/dev/null || echo '{"revision":"unknown"}') @@ -95,12 +133,20 @@ case "$ACTION" in restart_website) echo ">>> [OPS-HELPER] Restarting motorsignal-website..." if ! restart_website_service; then - echo '{"status":"failed","failure_code":"service_restart_failed","status_message":"Failed to restart motorsignal-website.service."}' + echo '{"status":"failed","failure_code":"service_restart_failed","status_message":"Failed to restart motorsignal-website."}' exit 1 fi - sleep 2 - if curl -fsS --max-time 5 "$INTERNAL_WEBSITE_URL" >/dev/null 2>&1; then + VERIFY_OK=0 + for i in {1..15}; do + if curl -fsS --max-time 3 "$INTERNAL_WEBSITE_URL" >/dev/null 2>&1; then + VERIFY_OK=1 + break + fi + sleep 1 + done + + if [[ "$VERIFY_OK" -eq 1 ]]; then echo '{"status":"succeeded","health_result":"healthy","status_message":"motorsignal-website restarted and verified active."}' exit 0 else @@ -117,7 +163,19 @@ case "$ACTION" in sudo -n systemd-run --unit="motorsignal-api-restart-$(date +%s)" \ --description="MotorSignal API Delayed Restart" \ bash -c 'sleep 2; systemctl restart motorsignal-api' >/dev/null 2>&1 || \ - (nohup bash -c 'sleep 2; sudo -n systemctl restart motorsignal-api || systemctl restart motorsignal-api' >/dev/null 2>&1 &) + (nohup bash -c ' + sleep 2 + if sudo -n systemctl restart motorsignal-api 2>/dev/null; then + exit 0 + elif systemctl restart motorsignal-api 2>/dev/null; then + exit 0 + else + API_PID=$(systemctl show -p MainPID --value motorsignal-api 2>/dev/null || pgrep -u motorsignal -f "uvicorn main:app" || echo "") + if [[ -n "$API_PID" && "$API_PID" -gt 0 ]]; then + kill -ABRT "$API_PID" 2>/dev/null || kill -KILL "$API_PID" 2>/dev/null || true + fi + fi + ' >/dev/null 2>&1 &) echo '{"status":"running","status_message":"motorsignal-api restart initiated; process will cycle momentarily."}' exit 0