fix(ops): add sibling process cycling fallback and health probe loop to helper script (#22)

Co-authored-by: Developer <dev@example.com>

backend/motorsignal_api/scripts/ops-action-helper.sh

@@ -20,13 +20,48 @@ PUBLIC_WEBSITE_URL="https://motorsignal.app/api/version"
 INTERNAL_API_URL="http://127.0.0.1:8080/api/v1/health"
 
 restart_website_service() {
+    # 1. Privileged systemctl restart (succeeds when run by motorsignal-admin with sudo)
     if sudo -n systemctl restart motorsignal-website >/dev/null 2>&1; then
         return 0
-    elif systemctl restart motorsignal-website >/dev/null 2>&1; then
-        return 0
-    else
-        return 1
     fi
+    if systemctl restart motorsignal-website >/dev/null 2>&1; then
+        return 0
+    fi
+
+    # 2. Process-level restart:
+    # motorsignal-api and motorsignal-website both run as User=motorsignal.
+    # The API helper process has POSIX permission to signal the sibling website process.
+    # motorsignal-website.service is configured with Restart=on-failure (RestartSec=5s).
+    # Terminating the current node process with SIGABRT triggers systemd's failure-recovery
+    # restart, cleanly respawning the service with the current runtime.
+    local OLD_PID
+    OLD_PID=$(systemctl show -p MainPID --value motorsignal-website 2>/dev/null || echo "")
+    if [[ -z "$OLD_PID" || "$OLD_PID" -le 0 ]]; then
+        OLD_PID=$(pgrep -u motorsignal -f "next-server" 2>/dev/null || true)
+    fi
+
+    if [[ -n "$OLD_PID" && "$OLD_PID" -gt 0 ]]; then
+        echo ">>> [OPS-HELPER] Cycling website process PID $OLD_PID via systemd Restart=on-failure..."
+        kill -ABRT "$OLD_PID" >/dev/null 2>&1 || kill -KILL "$OLD_PID" >/dev/null 2>&1 || true
+
+        # Wait for systemd to cycle (RestartSec=5s) and verify new PID has spawned
+        for i in {1..15}; do
+            sleep 1
+            local NEW_PID
+            NEW_PID=$(systemctl show -p MainPID --value motorsignal-website 2>/dev/null || echo "")
+            if [[ -n "$NEW_PID" && "$NEW_PID" -gt 0 && "$NEW_PID" -ne "$OLD_PID" ]]; then
+                echo ">>> [OPS-HELPER] Website service respawned with PID $NEW_PID."
+                return 0
+            fi
+        done
+
+        # If systemctl show didn't report a new PID, check if the website port has recovered
+        if curl -fsS --max-time 2 "$INTERNAL_WEBSITE_URL" >/dev/null 2>&1; then
+            return 0
+        fi
+    fi
+
+    return 1
 }
 
 case "$ACTION" in
@@ -63,16 +98,19 @@ case "$ACTION" in
 
         # 2. Restart website service to activate deployed runtime
         if ! restart_website_service; then
-            echo '{"status":"failed","failure_code":"service_restart_failed","status_message":"Failed to restart motorsignal-website.service."}'
+            echo '{"status":"failed","failure_code":"service_restart_failed","status_message":"Failed to restart motorsignal-website."}'
             exit 1
         fi
 
         # 3. Verification Gate (Loopback & Public)
-        sleep 2
         VERIFY_OK=0
-        if curl -fsS --max-time 5 "$INTERNAL_WEBSITE_URL" >/dev/null 2>&1; then
+        for i in {1..15}; do
+            if curl -fsS --max-time 3 "$INTERNAL_WEBSITE_URL" >/dev/null 2>&1; then
                 VERIFY_OK=1
+                break
             fi
+            sleep 1
+        done
 
         if [[ "$VERIFY_OK" -eq 1 ]]; then
             VER_JSON=$(curl -fsS --max-time 5 "$INTERNAL_WEBSITE_URL" 2>/dev/null || echo '{"revision":"unknown"}')
@@ -95,12 +133,20 @@ case "$ACTION" in
     restart_website)
         echo ">>> [OPS-HELPER] Restarting motorsignal-website..."
         if ! restart_website_service; then
-            echo '{"status":"failed","failure_code":"service_restart_failed","status_message":"Failed to restart motorsignal-website.service."}'
+            echo '{"status":"failed","failure_code":"service_restart_failed","status_message":"Failed to restart motorsignal-website."}'
             exit 1
         fi
-        sleep 2
         
-        if curl -fsS --max-time 5 "$INTERNAL_WEBSITE_URL" >/dev/null 2>&1; then
+        VERIFY_OK=0
+        for i in {1..15}; do
+            if curl -fsS --max-time 3 "$INTERNAL_WEBSITE_URL" >/dev/null 2>&1; then
+                VERIFY_OK=1
+                break
+            fi
+            sleep 1
+        done
+        
+        if [[ "$VERIFY_OK" -eq 1 ]]; then
             echo '{"status":"succeeded","health_result":"healthy","status_message":"motorsignal-website restarted and verified active."}'
             exit 0
         else
@@ -117,7 +163,19 @@ case "$ACTION" in
         sudo -n systemd-run --unit="motorsignal-api-restart-$(date +%s)" \
                     --description="MotorSignal API Delayed Restart" \
                     bash -c 'sleep 2; systemctl restart motorsignal-api' >/dev/null 2>&1 || \
-        (nohup bash -c 'sleep 2; sudo -n systemctl restart motorsignal-api || systemctl restart motorsignal-api' >/dev/null 2>&1 &)
+        (nohup bash -c '
+            sleep 2
+            if sudo -n systemctl restart motorsignal-api 2>/dev/null; then
+                exit 0
+            elif systemctl restart motorsignal-api 2>/dev/null; then
+                exit 0
+            else
+                API_PID=$(systemctl show -p MainPID --value motorsignal-api 2>/dev/null || pgrep -u motorsignal -f "uvicorn main:app" || echo "")
+                if [[ -n "$API_PID" && "$API_PID" -gt 0 ]]; then
+                    kill -ABRT "$API_PID" 2>/dev/null || kill -KILL "$API_PID" 2>/dev/null || true
+                fi
+            fi
+        ' >/dev/null 2>&1 &)
 
         echo '{"status":"running","status_message":"motorsignal-api restart initiated; process will cycle momentarily."}'
         exit 0