motorsignal
/
MotorSignal-App
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(ops): add sibling process cycling fallback and health probe loop to helper script (#22) 

Co-authored-by: Developer <dev@example.com>
This commit is contained in:
Billie Ponder 2026-09-22 01:18:39 -04:00 committed by GitHub
parent f20c1d13e6
commit e8cde3a9a1
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 71 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

84
backend/motorsignal_api/scripts/ops-action-helper.sh
View File

@ -20,13 +20,48 @@ PUBLIC_WEBSITE_URL="https://motorsignal.app/api/version"
INTERNAL_API_URL="http://127.0.0.1:8080/api/v1/health"
restart_website_service() {
# 1. Privileged systemctl restart (succeeds when run by motorsignal-admin with sudo)
if sudo -n systemctl restart motorsignal-website >/dev/null 2>&1; then
return 0
elif systemctl restart motorsignal-website >/dev/null 2>&1; then
return 0
else
return 1
fi
if systemctl restart motorsignal-website >/dev/null 2>&1; then
return 0
fi
# 2. Process-level restart:
# motorsignal-api and motorsignal-website both run as User=motorsignal.
# The API helper process has POSIX permission to signal the sibling website process.
# motorsignal-website.service is configured with Restart=on-failure (RestartSec=5s).
# Terminating the current node process with SIGABRT triggers systemd's failure-recovery
# restart, cleanly respawning the service with the current runtime.
local OLD_PID
OLD_PID=$(systemctl show -p MainPID --value motorsignal-website 2>/dev/null || echo "")
if [[ -z "$OLD_PID" || "$OLD_PID" -le 0 ]]; then
OLD_PID=$(pgrep -u motorsignal -f "next-server" 2>/dev/null || true)
fi
if [[ -n "$OLD_PID" && "$OLD_PID" -gt 0 ]]; then
echo ">>> [OPS-HELPER] Cycling website process PID $OLD_PID via systemd Restart=on-failure..."
kill -ABRT "$OLD_PID" >/dev/null 2>&1 || kill -KILL "$OLD_PID" >/dev/null 2>&1 || true
# Wait for systemd to cycle (RestartSec=5s) and verify new PID has spawned
for i in {1..15}; do
sleep 1
local NEW_PID
NEW_PID=$(systemctl show -p MainPID --value motorsignal-website 2>/dev/null || echo "")
if [[ -n "$NEW_PID" && "$NEW_PID" -gt 0 && "$NEW_PID" -ne "$OLD_PID" ]]; then
echo ">>> [OPS-HELPER] Website service respawned with PID $NEW_PID."
return 0
fi
done
# If systemctl show didn't report a new PID, check if the website port has recovered
if curl -fsS --max-time 2 "$INTERNAL_WEBSITE_URL" >/dev/null 2>&1; then
return 0
fi
fi
return 1
}
case "$ACTION" in
@ -63,16 +98,19 @@ case "$ACTION" in
# 2. Restart website service to activate deployed runtime
if ! restart_website_service; then
echo '{"status":"failed","failure_code":"service_restart_failed","status_message":"Failed to restart motorsignal-website.service."}'
echo '{"status":"failed","failure_code":"service_restart_failed","status_message":"Failed to restart motorsignal-website."}'
exit 1
fi
# 3. Verification Gate (Loopback & Public)
sleep 2
VERIFY_OK=0
if curl -fsS --max-time 5 "$INTERNAL_WEBSITE_URL" >/dev/null 2>&1; then
VERIFY_OK=1
fi
for i in {1..15}; do
if curl -fsS --max-time 3 "$INTERNAL_WEBSITE_URL" >/dev/null 2>&1; then
VERIFY_OK=1
break
fi
sleep 1
done
if [[ "$VERIFY_OK" -eq 1 ]]; then
VER_JSON=$(curl -fsS --max-time 5 "$INTERNAL_WEBSITE_URL" 2>/dev/null || echo '{"revision":"unknown"}')
@ -95,12 +133,20 @@ case "$ACTION" in
restart_website)
echo ">>> [OPS-HELPER] Restarting motorsignal-website..."
if ! restart_website_service; then
echo '{"status":"failed","failure_code":"service_restart_failed","status_message":"Failed to restart motorsignal-website.service."}'
echo '{"status":"failed","failure_code":"service_restart_failed","status_message":"Failed to restart motorsignal-website."}'
exit 1
fi
sleep 2
if curl -fsS --max-time 5 "$INTERNAL_WEBSITE_URL" >/dev/null 2>&1; then
VERIFY_OK=0
for i in {1..15}; do
if curl -fsS --max-time 3 "$INTERNAL_WEBSITE_URL" >/dev/null 2>&1; then
VERIFY_OK=1
break
fi
sleep 1
done
if [[ "$VERIFY_OK" -eq 1 ]]; then
echo '{"status":"succeeded","health_result":"healthy","status_message":"motorsignal-website restarted and verified active."}'
exit 0
else
@ -117,7 +163,19 @@ case "$ACTION" in
sudo -n systemd-run --unit="motorsignal-api-restart-$(date +%s)" \
--description="MotorSignal API Delayed Restart" \
bash -c 'sleep 2; systemctl restart motorsignal-api' >/dev/null 2>&1 || \
(nohup bash -c 'sleep 2; sudo -n systemctl restart motorsignal-api || systemctl restart motorsignal-api' >/dev/null 2>&1 &)
(nohup bash -c '
sleep 2
if sudo -n systemctl restart motorsignal-api 2>/dev/null; then
exit 0
elif systemctl restart motorsignal-api 2>/dev/null; then
exit 0
else
API_PID=$(systemctl show -p MainPID --value motorsignal-api 2>/dev/null || pgrep -u motorsignal -f "uvicorn main:app" || echo "")
if [[ -n "$API_PID" && "$API_PID" -gt 0 ]]; then
kill -ABRT "$API_PID" 2>/dev/null || kill -KILL "$API_PID" 2>/dev/null || true
fi
fi
' >/dev/null 2>&1 &)
echo '{"status":"running","status_message":"motorsignal-api restart initiated; process will cycle momentarily."}'
exit 0