#!/bin/bash # Prüft Background-Jobs anhand ihrer Heartbeat-Dateien. # Per Cron alle 5 Minuten aufrufen: # */5 * * * * /path/to/antragstracker/scripts/check-background-jobs.sh >> /path/to/antragstracker/data/job-checker.log 2>&1 set -euo pipefail cd "$(dirname "$0")/.." DATA_DIR="data" MAX_STALE_SECONDS=1800 # 30 Minuten ohne Update = stale now=$(date +%s) check_job() { local name="$1" local heartbeat_file="$2" local restart_cmd="$3" if [ ! -f "$heartbeat_file" ]; then return 0 # Kein Heartbeat = Job nicht aktiv, OK fi local status=$(python3 -c "import json; d=json.load(open('$heartbeat_file')); print(d.get('status','unknown'))" 2>/dev/null || echo "error") if [ "$status" = "completed" ]; then echo "[$(date)] ✅ $name: fertig" return 0 fi # PID prüfen local pid=$(python3 -c "import json; d=json.load(open('$heartbeat_file')); print(d.get('pid',''))" 2>/dev/null || echo "") if [ -n "$pid" ] && [ "$pid" != "null" ] && [ "$pid" != "None" ]; then if ! kill -0 "$pid" 2>/dev/null; then echo "[$(date)] ❌ $name: Prozess $pid tot → Neustart" eval "$restart_cmd" return 0 fi fi # Timestamp prüfen local last_batch=$(python3 -c " import json, datetime d = json.load(open('$heartbeat_file')) ts = d.get('last_batch_at') or d.get('started_at') or '' if ts: dt = datetime.datetime.fromisoformat(ts.replace('Z', '+00:00')) print(int(dt.timestamp())) else: print(0) " 2>/dev/null || echo "0") if [ "$last_batch" != "0" ]; then local age=$((now - last_batch)) if [ "$age" -gt "$MAX_STALE_SECONDS" ]; then echo "[$(date)] ⚠️ $name: Heartbeat ${age}s alt (max ${MAX_STALE_SECONDS}s) → Neustart" # Kill alten Prozess [ -n "$pid" ] && [ "$pid" != "null" ] && kill "$pid" 2>/dev/null || true sleep 2 eval "$restart_cmd" return 0 fi fi # Alles gut local progress=$(python3 -c " import json d = json.load(open('$heartbeat_file')) done = d.get('total_geocoded', d.get('total_done', '?')) pending = d.get('total_pending', '?') print(f'{done} done, {pending} pending') " 2>/dev/null || echo "?") echo "[$(date)] ✓ $name: läuft (PID $pid, $progress)" } # === Jobs registrieren === PROJ_DIR="$(pwd)" check_job "Geocoding" \ "$DATA_DIR/geocode-heartbeat.json" \ "cd '$PROJ_DIR' && nohup bash scripts/geocode_background.sh >> data/geocode.log 2>&1 &" # Weitere Jobs hier eintragen: # check_job "OParl-Sync" "$DATA_DIR/sync-heartbeat.json" "..."