45 lines
1.2 KiB
Bash
45 lines
1.2 KiB
Bash
|
#!/bin/bash
|
||
|
|
# Läuft Beratungsfolge-Scraper in Tranchen bis alles fertig ist
|
||
|
|
|
||
|
|
cd "$(dirname "$0")/.."
|
||
|
|
source .venv/bin/activate
|
||
|
|
|
||
|
|
LOG_FILE="data/beratungen_batches.log"
|
||
|
|
BATCH_SIZE=200
|
||
|
|
PAUSE_SECONDS=10
|
||
|
|
|
||
|
|
echo "=== Beratungsfolge-Batch-Runner gestartet $(date) ===" | tee -a "$LOG_FILE"
|
||
|
|
|
||
|
|
while true; do
|
||
|
|
echo "" | tee -a "$LOG_FILE"
|
||
|
|
echo "--- Starte Batch $(date +%H:%M:%S) ---" | tee -a "$LOG_FILE"
|
||
|
|
|
||
|
|
python scripts/scrape_beratungsfolge.py --limit $BATCH_SIZE 2>&1 | tee -a "$LOG_FILE"
|
||
|
|
|
||
|
|
# Prüfe ob noch Vorlagen ohne Beschlusstext übrig sind
|
||
|
|
REMAINING=$(python -c "
|
||
|
|
import sqlite3
|
||
|
|
conn = sqlite3.connect('data/tracker_remote.db')
|
||
|
|
r = conn.execute('''
|
||
|
|
SELECT COUNT(DISTINCT v.id)
|
||
|
|
FROM vorlagen v
|
||
|
|
LEFT JOIN beratungen b ON v.id = b.vorlage_id AND b.beschlusstext IS NOT NULL
|
||
|
|
WHERE v.web_url IS NOT NULL AND b.id IS NULL
|
||
|
|
''').fetchone()[0]
|
||
|
|
print(r)
|
||
|
|
")
|
||
|
|
|
||
|
|
echo "Verbleibend: $REMAINING" | tee -a "$LOG_FILE"
|
||
|
|
|
||
|
|
if [ "$REMAINING" -eq 0 ]; then
|
||
|
|
echo "" | tee -a "$LOG_FILE"
|
||
|
|
echo "=== ALLE FERTIG $(date) ===" | tee -a "$LOG_FILE"
|
||
|
|
break
|
||
|
|
fi
|
||
|
|
|
||
|
|
echo "Pause ${PAUSE_SECONDS}s..." | tee -a "$LOG_FILE"
|
||
|
|
sleep $PAUSE_SECONDS
|
||
|
|
done
|
||
|
|
|
||
|
|
echo "Runner beendet." | tee -a "$LOG_FILE"
|