feat(#172): Vote-Orphans-Banner + Bulk-Auto-Bewerten-Endpoint
Datenlage auf dev: 7281 Plenum-Votes, 96 Bewertungen, nur 19 Matches. Stimmverhalten-Tab zeigt fast nichts, weil die meisten Vote-Drucksachen keine Bewertung haben. Issue #172 schliesst die Luecke. **Banner im Stimmverhalten-Tab:** - Zeigt Anzahl + Verteilung pro BL der "Vote-only"-Drucksachen - Nur sichtbar wenn count > 0 - Aktion: "Auto-Bewerten Top-N" mit Limit-Selector (5/10/20) **Endpoint `GET /api/auswertungen/vote-orphans`:** LEFT JOIN plenum_vote_results vs assessments, count + by_bundesland + Top-N items sortiert nach parsed_at desc. **Endpoint `POST /api/auswertungen/vote-orphans/auto-rate`:** Admin-only, rate-limited 3/min. Nimmt Top-N Orphans, lädt Antragstext per Adapter, enqueued einen Bewertungs-Job pro Drucksache. Defaults limit=10, max 50. Per-skipped-reason-Liste in der Response (Adapter fehlt, Empty-Text, Queue-full, etc.). **Tests:** 4 neue (`TestGetVoteOrphans`), Suite 1071 gruen. Helper `_enqueue_for_bl` aus dem Batch-Endpoint wird hier indirekt wiederverwendet (gleiche Job-Queue-Pipeline). Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
48a272a87d
commit
8136a1a10b
@ -366,6 +366,78 @@ def _avg(values: list[float]) -> Optional[float]:
|
|||||||
return round(sum(values) / len(values), 2) if values else None
|
return round(sum(values) / len(values), 2) if values else None
|
||||||
|
|
||||||
|
|
||||||
|
def get_vote_orphans(
|
||||||
|
filter_bl: Optional[str] = None,
|
||||||
|
limit: int = 200,
|
||||||
|
db_path: Optional[Path] = None,
|
||||||
|
) -> dict:
|
||||||
|
"""Drucksachen mit Plenum-Vote, aber ohne GWÖ-Bewertung (#172).
|
||||||
|
|
||||||
|
Liefert Anzahl + Top-N nach Datum desc. Top-N ist die Ranking-Basis
|
||||||
|
fuer das Bulk-Auto-Bewerten — neueste Anträge zuerst, weil aktuell
|
||||||
|
relevante Themen hoehere Pressewirkung haben.
|
||||||
|
"""
|
||||||
|
path = db_path or settings.db_path
|
||||||
|
if not Path(path).exists():
|
||||||
|
return {"count": 0, "items": [], "by_bundesland": {}}
|
||||||
|
|
||||||
|
conn = sqlite3.connect(str(path))
|
||||||
|
try:
|
||||||
|
# Pro BL: Anzahl Vote-only Drucksachen
|
||||||
|
sql_by_bl = """
|
||||||
|
SELECT p.bundesland, COUNT(DISTINCT p.drucksache) AS n
|
||||||
|
FROM plenum_vote_results p
|
||||||
|
LEFT JOIN assessments a
|
||||||
|
ON a.bundesland = p.bundesland AND a.drucksache = p.drucksache
|
||||||
|
WHERE a.drucksache IS NULL
|
||||||
|
"""
|
||||||
|
params = ()
|
||||||
|
if filter_bl:
|
||||||
|
sql_by_bl += " AND p.bundesland = ?"
|
||||||
|
params = (filter_bl,)
|
||||||
|
sql_by_bl += " GROUP BY p.bundesland ORDER BY n DESC"
|
||||||
|
by_bl = dict(conn.execute(sql_by_bl, params).fetchall())
|
||||||
|
|
||||||
|
# Top-N Drucksachen mit Vote-Result, ohne Assessment
|
||||||
|
sql_top = """
|
||||||
|
SELECT p.bundesland, p.drucksache,
|
||||||
|
MAX(p.parsed_at) AS latest_parsed,
|
||||||
|
p.ergebnis,
|
||||||
|
p.quelle_protokoll
|
||||||
|
FROM plenum_vote_results p
|
||||||
|
LEFT JOIN assessments a
|
||||||
|
ON a.bundesland = p.bundesland AND a.drucksache = p.drucksache
|
||||||
|
WHERE a.drucksache IS NULL
|
||||||
|
"""
|
||||||
|
if filter_bl:
|
||||||
|
sql_top += " AND p.bundesland = ?"
|
||||||
|
sql_top += """
|
||||||
|
GROUP BY p.bundesland, p.drucksache
|
||||||
|
ORDER BY latest_parsed DESC
|
||||||
|
LIMIT ?
|
||||||
|
"""
|
||||||
|
params2 = ((filter_bl, limit) if filter_bl else (limit,))
|
||||||
|
rows = conn.execute(sql_top, params2).fetchall()
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
items = [
|
||||||
|
{
|
||||||
|
"bundesland": r[0],
|
||||||
|
"drucksache": r[1],
|
||||||
|
"parsed_at": r[2],
|
||||||
|
"ergebnis": r[3],
|
||||||
|
"quelle_protokoll": r[4],
|
||||||
|
}
|
||||||
|
for r in rows
|
||||||
|
]
|
||||||
|
return {
|
||||||
|
"count": sum(by_bl.values()),
|
||||||
|
"items": items,
|
||||||
|
"by_bundesland": by_bl,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def aggregate_stimm_index(
|
def aggregate_stimm_index(
|
||||||
filter_bl: Optional[str] = None,
|
filter_bl: Optional[str] = None,
|
||||||
filter_wp: Optional[str] = None,
|
filter_wp: Optional[str] = None,
|
||||||
|
|||||||
82
app/main.py
82
app/main.py
@ -2628,6 +2628,88 @@ async def auswertungen_stimm_index_pro_gruppe(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/api/auswertungen/vote-orphans")
|
||||||
|
async def api_vote_orphans(bundesland: Optional[str] = None, limit: int = 200):
|
||||||
|
"""Drucksachen mit Plenum-Vote aber ohne GWÖ-Bewertung (#172)."""
|
||||||
|
from .auswertungen import get_vote_orphans
|
||||||
|
return get_vote_orphans(filter_bl=bundesland, limit=limit)
|
||||||
|
|
||||||
|
|
||||||
|
@app.post("/api/auswertungen/vote-orphans/auto-rate")
|
||||||
|
@limiter.limit("3/minute")
|
||||||
|
async def api_auto_rate_vote_orphans(
|
||||||
|
request: Request,
|
||||||
|
bundesland: Optional[str] = Form(None),
|
||||||
|
limit: int = Form(10),
|
||||||
|
user: dict = Depends(require_admin),
|
||||||
|
):
|
||||||
|
"""Bulk-Auto-Bewerten der Top-N Vote-Orphans (#172).
|
||||||
|
|
||||||
|
Admin-only + rate-limited. Nimmt die neuesten Drucksachen aus
|
||||||
|
`vote-orphans`, laedt den Antragstext per Adapter herunter und
|
||||||
|
enqueued einen Job pro Drucksache. Konservatives Default-Limit 10.
|
||||||
|
"""
|
||||||
|
if limit < 1 or limit > 50:
|
||||||
|
raise HTTPException(status_code=400, detail="limit muss 1-50 sein")
|
||||||
|
|
||||||
|
from .auswertungen import get_vote_orphans
|
||||||
|
from .queue import enqueue, QueueFullError
|
||||||
|
|
||||||
|
orphans = get_vote_orphans(filter_bl=bundesland, limit=limit)
|
||||||
|
|
||||||
|
enqueued = []
|
||||||
|
skipped = []
|
||||||
|
for item in orphans["items"]:
|
||||||
|
if len(enqueued) >= limit:
|
||||||
|
break
|
||||||
|
bl = item["bundesland"]
|
||||||
|
ds = item["drucksache"]
|
||||||
|
# Defensive: nochmal pruefen
|
||||||
|
existing = await get_assessment(ds)
|
||||||
|
if existing:
|
||||||
|
skipped.append({"drucksache": ds, "reason": "already_rated"})
|
||||||
|
continue
|
||||||
|
adapter = get_adapter(bl)
|
||||||
|
if not adapter:
|
||||||
|
skipped.append({"drucksache": ds, "reason": f"no_adapter_for_{bl}"})
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
text = await adapter.download_text(ds)
|
||||||
|
except Exception as e:
|
||||||
|
skipped.append({"drucksache": ds, "reason": f"download_error: {str(e)[:80]}"})
|
||||||
|
continue
|
||||||
|
if not text:
|
||||||
|
skipped.append({"drucksache": ds, "reason": "empty_text"})
|
||||||
|
continue
|
||||||
|
# Doc-Stub (ohne adapter.search)
|
||||||
|
from .parlamente import Drucksache
|
||||||
|
doc = Drucksache(
|
||||||
|
drucksache=ds, title=ds, fraktionen=[], datum="",
|
||||||
|
link="", bundesland=bl,
|
||||||
|
)
|
||||||
|
job_id = str(uuid.uuid4())
|
||||||
|
await create_job(job_id, text[:500], bl, "qwen-plus", drucksache=ds)
|
||||||
|
try:
|
||||||
|
position = await enqueue(
|
||||||
|
job_id, run_drucksache_analysis,
|
||||||
|
job_id, ds, text, bl, "qwen-plus", doc,
|
||||||
|
drucksache=ds,
|
||||||
|
)
|
||||||
|
enqueued.append({
|
||||||
|
"drucksache": ds, "bundesland": bl,
|
||||||
|
"job_id": job_id, "queue_position": position,
|
||||||
|
})
|
||||||
|
except QueueFullError:
|
||||||
|
skipped.append({"drucksache": ds, "reason": "queue_full"})
|
||||||
|
break
|
||||||
|
return {
|
||||||
|
"status": "auto_rate_enqueued",
|
||||||
|
"enqueued": len(enqueued),
|
||||||
|
"skipped": skipped,
|
||||||
|
"jobs": enqueued,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
@app.get("/api/auswertungen/empfehlungs-konsistenz")
|
@app.get("/api/auswertungen/empfehlungs-konsistenz")
|
||||||
async def auswertungen_empfehlungs_konsistenz(
|
async def auswertungen_empfehlungs_konsistenz(
|
||||||
bundesland: Optional[str] = None,
|
bundesland: Optional[str] = None,
|
||||||
|
|||||||
@ -201,6 +201,28 @@ table.modal-table th { background: var(--ecg-bg-subtle); font-weight: 700; }
|
|||||||
|
|
||||||
<!-- Panel 3: Stimmverhalten × Gemeinwohl -->
|
<!-- Panel 3: Stimmverhalten × Gemeinwohl -->
|
||||||
<div class="auswert-panel" id="panel-stimmverhalten">
|
<div class="auswert-panel" id="panel-stimmverhalten">
|
||||||
|
<!-- Vote-Orphans-Banner: Drucksachen mit Vote, ohne Bewertung (#172) -->
|
||||||
|
<div id="sv-orphans-banner" class="v2-kasten" style="display:none;background:rgba(247,148,29,0.10);border-color:rgba(247,148,29,0.4);margin-bottom:0.75rem;">
|
||||||
|
<p style="margin:0;font-size:12px;line-height:1.5;">
|
||||||
|
<strong id="sv-orphans-count">…</strong> Drucksachen haben einen Plenum-Vote,
|
||||||
|
aber noch keine GWÖ-Bewertung. <span id="sv-orphans-by-bl" style="opacity:0.7;font-family:var(--font-mono);font-size:11px;"></span>
|
||||||
|
<br>
|
||||||
|
Das Stimmverhalten-Feature greift erst, wenn beide Seiten vorhanden sind.
|
||||||
|
</p>
|
||||||
|
<div style="margin-top:8px;display:flex;gap:8px;align-items:center;flex-wrap:wrap;">
|
||||||
|
<label style="font-size:11px;font-family:var(--font-mono);">Auto-Bewerten Top-N:</label>
|
||||||
|
<select id="sv-orphans-limit" style="font-family:var(--font-mono);font-size:11px;padding:3px 8px;border:1px solid var(--ecg-border);border-radius:3px;background:var(--ecg-card-bg);">
|
||||||
|
<option value="5">5</option>
|
||||||
|
<option value="10" selected>10</option>
|
||||||
|
<option value="20">20</option>
|
||||||
|
</select>
|
||||||
|
<button type="button" onclick="bulkRateOrphans()"
|
||||||
|
style="font-family:var(--font-mono);font-size:11px;padding:5px 12px;border:1px solid var(--ecg-teal);background:var(--ecg-teal);color:#fff;border-radius:3px;cursor:pointer;">
|
||||||
|
Top-N bewerten lassen
|
||||||
|
</button>
|
||||||
|
<span id="sv-orphans-result" style="font-size:11px;font-family:var(--font-mono);opacity:0.7;"></span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
<div class="v2-kasten outline-blue" style="margin-bottom:1rem;">
|
<div class="v2-kasten outline-blue" style="margin-bottom:1rem;">
|
||||||
<h4 style="margin-top:0;">Stimmverhalten × Gemeinwohl-Orientierung</h4>
|
<h4 style="margin-top:0;">Stimmverhalten × Gemeinwohl-Orientierung</h4>
|
||||||
<p style="font-size:12px;line-height:1.5;">
|
<p style="font-size:12px;line-height:1.5;">
|
||||||
@ -616,6 +638,7 @@ async function loadStimmverhalten() {
|
|||||||
const bl = svGetBl();
|
const bl = svGetBl();
|
||||||
const exclude = document.getElementById('sv-exclude-antragsteller').checked ? '1' : '0';
|
const exclude = document.getElementById('sv-exclude-antragsteller').checked ? '1' : '0';
|
||||||
|
|
||||||
|
loadVoteOrphansBanner(bl);
|
||||||
loadStimmIndex(bl, exclude);
|
loadStimmIndex(bl, exclude);
|
||||||
loadHeuchelei(bl);
|
loadHeuchelei(bl);
|
||||||
loadMatrixHeatmap();
|
loadMatrixHeatmap();
|
||||||
@ -624,6 +647,54 @@ async function loadStimmverhalten() {
|
|||||||
loadStimmIndexCrossBl(exclude);
|
loadStimmIndexCrossBl(exclude);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function loadVoteOrphansBanner(bl) {
|
||||||
|
const banner = document.getElementById('sv-orphans-banner');
|
||||||
|
const countEl = document.getElementById('sv-orphans-count');
|
||||||
|
const byBlEl = document.getElementById('sv-orphans-by-bl');
|
||||||
|
let url = '/api/auswertungen/vote-orphans?limit=10';
|
||||||
|
if (bl) url += '&bundesland=' + encodeURIComponent(bl);
|
||||||
|
try {
|
||||||
|
const r = await fetch(url);
|
||||||
|
const d = await r.json();
|
||||||
|
if (!d.count) {
|
||||||
|
banner.style.display = 'none';
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
banner.style.display = '';
|
||||||
|
countEl.textContent = d.count.toLocaleString('de-DE');
|
||||||
|
const sortedBl = Object.entries(d.by_bundesland).sort((a, b) => b[1] - a[1]).slice(0, 8);
|
||||||
|
byBlEl.textContent = sortedBl.map(e => `${e[0]}:${e[1]}`).join(' · ');
|
||||||
|
} catch (e) {
|
||||||
|
banner.style.display = 'none';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function bulkRateOrphans() {
|
||||||
|
const bl = svGetBl();
|
||||||
|
const limit = document.getElementById('sv-orphans-limit').value;
|
||||||
|
const result = document.getElementById('sv-orphans-result');
|
||||||
|
result.textContent = '… enqueue läuft';
|
||||||
|
const fd = new FormData();
|
||||||
|
if (bl) fd.append('bundesland', bl);
|
||||||
|
fd.append('limit', limit);
|
||||||
|
try {
|
||||||
|
const r = await fetch('/api/auswertungen/vote-orphans/auto-rate', { method: 'POST', body: fd });
|
||||||
|
if (r.status === 403) { result.textContent = 'Admin-Rechte fehlen'; return; }
|
||||||
|
if (r.status === 429) { result.textContent = 'Rate-Limit (3/min)'; return; }
|
||||||
|
if (!r.ok) {
|
||||||
|
const err = await r.json().catch(() => ({}));
|
||||||
|
result.textContent = 'Fehler: ' + (err.detail || r.statusText);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const data = await r.json();
|
||||||
|
const skip = (data.skipped || []).length;
|
||||||
|
result.innerHTML = `${data.enqueued} enqueued${skip ? `, ${skip} skipped` : ''} — <a href="/v2/admin/queue" style="color:var(--ecg-teal);">Queue ansehen →</a>`;
|
||||||
|
setTimeout(() => loadVoteOrphansBanner(bl), 800);
|
||||||
|
} catch (e) {
|
||||||
|
result.textContent = 'Fehler: ' + e;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
function downloadStimmverhaltenCsv() {
|
function downloadStimmverhaltenCsv() {
|
||||||
const bl = svGetBl();
|
const bl = svGetBl();
|
||||||
const exclude = document.getElementById('sv-exclude-antragsteller').checked ? '1' : '0';
|
const exclude = document.getElementById('sv-exclude-antragsteller').checked ? '1' : '0';
|
||||||
|
|||||||
@ -312,3 +312,84 @@ class TestGetWahlperioden:
|
|||||||
assert wps == sorted(wps)
|
assert wps == sorted(wps)
|
||||||
# Sample-DB enthaelt NRW-WP18, MV-WP8, MV-WP7 sowie BB-WP8
|
# Sample-DB enthaelt NRW-WP18, MV-WP8, MV-WP7 sowie BB-WP8
|
||||||
assert any("NRW" in w for w in wps)
|
assert any("NRW" in w for w in wps)
|
||||||
|
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
# get_vote_orphans (#172)
|
||||||
|
# ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestGetVoteOrphans:
|
||||||
|
@pytest.fixture
|
||||||
|
def orphan_db(self, tmp_path):
|
||||||
|
db = tmp_path / "orphans.db"
|
||||||
|
conn = sqlite3.connect(str(db))
|
||||||
|
conn.execute("""
|
||||||
|
CREATE TABLE assessments (
|
||||||
|
drucksache TEXT PRIMARY KEY, title TEXT, fraktionen TEXT,
|
||||||
|
datum TEXT, bundesland TEXT, gwoe_score REAL,
|
||||||
|
source TEXT, model TEXT, created_at TEXT, updated_at TEXT
|
||||||
|
)
|
||||||
|
""")
|
||||||
|
conn.execute("""
|
||||||
|
CREATE TABLE plenum_vote_results (
|
||||||
|
bundesland TEXT NOT NULL, drucksache TEXT NOT NULL,
|
||||||
|
ergebnis TEXT, einstimmig INTEGER DEFAULT 0,
|
||||||
|
fraktionen_ja TEXT DEFAULT '[]',
|
||||||
|
fraktionen_nein TEXT DEFAULT '[]',
|
||||||
|
fraktionen_enthaltung TEXT DEFAULT '[]',
|
||||||
|
quelle_protokoll TEXT NOT NULL,
|
||||||
|
quelle_url TEXT,
|
||||||
|
parsed_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
PRIMARY KEY (bundesland, drucksache, quelle_protokoll)
|
||||||
|
)
|
||||||
|
""")
|
||||||
|
# 3 Votes: 2 davon mit Bewertung, 1 orphan
|
||||||
|
conn.execute("INSERT INTO assessments VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
||||||
|
("18/1", "T", '["CDU"]', "2024-01-01", "NRW", 7.0,
|
||||||
|
"test", "qwen", "now", "now"))
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO plenum_vote_results "
|
||||||
|
"(bundesland, drucksache, ergebnis, quelle_protokoll, parsed_at) "
|
||||||
|
"VALUES (?, ?, ?, ?, ?)",
|
||||||
|
("NRW", "18/1", "angenommen", "MMP18-1", "2024-01-02"),
|
||||||
|
)
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO plenum_vote_results "
|
||||||
|
"(bundesland, drucksache, ergebnis, quelle_protokoll, parsed_at) "
|
||||||
|
"VALUES (?, ?, ?, ?, ?)",
|
||||||
|
("NRW", "18/2", "abgelehnt", "MMP18-2", "2024-01-05"),
|
||||||
|
)
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO plenum_vote_results "
|
||||||
|
"(bundesland, drucksache, ergebnis, quelle_protokoll, parsed_at) "
|
||||||
|
"VALUES (?, ?, ?, ?, ?)",
|
||||||
|
("BB", "8/3", "angenommen", "BB8-1", "2024-01-04"),
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
conn.close()
|
||||||
|
return db
|
||||||
|
|
||||||
|
def test_count_excludes_rated(self, orphan_db):
|
||||||
|
from app.auswertungen import get_vote_orphans
|
||||||
|
result = get_vote_orphans(db_path=orphan_db)
|
||||||
|
# 18/1 hat assessment → nicht orphan. 18/2 (NRW) + 8/3 (BB) sind orphans.
|
||||||
|
assert result["count"] == 2
|
||||||
|
assert result["by_bundesland"] == {"NRW": 1, "BB": 1}
|
||||||
|
|
||||||
|
def test_filter_bl(self, orphan_db):
|
||||||
|
from app.auswertungen import get_vote_orphans
|
||||||
|
nrw = get_vote_orphans(filter_bl="NRW", db_path=orphan_db)
|
||||||
|
assert nrw["count"] == 1
|
||||||
|
assert nrw["items"][0]["drucksache"] == "18/2"
|
||||||
|
|
||||||
|
def test_sort_by_latest_parsed_desc(self, orphan_db):
|
||||||
|
from app.auswertungen import get_vote_orphans
|
||||||
|
result = get_vote_orphans(db_path=orphan_db)
|
||||||
|
# 18/2 ist 2024-01-05, 8/3 ist 2024-01-04 → 18/2 zuerst
|
||||||
|
assert result["items"][0]["drucksache"] == "18/2"
|
||||||
|
|
||||||
|
def test_empty_db(self, tmp_path):
|
||||||
|
from app.auswertungen import get_vote_orphans
|
||||||
|
result = get_vote_orphans(db_path=tmp_path / "missing.db")
|
||||||
|
assert result == {"count": 0, "items": [], "by_bundesland": {}}
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user