From d30fcb132af50a5d3f1fe8dcb953c60f1f164459 Mon Sep 17 00:00:00 2001 From: Dotty Dotter Date: Wed, 6 May 2026 02:49:06 +0200 Subject: [PATCH] feat: Stand-Dashboard, Score-Histogram, PM-Markdown, Live-Polling, Cluster-Indicator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sechs zusammengehoerige UX/Performance-Erweiterungen: **1. /v2/admin/stand β€” System-Stand-Dashboard** KPI-Kacheln (Bewertungen, Plenum-Votes, Match, Vote-Orphans, News, PM- Drafts, Bookmarks) + GWΓ–-Score-Histogram + Per-BL-Tabelle + News-Source- Tabelle. Auto-Refresh 30 s. Endpoint /api/admin/stand liefert alles in einem Roundtrip. Nav-Eintrag "Stand" in der Admin-Sektion. **2. /auswertungen Score-Histogram-Tab** 4. Tab "Score-Verteilung" mit Bar-Chart 0–10. Endpoint /api/auswertungen/score-histogram liefert Buckets, optional gefiltert nach Bundesland + Wahlperiode. Reagiert auf den globalen BL-Filter. **3. PM-Body Markdown-Rendering** Mini-Renderer im Modal: **bold** / __bold__ / *italic* / _italic_ / - list-bullets / Doppel-Newline-Paragraphen. Kein externer Markdown- Parser, keine neue Dependency. Body wird HTML-escaped, Patterns dann zu Tags umgesetzt. **4. Performance-Cache fuer themen_matching** TTL-Cache (60 s) fuer aggregate_top_themen und aggregate_news_cluster. Cache-Key inkl. aller Filter-Parameter. Automatische Invalidation in news_aggregator.run_aggregator nach erfolgreichem Insert/Embed. 4 neue Tests fuer cache_get/set/clear-Verhalten. **5. Stimmverhalten Banner Live-Update** Statt setTimeout(800) jetzt pollQueueUntilDrained: alle 4 s GET /api/queue/status, Banner zeigt pending + elapsed live. Bei pending=0 zwei Polls in Folge: Banner + Stimmverhalten-Charts neu laden. Max 5 Min Polling-Timeout. Bricht ab wenn Tab gewechselt wird. **6. Antrag-Detail Cluster-Indicator** News-Match-Box im Antrag-Detail laedt parallel /aktuelle-themen/cluster und mappt URL β†’ Cluster. Pro News-Card ein "πŸ”— Cluster (N News)"-Badge mit Hover-Tooltip der anderen Cluster-Members. Macht thematische BΓΌndel sichtbar, ohne Pop-Out auf den Cluster-Tab. Suite: 1088 β†’ 1092 grΓΌn (4 Cache-Tests). Co-Authored-By: Claude Opus 4.7 (1M context) --- app/main.py | 160 ++++++++++++ app/news_aggregator.py | 10 +- app/templates/v2/base.html | 1 + app/templates/v2/screens/admin_stand.html | 242 ++++++++++++++++++ app/templates/v2/screens/aktuelle-themen.html | 42 ++- app/templates/v2/screens/antrag_detail.html | 46 +++- app/templates/v2/screens/auswertungen.html | 114 ++++++++- app/themen_matching.py | 58 ++++- tests/test_themen_matching.py | 40 +++ 9 files changed, 696 insertions(+), 17 deletions(-) create mode 100644 app/templates/v2/screens/admin_stand.html diff --git a/app/main.py b/app/main.py index 62bb5bd..43dcbf5 100644 --- a/app/main.py +++ b/app/main.py @@ -2228,6 +2228,46 @@ async def api_draft_versions(drucksache: str, news_url: str): } +@app.get("/api/auswertungen/score-histogram") +async def auswertungen_score_histogram( + bundesland: Optional[str] = None, + wahlperiode: Optional[str] = None, +): + """GWΓ–-Score-Verteilung (0–10) ueber alle Bewertungen. + + Liefert ein Bucket-Array fuer einen Histogramm-Chart. Filterbar + ueber Bundesland + Wahlperiode (gleicher Pattern wie /matrix). + """ + import sqlite3 + from .auswertungen import _load_assessments + rows = _load_assessments() + from .wahlperioden import wahlperiode_for + + buckets = [0] * 11 + total = 0 + for r in rows: + if bundesland and r["bundesland"] != bundesland: + continue + if wahlperiode is not None: + wp = wahlperiode_for(r["datum"], r["bundesland"]) + if wp != wahlperiode: + continue + score = r["gwoe_score"] + if score is None: + continue + bucket = min(10, max(0, int(score))) + buckets[bucket] += 1 + total += 1 + return { + "buckets": [ + {"score_min": i, "score_max": i + 1, "count": c} + for i, c in enumerate(buckets) + ], + "total": total, + "filter": {"bundesland": bundesland, "wahlperiode": wahlperiode}, + } + + @app.get("/api/auswertungen/matrix") async def auswertungen_matrix( wahlperiode: Optional[str] = None, @@ -2902,6 +2942,126 @@ async def v2_admin_queue(request: Request, user: dict = Depends(require_admin)): }) +@app.get("/v2/admin/stand", response_class=HTMLResponse) +async def v2_admin_stand(request: Request, user: dict = Depends(require_admin)): + """System-Stand-Dashboard β€” Ueberblick ueber alle Datenmengen.""" + return templates.TemplateResponse("v2/screens/admin_stand.html", { + "request": request, + "v2_active_nav": "admin_stand", + **_v2_template_context(user), + }) + + +@app.get("/api/admin/stand") +async def api_admin_stand(user: dict = Depends(require_admin)): + """Datenstand-Aggregation fΓΌr das Stand-Dashboard. + + Liefert Gesamt + Per-Quelle + Letzte-7-Tage in einem Roundtrip. + """ + import sqlite3 + from datetime import datetime, timedelta, timezone + + db = sqlite3.connect(str(settings.db_path)) + cutoff_7d = (datetime.now(timezone.utc) - timedelta(days=7)).isoformat() + try: + # Assessments + n_ass_total = db.execute( + "SELECT COUNT(*) FROM assessments WHERE gwoe_score IS NOT NULL" + ).fetchone()[0] + n_ass_7d = db.execute( + "SELECT COUNT(*) FROM assessments " + "WHERE gwoe_score IS NOT NULL AND created_at >= ?", + (cutoff_7d,), + ).fetchone()[0] + ass_per_bl = dict(db.execute( + "SELECT bundesland, COUNT(*) FROM assessments " + "WHERE gwoe_score IS NOT NULL GROUP BY bundesland ORDER BY 2 DESC" + ).fetchall()) + + # Score-Verteilung + score_dist = dict(db.execute( + "SELECT CAST(gwoe_score AS INTEGER), COUNT(*) FROM assessments " + "WHERE gwoe_score IS NOT NULL GROUP BY CAST(gwoe_score AS INTEGER)" + ).fetchall()) + + # Plenum-Votes + n_votes = db.execute("SELECT COUNT(*) FROM plenum_vote_results").fetchone()[0] + votes_per_bl = dict(db.execute( + "SELECT bundesland, COUNT(*) FROM plenum_vote_results " + "GROUP BY bundesland ORDER BY 2 DESC" + ).fetchall()) + + # Match (assessment ∩ vote) + n_match = db.execute(""" + SELECT COUNT(DISTINCT a.drucksache) FROM assessments a + INNER JOIN plenum_vote_results p + ON a.bundesland=p.bundesland AND a.drucksache=p.drucksache + WHERE a.gwoe_score IS NOT NULL + """).fetchone()[0] + n_orphans = db.execute(""" + SELECT COUNT(DISTINCT p.bundesland || '/' || p.drucksache) + FROM plenum_vote_results p + LEFT JOIN assessments a + ON a.bundesland=p.bundesland AND a.drucksache=p.drucksache + WHERE a.drucksache IS NULL + """).fetchone()[0] + + # News + n_news = db.execute("SELECT COUNT(*) FROM news_articles").fetchone()[0] + n_news_emb = db.execute( + "SELECT COUNT(*) FROM news_articles WHERE summary_embedding IS NOT NULL" + ).fetchone()[0] + n_news_7d = db.execute( + "SELECT COUNT(*) FROM news_articles WHERE datum >= ?", (cutoff_7d,), + ).fetchone()[0] + news_per_source = dict(db.execute( + "SELECT source, COUNT(*) FROM news_articles GROUP BY source" + ).fetchall()) + + # PM-Drafts + n_drafts = db.execute("SELECT COUNT(*) FROM presse_drafts").fetchone()[0] + n_drafts_7d = db.execute( + "SELECT COUNT(*) FROM presse_drafts WHERE created_at >= ?", + (cutoff_7d,), + ).fetchone()[0] + + # Bookmarks + try: + n_bookmarks = db.execute("SELECT COUNT(*) FROM bookmarks").fetchone()[0] + except sqlite3.OperationalError: + n_bookmarks = 0 + finally: + db.close() + + return { + "assessments": { + "total": n_ass_total, + "last_7_days": n_ass_7d, + "by_bundesland": ass_per_bl, + "score_distribution": score_dist, + }, + "plenum_votes": { + "total": n_votes, + "by_bundesland": votes_per_bl, + }, + "match": { + "with_assessment_and_vote": n_match, + "vote_orphans": n_orphans, + }, + "news": { + "total": n_news, + "embedded": n_news_emb, + "last_7_days": n_news_7d, + "by_source": news_per_source, + }, + "presse_drafts": { + "total": n_drafts, + "last_7_days": n_drafts_7d, + }, + "bookmarks": n_bookmarks, + } + + @app.get("/v2/admin/abos", response_class=HTMLResponse) async def v2_admin_abos(request: Request, user: dict = Depends(require_admin)): """Abo-Verwaltung β€” alle E-Mail-Abonnements (Admin).""" diff --git a/app/news_aggregator.py b/app/news_aggregator.py index ee1c324..35b7d84 100644 --- a/app/news_aggregator.py +++ b/app/news_aggregator.py @@ -344,4 +344,12 @@ def run_aggregator(db_path: Optional[Path] = None, embed: bool = True) -> dict: geworfen. """ articles = fetch_all() - return upsert_articles(articles, db_path=db_path, embed=embed) + result = upsert_articles(articles, db_path=db_path, embed=embed) + # Cache invalidieren, damit das Dashboard die neuen News sofort zeigt. + if result.get("inserted", 0) > 0 or result.get("embedded", 0) > 0: + try: + from . import themen_matching + themen_matching.cache_clear() + except Exception: + logger.exception("themen_matching cache_clear failed") + return result diff --git a/app/templates/v2/base.html b/app/templates/v2/base.html index a191473..f41bba4 100644 --- a/app/templates/v2/base.html +++ b/app/templates/v2/base.html @@ -66,6 +66,7 @@ {% if is_admin %}
β€” Administration
+ {{ icon("info", 14) }} Stand {{ icon("user-check", 14) }} Freischaltungen {{ icon("list-checks", 14) }} Queue {{ icon("envelope-simple", 14) }} Abos diff --git a/app/templates/v2/screens/admin_stand.html b/app/templates/v2/screens/admin_stand.html new file mode 100644 index 0000000..8c22f94 --- /dev/null +++ b/app/templates/v2/screens/admin_stand.html @@ -0,0 +1,242 @@ +{% extends "v2/base.html" %} + +{% block title %}System-Stand β€” GWΓ–-AntragsprΓΌfer{% endblock %} + +{% set v2_active_nav = "admin_stand" %} + +{% block head_extra %} + + +{% endblock %} + +{% block main %} +
+

System-Stand

+

+ DatenΓΌberblick Β· automatische Aktualisierung alle 30 s +

+
+ +
Lade Stand …
+ + + +{% endblock %} + +{% block body_scripts %} + +{% endblock %} diff --git a/app/templates/v2/screens/aktuelle-themen.html b/app/templates/v2/screens/aktuelle-themen.html index 0a78cf8..63afa61 100644 --- a/app/templates/v2/screens/aktuelle-themen.html +++ b/app/templates/v2/screens/aktuelle-themen.html @@ -665,10 +665,50 @@ async function showDraftFromData(d) { DS ${d.drucksache} (${d.bundesland}) Β· Bezug zu: ${d.news_titel}
` + actionRow + - `
${d.body.replace(/`; + `
${renderPmBody(d.body)}
`; backdrop.style.display = 'flex'; } +// Mini-Markdown-Renderer fuer PM-Body β€” interpretiert **bold**, *italic*, +// __bold__, _italic_ + Listen + Paragraphen-Breaks. KEIN externer Markdown- +// Parser noetig (kein dependency, schnell, isoliert testbar). +function renderPmBody(body) { + if (!body) return ''; + // 1. HTML escapen + let s = body.replace(/&/g, '&').replace(//g, '>'); + // 2. **bold** + __bold__ + s = s.replace(/\*\*([^*\n]+?)\*\*/g, '$1'); + s = s.replace(/__([^_\n]+?)__/g, '$1'); + // 3. *italic* + _italic_ (vorsichtig β€” nur wenn nicht zw. Ziffern) + s = s.replace(/(?$1'); + s = s.replace(/(?$1'); + // 4. Listen: "- " oder "* " am Zeilenanfang β†’ Bullet-Punkte + // (vorsichtig: Bindestrich mitten im Wort soll bleiben) + const lines = s.split('\n'); + const out = []; + let inList = false; + for (const line of lines) { + if (/^\s*[-*]\s+/.test(line)) { + if (!inList) { out.push('
    '); inList = true; } + out.push('
  • ' + line.replace(/^\s*[-*]\s+/, '') + '
    • '); + } else { + if (inList) { out.push('
'); inList = false; } + out.push(line); + } + } + if (inList) out.push(''); + s = out.join('\n'); + // 5. Doppel-Newlines β†’

+ const paras = s.split(/\n\s*\n/); + return paras.map(p => { + const trimmed = p.trim(); + if (!trimmed) return ''; + // Wenn der Block schon mit Tag startet (z.B.

    ), kein

    -Wrap + if (trimmed.startsWith('<')) return trimmed; + return '

    ' + trimmed.replace(/\n/g, '
    ') + '

    '; + }).filter(Boolean).join('\n'); +} + async function loadVersion(draftId) { try { const r = await fetch(`/api/aktuelle-themen/drafts/${draftId}`); diff --git a/app/templates/v2/screens/antrag_detail.html b/app/templates/v2/screens/antrag_detail.html index 8dd0c63..61118d0 100644 --- a/app/templates/v2/screens/antrag_detail.html +++ b/app/templates/v2/screens/antrag_detail.html @@ -890,15 +890,35 @@ window.v2ShowMatrixFieldInfo = function(field) { const list = document.getElementById('ad-news-list'); if (!box || !list) return; try { - const r = await fetch('/api/aktuelle-themen/news-fuer-antrag?drucksache=' - + encodeURIComponent(ds) - + '&top_k=5&min_similarity=0.4&days=90'); - const data = await r.json(); - const matches = data.matches || []; - if (!matches.length) { - // Box bleibt unsichtbar, kein Stoerfaktor - return; - } + // Parallel: News-Matches + Cluster-Map (fuer Cluster-Indicator) + const [matchesResp, clusterResp] = await Promise.all([ + fetch('/api/aktuelle-themen/news-fuer-antrag?drucksache=' + + encodeURIComponent(ds) + + '&top_k=5&min_similarity=0.4&days=90'), + fetch('/api/aktuelle-themen/cluster?days=90&min_cluster_size=2'), + ]); + const matchData = await matchesResp.json(); + const matches = matchData.matches || []; + if (!matches.length) return; // Box bleibt unsichtbar + + // Cluster-Lookup-Map: URL β†’ {clusterIdx, size, otherTitles} + let clusterByUrl = {}; + try { + const clusterData = await clusterResp.json(); + (clusterData.clusters || []).forEach((c, i) => { + (c.members || []).forEach(m => { + clusterByUrl[m.url] = { + size: c.size, + tags: c.top_tags || [], + others: (c.members || []) + .filter(o => o.url !== m.url) + .map(o => `${o.source}: ${o.titel}`) + .slice(0, 5), + }; + }); + }); + } catch (_) { /* clusters optional */ } + box.style.display = ''; let html = ''; for (const n of matches) { @@ -909,10 +929,16 @@ window.v2ShowMatrixFieldInfo = function(field) { const summary = n.summary ? '

    ' + n.summary + '

    ' : ''; + const clusterInfo = clusterByUrl[n.url]; + const clusterBadge = clusterInfo + ? '' + + 'πŸ”— Cluster (' + clusterInfo.size + ' News)' + : ''; html += '
    '; html += '
    ' + d + ' Β· ' + n.source + (n.ressort ? ' / ' + n.ressort : '') - + ' Β· sim ' + n.similarity + '
    '; + + ' Β· sim ' + n.similarity + clusterBadge + '
    '; html += '' + n.titel + ''; diff --git a/app/templates/v2/screens/auswertungen.html b/app/templates/v2/screens/auswertungen.html index fefaccb..c32c06c 100644 --- a/app/templates/v2/screens/auswertungen.html +++ b/app/templates/v2/screens/auswertungen.html @@ -169,6 +169,7 @@ table.modal-table th { background: var(--ecg-bg-subtle); font-weight: 700; } +
@@ -366,7 +367,19 @@ table.modal-table th { background: var(--ecg-bg-subtle); font-weight: 700; }
- + +
+

+ Verteilung der GWΓ–-Scores (0–10) ueber alle Bewertungen β€” zeigt + auf einen Blick, wo der Antrags-Pool inhaltlich liegt. +

+
+ +
+
+
+ +

Cluster-Ansicht

@@ -397,7 +410,8 @@ table.modal-table th { background: var(--ecg-bg-subtle); font-weight: 700; } {% block body_scripts %}