feat: Stand-Dashboard, Score-Histogram, PM-Markdown, Live-Polling, Cluster-Indicator

Sechs zusammengehoerige UX/Performance-Erweiterungen:

**1. /v2/admin/stand — System-Stand-Dashboard**
KPI-Kacheln (Bewertungen, Plenum-Votes, Match, Vote-Orphans, News, PM-
Drafts, Bookmarks) + GWÖ-Score-Histogram + Per-BL-Tabelle + News-Source-
Tabelle. Auto-Refresh 30 s. Endpoint /api/admin/stand liefert alles in
einem Roundtrip. Nav-Eintrag "Stand" in der Admin-Sektion.

**2. /auswertungen Score-Histogram-Tab**
4. Tab "Score-Verteilung" mit Bar-Chart 0–10. Endpoint
/api/auswertungen/score-histogram liefert Buckets, optional gefiltert
nach Bundesland + Wahlperiode. Reagiert auf den globalen BL-Filter.

**3. PM-Body Markdown-Rendering**
Mini-Renderer im Modal: **bold** / __bold__ / *italic* / _italic_ /
- list-bullets / Doppel-Newline-Paragraphen. Kein externer Markdown-
Parser, keine neue Dependency. Body wird HTML-escaped, Patterns dann
zu Tags umgesetzt.

**4. Performance-Cache fuer themen_matching**
TTL-Cache (60 s) fuer aggregate_top_themen und aggregate_news_cluster.
Cache-Key inkl. aller Filter-Parameter. Automatische Invalidation in
news_aggregator.run_aggregator nach erfolgreichem Insert/Embed.
4 neue Tests fuer cache_get/set/clear-Verhalten.

**5. Stimmverhalten Banner Live-Update**
Statt setTimeout(800) jetzt pollQueueUntilDrained: alle 4 s
GET /api/queue/status, Banner zeigt pending + elapsed live. Bei
pending=0 zwei Polls in Folge: Banner + Stimmverhalten-Charts neu
laden. Max 5 Min Polling-Timeout. Bricht ab wenn Tab gewechselt wird.

**6. Antrag-Detail Cluster-Indicator**
News-Match-Box im Antrag-Detail laedt parallel /aktuelle-themen/cluster
und mappt URL → Cluster. Pro News-Card ein "🔗 Cluster (N News)"-Badge
mit Hover-Tooltip der anderen Cluster-Members. Macht thematische
Bündel sichtbar, ohne Pop-Out auf den Cluster-Tab.

Suite: 1088 → 1092 grün (4 Cache-Tests).

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Dotty Dotter 2026-05-06 02:49:06 +02:00
parent 0377cf4bd9
commit d30fcb132a
9 changed files with 696 additions and 17 deletions

View File

@ -2228,6 +2228,46 @@ async def api_draft_versions(drucksache: str, news_url: str):
} }
@app.get("/api/auswertungen/score-histogram")
async def auswertungen_score_histogram(
bundesland: Optional[str] = None,
wahlperiode: Optional[str] = None,
):
"""GWÖ-Score-Verteilung (010) ueber alle Bewertungen.
Liefert ein Bucket-Array fuer einen Histogramm-Chart. Filterbar
ueber Bundesland + Wahlperiode (gleicher Pattern wie /matrix).
"""
import sqlite3
from .auswertungen import _load_assessments
rows = _load_assessments()
from .wahlperioden import wahlperiode_for
buckets = [0] * 11
total = 0
for r in rows:
if bundesland and r["bundesland"] != bundesland:
continue
if wahlperiode is not None:
wp = wahlperiode_for(r["datum"], r["bundesland"])
if wp != wahlperiode:
continue
score = r["gwoe_score"]
if score is None:
continue
bucket = min(10, max(0, int(score)))
buckets[bucket] += 1
total += 1
return {
"buckets": [
{"score_min": i, "score_max": i + 1, "count": c}
for i, c in enumerate(buckets)
],
"total": total,
"filter": {"bundesland": bundesland, "wahlperiode": wahlperiode},
}
@app.get("/api/auswertungen/matrix") @app.get("/api/auswertungen/matrix")
async def auswertungen_matrix( async def auswertungen_matrix(
wahlperiode: Optional[str] = None, wahlperiode: Optional[str] = None,
@ -2902,6 +2942,126 @@ async def v2_admin_queue(request: Request, user: dict = Depends(require_admin)):
}) })
@app.get("/v2/admin/stand", response_class=HTMLResponse)
async def v2_admin_stand(request: Request, user: dict = Depends(require_admin)):
"""System-Stand-Dashboard — Ueberblick ueber alle Datenmengen."""
return templates.TemplateResponse("v2/screens/admin_stand.html", {
"request": request,
"v2_active_nav": "admin_stand",
**_v2_template_context(user),
})
@app.get("/api/admin/stand")
async def api_admin_stand(user: dict = Depends(require_admin)):
"""Datenstand-Aggregation für das Stand-Dashboard.
Liefert Gesamt + Per-Quelle + Letzte-7-Tage in einem Roundtrip.
"""
import sqlite3
from datetime import datetime, timedelta, timezone
db = sqlite3.connect(str(settings.db_path))
cutoff_7d = (datetime.now(timezone.utc) - timedelta(days=7)).isoformat()
try:
# Assessments
n_ass_total = db.execute(
"SELECT COUNT(*) FROM assessments WHERE gwoe_score IS NOT NULL"
).fetchone()[0]
n_ass_7d = db.execute(
"SELECT COUNT(*) FROM assessments "
"WHERE gwoe_score IS NOT NULL AND created_at >= ?",
(cutoff_7d,),
).fetchone()[0]
ass_per_bl = dict(db.execute(
"SELECT bundesland, COUNT(*) FROM assessments "
"WHERE gwoe_score IS NOT NULL GROUP BY bundesland ORDER BY 2 DESC"
).fetchall())
# Score-Verteilung
score_dist = dict(db.execute(
"SELECT CAST(gwoe_score AS INTEGER), COUNT(*) FROM assessments "
"WHERE gwoe_score IS NOT NULL GROUP BY CAST(gwoe_score AS INTEGER)"
).fetchall())
# Plenum-Votes
n_votes = db.execute("SELECT COUNT(*) FROM plenum_vote_results").fetchone()[0]
votes_per_bl = dict(db.execute(
"SELECT bundesland, COUNT(*) FROM plenum_vote_results "
"GROUP BY bundesland ORDER BY 2 DESC"
).fetchall())
# Match (assessment ∩ vote)
n_match = db.execute("""
SELECT COUNT(DISTINCT a.drucksache) FROM assessments a
INNER JOIN plenum_vote_results p
ON a.bundesland=p.bundesland AND a.drucksache=p.drucksache
WHERE a.gwoe_score IS NOT NULL
""").fetchone()[0]
n_orphans = db.execute("""
SELECT COUNT(DISTINCT p.bundesland || '/' || p.drucksache)
FROM plenum_vote_results p
LEFT JOIN assessments a
ON a.bundesland=p.bundesland AND a.drucksache=p.drucksache
WHERE a.drucksache IS NULL
""").fetchone()[0]
# News
n_news = db.execute("SELECT COUNT(*) FROM news_articles").fetchone()[0]
n_news_emb = db.execute(
"SELECT COUNT(*) FROM news_articles WHERE summary_embedding IS NOT NULL"
).fetchone()[0]
n_news_7d = db.execute(
"SELECT COUNT(*) FROM news_articles WHERE datum >= ?", (cutoff_7d,),
).fetchone()[0]
news_per_source = dict(db.execute(
"SELECT source, COUNT(*) FROM news_articles GROUP BY source"
).fetchall())
# PM-Drafts
n_drafts = db.execute("SELECT COUNT(*) FROM presse_drafts").fetchone()[0]
n_drafts_7d = db.execute(
"SELECT COUNT(*) FROM presse_drafts WHERE created_at >= ?",
(cutoff_7d,),
).fetchone()[0]
# Bookmarks
try:
n_bookmarks = db.execute("SELECT COUNT(*) FROM bookmarks").fetchone()[0]
except sqlite3.OperationalError:
n_bookmarks = 0
finally:
db.close()
return {
"assessments": {
"total": n_ass_total,
"last_7_days": n_ass_7d,
"by_bundesland": ass_per_bl,
"score_distribution": score_dist,
},
"plenum_votes": {
"total": n_votes,
"by_bundesland": votes_per_bl,
},
"match": {
"with_assessment_and_vote": n_match,
"vote_orphans": n_orphans,
},
"news": {
"total": n_news,
"embedded": n_news_emb,
"last_7_days": n_news_7d,
"by_source": news_per_source,
},
"presse_drafts": {
"total": n_drafts,
"last_7_days": n_drafts_7d,
},
"bookmarks": n_bookmarks,
}
@app.get("/v2/admin/abos", response_class=HTMLResponse) @app.get("/v2/admin/abos", response_class=HTMLResponse)
async def v2_admin_abos(request: Request, user: dict = Depends(require_admin)): async def v2_admin_abos(request: Request, user: dict = Depends(require_admin)):
"""Abo-Verwaltung — alle E-Mail-Abonnements (Admin).""" """Abo-Verwaltung — alle E-Mail-Abonnements (Admin)."""

View File

@ -344,4 +344,12 @@ def run_aggregator(db_path: Optional[Path] = None, embed: bool = True) -> dict:
geworfen. geworfen.
""" """
articles = fetch_all() articles = fetch_all()
return upsert_articles(articles, db_path=db_path, embed=embed) result = upsert_articles(articles, db_path=db_path, embed=embed)
# Cache invalidieren, damit das Dashboard die neuen News sofort zeigt.
if result.get("inserted", 0) > 0 or result.get("embedded", 0) > 0:
try:
from . import themen_matching
themen_matching.cache_clear()
except Exception:
logger.exception("themen_matching cache_clear failed")
return result

View File

@ -66,6 +66,7 @@
{% if is_admin %} {% if is_admin %}
<div class="v2-nav-group"> <div class="v2-nav-group">
<div class="v2-nav-label">— Administration</div> <div class="v2-nav-label">— Administration</div>
<a href="/v2/admin/stand" class="v2-nav-item {% if v2_active_nav == 'admin_stand' %}active{% endif %}">{{ icon("info", 14) }} Stand</a>
<a href="/v2/admin/freischaltungen" class="v2-nav-item">{{ icon("user-check", 14) }} Freischaltungen</a> <a href="/v2/admin/freischaltungen" class="v2-nav-item">{{ icon("user-check", 14) }} Freischaltungen</a>
<a href="/v2/admin/queue" class="v2-nav-item">{{ icon("list-checks", 14) }} Queue</a> <a href="/v2/admin/queue" class="v2-nav-item">{{ icon("list-checks", 14) }} Queue</a>
<a href="/v2/admin/abos" class="v2-nav-item">{{ icon("envelope-simple", 14) }} Abos</a> <a href="/v2/admin/abos" class="v2-nav-item">{{ icon("envelope-simple", 14) }} Abos</a>

View File

@ -0,0 +1,242 @@
{% extends "v2/base.html" %}
{% block title %}System-Stand — GWÖ-Antragsprüfer{% endblock %}
{% set v2_active_nav = "admin_stand" %}
{% block head_extra %}
<script src="/static/chart.umd.min.js"></script>
<style>
.stand-grid {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(220px, 1fr));
gap: 14px;
margin-bottom: 1.5rem;
}
.stand-kpi {
background: var(--ecg-card-bg);
border: 1px solid var(--ecg-border);
border-radius: 6px;
padding: 16px;
text-align: center;
}
.stand-kpi-value {
font-family: var(--font-mono);
font-size: 30px;
font-weight: 700;
color: var(--ecg-teal);
line-height: 1.1;
}
.stand-kpi-label {
font-family: var(--font-mono);
font-size: 11px;
text-transform: uppercase;
letter-spacing: 0.06em;
opacity: 0.6;
margin-top: 4px;
}
.stand-kpi-sub {
font-family: var(--font-mono);
font-size: 11px;
opacity: 0.55;
margin-top: 6px;
}
.stand-section {
margin-top: 2rem;
}
.stand-section h2 {
font-family: var(--font-display);
font-size: 16px;
color: var(--ecg-teal);
margin: 0 0 10px;
}
.stand-table {
width: 100%;
border-collapse: collapse;
font-family: var(--font-mono);
font-size: 12px;
}
.stand-table td, .stand-table th {
border-bottom: 1px solid var(--ecg-border);
padding: 4px 8px;
text-align: left;
}
.stand-table th {
font-size: 10px;
text-transform: uppercase;
letter-spacing: 0.05em;
opacity: 0.6;
font-weight: 700;
}
.stand-table td:nth-child(2) {
text-align: right;
}
</style>
{% endblock %}
{% block main %}
<div style="padding:0 0 1.5rem;">
<h1 style="font-family:var(--font-display);font-size:22px;color:var(--ecg-teal);margin:0 0 4px;">System-Stand</h1>
<p style="font-size:12px;font-family:var(--font-mono);color:var(--ecg-dark);opacity:0.6;">
Datenüberblick · automatische Aktualisierung alle 30 s
</p>
</div>
<div id="stand-loading" style="font-family:var(--font-mono);font-size:12px;opacity:0.5;padding:16px 0;">Lade Stand …</div>
<div id="stand-content" style="display:none;">
<!-- KPI-Kacheln -->
<div class="stand-grid">
<div class="stand-kpi">
<div class="stand-kpi-value" id="kpi-ass"></div>
<div class="stand-kpi-label">Bewertungen</div>
<div class="stand-kpi-sub" id="kpi-ass-7d">— in 7 Tagen</div>
</div>
<div class="stand-kpi">
<div class="stand-kpi-value" id="kpi-votes"></div>
<div class="stand-kpi-label">Plenum-Votes</div>
<div class="stand-kpi-sub">aus Plenarprotokollen</div>
</div>
<div class="stand-kpi">
<div class="stand-kpi-value" id="kpi-match"></div>
<div class="stand-kpi-label">Bewertung ∩ Vote</div>
<div class="stand-kpi-sub" id="kpi-orphans">— Vote-Orphans</div>
</div>
<div class="stand-kpi">
<div class="stand-kpi-value" id="kpi-news"></div>
<div class="stand-kpi-label">News</div>
<div class="stand-kpi-sub" id="kpi-news-7d">— in 7 Tagen</div>
</div>
<div class="stand-kpi">
<div class="stand-kpi-value" id="kpi-drafts"></div>
<div class="stand-kpi-label">PM-Entwürfe</div>
<div class="stand-kpi-sub" id="kpi-drafts-7d">— in 7 Tagen</div>
</div>
<div class="stand-kpi">
<div class="stand-kpi-value" id="kpi-bookmarks"></div>
<div class="stand-kpi-label">Merklisten-Einträge</div>
<div class="stand-kpi-sub">user-übergreifend</div>
</div>
</div>
<!-- Score-Histogram -->
<div class="stand-section">
<h2>GWÖ-Score-Verteilung</h2>
<div style="background:var(--ecg-card-bg);border:1px solid var(--ecg-border);border-radius:6px;padding:14px;">
<canvas id="stand-score-chart" style="max-height:240px;"></canvas>
</div>
</div>
<!-- Per-Bundesland -->
<div class="stand-section">
<h2>Bewertungen + Votes pro Bundesland</h2>
<table class="stand-table">
<thead><tr><th>BL</th><th>Bewertungen</th><th>Plenum-Votes</th></tr></thead>
<tbody id="stand-bl-rows"></tbody>
</table>
</div>
<!-- News-Quellen -->
<div class="stand-section">
<h2>News pro Quelle</h2>
<table class="stand-table">
<thead><tr><th>Quelle</th><th>Anzahl</th></tr></thead>
<tbody id="stand-news-rows"></tbody>
</table>
</div>
<div id="stand-meta" style="font-family:var(--font-mono);font-size:11px;opacity:0.5;margin-top:1.5rem;"></div>
</div>
{% endblock %}
{% block body_scripts %}
<script>
let _scoreChart = null;
function fmtN(n) {
return (n == null) ? '—' : n.toLocaleString('de-DE');
}
async function loadStand() {
try {
const r = await fetch('/api/admin/stand');
if (!r.ok) {
document.getElementById('stand-loading').textContent = 'Fehler ' + r.status;
return;
}
const d = await r.json();
document.getElementById('stand-loading').style.display = 'none';
document.getElementById('stand-content').style.display = '';
// KPIs
document.getElementById('kpi-ass').textContent = fmtN(d.assessments.total);
document.getElementById('kpi-ass-7d').textContent = '+' + fmtN(d.assessments.last_7_days) + ' in 7 Tagen';
document.getElementById('kpi-votes').textContent = fmtN(d.plenum_votes.total);
document.getElementById('kpi-match').textContent = fmtN(d.match.with_assessment_and_vote);
document.getElementById('kpi-orphans').textContent = fmtN(d.match.vote_orphans) + ' Vote-Orphans';
document.getElementById('kpi-news').textContent = fmtN(d.news.total);
document.getElementById('kpi-news-7d').textContent =
'+' + fmtN(d.news.last_7_days) + ' in 7 Tagen, ' + fmtN(d.news.embedded) + ' embedded';
document.getElementById('kpi-drafts').textContent = fmtN(d.presse_drafts.total);
document.getElementById('kpi-drafts-7d').textContent = '+' + fmtN(d.presse_drafts.last_7_days) + ' in 7 Tagen';
document.getElementById('kpi-bookmarks').textContent = fmtN(d.bookmarks);
// Score-Histogram
const dist = d.assessments.score_distribution || {};
const buckets = [0,1,2,3,4,5,6,7,8,9,10];
const values = buckets.map(b => dist[String(b)] || 0);
const colors = buckets.map(b => {
if (b <= 2) return 'rgba(200,30,30,0.6)';
if (b <= 4) return 'rgba(247,148,29,0.6)';
if (b <= 6) return 'rgba(150,150,150,0.5)';
if (b <= 8) return 'rgba(136,158,51,0.6)';
return 'rgba(0,157,165,0.7)';
});
if (_scoreChart) _scoreChart.destroy();
_scoreChart = new Chart(document.getElementById('stand-score-chart'), {
type: 'bar',
data: { labels: buckets.map(b => b + '' + (b+1)), datasets: [{
label: 'Bewertungen',
data: values,
backgroundColor: colors,
}]},
options: {
responsive: true,
plugins: { legend: { display: false } },
scales: {
y: { beginAtZero: true, title: { display: true, text: 'Anzahl' } },
x: { title: { display: true, text: 'GWÖ-Score-Bucket' } },
},
},
});
// Per-BL-Tabelle
const ass_bl = d.assessments.by_bundesland || {};
const vote_bl = d.plenum_votes.by_bundesland || {};
const bl_set = new Set([...Object.keys(ass_bl), ...Object.keys(vote_bl)]);
const bl_rows = [...bl_set].sort();
document.getElementById('stand-bl-rows').innerHTML = bl_rows.map(bl => `
<tr>
<td>${bl}</td>
<td>${fmtN(ass_bl[bl] || 0)}</td>
<td>${fmtN(vote_bl[bl] || 0)}</td>
</tr>`).join('');
// News-Source-Tabelle
const ns = d.news.by_source || {};
document.getElementById('stand-news-rows').innerHTML =
Object.entries(ns).sort((a, b) => b[1] - a[1]).map(([s, n]) => `
<tr><td>${s}</td><td>${fmtN(n)}</td></tr>`).join('');
document.getElementById('stand-meta').textContent =
'Aktualisiert: ' + new Date().toLocaleTimeString('de-DE');
} catch (e) {
document.getElementById('stand-loading').textContent = 'Fehler: ' + e;
}
}
loadStand();
setInterval(loadStand, 30000);
</script>
{% endblock %}

View File

@ -665,10 +665,50 @@ async function showDraftFromData(d) {
DS ${d.drucksache} (${d.bundesland}) · Bezug zu: <a href="${d.news_url}" target="_blank" rel="noopener" style="color:var(--ecg-teal);">${d.news_titel}</a> DS ${d.drucksache} (${d.bundesland}) · Bezug zu: <a href="${d.news_url}" target="_blank" rel="noopener" style="color:var(--ecg-teal);">${d.news_titel}</a>
</div>` + </div>` +
actionRow + actionRow +
`<div style="white-space:pre-wrap;font-size:13px;line-height:1.5;">${d.body.replace(/</g, '&lt;')}</div>`; `<div style="font-size:13px;line-height:1.6;">${renderPmBody(d.body)}</div>`;
backdrop.style.display = 'flex'; backdrop.style.display = 'flex';
} }
// Mini-Markdown-Renderer fuer PM-Body — interpretiert **bold**, *italic*,
// __bold__, _italic_ + Listen + Paragraphen-Breaks. KEIN externer Markdown-
// Parser noetig (kein dependency, schnell, isoliert testbar).
function renderPmBody(body) {
if (!body) return '';
// 1. HTML escapen
let s = body.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;');
// 2. **bold** + __bold__
s = s.replace(/\*\*([^*\n]+?)\*\*/g, '<strong>$1</strong>');
s = s.replace(/__([^_\n]+?)__/g, '<strong>$1</strong>');
// 3. *italic* + _italic_ (vorsichtig — nur wenn nicht zw. Ziffern)
s = s.replace(/(?<![*\w])\*([^*\n]+?)\*(?![*\w])/g, '<em>$1</em>');
s = s.replace(/(?<![_\w])_([^_\n]+?)_(?![_\w])/g, '<em>$1</em>');
// 4. Listen: "- " oder "* " am Zeilenanfang → Bullet-Punkte
// (vorsichtig: Bindestrich mitten im Wort soll bleiben)
const lines = s.split('\n');
const out = [];
let inList = false;
for (const line of lines) {
if (/^\s*[-*]\s+/.test(line)) {
if (!inList) { out.push('<ul style="margin:0.5em 0;padding-left:1.4em;">'); inList = true; }
out.push('<li>' + line.replace(/^\s*[-*]\s+/, '') + '</li>');
} else {
if (inList) { out.push('</ul>'); inList = false; }
out.push(line);
}
}
if (inList) out.push('</ul>');
s = out.join('\n');
// 5. Doppel-Newlines → </p><p>
const paras = s.split(/\n\s*\n/);
return paras.map(p => {
const trimmed = p.trim();
if (!trimmed) return '';
// Wenn der Block schon mit Tag startet (z.B. <ul>), kein <p>-Wrap
if (trimmed.startsWith('<')) return trimmed;
return '<p style="margin:0 0 0.9em;">' + trimmed.replace(/\n/g, '<br>') + '</p>';
}).filter(Boolean).join('\n');
}
async function loadVersion(draftId) { async function loadVersion(draftId) {
try { try {
const r = await fetch(`/api/aktuelle-themen/drafts/${draftId}`); const r = await fetch(`/api/aktuelle-themen/drafts/${draftId}`);

View File

@ -890,15 +890,35 @@ window.v2ShowMatrixFieldInfo = function(field) {
const list = document.getElementById('ad-news-list'); const list = document.getElementById('ad-news-list');
if (!box || !list) return; if (!box || !list) return;
try { try {
const r = await fetch('/api/aktuelle-themen/news-fuer-antrag?drucksache=' // Parallel: News-Matches + Cluster-Map (fuer Cluster-Indicator)
const [matchesResp, clusterResp] = await Promise.all([
fetch('/api/aktuelle-themen/news-fuer-antrag?drucksache='
+ encodeURIComponent(ds) + encodeURIComponent(ds)
+ '&top_k=5&min_similarity=0.4&days=90'); + '&top_k=5&min_similarity=0.4&days=90'),
const data = await r.json(); fetch('/api/aktuelle-themen/cluster?days=90&min_cluster_size=2'),
const matches = data.matches || []; ]);
if (!matches.length) { const matchData = await matchesResp.json();
// Box bleibt unsichtbar, kein Stoerfaktor const matches = matchData.matches || [];
return; if (!matches.length) return; // Box bleibt unsichtbar
}
// Cluster-Lookup-Map: URL → {clusterIdx, size, otherTitles}
let clusterByUrl = {};
try {
const clusterData = await clusterResp.json();
(clusterData.clusters || []).forEach((c, i) => {
(c.members || []).forEach(m => {
clusterByUrl[m.url] = {
size: c.size,
tags: c.top_tags || [],
others: (c.members || [])
.filter(o => o.url !== m.url)
.map(o => `${o.source}: ${o.titel}`)
.slice(0, 5),
};
});
});
} catch (_) { /* clusters optional */ }
box.style.display = ''; box.style.display = '';
let html = ''; let html = '';
for (const n of matches) { for (const n of matches) {
@ -909,10 +929,16 @@ window.v2ShowMatrixFieldInfo = function(field) {
const summary = n.summary const summary = n.summary
? '<p style="font-size:12px;margin:4px 0 8px;opacity:0.85;line-height:1.5;">' + n.summary + '</p>' ? '<p style="font-size:12px;margin:4px 0 8px;opacity:0.85;line-height:1.5;">' + n.summary + '</p>'
: ''; : '';
const clusterInfo = clusterByUrl[n.url];
const clusterBadge = clusterInfo
? '<span style="display:inline-block;padding:1px 7px;background:rgba(0,157,165,0.15);border-radius:11px;font-family:var(--font-mono);font-size:10px;margin-left:6px;color:var(--ecg-teal);" '
+ 'title="' + clusterInfo.others.map(s => s.replace(/"/g, '&quot;')).join(' • ') + '">'
+ '🔗 Cluster (' + clusterInfo.size + ' News)</span>'
: '';
html += '<div style="border-bottom:1px dotted var(--ecg-border);padding:10px 0;">'; html += '<div style="border-bottom:1px dotted var(--ecg-border);padding:10px 0;">';
html += '<div style="font-family:var(--font-mono);font-size:10px;opacity:0.6;margin-bottom:3px;">' html += '<div style="font-family:var(--font-mono);font-size:10px;opacity:0.6;margin-bottom:3px;">'
+ d + ' · ' + n.source + (n.ressort ? ' / ' + n.ressort : '') + d + ' · ' + n.source + (n.ressort ? ' / ' + n.ressort : '')
+ ' · sim ' + n.similarity + '</div>'; + ' · sim ' + n.similarity + clusterBadge + '</div>';
html += '<a href="' + n.url + '" target="_blank" rel="noopener" ' html += '<a href="' + n.url + '" target="_blank" rel="noopener" '
+ 'style="color:var(--ecg-teal);text-decoration:none;font-weight:500;">' + 'style="color:var(--ecg-teal);text-decoration:none;font-weight:500;">'
+ n.titel + '</a>'; + n.titel + '</a>';

View File

@ -169,6 +169,7 @@ table.modal-table th { background: var(--ecg-bg-subtle); font-weight: 700; }
<button class="auswert-tab active" role="tab" onclick="switchTab('bl-partei', this)">BL × Partei</button> <button class="auswert-tab active" role="tab" onclick="switchTab('bl-partei', this)">BL × Partei</button>
<button class="auswert-tab" role="tab" onclick="switchTab('themen', this)">Thema × Fraktion</button> <button class="auswert-tab" role="tab" onclick="switchTab('themen', this)">Thema × Fraktion</button>
<button class="auswert-tab" role="tab" onclick="switchTab('stimmverhalten', this)">Stimmverhalten</button> <button class="auswert-tab" role="tab" onclick="switchTab('stimmverhalten', this)">Stimmverhalten</button>
<button class="auswert-tab" role="tab" onclick="switchTab('histogram', this)">Score-Verteilung</button>
<button class="auswert-tab" role="tab" onclick="switchTab('cluster', this)">Cluster</button> <button class="auswert-tab" role="tab" onclick="switchTab('cluster', this)">Cluster</button>
</div> </div>
@ -366,7 +367,19 @@ table.modal-table th { background: var(--ecg-bg-subtle); font-weight: 700; }
<div id="sv-cross-bl-meta" class="meta-line"></div> <div id="sv-cross-bl-meta" class="meta-line"></div>
</div> </div>
<!-- Panel 4: Cluster-Link --> <!-- Panel 4: Score-Histogram -->
<div class="auswert-panel" id="panel-histogram">
<p style="font-size:11px;font-family:var(--font-mono);opacity:0.7;margin:0 0 0.5rem;">
Verteilung der GWÖ-Scores (010) ueber alle Bewertungen — zeigt
auf einen Blick, wo der Antrags-Pool inhaltlich liegt.
</p>
<div class="matrix-wrap" style="background:var(--ecg-card-bg);border:1px solid var(--ecg-border);border-radius:4px;padding:14px;">
<canvas id="hist-chart" style="max-height:340px;"></canvas>
</div>
<div id="hist-meta" style="font-family:var(--font-mono);font-size:11px;opacity:0.6;margin:8px 0 1.5rem;"></div>
</div>
<!-- Panel 5: Cluster-Link -->
<div class="auswert-panel" id="panel-cluster"> <div class="auswert-panel" id="panel-cluster">
<div class="v2-kasten outline-blue"> <div class="v2-kasten outline-blue">
<h4>Cluster-Ansicht</h4> <h4>Cluster-Ansicht</h4>
@ -397,7 +410,8 @@ table.modal-table th { background: var(--ecg-bg-subtle); font-weight: 700; }
{% block body_scripts %} {% block body_scripts %}
<script> <script>
let _tabLoaded = { 'bl-partei': false, 'themen': false, 'stimmverhalten': false }; let _tabLoaded = { 'bl-partei': false, 'themen': false, 'stimmverhalten': false, 'histogram': false };
let _histChart = null;
let _svCharts = { index: null, heuchelei: null, empfehlung: null, crossBl: null, zeitreihe: null }; let _svCharts = { index: null, heuchelei: null, empfehlung: null, crossBl: null, zeitreihe: null };
let _svMatrixAxis = 'werte'; // 'werte' or 'gruppen' let _svMatrixAxis = 'werte'; // 'werte' or 'gruppen'
@ -443,6 +457,54 @@ function switchTab(id, btn) {
loadStimmverhalten(); loadStimmverhalten();
_tabLoaded.stimmverhalten = true; _tabLoaded.stimmverhalten = true;
} }
if (id === 'histogram' && !_tabLoaded.histogram) {
loadHistogram();
_tabLoaded.histogram = true;
}
}
async function loadHistogram() {
const blRaw = (window.v2GetGlobalBl && window.v2GetGlobalBl()) || 'ALL';
const bl = (blRaw === 'ALL') ? '' : blRaw;
const wp = document.getElementById('wp-filter') ? document.getElementById('wp-filter').value : '';
const meta = document.getElementById('hist-meta');
let url = '/api/auswertungen/score-histogram';
const params = [];
if (bl) params.push('bundesland=' + encodeURIComponent(bl));
if (wp) params.push('wahlperiode=' + encodeURIComponent(wp));
if (params.length) url += '?' + params.join('&');
try {
const r = await fetch(url);
const data = await r.json();
if (_histChart) _histChart.destroy();
const labels = data.buckets.map(b => `${b.score_min}${b.score_max}`);
const values = data.buckets.map(b => b.count);
const colors = data.buckets.map(b => {
const v = b.score_min;
if (v <= 2) return 'rgba(200,30,30,0.6)';
if (v <= 4) return 'rgba(247,148,29,0.6)';
if (v <= 6) return 'rgba(150,150,150,0.5)';
if (v <= 8) return 'rgba(136,158,51,0.6)';
return 'rgba(0,157,165,0.7)';
});
_histChart = new Chart(document.getElementById('hist-chart'), {
type: 'bar',
data: { labels: labels, datasets: [{ label: 'Bewertungen', data: values, backgroundColor: colors }] },
options: {
responsive: true,
plugins: { legend: { display: false } },
scales: {
y: { beginAtZero: true, title: { display: true, text: 'Anzahl Bewertungen' } },
x: { title: { display: true, text: 'GWÖ-Score-Bucket' } },
},
},
});
meta.textContent = `${data.total} Bewertungen` +
(data.filter.bundesland ? ` · BL: ${data.filter.bundesland}` : '') +
(data.filter.wahlperiode ? ` · WP: ${data.filter.wahlperiode}` : '');
} catch (e) {
meta.textContent = 'Fehler: ' + e;
}
} }
// Bei globalem BL-Wechsel aktive Panels neu laden — ABER NICHT // Bei globalem BL-Wechsel aktive Panels neu laden — ABER NICHT
@ -452,6 +514,7 @@ window.addEventListener('v2-bl-changed', function () {
if (!activePanel) return; if (!activePanel) return;
if (activePanel.id === 'panel-bl-partei') loadBlMatrix(); if (activePanel.id === 'panel-bl-partei') loadBlMatrix();
if (activePanel.id === 'panel-themen') loadThemenMatrix(); if (activePanel.id === 'panel-themen') loadThemenMatrix();
if (activePanel.id === 'panel-histogram') loadHistogram();
// Stimmverhalten reagiert NICHT auf globalen BL-Filter — eigener Selector. // Stimmverhalten reagiert NICHT auf globalen BL-Filter — eigener Selector.
}); });
@ -688,13 +751,58 @@ async function bulkRateOrphans() {
} }
const data = await r.json(); const data = await r.json();
const skip = (data.skipped || []).length; const skip = (data.skipped || []).length;
const expectedJobs = (data.jobs || []).length;
result.innerHTML = `${data.enqueued} enqueued${skip ? `, ${skip} skipped` : ''} — <a href="/v2/admin/queue" style="color:var(--ecg-teal);">Queue ansehen →</a>`; result.innerHTML = `${data.enqueued} enqueued${skip ? `, ${skip} skipped` : ''} — <a href="/v2/admin/queue" style="color:var(--ecg-teal);">Queue ansehen →</a>`;
setTimeout(() => loadVoteOrphansBanner(bl), 800); // Live-Polling: Queue-Status prüfen alle 4s, bis pending=0 oder
// 5 Min vorbei sind. Banner danach neu laden.
pollQueueUntilDrained(bl, expectedJobs, result);
} catch (e) { } catch (e) {
result.textContent = 'Fehler: ' + e; result.textContent = 'Fehler: ' + e;
} }
} }
async function pollQueueUntilDrained(bl, expectedJobs, resultEl) {
const startTs = Date.now();
const maxMs = 5 * 60 * 1000;
let lastPending = -1;
while (Date.now() - startTs < maxMs) {
await new Promise(r => setTimeout(r, 4000));
try {
const q = await fetch('/api/queue/status');
const data = await q.json();
const pending = data.pending != null ? data.pending : 0;
const completed = data.processed_total || 0;
// Wenn das Banner das Tab gewechselt hat, abbrechen
if (!document.getElementById('panel-stimmverhalten').classList.contains('active')) {
// Stimmverhalten-Tab geschlossen, hör auf zu pollen
return;
}
if (resultEl) {
const elapsed = Math.round((Date.now() - startTs) / 1000);
resultEl.innerHTML =
`${expectedJobs} enqueued · pending: ${pending} · ` +
`${elapsed}s gewartet — <a href="/v2/admin/queue" style="color:var(--ecg-teal);">Queue ansehen →</a>`;
}
if (pending === 0 && lastPending === 0) {
// Zwei Polls in Folge mit pending=0 → fertig
loadVoteOrphansBanner(bl);
// Auch die anderen Stimmverhalten-Charts neu laden
loadStimmverhalten();
if (resultEl) resultEl.textContent =
`Fertig — Bewertungen ${expectedJobs} verarbeitet, Banner aktualisiert`;
return;
}
lastPending = pending;
} catch (e) {
// Ignore, retry
}
}
// Timeout
if (resultEl) resultEl.textContent =
`Polling-Timeout (5 Min) — Banner wird normal aktualisiert`;
loadVoteOrphansBanner(bl);
}
function downloadStimmverhaltenCsv() { function downloadStimmverhaltenCsv() {
const bl = svGetBl(); const bl = svGetBl();
const exclude = document.getElementById('sv-exclude-antragsteller').checked ? '1' : '0'; const exclude = document.getElementById('sv-exclude-antragsteller').checked ? '1' : '0';

View File

@ -10,12 +10,19 @@ Reuse:
- Beide Tabellen nutzen denselben Embedding-Modell-Vektorraum (qwen v4), - Beide Tabellen nutzen denselben Embedding-Modell-Vektorraum (qwen v4),
daher direkter Cross-Vergleich moeglich daher direkter Cross-Vergleich moeglich
- Filter ueber ``embedding_model``-Spalte, falls Migration laueft - Filter ueber ``embedding_model``-Spalte, falls Migration laueft
**Performance-Cache:** ``aggregate_top_themen`` und ``aggregate_news_cluster``
sind teuer (cosine über ~300 News × ~100 Bewertungen = 30k Ops). Daher
TTL-Cache: gleiche Filter-Tuples werden 60 s lang aus Memory geliefert,
danach neu berechnet. Cache wird beim Modul-Import geleert (keine
persistente Stale-Gefahr nach Deploy).
""" """
from __future__ import annotations from __future__ import annotations
import json import json
import logging import logging
import sqlite3 import sqlite3
import time
from collections import defaultdict from collections import defaultdict
from datetime import datetime, timezone from datetime import datetime, timezone
from pathlib import Path from pathlib import Path
@ -23,6 +30,30 @@ from typing import Optional
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
_CACHE: dict[tuple, tuple[float, dict]] = {}
_CACHE_TTL_SECONDS = 60
def _cache_get(key: tuple) -> Optional[dict]:
entry = _CACHE.get(key)
if entry is None:
return None
expires_at, value = entry
if time.time() > expires_at:
_CACHE.pop(key, None)
return None
return value
def _cache_set(key: tuple, value: dict) -> None:
_CACHE[key] = (time.time() + _CACHE_TTL_SECONDS, value)
def cache_clear() -> None:
"""Leert den TTL-Cache. Aufruf z.B. nach News-Aggregator-Lauf,
damit neue News sofort sichtbar werden."""
_CACHE.clear()
def _load_embeddings( def _load_embeddings(
db_path: Path, db_path: Path,
@ -286,6 +317,15 @@ def aggregate_top_themen(
"filter": {...} "filter": {...}
}`` }``
""" """
# Cache-Key (db_path nur wenn Test-Override; sonst per Default)
cache_key = (
"top_themen", days_window, top_k, round(min_similarity, 3),
matches_per_news, only_relevant, single_date, str(db_path or ""),
)
cached = _cache_get(cache_key)
if cached is not None:
return cached
from .config import settings from .config import settings
from . import embeddings as emb from . import embeddings as emb
@ -392,7 +432,7 @@ def aggregate_top_themen(
reverse=True, reverse=True,
) )
return { result = {
"buckets": buckets, "buckets": buckets,
"n_total_news": len(news_rows), "n_total_news": len(news_rows),
"n_in_window": n_in_window, "n_in_window": n_in_window,
@ -406,6 +446,8 @@ def aggregate_top_themen(
"single_date": single_date, "single_date": single_date,
}, },
} }
_cache_set(cache_key, result)
return result
def aggregate_themen_zeitreihe( def aggregate_themen_zeitreihe(
@ -472,6 +514,8 @@ def aggregate_news_cluster(
) -> dict: ) -> dict:
"""News-zu-News-Clustering ueber Embeddings. """News-zu-News-Clustering ueber Embeddings.
Cached (60s TTL).
Greedy: jede ungeclusterte News wird Cluster-Seed, alle anderen mit Greedy: jede ungeclusterte News wird Cluster-Seed, alle anderen mit
cosine >= ``intra_threshold`` werden eingeschlossen. Cluster mit cosine >= ``intra_threshold`` werden eingeschlossen. Cluster mit
weniger als ``min_cluster_size`` News werden verworfen (nicht als weniger als ``min_cluster_size`` News werden verworfen (nicht als
@ -479,6 +523,14 @@ def aggregate_news_cluster(
Pro Cluster: zentralster Antrag-Match aus den GWÖ-bewerteten Antraegen. Pro Cluster: zentralster Antrag-Match aus den GWÖ-bewerteten Antraegen.
""" """
cache_key = (
"cluster", days_window, round(intra_threshold, 3),
round(antrag_threshold, 3), min_cluster_size, str(db_path or ""),
)
cached = _cache_get(cache_key)
if cached is not None:
return cached
from .config import settings from .config import settings
from . import embeddings as emb from . import embeddings as emb
@ -592,7 +644,7 @@ def aggregate_news_cluster(
), ),
reverse=True, reverse=True,
) )
return { result = {
"clusters": out_clusters, "clusters": out_clusters,
"n_total_news": len(fresh), "n_total_news": len(fresh),
"filter": { "filter": {
@ -602,6 +654,8 @@ def aggregate_news_cluster(
"min_cluster_size": min_cluster_size, "min_cluster_size": min_cluster_size,
}, },
} }
_cache_set(cache_key, result)
return result
def aggregate_top_antraege_with_news( def aggregate_top_antraege_with_news(

View File

@ -14,6 +14,7 @@ from app.themen_matching import (
aggregate_themen_zeitreihe, aggregate_themen_zeitreihe,
aggregate_top_antraege_with_news, aggregate_top_antraege_with_news,
aggregate_top_themen, aggregate_top_themen,
cache_clear,
compute_relevance, compute_relevance,
find_anträge_for_news, find_anträge_for_news,
find_news_for_antrag, find_news_for_antrag,
@ -452,3 +453,42 @@ class TestTopAntraegeWithNews:
elif first_without == len(result["antraege"]): elif first_without == len(result["antraege"]):
first_without = i first_without = i
assert last_with_news < first_without assert last_with_news < first_without
# ─────────────────────────────────────────────────────────────────────────────
# TTL-Cache (Performance #170 followup)
# ─────────────────────────────────────────────────────────────────────────────
class TestPerformanceCache:
def test_top_themen_cache_hit_returns_same_object(self, populated_db):
"""Zweiter Call mit gleichen Args sollte den gleichen dict liefern."""
cache_clear()
a = aggregate_top_themen(db_path=populated_db, min_similarity=0.5)
b = aggregate_top_themen(db_path=populated_db, min_similarity=0.5)
# Cache liefert dasselbe Objekt (identity check)
assert a is b
def test_top_themen_cache_miss_different_args(self, populated_db):
"""Andere Args → neuer Eintrag, anderer dict."""
cache_clear()
a = aggregate_top_themen(db_path=populated_db, min_similarity=0.5)
b = aggregate_top_themen(db_path=populated_db, min_similarity=0.6)
# Different filter values → different cache-keys
assert a is not b
def test_cache_clear_invalidates(self, populated_db):
cache_clear()
a = aggregate_top_themen(db_path=populated_db, min_similarity=0.5)
cache_clear()
b = aggregate_top_themen(db_path=populated_db, min_similarity=0.5)
# Nach clear: neuer Aufruf gibt neues Objekt zurueck
assert a is not b
# Inhaltlich identisch
assert len(a["buckets"]) == len(b["buckets"])
def test_cluster_cached_too(self, populated_db):
cache_clear()
a = aggregate_news_cluster(db_path=populated_db, min_cluster_size=1)
b = aggregate_news_cluster(db_path=populated_db, min_cluster_size=1)
assert a is b