podcast-mindmap/scripts/curate_debates.py

#!/usr/bin/env python3
"""#18 Cross-Podcast-Debatte: Kuratiere Gegenüberstellungen zu gemeinsamen Themen.

Nimmt die stärksten Cross-Podcast-Paare und lässt Qwen Übereinstimmungen/Divergenzen zusammenfassen.

Nutzung:
    DASHSCOPE_API_KEY=... python3 curate_debates.py [db-pfad] [limit]
"""

import json
import os
import sys
import time
import sqlite3

from openai import OpenAI

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from json_utils import parse_llm_json

DB_PATH = sys.argv[1] if len(sys.argv) > 1 else "data/db.sqlite"
LIMIT = int(sys.argv[2]) if len(sys.argv) > 2 and not sys.argv[2].startswith("--") else 100
RERUN_ERRORS = "--rerun-errors" in sys.argv
API_KEY = os.environ.get("DASHSCOPE_API_KEY", "")
BASE_URL = "https://dashscope-intl.aliyuncs.com/compatible-mode/v1"
MODEL = "qwen-plus"

SYSTEM_PROMPT = """Du bist ein Diskursanalyst. Du erhältst zwei Textabschnitte aus VERSCHIEDENEN Podcasts, die dasselbe Thema behandeln.

Erstelle eine kurze Gegenüberstellung. Antworte NUR mit JSON:

{
  "topic": "Das gemeinsame Thema in 3-5 Wörtern",
  "agreement": "Worin stimmen beide überein? (1-2 Sätze)",
  "divergence": "Worin unterscheiden sie sich? (1-2 Sätze, oder 'keine wesentliche Divergenz')",
  "insight": "Was lernt man durch die Gegenüberstellung, das man aus keinem der beiden allein lernen würde? (1 Satz)"
}"""


def curate_pair(client, text_a, meta_a, text_b, meta_b):
    user_msg = f"""Podcast A — {meta_a}:
"{text_a}"

Podcast B — {meta_b}:
"{text_b}"

Erstelle die Gegenüberstellung."""

    last_err = None
    for attempt in range(3):
        try:
            resp = client.chat.completions.create(
                model=MODEL,
                messages=[
                    {"role": "system", "content": SYSTEM_PROMPT},
                    {"role": "user", "content": user_msg},
                ],
                temperature=0.2,
                max_tokens=400,
            )
            content = resp.choices[0].message.content
            usage = getattr(resp, "usage", None)
            tokens = (usage.prompt_tokens, usage.completion_tokens) if usage else (0, 0)
            try:
                parsed = parse_llm_json(content, expect="object")
                parsed["_tokens"] = tokens
                return parsed
            except ValueError as pe:
                last_err = f"parse: {pe}"
                break
        except Exception as e:
            last_err = str(e)
            if attempt < 2:
                time.sleep(2 ** attempt)
                continue
            break
    return {"topic": "error", "agreement": "", "divergence": "", "insight": str(last_err), "_tokens": (0, 0)}


def main():
    if not API_KEY:
        print("DASHSCOPE_API_KEY nicht gesetzt.")
        sys.exit(1)

    client = OpenAI(api_key=API_KEY, base_url=BASE_URL)
    db = sqlite3.connect(DB_PATH, timeout=60.0)
    db.execute("PRAGMA busy_timeout=60000")
    db.row_factory = sqlite3.Row

    db.executescript("""
    CREATE TABLE IF NOT EXISTS debates (
        id INTEGER PRIMARY KEY AUTOINCREMENT,
        topic TEXT,
        source_podcast TEXT, source_episode TEXT, source_idx INTEGER,
        target_podcast TEXT, target_episode TEXT, target_idx INTEGER,
        agreement TEXT, divergence TEXT, insight TEXT, score REAL
    );
    CREATE INDEX IF NOT EXISTS idx_debates_topic ON debates(topic);
    """)

    if RERUN_ERRORS:
        rows = db.execute("""
            SELECT d.source_podcast as podcast_id, d.source_episode, d.source_idx,
                   d.target_podcast, d.target_episode, d.target_idx, d.score,
                   p1.text as source_text, p2.text as target_text,
                   pc1.name as source_podcast_name, pc2.name as target_podcast_name,
                   e1.title as source_title, e1.guest as source_guest,
                   e2.title as target_title, e2.guest as target_guest
            FROM debates d
            JOIN paragraphs p1 ON d.source_podcast = p1.podcast_id AND d.source_episode = p1.episode_id AND d.source_idx = p1.idx
            JOIN paragraphs p2 ON d.target_podcast = p2.podcast_id AND d.target_episode = p2.episode_id AND d.target_idx = p2.idx
            JOIN episodes e1 ON d.source_podcast = e1.podcast_id AND d.source_episode = e1.id
            JOIN episodes e2 ON d.target_podcast = e2.podcast_id AND d.target_episode = e2.id
            JOIN podcasts pc1 ON d.source_podcast = pc1.id
            JOIN podcasts pc2 ON d.target_podcast = pc2.id
            WHERE d.topic = 'error'
        """).fetchall()
        del_count = db.execute("DELETE FROM debates WHERE topic='error'").rowcount
        db.commit()
        print(f"RE-RUN: {del_count} error-Records geloescht, {len(rows)} werden neu kuratiert.")
        existing = set()
    else:
        # Get strongest cross-podcast links
        rows = db.execute("""
            SELECT sl.podcast_id, sl.source_episode, sl.source_idx,
                   sl.target_podcast, sl.target_episode, sl.target_idx, sl.score,
                   p1.text as source_text, p2.text as target_text,
                   pc1.name as source_podcast_name, pc2.name as target_podcast_name,
                   e1.title as source_title, e1.guest as source_guest,
                   e2.title as target_title, e2.guest as target_guest
            FROM semantic_links sl
            JOIN paragraphs p1 ON sl.podcast_id = p1.podcast_id AND sl.source_episode = p1.episode_id AND sl.source_idx = p1.idx
            JOIN paragraphs p2 ON sl.target_podcast = p2.podcast_id AND sl.target_episode = p2.episode_id AND sl.target_idx = p2.idx
            JOIN episodes e1 ON sl.podcast_id = e1.podcast_id AND sl.source_episode = e1.id
            JOIN episodes e2 ON sl.target_podcast = e2.podcast_id AND sl.target_episode = e2.id
            JOIN podcasts pc1 ON sl.podcast_id = pc1.id
            JOIN podcasts pc2 ON sl.target_podcast = pc2.id
            WHERE sl.podcast_id != sl.target_podcast
            ORDER BY sl.score DESC
            LIMIT ?
        """, (LIMIT,)).fetchall()

        print(f"Kuratiere {len(rows)} Cross-Podcast-Debatten mit {MODEL}…")

        existing = set()
        try:
            for r in db.execute("SELECT source_podcast||source_episode||source_idx||target_podcast||target_episode||target_idx as k FROM debates").fetchall():
                existing.add(r["k"])
        except Exception:
            pass

    processed = 0
    total_in_tokens = 0
    total_out_tokens = 0
    for i, row in enumerate(rows):
        key = f"{row['podcast_id']}{row['source_episode']}{row['source_idx']}{row['target_podcast']}{row['target_episode']}{row['target_idx']}"
        if key in existing:
            continue

        meta_a = f"{row['source_podcast_name']} / {row['source_episode']}: {row['source_title']} ({row['source_guest']})"
        meta_b = f"{row['target_podcast_name']} / {row['target_episode']}: {row['target_title']} ({row['target_guest']})"

        result = curate_pair(client, row["source_text"][:800], meta_a, row["target_text"][:800], meta_b)
        in_t, out_t = result.pop("_tokens", (0, 0))
        total_in_tokens += in_t
        total_out_tokens += out_t

        db.execute(
            "INSERT INTO debates (topic, source_podcast, source_episode, source_idx, "
            "target_podcast, target_episode, target_idx, agreement, divergence, insight, score) "
            "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
            (result.get("topic", ""), row["podcast_id"], row["source_episode"], row["source_idx"],
             row["target_podcast"], row["target_episode"], row["target_idx"],
             result.get("agreement", ""), result.get("divergence", ""),
             result.get("insight", ""), row["score"])
        )

        processed += 1
        if processed % 10 == 0:
            db.commit()
            print(f"  {processed} kuratiert…")

        time.sleep(0.3)

    db.commit()

    topics = db.execute("SELECT topic, COUNT(*) as c FROM debates GROUP BY topic ORDER BY c DESC LIMIT 20").fetchall()
    print(f"\nFertig: {processed} Debatten kuratiert.")
    print("Top-Themen:")
    for t in topics:
        print(f"  {t['topic']}: {t['c']}")
    cost = total_in_tokens / 1e6 * 0.40 + total_out_tokens / 1e6 * 1.20
    print(f"Tokens: in={total_in_tokens} out={total_out_tokens} ~${cost:.4f}")

    db.close()


if __name__ == "__main__":
    main()