Transformer Math

import json
from openai import OpenAI

def evaluate_model(model: str, test_cases: list[dict]) -> dict:
    """Run eval suite and compute pass rates by category."""
    client = OpenAI()
    results = {"correct": 0, "total": 0, "by_category": {}}

    for case in test_cases:
        response = client.chat.completions.create(
            model=model,
            messages=[{"role": "user", "content": case["prompt"]}],
            temperature=0,  # deterministic for eval
        )
        answer = response.choices[0].message.content
        correct = case["check_fn"](answer, case["expected"])

        results["total"] += 1
        results["correct"] += int(correct)
        cat = case.get("category", "general")
        results["by_category"].setdefault(cat, {"correct": 0, "total": 0})
        results["by_category"][cat]["total"] += 1
        results["by_category"][cat]["correct"] += int(correct)

    results["accuracy"] = results["correct"] / results["total"]
    return results

JUDGE_SYSTEM = """You are an expert evaluator. Score the response on the following criteria.
Think step by step before assigning a score."""

JUDGE_TEMPLATE = """
## Task
{task_description}

## User Query
{query}

## Response to Evaluate
{response}

## Evaluation Rubric
Score each dimension 1-5 (5 = best):

**Faithfulness** — Does every claim in the response follow from the provided context?
1 = Hallucinated facts unrelated to context
5 = Every claim is directly supported by context

**Relevance** — Does the response address what the user actually asked?
1 = Completely off-topic
5 = Directly answers the question with appropriate scope

**Completeness** — Are all parts of the question addressed?
1 = Misses key aspects of the query
5 = Covers all relevant aspects

## Chain-of-Thought Reasoning
Think through each dimension before scoring:

## Scores (JSON)
{{"faithfulness": <1-5>, "relevance": <1-5>, "completeness": <1-5>}}
"""

def llm_judge(query: str, response: str, context: str, task_desc: str) -> dict:
    """Call GPT-4 as judge, parse JSON scores from response."""
    from openai import OpenAI
    client = OpenAI()

    prompt = JUDGE_TEMPLATE.format(
        task_description=task_desc,
        query=query,
        response=response,
        # context injected into task_description in practice
    )
    result = client.chat.completions.create(
        model="gpt-4o",
        messages=[
            {"role": "system", "content": JUDGE_SYSTEM},
            {"role": "user", "content": prompt},
        ],
        temperature=0,
        response_format={"type": "json_object"},
    )
    import json
    return json.loads(result.choices[0].message.content)

# pip install nltk rouge-score
from nltk.translate.bleu_score import sentence_bleu, SmoothingFunction
from rouge_score import rouge_scorer

def compute_bleu(reference: str, hypothesis: str) -> float:
    """Compute sentence-level BLEU-4. Range [0, 1]."""
    ref_tokens = reference.lower().split()
    hyp_tokens = hypothesis.lower().split()
    smoothie = SmoothingFunction().method1  # avoid 0 on short sentences
    return sentence_bleu([ref_tokens], hyp_tokens, smoothing_function=smoothie)

def compute_rouge(reference: str, hypothesis: str) -> dict:
    """Compute ROUGE-1, ROUGE-2, ROUGE-L F1 scores."""
    scorer = rouge_scorer.RougeScorer(["rouge1", "rouge2", "rougeL"], use_stemmer=True)
    scores = scorer.score(reference, hypothesis)
    return {
        "rouge1": round(scores["rouge1"].fmeasure, 4),
        "rouge2": round(scores["rouge2"].fmeasure, 4),
        "rougeL": round(scores["rougeL"].fmeasure, 4),
    }

# Example: summarization quality check
reference = "The transformer uses self-attention to process sequences in parallel."
hypothesis = "Transformers apply attention mechanisms across the full sequence simultaneously."

print("BLEU-4:", compute_bleu(reference, hypothesis))
# → BLEU-4: 0.089  (low — different wording, same meaning — BLEU misses this)

print("ROUGE:", compute_rouge(reference, hypothesis))
# → {'rouge1': 0.35, 'rouge2': 0.08, 'rougeL': 0.25}
# ROUGE-1 is better but still low — use BERTScore for semantic similarity