Day 2 · Gold standards & evaluation

Day 2 — Linguistic Data Analysis II

How to use this notebook

This is your single submission for the day. It has two parts:

• Part A · Tutorial — load a gold standard → evaluate a frozen set of model predictions → read precision / recall / F1 + a confusion matrix → inspect the errors, on an easy-to-judge task (CEFR sentence level).
• Part B · Corpus Lab — code the evaluation metrics (precision, recall, F1, Cohen’s κ) by hand, then check them against scikit-learn.

You only edit the cells marked ✏️ YOU EDIT. Cells marked 🔧 Library cell are pre-written — run them, don’t change them.

➡️ Work top to bottom. When you’re done, Runtime → Run all, then File → Download → Download .ipynb and submit that file.

Part A · Tutorial — the pipeline on CEFR-SP

The task (CEFR sentence level, A1–C2) is easy to judge on purpose — today the mechanics are the lesson, not the labeling.

NoteToday runs on frozen predictions — no API key, no live model

You met the live model on Day 1 and saw its answers change from run to run. When you’re learning to measure quality, that wobble is just noise fighting the lesson — so today the model’s predictions are pre-computed and committed to a file. You load them and everyone’s precision / recall / F1 come out identical every run. You’ll run the model yourself (with the key) from Day 3 on.

Setup — run this first

#@title 📦 Setup — run me first { display-mode: "form" }
# Imports + the LLM backend. No pip install needed in Colab.
import json, re, urllib.request, os
from sklearn.metrics import classification_report, confusion_matrix
import pandas as pd, seaborn as sns, matplotlib.pyplot as plt

MODEL_ID = "gemini-2.5-flash"   # pinned model for the reproducible (API) backend

def _resolve_gemini_key():
    """Find a Gemini API key: Colab Secrets first (not auto-exported to env), then env."""
    try:
        from google.colab import userdata      # only exists in Colab
        key = userdata.get("GEMINI_API_KEY")
        if key:
            return key
    except Exception:
        pass                                    # not in Colab, or secret not set
    return os.environ.get("GEMINI_API_KEY")

def _make_api_backend(key):
    """Reproducible backend: Gemini API with temperature=0 + a fixed seed."""
    from google import genai
    from google.genai import types
    client = genai.Client(api_key=key)
    cfg = types.GenerateContentConfig(temperature=0, seed=42)
    return (lambda p: client.models.generate_content(
                model=MODEL_ID, contents=p, config=cfg).text,
            f"Gemini API ({MODEL_ID}, temperature=0, seed=42)")

# Prefer the API key when set (reproducible); else fall back to colab.ai (demo).
_key = _resolve_gemini_key()
if _key:
    generate_text, _backend = _make_api_backend(_key)
else:
    try:
        from google.colab import ai            # Colab's built-in Gemini — no key
        generate_text, _backend = (lambda p: ai.generate_text(p)), "Colab Gemini (demo, non-reproducible)"
    except ImportError:
        raise RuntimeError(
            "No LLM backend found. Run this notebook in Google Colab (free built-in "
            "Gemini, no key needed), or set GEMINI_API_KEY — in Colab via the Secrets "
            "panel, or as an environment variable when running locally. "
            "See resources/tools/gemini-api-key.md.")

# CEFR-SP gold set (72 sentences, 12 per level), fetched from the course repo.
GOLD_URL = "https://raw.githubusercontent.com/egumasa/linguistic-data-analysis-II-2026/main/sources/resources/datasets/gold/cefr_sentences.json"
LEVELS = ["A1", "A2", "B1", "B2", "C1", "C2"]
PREDICTIONS_URL = "https://raw.githubusercontent.com/egumasa/linguistic-data-analysis-II-2026/main/sources/resources/datasets/gold/predictions_day2.json"   # frozen model predictions
print(f"Setup done. LLM backend: {_backend}. scikit-learn ready.")

#@title 🔧 Library cell: load_gold(url_or_path) → gold { display-mode: "form" }
def load_gold(url_or_path):
    """Read the canonical gold JSON: [{'id','text','label'}, ...]."""
    if str(url_or_path).startswith("http"):
        raw = urllib.request.urlopen(url_or_path).read().decode("utf-8")
        gold = json.loads(raw)
    else:
        gold = json.loads(open(url_or_path, encoding="utf-8").read())
    print(f"Loaded {len(gold)} items. First one:", gold[0])
    return gold

#@title 🔧 Library cell: run_prompt(prompt, gold) → predictions { display-mode: "form" }
def _extract_level(text):
    """Pull the first A1/A2/B1/B2/C1/C2 out of the model's reply."""
    m = re.search(r"\b([ABC][12])\b", str(text).upper())
    return m.group(1) if m else "??"

def run_prompt(prompt, gold):
    """Send each item's `text` to the LLM via {text}, collect predicted labels."""
    predictions = []
    for i, item in enumerate(gold, 1):
        reply = generate_text(prompt.format(text=item["text"]))
        predictions.append(_extract_level(reply))
        if i % 12 == 0:
            print(f"  ...{i}/{len(gold)} done")
    print(f"Got {len(predictions)} predictions.")
    return predictions

#@title 🔧 Library cell: evaluate(gold, predictions) → P/R/F1 + confusion matrix { display-mode: "form" }
def evaluate(gold, predictions):
    y_true = [item["label"] for item in gold]
    y_pred = predictions
    print(classification_report(y_true, y_pred, labels=LEVELS, zero_division=0))
    cm = confusion_matrix(y_true, y_pred, labels=LEVELS)
    plt.figure(figsize=(5.5, 4.5))
    sns.heatmap(cm, annot=True, fmt="d", cmap="Blues",
                xticklabels=LEVELS, yticklabels=LEVELS)
    plt.xlabel("Predicted"); plt.ylabel("Gold"); plt.title("Confusion matrix")
    plt.tight_layout(); plt.show()

#@title 🔧 Library cell: show_errors(gold, predictions) → misclassified table { display-mode: "form" }
def show_errors(gold, predictions):
    rows = [{"id": g["id"], "gold": g["label"], "pred": p, "text": g["text"]}
            for g, p in zip(gold, predictions) if g["label"] != p]
    print(f"{len(rows)} of {len(gold)} wrong.")
    return pd.DataFrame(rows)

#@title 🔧 Library cell: save_predictions / load_predictions { display-mode: "form" }
def save_predictions(predictions, path):
    """Freeze a list of predicted labels to JSON."""
    with open(path, "w", encoding="utf-8") as f:
        json.dump(predictions, f)
    print(f"Saved {len(predictions)} predictions to {path}")

def load_predictions(url_or_path):
    """Read a frozen predictions list — a committed URL or a local path."""
    if str(url_or_path).startswith("http"):
        raw = urllib.request.urlopen(url_or_path).read().decode("utf-8")
        predictions = json.loads(raw)
    else:
        predictions = json.loads(open(url_or_path, encoding="utf-8").read())
    print(f"Loaded {len(predictions)} frozen predictions.")
    return predictions

Step 1 — Load the gold standard

Notice the shape: every dataset this week is {"id", "text", "label"}. That is the only data shape you have to learn.

gold = load_gold(GOLD_URL)

Step 2 — The prompt, and the frozen predictions it produced

Here is the prompt we sent the model — {text} is where each sentence gets slotted in. We ran it once over the gold set and committed the predictions, so today you load that frozen file rather than call the model. (From Day 3 you’ll run prompts like this yourself.)

# The prompt used to produce the frozen predictions (shown for reference — not run today):
PROMPT = """You are an expert rater of English sentence difficulty using the CEFR scale.
Classify the sentence into exactly one of: A1, A2, B1, B2, C1, C2.
Answer with the level only.

Sentence: {text}"""

# Load the pre-computed predictions (same order as `gold`):
predictions = load_predictions(PREDICTIONS_URL)

Step 3 — Read the evaluation

Per-level precision / recall / F1 (plus a macro average), then the confusion matrix. For CEFR you’ll usually see confusions between adjacent levels (B1 ↔︎ B2), rarely far-apart ones (A1 ↔︎ C2) — the model has the right idea, imprecise thresholds. Because the predictions are frozen, these numbers are the same every time you run — that’s the point.

evaluate(gold, predictions)

Step 4 — Error analysis

For each miss, ask: is the gold defensible, or is this a genuinely borderline sentence? “Is the disagreement the model’s fault or the scheme’s?” is the heart of annotation work — and it returns with force in the Day 3 discourse task.

show_errors(gold, predictions)

Part B · Corpus Lab — code the metrics from scratch

evaluate() above printed precision, recall, F1 and a confusion matrix for you. Now you write those formulas yourself, from plain lists of labels — no imports, no numpy. Once you have coded them, the numbers scikit-learn reports in your final project will never be a black box.

Fill in each function (replace its raise NotImplementedError(...)), then run the self-check cell — it compares your functions against scikit-learn on a small example and prints ✅ / ❌ for each.

# ✏️ YOU EDIT — implement each function. All take plain Python lists of labels.

def confusion_counts(gold, pred, label):
    """Return {"tp","fp","fn","tn"} for ONE label, treating it as the positive class.
        tp: gold IS label AND pred IS label
        fp: gold is NOT label BUT pred IS label
        fn: gold IS label BUT pred is NOT label
        tn: gold is NOT label AND pred is NOT label
    Example (label="B2", gold=["A1","B2","B2"], pred=["A1","B2","C1"]):
        {"tp": 1, "fp": 0, "fn": 1, "tn": 1}
    """
    # HINT: start tp=fp=fn=tn=0; loop `for g, p in zip(gold, pred):`;
    #       if/elif on whether g == label / p == label.
    raise NotImplementedError("Count tp, fp, fn, tn and return them in a dict.")


def precision(gold, pred, label):
    """tp / (tp + fp). Of all items CALLED `label`, what fraction really were?
    Return 0.0 if tp + fp == 0 (never divide by zero)."""
    raise NotImplementedError("Return tp / (tp + fp), or 0.0 if tp + fp == 0.")


def recall(gold, pred, label):
    """tp / (tp + fn). Of all items that TRULY were `label`, what fraction found?
    Return 0.0 if tp + fn == 0."""
    raise NotImplementedError("Return tp / (tp + fn), or 0.0 if tp + fn == 0.")


def f1(gold, pred, label):
    """Harmonic mean of precision and recall:
        2 * p * r / (p + r).   Return 0.0 if p + r == 0."""
    # HINT: reuse precision(...) and recall(...) above.
    raise NotImplementedError("Return the harmonic mean of precision and recall.")


def macro_f1(gold, pred, labels):
    """Plain average of f1(gold, pred, label) over every label in `labels`
    (every class counts equally, no matter how many items it has)."""
    raise NotImplementedError("Average f1 over every label in `labels`.")


def percent_agreement(a, b):
    """Fraction of positions where two annotators agree: a[i] == b[i]. 0.0–1.0."""
    raise NotImplementedError("Return the fraction of positions where a[i] == b[i].")


def cohen_kappa(a, b):
    """Agreement corrected for chance:  (p_o - p_e) / (1 - p_e).
        p_o = observed agreement = percent_agreement(a, b)
        p_e = chance agreement = sum over labels of prop_a(label) * prop_b(label),
              where prop_x(label) = (# times x used label) / N.
    Return 0.0 if 1 - p_e == 0."""
    # HINT: labels = set(a) | set(b); N = len(a); a.count(label) counts uses.
    raise NotImplementedError("Return (p_o - p_e) / (1 - p_e), or 0.0 if 1 - p_e == 0.")

#@title 🔎 Self-check against scikit-learn — run me { display-mode: "form" }
from sklearn.metrics import (precision_score, recall_score, f1_score,
                             cohen_kappa_score)

# A small fixture with imbalance and a few mistakes — enough to catch bugs.
LAB = ["A1", "A2", "B1", "B2"]
g = ["A1", "A1", "A2", "A2", "B1", "B1", "B1", "B2", "B2", "B2"]
p = ["A1", "A2", "A2", "A2", "B1", "B1", "B2", "B2", "B2", "A2"]
ann_a = ["A1", "A2", "B1", "B1", "B2", "A1", "A2", "B2", "B1", "A1"]
ann_b = ["A1", "A2", "B1", "B2", "B2", "A2", "A2", "B2", "B1", "A1"]

TOL, results = 1e-9, []
def _chk(name, got, exp):
    ok = abs(got - exp) < TOL
    results.append(ok)
    print(("✅" if ok else "❌"), f"{name:<22} yours={got:.6f}  sklearn={exp:.6f}")

for label in LAB:
    _chk(f"precision({label})", precision(g, p, label),
         precision_score(g, p, labels=[label], average="micro", zero_division=0))
    _chk(f"recall({label})", recall(g, p, label),
         recall_score(g, p, labels=[label], average="micro", zero_division=0))
    _chk(f"f1({label})", f1(g, p, label),
         f1_score(g, p, labels=[label], average="micro", zero_division=0))
_chk("macro_f1", macro_f1(g, p, LAB),
     f1_score(g, p, labels=LAB, average="macro", zero_division=0))
_chk("percent_agreement", percent_agreement(ann_a, ann_b),
     sum(1 for x, y in zip(ann_a, ann_b) if x == y) / len(ann_a))
_chk("cohen_kappa", cohen_kappa(ann_a, ann_b), cohen_kappa_score(ann_a, ann_b))

print("-" * 55)
print(f"All {len(results)} checks passed ✅  — your metrics match scikit-learn."
      if all(results) else
      f"{results.count(False)} of {len(results)} checks FAILED — fix and re-run.")

---

✅ Before you submit

1. Runtime → Run all and check every cell ran without error.
2. Part A outputs are visible (tables / charts / the model’s answers).
3. Part B self-check prints ✅ (or your TODO answers are filled in).
4. File → Download → Download .ipynb and upload that one file.