Day 4 · Pipeline assembly & sampling your gold set

Day 4 — Linguistic Data Analysis II

How to use this notebook

This is your single submission for the day. It has two parts:

  • Part A · Tutorial — sample a balanced gold subset from a dataset pool, ready for QC.
  • Part B · Corpus Lab — quality-control and adjudicate your sampled gold set (to be written).

You only edit the cells marked ✏️ YOU EDIT. Cells marked 🔧 Library cell are pre-written — run them, don’t change them.

➡️ Work top to bottom. When you’re done, Runtime → Run all, then File → Download → Download .ipynb and submit that file.

Part A · Tutorial — sample a balanced gold subset

For your mini-project you build your own gold set by sampling from a pool. A balanced sample (equal items per label) keeps precision/recall/F1 and the confusion matrix meaningful. Here we demo it on the familiar CEFR pool; swap POOL_URL for your track’s pool. See the mini-project tracks for the full list.

Setup — run this first

Show code 
#@title 📦 Setup — run me first { display-mode: "form" }
# Imports + the LLM backend. No pip install needed in Colab.
import json, re, urllib.request, os
from sklearn.metrics import classification_report, confusion_matrix
import pandas as pd, seaborn as sns, matplotlib.pyplot as plt

MODEL_ID = "gemini-2.5-flash"   # pinned model for the reproducible (API) backend

def _resolve_gemini_key():
    """Find a Gemini API key: Colab Secrets first (not auto-exported to env), then env."""
    try:
        from google.colab import userdata      # only exists in Colab
        key = userdata.get("GEMINI_API_KEY")
        if key:
            return key
    except Exception:
        pass                                    # not in Colab, or secret not set
    return os.environ.get("GEMINI_API_KEY")

def _make_api_backend(key):
    """Reproducible backend: Gemini API with temperature=0 + a fixed seed."""
    from google import genai
    from google.genai import types
    client = genai.Client(api_key=key)
    cfg = types.GenerateContentConfig(temperature=0, seed=42)
    return (lambda p: client.models.generate_content(
                model=MODEL_ID, contents=p, config=cfg).text,
            f"Gemini API ({MODEL_ID}, temperature=0, seed=42)")

# Prefer the API key when set (reproducible); else fall back to colab.ai (demo).
_key = _resolve_gemini_key()
if _key:
    generate_text, _backend = _make_api_backend(_key)
else:
    try:
        from google.colab import ai            # Colab's built-in Gemini — no key
        generate_text, _backend = (lambda p: ai.generate_text(p)), "Colab Gemini (demo, non-reproducible)"
    except ImportError:
        raise RuntimeError(
            "No LLM backend found. Run this notebook in Google Colab (free built-in "
            "Gemini, no key needed), or set GEMINI_API_KEY — in Colab via the Secrets "
            "panel, or as an environment variable when running locally. "
            "See resources/tools/gemini-api-key.md.")

# Default gold + LEVELS for CEFR; change LEVELS to match your own track's labels.
GOLD_URL = "https://raw.githubusercontent.com/egumasa/linguistic-data-analysis-II-2026/main/sources/resources/datasets/gold/cefr_sentences.json"
LEVELS = ["A1", "A2", "B1", "B2", "C1", "C2"]
print(f"Setup done. LLM backend: {_backend}. scikit-learn ready.")
Show code 
#@title 🔧 Library cell: load_gold(url_or_path) → gold { display-mode: "form" }
def load_gold(url_or_path):
    """Read the canonical gold JSON: [{'id','text','label'}, ...]."""
    if str(url_or_path).startswith("http"):
        raw = urllib.request.urlopen(url_or_path).read().decode("utf-8")
        gold = json.loads(raw)
    else:
        gold = json.loads(open(url_or_path, encoding="utf-8").read())
    print(f"Loaded {len(gold)} items. First one:", gold[0])
    return gold
Show code 
#@title 🔧 Library cell: run_prompt(prompt, gold) → predictions { display-mode: "form" }
def _extract_level(text):
    """Pull the first A1/A2/B1/B2/C1/C2 out of the model's reply."""
    m = re.search(r"\b([ABC][12])\b", str(text).upper())
    return m.group(1) if m else "??"

def run_prompt(prompt, gold):
    """Send each item's `text` to the LLM via {text}, collect predicted labels."""
    predictions = []
    for i, item in enumerate(gold, 1):
        reply = generate_text(prompt.format(text=item["text"]))
        predictions.append(_extract_level(reply))
        if i % 12 == 0:
            print(f"  ...{i}/{len(gold)} done")
    print(f"Got {len(predictions)} predictions.")
    return predictions
Show code 
#@title 🔧 Library cell: evaluate(gold, predictions) → P/R/F1 + confusion matrix { display-mode: "form" }
def evaluate(gold, predictions):
    y_true = [item["label"] for item in gold]
    y_pred = predictions
    print(classification_report(y_true, y_pred, labels=LEVELS, zero_division=0))
    cm = confusion_matrix(y_true, y_pred, labels=LEVELS)
    plt.figure(figsize=(5.5, 4.5))
    sns.heatmap(cm, annot=True, fmt="d", cmap="Blues",
                xticklabels=LEVELS, yticklabels=LEVELS)
    plt.xlabel("Predicted"); plt.ylabel("Gold"); plt.title("Confusion matrix")
    plt.tight_layout(); plt.show()
Show code 
#@title 🔧 Library cell: show_errors(gold, predictions) → misclassified table { display-mode: "form" }
def show_errors(gold, predictions):
    rows = [{"id": g["id"], "gold": g["label"], "pred": p, "text": g["text"]}
            for g, p in zip(gold, predictions) if g["label"] != p]
    print(f"{len(rows)} of {len(gold)} wrong.")
    return pd.DataFrame(rows)
Show code 
#@title 🔧 Library cell: save_predictions / load_predictions { display-mode: "form" }
def save_predictions(predictions, path):
    """Freeze a list of predicted labels to JSON."""
    with open(path, "w", encoding="utf-8") as f:
        json.dump(predictions, f)
    print(f"Saved {len(predictions)} predictions to {path}")

def load_predictions(url_or_path):
    """Read a frozen predictions list — a committed URL or a local path."""
    if str(url_or_path).startswith("http"):
        raw = urllib.request.urlopen(url_or_path).read().decode("utf-8")
        predictions = json.loads(raw)
    else:
        predictions = json.loads(open(url_or_path, encoding="utf-8").read())
    print(f"Loaded {len(predictions)} frozen predictions.")
    return predictions
Show code 
POOL_URL = "https://raw.githubusercontent.com/egumasa/linguistic-data-analysis-II-2026/main/sources/resources/datasets/gold/cefr_pool.json"   # ✏️ swap for your track's pool
pool = load_gold(POOL_URL)

Draw a balanced sample ✏️ YOU EDIT

PER_LABEL items per label, with a fixed random seed so the sample is reproducible (same every run). Rare classes simply yield fewer — that’s a property of the data.

Show code 
import random
from collections import defaultdict, Counter

PER_LABEL = 8            # how many items per label
random.seed(42)         # fixed seed = same sample every run

by_label = defaultdict(list)
for item in pool:
    by_label[item["label"]].append(item)

gold = []
for label in sorted(by_label):
    bucket = by_label[label]
    random.shuffle(bucket)
    gold.extend(bucket[:PER_LABEL])
random.shuffle(gold)
gold = [{"id": i + 1, "text": x["text"], "label": x["label"]}
        for i, x in enumerate(gold)]

print("sampled:", len(gold), "| per label:",
      dict(Counter(x["label"] for x in gold)))

Save your gold set to Google Drive

Keep your own gold set in your Drive (not the course repo). See Housing your data in Google Drive for the mount → save → load round-trip.

Show code 
# ✏️ Uncomment in Colab to save to your Drive:
# from google.colab import drive; drive.mount("/content/drive")
# import json
# with open("/content/drive/MyDrive/my_gold.json", "w", encoding="utf-8") as f:
#     json.dump(gold, f, ensure_ascii=False, indent=2)
# print("saved", len(gold), "items")

Part B · Corpus Lab — QC & adjudicate your gold set

Warning🚧 TODO — to be written

This lab is still being written. When it is, you will:

  1. Each group member independently re-checks part of the sampled set against the scheme.
  2. Flag disagreements with the published label.
  3. Discuss and resolve them — and record what changed (this feeds your report’s “Scheme & gold” section).

✅ Before you submit

  1. Runtime → Run all and check every cell ran without error.
  2. Part A outputs are visible (tables / charts / the model’s answers).
  3. Part B self-check prints ✅ (or your TODO answers are filled in).
  4. File → Download → Download .ipynb and upload that one file.