from corpusgen.inventory import PhoibleDataset
PhoibleDataset().download()  # cached at ~/.corpusgen/phoible.csv (~24 MB)

import corpusgen

report = corpusgen.evaluate(
    ["The quick brown fox jumps over the lazy dog.",
     "She sells seashells by the seashore.",
     "Pack my box with five dozen liquor jugs."],
    language="en-us",
    target_phonemes="phoible",
)

print(report.render())
print(report.coverage)           # 0.65
print(report.missing_phonemes)   # {'ʒ', 'ð', 'θ', ...}

import corpusgen

candidates = [
    "The quick brown fox jumps over the lazy dog.",
    "She sells seashells by the seashore.",
    "Peter Piper picked a peck of pickled peppers.",
    "How much wood would a woodchuck chuck?",
    "To be or not to be, that is the question.",
]

result = corpusgen.select_sentences(
    candidates,
    language="en-us",
    algorithm="greedy",  # or "celf", "stochastic", "ilp", "distribution", "nsga2"
)

print(f"Selected {result.num_selected} of {len(candidates)} sentences")
print(f"Coverage: {result.coverage:.1%}")

from corpusgen.generate.phon_ctg.targets import PhoneticTargetInventory
from corpusgen.generate.phon_ctg.scorer import PhoneticScorer
from corpusgen.generate.phon_ctg.loop import GenerationLoop, StoppingCriteria
from corpusgen.generate.backends.repository import RepositoryBackend
from corpusgen.g2p.manager import G2PManager

# 1. Phonemize a sentence pool
g2p = G2PManager()
sentences = ["The cat sat on the mat.", "Big dogs bark loudly.", ...]
results = g2p.phonemize_batch(sentences, language="en-us")
pool = [
    {"text": s, "phonemes": r.phonemes}
    for s, r in zip(sentences, results) if r.phonemes
]

# 2. Set up targets, scorer, and backend
targets = PhoneticTargetInventory(
    target_phonemes=["p", "b", "t", "d", "k", "g"],
    unit="phoneme",
)
scorer = PhoneticScorer(targets=targets, coverage_weight=1.0)
backend = RepositoryBackend(pool=pool)

# 3. Run the generation loop
loop = GenerationLoop(
    backend=backend,
    targets=targets,
    scorer=scorer,
    stopping_criteria=StoppingCriteria(
        target_coverage=0.9,
        max_sentences=20,
    ),
)
result = loop.run()

print(f"Generated {result.num_generated} sentences, coverage: {result.coverage:.1%}")

from corpusgen.generate.backends.llm_api import LLMBackend

# Requires: poetry install --with llm
# Set your API key: export OPENAI_API_KEY=...
backend = LLMBackend(
    model="gpt-4o-mini",
    language="en-us",
)

# Use with the same GenerationLoop as above
loop = GenerationLoop(
    backend=backend,
    targets=targets,
    scorer=scorer,
    stopping_criteria=StoppingCriteria(target_coverage=0.9),
)
result = loop.run()

from corpusgen.generate.phon_ctg.targets import PhoneticTargetInventory
from corpusgen.generate.phon_rl.reward import PhoneticReward
from corpusgen.generate.phon_rl.trainer import PhonRLTrainer, TrainingConfig

# Requires: poetry install --with local

# 1. Define targets and reward
targets = PhoneticTargetInventory(
    target_phonemes=["p", "b", "t", "d", "k"],
    unit="phoneme",
)
reward = PhoneticReward(targets=targets, coverage_weight=1.0)

# 2. Configure PPO training
config = TrainingConfig(
    model_name="gpt2",
    num_steps=100,
    learning_rate=1e-5,
    kl_coeff=0.1,
    use_peft=True,     # LoRA for parameter-efficient training
    peft_r=8,
    peft_alpha=16,
    device=None,        # auto-detect GPU
)

# 3. Train with dynamic prompts that adapt to coverage gaps
def make_prompt(targets):
    missing = targets.next_targets(5)
    return f"Write a sentence using these sounds: {', '.join(missing)}"

trainer = PhonRLTrainer(reward=reward, config=config)
result = trainer.train(prompt_fn=make_prompt)

print(f"Final coverage: {result.final_coverage:.1%}")
trainer.save_checkpoint("./phon_rl_checkpoint")

from corpusgen import get_inventory

inv = get_inventory("en-us")
print(inv.language_name)          # 'English'
print(inv.consonants)             # ['p', 'b', 't', 'd', 'k', ...]
print(inv.vowels)                 # ['iː', 'ɪ', 'ɛ', 'æ', ...]

# Query by distinctive features
nasals = inv.segments_with_feature("nasal", "+")
print([s.phoneme for s in nasals])  # ['m', 'n', 'ŋ']

import corpusgen

report = corpusgen.evaluate(
    ["The quick brown fox jumps."],
    language="en-us",
    target_phonemes="phoible",
    unit="diphone",
)
print(f"Diphone coverage: {report.coverage:.1%}")

import corpusgen

report = corpusgen.evaluate(
    ["The quick brown fox."],
    language="en-us",
    target_phonemes="phoible",
)

# JSON
print(report.to_json(indent=2))

# JSON-LD (linked data)
doc = report.to_jsonld_ex()

# Human-readable at different verbosity levels
from corpusgen.evaluate.report import Verbosity
print(report.render(verbosity=Verbosity.MINIMAL))
print(report.render(verbosity=Verbosity.NORMAL))
print(report.render(verbosity=Verbosity.VERBOSE))

import corpusgen

report = corpusgen.evaluate(
    ["The cat sat on the mat.", "Big dogs dig deep holes."],
    language="en-us",
    target_phonemes="phoible",
)

# Distribution metrics are auto-computed
dm = report.distribution
print(f"Normalized entropy: {dm.normalized_entropy:.4f}")  # 1.0 = perfectly uniform
print(f"JSD vs uniform: {dm.jsd_uniform:.6f}")             # 0.0 = perfectly uniform
print(f"PCD (uniform): {dm.pcd_uniform:.4f}")              # coverage × (1 - JSD)

# Compare against a natural language reference distribution
from corpusgen.evaluate.distribution import compute_distribution_metrics

reference = {"p": 0.04, "t": 0.07, "k": 0.03, "ə": 0.12}  # example frequencies
dm_ref = compute_distribution_metrics(
    report.phoneme_counts, report.target_phonemes, reference_distribution=reference
)
print(f"JSD vs reference: {dm_ref.jsd_reference:.6f}")
print(f"Pearson correlation: {dm_ref.pearson_correlation}")

from corpusgen.evaluate.trajectory import compute_coverage_trajectory

# From a SelectionResult
traj = compute_coverage_trajectory(
    [candidate_phonemes[i] for i in result.selected_indices],
    target_units=result.covered_units | result.missing_units,
    unit=result.unit,
)

# Easy plotting
import matplotlib.pyplot as plt
plt.plot(range(len(traj.coverages)), traj.coverages)
plt.xlabel("Sentences")
plt.ylabel("Coverage")
plt.title("Coverage Saturation Curve")
plt.show()

# Access marginal gains per sentence
print(traj.gains)  # [5, 3, 2, 1, 1, 0, ...]

import corpusgen

report = corpusgen.evaluate(
    ["The cat sat on the mat.", "Big dogs dig deep holes."],
    language="en-us",
)

# Text quality metrics are auto-computed
tq = report.text_quality
print(f"Type-Token Ratio: {tq.type_token_ratio:.3f}")
print(f"Flesch Reading Ease: {tq.flesch_reading_ease:.1f}")
print(f"Avg sentence length: {tq.sentence_length_words_mean:.1f} words")

from corpusgen.evaluate.perplexity import compute_corpus_perplexity

# Simple — loads GPT-2 automatically (requires: poetry install --with local)
metrics = compute_corpus_perplexity(
    ["The cat sat on the mat.", "Big dogs dig deep holes."],
    model_name="gpt2",
)

print(f"Corpus perplexity: {metrics.corpus_perplexity:.2f}")  # token-weighted (standard LM metric)
print(f"Mean perplexity:   {metrics.mean_perplexity:.2f}")    # sentence-weighted
print(f"Median:            {metrics.median_perplexity:.2f}")
print(f"Total tokens:      {metrics.num_tokens}")

# Per-sentence breakdown
for i, ppl in enumerate(metrics.per_sentence):
    print(f"  Sentence {i}: PPL = {ppl:.2f}")

# Shared model — avoids loading the same model twice when you are
# also using PerplexityFluencyScorer during generation:
from corpusgen.generate.scorers.fluency import PerplexityFluencyScorer

scorer = PerplexityFluencyScorer(model_name="gpt2", device="cuda")
scorer("warm-up call to trigger lazy load")

metrics = compute_corpus_perplexity(
    sentences,
    model=scorer._model,
    tokenizer=scorer._tokenizer,
)

from corpusgen.evaluate.error_rates import compute_error_rates

result = compute_error_rates(
    references=["the cat sat on the mat", "big dogs dig deep holes"],
    hypotheses=["the cat sat on a mat", "big dog dig deep hole"],
)

print(f"WER: {result.wer:.2%}")   # corpus-level, micro-averaged
print(f"CER: {result.cer:.2%}")
print(f"SER: {result.ser:.2%}")

# With phoneme-level comparison
result = compute_error_rates(
    references=["the cat"],
    hypotheses=["a cat"],
    reference_phonemes=[["\u00f0", "\u0259", "k", "\u00e6", "t"]],
    hypothesis_phonemes=[["\u0259", "k", "\u00e6", "t"]],
)
print(f"PER: {result.per:.2%}")

# Per-sentence breakdown
for d in result.details:
    print(f"  [{d.index}] WER={d.wer:.2%} CER={d.cer:.2%}")