{
  "artifact": "OreAyo ASR v1 — published benchmark figures",
  "generated": "2026-08-27",
  "release": "ASR v1 (stable, deployed)",
  "evaluation_date": "2026-06-22",
  "base_model": {
    "name": "facebook/mms-1b-all",
    "total_parameters": 964768990,
    "trainable_parameters": 2271582,
    "trainable_fraction": "0.24%",
    "adaptation": "per-language adapter + CTC head; base frozen"
  },
  "scoring": {
    "metric": "WER/CER via jiwer",
    "diacritics": "PRESERVED — scored strictly",
    "normalisation": [
      "Unicode NFC",
      "lowercase",
      "punctuation removed",
      "zero-width stripped",
      "whitespace collapsed"
    ],
    "digits": "records containing digits excluded",
    "note": "No diacritic stripped before scoring. Tone marks and sub-dots are scored as part of the word."
  },
  "eval_protocol": {
    "held_out": "corpus validation split, disjoint utterances",
    "utterance_overlap_with_train": 0,
    "speaker_disjoint": false,
    "caveat": "Eval speakers also appear in training; figures are an optimistic bound."
  },
  "results": {
    "yoruba": {
      "wer": 0.588,
      "cer": 0.214,
      "training_job": "P1-rerun (step 4000)",
      "eval_set": {
        "file": "oreayo/data/eval_sets/yoruba_val_frozen_2000.jsonl",
        "sha256": "9af107f93939e67419cfd0b1b680b071c39b075fa87c0e5fb0217d3068c7f2a6",
        "utterances": 2000,
        "audio_hours": 2.04,
        "distinct_speakers": 975,
        "mean_utterance_s": 3.67
      }
    },
    "igbo": {
      "wer": 0.484,
      "cer": 0.15,
      "training_job": "oreayo-mms-igbo-1782079548 (step 4000)",
      "eval_set": {
        "file": "oreayo/data/eval_sets/igbo_val_frozen_2000.jsonl",
        "sha256": "d1a100c3e475bc258d0653b0e464e8bae1d3dab47f9a34be577d66e71ae46f98",
        "utterances": 2000,
        "audio_hours": 2.14,
        "distinct_speakers": 1015,
        "mean_utterance_s": 3.85
      }
    },
    "hausa": {
      "wer": 0.422,
      "cer": 0.104,
      "training_job": "oreayo-mms-hausa-1782110241 (step 4000)",
      "eval_set": {
        "file": "oreayo/data/eval_sets/hausa_val_frozen_2000.jsonl",
        "sha256": "49805ae0e96ca9fec14b25fe4b334025d9715f9acf80aa28f65ffce1bb874521",
        "utterances": 2000,
        "audio_hours": 1.74,
        "distinct_speakers": 954,
        "mean_utterance_s": 3.13
      }
    }
  },
  "baseline": {
    "off_the_shelf_zero_shot": "MEASURED",
    "measured": "2026-08-27",
    "model": "facebook/mms-1b-all with stock per-language adapters",
    "protocol": "identical eval sets, normalisation and metric; only the adapter differs",
    "note": "Negative relative change = adaptation improved on the baseline. Hausa is positive: the stock adapter scores better than the fine-tuned one.",
    "caveat": "One run per language, no confidence intervals. Not a controlled training comparison — the stock adapters were trained by Meta on their own data at their own scale.",
    "results": {
      "yoruba": {
        "wer": 0.6427,
        "cer": 0.243,
        "eval_set_sha256": "9af107f93939e67419cfd0b1b680b071c39b075fa87c0e5fb0217d3068c7f2a6",
        "utterances": 2000,
        "relative_wer_change_vs_adapted": -8.5
      },
      "igbo": {
        "wer": 0.6215,
        "cer": 0.1908,
        "eval_set_sha256": "d1a100c3e475bc258d0653b0e464e8bae1d3dab47f9a34be577d66e71ae46f98",
        "utterances": 2000,
        "relative_wer_change_vs_adapted": -22.1
      },
      "hausa": {
        "wer": 0.4119,
        "cer": 0.1036,
        "eval_set_sha256": "49805ae0e96ca9fec14b25fe4b334025d9715f9acf80aa28f65ffce1bb874521",
        "utterances": 2000,
        "relative_wer_change_vs_adapted": 2.5
      }
    }
  }
}