{ "winner": "v32", "digests": { "qwen35-2b": "09de712676e7ed373b82ebfd33bfeb7fa70d00c38a3ff63fe104b89fd8a2334f", "gemma4-e2b": "b37049369adfe3d2b653af0ab301a062ca5cbe96ace6aa0d3f2559ee9b563fc2" }, "baseline_reference": "precise", "screen": { "v01": { "n": 8, "passed": 0, "partial": 3, "failed": 5, "score": 3, "issues": { "content": 3, "numbers": 2, "fidelity": 2, "format": 2, "display": 1, "terms": 1, "quote": 1 }, "median_ms": 397.19 }, "v02": { "n": 8, "passed": 0, "partial": 4, "failed": 4, "score": 4, "issues": { "content": 4, "format": 4, "terms": 3, "numbers": 2, "display": 1, "quote": 1 }, "median_ms": 421.62 }, "v03": { "n": 8, "passed": 0, "partial": 5, "failed": 3, "score": 5, "issues": { "content": 4, "terms": 3, "format": 3, "numbers": 2, "display": 1, "quote": 1 }, "median_ms": 418.87 }, "v04": { "n": 8, "passed": 1, "partial": 4, "failed": 3, "score": 6, "issues": { "content": 3, "format": 2, "terms": 2, "numbers": 2, "display": 1, "fidelity": 1, "quote": 1 }, "median_ms": 366.31 }, "v05": { "n": 8, "passed": 0, "partial": 5, "failed": 3, "score": 5, "issues": { "content": 4, "terms": 3, "numbers": 2, "display": 1, "format": 1, "quote": 1 }, "median_ms": 388.05 }, "v06": { "n": 8, "passed": 0, "partial": 5, "failed": 3, "score": 5, "issues": { "content": 4, "format": 3, "terms": 3, "numbers": 2, "display": 1, "quote": 1 }, "median_ms": 388.6 }, "v07": { "n": 8, "passed": 0, "partial": 4, "failed": 4, "score": 4, "issues": { "content": 4, "terms": 3, "format": 2, "numbers": 2, "display": 1, "quote": 1 }, "median_ms": 384.29 }, "v08": { "n": 8, "passed": 0, "partial": 5, "failed": 3, "score": 5, "issues": { "content": 4, "terms": 3, "numbers": 2, "display": 1, "format": 1, "quote": 1 }, "median_ms": 399.83 }, "v09": { "n": 8, "passed": 0, "partial": 4, "failed": 4, "score": 4, "issues": { "format": 5, "content": 4, "terms": 3, "numbers": 2, "display": 1, "quote": 1 }, "median_ms": 386.52 }, "v10": { "n": 8, "passed": 0, "partial": 4, "failed": 4, "score": 4, "issues": { "content": 4, "terms": 3, "format": 2, "display": 1, "numbers": 1, "quote": 1 }, "median_ms": 418.58 }, "v11": { "n": 8, "passed": 1, "partial": 3, "failed": 4, "score": 5, "issues": { "content": 3, "format": 2, "terms": 2, "numbers": 2, "display": 1, "fidelity": 1, "quote": 1 }, "median_ms": 397.49 }, "v12": { "n": 8, "passed": 1, "partial": 4, "failed": 3, "score": 6, "issues": { "content": 3, "terms": 2, "numbers": 2, "display": 1, "fidelity": 1, "quote": 1 }, "median_ms": 298.35 }, "v13": { "n": 8, "passed": 1, "partial": 3, "failed": 4, "score": 5, "issues": { "content": 3, "terms": 2, "numbers": 2, "display": 1, "fidelity": 1, "quote": 1 }, "median_ms": 371.45 }, "v14": { "n": 8, "passed": 0, "partial": 3, "failed": 5, "score": 3, "issues": { "content": 4, "terms": 2, "numbers": 2, "display": 1, "fidelity": 1, "quote": 1 }, "median_ms": 368.15 }, "v15": { "n": 8, "passed": 0, "partial": 5, "failed": 3, "score": 5, "issues": { "content": 4, "terms": 2, "numbers": 2, "display": 1, "fidelity": 1, "quote": 1, "format": 1 }, "median_ms": 337.53 }, "v16": { "n": 8, "passed": 1, "partial": 3, "failed": 4, "score": 5, "issues": { "content": 3, "terms": 2, "numbers": 2, "display": 1, "fidelity": 1, "quote": 1 }, "median_ms": 358.25 }, "v17": { "n": 8, "passed": 1, "partial": 3, "failed": 4, "score": 5, "issues": { "content": 3, "terms": 2, "numbers": 2, "display": 1, "format": 1, "fidelity": 1, "quote": 1 }, "median_ms": 380.89 }, "v18": { "n": 8, "passed": 1, "partial": 5, "failed": 2, "score": 7, "issues": { "content": 3, "terms": 3, "format": 2, "numbers": 2, "display": 1, "quote": 1 }, "median_ms": 384.88 }, "v19": { "n": 8, "passed": 0, "partial": 5, "failed": 3, "score": 5, "issues": { "content": 4, "terms": 3, "format": 2, "numbers": 2, "display": 1, "repetition": 1 }, "median_ms": 426.27 }, "v20": { "n": 8, "passed": 1, "partial": 4, "failed": 3, "score": 6, "issues": { "content": 3, "terms": 3, "numbers": 2, "display": 1, "format": 1, "quote": 1 }, "median_ms": 345.95 }, "v21": { "n": 8, "passed": 0, "partial": 5, "failed": 3, "score": 5, "issues": { "content": 4, "terms": 3, "numbers": 2, "display": 1, "format": 1, "quote": 1 }, "median_ms": 379.62 }, "v22": { "n": 8, "passed": 1, "partial": 5, "failed": 2, "score": 7, "issues": { "content": 3, "terms": 3, "format": 2, "numbers": 2, "display": 1, "quote": 1 }, "median_ms": 420.34 }, "v23": { "n": 8, "passed": 1, "partial": 4, "failed": 3, "score": 6, "issues": { "content": 3, "terms": 3, "numbers": 2, "display": 1, "format": 1, "quote": 1 }, "median_ms": 366.79 }, "v24": { "n": 8, "passed": 0, "partial": 5, "failed": 3, "score": 5, "issues": { "content": 4, "terms": 3, "numbers": 2, "display": 1, "format": 1, "quote": 1 }, "median_ms": 365.96 }, "v25": { "n": 8, "passed": 1, "partial": 5, "failed": 2, "score": 7, "issues": { "content": 3, "terms": 3, "format": 2, "numbers": 2, "display": 1, "quote": 1 }, "median_ms": 555.97 }, "v26": { "n": 8, "passed": 1, "partial": 5, "failed": 2, "score": 7, "issues": { "content": 3, "terms": 3, "format": 2, "numbers": 2, "display": 1, "quote": 1 }, "median_ms": 376.32 }, "v27": { "n": 8, "passed": 1, "partial": 5, "failed": 2, "score": 7, "issues": { "content": 3, "terms": 2, "numbers": 2, "display": 1, "punctuation": 1, "format": 1, "quote": 1 }, "median_ms": 430.05 }, "v28": { "n": 8, "passed": 0, "partial": 6, "failed": 2, "score": 6, "issues": { "content": 3, "terms": 3, "format": 2, "numbers": 2, "display": 1, "quote": 1 }, "median_ms": 450.54 }, "v29": { "n": 8, "passed": 1, "partial": 4, "failed": 3, "score": 6, "issues": { "content": 3, "numbers": 2, "terms": 1, "repetition": 1, "format": 1, "fidelity": 1, "quote": 1 }, "median_ms": 357.47 }, "v30": { "n": 8, "passed": 0, "partial": 5, "failed": 3, "score": 5, "issues": { "content": 4, "format": 3, "terms": 3, "numbers": 2, "display": 1, "quote": 1 }, "median_ms": 384.02 }, "v31": { "n": 8, "passed": 1, "partial": 4, "failed": 3, "score": 6, "issues": { "content": 3, "format": 2, "numbers": 2, "display": 1, "terms": 1, "punctuation": 1, "fidelity": 1, "quote": 1 }, "median_ms": 392.52 }, "v32": { "n": 8, "passed": 0, "partial": 7, "failed": 1, "score": 7, "issues": { "content": 4, "terms": 3, "numbers": 2, "format": 2, "quote": 1 }, "median_ms": 457.01 }, "v33": { "n": 8, "passed": 1, "partial": 5, "failed": 2, "score": 7, "issues": { "content": 3, "format": 3, "terms": 3, "numbers": 2, "display": 1, "quote": 1 }, "median_ms": 401.13 }, "v34": { "n": 8, "passed": 1, "partial": 4, "failed": 3, "score": 6, "issues": { "content": 3, "numbers": 2, "terms": 1, "punctuation": 1, "fidelity": 1, "format": 1, "quote": 1 }, "median_ms": 330.91 }, "v35": { "n": 8, "passed": 1, "partial": 5, "failed": 2, "score": 7, "issues": { "content": 3, "terms": 3, "format": 2, "numbers": 2, "display": 1, "quote": 1 }, "median_ms": 363.79 }, "v36": { "n": 8, "passed": 0, "partial": 6, "failed": 2, "score": 6, "issues": { "content": 4, "format": 3, "terms": 3, "numbers": 2, "display": 1, "quote": 1 }, "median_ms": 483.96 }, "v37": { "n": 8, "passed": 0, "partial": 5, "failed": 3, "score": 5, "issues": { "content": 4, "terms": 3, "numbers": 2, "format": 2, "display": 1, "quote": 1 }, "median_ms": 462.57 }, "v38": { "n": 8, "passed": 0, "partial": 6, "failed": 2, "score": 6, "issues": { "content": 4, "format": 3, "terms": 3, "numbers": 2, "display": 1, "quote": 1 }, "median_ms": 456.38 }, "v39": { "n": 8, "passed": 1, "partial": 4, "failed": 3, "score": 6, "issues": { "content": 3, "terms": 3, "numbers": 2, "display": 1, "quote": 1 }, "median_ms": 376.98 }, "v40": { "n": 8, "passed": 0, "partial": 5, "failed": 3, "score": 5, "issues": { "content": 4, "terms": 3, "format": 2, "numbers": 2, "display": 1, "quote": 1 }, "median_ms": 390.16 }, "v41": { "n": 8, "passed": 0, "partial": 3, "failed": 5, "score": 3, "issues": { "format": 4, "content": 4, "numbers": 2, "terms": 2, "display": 1, "repetition": 1, "quote": 1 }, "median_ms": 388.22 }, "v42": { "n": 8, "passed": 1, "partial": 3, "failed": 4, "score": 5, "issues": { "repetition": 4, "format": 3, "content": 3, "numbers": 2, "display": 1, "terms": 1, "quote": 1 }, "median_ms": 648.66 }, "v43": { "n": 8, "passed": 1, "partial": 3, "failed": 4, "score": 5, "issues": { "content": 3, "format": 2, "repetition": 2, "terms": 2, "numbers": 2, "display": 1, "fidelity": 1 }, "median_ms": 526.79 }, "v44": { "n": 8, "passed": 0, "partial": 2, "failed": 6, "score": 2, "issues": { "content": 3, "fidelity": 2, "numbers": 2, "repetition": 2, "display": 1, "terms": 1, "quote": 1, "format": 1 }, "median_ms": 613.15 }, "v45": { "n": 8, "passed": 0, "partial": 6, "failed": 2, "score": 6, "issues": { "format": 5, "content": 4, "terms": 3, "numbers": 2, "display": 1, "quote": 1 }, "median_ms": 408.87 }, "v46": { "n": 8, "passed": 0, "partial": 4, "failed": 4, "score": 4, "issues": { "format": 5, "content": 3, "repetition": 3, "terms": 3, "display": 1, "quote": 1, "numbers": 1 }, "median_ms": 701.54 }, "v47": { "n": 8, "passed": 1, "partial": 3, "failed": 4, "score": 5, "issues": { "content": 3, "repetition": 2, "terms": 2, "display": 1, "numbers": 1, "fidelity": 1 }, "median_ms": 439.07 }, "v48": { "n": 8, "passed": 1, "partial": 3, "failed": 4, "score": 5, "issues": { "content": 3, "format": 2, "terms": 2, "numbers": 2, "display": 1, "fidelity": 1, "repetition": 1, "quote": 1 }, "median_ms": 539.63 }, "v49": { "n": 8, "passed": 0, "partial": 6, "failed": 2, "score": 6, "issues": { "content": 4, "format": 3, "terms": 3, "numbers": 2, "display": 1, "repetition": 1 }, "median_ms": 426.54 }, "v50": { "n": 8, "passed": 1, "partial": 4, "failed": 3, "score": 6, "issues": { "content": 3, "terms": 2, "numbers": 2, "display": 1, "fidelity": 1, "quote": 1 }, "median_ms": 408.79 } }, "finalists": [ "v32", "v35", "v26", "v18", "v33", "v22" ], "combined": { "v32": { "n": 20, "passed": 3, "partial": 16, "failed": 1, "score": 22, "issues": { "terms": 8, "numbers": 5, "content": 4, "format": 2, "quote": 1, "display": 1 }, "median_ms": 402.15 }, "v35": { "n": 20, "passed": 3, "partial": 15, "failed": 2, "score": 21, "issues": { "terms": 7, "numbers": 5, "content": 3, "format": 2, "display": 2, "quote": 1, "fidelity": 1, "punctuation": 1 }, "median_ms": 365.23 }, "v26": { "n": 20, "passed": 3, "partial": 13, "failed": 4, "score": 19, "issues": { "terms": 8, "numbers": 5, "content": 3, "format": 2, "display": 2, "quote": 1, "punctuation": 1 }, "median_ms": 359.3 }, "v18": { "n": 20, "passed": 3, "partial": 14, "failed": 3, "score": 20, "issues": { "terms": 8, "numbers": 5, "content": 4, "display": 2, "format": 2, "quote": 1, "punctuation": 1 }, "median_ms": 365.4 }, "v33": { "n": 20, "passed": 4, "partial": 11, "failed": 5, "score": 19, "issues": { "terms": 7, "format": 6, "numbers": 5, "content": 3, "display": 2, "quote": 1, "fidelity": 1 }, "median_ms": 370.08 }, "v22": { "n": 20, "passed": 3, "partial": 14, "failed": 3, "score": 20, "issues": { "terms": 8, "numbers": 5, "content": 4, "display": 2, "format": 2, "quote": 1, "punctuation": 1 }, "median_ms": 389.05 } }, "holdout": { "n": 30, "passed": 3, "partial": 24, "failed": 3, "score": 30, "issues": { "numbers": 9, "content": 7, "terms": 6, "punctuation": 4, "fidelity": 3, "display": 3, "quote": 2, "format": 2 }, "median_ms": 354.24 }, "full": { "qwen35-2b": { "n": 50, "passed": 6, "partial": 40, "failed": 4, "score": 52, "issues": { "terms": 14, "numbers": 14, "content": 11, "display": 4, "punctuation": 4, "format": 4, "fidelity": 3, "quote": 3 }, "median_ms": 358.94 }, "gemma4-e2b": { "n": 50, "passed": 15, "partial": 18, "failed": 17, "score": 48, "issues": { "numbers": 10, "content": 8, "fidelity": 7, "display": 6, "terms": 6, "punctuation": 3, "quote": 2 }, "median_ms": 350.73 } } }