A Developer’s Guide to OmniParseBench: Leaderboard Benchmarks and OCR Testing

In this tutorial, we implement OmniParseBench, Datalab’s open benchmark for document parsers, which grades a parser’s output with 16,288 small pass-or-fail tests across 2,937 pages, from scanned forms and handwriting to arXiv math and dense tables. Rather than trust the leaderboard, we rebuild it from the verdicts Datalab released for 27 parsers, after feeding the scorer a hand-made page to see which mistakes flip a verdict. We then measure ranking certainty, slice it by document type, test whether the two models the card names as helping build tests behave differently on tests a model corrected, and run Tesseract and Indra OCR ourselves on a 50-page sample. Along the way, we find that the top two parsers are statistically tied, the leader changes on most slices, and one launch figure doesn’t match the released files. Copy CodeCopiedUse a different Browser

import os
import re
import sys
import json
import time
import html
import shutil
import platform
import traceback
import subprocess
import collections
import urllib.request
from pathlib import Path
 
RESULTS = {}          # step -> one-line result, for the summary table
CLAIMS = []           # (step, claim, held)
CURRENT = ["1"]
 
 
def banner(title):
    print("\n" + "=" * 78)
    print(title)
    print("=" * 78)
 
 
def section(name):
    def wrap(fn):
        def run(*a, **kw):
            CURRENT[0] = name.split(".")[0]
            banner(name)
            t0 = time.time()
            try:
                out = fn(*a, **kw)
                RESULTS[name] = out if isinstance(out, str) else "ok"
                return out
            except Exception as e:
                RESULTS[name] = f"SKIPPED / FAILED -> {type(e).__name__}: {e}"
                print(f"\n[!] {name} did not complete: {type(e).__name__}: {e}")
                traceback.print_exc(limit=3)
                return None
            finally:
                print(f"\n[{time.time() - t0:.1f} s]")
        return run
    return wrap
 
 
def claim(text, held):
    CLAIMS.append((CURRENT[0], text, bool(held)))
    return held
 
 
banner("1. Install omni-parse-bench 1.0.0rc1 and Tesseract, and pin the dataset revision")
SYSTEM = platform.system()
OPB_VERSION = "1.0.0rc1"        # a pre-release: a bare `pip install omni-parse-bench` resolves to 0.2.0, whose API differs
REPO = "datalab-to/omni_parse_bench"
REV = "961c0f89a2da701f5ce53afbd5e80aa469b0369a"   # the dataset with the published verdicts of 27 parsers
INDRA = ["indra-ocr==0.3.2", "numpy==2.5.4", "onnxruntime==1.31.0", "opencv-python-headless==5.0.0.93",
         "pypdfium2==5.14.0", "huggingface_hub==2.2.0"]   # the pins of the package's own Indra server image
WORK = Path(os.environ.get("OPB_WORK") or ("/content/opb" if os.path.isdir("/content") else "opb_work")).resolve()
WORK.mkdir(parents=True, exist_ok=True)
subprocess.run([sys.executable, "-m", "pip", "install", "-q", f"omni-parse-bench[benchmark]=={OPB_VERSION}", "uv"], check=True)
if SYSTEM == "Linux" and shutil.which("tesseract") is None:
    scripts = ("arab armn beng cans cher cyrl deva ethi frak geor grek gujr guru hang hang-vert hans hans-vert hant "
               "hant-vert hebr jpan jpan-vert knda khmr laoo latn mlym mymr orya sinh syrc taml telu thaa thai tibt viet").split()
    subprocess.run(["apt-get", "-qq", "update"], check=True)
    subprocess.run(["apt-get", "-qq", "install", "-y", "tesseract-ocr", "tesseract-ocr-eng", "tesseract-ocr-osd"]
                   + [f"tesseract-ocr-script-{s}" for s in scripts], check=True, stdout=subprocess.DEVNULL)
 
import numpy as np
import pandas as pd
import importlib.metadata as md
from huggingface_hub import snapshot_download
from omni_parse_bench import benchmark, data, harness, markdown, vocab
from omni_parse_bench.scorers import unit_tests
 
print(f"omni-parse-bench {md.version('omni-parse-bench')}, Python {platform.python_version()}, {SYSTEM} {platform.machine()}, "
      f"{os.cpu_count()} CPU cores")
if SYSTEM == "Linux":
    print("os:", open("/etc/os-release").read().split('PRETTY_NAME="')[1].split('"')[0])
tess = subprocess.run(["tesseract", "--version"], capture_output=True, text=True)
print("tesseract:", (tess.stdout or tess.stderr).splitlines()[0])
TESS_MAJOR = int((tess.stdout or tess.stderr).split()[1].split(".")[0])
print("test families:", {f: [t for t, g in vocab.FAMILIES.items() if g == f] for f in dict.fromkeys(vocab.FAMILIES.values())})
print(f"dataset {REPO} @ {REV[:7]}; work folder {WORK}")
RESULTS["1. Install omni-parse-bench 1.0.0rc1 and Tesseract, and pin the dataset revision"] = (
    f"omni-parse-bench {md.version('omni-parse-bench')}, tesseract {(tess.stdout or tess.stderr).split()[1]}, dataset @ {REV[:7]}")

We install omni-parse-bench pinned to 1.0.0rc1, because a bare pip install still resolves to 0.2.0, an older release with a different API. The benchmark extra brings the parser adapters, the Hugging Face client, and pyarrow for the manifest, and on Colab we add Tesseract with its script models through apt. We also pin the dataset to one commit, since the repository changes as parsers are added. Printing the families shows the structure the benchmark rests on: present, order, and repeat tests form the text family; table_cell forms the tables family; and layout_kind forms the layout family. Copy CodeCopiedUse a different Browser

@section("2. How the scorer decides: one hand-built page, five test types, one change at a time")
def scorer_anatomy():
    T = lambda i, tt, out, tags, args, st=None: {"id": i, "test_type": tt, "output_type": out, "derivation": "authored",
                                                 "tags": tags, "args": args, "settings": st or {}}
    exact = lambda s: {"text": s, "max_diffs": 0}
    gold = [
        T("title", "present", "html", ["heading"], {"content": exact("Quarterly Report")}),
        T("total", "present", "html", ["eng"], {"content": exact("Total due")}),
        T("growth", "present", "html", ["eng"], {"content": {"number": "-4.8%"}}),
        T("amount", "present", "html", ["eng"], {"content": {"number": "1,240.00"}}),
        T("eq_ratio", "present", "html", ["math"], {"content": {"latex": r"\alpha_{1} + \beta"}}),
        T("eq_half", "present", "html", ["math"], {"content": {"latex": r"\frac{1}{2} \to x"}}),
        T("order", "order", "html", ["eng"], {"before": exact("Introduction: this report covers the second quarter."),
                                              "after": exact("Conclusion: the plan holds.")}),
        T("repeat", "repeat", "html", ["eng"], {"content": exact("Revenue")}, {"expected_count": 2}),
        T("cell_top", "table_cell", "html", ["table"], {"cell": exact("1,240"), "top_heading": exact("2024")}),
        T("cell_left", "table_cell", "html", ["table"], {"cell": exact("1,240"), "left_heading": exact("Income")}),
        T("cell_price", "table_cell", "html", ["table"], {"cell": exact("9.50"), "top_heading": exact("Price"),
                                                          "left": exact("Widget")}),
        T("block_title", "layout_kind", "blocks", ["heading"],
          {"content": dict(exact("Quarterly Report"), place=[[0.10, 0.04, 0.90, 0.09]])}, {"kind": "heading"}),
        T("block_total", "layout_kind", "blocks", ["eng"],
          {"content": dict(exact("Total due"), place=[[0.10, 0.12, 0.90, 0.16]])}, {"kind": "text"}),
    ]
    gold_file = WORK / "fixture" / "gold" / "demo-p1.json"           # a gold file is named after its page
    gold_file.parent.mkdir(parents=True, exist_ok=True)
    gold_file.write_text(json.dumps({"tests": gold}, indent=1))
    page = data.Page(page_id="demo-p1", document="demo", suite="demo", doc_path=WORK / "fixture" / "demo-p1.pdf",
                     gt_path=gold_file, outputs=("blocks", "html"))
    tests = unit_tests.gold(page)
 
    table_a = ("20232024Income1,0001,240"
               "Costs600700")
    table_b = "ProductPriceWidget9.50"
    md_table_b = "| Product | Price |\n|---|---|\n| Widget | 9.50 |"
    parts = {"title": "Quarterly Report", "total": "Total due: 1,240.00 USD.",
             "growth": "Growth was -4.8% year on year.", "ratio": r"Ratio: 1+ holds.",
             "half": r"Half: 12x in the model.",
             "intro": "Introduction: this report covers the second quarter.",
             "revenue": "Revenue is shown in thousands. Revenue grew in the second quarter.",
             "table_a": table_a, "conclusion": "Conclusion: the plan holds.", "table_b": table_b}
    blocks = [{"label": "heading", "bbox": [0.10, 0.04, 0.90, 0.09], "text": "Quarterly Report"},
              {"label": "text", "bbox": [0.10, 0.12, 0.90, 0.16], "text": "Total due: 1,240.00 USD."},
              {"label": "table", "bbox": [0.10, 0.40, 0.90, 0.60], "text": "Income 1,000 1,240 Costs 600 700"}]
 
    def pred(change=None, order=None, new_blocks=None):
        p = dict(parts, **(change or {}))
        return {"html": "\n".join(p[k] for k in (order or list(parts))), "blocks": blocks if new_blocks is None else new_blocks}
 
    base = {v.id: v.result for v in unit_tests.score(pred(), tests)}
    print(f"base prediction: {sum(r == 'pass' for r in base.values())} of {len(tests)} tests pass\n")
    swapped = ["title", "total", "growth", "ratio", "half", "conclusion", "intro", "revenue", "table_a", "table_b"]
    variants = [
        ("flip", "a typo: 'Total dve'", pred({"total": "Total dve: 1,240.00 USD."}), ()),
        ("flip", "the minus sign dropped: 4.8%", pred({"growth": "Growth was 4.8% year on year."}), ()),
        ("flip", "another symbol: alpha_2", pred({"ratio": r"Ratio: 2+ holds."}), ()),
        ("flip", "conclusion before introduction", pred(order=swapped), ()),
        ("flip", "'Revenues' for the second 'Revenue'",
         pred({"revenue": "Revenue is shown in thousands. Revenues grew in the second quarter."}), ()),
        ("flip", "2023 and 2024 headings swapped", pred({"table_a": table_a.replace("20232024",
                                                                                     "20242023")}), ()),
        ("flip", "row label 'Margin' for 'Income'", pred({"table_a": table_a.replace("Income", "Margin")}), ()),
        ("flip", "a markdown table left in the html", pred({"table_b": md_table_b}), ()),
        ("flip", "the title's block labelled text", pred(new_blocks=[dict(blocks[0], label="text")] + blocks[1:]), ()),
        ("control", r"\dfrac12\rightarrow for \frac{1}{2}\to", pred({"half": r"Half: $\dfrac12\rightarrow x$ in the model."}), ()),
        ("control", "'TOTAL DUE' and '1240.00'", pred({"total": "TOTAL DUE: 1240.00 USD."}), ()),
        ("control", "markdown table through markdown.to_html", pred({"table_b": markdown.to_html(md_table_b)}), ()),
        ("unsupported", "no blocks, declared unsupported", {"html": pred()["html"]}, ("blocks",)),
    ]
    as_expected = 0
    for kind, name, p, unsupported in variants:
        changed = [v for v in unit_tests.score(p, tests, unsupported) if v.result != base[v.id]]
        if kind == "flip":
            ok = len(changed) == 1 and changed[0].result == "fail"
        elif kind == "control":
            ok = not changed
        else:
            ok = bool(changed) and all(v.result == "unsupported" for v in changed)
        as_expected += ok
        shown = "; ".join(f"{v.id} -> {v.result}" + (f" ({v.why})" if v.result == "fail" else "") for v in changed) or "no verdict changes"
        print(f"{kind:11s} {name:42s} {shown}")
    s = unit_tests.summarize(unit_tests.lines(unit_tests.score(variants[0][2], tests), tests, page))
    print(f"\nsummary of the typo variant: headline {s['headline']:.4f} = mean of "
          + ", ".join(f"{k} {v['score']:.3f} ({v['tests']} tests)" for k, v in s["families"].items()))
    claim("every one-change variant flips exactly one verdict to fail, and no control flips any", as_expected == len(variants))
    return f"{as_expected} of {len(variants)} hand-built variants behave as expected (9 flips, 3 controls, 1 unsupported