Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
153 changes: 153 additions & 0 deletions .github/workflows/python-e2e-golden.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,153 @@
name: Python e2e & golden tests

# Two complementary suites for the 1.0 bindings rewrite (bindings/python/tests):
# e2e (tests/e2e) — transformers scenario tests. Run on the wheel
# transformers resolves (required green) and on the
# in-tree build (informational while incomplete).
# golden (tests/golden) — exact-output conformance against committed goldens
# generated from the released wheel. Required: the
# goldens must reproduce from the release. The
# in-tree run is informational while incomplete.

on:
push:
branches:
- main
paths-ignore:
- bindings/node/**
pull_request:
paths-ignore:
- bindings/node/**

env:
# CI runners have no GPU: resolve torch from the CPU index instead of
# pulling ~2 GB of CUDA wheels.
UV_TORCH_BACKEND: cpu

jobs:
e2e-release:
name: e2e on released tokenizers (must pass)
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

- name: Install uv
uses: astral-sh/setup-uv@v6

- name: Cache Hub downloads
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
with:
path: ~/.cache/huggingface
key: e2e-hub-${{ hashFiles('bindings/python/tests/e2e/*') }}
restore-keys: e2e-hub-

- name: Run the e2e suite
working-directory: ./bindings/python
env:
HF_TOKEN: ${{ secrets.HF_TOKEN }}
run: make e2e-release

golden-freshness:
name: goldens reproduce from the released wheel (must pass)
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

- name: Install uv
uses: astral-sh/setup-uv@v6

- name: Cache test data
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
with:
path: tokenizers/data
key: golden-data-${{ hashFiles('tokenizers/Makefile', 'tokenizers/tk-encode/examples/bench_models.json') }}

- name: Download model tokenizers and fixture corpora
working-directory: tokenizers
env:
HF_TOKEN: ${{ secrets.HF_TOKEN }}
run: make bench-models fixtures HF="uvx --from huggingface_hub hf"

# Regenerating must reproduce the committed goldens byte for byte. A
# difference means either an uncommitted regen, or a new tokenizers
# release that changed behavior — both need a human decision.
- name: Regenerate and diff the goldens
working-directory: ./bindings/python
run: |
make golden-regen
if [ -n "$(git status --porcelain -- tests/golden/goldens)" ]; then
git status --porcelain -- tests/golden/goldens
git diff -- tests/golden/goldens | head -100
echo "::error::committed goldens do not reproduce from the released wheel — run 'make golden-regen' and commit, or investigate the release"
exit 1
fi

in-tree:
name: e2e + golden on the in-tree build (informational)
runs-on: ubuntu-latest
continue-on-error: true
steps:
- name: Checkout repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

- name: Install Rust
uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable

- name: Cache cargo registry / git / target
uses: Swatinem/rust-cache@23869a5bd66c73db3c0ac40331f3206eb23791dc # v2.9.1
with:
workspaces: bindings/python
shared-key: python
cache-bin: false

- name: Install uv
uses: astral-sh/setup-uv@v6

- name: Cache test data
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
with:
path: tokenizers/data
key: golden-data-${{ hashFiles('tokenizers/Makefile', 'tokenizers/tk-encode/examples/bench_models.json') }}

- name: Cache Hub downloads
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
with:
path: ~/.cache/huggingface
key: e2e-hub-${{ hashFiles('bindings/python/tests/e2e/*') }}
restore-keys: e2e-hub-

- name: Download model tokenizers and fixture corpora
working-directory: tokenizers
env:
HF_TOKEN: ${{ secrets.HF_TOKEN }}
run: make bench-models fixtures HF="uvx --from huggingface_hub hf"

- name: Run the golden conformance layer
working-directory: ./bindings/python
run: |
set -o pipefail
make golden 2>&1 | tee golden.log

- name: Run the e2e suite
if: always()
working-directory: ./bindings/python
env:
HF_TOKEN: ${{ secrets.HF_TOKEN }}
run: |
set -o pipefail
make e2e 2>&1 | tee e2e.log

# The point of these runs while the rewrite is in progress: surface
# what still fails, without hunting through the logs.
- name: Post the failure overview to the step summary
if: always()
working-directory: ./bindings/python
run: |
for suite in golden e2e; do
echo "## ${suite} on the in-tree build"
echo '```'
sed -n '/short test summary info/,$p' "${suite}.log" 2>/dev/null || echo "no ${suite} results"
echo '```'
done >> "$GITHUB_STEP_SUMMARY"
6 changes: 4 additions & 2 deletions .github/workflows/python.yml
Original file line number Diff line number Diff line change
Expand Up @@ -170,11 +170,13 @@ jobs:
- name: Install uv
uses: astral-sh/setup-uv@v6

# Pinned: a ruff release can change the default rule set and break CI
# without any code change (0.16.0 did). Bump deliberately.
- name: Lint the Python sources with ruff
working-directory: ./bindings/python
run: |
uvx ruff check benches examples tests
uvx ruff format --check benches examples tests
uvx ruff@0.16.0 check benches examples tests
uvx ruff@0.16.0 format --check benches examples tests

audit:
name: Audit dependencies
Expand Down
4 changes: 4 additions & 0 deletions bindings/python/.gitignore
Original file line number Diff line number Diff line change
@@ -1,6 +1,10 @@
.venv/
.venv-ft/
.venv-e2e/
.venv-e2e-release/
.release/
e2e.log
golden.log
python_bench.json
python_bench.md
.smoke/
52 changes: 49 additions & 3 deletions bindings/python/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,11 @@ PYTHON := .venv/bin/python
dev: .venv
. .venv/bin/activate && maturin develop --release

# huggingface_hub backs from_pretrained (the wheel's optional `hub` extra);
# the pretrained tests and example 01 need it.
.venv:
uv venv .venv
uv pip install --python $(PYTHON) maturin numpy pytest ruff mypy
uv pip install --python $(PYTHON) maturin numpy pytest ruff mypy huggingface_hub

# Regenerate the .pyi stubs from the built extension. Run after `make dev`.
.PHONY: stubs
Expand All @@ -19,10 +21,17 @@ test: dev
$(PYTHON) -m pytest -q
$(PYTHON) -m mypy.stubtest tokenizers --allowlist stubtest_allowlist.txt

# Examples 01-03 read this corpus; fetch it through the tokenizers repo's
# data rule, pointing it at the dev venv's `hf` CLI so nothing global is
# needed.
TOKENIZERS_DIR := ../../tokenizers
$(TOKENIZERS_DIR)/data/big.txt: | .venv
$(MAKE) -C $(TOKENIZERS_DIR) data/big.txt HF=$(abspath .venv/bin/hf)

# Run the end-to-end examples. `datasets` (for 06) is installed on demand so
# `make dev` stays lean; 06 downloads wikitext-2 (~12 MB) on first run.
.PHONY: examples
examples: dev
examples: dev $(TOKENIZERS_DIR)/data/big.txt
uv pip install --python $(PYTHON) datasets
$(PYTHON) examples/01_pretrained.py
$(PYTHON) examples/02_train_and_encode.py
Expand All @@ -31,6 +40,43 @@ examples: dev
$(PYTHON) examples/05_train_bytelevel_bpe.py
$(PYTHON) examples/06_train_with_datasets.py

# End-to-end tests (tests/e2e): the transformers scenario suite, two builds.
# e2e — the in-tree build; failures list what the rewrite still misses
# e2e-release — the tokenizers wheel transformers resolves; must always pass
# Dedicated venvs: transformers+torch are heavy, and each venv carries its own
# tokenizers. The dev venv also needs transformers' import-time tokenizers pin
# lifted, or it refuses our 1.0.0-dev version (see relax_transformers_pin.py).
.venv-e2e:
uv venv $@ --python 3.12
uv pip install --python $@/bin/python -r tests/e2e/requirements.txt maturin
$@/bin/python tests/e2e/relax_transformers_pin.py

.PHONY: e2e
e2e: .venv-e2e
. .venv-e2e/bin/activate && maturin develop --release
.venv-e2e/bin/python -m pytest tests/e2e -q -ra --continue-on-collection-errors

.venv-e2e-release:
uv venv $@ --python 3.12
uv pip install --python $@/bin/python -r tests/e2e/requirements.txt

.PHONY: e2e-release
e2e-release: .venv-e2e-release
.venv-e2e-release/bin/python -m pytest tests/e2e -q

# Golden conformance layer (tests/golden): exact encode/decode outputs on the
# data/ fixtures, diffed against committed goldens. The goldens are only ever
# regenerated from the released wheel (golden-regen) — never hand-edited.
# `golden` is opt-in (env gate) while the rewrite is incomplete; drop the gate
# once it runs green so plain pytest enforces conformance everywhere.
.PHONY: golden
golden: dev
TOKENIZERS_GOLDEN=1 $(PYTHON) -m pytest tests/golden -q -ra

.PHONY: golden-regen
golden-regen: .venv .release
PYTHONPATH=.release $(PYTHON) tests/golden/generate.py

# The released PyPI wheel shares our package name, so it lives in its own
# directory that the bench subprocess puts on PYTHONPATH.
.release: | .venv
Expand All @@ -50,4 +96,4 @@ lint: .venv

.PHONY: clean
clean:
rm -rf .venv .release target tools/stub-gen/target
rm -rf .venv .venv-e2e .venv-e2e-release .release target tools/stub-gen/target
14 changes: 9 additions & 5 deletions bindings/python/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -136,11 +136,15 @@ per-version with `maturin build --no-default-features`.
Other targets:

```sh
make test # pytest suite in tests/
make examples # run the end-to-end examples (needs ../../tokenizers/data)
make bench # benchmark against the released tokenizers wheel from PyPI
make stubs # regenerate the .pyi type stubs from the built extension
make lint # cargo fmt + clippy, ruff over the python sources
make test # pytest suite in tests/
make examples # run the end-to-end examples (needs ../../tokenizers/data)
make bench # benchmark against the released tokenizers wheel from PyPI
make e2e # transformers end-to-end suite (tests/e2e) on this build
make e2e-release # same suite on the released wheel — must always pass
make golden # exact-output conformance vs the committed goldens (tests/golden)
make golden-regen # regenerate the goldens from the released wheel
make stubs # regenerate the .pyi type stubs from the built extension
make lint # cargo fmt + clippy, ruff over the python sources
```

The examples and the benchmark read test data from `../../tokenizers/data`.
Expand Down
25 changes: 14 additions & 11 deletions bindings/python/benches/bench_vs_release.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,6 @@
from pathlib import Path

import numpy as np

import tokenizers

# Keep in sync with fixture_bench.rs (CHUNK_BYTES, MAX_CHUNKS).
Expand Down Expand Up @@ -117,15 +116,15 @@ def bench_release_side(models: list[dict], fixtures: list[dict], iters: int) ->
for fixture in fixtures:
chunks = fixture["chunks"]
encoded = tok.encode_batch_fast(chunks, add_special_tokens=False)
t = timed(lambda: tok.encode_batch_fast(chunks, add_special_tokens=False), iters)
t = timed(lambda tok=tok, chunks=chunks: tok.encode_batch_fast(chunks, add_special_tokens=False), iters)
rows.append(
{
"mbps": fixture["bytes"] / t / 1e6,
"digests": digests([e.ids for e in encoded]),
}
)
os.environ["TOKENIZERS_PARALLELISM"] = "true"
t = timed(lambda: tok.encode_batch_fast(all_chunks, add_special_tokens=False), iters)
t = timed(lambda tok=tok: tok.encode_batch_fast(all_chunks, add_special_tokens=False), iters)
out["models"][model["name"]] = {"fixtures": rows, "multi_thread_mbps": nbytes / t / 1e6}
return out

Expand All @@ -136,7 +135,7 @@ def bench_local_side(tok, fixtures: list[dict], release_row: dict, iters: int) -
for fixture, rel in zip(fixtures, release_row["fixtures"], strict=True):
chunks = fixture["chunks"]
encoded = tok.encode_batch_ids(chunks, add_special_tokens=False)
t = timed(lambda: tok.encode_batch_ids(chunks, add_special_tokens=False), iters)
t = timed(lambda chunks=chunks: tok.encode_batch_ids(chunks, add_special_tokens=False), iters)
mbps = fixture["bytes"] / t / 1e6
row["fixtures"].append(
{
Expand Down Expand Up @@ -169,13 +168,17 @@ def render_markdown(report: dict) -> str:
lines = [
f"## Python bindings: this branch vs `tokenizers` {report['release_version']} (PyPI)",
"",
f"{report['fixture_count']} fixtures (~10 KiB chunks, ≤100/fixture), median of "
f"{report['iters']} runs, {report['cpus']} CPUs. Single-thread numbers aggregate "
"all fixtures (speedup range = slowest…fastest fixture); multi-thread runs the "
"flattened corpus. Speedup >1 means this branch is faster.",
(
f"{report['fixture_count']} fixtures (~10 KiB chunks, ≤100/fixture), median of "
f"{report['iters']} runs, {report['cpus']} CPUs. Single-thread numbers aggregate "
"all fixtures (speedup range = slowest…fastest fixture); multi-thread runs the "
"flattened corpus. Speedup >1 means this branch is faster."
),
"",
"| model | ids | branch 1t (MB/s) | release 1t (MB/s) | speedup 1t (range) "
"| branch mt (MB/s) | release mt (MB/s) | speedup mt |",
(
"| model | ids | branch 1t (MB/s) | release 1t (MB/s) | speedup 1t (range) "
"| branch mt (MB/s) | release mt (MB/s) | speedup mt |"
),
"|---|---|---|---|---|---|---|---|",
]
for row in report["models"]:
Expand Down Expand Up @@ -224,7 +227,7 @@ def main() -> int:
f"`uv pip install --target {args.release_dir} tokenizers`"
)

models = json.load(open(args.manifest)) if args.manifest else DEFAULT_MODELS
models = json.loads(args.manifest.read_text()) if args.manifest else DEFAULT_MODELS
for model in models:
model["path"] = str(args.data_dir / model.get("file", model["name"] + ".json"))
missing = [m["name"] for m in models if not Path(m["path"]).is_file()]
Expand Down
1 change: 0 additions & 1 deletion bindings/python/examples/01_pretrained.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@
from pathlib import Path

import numpy as np

from tokenizers import Tokenizer, TokenizersError

DATA = Path(__file__).resolve().parents[3] / "tokenizers" / "data"
Expand Down
1 change: 0 additions & 1 deletion bindings/python/examples/02_train_and_encode.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
from pathlib import Path

import numpy as np

from tokenizers import AddedToken, Tokenizer, models, normalizers, pre_tokenizers, trainers

DATA = Path(__file__).resolve().parents[3] / "tokenizers" / "data"
Expand Down
1 change: 0 additions & 1 deletion bindings/python/examples/06_train_with_datasets.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
Needs the `datasets` package; downloads wikitext-2 (~12 MB) on first run."""

import datasets

from tokenizers import Tokenizer, models, normalizers, pre_tokenizers

tokenizer = Tokenizer(models.BPE())
Expand Down
1 change: 0 additions & 1 deletion bindings/python/tests/conftest.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
from pathlib import Path

import pytest

from tokenizers import Tokenizer, models, pre_tokenizers, trainers

DATA = Path(__file__).resolve().parents[3] / "tokenizers" / "data"
Expand Down
Empty file.
Loading
Loading