#!/usr/bin/env python3
"""Measure char count, byte count, and token counts for the three benchmark samples.

Run:
    tmp/markdown-vs-html-benchmark/venv/bin/python tmp/markdown-vs-html-benchmark/measure.py
"""
from pathlib import Path
import tiktoken

SAMPLES_DIR = Path(__file__).parent
FILES = ["sample.html", "sample.md", "sample.txt"]
ENCODINGS = ["cl100k_base", "o200k_base"]


def main() -> None:
    encoders = {name: tiktoken.get_encoding(name) for name in ENCODINGS}

    print(f"tiktoken version: {tiktoken.__version__}")
    print()
    header = (
        f"{'file':<12}"
        f"{'chars':>8}"
        f"{'bytes':>8}"
        + "".join(f"{name + ' tok':>18}" for name in ENCODINGS)
    )
    print(header)
    print("-" * len(header))

    for fname in FILES:
        path = SAMPLES_DIR / fname
        text = path.read_text(encoding="utf-8")
        chars = len(text)
        bytes_ = len(text.encode("utf-8"))
        token_cols = "".join(
            f"{len(enc.encode(text)):>18}" for enc in encoders.values()
        )
        print(f"{fname:<12}{chars:>8}{bytes_:>8}{token_cols}")


if __name__ == "__main__":
    main()