{
  "added_at": "2026-11-06",
  "category": "native-analysis",
  "description_en": "Binary BPE Tokenizer for Executable Analysis studies byte-pair encoding of ELF, PE, Mach-O and APK files for transformer-based binary analysis. The repository provides paper sources, 4K to 64K tokenizer vocabularies, Python evaluation scripts and a sample archive. The author reports 2.89 bytes per token for the 64K vocabulary.",
  "description_zh": "Binary BPE Tokenizer for Executable Analysis 研究面向 Transformer 二进制分析的 ELF、PE、Mach-O 和 APK 字节对编码。仓库包含论文源文件、4K 至 64K 词表、Python 评测脚本和样本压缩包。作者报告 64K 词表平均每个 token 对应 2.89 字节。",
  "entry_url": "https://awesomerea.org/resources/binary-bpe-tokenizer-research/",
  "entry_url_zh": "https://awesomerea.org/zh/resources/binary-bpe-tokenizer-research/",
  "evidence_url": "https://github.com/mjbommar/binary-tokenizer-paper/blob/7ef1ec7739a37437c229423847c04d9295c82a0e/README.md",
  "group": "learn",
  "id": "binary-bpe-tokenizer-research",
  "json_url": "https://awesomerea.org/resources/binary-bpe-tokenizer-research/record.json",
  "kind": "research",
  "markdown_url": "https://awesomerea.org/resources/binary-bpe-tokenizer-research/index.md",
  "markdown_url_zh": "https://awesomerea.org/zh/resources/binary-bpe-tokenizer-research/index.md",
  "name": "Binary BPE Tokenizer for Executable Analysis",
  "name_zh": "Binary BPE Tokenizer for Executable Analysis 可执行文件分析分词研究",
  "record_type": "resource",
  "relationship": "learning",
  "requirement_facets": [
    "prerequisites",
    "prerequisites",
    "license"
  ],
  "requirements": [
    "Python 3.10+ with uv; TeX Live 2023+ is required to compile the paper.",
    "Rust 1.70+ is optional for retraining the tokenizer from scratch.",
    "No licence file in the repository."
  ],
  "requirements_zh": [
    "需要 Python 3.10 或更高版本及 uv；编译论文还需要 TeX Live 2023 或更高版本。",
    "从头训练分词器可选用 Rust 1.70 或更高版本。",
    "仓库中没有许可证文件。"
  ],
  "targets": [
    "app",
    "binary"
  ],
  "url": "https://github.com/mjbommar/binary-tokenizer-paper",
  "verification": "source-reviewed",
  "verified_at": "2026-11-06"
}