{"enrichment":{"capability":"Tokenizers provides fast, production-grade implementations of modern NLP tokenizers (BPE, WordPiece, SentencePiece) with Rust-backed performance, supporting training, encoding, and full preprocessing pipelines.","verdict":"Tokenizers 0.23.1 is a mature, actively maintained library with no known vulnerabilities, permissive licensing, and broad platform support via prebuilt wheels. Medium install friction is offset by comprehensive wheel coverage and active development. Suitable for production NLP pipelines."},"id":"tokenizers","links":{"html":"https://skillfed.io/packages/tokenizers","md":"https://skillfed.io/packages/tokenizers.md","pypi":"https://pypi.org/project/tokenizers/"},"maintenance":{"status":"active"},"meta":{"latest_release":"2026-04-27","license_spdx":null,"license_treatment":"permissive","name":"tokenizers","python_support":"supports_current","summary":null},"popularity":{"tier":"top_1000"},"security":{"n_vulnerabilities":0},"version":"0.23.1"}