diff --git a/sci-ml/tokenizers/Manifest b/sci-ml/tokenizers/Manifest index 94c91136ea519..5d875be69a420 100644 --- a/sci-ml/tokenizers/Manifest +++ b/sci-ml/tokenizers/Manifest @@ -1,3 +1,6 @@ DIST tokenizers-0.22.0-crates.tar.xz 20740996 BLAKE2B fcd0fcd907ae8e9c9aa86eb31ab8aaef5767e9279692dc2c2218673a4b7be73795ca41f91c662e10bfd31eb91af1199e01f8fde1ceb69c189a9bb0c279196b47 SHA512 30bd9df1975ff6388708057e6266f981a8dc65664aab6dd81fb9f884f3631fd56c2ea8f1ac2872f2c566f8a656cf2f36ef1ee4fcebb2cbd74c6b0a0e1b14c373 DIST tokenizers-0.22.1.gh.tar.gz 1563505 BLAKE2B 798193afeef89a93fdd007d1ffa42113df0ed2d9de92c290a4789ab205cc6c34cf68ba62cb44e50606ec54473f05fbd405b295f9193c1c1f8af845c9ce3104c2 SHA512 1a25ee6b218232112f10bc1e082e71ed1960ab7fa62c7341b38cc5f5dc0a735cf35b3e90d4a1c1cc0aedb7c198182c2e559662459a668a2fef4e633096a1cccc +DIST tokenizers-0.22.2-crates.tar.xz 21576112 BLAKE2B af30c1a4d2226444104349e01d7a1b60738ac2ec9d98795bda0da03dd0090a138b2d6b5ba6446f4c1dcc2d031b9a4b9f6af2b6240b38c051c63b586b0cac3d00 SHA512 7eed65dbdf1127022ae2ff9858bdff00f21cadb3dd7af7dea7f46424cb294f0bbc8819a2b56bba3a0b04f9c6245bc4619a72feda9cf929766d08fcc148422af2 +DIST tokenizers-0.22.2.gh.tar.gz 1568453 BLAKE2B bc429661a13d63b403089a4b8ef2746b64ddbc0c63176ae4bff02d32f092bf780367570073abe85e43052af3a82b4bbb784e70d157ba0f14e82b8149c99fe50b SHA512 6df9f9e30cde94418a160190a8d31e272d768e9cc5321e38e820290c2164d55650f8904e4d58be9c5d5e2cc87a656a6bbd82f12e1828ff12384dd9e1bf1cfde4 DIST tokenizers-python-0.22.1-crates.tar.xz 14184544 BLAKE2B 718c7246fb77568c1aa54e168fce682ef3130e2d51f8b6b0ed5dea96f4dabedfc92e395ef083b514732b8d04fed1d5a5ad1ba4a9667eee0e993616bd5f03184d SHA512 1226e024ab7ad7dc06e191951e2ffa8d30fd575d867d949ad37ffd4fe6a7e491b66a2a767e5d0dbb4b55f8b8eff809baceb11950b811e4e5d685074507060e23 +DIST tokenizers-python-0.22.2-crates.tar.xz 9862444 BLAKE2B c4ca8223a0c5cf1644f6871b01f822993d2f765b6e8c88fc00c0a57b799a099c50cd1bde08794eede404e6b3e8ed6319b7db761fd7f43d23d795869911abd750 SHA512 a47dc3eac62a2a59f77f93cbe7444080dbe6d636ce0dadc260208e448c8b3173fa586df84069d98d4af146fe02e2465f87486c2cff25525ce61869d435a8d736 diff --git a/sci-ml/tokenizers/tokenizers-0.22.2.ebuild b/sci-ml/tokenizers/tokenizers-0.22.2.ebuild new file mode 100644 index 0000000000000..e6a565e17b29f --- /dev/null +++ b/sci-ml/tokenizers/tokenizers-0.22.2.ebuild @@ -0,0 +1,156 @@ +# Copyright 2023-2026 Gentoo Authors +# Distributed under the terms of the GNU General Public License v2 + +# Autogenerated by pycargoebuild 0.15.0 + +EAPI=8 + +DISTUTILS_USE_PEP517=maturin +PYTHON_COMPAT=( python3_{12..14} ) +DISTUTILS_EXT=1 +DISTUTILS_SINGLE_IMPL=1 +RUST_MIN_VER="1.82.0" + +CRATES=" +" + +inherit cargo distutils-r1 + +DESCRIPTION="Implementation of today's most used tokenizers" +HOMEPAGE="https://github.com/huggingface/tokenizers" +SRC_URI=" + https://github.com/huggingface/${PN}/archive/refs/tags/v${PV}.tar.gz + -> ${P}.gh.tar.gz + ${CARGO_CRATE_URIS} +" +if [[ ${PKGBUMPING} != ${PVR} ]]; then + SRC_URI+=" + https://dev.gentoo.org/~tupone/distfiles/${P}-crates.tar.xz + https://dev.gentoo.org/~tupone/distfiles/${PN}-python-${PV}-crates.tar.xz + " +fi + +LICENSE="Apache-2.0" +# Dependent crate licenses +LICENSE+=" + Apache-2.0 Apache-2.0-with-LLVM-exceptions BSD-2 BSD ISC MIT MPL-2.0 + Unicode-DFS-2016 +" +SLOT="0" +KEYWORDS="~amd64" + +RDEPEND="dev-libs/oniguruma" +BDEPEND=" + test? ( sci-ml/datasets[${PYTHON_SINGLE_USEDEP}] ) + $(python_gen_cond_dep ' + dev-python/setuptools-rust[${PYTHON_USEDEP}] + ') +" + +EPYTEST_PLUGINS=( ) +distutils_enable_tests pytest + +QA_FLAGS_IGNORED=".*/site-packages/tokenizers/.*so" + +src_unpack() { + cargo_src_unpack +} + +pkg_setup() { + python-single-r1_pkg_setup + rust_pkg_setup +} + +src_prepare() { + default + cd bindings/python + eapply "${FILESDIR}"/${PN}-0.21.2-test.patch + distutils-r1_src_prepare +} + +src_configure() { + cd tokenizers + cargo_src_configure + cd ../bindings/python + distutils-r1_src_configure +} + +src_compile() { + export RUSTONIG_SYSTEM_LIBONIG=1 + cd tokenizers + cargo_src_compile + cd ../bindings/python + distutils-r1_src_compile +} + +src_test() { + cd tokenizers + # Tests do not work + #cargo_src_test + cd ../bindings/python + local -x EPYTEST_IGNORE=( benches/test_tiktoken.py ) + local -x EPYTEST_DESELECT=( + tests/bindings/test_encoding.py::TestEncoding::test_char_to_token + tests/bindings/test_encoding.py::TestEncoding::test_char_to_word + tests/bindings/test_encoding.py::TestEncoding::test_invalid_truncate_direction + tests/bindings/test_encoding.py::TestEncoding::test_n_sequences + tests/bindings/test_encoding.py::TestEncoding::test_sequence_ids + tests/bindings/test_encoding.py::TestEncoding::test_token_to_chars + tests/bindings/test_encoding.py::TestEncoding::test_token_to_sequence + tests/bindings/test_encoding.py::TestEncoding::test_token_to_word + tests/bindings/test_encoding.py::TestEncoding::test_truncation + tests/bindings/test_encoding.py::TestEncoding::test_word_to_chars + tests/bindings/test_encoding.py::TestEncoding::test_word_to_tokens + tests/bindings/test_models.py::TestBPE::test_instantiate + tests/bindings/test_models.py::TestWordLevel::test_instantiate + tests/bindings/test_models.py::TestWordPiece::test_instantiate + tests/bindings/test_processors.py::TestByteLevelProcessing::test_processing + tests/bindings/test_tokenizer.py::TestAsyncTokenizer::test_async_methods_existence + tests/bindings/test_tokenizer.py::TestAsyncTokenizer::test_basic_encoding + tests/bindings/test_tokenizer.py::TestAsyncTokenizer::test_concurrency + tests/bindings/test_tokenizer.py::TestAsyncTokenizer::test_decode + tests/bindings/test_tokenizer.py::TestAsyncTokenizer::test_encode + tests/bindings/test_tokenizer.py::TestAsyncTokenizer::test_error_handling + tests/bindings/test_tokenizer.py::TestAsyncTokenizer::test_large_batch + tests/bindings/test_tokenizer.py::TestAsyncTokenizer::test_numpy_inputs + tests/bindings/test_tokenizer.py::TestAsyncTokenizer::test_various_input_formats + tests/bindings/test_tokenizer.py::TestAsyncTokenizer::test_performance_comparison + tests/bindings/test_tokenizer.py::TestAsyncTokenizer::test_with_special_tokens + tests/bindings/test_tokenizer.py::TestAsyncTokenizer::test_with_truncation_padding + tests/bindings/test_tokenizer.py::TestTokenizer::test_encode_add_special_tokens + tests/bindings/test_tokenizer.py::TestTokenizer::test_encode_formats + tests/bindings/test_tokenizer.py::TestTokenizer::test_encode_special_tokens + tests/bindings/test_tokenizer.py::TestTokenizer::test_decode_skip_special_tokens + tests/bindings/test_tokenizer.py::TestTokenizer::test_decode_stream_fallback + tests/bindings/test_tokenizer.py::TestTokenizer::test_from_pretrained + tests/bindings/test_tokenizer.py::TestTokenizer::test_from_pretrained_revision + tests/bindings/test_tokenizer.py::TestTokenizer::test_splitting + tests/bindings/test_trainers.py::TestUnigram::test_continuing_prefix_trainer_mismatch + tests/bindings/test_trainers.py::TestUnigram::test_train + tests/bindings/test_trainers.py::TestUnigram::test_train_parallelism_with_custom_pretokenizer + tests/documentation/test_pipeline.py::TestPipeline::test_bert_example + tests/documentation/test_pipeline.py::TestPipeline::test_pipeline + tests/documentation/test_quicktour.py::TestQuicktour::test_quicktour + tests/documentation/test_tutorial_train_from_iterators.py::TestTrainFromIterators::test_datasets + tests/documentation/test_tutorial_train_from_iterators.py::TestTrainFromIterators::test_gzip + tests/implementations/test_byte_level_bpe.py::TestByteLevelBPE::test_add_prefix_space + tests/implementations/test_byte_level_bpe.py::TestByteLevelBPE::test_basic_encode + tests/implementations/test_byte_level_bpe.py::TestByteLevelBPE::test_lowerspace + tests/implementations/test_byte_level_bpe.py::TestByteLevelBPE::test_multiprocessing_with_parallelism + tests/implementations/test_bert_wordpiece.py::TestBertWordPieceTokenizer::test_basic_encode + tests/implementations/test_bert_wordpiece.py::TestBertWordPieceTokenizer::test_multiprocessing_with_parallelism + tests/implementations/test_char_bpe.py::TestCharBPETokenizer::test_basic_encode + tests/implementations/test_char_bpe.py::TestCharBPETokenizer::test_decoding + tests/implementations/test_char_bpe.py::TestCharBPETokenizer::test_lowercase + tests/implementations/test_char_bpe.py::TestCharBPETokenizer::test_multiprocessing_with_parallelism + tests/test_serialization.py::TestSerialization::test_full_serialization_albert + tests/test_serialization.py::TestSerialization::test_str_big + ) + distutils-r1_src_test +} + +src_install() { + cd tokenizers + cd ../bindings/python + distutils-r1_src_install +}