mirror of
https://github.com/gentoo-mirror/gentoo.git
synced 2026-09-24 04:59:14 -07:00
sci-ml/tokenizers: add 0.23.2
Signed-off-by: Alfredo Tupone <tupone@gentoo.org>
This commit is contained in:
Alfredo Tupone
@@ -4,6 +4,9 @@ DIST tokenizers-0.22.2-crates.tar.xz 21576112 BLAKE2B af30c1a4d2226444104349e01d
|
||||
DIST tokenizers-0.22.2.gh.tar.gz 1568453 BLAKE2B bc429661a13d63b403089a4b8ef2746b64ddbc0c63176ae4bff02d32f092bf780367570073abe85e43052af3a82b4bbb784e70d157ba0f14e82b8149c99fe50b SHA512 6df9f9e30cde94418a160190a8d31e272d768e9cc5321e38e820290c2164d55650f8904e4d58be9c5d5e2cc87a656a6bbd82f12e1828ff12384dd9e1bf1cfde4
|
||||
DIST tokenizers-0.23.1-crates.tar.xz 21626500 BLAKE2B 7d772e184af4c00e871357e9f146ce76c20dbd47b6e4e7416253fce464113181041b55cdd643436c1d9fc262298a3a6aed523ed10926d4a9c29246c26ccb3e15 SHA512 137e3892805fec33c19c9d005d1d065c5174f01973d6b6df528c8e6a349c6a1e00182b04a0df50b5880e32085e0af1ba0d167e441b8d8730c73650e863e5a527
|
||||
DIST tokenizers-0.23.1.gh.tar.gz 1613786 BLAKE2B 6b401812501fc3ced28a89264249b6b95b29346b25853f567d9980900f95e51a85ff116ffa93f55dc3f9cc134935d472ca683252fcdbeeb5f3cdda602c943c55 SHA512 4ede0677d0c61f00e15a9b9420b799d4ee1ae0af9510ee182b2eaa4d95d384629ad6c656d3fcda96a2d9388df836e33d8f04a242907b0901fa3aab17484fbcdd
|
||||
DIST tokenizers-0.23.2-crates.tar.xz 21774012 BLAKE2B 859d9574e322257c1de4c70ad1585303cef09674a886c53bbdcc43a677febaec44a5d97173120a4fd5f12b90dad8b7c2deb7b44c402314c879578ee419db19ce SHA512 a7517e4958e8a7b27e0de67cb1335f01f7061f9f4fde9f3268594282de6cd9c2eeb3560ea92d8bd19a3f0c9a0b4f1bd4e87122e382778784062c36bf7eae906a
|
||||
DIST tokenizers-0.23.2.gh.tar.gz 1475786 BLAKE2B 362ed1e83a8f4bf64f91168e81e7a5093f5a585de138c83f10bb2a4abce9c731dcc9cdf7870168b0b7055322a356805fdab87a2b87f3e75f6086402d8debe337 SHA512 bfdb092144f5ed8dcf51665ae72571c81d3194aa2fff8413b14ab4bc09664e24c89e0c4ecb72342bcf5cf3ec31b0c9fb3587e670a2b26b83ceb7144f00f6d4d7
|
||||
DIST tokenizers-python-0.22.1-crates.tar.xz 14184544 BLAKE2B 718c7246fb77568c1aa54e168fce682ef3130e2d51f8b6b0ed5dea96f4dabedfc92e395ef083b514732b8d04fed1d5a5ad1ba4a9667eee0e993616bd5f03184d SHA512 1226e024ab7ad7dc06e191951e2ffa8d30fd575d867d949ad37ffd4fe6a7e491b66a2a767e5d0dbb4b55f8b8eff809baceb11950b811e4e5d685074507060e23
|
||||
DIST tokenizers-python-0.22.2-crates.tar.xz 9862444 BLAKE2B c4ca8223a0c5cf1644f6871b01f822993d2f765b6e8c88fc00c0a57b799a099c50cd1bde08794eede404e6b3e8ed6319b7db761fd7f43d23d795869911abd750 SHA512 a47dc3eac62a2a59f77f93cbe7444080dbe6d636ce0dadc260208e448c8b3173fa586df84069d98d4af146fe02e2465f87486c2cff25525ce61869d435a8d736
|
||||
DIST tokenizers-python-0.23.1-crates.tar.xz 10988872 BLAKE2B 0bd16776118cc799edafd15abe5788ab67be6e4ec8c26c4b97556d09d949223121d18d6202cdd7fec4702674c958018705383be9b570b0c1f8133ab1f6189511 SHA512 9ab7173018cd80f191d05219e9e163ddde32451585c3f11a68be36c3403514ffb59b9ea307efe050b346b066e95d0b2fe54134b2b89f67e5032aa256340c93d8
|
||||
DIST tokenizers-python-0.23.2-crates.tar.xz 10401308 BLAKE2B 51405dd7107a9cd17bd468b9db6187d5cd3b92a80f96f5ca20eb7b58e7da8404e6d4e1187e184be2d64434ea3dc1d30b32059055c95a6b73b2b0e819d1a5b402 SHA512 2d84aaa18eec6b74b440549d60f94ef411554275519b2d7f71fa31700b839fcb177997223f73fdadc5fb629947de8554e5d7e82f09aeeecce4c1cf401a5dba46
|
||||
|
||||
155
sci-ml/tokenizers/tokenizers-0.23.2.ebuild
Normal file
155
sci-ml/tokenizers/tokenizers-0.23.2.ebuild
Normal file
@@ -0,0 +1,155 @@
|
||||
# Copyright 2023-2026 Gentoo Authors
|
||||
# Distributed under the terms of the GNU General Public License v2
|
||||
|
||||
# Autogenerated by pycargoebuild 0.15.0
|
||||
|
||||
EAPI=8
|
||||
|
||||
DISTUTILS_USE_PEP517=maturin
|
||||
PYTHON_COMPAT=( python3_{12..14} )
|
||||
DISTUTILS_EXT=1
|
||||
DISTUTILS_SINGLE_IMPL=1
|
||||
RUST_MIN_VER="1.87.0"
|
||||
|
||||
CRATES="
|
||||
"
|
||||
|
||||
inherit cargo distutils-r1
|
||||
|
||||
DESCRIPTION="Implementation of today's most used tokenizers"
|
||||
HOMEPAGE="https://github.com/huggingface/tokenizers"
|
||||
SRC_URI="
|
||||
https://github.com/huggingface/${PN}/archive/refs/tags/v${PV}.tar.gz
|
||||
-> ${P}.gh.tar.gz
|
||||
${CARGO_CRATE_URIS}
|
||||
"
|
||||
if [[ ${PKGBUMPING} != ${PVR} ]]; then
|
||||
SRC_URI+="
|
||||
https://dev.gentoo.org/~tupone/distfiles/${P}-crates.tar.xz
|
||||
https://dev.gentoo.org/~tupone/distfiles/${PN}-python-${PV}-crates.tar.xz
|
||||
"
|
||||
fi
|
||||
|
||||
LICENSE="Apache-2.0"
|
||||
# Dependent crate licenses
|
||||
LICENSE+="
|
||||
Apache-2.0 Apache-2.0-with-LLVM-exceptions BSD-2 BSD ISC MIT MPL-2.0
|
||||
Unicode-DFS-2016
|
||||
"
|
||||
SLOT="0"
|
||||
KEYWORDS="~amd64"
|
||||
|
||||
RDEPEND="dev-libs/oniguruma"
|
||||
BDEPEND="
|
||||
test? ( sci-ml/datasets[${PYTHON_SINGLE_USEDEP}] )
|
||||
$(python_gen_cond_dep '
|
||||
dev-python/setuptools-rust[${PYTHON_USEDEP}]
|
||||
')
|
||||
"
|
||||
|
||||
EPYTEST_PLUGINS=( )
|
||||
distutils_enable_tests pytest
|
||||
|
||||
QA_FLAGS_IGNORED=".*/site-packages/tokenizers/.*so"
|
||||
|
||||
src_unpack() {
|
||||
cargo_src_unpack
|
||||
}
|
||||
|
||||
pkg_setup() {
|
||||
python-single-r1_pkg_setup
|
||||
rust_pkg_setup
|
||||
}
|
||||
|
||||
src_prepare() {
|
||||
default
|
||||
cd bindings/python
|
||||
eapply "${FILESDIR}"/${PN}-0.21.2-test.patch
|
||||
distutils-r1_src_prepare
|
||||
}
|
||||
|
||||
src_configure() {
|
||||
cd tokenizers
|
||||
cargo_src_configure
|
||||
cd ../bindings/python
|
||||
distutils-r1_src_configure
|
||||
}
|
||||
|
||||
src_compile() {
|
||||
export RUSTONIG_SYSTEM_LIBONIG=1
|
||||
cd tokenizers
|
||||
cargo_src_compile
|
||||
cd ../bindings/python
|
||||
distutils-r1_src_compile
|
||||
}
|
||||
|
||||
src_test() {
|
||||
cd tokenizers
|
||||
# Tests do not work
|
||||
#cargo_src_test
|
||||
cd ../bindings/python
|
||||
local -x EPYTEST_IGNORE=( benches/test_tiktoken.py )
|
||||
local -x EPYTEST_DESELECT=(
|
||||
tests/bindings/test_encoding.py::TestEncoding::test_char_to_token
|
||||
tests/bindings/test_encoding.py::TestEncoding::test_char_to_word
|
||||
tests/bindings/test_encoding.py::TestEncoding::test_invalid_truncate_direction
|
||||
tests/bindings/test_encoding.py::TestEncoding::test_n_sequences
|
||||
tests/bindings/test_encoding.py::TestEncoding::test_sequence_ids
|
||||
tests/bindings/test_encoding.py::TestEncoding::test_token_to_chars
|
||||
tests/bindings/test_encoding.py::TestEncoding::test_token_to_sequence
|
||||
tests/bindings/test_encoding.py::TestEncoding::test_token_to_word
|
||||
tests/bindings/test_encoding.py::TestEncoding::test_truncation
|
||||
tests/bindings/test_encoding.py::TestEncoding::test_word_to_chars
|
||||
tests/bindings/test_encoding.py::TestEncoding::test_word_to_tokens
|
||||
tests/bindings/test_models.py::TestWordLevel::test_instantiate
|
||||
tests/bindings/test_models.py::TestWordPiece::test_instantiate
|
||||
tests/bindings/test_processors.py::TestByteLevelProcessing::test_processing
|
||||
tests/bindings/test_tokenizer.py::TestAsyncTokenizer::test_async_methods_existence
|
||||
tests/bindings/test_tokenizer.py::TestAsyncTokenizer::test_basic_encoding
|
||||
tests/bindings/test_tokenizer.py::TestAsyncTokenizer::test_concurrency
|
||||
tests/bindings/test_tokenizer.py::TestAsyncTokenizer::test_decode
|
||||
tests/bindings/test_tokenizer.py::TestAsyncTokenizer::test_encode
|
||||
tests/bindings/test_tokenizer.py::TestAsyncTokenizer::test_error_handling
|
||||
tests/bindings/test_tokenizer.py::TestAsyncTokenizer::test_large_batch
|
||||
tests/bindings/test_tokenizer.py::TestAsyncTokenizer::test_numpy_inputs
|
||||
tests/bindings/test_tokenizer.py::TestAsyncTokenizer::test_various_input_formats
|
||||
tests/bindings/test_tokenizer.py::TestAsyncTokenizer::test_performance_comparison
|
||||
tests/bindings/test_tokenizer.py::TestAsyncTokenizer::test_with_special_tokens
|
||||
tests/bindings/test_tokenizer.py::TestAsyncTokenizer::test_with_truncation_padding
|
||||
tests/bindings/test_tokenizer.py::TestTokenizer::test_encode_add_special_tokens
|
||||
tests/bindings/test_tokenizer.py::TestTokenizer::test_encode_formats
|
||||
tests/bindings/test_tokenizer.py::TestTokenizer::test_encode_special_tokens
|
||||
tests/bindings/test_tokenizer.py::TestTokenizer::test_decode_skip_special_tokens
|
||||
tests/bindings/test_tokenizer.py::TestTokenizer::test_decode_stream_fallback
|
||||
tests/bindings/test_tokenizer.py::TestTokenizer::test_from_pretrained
|
||||
tests/bindings/test_tokenizer.py::TestTokenizer::test_from_pretrained_revision
|
||||
tests/bindings/test_tokenizer.py::TestTokenizer::test_splitting
|
||||
tests/bindings/test_trainers.py::TestUnigram::test_continuing_prefix_trainer_mismatch
|
||||
tests/bindings/test_trainers.py::TestUnigram::test_train
|
||||
tests/bindings/test_trainers.py::TestUnigram::test_train_parallelism_with_custom_pretokenizer
|
||||
tests/documentation/test_pipeline.py::TestPipeline::test_bert_example
|
||||
tests/documentation/test_pipeline.py::TestPipeline::test_pipeline
|
||||
tests/documentation/test_quicktour.py::TestQuicktour::test_quicktour
|
||||
tests/documentation/test_tutorial_train_from_iterators.py::TestTrainFromIterators::test_datasets
|
||||
tests/documentation/test_tutorial_train_from_iterators.py::TestTrainFromIterators::test_gzip
|
||||
tests/implementations/test_byte_level_bpe.py::TestByteLevelBPE::test_add_prefix_space
|
||||
tests/implementations/test_byte_level_bpe.py::TestByteLevelBPE::test_basic_encode
|
||||
tests/implementations/test_byte_level_bpe.py::TestByteLevelBPE::test_lowerspace
|
||||
tests/implementations/test_byte_level_bpe.py::TestByteLevelBPE::test_multiprocessing_with_parallelism
|
||||
tests/implementations/test_bert_wordpiece.py::TestBertWordPieceTokenizer::test_basic_encode
|
||||
tests/implementations/test_bert_wordpiece.py::TestBertWordPieceTokenizer::test_multiprocessing_with_parallelism
|
||||
tests/implementations/test_char_bpe.py::TestCharBPETokenizer::test_basic_encode
|
||||
tests/implementations/test_char_bpe.py::TestCharBPETokenizer::test_decoding
|
||||
tests/implementations/test_char_bpe.py::TestCharBPETokenizer::test_lowercase
|
||||
tests/implementations/test_char_bpe.py::TestCharBPETokenizer::test_multiprocessing_with_parallelism
|
||||
tests/test_serialization.py::TestSerialization::test_full_serialization_albert
|
||||
tests/test_serialization.py::TestSerialization::test_str_big
|
||||
)
|
||||
distutils-r1_src_test
|
||||
}
|
||||
|
||||
src_install() {
|
||||
cd tokenizers
|
||||
cd ../bindings/python
|
||||
distutils-r1_src_install
|
||||
}
|
||||
Reference in New Issue
Block a user