macports-ports/python/py-tokenizers/Portfile

# -*- coding: utf-8; mode: tcl; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- vim:fenc=utf-8:ft=tcl:et:sw=4:ts=4:sts=4

PortSystem          1.0
PortGroup           python 1.0
PortGroup           rust 1.0

name                py-tokenizers
version             0.22.2
revision            0
license             Apache-2
maintainers         nomaintainer

description         Fast and Customizable Tokenizers
long_description    Tokenizers provides an implementation of today's \
                    most used tokenizers, with a focus on performance \
                    and versatility. Includes BPE, WordPiece, and \
                    Unigram tokenizer implementations.

homepage            https://github.com/huggingface/tokenizers

checksums           ${distname}${extract.suffix} \
                    rmd160  679ff19184035d437252ad9cfbe058d347077c0e \
                    sha256  473b83b915e547aa366d1eee11806deaf419e17be16310ac0a14077f1e28f917 \
                    size    372115

python.versions     310 311 312 313 314
python.pep517_backend \
                    maturin

if {${name} ne ${subport}} {
    depends_lib-append \
                    port:py${python.version}-huggingface_hub

    # cd ${worksrcpath}/bindings/python
    # awk -F'"' '/^name = /{name=$2} /^version = /{ver=$2} /^checksum = /{print name, ver, $2, "\\\\"}' Cargo.lock | pbcopy
    cargo.crates \
        ahash 0.8.12 5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75 \
        aho-corasick 1.1.4 ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301 \
        anstream 0.6.21 43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a \
        anstyle 1.0.13 5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78 \
        anstyle-parse 0.2.7 4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2 \
        anstyle-query 1.1.5 40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc \
        anstyle-wincon 3.0.11 291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d \
        autocfg 1.5.0 c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8 \
        base64 0.13.1 9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8 \
        bitflags 2.10.0 812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3 \
        bumpalo 3.19.0 46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43 \
        castaway 0.2.4 dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a \
        cc 1.2.48 c481bdbf0ed3b892f6f806287d72acd515b352a4ec27a208489b8c1bc839633a \
        cfg-if 1.0.4 9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801 \
        colorchoice 1.0.4 b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75 \
        compact_str 0.9.0 3fdb1325a1cece981e8a296ab8f0f9b63ae357bd0784a9faaf548cc7b480707a \
        console 0.16.1 b430743a6eb14e9764d4260d4c0d8123087d504eeb9c48f2b2a5e810dd369df4 \
        crossbeam-deque 0.8.6 9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51 \
        crossbeam-epoch 0.9.18 5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e \
        crossbeam-utils 0.8.21 d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28 \
        darling 0.20.11 fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee \
        darling_core 0.20.11 0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e \
        darling_macro 0.20.11 fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead \
        dary_heap 0.3.8 06d2e3287df1c007e74221c49ca10a95d557349e54b3a75dc2fb14712c751f04 \
        derive_builder 0.20.2 507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947 \
        derive_builder_core 0.20.2 2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8 \
        derive_builder_macro 0.20.2 ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c \
        either 1.15.0 48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719 \
        encode_unicode 1.0.0 34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0 \
        env_filter 0.1.4 1bf3c259d255ca70051b30e2e95b5446cdb8949ac4cd22c0d7fd634d89f568e2 \
        env_logger 0.11.8 13c863f0904021b108aa8b2f55046443e6b1ebde8fd4a15c399893aae4fa069f \
        errno 0.3.14 39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb \
        esaxx-rs 0.1.10 d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6 \
        fastrand 2.3.0 37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be \
        find-msvc-tools 0.1.5 3a3076410a55c90011c298b04d0cfa770b00fa04e1e3c97d3f6c9de105a03844 \
        fnv 1.0.7 3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1 \
        futures 0.3.31 65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876 \
        futures-channel 0.3.31 2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10 \
        futures-core 0.3.31 05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e \
        futures-executor 0.3.31 1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f \
        futures-io 0.3.31 9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6 \
        futures-macro 0.3.31 162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650 \
        futures-sink 0.3.31 e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7 \
        futures-task 0.3.31 f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988 \
        futures-util 0.3.31 9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81 \
        getrandom 0.3.4 899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd \
        heck 0.5.0 2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea \
        ident_case 1.0.1 b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39 \
        indicatif 0.18.3 9375e112e4b463ec1b1c6c011953545c65a30164fbab5b581df32b3abf0dcb88 \
        indoc 2.0.7 79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706 \
        is_terminal_polyfill 1.70.2 a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695 \
        itertools 0.14.0 2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285 \
        itoa 1.0.15 4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c \
        jiff 0.2.16 49cce2b81f2098e7e3efc35bc2e0a6b7abec9d34128283d7a26fa8f32a6dbb35 \
        jiff-static 0.2.16 980af8b43c3ad5d8d349ace167ec8170839f753a42d233ba19e08afe1850fa69 \
        js-sys 0.3.83 464a3709c7f55f1f721e5389aa6ea4e3bc6aba669353300af094b29ffbdde1d8 \
        libc 0.2.177 2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976 \
        linux-raw-sys 0.11.0 df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039 \
        log 0.4.28 34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432 \
        macro_rules_attribute 0.2.2 65049d7923698040cd0b1ddcced9b0eb14dd22c5f86ae59c3740eab64a676520 \
        macro_rules_attribute-proc_macro 0.2.2 670fdfda89751bc4a84ac13eaa63e205cf0fd22b4c9a5fbfa085b63c1f1d3a30 \
        matrixmultiply 0.3.10 a06de3016e9fae57a36fd14dba131fccf49f74b40b7fbdb472f96e361ec71a08 \
        memchr 2.7.6 f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273 \
        memoffset 0.9.1 488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a \
        minimal-lexical 0.2.1 68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a \
        mio 1.1.0 69d83b0086dc8ecf3ce9ae2874b2d1290252e2a30720bea58a5c6639b0092873 \
        monostate 0.1.18 3341a273f6c9d5bef1908f17b7267bbab0e95c9bf69a0d4dcf8e9e1b2c76ef67 \
        monostate-impl 0.1.18 e4db6d5580af57bf992f59068d4ea26fd518574ff48d7639b255a36f9de6e7e9 \
        ndarray 0.16.1 882ed72dce9365842bf196bdeedf5055305f11fc8c03dee7bb0194a6cad34841 \
        nom 7.1.3 d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a \
        num-complex 0.4.6 73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495 \
        num-integer 0.1.46 7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f \
        num-traits 0.2.19 071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841 \
        numpy 0.26.0 9b2dba356160b54f5371b550575b78130a54718b4c6e46b3f33a6da74a27e78b \
        once_cell 1.21.3 42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d \
        once_cell_polyfill 1.70.2 384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe \
        onig 6.5.1 336b9c63443aceef14bea841b899035ae3abe89b7c486aaf4c5bd8aafedac3f0 \
        onig_sys 69.9.1 c7f86c6eef3d6df15f23bcfb6af487cbd2fed4e5581d58d5bf1f5f8b7f6727dc \
        paste 1.0.15 57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a \
        pin-project-lite 0.2.16 3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b \
        pin-utils 0.1.0 8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184 \
        pkg-config 0.3.32 7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c \
        portable-atomic 1.11.1 f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483 \
        portable-atomic-util 0.2.4 d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507 \
        ppv-lite86 0.2.21 85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9 \
        proc-macro2 1.0.103 5ee95bc4ef87b8d5ba32e8b7714ccc834865276eab0aed5c9958d00ec45f49e8 \
        pyo3 0.26.0 7ba0117f4212101ee6544044dae45abe1083d30ce7b29c4b5cbdfa2354e07383 \
        pyo3-async-runtimes 0.26.0 e6ee6d4cb3e8d5b925f5cdb38da183e0ff18122eb2048d4041c9e7034d026e23 \
        pyo3-build-config 0.26.0 4fc6ddaf24947d12a9aa31ac65431fb1b851b8f4365426e182901eabfb87df5f \
        pyo3-ffi 0.26.0 025474d3928738efb38ac36d4744a74a400c901c7596199e20e45d98eb194105 \
        pyo3-macros 0.26.0 2e64eb489f22fe1c95911b77c44cc41e7c19f3082fc81cce90f657cdc42ffded \
        pyo3-macros-backend 0.26.0 100246c0ecf400b475341b8455a9213344569af29a3c841d29270e53102e0fcf \
        quote 1.0.42 a338cc41d27e6cc6dce6cefc13a0729dfbb81c262b1f519331575dd80ef3067f \
        r-efi 5.3.0 69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f \
        rand 0.9.2 6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1 \
        rand_chacha 0.9.0 d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb \
        rand_core 0.9.3 99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38 \
        rawpointer 0.2.1 60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3 \
        rayon 1.11.0 368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f \
        rayon-cond 0.4.0 2964d0cf57a3e7a06e8183d14a8b527195c706b7983549cd5462d5aa3747438f \
        rayon-core 1.13.0 22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91 \
        regex 1.12.2 843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4 \
        regex-automata 0.4.13 5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c \
        regex-syntax 0.8.8 7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58 \
        rustc-hash 2.1.1 357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d \
        rustix 1.1.2 cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e \
        rustversion 1.0.22 b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d \
        ryu 1.0.20 28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f \
        serde 1.0.228 9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e \
        serde_core 1.0.228 41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad \
        serde_derive 1.0.228 d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79 \
        serde_json 1.0.145 402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c \
        shlex 1.3.0 0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64 \
        signal-hook-registry 1.4.7 7664a098b8e616bdfcc2dc0e9ac44eb231eedf41db4e9fe95d8d32ec728dedad \
        slab 0.4.11 7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589 \
        smallvec 1.15.1 67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03 \
        spm_precompiled 0.1.4 5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326 \
        static_assertions 1.1.0 a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f \
        strsim 0.11.1 7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f \
        syn 2.0.111 390cc9a294ab71bdb1aa2e99d13be9c753cd2d7bd6560c77118597410c4d2e87 \
        target-lexicon 0.13.3 df7f62577c25e07834649fc3b39fafdc597c0a3527dc1c60129201ccfcbaa50c \
        tempfile 3.23.0 2d31c77bdf42a745371d260a26ca7163f1e0924b64afa0b688e61b5a9fa02f16 \
        thiserror 2.0.17 f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8 \
        thiserror-impl 2.0.17 3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913 \
        tokio 1.48.0 ff360e02eab121e0bc37a2d3b4d4dc622e6eda3a8e5253d5435ecf5bd4c68408 \
        tokio-macros 2.6.0 af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5 \
        unicode-ident 1.0.22 9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5 \
        unicode-normalization-alignments 0.1.12 43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de \
        unicode-segmentation 1.12.0 f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493 \
        unicode-width 0.2.2 b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254 \
        unicode_categories 0.1.1 39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e \
        unindent 0.2.4 7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3 \
        unit-prefix 0.5.2 81e544489bf3d8ef66c953931f56617f423cd4b5494be343d9b9d3dda037b9a3 \
        utf8parse 0.2.2 06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821 \
        version_check 0.9.5 0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a \
        wasi 0.11.1+wasi-snapshot-preview1 ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b \
        wasip2 1.0.1+wasi-0.2.4 0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7 \
        wasm-bindgen 0.2.106 0d759f433fa64a2d763d1340820e46e111a7a5ab75f993d1852d70b03dbb80fd \
        wasm-bindgen-macro 0.2.106 48cb0d2638f8baedbc542ed444afc0644a29166f1595371af4fecf8ce1e7eeb3 \
        wasm-bindgen-macro-support 0.2.106 cefb59d5cd5f92d9dcf80e4683949f15ca4b511f4ac0a6e14d4e1ac60c6ecd40 \
        wasm-bindgen-shared 0.2.106 cbc538057e648b67f72a982e708d485b2efa771e1ac05fec311f9f63e5800db4 \
        web-time 1.1.0 5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb \
        windows-link 0.2.1 f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5 \
        windows-sys 0.61.2 ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc \
        wit-bindgen 0.46.0 f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59 \
        zerocopy 0.8.31 fd74ec98b9250adb3ca554bdde269adf631549f51d8a8f8f0a10b50f1cb298c3 \
        zerocopy-derive 0.8.31 d8a8d209fdf45cf5138cbb5a506f6b52522a25afccc534d1475dad8e31105c6a
}