tokenizers-ruby
Fast state-of-the-art tokenizers for Ruby
File Explorer
Download Latest Version (.zip)- build.yml
- release.yml
- mod.rs
- normalization.rs
- regex.rs
- decoders.rs
- encoding.rs
- error.rs
- lib.rs
- models.rs
- normalizers.rs
- pre_tokenizers.rs
- processors.rs
- ruby.rs
- tokenizer.rs
- trainers.rs
- Cargo.toml
- extconf.rb
- bpe_decoder.rb
- ctc.rb
- metaspace.rb
- strip.rb
- word_piece.rb
- bpe.rb
- unigram.rb
- word_level.rb
- word_piece.rb
- bert_normalizer.rb
- prepend.rb
- strip.rb
- byte_level.rb
- digits.rb
- metaspace.rb
- punctuation.rb
- split.rb
- byte_level.rb
- roberta_processing.rb
- sequence.rb
- template_processing.rb
- bpe_trainer.rb
- unigram_trainer.rb
- word_level_trainer.rb
- word_piece_trainer.rb
- added_token.rb
- char_bpe_tokenizer.rb
- encoding.rb
- from_pretrained.rb
- tokenizer.rb
- version.rb
- tokenizers.rb
- added_token_test.rb
- decoder_test.rb
- model_test.rb
- normalizer_test.rb
- pre_tokenizer_test.rb
- processor_test.rb
- quicktour_test.rb
- test_helper.rb
- tokenizer_test.rb
- trainer_test.rb
- .gitignore
- Cargo.lock
- Cargo.toml
- CHANGELOG.md
- Gemfile
- LICENSE.txt
- Rakefile
- README.md
- tokenizers.gemspec
// repository documentation
Was this content helpful?
(0 ratings)