1 reference to ToTokens
Microsoft.ML.Tokenizers (1)
Model\BPETokenizer.cs (1)
1301
internal void WordToTokens(ref Word word, List<EncodedToken> tokens, int offset, ReadOnlySpan<int> mapping) => word.
ToTokens
(VocabReverse, tokens, offset, mapping);