1 reference to ToTokens
Microsoft.ML.Tokenizers (1)
Model\BPETokenizer.cs (1)
935
internal void WordToTokens(ref Word word, List<EncodedToken> tokens, int offset) => word.
ToTokens
(VocabReverse, tokens, offset);