1 write to EndOfSentenceId
Microsoft.ML.Tokenizers (1)
Model\BPETokenizer.cs (1)
334EndOfSentenceId = aId;
2 references to EndOfSentenceId
Microsoft.ML.Tokenizers (2)
Model\BPETokenizer.cs (2)
471tokens.Add(new EncodedToken(EndOfSentenceId, EndOfSentenceToken, new Range(charsConsumed, charsConsumed))); 538ids.Add(EndOfSentenceId);