TryDecodeAsSpecialToken

5 references to TryDecodeAsSpecialToken

Microsoft.ML.Tokenizers (5)

Model\SentencePieceBaseModel.cs (5)

175TryDecodeAsSpecialToken(this, current, considerSpecialTokens, ref sb); 190else if (!TryDecodeAsSpecialToken(this, current, considerSpecialTokens, ref sb) && TryMapIdToToken(current, out string? token)) 195else if (!TryDecodeAsSpecialToken(this, current, considerSpecialTokens, ref sb) && TryMapIdToToken(current, out string? token)) 213TryDecodeAsSpecialToken(this, current, considerSpecialTokens, ref sb); 243if (!TryDecodeAsSpecialToken(this, current, considerSpecialTokens, ref sb) && TryMapIdToToken(current, out string? token))