2 types derived from SentencePieceBaseModel
Microsoft.ML.Tokenizers (2)
Model\SentencePieceBpeModel.cs (1)
19
internal sealed class SentencePieceBpeModel :
SentencePieceBaseModel
Model\SentencePieceUnigramModel.cs (1)
19
internal sealed class SentencePieceUnigramModel :
SentencePieceBaseModel
4 references to SentencePieceBaseModel
Microsoft.ML.Tokenizers (4)
Model\SentencePieceBaseModel.cs (2)
382
static bool TryDecodeAsSpecialToken(
SentencePieceBaseModel
model, int id, bool considerSpecialTokens, ref ValueStringBuilder sb)
620
static OperationStatus TryDecodeAsSpecialToken(
SentencePieceBaseModel
model, int id, bool considerSpecialTokens, Span<char> buffer, ref int charsWritten, out bool isSpecialToken)
Model\SentencePieceTokenizer.cs (2)
22
private readonly
SentencePieceBaseModel
_model;
34
private SentencePieceTokenizer(
SentencePieceBaseModel
model)