2 references to _codegen350MMonoTokenizerWithSpace
Microsoft.ML.Tokenizers.Tests (2)
CodeGenTests.cs (2)
221
TestTokenizer(
_codegen350MMonoTokenizerWithSpace
, text, expectedTokens, expectedOffsets, expectedIds, expectedTokensWithSpace, expectedOffsetsWithSpace, expectedIdsWithSpace);
227
TestDecoding(
_codegen350MMonoTokenizerWithSpace
, text);