1 write to MaxInputCharsPerWord
Microsoft.ML.Tokenizers (1)
Model\WordPieceTokenizer.cs (1)
70
MaxInputCharsPerWord
= options.MaxInputCharsPerWord;
5 references to MaxInputCharsPerWord
Microsoft.ML.Tokenizers (4)
Model\WordPieceTokenizer.cs (4)
317
if (text.Length >
MaxInputCharsPerWord
)
323
int maxLength =
MaxInputCharsPerWord
+ ContinuingSubwordPrefix.Length;
456
if (text.Length >
MaxInputCharsPerWord
)
463
int maxLength =
MaxInputCharsPerWord
+ ContinuingSubwordPrefix.Length;
Microsoft.ML.Tokenizers.Tests (1)
WordPieceTests.cs (1)
41
Assert.Equal(100, tokenizer.
MaxInputCharsPerWord
);