1 write to MaxInputCharsPerWord
Microsoft.ML.Tokenizers (1)
Model\WordPieceTokenizer.cs (1)
75MaxInputCharsPerWord = maxInputCharsPerWord;
5 references to MaxInputCharsPerWord
Microsoft.ML.Tokenizers (4)
Model\WordPieceTokenizer.cs (4)
372if (text.Length > MaxInputCharsPerWord) 378int maxLength = MaxInputCharsPerWord + ContinuingSubwordPrefix.Length; 511if (text.Length > MaxInputCharsPerWord) 518int maxLength = MaxInputCharsPerWord + ContinuingSubwordPrefix.Length;
Microsoft.ML.Tokenizers.Tests (1)
WordPieceTests.cs (1)
41Assert.Equal(100, tokenizer.MaxInputCharsPerWord);