1 write to _normalized
Microsoft.ML.Tokenizers (1)
Normalizer\SentencePieceNormalizer.cs (1)
60
DecodePrecompiledCharsMap(precompiledCharsMap, out DoubleArrayUnit[]? trieBlob, out
_normalized
);
4 references to _normalized
Microsoft.ML.Tokenizers (4)
Normalizer\SentencePieceNormalizer.cs (4)
367
Debug.Assert(
_normalized
is not null);
373
while (normalizedLength <
_normalized
!.Length &&
_normalized
[normalizedLength] != 0)
377
normalizedPrefix = new Memory<byte>(
_normalized
, longestValue, normalizedLength - longestValue);