1 write to _normalized
Microsoft.ML.Tokenizers (1)
Normalizer\SentencePieceNormalizer.cs (1)
53DecodePrecompiledCharsMap(precompiledCharsMap, out DoubleArrayUnit[]? trieBlob, out _normalized);
4 references to _normalized
Microsoft.ML.Tokenizers (4)
Normalizer\SentencePieceNormalizer.cs (4)
319Debug.Assert(_normalized is not null); 325while (normalizedLength < _normalized!.Length && _normalized[normalizedLength] != 0) 329normalizedPrefix = new Memory<byte>(_normalized, longestValue, normalizedLength - longestValue);