1 write to UnicodeToByte
Microsoft.ML.Tokenizers (1)
Utils\ByteToUnicodeEncoding.cs (1)
36
UnicodeToByte
= ByteToUnicode.ToDictionary(kv => kv.Value, kv => kv.Key);
4 references to UnicodeToByte
Microsoft.ML.Tokenizers (4)
Model\BPETokenizer.cs (1)
1049
current[i] = (byte)byteToUnicodeEncoding.
UnicodeToByte
[span[i]];
Model\CodeGenTokenizer.cs (1)
1479
current[i] = (byte)byteToUnicodeEncoding.
UnicodeToByte
[span[i]];
Model\EnglishRobertaTokenizer.cs (1)
286
IReadOnlyDictionary<char, char> unicodeToByte = ByteToUnicodeEncoding.Instance.
UnicodeToByte
;
Utils\Helpers.cs (1)
216
IReadOnlyDictionary<char, char> unicodeToByte = ByteToUnicodeEncoding.Instance.
UnicodeToByte
;