1 instantiation of ByteToUnicodeEncoding
Microsoft.ML.Tokenizers (1)
Utils\ByteToUnicodeEncoding.cs (1)
15public static ByteToUnicodeEncoding Instance { get; } = new ByteToUnicodeEncoding();
16 references to ByteToUnicodeEncoding
Microsoft.ML.Tokenizers (16)
Model\BPETokenizer.cs (2)
992ByteToUnicodeEncoding byteToUnicodeEncoding = ByteToUnicodeEncoding.Instance;
Model\CodeGenTokenizer.cs (4)
1363ByteToUnicodeEncoding byteToUnicodeEncoding = ByteToUnicodeEncoding.Instance; 1525private static readonly char _transformedSpace = ByteToUnicodeEncoding.Instance.ByteToUnicode[' ']; 1608string[] charToString = ByteToUnicodeEncoding.Instance.CharToString;
Model\EnglishRobertaTokenizer.cs (6)
286IReadOnlyDictionary<char, char> unicodeToByte = ByteToUnicodeEncoding.Instance.UnicodeToByte; 367IReadOnlyDictionary<char, char> byteToUnicode = ByteToUnicodeEncoding.Instance.ByteToUnicode; 679IReadOnlyDictionary<char, char> byteToUnicode = ByteToUnicodeEncoding.Instance.ByteToUnicode; 724IReadOnlyDictionary<char, char> byteToUnicode = ByteToUnicodeEncoding.Instance.ByteToUnicode; 955string[] charToString = ByteToUnicodeEncoding.Instance.CharToString; 1083public bool IsSupportedChar(char ch) => ByteToUnicodeEncoding.Instance.ByteToUnicode.ContainsKey(ch);
Utils\ByteToUnicodeEncoding.cs (1)
15public static ByteToUnicodeEncoding Instance { get; } = new ByteToUnicodeEncoding();
Utils\Helpers.cs (3)
165ByteToUnicodeEncoding byteToUnicodeEncoder = ByteToUnicodeEncoding.Instance; 216IReadOnlyDictionary<char, char> unicodeToByte = ByteToUnicodeEncoding.Instance.UnicodeToByte;