1 instantiation of ByteToUnicodeEncoding
Microsoft.ML.Tokenizers (1)
Utils\ByteToUnicodeEncoding.cs (1)
15public static ByteToUnicodeEncoding Instance { get; } = new ByteToUnicodeEncoding();
14 references to ByteToUnicodeEncoding
Microsoft.ML.Tokenizers (14)
Model\CodeGenTokenizer.cs (5)
1360ByteToUnicodeEncoding byteToUnicodeEncoding = ByteToUnicodeEncoding.Instance; 1522private static readonly char _transformedSpace = ByteToUnicodeEncoding.Instance.ByteToUnicode[' ']; 1566IReadOnlyDictionary<char, char> unicodeToByte = ByteToUnicodeEncoding.Instance.UnicodeToByte; 1626string[] charToString = ByteToUnicodeEncoding.Instance.CharToString;
Model\EnglishRobertaTokenizer.cs (6)
286IReadOnlyDictionary<char, char> unicodeToByte = ByteToUnicodeEncoding.Instance.UnicodeToByte; 367IReadOnlyDictionary<char, char> byteToUnicode = ByteToUnicodeEncoding.Instance.ByteToUnicode; 679IReadOnlyDictionary<char, char> byteToUnicode = ByteToUnicodeEncoding.Instance.ByteToUnicode; 724IReadOnlyDictionary<char, char> byteToUnicode = ByteToUnicodeEncoding.Instance.ByteToUnicode; 955string[] charToString = ByteToUnicodeEncoding.Instance.CharToString; 1083public bool IsSupportedChar(char ch) => ByteToUnicodeEncoding.Instance.ByteToUnicode.ContainsKey(ch);
Utils\ByteToUnicodeEncoding.cs (1)
15public static ByteToUnicodeEncoding Instance { get; } = new ByteToUnicodeEncoding();
Utils\Helpers.cs (2)
82ByteToUnicodeEncoding byteToUnicodeEncoder = ByteToUnicodeEncoding.Instance;