1 instantiation of ByteToUnicodeEncoding
Microsoft.ML.Tokenizers (1)
Utils\ByteToUnicodeEncoding.cs (1)
15public static ByteToUnicodeEncoding Instance { get; } = new ByteToUnicodeEncoding();
14 references to ByteToUnicodeEncoding
Microsoft.ML.Tokenizers (14)
Model\CodeGenTokenizer.cs (5)
1358ByteToUnicodeEncoding byteToUnicodeEncoding = ByteToUnicodeEncoding.Instance; 1520private static readonly char _transformedSpace = ByteToUnicodeEncoding.Instance.ByteToUnicode[' ']; 1564IReadOnlyDictionary<char, char> unicodeToByte = ByteToUnicodeEncoding.Instance.UnicodeToByte; 1623string[] charToString = ByteToUnicodeEncoding.Instance.CharToString;
Model\EnglishRobertaTokenizer.cs (6)
275IReadOnlyDictionary<char, char> unicodeToByte = ByteToUnicodeEncoding.Instance.UnicodeToByte; 356IReadOnlyDictionary<char, char> byteToUnicode = ByteToUnicodeEncoding.Instance.ByteToUnicode; 668IReadOnlyDictionary<char, char> byteToUnicode = ByteToUnicodeEncoding.Instance.ByteToUnicode; 713IReadOnlyDictionary<char, char> byteToUnicode = ByteToUnicodeEncoding.Instance.ByteToUnicode; 944string[] charToString = ByteToUnicodeEncoding.Instance.CharToString; 1072public bool IsSupportedChar(char ch) => ByteToUnicodeEncoding.Instance.ByteToUnicode.ContainsKey(ch);
Utils\ByteToUnicodeEncoding.cs (1)
15public static ByteToUnicodeEncoding Instance { get; } = new ByteToUnicodeEncoding();
Utils\Helpers.cs (2)
82ByteToUnicodeEncoding byteToUnicodeEncoder = ByteToUnicodeEncoding.Instance;