1 write to UnknownToken
Microsoft.ML.Tokenizers (1)
Model\CodeGenTokenizer.cs (1)
141UnknownToken = unknownToken;
10 references to UnknownToken
Microsoft.ML.Tokenizers (9)
Model\CodeGenTokenizer.cs (9)
149if (!string.IsNullOrEmpty(UnknownToken)) 151if (!_vocab.TryGetValue(UnknownToken!, out (int unknownId, string token) value)) 153throw new ArgumentException($"The Unknown token '{UnknownToken}' is not found in the vocabulary."); 1284Helpers.AppendToBytesArray(UnknownToken!.AsSpan(), ref bytes, ref bytesIndex); 1425if (UnknownToken!.Length > buffer.Length) 1430UnknownToken.AsSpan().CopyTo(buffer); 1431buffer = buffer.Slice(UnknownToken.Length); 1432charsWritten += UnknownToken.Length; 1669result.Add(GetToken(UnknownTokenId.Value, UnknownToken!, symbols[index].pieceSpan.Index, symbols[index].pieceSpan.Length, originalText, mapping));
Microsoft.ML.Tokenizers.Tests (1)
CodeGenTests.cs (1)
980Assert.Equal(DefaultSpecialToken, codeGenTokenizer.UnknownToken);