1 write to UnknownToken
Microsoft.ML.Tokenizers (1)
Model\CodeGenTokenizer.cs (1)
138UnknownToken = unknownToken;
10 references to UnknownToken
Microsoft.ML.Tokenizers (9)
Model\CodeGenTokenizer.cs (9)
146if (!string.IsNullOrEmpty(UnknownToken)) 148if (!_vocab.TryGetValue(UnknownToken!, out (int unknownId, string token) value)) 150throw new ArgumentException($"The Unknown token '{UnknownToken}' is not found in the vocabulary."); 1281AppendToBytesArray(UnknownToken!.AsSpan(), ref bytes, ref bytesIndex); 1422if (UnknownToken!.Length > buffer.Length) 1427UnknownToken.AsSpan().CopyTo(buffer); 1428buffer = buffer.Slice(UnknownToken.Length); 1429charsWritten += UnknownToken.Length; 1687result.Add(GetToken(UnknownTokenId.Value, UnknownToken!, symbols[index].pieceSpan.Index, symbols[index].pieceSpan.Length, originalText, mapping));
Microsoft.ML.Tokenizers.Tests (1)
CodeGenTests.cs (1)
980Assert.Equal(DefaultSpecialToken, codeGenTokenizer.UnknownToken);