1 write to UnknownTokenId
Microsoft.ML.Tokenizers (1)
Model\CodeGenTokenizer.cs (1)
153UnknownTokenId = value.unknownId;
7 references to UnknownTokenId
Microsoft.ML.Tokenizers (6)
Model\CodeGenTokenizer.cs (6)
1277if (UnknownTokenId.HasValue && id == UnknownTokenId.Value) 1413if (UnknownTokenId.HasValue && id == UnknownTokenId.Value) 1685else if (UnknownTokenId.HasValue) 1687result.Add(GetToken(UnknownTokenId.Value, UnknownToken!, symbols[index].pieceSpan.Index, symbols[index].pieceSpan.Length, originalText, mapping));
Microsoft.ML.Tokenizers.Tests (1)
CodeGenTests.cs (1)
976Assert.Equal(codeGenTokenizer.EncodeToIds(DefaultSpecialToken)[0], codeGenTokenizer.UnknownTokenId!.Value);