1 write to EndOfSentenceId
Microsoft.ML.Tokenizers (1)
Model\CodeGenTokenizer.cs (1)
173EndOfSentenceId = value.endOfSentenceId;
33 references to EndOfSentenceId
Microsoft.ML.GenAI.Phi.Tests (1)
Phi2Tests.cs (1)
45tokenizer.EndOfSentenceId.Should().Be(50256);
Microsoft.ML.Tokenizers (10)
Model\CodeGenTokenizer.cs (10)
396if (addEos && EndOfSentenceId.HasValue) 399tokens.Add(new EncodedToken(EndOfSentenceId.Value, EndOfSentenceToken!, new Range(index, index))); 643if (addEndOfSentence && EndOfSentenceId.HasValue && ids.Count < maxTokenCount) 645ids.Add(EndOfSentenceId.Value); 855if (addEndOfSentence && EndOfSentenceId.HasValue && count < maxTokenCount) 973tokenCount = (addEndOfSentence && EndOfSentenceId.HasValue) ? 1 : 0; 1268if (EndOfSentenceId.HasValue && id == EndOfSentenceId.Value) 1390if (EndOfSentenceId.HasValue && id == EndOfSentenceId.Value)
Microsoft.ML.Tokenizers.Tests (22)
CodeGenTests.cs (22)
684Assert.True(codeGenTokenizer.EndOfSentenceId.HasValue); 686idList.Add(codeGenTokenizer.EndOfSentenceId!.Value); 709idList.Add(codeGenTokenizer.EndOfSentenceId!.Value); 743Assert.Equal(codeGenTokenizer.EndOfSentenceId.Value, ids[ids.Count - 1]); 745Assert.Equal(codeGenTokenizer.EndOfSentenceId.Value, ids[ids.Count - 1]); 747Assert.Equal(codeGenTokenizer.EndOfSentenceId.Value, ids[ids.Count - 1]); 749Assert.Equal(codeGenTokenizer.EndOfSentenceId.Value, ids[ids.Count - 1]); 751Assert.NotEqual(codeGenTokenizer.EndOfSentenceId.Value, ids[ids.Count - 1]); 753Assert.NotEqual(codeGenTokenizer.EndOfSentenceId.Value, ids[ids.Count - 1]); 755Assert.Equal(codeGenTokenizer.EndOfSentenceId.Value, ids[ids.Count - 1]); 757Assert.Equal(codeGenTokenizer.EndOfSentenceId.Value, ids[ids.Count - 1]); 820idList.Add(codeGenTokenizer.EndOfSentenceId!.Value); 849idList.Add(codeGenTokenizer.EndOfSentenceId!.Value); 891Assert.Equal(codeGenTokenizer.EndOfSentenceId.Value, ids[ids.Count - 1]); 894Assert.Equal(codeGenTokenizer.EndOfSentenceId.Value, ids[ids.Count - 1]); 897Assert.Equal(codeGenTokenizer.EndOfSentenceId.Value, ids[ids.Count - 1]); 900Assert.Equal(codeGenTokenizer.EndOfSentenceId.Value, ids[ids.Count - 1]); 903Assert.NotEqual(codeGenTokenizer.EndOfSentenceId.Value, ids[ids.Count - 1]); 906Assert.NotEqual(codeGenTokenizer.EndOfSentenceId.Value, ids[ids.Count - 1]); 909Assert.Equal(codeGenTokenizer.EndOfSentenceId.Value, ids[ids.Count - 1]); 912Assert.Equal(codeGenTokenizer.EndOfSentenceId.Value, ids[ids.Count - 1]); 975Assert.Equal(codeGenTokenizer.EncodeToIds(DefaultSpecialToken)[0], codeGenTokenizer.EndOfSentenceId!.Value);