1 write to EndOfSentenceId
Microsoft.ML.Tokenizers (1)
Model\CodeGenTokenizer.cs (1)
173
EndOfSentenceId
= value.endOfSentenceId;
33 references to EndOfSentenceId
Microsoft.ML.GenAI.Phi.Tests (1)
Phi2Tests.cs (1)
45
tokenizer.
EndOfSentenceId
.Should().Be(50256);
Microsoft.ML.Tokenizers (10)
Model\CodeGenTokenizer.cs (10)
396
if (addEos &&
EndOfSentenceId
.HasValue)
399
tokens.Add(new EncodedToken(
EndOfSentenceId
.Value, EndOfSentenceToken!, new Range(index, index)));
643
if (addEndOfSentence &&
EndOfSentenceId
.HasValue && ids.Count < maxTokenCount)
645
ids.Add(
EndOfSentenceId
.Value);
855
if (addEndOfSentence &&
EndOfSentenceId
.HasValue && count < maxTokenCount)
973
tokenCount = (addEndOfSentence &&
EndOfSentenceId
.HasValue) ? 1 : 0;
1268
if (
EndOfSentenceId
.HasValue && id ==
EndOfSentenceId
.Value)
1390
if (
EndOfSentenceId
.HasValue && id ==
EndOfSentenceId
.Value)
Microsoft.ML.Tokenizers.Tests (22)
CodeGenTests.cs (22)
684
Assert.True(codeGenTokenizer.
EndOfSentenceId
.HasValue);
686
idList.Add(codeGenTokenizer.
EndOfSentenceId
!.Value);
709
idList.Add(codeGenTokenizer.
EndOfSentenceId
!.Value);
743
Assert.Equal(codeGenTokenizer.
EndOfSentenceId
.Value, ids[ids.Count - 1]);
745
Assert.Equal(codeGenTokenizer.
EndOfSentenceId
.Value, ids[ids.Count - 1]);
747
Assert.Equal(codeGenTokenizer.
EndOfSentenceId
.Value, ids[ids.Count - 1]);
749
Assert.Equal(codeGenTokenizer.
EndOfSentenceId
.Value, ids[ids.Count - 1]);
751
Assert.NotEqual(codeGenTokenizer.
EndOfSentenceId
.Value, ids[ids.Count - 1]);
753
Assert.NotEqual(codeGenTokenizer.
EndOfSentenceId
.Value, ids[ids.Count - 1]);
755
Assert.Equal(codeGenTokenizer.
EndOfSentenceId
.Value, ids[ids.Count - 1]);
757
Assert.Equal(codeGenTokenizer.
EndOfSentenceId
.Value, ids[ids.Count - 1]);
820
idList.Add(codeGenTokenizer.
EndOfSentenceId
!.Value);
849
idList.Add(codeGenTokenizer.
EndOfSentenceId
!.Value);
891
Assert.Equal(codeGenTokenizer.
EndOfSentenceId
.Value, ids[ids.Count - 1]);
894
Assert.Equal(codeGenTokenizer.
EndOfSentenceId
.Value, ids[ids.Count - 1]);
897
Assert.Equal(codeGenTokenizer.
EndOfSentenceId
.Value, ids[ids.Count - 1]);
900
Assert.Equal(codeGenTokenizer.
EndOfSentenceId
.Value, ids[ids.Count - 1]);
903
Assert.NotEqual(codeGenTokenizer.
EndOfSentenceId
.Value, ids[ids.Count - 1]);
906
Assert.NotEqual(codeGenTokenizer.
EndOfSentenceId
.Value, ids[ids.Count - 1]);
909
Assert.Equal(codeGenTokenizer.
EndOfSentenceId
.Value, ids[ids.Count - 1]);
912
Assert.Equal(codeGenTokenizer.
EndOfSentenceId
.Value, ids[ids.Count - 1]);
975
Assert.Equal(codeGenTokenizer.EncodeToIds(DefaultSpecialToken)[0], codeGenTokenizer.
EndOfSentenceId
!.Value);