1 write to BeginningOfSentenceId
Microsoft.ML.Tokenizers (1)
Model\CodeGenTokenizer.cs (1)
163
BeginningOfSentenceId
= value.beggingOfSentenceId;
34 references to BeginningOfSentenceId
Microsoft.ML.GenAI.Phi.Tests (1)
Phi2Tests.cs (1)
46
tokenizer.
BeginningOfSentenceId
.Should().Be(50256);
Microsoft.ML.Tokenizers (10)
Model\CodeGenTokenizer.cs (10)
377
if (addBos &&
BeginningOfSentenceId
.HasValue)
379
tokens.Add(new EncodedToken(
BeginningOfSentenceId
.Value, BeginningOfSentenceToken!, new Range(0, 0)));
617
if (addBeginningOfSentence &&
BeginningOfSentenceId
.HasValue)
619
ids.Add(
BeginningOfSentenceId
.Value);
836
int count = (addBeginningOfSentence &&
BeginningOfSentenceId
.HasValue) ? 1 : 0;
994
if (addBeginningOfSentence &&
BeginningOfSentenceId
.HasValue && tokenCount < maxTokenCount)
1259
if (
BeginningOfSentenceId
.HasValue && id ==
BeginningOfSentenceId
.Value)
1367
if (
BeginningOfSentenceId
.HasValue && id ==
BeginningOfSentenceId
.Value)
Microsoft.ML.Tokenizers.Tests (23)
CodeGenTests.cs (23)
551
Assert.True(codeGenTokenizer.
BeginningOfSentenceId
.HasValue);
553
idList.Insert(0, codeGenTokenizer.
BeginningOfSentenceId
!.Value);
576
idList.Insert(0, codeGenTokenizer.
BeginningOfSentenceId
!.Value);
610
Assert.Equal(codeGenTokenizer.
BeginningOfSentenceId
.Value, ids[0]);
612
Assert.Equal(codeGenTokenizer.
BeginningOfSentenceId
.Value, ids[0]);
614
Assert.Equal(codeGenTokenizer.
BeginningOfSentenceId
.Value, ids[0]);
616
Assert.Equal(codeGenTokenizer.
BeginningOfSentenceId
.Value, ids[0]);
618
Assert.NotEqual(codeGenTokenizer.
BeginningOfSentenceId
.Value, ids[0]);
620
Assert.NotEqual(codeGenTokenizer.
BeginningOfSentenceId
.Value, ids[0]);
622
Assert.Equal(codeGenTokenizer.
BeginningOfSentenceId
.Value, ids[0]);
624
Assert.Equal(codeGenTokenizer.
BeginningOfSentenceId
.Value, ids[0]);
817
Assert.True(codeGenTokenizer.
BeginningOfSentenceId
.HasValue);
819
idList.Insert(0, codeGenTokenizer.
BeginningOfSentenceId
!.Value);
848
idList.Insert(0, codeGenTokenizer.
BeginningOfSentenceId
!.Value);
890
Assert.Equal(codeGenTokenizer.
BeginningOfSentenceId
.Value, ids[0]);
893
Assert.Equal(codeGenTokenizer.
BeginningOfSentenceId
.Value, ids[0]);
896
Assert.Equal(codeGenTokenizer.
BeginningOfSentenceId
.Value, ids[0]);
899
Assert.Equal(codeGenTokenizer.
BeginningOfSentenceId
.Value, ids[0]);
902
Assert.NotEqual(codeGenTokenizer.
BeginningOfSentenceId
.Value, ids[0]);
905
Assert.NotEqual(codeGenTokenizer.
BeginningOfSentenceId
.Value, ids[0]);
908
Assert.Equal(codeGenTokenizer.
BeginningOfSentenceId
.Value, ids[0]);
911
Assert.Equal(codeGenTokenizer.
BeginningOfSentenceId
.Value, ids[0]);
974
Assert.Equal(codeGenTokenizer.EncodeToIds(DefaultSpecialToken)[0], codeGenTokenizer.
BeginningOfSentenceId
!.Value);