1 write to BeginningOfSentenceId
Microsoft.ML.Tokenizers (1)
Model\CodeGenTokenizer.cs (1)
163BeginningOfSentenceId = value.beggingOfSentenceId;
34 references to BeginningOfSentenceId
Microsoft.ML.GenAI.Phi.Tests (1)
Phi2Tests.cs (1)
46tokenizer.BeginningOfSentenceId.Should().Be(50256);
Microsoft.ML.Tokenizers (10)
Model\CodeGenTokenizer.cs (10)
377if (addBos && BeginningOfSentenceId.HasValue) 379tokens.Add(new EncodedToken(BeginningOfSentenceId.Value, BeginningOfSentenceToken!, new Range(0, 0))); 617if (addBeginningOfSentence && BeginningOfSentenceId.HasValue) 619ids.Add(BeginningOfSentenceId.Value); 836int count = (addBeginningOfSentence && BeginningOfSentenceId.HasValue) ? 1 : 0; 994if (addBeginningOfSentence && BeginningOfSentenceId.HasValue && tokenCount < maxTokenCount) 1259if (BeginningOfSentenceId.HasValue && id == BeginningOfSentenceId.Value) 1367if (BeginningOfSentenceId.HasValue && id == BeginningOfSentenceId.Value)
Microsoft.ML.Tokenizers.Tests (23)
CodeGenTests.cs (23)
551Assert.True(codeGenTokenizer.BeginningOfSentenceId.HasValue); 553idList.Insert(0, codeGenTokenizer.BeginningOfSentenceId!.Value); 576idList.Insert(0, codeGenTokenizer.BeginningOfSentenceId!.Value); 610Assert.Equal(codeGenTokenizer.BeginningOfSentenceId.Value, ids[0]); 612Assert.Equal(codeGenTokenizer.BeginningOfSentenceId.Value, ids[0]); 614Assert.Equal(codeGenTokenizer.BeginningOfSentenceId.Value, ids[0]); 616Assert.Equal(codeGenTokenizer.BeginningOfSentenceId.Value, ids[0]); 618Assert.NotEqual(codeGenTokenizer.BeginningOfSentenceId.Value, ids[0]); 620Assert.NotEqual(codeGenTokenizer.BeginningOfSentenceId.Value, ids[0]); 622Assert.Equal(codeGenTokenizer.BeginningOfSentenceId.Value, ids[0]); 624Assert.Equal(codeGenTokenizer.BeginningOfSentenceId.Value, ids[0]); 817Assert.True(codeGenTokenizer.BeginningOfSentenceId.HasValue); 819idList.Insert(0, codeGenTokenizer.BeginningOfSentenceId!.Value); 848idList.Insert(0, codeGenTokenizer.BeginningOfSentenceId!.Value); 890Assert.Equal(codeGenTokenizer.BeginningOfSentenceId.Value, ids[0]); 893Assert.Equal(codeGenTokenizer.BeginningOfSentenceId.Value, ids[0]); 896Assert.Equal(codeGenTokenizer.BeginningOfSentenceId.Value, ids[0]); 899Assert.Equal(codeGenTokenizer.BeginningOfSentenceId.Value, ids[0]); 902Assert.NotEqual(codeGenTokenizer.BeginningOfSentenceId.Value, ids[0]); 905Assert.NotEqual(codeGenTokenizer.BeginningOfSentenceId.Value, ids[0]); 908Assert.Equal(codeGenTokenizer.BeginningOfSentenceId.Value, ids[0]); 911Assert.Equal(codeGenTokenizer.BeginningOfSentenceId.Value, ids[0]); 974Assert.Equal(codeGenTokenizer.EncodeToIds(DefaultSpecialToken)[0], codeGenTokenizer.BeginningOfSentenceId!.Value);