1 write to EndOfSentenceToken
Microsoft.ML.Tokenizers (1)
Model\CodeGenTokenizer.cs (1)
143EndOfSentenceToken = endOfSentenceToken;
17 references to EndOfSentenceToken
Microsoft.ML.Tokenizers (10)
Model\CodeGenTokenizer.cs (10)
169if (!string.IsNullOrEmpty(EndOfSentenceToken)) 171if (!_vocab.TryGetValue(EndOfSentenceToken!, out (int endOfSentenceId, string token) value)) 173throw new ArgumentException($"The end of sentence token '{EndOfSentenceToken}' is not found in the vocabulary."); 184if (AddEndOfSentence && string.IsNullOrEmpty(EndOfSentenceToken)) 402tokens.Add(new EncodedToken(EndOfSentenceId.Value, EndOfSentenceToken!, new Range(index, index))); 1275Helpers.AppendToBytesArray(EndOfSentenceToken!.AsSpan(), ref bytes, ref bytesIndex); 1402if (EndOfSentenceToken!.Length > buffer.Length) 1407EndOfSentenceToken.AsSpan().CopyTo(buffer); 1408buffer = buffer.Slice(EndOfSentenceToken.Length); 1409charsWritten += EndOfSentenceToken.Length;
Microsoft.ML.Tokenizers.Tests (7)
CodeGenTests.cs (7)
294string targetText = $"{codeGenTokenizer.BeginningOfSentenceToken}{text}{codeGenTokenizer.EndOfSentenceToken}"; 683Assert.True(codeGenTokenizer.EndOfSentenceToken is not null); 688tokensList.Add(codeGenTokenizer.EndOfSentenceToken!); 711tokensList.Add(codeGenTokenizer.EndOfSentenceToken!); 823tokensList.Add(codeGenTokenizer.EndOfSentenceToken!); 852tokensList.Add(codeGenTokenizer.EndOfSentenceToken!); 979Assert.Equal(DefaultSpecialToken, codeGenTokenizer.EndOfSentenceToken);