1 write to Value
Microsoft.ML.Tokenizers (1)
EncodedToken.cs (1)
38
Value
= value;
70 references to Value
Microsoft.ML.Tokenizers (10)
Model\CodeGenTokenizer.cs (3)
1593
tokens.Add(new EncodedToken(tokensToAdd[0].Id, tokensToAdd[0].
Value
, (offset == 0 ? tokensToAdd[0].Offset.Index : tokensToAdd[0].Offset.Index + offset - 1, offset == 0 ? tokensToAdd[0].Offset.Length - 1 : tokensToAdd[0].Offset.Length)));
1597
tokens.Add(new EncodedToken(tokensToAdd[i].Id, tokensToAdd[i].
Value
, (tokensToAdd[i].Offset.Index + offset - 1, tokensToAdd[i].Offset.Length)));
1605
tokens.Add(new EncodedToken(t.Id, t.
Value
, (t.Offset.Index + offset, t.Offset.Length)));
Model\EnglishRobertaTokenizer.cs (7)
328
tokens.Add(new EncodedToken(t.Id, t.
Value
, (split.Offset + t.Offset.Index, t.Offset.Length)));
906
Debug.Assert(index + tokens[i].
Value
.Length <= indexMapping.Length);
908
if (tokens[i].Offset != (indexMapping[index], tokens[i].
Value
.Length))
918
list.Add(new EncodedToken(tokens[j].Id, tokens[j].
Value
, (indexMapping[index], tokens[j].
Value
.Length)));
919
index += tokens[j].
Value
.Length;
925
index += tokens[i].
Value
.Length;
Microsoft.ML.Tokenizers.Tests (56)
BpeTests.cs (5)
276
Assert.Equal(expectedTokens[i], encoding[i].
Value
);
280
Assert.Equal(encoding[i].
Value
, reverseVocabulary[encodingIds[i]]);
281
Assert.Equal(encodingIds[i], bpe.Vocabulary[encoding[i].
Value
]);
432
Assert.Equal(expectedTokens, encoding.Select(t => t.
Value
).ToArray());
436
Assert.Equal(expectedTokens, encoding1.Select(t => t.
Value
).ToArray());
CodeGenTests.cs (32)
237
Assert.Equal(expectedTokensWithSpace, encoding.Select(t => t.
Value
).ToArray());
243
Assert.Equal(expectedTokens, encoding.Select(t => t.
Value
).ToArray());
557
Assert.Equal(tokensList, encoding.Select(t => t.
Value
).ToArray());
562
Assert.Equal(tokensList, encoding.Select(t => t.
Value
).ToArray());
567
Assert.Equal(tokensList, encoding.Select(t => t.
Value
).ToArray());
572
Assert.Equal(tokensList, encoding.Select(t => t.
Value
).ToArray());
581
Assert.Equal(tokensList, encoding.Select(t => t.
Value
).ToArray());
586
Assert.Equal(tokensList, encoding.Select(t => t.
Value
).ToArray());
591
Assert.Equal(expectedTokens, encoding.Select(t => t.
Value
).ToArray());
596
Assert.Equal(expectedTokens, encoding.Select(t => t.
Value
).ToArray());
601
Assert.Equal(expectedTokensWithSpace, encoding.Select(t => t.
Value
).ToArray());
606
Assert.Equal(expectedTokensWithSpace, encoding.Select(t => t.
Value
).ToArray());
690
Assert.Equal(tokensList, encoding.Select(t => t.
Value
).ToArray());
695
Assert.Equal(tokensList, encoding.Select(t => t.
Value
).ToArray());
700
Assert.Equal(tokensList, encoding.Select(t => t.
Value
).ToArray());
705
Assert.Equal(tokensList, encoding.Select(t => t.
Value
).ToArray());
714
Assert.Equal(tokensList, encoding.Select(t => t.
Value
).ToArray());
719
Assert.Equal(tokensList, encoding.Select(t => t.
Value
).ToArray());
724
Assert.Equal(expectedTokens, encoding.Select(t => t.
Value
).ToArray());
729
Assert.Equal(expectedTokens, encoding.Select(t => t.
Value
).ToArray());
734
Assert.Equal(expectedTokensWithSpace, encoding.Select(t => t.
Value
).ToArray());
739
Assert.Equal(expectedTokensWithSpace, encoding.Select(t => t.
Value
).ToArray());
825
Assert.Equal(tokensList, encoding.Select(t => t.
Value
).ToArray());
831
Assert.Equal(tokensList, encoding.Select(t => t.
Value
).ToArray());
837
Assert.Equal(tokensList, encoding.Select(t => t.
Value
).ToArray());
843
Assert.Equal(tokensList, encoding.Select(t => t.
Value
).ToArray());
855
Assert.Equal(tokensList, encoding.Select(t => t.
Value
).ToArray());
861
Assert.Equal(tokensList, encoding.Select(t => t.
Value
).ToArray());
867
Assert.Equal(expectedTokens, encoding.Select(t => t.
Value
).ToArray());
873
Assert.Equal(expectedTokens, encoding.Select(t => t.
Value
).ToArray());
879
Assert.Equal(expectedTokensWithSpace, encoding.Select(t => t.
Value
).ToArray());
885
Assert.Equal(expectedTokensWithSpace, encoding.Select(t => t.
Value
).ToArray());
EnglishRobertaTests.cs (3)
184
Assert.Equal(expectedTokens, encoding.Select(t => t.
Value
).ToArray());
188
Assert.Equal(expectedTokens, encoding1.Select(t => t.
Value
).ToArray());
268
string[] tokens = encoding.Select(t => t.
Value
).ToArray();
LlamaTests.cs (8)
246
Assert.Equal(tokens, result.Select(t => t.
Value
).ToArray());
268
Assert.Equal(tokens.Skip(1), bpeTokens.Select(token => token.
Value
));
278
Assert.Equal(isEmptyInput ? Array.Empty<string>() : tokens.Skip(1).Concat(new[] { bpe.EndOfSentenceToken }), bpeTokens.Select(token => token.
Value
));
288
Assert.Equal(isEmptyInput ? Array.Empty<string>() : tokens.Concat(new[] { bpe.EndOfSentenceToken }), bpeTokens.Select(token => token.
Value
));
503
Assert.Equal(expectedTokens, encoding.Select(t => t.
Value
).ToArray());
507
Assert.Equal(expectedTokens, encoding1.Select(t => t.
Value
).ToArray());
541
Assert.Equal(expectedTokens1, encoding.Select(t => t.
Value
).ToArray());
665
Assert.Equal(new[] { tokenizer.BeginningOfSentenceToken, kvp.Key }, encodedTokens.Select(et => et.
Value
).ToArray());
TitokenTests.cs (8)
146
string[] tokens = result.Select(token => token.
Value
).ToArray();
199
string[] tokens = result.Select(token => token.
Value
).ToArray();
240
string[] tokens = result.Select(token => token.
Value
).ToArray();
276
Assert.Equal(new string[] { "<|im_start|>", "Hello", " ⭐", "⭐", " World", "<|im_end|>" }, result.Select(token => token.
Value
).ToArray());
311
Assert.Equal(new string[] { "<|endoftext|>", "Hello", " ⭐", " World", "<|endofprompt|>" }, result.Select(token => token.
Value
).ToArray());
567
Assert.Equal(expectedTokens, encoding.Select(t => t.
Value
).ToArray());
571
Assert.Equal(expectedTokens, encoding1.Select(t => t.
Value
).ToArray());
682
Assert.Equal(expectedTokens, result.Select(r => r.
Value
));
Microsoft.ML.TorchSharp (4)
NasBert\NerTrainer.cs (3)
179
if (NerTrainer.TokenStartsWithSpace(encoding[i].
Value
))
389
if (NerTrainer.TokenStartsWithSpace(encoding[i].
Value
))
403
if (NerTrainer.TokenStartsWithSpace(encoding[i].
Value
))
Roberta\QATrainer.cs (1)
405
var contextToken = contextTokens.Select(t => t.
Value
).ToArray();