1 write to Value
Microsoft.ML.Tokenizers (1)
EncodedToken.cs (1)
38Value = value;
70 references to Value
Microsoft.ML.Tokenizers (10)
Model\CodeGenTokenizer.cs (3)
1593tokens.Add(new EncodedToken(tokensToAdd[0].Id, tokensToAdd[0].Value, (offset == 0 ? tokensToAdd[0].Offset.Index : tokensToAdd[0].Offset.Index + offset - 1, offset == 0 ? tokensToAdd[0].Offset.Length - 1 : tokensToAdd[0].Offset.Length))); 1597tokens.Add(new EncodedToken(tokensToAdd[i].Id, tokensToAdd[i].Value, (tokensToAdd[i].Offset.Index + offset - 1, tokensToAdd[i].Offset.Length))); 1605tokens.Add(new EncodedToken(t.Id, t.Value, (t.Offset.Index + offset, t.Offset.Length)));
Model\EnglishRobertaTokenizer.cs (7)
328tokens.Add(new EncodedToken(t.Id, t.Value, (split.Offset + t.Offset.Index, t.Offset.Length))); 906Debug.Assert(index + tokens[i].Value.Length <= indexMapping.Length); 908if (tokens[i].Offset != (indexMapping[index], tokens[i].Value.Length)) 918list.Add(new EncodedToken(tokens[j].Id, tokens[j].Value, (indexMapping[index], tokens[j].Value.Length))); 919index += tokens[j].Value.Length; 925index += tokens[i].Value.Length;
Microsoft.ML.Tokenizers.Tests (56)
BpeTests.cs (5)
276Assert.Equal(expectedTokens[i], encoding[i].Value); 280Assert.Equal(encoding[i].Value, reverseVocabulary[encodingIds[i]]); 281Assert.Equal(encodingIds[i], bpe.Vocabulary[encoding[i].Value]); 432Assert.Equal(expectedTokens, encoding.Select(t => t.Value).ToArray()); 436Assert.Equal(expectedTokens, encoding1.Select(t => t.Value).ToArray());
CodeGenTests.cs (32)
237Assert.Equal(expectedTokensWithSpace, encoding.Select(t => t.Value).ToArray()); 243Assert.Equal(expectedTokens, encoding.Select(t => t.Value).ToArray()); 557Assert.Equal(tokensList, encoding.Select(t => t.Value).ToArray()); 562Assert.Equal(tokensList, encoding.Select(t => t.Value).ToArray()); 567Assert.Equal(tokensList, encoding.Select(t => t.Value).ToArray()); 572Assert.Equal(tokensList, encoding.Select(t => t.Value).ToArray()); 581Assert.Equal(tokensList, encoding.Select(t => t.Value).ToArray()); 586Assert.Equal(tokensList, encoding.Select(t => t.Value).ToArray()); 591Assert.Equal(expectedTokens, encoding.Select(t => t.Value).ToArray()); 596Assert.Equal(expectedTokens, encoding.Select(t => t.Value).ToArray()); 601Assert.Equal(expectedTokensWithSpace, encoding.Select(t => t.Value).ToArray()); 606Assert.Equal(expectedTokensWithSpace, encoding.Select(t => t.Value).ToArray()); 690Assert.Equal(tokensList, encoding.Select(t => t.Value).ToArray()); 695Assert.Equal(tokensList, encoding.Select(t => t.Value).ToArray()); 700Assert.Equal(tokensList, encoding.Select(t => t.Value).ToArray()); 705Assert.Equal(tokensList, encoding.Select(t => t.Value).ToArray()); 714Assert.Equal(tokensList, encoding.Select(t => t.Value).ToArray()); 719Assert.Equal(tokensList, encoding.Select(t => t.Value).ToArray()); 724Assert.Equal(expectedTokens, encoding.Select(t => t.Value).ToArray()); 729Assert.Equal(expectedTokens, encoding.Select(t => t.Value).ToArray()); 734Assert.Equal(expectedTokensWithSpace, encoding.Select(t => t.Value).ToArray()); 739Assert.Equal(expectedTokensWithSpace, encoding.Select(t => t.Value).ToArray()); 825Assert.Equal(tokensList, encoding.Select(t => t.Value).ToArray()); 831Assert.Equal(tokensList, encoding.Select(t => t.Value).ToArray()); 837Assert.Equal(tokensList, encoding.Select(t => t.Value).ToArray()); 843Assert.Equal(tokensList, encoding.Select(t => t.Value).ToArray()); 855Assert.Equal(tokensList, encoding.Select(t => t.Value).ToArray()); 861Assert.Equal(tokensList, encoding.Select(t => t.Value).ToArray()); 867Assert.Equal(expectedTokens, encoding.Select(t => t.Value).ToArray()); 873Assert.Equal(expectedTokens, encoding.Select(t => t.Value).ToArray()); 879Assert.Equal(expectedTokensWithSpace, encoding.Select(t => t.Value).ToArray()); 885Assert.Equal(expectedTokensWithSpace, encoding.Select(t => t.Value).ToArray());
EnglishRobertaTests.cs (3)
184Assert.Equal(expectedTokens, encoding.Select(t => t.Value).ToArray()); 188Assert.Equal(expectedTokens, encoding1.Select(t => t.Value).ToArray()); 268string[] tokens = encoding.Select(t => t.Value).ToArray();
LlamaTests.cs (8)
246Assert.Equal(tokens, result.Select(t => t.Value).ToArray()); 268Assert.Equal(tokens.Skip(1), bpeTokens.Select(token => token.Value)); 278Assert.Equal(isEmptyInput ? Array.Empty<string>() : tokens.Skip(1).Concat(new[] { bpe.EndOfSentenceToken }), bpeTokens.Select(token => token.Value)); 288Assert.Equal(isEmptyInput ? Array.Empty<string>() : tokens.Concat(new[] { bpe.EndOfSentenceToken }), bpeTokens.Select(token => token.Value)); 503Assert.Equal(expectedTokens, encoding.Select(t => t.Value).ToArray()); 507Assert.Equal(expectedTokens, encoding1.Select(t => t.Value).ToArray()); 541Assert.Equal(expectedTokens1, encoding.Select(t => t.Value).ToArray()); 665Assert.Equal(new[] { tokenizer.BeginningOfSentenceToken, kvp.Key }, encodedTokens.Select(et => et.Value).ToArray());
TitokenTests.cs (8)
146string[] tokens = result.Select(token => token.Value).ToArray(); 199string[] tokens = result.Select(token => token.Value).ToArray(); 240string[] tokens = result.Select(token => token.Value).ToArray(); 276Assert.Equal(new string[] { "<|im_start|>", "Hello", " ⭐", "⭐", " World", "<|im_end|>" }, result.Select(token => token.Value).ToArray()); 311Assert.Equal(new string[] { "<|endoftext|>", "Hello", " ⭐", " World", "<|endofprompt|>" }, result.Select(token => token.Value).ToArray()); 567Assert.Equal(expectedTokens, encoding.Select(t => t.Value).ToArray()); 571Assert.Equal(expectedTokens, encoding1.Select(t => t.Value).ToArray()); 682Assert.Equal(expectedTokens, result.Select(r => r.Value));
Microsoft.ML.TorchSharp (4)
NasBert\NerTrainer.cs (3)
179if (NerTrainer.TokenStartsWithSpace(encoding[i].Value)) 389if (NerTrainer.TokenStartsWithSpace(encoding[i].Value)) 403if (NerTrainer.TokenStartsWithSpace(encoding[i].Value))
Roberta\QATrainer.cs (1)
405var contextToken = contextTokens.Select(t => t.Value).ToArray();