1 write to Value
Microsoft.ML.Tokenizers (1)
EncodedToken.cs (1)
40Value = value;
73 references to Value
Microsoft.ML.Tokenizers (13)
EncodedToken.cs (2)
44public bool Equals(EncodedToken other) => Id == other.Id && Value == other.Value && Offset.Equals(other.Offset);
Model\CodeGenTokenizer.cs (3)
1596tokens.Add(new EncodedToken(tokensToAdd[0].Id, tokensToAdd[0].Value, new Range(r.s, r.e))); 1600tokens.Add(new EncodedToken(tokensToAdd[i].Id, tokensToAdd[i].Value, new Range(tokensToAdd[i].Offset.Start.Value + offset - 1, tokensToAdd[i].Offset.End.Value + offset - 1))); 1608tokens.Add(new EncodedToken(t.Id, t.Value, new Range(t.Offset.Start.Value + offset, t.Offset.End.Value + offset)));
Model\EnglishRobertaTokenizer.cs (7)
339tokens.Add(new EncodedToken(t.Id, t.Value, new Range(split.Offset + t.Offset.Start.Value, split.Offset + t.Offset.End.Value))); 917Debug.Assert(index + tokens[i].Value.Length <= indexMapping.Length); 919if (tokens[i].Offset.Start.Value != indexMapping[index] || tokens[i].Offset.End.Value != indexMapping[index] + tokens[i].Value.Length) 929list.Add(new EncodedToken(tokens[j].Id, tokens[j].Value, new Range(indexMapping[index], indexMapping[index] + tokens[j].Value.Length))); 930index += tokens[j].Value.Length; 936index += tokens[i].Value.Length;
Model\WordPieceTokenizer.cs (1)
359if (curToken.Value is null)
Microsoft.ML.Tokenizers.Tests (56)
BpeTests.cs (5)
276Assert.Equal(expectedTokens[i], encoding[i].Value); 280Assert.Equal(encoding[i].Value, reverseVocabulary[encodingIds[i]]); 281Assert.Equal(encodingIds[i], bpe.Vocabulary[encoding[i].Value]); 432Assert.Equal(expectedTokens, encoding.Select(t => t.Value).ToArray()); 436Assert.Equal(expectedTokens, encoding1.Select(t => t.Value).ToArray());
CodeGenTests.cs (32)
237Assert.Equal(expectedTokensWithSpace, encoding.Select(t => t.Value).ToArray()); 243Assert.Equal(expectedTokens, encoding.Select(t => t.Value).ToArray()); 557Assert.Equal(tokensList, encoding.Select(t => t.Value).ToArray()); 562Assert.Equal(tokensList, encoding.Select(t => t.Value).ToArray()); 567Assert.Equal(tokensList, encoding.Select(t => t.Value).ToArray()); 572Assert.Equal(tokensList, encoding.Select(t => t.Value).ToArray()); 581Assert.Equal(tokensList, encoding.Select(t => t.Value).ToArray()); 586Assert.Equal(tokensList, encoding.Select(t => t.Value).ToArray()); 591Assert.Equal(expectedTokens, encoding.Select(t => t.Value).ToArray()); 596Assert.Equal(expectedTokens, encoding.Select(t => t.Value).ToArray()); 601Assert.Equal(expectedTokensWithSpace, encoding.Select(t => t.Value).ToArray()); 606Assert.Equal(expectedTokensWithSpace, encoding.Select(t => t.Value).ToArray()); 690Assert.Equal(tokensList, encoding.Select(t => t.Value).ToArray()); 695Assert.Equal(tokensList, encoding.Select(t => t.Value).ToArray()); 700Assert.Equal(tokensList, encoding.Select(t => t.Value).ToArray()); 705Assert.Equal(tokensList, encoding.Select(t => t.Value).ToArray()); 714Assert.Equal(tokensList, encoding.Select(t => t.Value).ToArray()); 719Assert.Equal(tokensList, encoding.Select(t => t.Value).ToArray()); 724Assert.Equal(expectedTokens, encoding.Select(t => t.Value).ToArray()); 729Assert.Equal(expectedTokens, encoding.Select(t => t.Value).ToArray()); 734Assert.Equal(expectedTokensWithSpace, encoding.Select(t => t.Value).ToArray()); 739Assert.Equal(expectedTokensWithSpace, encoding.Select(t => t.Value).ToArray()); 825Assert.Equal(tokensList, encoding.Select(t => t.Value).ToArray()); 831Assert.Equal(tokensList, encoding.Select(t => t.Value).ToArray()); 837Assert.Equal(tokensList, encoding.Select(t => t.Value).ToArray()); 843Assert.Equal(tokensList, encoding.Select(t => t.Value).ToArray()); 855Assert.Equal(tokensList, encoding.Select(t => t.Value).ToArray()); 861Assert.Equal(tokensList, encoding.Select(t => t.Value).ToArray()); 867Assert.Equal(expectedTokens, encoding.Select(t => t.Value).ToArray()); 873Assert.Equal(expectedTokens, encoding.Select(t => t.Value).ToArray()); 879Assert.Equal(expectedTokensWithSpace, encoding.Select(t => t.Value).ToArray()); 885Assert.Equal(expectedTokensWithSpace, encoding.Select(t => t.Value).ToArray());
EnglishRobertaTests.cs (3)
184Assert.Equal(expectedTokens, encoding.Select(t => t.Value).ToArray()); 188Assert.Equal(expectedTokens, encoding1.Select(t => t.Value).ToArray()); 268string[] tokens = encoding.Select(t => t.Value).ToArray();
LlamaTests.cs (8)
246Assert.Equal(tokens, result.Select(t => t.Value).ToArray()); 268Assert.Equal(tokens.Skip(1), bpeTokens.Select(token => token.Value)); 278Assert.Equal(isEmptyInput ? Array.Empty<string>() : tokens.Skip(1).Concat(new[] { bpe.EndOfSentenceToken }), bpeTokens.Select(token => token.Value)); 288Assert.Equal(isEmptyInput ? Array.Empty<string>() : tokens.Concat(new[] { bpe.EndOfSentenceToken }), bpeTokens.Select(token => token.Value)); 503Assert.Equal(expectedTokens, encoding.Select(t => t.Value).ToArray()); 507Assert.Equal(expectedTokens, encoding1.Select(t => t.Value).ToArray()); 541Assert.Equal(expectedTokens1, encoding.Select(t => t.Value).ToArray()); 665Assert.Equal(new[] { tokenizer.BeginningOfSentenceToken, kvp.Key }, encodedTokens.Select(et => et.Value).ToArray());
TiktokenTests.cs (8)
147string[] tokens = result.Select(token => token.Value).ToArray(); 200string[] tokens = result.Select(token => token.Value).ToArray(); 241string[] tokens = result.Select(token => token.Value).ToArray(); 277Assert.Equal(new string[] { "<|im_start|>", "Hello", " ⭐", "⭐", " World", "<|im_end|>" }, result.Select(token => token.Value).ToArray()); 312Assert.Equal(new string[] { "<|endoftext|>", "Hello", " ⭐", " World", "<|endofprompt|>" }, result.Select(token => token.Value).ToArray()); 571Assert.Equal(expectedTokens, encoding.Select(t => t.Value).ToArray()); 575Assert.Equal(expectedTokens, encoding1.Select(t => t.Value).ToArray()); 686Assert.Equal(expectedTokens, result.Select(r => r.Value));
Microsoft.ML.TorchSharp (4)
NasBert\NerTrainer.cs (3)
179if (NerTrainer.TokenStartsWithSpace(encoding[i].Value)) 389if (NerTrainer.TokenStartsWithSpace(encoding[i].Value)) 403if (NerTrainer.TokenStartsWithSpace(encoding[i].Value))
Roberta\QATrainer.cs (1)
405var contextToken = contextTokens.Select(t => t.Value).ToArray();