36Id = id;

1018accumulatedIds.Add(t.Id); 1044accumulatedIds?.Add(tokens[k].Id); 1056accumulatedIds?.Add(tokens[tokenCount].Id); 1074accumulatedIds.Add(t.Id); 1092accumulatedIds?.Add(tokens[i].Id); 1593tokens.Add(new EncodedToken(tokensToAdd[0].Id, tokensToAdd[0].Value, (offset == 0 ? tokensToAdd[0].Offset.Index : tokensToAdd[0].Offset.Index + offset - 1, offset == 0 ? tokensToAdd[0].Offset.Length - 1 : tokensToAdd[0].Offset.Length))); 1597tokens.Add(new EncodedToken(tokensToAdd[i].Id, tokensToAdd[i].Value, (tokensToAdd[i].Offset.Index + offset - 1, tokensToAdd[i].Offset.Length))); 1605tokens.Add(new EncodedToken(t.Id, t.Value, (t.Offset.Index + offset, t.Offset.Length)));

328tokens.Add(new EncodedToken(t.Id, t.Value, (split.Offset + t.Offset.Index, t.Offset.Length))); 587accumulatedIds.Add(t.Id); 599accumulatedIds.Add(tokens[i].Id); 624accumulatedIds.Add(t.Id); 636accumulatedIds.Add(tokens[i].Id); 918list.Add(new EncodedToken(tokens[j].Id, tokens[j].Value, (indexMapping[index], tokens[j].Value.Length)));

49ids[i] = results.Tokens[i].Id;

258int[] encodingIds = encoding.Select(t => t.Id).ToArray(); 278Assert.Equal(ids[i], encoding[i].Id); 380Assert.Equal(encoding.Select(t => t.Id).ToArray(), ids); 434Assert.Equal(expectedIds, encoding.Select(t => t.Id).ToArray()); 438Assert.Equal(expectedIds, encoding1.Select(t => t.Id).ToArray());

236Assert.Equal(expectedIdsWithSpace, encoding.Select(t => t.Id).ToArray()); 242Assert.Equal(expectedIds, encoding.Select(t => t.Id).ToArray()); 253int[] ids = encoding.Select(t => t.Id).ToArray(); 256ids = encoding.Select(t => t.Id).ToArray(); 261ids = encoding.Select(t => t.Id).ToArray(); 264ids = encoding.Select(t => t.Id).ToArray(); 269ids = encoding.Select(t => t.Id).ToArray(); 272ids = encoding.Select(t => t.Id).ToArray(); 277ids = encoding.Select(t => t.Id).ToArray(); 280ids = encoding.Select(t => t.Id).ToArray(); 285ids = encoding.Select(t => t.Id).ToArray(); 288ids = encoding.Select(t => t.Id).ToArray(); 293ids = encoding.Select(t => t.Id).ToArray(); 297ids = encoding.Select(t => t.Id).ToArray(); 556Assert.Equal(idList, encoding.Select(t => t.Id).ToArray()); 561Assert.Equal(idList, encoding.Select(t => t.Id).ToArray()); 566Assert.Equal(idList, encoding.Select(t => t.Id).ToArray()); 571Assert.Equal(idList, encoding.Select(t => t.Id).ToArray()); 580Assert.Equal(idList, encoding.Select(t => t.Id).ToArray()); 585Assert.Equal(idList, encoding.Select(t => t.Id).ToArray()); 590Assert.Equal(expectedIds, encoding.Select(t => t.Id).ToArray()); 595Assert.Equal(expectedIds, encoding.Select(t => t.Id).ToArray()); 600Assert.Equal(expectedIdsWithSpace, encoding.Select(t => t.Id).ToArray()); 605Assert.Equal(expectedIdsWithSpace, encoding.Select(t => t.Id).ToArray()); 689Assert.Equal(idList, encoding.Select(t => t.Id).ToArray()); 694Assert.Equal(idList, encoding.Select(t => t.Id).ToArray()); 699Assert.Equal(idList, encoding.Select(t => t.Id).ToArray()); 704Assert.Equal(idList, encoding.Select(t => t.Id).ToArray()); 713Assert.Equal(idList, encoding.Select(t => t.Id).ToArray()); 718Assert.Equal(idList, encoding.Select(t => t.Id).ToArray()); 723Assert.Equal(expectedIds, encoding.Select(t => t.Id).ToArray()); 728Assert.Equal(expectedIds, encoding.Select(t => t.Id).ToArray()); 733Assert.Equal(expectedIdsWithSpace, encoding.Select(t => t.Id).ToArray()); 738Assert.Equal(expectedIdsWithSpace, encoding.Select(t => t.Id).ToArray()); 824Assert.Equal(idList, encoding.Select(t => t.Id).ToArray()); 830Assert.Equal(idList, encoding.Select(t => t.Id).ToArray()); 836Assert.Equal(idList, encoding.Select(t => t.Id).ToArray()); 842Assert.Equal(idList, encoding.Select(t => t.Id).ToArray()); 854Assert.Equal(idList, encoding.Select(t => t.Id).ToArray()); 860Assert.Equal(idList, encoding.Select(t => t.Id).ToArray()); 866Assert.Equal(expectedIds, encoding.Select(t => t.Id).ToArray()); 872Assert.Equal(expectedIds, encoding.Select(t => t.Id).ToArray()); 878Assert.Equal(expectedIdsWithSpace, encoding.Select(t => t.Id).ToArray()); 884Assert.Equal(expectedIdsWithSpace, encoding.Select(t => t.Id).ToArray());

186Assert.Equal(expectedIds, encoding.Select(t => t.Id).ToArray()); 190Assert.Equal(expectedIds, encoding1.Select(t => t.Id).ToArray()); 266int[] encodingIds = encoding.Select(t => t.Id).ToArray();

245Assert.Equal(ids, result.Select(t => t.Id).ToArray()); 267Assert.Equal(ids.Skip(1), bpeTokens.Select(token => token.Id)); 269int[] extractedIds = bpeTokens.Select(token => token.Id).ToArray(); 277Assert.Equal(isEmptyInput ? Array.Empty<int>() : ids.Skip(1).Concat(new[] { bpe.EndOfSentenceId }), bpeTokens.Select(token => token.Id)); 279extractedIds = bpeTokens.Select(token => token.Id).ToArray(); 287Assert.Equal(isEmptyInput ? Array.Empty<int>() : ids.Concat(new[] { bpe.EndOfSentenceId }), bpeTokens.Select(token => token.Id)); 289extractedIds = bpeTokens.Select(token => token.Id).ToArray(); 505Assert.Equal(expectedIds, encoding.Select(t => t.Id).ToArray()); 509Assert.Equal(expectedIds, encoding1.Select(t => t.Id).ToArray()); 543Assert.Equal(expectedIds1, encoding.Select(t => t.Id).ToArray()); 666Assert.Equal(new[] { tokenizer.BeginningOfSentenceId, kvp.Value }, encodedTokens.Select(et => et.Id).ToArray()); 670Assert.Equal(encodedIds, encodedTokens.Select(et => et.Id).ToArray());

145int[] ids = result.Select(token => token.Id).ToArray(); 198int[] ids = result.Select(token => token.Id).ToArray(); 239int[] ids = result.Select(token => token.Id).ToArray(); 274Assert.Equal(encoded, result.Select(token => token.Id).ToArray()); 309Assert.Equal(encoded, result.Select(token => token.Id).ToArray()); 569Assert.Equal(expectedIds, encoding.Select(t => t.Id).ToArray()); 573Assert.Equal(expectedIds, encoding1.Select(t => t.Id).ToArray()); 680int[] ids = result.Select(r => r.Id).ToArray();

190t = torch.tensor((ZeroArray).Concat(Tokenizer.RobertaModel().ConvertIdsToOccurrenceRanks(encoding.Select(t => t.Id).ToArray())).ToList(), device: Device);

406var contextTokenId = Tokenizer.RobertaModel().ConvertIdsToOccurrenceRanks(contextTokens.Select(t => t.Id).ToArray());