38Id = id;

44public bool Equals(EncodedToken other) => Id == other.Id && Value == other.Value && Offset.Equals(other.Offset);

1020accumulatedIds.Add(t.Id); 1046accumulatedIds?.Add(tokens[k].Id); 1058accumulatedIds?.Add(tokens[tokenCount].Id); 1076accumulatedIds.Add(t.Id); 1094accumulatedIds?.Add(tokens[i].Id); 1596tokens.Add(new EncodedToken(tokensToAdd[0].Id, tokensToAdd[0].Value, new Range(r.s, r.e))); 1600tokens.Add(new EncodedToken(tokensToAdd[i].Id, tokensToAdd[i].Value, new Range(tokensToAdd[i].Offset.Start.Value + offset - 1, tokensToAdd[i].Offset.End.Value + offset - 1))); 1608tokens.Add(new EncodedToken(t.Id, t.Value, new Range(t.Offset.Start.Value + offset, t.Offset.End.Value + offset)));

339tokens.Add(new EncodedToken(t.Id, t.Value, new Range(split.Offset + t.Offset.Start.Value, split.Offset + t.Offset.End.Value))); 598accumulatedIds.Add(t.Id); 610accumulatedIds.Add(tokens[i].Id); 635accumulatedIds.Add(t.Id); 647accumulatedIds.Add(tokens[i].Id); 929list.Add(new EncodedToken(tokens[j].Id, tokens[j].Value, new Range(indexMapping[index], indexMapping[index] + tokens[j].Value.Length)));

49ids[i] = results.Tokens[i].Id;

258int[] encodingIds = encoding.Select(t => t.Id).ToArray(); 278Assert.Equal(ids[i], encoding[i].Id); 380Assert.Equal(encoding.Select(t => t.Id).ToArray(), ids); 434Assert.Equal(expectedIds, encoding.Select(t => t.Id).ToArray()); 438Assert.Equal(expectedIds, encoding1.Select(t => t.Id).ToArray()); 528Assert.Equal(expectedTokens.Select(t => t.Id).ToArray(), ids);

236Assert.Equal(expectedIdsWithSpace, encoding.Select(t => t.Id).ToArray()); 242Assert.Equal(expectedIds, encoding.Select(t => t.Id).ToArray()); 253int[] ids = encoding.Select(t => t.Id).ToArray(); 256ids = encoding.Select(t => t.Id).ToArray(); 261ids = encoding.Select(t => t.Id).ToArray(); 264ids = encoding.Select(t => t.Id).ToArray(); 269ids = encoding.Select(t => t.Id).ToArray(); 272ids = encoding.Select(t => t.Id).ToArray(); 277ids = encoding.Select(t => t.Id).ToArray(); 280ids = encoding.Select(t => t.Id).ToArray(); 285ids = encoding.Select(t => t.Id).ToArray(); 288ids = encoding.Select(t => t.Id).ToArray(); 293ids = encoding.Select(t => t.Id).ToArray(); 297ids = encoding.Select(t => t.Id).ToArray(); 556Assert.Equal(idList, encoding.Select(t => t.Id).ToArray()); 561Assert.Equal(idList, encoding.Select(t => t.Id).ToArray()); 566Assert.Equal(idList, encoding.Select(t => t.Id).ToArray()); 571Assert.Equal(idList, encoding.Select(t => t.Id).ToArray()); 580Assert.Equal(idList, encoding.Select(t => t.Id).ToArray()); 585Assert.Equal(idList, encoding.Select(t => t.Id).ToArray()); 590Assert.Equal(expectedIds, encoding.Select(t => t.Id).ToArray()); 595Assert.Equal(expectedIds, encoding.Select(t => t.Id).ToArray()); 600Assert.Equal(expectedIdsWithSpace, encoding.Select(t => t.Id).ToArray()); 605Assert.Equal(expectedIdsWithSpace, encoding.Select(t => t.Id).ToArray()); 689Assert.Equal(idList, encoding.Select(t => t.Id).ToArray()); 694Assert.Equal(idList, encoding.Select(t => t.Id).ToArray()); 699Assert.Equal(idList, encoding.Select(t => t.Id).ToArray()); 704Assert.Equal(idList, encoding.Select(t => t.Id).ToArray()); 713Assert.Equal(idList, encoding.Select(t => t.Id).ToArray()); 718Assert.Equal(idList, encoding.Select(t => t.Id).ToArray()); 723Assert.Equal(expectedIds, encoding.Select(t => t.Id).ToArray()); 728Assert.Equal(expectedIds, encoding.Select(t => t.Id).ToArray()); 733Assert.Equal(expectedIdsWithSpace, encoding.Select(t => t.Id).ToArray()); 738Assert.Equal(expectedIdsWithSpace, encoding.Select(t => t.Id).ToArray()); 824Assert.Equal(idList, encoding.Select(t => t.Id).ToArray()); 830Assert.Equal(idList, encoding.Select(t => t.Id).ToArray()); 836Assert.Equal(idList, encoding.Select(t => t.Id).ToArray()); 842Assert.Equal(idList, encoding.Select(t => t.Id).ToArray()); 854Assert.Equal(idList, encoding.Select(t => t.Id).ToArray()); 860Assert.Equal(idList, encoding.Select(t => t.Id).ToArray()); 866Assert.Equal(expectedIds, encoding.Select(t => t.Id).ToArray()); 872Assert.Equal(expectedIds, encoding.Select(t => t.Id).ToArray()); 878Assert.Equal(expectedIdsWithSpace, encoding.Select(t => t.Id).ToArray()); 884Assert.Equal(expectedIdsWithSpace, encoding.Select(t => t.Id).ToArray());

186Assert.Equal(expectedIds, encoding.Select(t => t.Id).ToArray()); 190Assert.Equal(expectedIds, encoding1.Select(t => t.Id).ToArray()); 266int[] encodingIds = encoding.Select(t => t.Id).ToArray();

245Assert.Equal(ids, result.Select(t => t.Id).ToArray()); 267Assert.Equal(ids.Skip(1), bpeTokens.Select(token => token.Id)); 269int[] extractedIds = bpeTokens.Select(token => token.Id).ToArray(); 277Assert.Equal(isEmptyInput ? Array.Empty<int>() : ids.Skip(1).Concat(new[] { bpe.EndOfSentenceId }), bpeTokens.Select(token => token.Id)); 279extractedIds = bpeTokens.Select(token => token.Id).ToArray(); 287Assert.Equal(isEmptyInput ? Array.Empty<int>() : ids.Concat(new[] { bpe.EndOfSentenceId }), bpeTokens.Select(token => token.Id)); 289extractedIds = bpeTokens.Select(token => token.Id).ToArray(); 505Assert.Equal(expectedIds, encoding.Select(t => t.Id).ToArray()); 509Assert.Equal(expectedIds, encoding1.Select(t => t.Id).ToArray()); 543Assert.Equal(expectedIds1, encoding.Select(t => t.Id).ToArray()); 666Assert.Equal(new[] { tokenizer.BeginningOfSentenceId, kvp.Value }, encodedTokens.Select(et => et.Id).ToArray()); 670Assert.Equal(encodedIds, encodedTokens.Select(et => et.Id).ToArray());

146int[] ids = result.Select(token => token.Id).ToArray(); 199int[] ids = result.Select(token => token.Id).ToArray(); 240int[] ids = result.Select(token => token.Id).ToArray(); 275Assert.Equal(encoded, result.Select(token => token.Id).ToArray()); 310Assert.Equal(encoded, result.Select(token => token.Id).ToArray()); 573Assert.Equal(expectedIds, encoding.Select(t => t.Id).ToArray()); 577Assert.Equal(expectedIds, encoding1.Select(t => t.Id).ToArray()); 684int[] ids = result.Select(r => r.Id).ToArray();

190t = torch.tensor((ZeroArray).Concat(Tokenizer.RobertaModel().ConvertIdsToOccurrenceRanks(encoding.Select(t => t.Id).ToArray())).ToList(), device: Device);

406var contextTokenId = Tokenizer.RobertaModel().ConvertIdsToOccurrenceRanks(contextTokens.Select(t => t.Id).ToArray());