382 references to Range
Microsoft.Build.Framework (3)
Range.cs (3)
69public static Range StartAt(Index start) => new Range(start, Index.End); 72public static Range EndAt(Index end) => new Range(Index.Start, end); 75public static Range All => new Range(Index.Start, Index.End);
Microsoft.CodeAnalysis.CSharp (3)
Parser\Lexer_StringLiteral.cs (3)
890new Range(openBracePosition, openBracePosition + 1), 892new Range(closeBracePosition, _lexer.TextWindow.Position))); 1059colonRange = new Range(_lexer.TextWindow.Position, _lexer.TextWindow.Position + 1);
Microsoft.CodeAnalysis.Features (1)
EditAndContinue\ActiveStatementsMap.cs (1)
324return new Range(start, end);
Microsoft.CodeAnalysis.Features.UnitTests (4)
EditAndContinue\ActiveStatementsMapTests.cs (4)
42Assert.Equal(new Range(s, e), ActiveStatementsMap.GetSpansStartingInSpan(span.Start, span.End, array, startPositionComparer: (x, y) => x.Start.CompareTo(y))); 60Assert.Equal(new Range(5, 6), ActiveStatementsMap.GetSpansStartingInSpan(span.Start, span.End, array, startPositionComparer: (x, y) => x.Start.CompareTo(y))); 74Assert.Equal(new Range(s, e), ActiveStatementsMap.GetSpansStartingInSpan(span.Start, span.End, array, startPositionComparer: (x, y) => x.Start.CompareTo(y))); 87Assert.Equal(new Range(s, e), ActiveStatementsMap.GetSpansStartingInSpan(span.Start, span.End, array, startPositionComparer: (x, y) => x.Start.CompareTo(y)));
Microsoft.ML.GenAI.Core (1)
Pipeline\CausalLMPipeline.cs (1)
262.Where(t => !t.Offset.Equals(new Range(0, 0)))
Microsoft.ML.Tokenizers (42)
Model\BPETokenizer.cs (3)
457tokens.Add(new EncodedToken(BeginningOfSentenceId, BeginningOfSentenceToken, new Range(0, 0))); 476tokens.Add(new EncodedToken(EndOfSentenceId, EndOfSentenceToken, new Range(charsConsumed, charsConsumed))); 1311tokens.Add(new EncodedToken(value.specialTokenId, value.specialToken, new Range(offset, offset + text.Length)));
Model\CodeGenTokenizer.cs (8)
382tokens.Add(new EncodedToken(BeginningOfSentenceId.Value, BeginningOfSentenceToken!, new Range(0, 0))); 402tokens.Add(new EncodedToken(EndOfSentenceId.Value, EndOfSentenceToken!, new Range(index, index))); 435tokens.Add(new EncodedToken(value.specialTokenId, value.specialToken, new Range(index, index + ((addPrefixSpace && offset == 0) ? textSpan.Length - 1 : textSpan.Length)))); 1578tokens.Add(new EncodedToken(tokensToAdd[0].Id, tokensToAdd[0].Value, new Range(r.s, r.e))); 1582tokens.Add(new EncodedToken(tokensToAdd[i].Id, tokensToAdd[i].Value, new Range(tokensToAdd[i].Offset.Start.Value + offset - 1, tokensToAdd[i].Offset.End.Value + offset - 1))); 1590tokens.Add(new EncodedToken(t.Id, t.Value, new Range(t.Offset.Start.Value + offset, t.Offset.End.Value + offset))); 1610return new List<EncodedToken> { new EncodedToken(_vocab[new StringSpanOrdinalKey(tokenValue)].Id, tokenValue, new Range(mapping[0], mapping[0] + 1)) }; 1683return new EncodedToken(id, token, new Range(mapping[index], endIndex));
Model\EnglishRobertaTokenizer.cs (4)
339tokens.Add(new EncodedToken(t.Id, t.Value, new Range(split.Offset + t.Offset.Start.Value, split.Offset + t.Offset.End.Value))); 929list.Add(new EncodedToken(tokens[j].Id, tokens[j].Value, new Range(indexMapping[index], indexMapping[index] + tokens[j].Value.Length))); 961return new List<EncodedToken> { new EncodedToken(_vocab[new StringSpanOrdinalKey(tokenValue)], tokenValue, new Range(indexMapping[0], indexMapping[0] + 1)) }; 1050tokens.Add(new EncodedToken(_vocab[new StringSpanOrdinalKey(w)], w, new Range(indexMapping[index], indexMapping[index] + w.Length)));
Model\SentencePieceBpeModel.cs (9)
111tokens.Add(new EncodedToken(BeginningOfSentenceId, BeginningOfSentenceToken, new Range(0, 0))); 125tokens.Add(new EncodedToken(id, SpecialTokensReverse![id], new Range(Offset, Offset + Length))); 138tokens.Add(new EncodedToken(EndOfSentenceId, EndOfSentenceToken, new Range(text.Length, text.Length))); 158tokens.Add(new EncodedToken(BeginningOfSentenceId, BeginningOfSentenceToken, new Range(0, 0))); 191new Range(symbols[index].pieceSpan.Index, symbols[index].pieceSpan.Index + symbols[index].pieceSpan.Length))); 203tokens.Add(new EncodedToken(EndOfSentenceId, EndOfSentenceToken, new Range(text.Length, text.Length))); 220tokens.Add(new EncodedToken(id, token, new Range(index + i, index + i + 1))); 244tokens.Add(new EncodedToken(id, token, new Range(index + i, index + i + length))); 272tokens.Add(new EncodedToken(id.Id, text.Slice(pieceSpan.Index, pieceSpan.Length).ToString(), new Range(pieceSpan.Index, pieceSpan.Index + pieceSpan.Length)));
Model\SentencePieceUnigramModel.cs (10)
221tokens.Add(new EncodedToken(BeginningOfSentenceId, BeginningOfSentenceToken, new Range(0, 0))); 237tokens.Add(new EncodedToken(id, SpecialTokensReverse![id], new Range(progressOffset, progressOffset + Length))); 253tokens.Add(new EncodedToken(EndOfSentenceId, EndOfSentenceToken, new Range(progressOffset, progressOffset))); 271tokens.Add(new EncodedToken(BeginningOfSentenceId, BeginningOfSentenceToken, new Range(0, 0))); 281tokens.Add(new EncodedToken(EndOfSentenceId, EndOfSentenceToken, new Range(progressOffset, progressOffset))); 365tokens.Add(new EncodedToken(node.Id, stringToken, new Range(0, tokenLength))); // we will update the range later. 382tokens[start] = new EncodedToken(tokens[start].Id, tokens[start].Value, new Range(tokensOffset, tokensOffset + tokenLength)); 393tokens[start] = new EncodedToken(tokens[start].Id, tokens[start].Value, new Range(tokensOffset, tokensOffset + tokenLength)); 434tokens.Insert(insertionStartPosition++, new EncodedToken(id, _vocabReverse[id].Piece, new Range(offsetStart, offsetStart + charLength))); 439tokens.Insert(insertionStartPosition++, new EncodedToken(id, _vocabReverse[id].Piece, new Range(offsetStart + charLength, offsetStart + charLength)));
Model\TiktokenTokenizer.cs (3)
310new Range(value[i].TokenIndex + offset, value[i].TokenIndex + offset + value[i].TokenLength))); 319tokens.Add(new EncodedToken(mappedId.Id, mappedId.Token, new Range(offset, offset + mappedId.Token.Length))); 351new Range(encodedTokens[i].TokenIndex + offset, encodedTokens[i].TokenIndex + offset + encodedTokens[i].TokenLength)));
Model\Word.cs (2)
301tokens.Add(new EncodedToken(_symbols[i].C, vocabReverse[_symbols[i].C], new Range(index + offset, index + offset + _symbols[i].Len))); 314tokens.Add(new EncodedToken(_symbols[i].C, vocabReverse[_symbols[i].C], new Range(mappedIndex + offset, mappedEndIndex + offset)));
Model\WordPieceTokenizer.cs (3)
319tokens.Add(new EncodedToken(UnknownTokenId, UnknownToken, new Range(offset, offset + text.Length))); 352curToken = new EncodedToken(id, _vocabReverse[id], new Range(offset + start, offset + end)); 373tokens.Add(new EncodedToken(UnknownTokenId, UnknownToken, new Range(offset, offset + textLength)));
Microsoft.ML.Tokenizers.Tests (320)
BertTokenizerTests.cs (66)
60new EncodedToken(8, "hello", new Range(0, 5)), 61new EncodedToken(6, ",", new Range(5, 6)), 62new EncodedToken(10, "how", new Range(7, 10)), 63new EncodedToken(11, "are", new Range(11, 14)), 64new EncodedToken(12, "you", new Range(15, 18)), 65new EncodedToken(13, "[SPECIAL]", new Range(19, 28)), 66new EncodedToken(7, "?", new Range(28, 29)) 80new EncodedToken(2, "[CLS]", new Range(0, 5)), 81new EncodedToken(8, "hello", new Range(6, 11)), 82new EncodedToken(6, ",", new Range(11, 12)), 83new EncodedToken(10, "how", new Range(13, 16)), 84new EncodedToken(11, "are", new Range(17, 20)), 85new EncodedToken(12, "you", new Range(21, 24)), 86new EncodedToken(13, "[SPECIAL]", new Range(25, 34)), 87new EncodedToken(7, "?", new Range(34, 35)), 88new EncodedToken(3, "[SEP]", new Range(36, 41)) 133new EncodedToken(8, "hello", new Range(0, 5)), 134new EncodedToken(6, ",", new Range(5, 6)), 135new EncodedToken(10, "how", new Range(7, 10)), 136new EncodedToken(11, "are", new Range(11, 14)), 137new EncodedToken(12, "you", new Range(15, 18)), 138new EncodedToken(7, "?", new Range(18, 19)) 152new EncodedToken(2, "[CLS]", new Range(0, 5)), 153new EncodedToken(8, "hello", new Range(6, 11)), 154new EncodedToken(6, ",", new Range(11, 12)), 155new EncodedToken(10, "how", new Range(13, 16)), 156new EncodedToken(11, "are", new Range(17, 20)), 157new EncodedToken(12, "you", new Range(21, 24)), 158new EncodedToken(7, "?", new Range(24, 25)), 159new EncodedToken(3, "[SEP]", new Range(26, 31)) 201new EncodedToken(1, "[UNK]", new Range(0, 5)), 202new EncodedToken(6, ",", new Range(5, 6)), 203new EncodedToken(1, "[UNK]", new Range(7, 10)), 204new EncodedToken(11, "are", new Range(11, 14)), 205new EncodedToken(12, "you", new Range(15, 18)), 206new EncodedToken(7, "?", new Range(18, 19)) 241new EncodedToken(10, "café", new Range(0, 4)), 242new EncodedToken(12, "über", new Range(5, 9)), 243new EncodedToken(15, "ångström", new Range(10, 18)), 244new EncodedToken(18, "résumé", new Range(19, 25)), 245new EncodedToken(5, "!", new Range(25, 26)), 256new EncodedToken(8, "Café", new Range(0, 4)), 257new EncodedToken(11, "Über", new Range(5, 9)), 258new EncodedToken(14, "Ångström", new Range(10, 18)), 259new EncodedToken(17, "Résumé", new Range(19, 25)), 260new EncodedToken(5, "!", new Range(25, 26)), 272new EncodedToken(9, "cafe", new Range(0, 4)), 273new EncodedToken(13, "uber", new Range(5, 9)), 274new EncodedToken(16, "angstrom", new Range(10, 18)), 275new EncodedToken(19, "resume", new Range(19, 25)), 276new EncodedToken(5, "!", new Range(25, 26)), 286new EncodedToken(20, "Cafe", new Range(0, 4)), 287new EncodedToken(21, "Uber", new Range(5, 9)), 288new EncodedToken(22, "Angstrom", new Range(10, 18)), 289new EncodedToken(23, "Resume", new Range(19, 25)), 290new EncodedToken(5, "!", new Range(25, 26)), 317new EncodedToken(9, "叟", new Range(1, 2)), 318new EncodedToken(11, "驷", new Range(4, 5)), 319new EncodedToken(10, "叢", new Range(8, 9)), 320new EncodedToken(12, "驸", new Range(11, 12)), 321new EncodedToken(5, "!", new Range(13, 14)) 335new EncodedToken(9, "叟", new Range(0, 1)), 336new EncodedToken(6, "##驷", new Range(1, 2)), 337new EncodedToken(10, "叢", new Range(3, 4)), 338new EncodedToken(7, "##驸", new Range(4, 5)), 339new EncodedToken(5, "!", new Range(5, 6))
BpeTests.cs (13)
556new EncodedToken(15496, "Hello", new Range(0, 5)), 557new EncodedToken(11, ",", new Range(5, 6)), 558new EncodedToken(88, "y", new Range(7, 8)), 559new EncodedToken(6, "'", new Range(8, 9)), 560new EncodedToken(439, "all", new Range(9, 12)), 561new EncodedToken(0, "!", new Range(12, 13)), 562new EncodedToken(9, "<issue_comment>", new Range(14, 29)), 563new EncodedToken(2437, "How", new Range(29, 32)), 564new EncodedToken(533, "are", new Range(33, 36)), 565new EncodedToken(5832, "you", new Range(37, 40)), 566new EncodedToken(50256, "<|endoftext|>", new Range(41, 43)), 567new EncodedToken(30, "?", new Range(44, 45)), 568new EncodedToken(0, "<|endoftext|>", new Range(45, 58))
CodeGenTests.cs (42)
592Assert.True(!encoding[0].Offset.Equals(new Range(0, 0)) || !encoding[1].Offset.Equals(new Range(0, 0))); 597Assert.True(!encoding[0].Offset.Equals(new Range(0, 0)) || !encoding[1].Offset.Equals(new Range(0, 0))); 602Assert.True(!encoding[0].Offset.Equals(new Range(0, 0)) || !encoding[1].Offset.Equals(new Range(0, 0))); 607Assert.True(!encoding[0].Offset.Equals(new Range(0, 0)) || !encoding[1].Offset.Equals(new Range(0, 0))); 691Assert.Equal(new Range(text.Length, text.Length), encoding[encoding.Count - 1].Offset); 696Assert.Equal(new Range(text.Length, text.Length), encoding[encoding.Count - 1].Offset); 701Assert.Equal(new Range(text.Length, text.Length), encoding[encoding.Count - 1].Offset); 706Assert.Equal(new Range(text.Length, text.Length), encoding[encoding.Count - 1].Offset); 715Assert.Equal(new Range(text.Length, text.Length), encoding[encoding.Count - 1].Offset); 720Assert.Equal(new Range(text.Length, text.Length), encoding[encoding.Count - 1].Offset); 725Assert.NotEqual(new Range(text.Length, text.Length), encoding[encoding.Count - 1].Offset); 730Assert.NotEqual(new Range(text.Length, text.Length), encoding[encoding.Count - 1].Offset); 735Assert.NotEqual(new Range(text.Length, text.Length), encoding[encoding.Count - 1].Offset); 740Assert.NotEqual(new Range(text.Length, text.Length), encoding[encoding.Count - 1].Offset); 826Assert.Equal(new Range(0, 0), encoding[0].Offset); 827Assert.Equal(new Range(text.Length, text.Length), encoding[encoding.Count - 1].Offset); 832Assert.Equal(new Range(0, 0), encoding[0].Offset); 833Assert.Equal(new Range(text.Length, text.Length), encoding[encoding.Count - 1].Offset); 838Assert.Equal(new Range(0, 0), encoding[0].Offset); 839Assert.Equal(new Range(text.Length, text.Length), encoding[encoding.Count - 1].Offset); 844Assert.Equal(new Range(0, 0), encoding[0].Offset); 845Assert.Equal(new Range(text.Length, text.Length), encoding[encoding.Count - 1].Offset); 856Assert.Equal(new Range(0, 0), encoding[0].Offset); 857Assert.Equal(new Range(text.Length, text.Length), encoding[encoding.Count - 1].Offset); 862Assert.Equal(new Range(0, 0), encoding[0].Offset); 863Assert.Equal(new Range(text.Length, text.Length), encoding[encoding.Count - 1].Offset); 868Assert.True(!encoding[0].Offset.Equals(new Range(0, 0)) || !encoding[1].Offset.Equals(new Range(0, 0))); 869Assert.NotEqual(new Range(text.Length, text.Length), encoding[encoding.Count - 1].Offset); 874Assert.True(!encoding[0].Offset.Equals(new Range(0, 0)) || !encoding[1].Offset.Equals(new Range(0, 0))); 875Assert.NotEqual(new Range(text.Length, text.Length), encoding[encoding.Count - 1].Offset); 880Assert.True(!encoding[0].Offset.Equals(new Range(0, 0)) || !encoding[1].Offset.Equals(new Range(0, 0))); 881Assert.NotEqual(new Range(text.Length, text.Length), encoding[encoding.Count - 1].Offset); 886Assert.True(!encoding[0].Offset.Equals(new Range(0, 0)) || !encoding[1].Offset.Equals(new Range(0, 0))); 887Assert.NotEqual(new Range(text.Length, text.Length), encoding[encoding.Count - 1].Offset);
TiktokenTests.cs (2)
160Assert.Equal(new List<Range> { new Range(0, 5), new Range(5, 11) }, offsets);
TokenizerTests.cs (1)
115tokens.Add(new EncodedToken(c - 'a', c.ToString(), new Range(count, count + 1)));
UnigramTests.cs (179)
80new Range[] { new Range(0, 6), new Range(6, 7), new Range(7, 13), new Range(13, 14) } 92new Range(0, 6), new Range(6, 7), new Range(7, 9), new Range(9, 10), new Range(10, 11), new Range(11, 12), 93new Range(12, 13), new Range(13, 15), new Range(15, 17), new Range(17, 18), new Range(18, 19), new Range(19, 21), 94new Range(21, 22), new Range(22, 24), new Range(24, 30), new Range(30, 31), new Range(31, 32), new Range(32, 34) 115new Range[] { new Range(0, 1), new Range(1, 2) } 125new Range[] { new Range(0, 2), new Range(2, 3) } 169new Range(0, 4), new Range(4, 8), new Range(8, 11), new Range(11, 15), new Range(15, 21), new Range(21, 25), 170new Range(25, 26), new Range(26, 33), new Range(33, 34), new Range(34, 42), new Range(42, 44), new Range(44, 49), new Range(49, 56), 171new Range(56, 60), new Range(60, 67), new Range(67, 71), new Range(71, 80), new Range(80, 84), new Range(84, 87), new Range(87, 88), 172new Range(88, 93), new Range(93, 94), new Range(94, 100), new Range(100, 102), new Range(102, 104), new Range(104, 109), new Range(109, 114), 173new Range(114, 117), new Range(117, 122), new Range(122, 123), new Range(123, 129), new Range(129, 136), new Range(136, 137), 174new Range(137, 143), new Range(143, 148), new Range(148, 151), new Range(151, 155), new Range(155, 159), new Range(159, 160), 175new Range(160, 162), new Range(162, 167), new Range(167, 169), new Range(169, 173), new Range(173, 176), new Range(176, 181), 176new Range(181, 184), new Range(184, 188), new Range(188, 194), new Range(194, 195), new Range(195, 196), new Range(196, 202), 177new Range(202, 205), new Range(205, 210), new Range(210, 213), new Range(213, 217), new Range(217, 222), new Range(222, 223), 178new Range(223, 226), new Range(226, 230), new Range(230, 232), new Range(232, 235), new Range(235, 243), new Range(243, 244), 179new Range(244, 247), new Range(247, 251), new Range(251, 260), new Range(260, 261), new Range(261, 265), new Range(265, 269), 180new Range(269, 272), new Range(272, 276), new Range(276, 279), new Range(279, 281), new Range(281, 284), new Range(284, 286), 181new Range(286, 293), new Range(293, 299), new Range(299, 301), new Range(301, 304), new Range(304, 306), new Range(306, 311), 182new Range(311, 312), new Range(312, 316), new Range(316, 319), new Range(319, 323), new Range(323, 326), new Range(326, 330), 183new Range(330, 335), new Range(335, 339), new Range(339, 343), new Range(343, 350), new Range(350, 355), new Range(355, 356), 184new Range(356, 359), new Range(359, 363), new Range(363, 365), new Range(365, 372), new Range(372, 374), new Range(374, 379), 185new Range(379, 382), new Range(382, 387), new Range(387, 388), new Range(388, 394), new Range(394, 401), new Range(401, 405), 186new Range(405, 409), new Range(409, 413), new Range(413, 414), new Range(414, 420), new Range(420, 421), new Range(421, 426), 187new Range(426, 436), new Range(436, 442), new Range(442, 444), new Range(444, 445) 198new Range[] { new Range(0, 5), new Range(5, 8), new Range(8, 9), new Range(9, 11), new Range(11, 12), new Range(12, 15), new Range(15, 17), new Range(17, 21), new Range(21, 22) } 208new Range[] { new Range(0, 1), new Range(1, 3), new Range(3, 4), new Range(4, 5), new Range(5, 7), new Range(7, 10), new Range(10, 11) } 218new Range[] { new Range(0, 2), new Range(2, 4), new Range(4, 6) } 228new Range[] { new Range(0, 1), new Range(1, 3) } 238new Range[] { new Range(0, 2), new Range(2, 5), new Range(5, 9), new Range(9, 12) } 248new Range[] { new Range(0, 6), new Range(6, 12), new Range(12, 13) } 258new Range[] { new Range(0, 2), new Range(2, 4), new Range(4, 5), new Range(5, 6), new Range(6, 7) } 268new Range[] { new Range(0, 9) } 278new Range[] { new Range(0, 1), new Range(1, 3) } 294Assert.True(writableTokens[0].Offset.Equals(new Range(0, 0))); 305Assert.True(writableTokens[writableTokens.Count - 1].Offset.Equals(new Range(normalized.Length, normalized.Length)));
WordPieceTests.cs (17)
78new EncodedToken(7, "un", new Range(0, 2)), 79new EncodedToken(4, "##want", new Range(2, 6)), 80new EncodedToken(5, "##ed", new Range(6, 8)), 81new EncodedToken(8, "runn", new Range(9, 13)), 82new EncodedToken(9, "##ing", new Range(13, 16)) 159new EncodedToken(0, "[UNK]", new Range(0, 9)), 160new EncodedToken(8, "runn", new Range(10, 14)), 161new EncodedToken(9, "##ing", new Range(14, 17)) 197new EncodedToken(0, "[UNK]", new Range(0, 5)), 198new EncodedToken(7, "un", new Range(6, 8)), 199new EncodedToken(4, "##want", new Range(8, 12)), 200new EncodedToken(5, "##ed", new Range(12, 14)), 201new EncodedToken(2, "[SEP]", new Range(15, 20)), 202new EncodedToken(1, "[CLS]", new Range(20, 25)), 203new EncodedToken(8, "runn", new Range(26, 30)), 204new EncodedToken(9, "##ing", new Range(30, 33)), 205new EncodedToken(1, "[CLS]", new Range(34, 39)),
System.Numerics.Tensors (4)
System\Buffers\NRange.cs (4)
133public static explicit operator Range(NRange value) => new Range((Index)value.Start, (Index)value.End); 139public static explicit operator checked Range(NRange value) => new Range(checked((Index)value.Start), checked((Index)value.End)); 145public Range ToRange() => new Range(checked((Index)Start), checked((Index)End)); 151public Range ToRangeUnchecked() => new Range((Index)Start, (Index)End);
System.Private.CoreLib (4)
src\libraries\System.Private.CoreLib\src\System\MemoryExtensions.cs (1)
5908public Range Current => new Range(_startCurrent, _endCurrent);
src\libraries\System.Private.CoreLib\src\System\Range.cs (3)
100public static Range StartAt(Index start) => new Range(start, Index.End); 103public static Range EndAt(Index end) => new Range(Index.Start, end); 106public static Range All => new Range(Index.Start, Index.End);