1505 references to AsSpan
Aspire.Dashboard (5)
Aspire.Hosting (2)
Aspire.Hosting.Azure (1)
Aspire.Hosting.Azure.AppContainers (1)
Aspire.Hosting.Azure.AppService (1)
Aspire.Hosting.Docker.Tests (1)
http2cat (2)
IIS.FunctionalTests (2)
IIS.LongTests (2)
IIS.NewHandler.FunctionalTests (2)
IIS.NewShim.FunctionalTests (2)
IIS.ShadowCopy.Tests (2)
IISExpress.FunctionalTests (2)
illink (2)
ILLink.RoslynAnalyzer (2)
InMemory.FunctionalTests (3)
Metrics (3)
Metrics.Legacy (3)
Microsoft.AspNetCore.Components (8)
Microsoft.AspNetCore.Components.Endpoints (5)
Microsoft.AspNetCore.Components.Server (2)
Microsoft.AspNetCore.Components.Web (2)
Microsoft.AspNetCore.CookiePolicy.Test (7)
Microsoft.AspNetCore.Http.Abstractions (5)
Microsoft.AspNetCore.Http.Connections.Common (1)
Microsoft.AspNetCore.Http.Extensions (13)
UriHelper.cs (7)
232var pathBaseSpan = uriParts.pathBase.AsSpan();
242index = CopyTextToBuffer(buffer, index, uriParts.scheme.AsSpan());
243index = CopyTextToBuffer(buffer, index, Uri.SchemeDelimiter.AsSpan());
244index = CopyTextToBuffer(buffer, index, uriParts.host.AsSpan());
246index = CopyTextToBuffer(buffer, index, uriParts.path.AsSpan());
247index = CopyTextToBuffer(buffer, index, uriParts.query.AsSpan());
248_ = CopyTextToBuffer(buffer, index, uriParts.fragment.AsSpan());
Microsoft.AspNetCore.HttpLogging (2)
Microsoft.AspNetCore.HttpOverrides (2)
Microsoft.AspNetCore.Identity.FunctionalTests (1)
Microsoft.AspNetCore.Localization (1)
Microsoft.AspNetCore.Mvc.Core (1)
Microsoft.AspNetCore.Mvc.Razor (5)
Microsoft.AspNetCore.Mvc.Razor.RuntimeCompilation (2)
Microsoft.AspNetCore.Mvc.RazorPages (2)
Microsoft.AspNetCore.OpenApi (1)
Microsoft.AspNetCore.OutputCaching (1)
Microsoft.AspNetCore.Rewrite (1)
Microsoft.AspNetCore.Routing (13)
Microsoft.AspNetCore.Server.HttpSys (2)
Microsoft.AspNetCore.Server.HttpSys.FunctionalTests (1)
Microsoft.AspNetCore.Server.IIS (3)
Microsoft.AspNetCore.Server.Kestrel.Core (6)
Microsoft.AspNetCore.Server.Kestrel.Core.Tests (2)
Microsoft.AspNetCore.Server.Kestrel.Transport.Quic (1)
Microsoft.AspNetCore.Shared.Tests (7)
Microsoft.AspNetCore.SignalR.Common (1)
Microsoft.AspNetCore.SignalR.Core (1)
Microsoft.AspNetCore.SignalR.Protocols.Json (1)
Microsoft.AspNetCore.SignalR.Protocols.NewtonsoftJson (1)
Microsoft.AspNetCore.WebSockets (5)
Microsoft.AspNetCore.WebSockets.Tests (3)
Microsoft.AspNetCore.WebUtilities (2)
Microsoft.Build (43)
Microsoft.Build.Engine.OM.UnitTests (2)
Microsoft.Build.Engine.UnitTests (1)
Microsoft.Build.Framework (1)
Microsoft.Build.Tasks.CodeAnalysis (5)
Microsoft.Build.Tasks.Core (9)
Microsoft.Build.Utilities.Core (3)
Microsoft.CodeAnalysis (14)
Microsoft.CodeAnalysis.Analyzers (3)
Microsoft.CodeAnalysis.AnalyzerUtilities (3)
Microsoft.CodeAnalysis.BannedApiAnalyzers (3)
Microsoft.CodeAnalysis.CodeStyle (3)
Microsoft.CodeAnalysis.CSharp (5)
Microsoft.CodeAnalysis.CSharp.Emit.UnitTests (2)
Microsoft.CodeAnalysis.CSharp.Emit3.UnitTests (3)
Microsoft.CodeAnalysis.ExternalAccess.Razor.Features (1)
Microsoft.CodeAnalysis.ExternalAccess.Razor.UnitTests (1)
Microsoft.CodeAnalysis.Features (4)
Microsoft.CodeAnalysis.Features.Test.Utilities (1)
Microsoft.CodeAnalysis.InteractiveHost (1)
Microsoft.CodeAnalysis.ResxSourceGenerator (3)
Microsoft.CodeAnalysis.Test.Utilities (2)
Microsoft.CodeAnalysis.UnitTests (5)
Microsoft.CodeAnalysis.Workspaces (11)
Microsoft.CodeAnalysis.Workspaces.MSBuild.BuildHost (3)
Microsoft.Data.Analysis (1)
Microsoft.DotNet.SwaggerGenerator.CodeGenerator (17)
Microsoft.DotNet.XUnitAssert.Tests (187)
SpanAssertsTests.cs (83)
18 Assert.Contains("wor".AsSpan(), "Hello, world!".AsSpan());
30 var ex = Record.Exception(() => Assert.Contains("WORLD".AsSpan(), "Hello, world!".AsSpan()));
58 Assert.Contains("WORLD".AsSpan(), "Hello, world!".AsSpan(), StringComparison.OrdinalIgnoreCase);
70 var ex = Record.Exception(() => Assert.Contains("foo".AsSpan(), default(string).AsSpan()));
199 Assert.DoesNotContain("hey".AsSpan(), "Hello, world!".AsSpan());
211 Assert.DoesNotContain("WORLD".AsSpan(), "Hello, world!".AsSpan());
223 var ex = Record.Exception(() => Assert.DoesNotContain("WORLD".AsSpan(), "Hello, world!".AsSpan(), StringComparison.OrdinalIgnoreCase));
253 var ex = Record.Exception(() => Assert.DoesNotContain("world".AsSpan(), "Hello, world!".AsSpan()));
283 Assert.DoesNotContain("foo".AsSpan(), default(string).AsSpan());
289 Assert.DoesNotContain("foo".Spanify(), default(string).AsSpan());
295 var ex = Record.Exception(() => Assert.DoesNotContain("world".AsSpan(), "Hello, world from a very long string that will end up being truncated".AsSpan()));
310 var ex = Record.Exception(() => Assert.DoesNotContain("world".AsSpan(), "This is a relatively long string that has 'Hello, world' placed in the middle so that we can dual trunaction".AsSpan()));
325 var ex = Record.Exception(() => Assert.DoesNotContain("world".AsSpan(), "This is a relatively long string that will from the front truncated, just to say 'Hello, world'".AsSpan()));
424 Assert.EndsWith("world!".AsSpan(), "Hello, world!".AsSpan());
425 Assert.EndsWith("world!".AsSpan(), "Hello, world!".Spanify());
426 Assert.EndsWith("world!".Spanify(), "Hello, world!".AsSpan());
446 assertFailure(() => Assert.EndsWith("hey".AsSpan(), "Hello, world!".AsSpan()));
447 assertFailure(() => Assert.EndsWith("hey".AsSpan(), "Hello, world!".Spanify()));
448 assertFailure(() => Assert.EndsWith("hey".Spanify(), "Hello, world!".AsSpan()));
468 assertFailure(() => Assert.EndsWith("WORLD!".AsSpan(), "world!".AsSpan()));
469 assertFailure(() => Assert.EndsWith("WORLD!".AsSpan(), "world!".Spanify()));
470 assertFailure(() => Assert.EndsWith("WORLD!".Spanify(), "world!".AsSpan()));
477 Assert.EndsWith("WORLD!".AsSpan(), "Hello, world!".AsSpan(), StringComparison.OrdinalIgnoreCase);
478 Assert.EndsWith("WORLD!".AsSpan(), "Hello, world!".Spanify(), StringComparison.OrdinalIgnoreCase);
479 Assert.EndsWith("WORLD!".Spanify(), "Hello, world!".AsSpan(), StringComparison.OrdinalIgnoreCase);
499 assertFailure(() => Assert.EndsWith("foo".AsSpan(), null));
522 assertFailure(() => Assert.EndsWith(expected.AsSpan(), actual.AsSpan()));
523 assertFailure(() => Assert.EndsWith(expected.AsSpan(), actual.Spanify()));
524 assertFailure(() => Assert.EndsWith(expected.Spanify(), actual.AsSpan()));
584 Assert.Equal(value1.AsSpan(), value2.AsSpan(), ignoreCase, ignoreLineEndingDifferences, ignoreWhiteSpaceDifferences, ignoreAllWhiteSpace);
585 Assert.Equal(value2.AsSpan(), value1.AsSpan(), ignoreCase, ignoreLineEndingDifferences, ignoreWhiteSpaceDifferences, ignoreAllWhiteSpace);
588 Assert.Equal(value1.AsSpan(), value2.Spanify(), ignoreCase, ignoreLineEndingDifferences, ignoreWhiteSpaceDifferences, ignoreAllWhiteSpace);
589 Assert.Equal(value2.AsSpan(), value1.Spanify(), ignoreCase, ignoreLineEndingDifferences, ignoreWhiteSpaceDifferences, ignoreAllWhiteSpace);
592 Assert.Equal(value1.Spanify(), value2.AsSpan(), ignoreCase, ignoreLineEndingDifferences, ignoreWhiteSpaceDifferences, ignoreAllWhiteSpace);
593 Assert.Equal(value2.Spanify(), value1.AsSpan(), ignoreCase, ignoreLineEndingDifferences, ignoreWhiteSpaceDifferences, ignoreAllWhiteSpace);
648 assertFailure(() => Assert.Equal(expected.AsSpan(), actual.AsSpan(), ignoreCase, ignoreLineEndingDifferences, ignoreWhiteSpaceDifferences, ignoreAllWhiteSpace));
649 assertFailure(() => Assert.Equal(expected.Spanify(), actual.AsSpan(), ignoreCase, ignoreLineEndingDifferences, ignoreWhiteSpaceDifferences, ignoreAllWhiteSpace));
650 assertFailure(() => Assert.Equal(expected.AsSpan(), actual.Spanify(), ignoreCase, ignoreLineEndingDifferences, ignoreWhiteSpaceDifferences, ignoreAllWhiteSpace));
674 "Why hello there world, you're a long string with some truncation!".AsSpan(),
675 "Why hello there world! You're a long string!".AsSpan()
680 "Why hello there world, you're a long string with some truncation!".AsSpan(),
687 "Why hello there world! You're a long string!".AsSpan()
841 Assert.StartsWith("Hello".AsSpan(), "Hello, world!".AsSpan());
842 Assert.StartsWith("Hello".AsSpan(), "Hello, world!".Spanify());
843 Assert.StartsWith("Hello".Spanify(), "Hello, world!".AsSpan());
863 assertFailure(() => Assert.StartsWith("hey".AsSpan(), "Hello, world!".AsSpan()));
864 assertFailure(() => Assert.StartsWith("hey".AsSpan(), "Hello, world!".Spanify()));
865 assertFailure(() => Assert.StartsWith("hey".Spanify(), "Hello, world!".AsSpan()));
885 assertFailure(() => Assert.StartsWith("WORLD!".AsSpan(), "world!".AsSpan()));
886 assertFailure(() => Assert.StartsWith("WORLD!".AsSpan(), "world!".Spanify()));
887 assertFailure(() => Assert.StartsWith("WORLD!".Spanify(), "world!".AsSpan()));
894 Assert.StartsWith("HELLO".AsSpan(), "Hello, world!".AsSpan(), StringComparison.OrdinalIgnoreCase);
895 Assert.StartsWith("HELLO".AsSpan(), "Hello, world!".Spanify(), StringComparison.OrdinalIgnoreCase);
896 Assert.StartsWith("HELLO".Spanify(), "Hello, world!".AsSpan(), StringComparison.OrdinalIgnoreCase);
916 assertFailure(() => Assert.StartsWith("foo".AsSpan(), null));
939 assertFailure(() => Assert.StartsWith(expected.AsSpan(), actual.AsSpan()));
940 assertFailure(() => Assert.StartsWith(expected.AsSpan(), actual.Spanify()));
941 assertFailure(() => Assert.StartsWith(expected.Spanify(), actual.AsSpan()));
StringAssertsTests.cs (104)
20 Assert.Contains("wor".AsSpan(), "Hello, world!".Spanify());
21 Assert.Contains("wor".Spanify(), "Hello, world!".AsSpan());
22 Assert.Contains("wor".AsSpan(), "Hello, world!".AsSpan());
49 verify(() => Assert.Contains("WORLD".AsSpan(), "Hello, world!".Spanify()));
50 verify(() => Assert.Contains("WORLD".Spanify(), "Hello, world!".AsSpan()));
51 verify(() => Assert.Contains("WORLD".AsSpan(), "Hello, world!".AsSpan()));
78 verify(() => Assert.Contains("hey".AsSpan(), "Hello, world!".Spanify()));
79 verify(() => Assert.Contains("hey".Spanify(), "Hello, world!".AsSpan()));
80 verify(() => Assert.Contains("hey".AsSpan(), "Hello, world!".AsSpan()));
124 verify(() => Assert.Contains(expected.AsSpan(), actual.Spanify()));
125 verify(() => Assert.Contains(expected.Spanify(), actual.AsSpan()));
126 verify(() => Assert.Contains(expected.AsSpan(), actual.AsSpan()));
140 Assert.Contains("WORLD".AsSpan(), "Hello, world!".Spanify(), StringComparison.OrdinalIgnoreCase);
141 Assert.Contains("WORLD".Spanify(), "Hello, world!".AsSpan(), StringComparison.OrdinalIgnoreCase);
142 Assert.Contains("WORLD".AsSpan(), "Hello, world!".AsSpan(), StringComparison.OrdinalIgnoreCase);
159 Assert.DoesNotContain("hey".AsSpan(), "Hello, world!".Spanify());
160 Assert.DoesNotContain("hey".Spanify(), "Hello, world!".AsSpan());
161 Assert.DoesNotContain("hey".AsSpan(), "Hello, world!".AsSpan());
175 Assert.DoesNotContain("WORLD".AsSpan(), "Hello, world!".Spanify());
176 Assert.DoesNotContain("WORLD".Spanify(), "Hello, world!".AsSpan());
177 Assert.DoesNotContain("WORLD".AsSpan(), "Hello, world!".AsSpan());
205 verify(() => Assert.DoesNotContain("world".AsSpan(), "Hello, world!".Spanify()));
206 verify(() => Assert.DoesNotContain("world".Spanify(), "Hello, world!".AsSpan()));
207 verify(() => Assert.DoesNotContain("world".AsSpan(), "Hello, world!".AsSpan()));
241 verify(() => Assert.DoesNotContain("world".AsSpan(), "Hello, world from a very long string that will end up being truncated".Spanify()));
242 verify(() => Assert.DoesNotContain("world".Spanify(), "Hello, world from a very long string that will end up being truncated".AsSpan()));
243 verify(() => Assert.DoesNotContain("world".AsSpan(), "Hello, world from a very long string that will end up being truncated".AsSpan()));
271 verify(() => Assert.DoesNotContain("world".AsSpan(), "This is a relatively long string that has 'Hello, world' placed in the middle so that we can dual trunaction".Spanify()));
272 verify(() => Assert.DoesNotContain("world".Spanify(), "This is a relatively long string that has 'Hello, world' placed in the middle so that we can dual trunaction".AsSpan()));
273 verify(() => Assert.DoesNotContain("world".AsSpan(), "This is a relatively long string that has 'Hello, world' placed in the middle so that we can dual trunaction".AsSpan()));
301 verify(() => Assert.DoesNotContain("world".AsSpan(), "This is a relatively long string that will from the front truncated, just to say 'Hello, world'".Spanify()));
302 verify(() => Assert.DoesNotContain("world".Spanify(), "This is a relatively long string that will from the front truncated, just to say 'Hello, world'".AsSpan()));
303 verify(() => Assert.DoesNotContain("world".AsSpan(), "This is a relatively long string that will from the front truncated, just to say 'Hello, world'".AsSpan()));
331 verify(() => Assert.DoesNotContain("WORLD".AsSpan(), "Hello, world!".Spanify(), StringComparison.OrdinalIgnoreCase));
332 verify(() => Assert.DoesNotContain("WORLD".Spanify(), "Hello, world!".AsSpan(), StringComparison.OrdinalIgnoreCase));
333 verify(() => Assert.DoesNotContain("WORLD".AsSpan(), "Hello, world!".AsSpan(), StringComparison.OrdinalIgnoreCase));
438 Assert.EndsWith("world!".AsSpan(), "Hello, world!".Spanify());
439 Assert.EndsWith("world!".Spanify(), "Hello, world!".AsSpan());
440 Assert.EndsWith("world!".AsSpan(), "Hello, world!".AsSpan());
467 verify(() => Assert.EndsWith("hey".AsSpan(), "Hello, world!".Spanify()));
468 verify(() => Assert.EndsWith("hey".Spanify(), "Hello, world!".AsSpan()));
469 verify(() => Assert.EndsWith("hey".AsSpan(), "Hello, world!".AsSpan()));
496 verify(() => Assert.EndsWith("WORLD!".AsSpan(), "world!".Spanify()));
497 verify(() => Assert.EndsWith("WORLD!".Spanify(), "world!".AsSpan()));
498 verify(() => Assert.EndsWith("WORLD!".AsSpan(), "world!".AsSpan()));
512 Assert.EndsWith("WORLD!".AsSpan(), "Hello, world!".Spanify(), StringComparison.OrdinalIgnoreCase);
513 Assert.EndsWith("WORLD!".Spanify(), "Hello, world!".AsSpan(), StringComparison.OrdinalIgnoreCase);
514 Assert.EndsWith("WORLD!".AsSpan(), "Hello, world!".AsSpan(), StringComparison.OrdinalIgnoreCase);
558 verify(() => Assert.EndsWith(expected.AsSpan(), actual.Spanify()));
559 verify(() => Assert.EndsWith(expected.Spanify(), actual.AsSpan()));
560 verify(() => Assert.EndsWith(expected.AsSpan(), actual.AsSpan()));
628 Assert.Equal(value1.AsSpan(), value2.Spanify(), ignoreCase, ignoreLineEndingDifferences, ignoreWhiteSpaceDifferences, ignoreAllWhiteSpace);
629 Assert.Equal(value2.AsSpan(), value1.Spanify(), ignoreCase, ignoreLineEndingDifferences, ignoreWhiteSpaceDifferences, ignoreAllWhiteSpace);
630 Assert.Equal(value1.Spanify(), value2.AsSpan(), ignoreCase, ignoreLineEndingDifferences, ignoreWhiteSpaceDifferences, ignoreAllWhiteSpace);
631 Assert.Equal(value2.Spanify(), value1.AsSpan(), ignoreCase, ignoreLineEndingDifferences, ignoreWhiteSpaceDifferences, ignoreAllWhiteSpace);
632 Assert.Equal(value1.AsSpan(), value2.AsSpan(), ignoreCase, ignoreLineEndingDifferences, ignoreWhiteSpaceDifferences, ignoreAllWhiteSpace);
633 Assert.Equal(value2.AsSpan(), value1.AsSpan(), ignoreCase, ignoreLineEndingDifferences, ignoreWhiteSpaceDifferences, ignoreAllWhiteSpace);
694 verify(() => Assert.Equal(expected.AsSpan(), actual.Spanify(), ignoreCase, ignoreLineEndingDifferences, ignoreWhiteSpaceDifferences, ignoreAllWhiteSpace));
695 verify(() => Assert.Equal(expected.Spanify(), actual.AsSpan(), ignoreCase, ignoreLineEndingDifferences, ignoreWhiteSpaceDifferences, ignoreAllWhiteSpace));
696 verify(() => Assert.Equal(expected.AsSpan(), actual.AsSpan(), ignoreCase, ignoreLineEndingDifferences, ignoreWhiteSpaceDifferences, ignoreAllWhiteSpace));
729 verify(() => Assert.Equal(expected.AsSpan(), actual.Spanify()));
730 verify(() => Assert.Equal(expected.Spanify(), actual.AsSpan()));
731 verify(() => Assert.Equal(expected.AsSpan(), actual.AsSpan()));
840 Assert.StartsWith("Hello".AsSpan(), "Hello, world!".Spanify());
841 Assert.StartsWith("Hello".Spanify(), "Hello, world!".AsSpan());
842 Assert.StartsWith("Hello".AsSpan(), "Hello, world!".AsSpan());
869 verify(() => Assert.StartsWith("hey".AsSpan(), "Hello, world!".Spanify()));
870 verify(() => Assert.StartsWith("hey".Spanify(), "Hello, world!".AsSpan()));
871 verify(() => Assert.StartsWith("hey".AsSpan(), "Hello, world!".AsSpan()));
898 verify(() => Assert.StartsWith("WORLD!".AsSpan(), "world!".Spanify()));
899 verify(() => Assert.StartsWith("WORLD!".Spanify(), "world!".AsSpan()));
900 verify(() => Assert.StartsWith("WORLD!".AsSpan(), "world!".AsSpan()));
914 Assert.StartsWith("HELLO".AsSpan(), "Hello, world!".Spanify(), StringComparison.OrdinalIgnoreCase);
915 Assert.StartsWith("HELLO".Spanify(), "Hello, world!".AsSpan(), StringComparison.OrdinalIgnoreCase);
916 Assert.StartsWith("HELLO".AsSpan(), "Hello, world!".AsSpan(), StringComparison.OrdinalIgnoreCase);
960 verify(() => Assert.StartsWith(expected.AsSpan(), actual.Spanify()));
961 verify(() => Assert.StartsWith(expected.Spanify(), actual.AsSpan()));
962 verify(() => Assert.StartsWith(expected.AsSpan(), actual.AsSpan()));
Microsoft.Extensions.AI.Abstractions (6)
Microsoft.Extensions.AI.Evaluation.Quality (1)
Microsoft.Extensions.AI.OpenAI (8)
Microsoft.Extensions.Caching.Hybrid (5)
Microsoft.Extensions.Compliance.Abstractions (7)
Microsoft.Extensions.Compliance.Abstractions.Tests (8)
Redaction\RedactorTest.cs (7)
20Assert.Equal(string.Empty, r.Redact(string.Empty.AsSpan()));
23Assert.True(r.TryRedact<object?>(null, new char[0], out _, string.Empty.AsSpan()));
248Assert.Equal(success, r.TryRedact(data, buffer, out int charsWritten, string.Empty.AsSpan(), null));
266Assert.Equal(success, r.TryRedact(data, buffer, out int charsWritten, string.Empty.AsSpan(), null));
284Assert.Equal(success, r.TryRedact(data, buffer, out int charsWritten, "R".AsSpan(), null));
298Assert.True(r.TryRedact(Array.Empty<char>(), buffer, out int charsWritten, string.Empty.AsSpan(), null));
314Assert.True(r.TryRedact(new char[0], buffer, out int charsWritten, string.Empty.AsSpan(), null));
Microsoft.Extensions.Compliance.Redaction (1)
Microsoft.Extensions.Compliance.Redaction.Tests (1)
Microsoft.Extensions.Compliance.Testing (1)
Microsoft.Extensions.Configuration (2)
Microsoft.Extensions.Diagnostics (3)
Microsoft.Extensions.Diagnostics.ResourceMonitoring (4)
Microsoft.Extensions.Diagnostics.ResourceMonitoring.Tests (1)
Microsoft.Extensions.FileProviders.Physical (2)
Microsoft.Extensions.Http.Diagnostics (13)
Microsoft.Extensions.Identity.Core (1)
Microsoft.Extensions.Identity.Stores (1)
Microsoft.Extensions.Logging (3)
Microsoft.Extensions.Logging.Console (1)
Microsoft.Extensions.Logging.Generators (5)
Microsoft.Extensions.Primitives (4)
Microsoft.Extensions.Telemetry (17)
Microsoft.Extensions.Telemetry.Tests (3)
Microsoft.Gen.Logging.Generated.Tests (1)
Microsoft.Maui.Controls.Xaml (2)
Microsoft.Maui.Graphics (9)
Microsoft.ML.Core (4)
Microsoft.ML.IntegrationTests (1)
Microsoft.ML.StandardTrainers (1)
Microsoft.ML.Tokenizers (72)
Tokenizer.cs (9)
68=> EncodeToIds(text, text.AsSpan(), new EncodeSettings { ConsiderPreTokenization = considerPreTokenization, ConsiderNormalization = considerNormalization }).Tokens;
92EncodeResults<int> result = EncodeToIds(text, text.AsSpan(),
150EncodeResults<EncodedToken> result = EncodeToTokens(text, text.AsSpan(), new EncodeSettings { ConsiderPreTokenization = considerPreTokenization, ConsiderNormalization = considerNormalization });
194=> CountTokens(text, text.AsSpan(), new EncodeSettings { ConsiderPreTokenization = considerPreTokenization, ConsiderNormalization = considerNormalization });
278text.AsSpan(),
324text.AsSpan(),
446textSpanToEncode = normalizedText.AsSpan();
468textSpanToEncode = normalizedText.AsSpan();
477textSpanToEncode = text.AsSpan();
Microsoft.ML.Tokenizers.Tests (160)
BpeTests.cs (7)
463IReadOnlyList<EncodedToken> encoding1 = tokenizer.EncodeToTokens(text.AsSpan(), out _);
474Assert.Equal(expectedIds, tokenizer.EncodeToIds(text.AsSpan()));
478Assert.Equal(expectedIds, tokenizer.EncodeToIds(text.AsSpan(), expectedIds.Length, out normalizedText, out length));
486Assert.Equal(expectedIds.Take(expectedIds.Length - 2), tokenizer.EncodeToIds(text.AsSpan(), expectedIds.Length - 2, out normalizedText, out length));
491Assert.Equal(expectedIds.Length, tokenizer.CountTokens(text.AsSpan()));
496Assert.Equal(expectedOffsets[expectedOffsets.Length - 4].Index + expectedOffsets[expectedOffsets.Length - 4].Length, tokenizer.GetIndexByTokenCount(text.AsSpan(), expectedIds.Length - 3, out normalizedText, out tokenCount));
503Assert.Equal(expectedOffsets[expectedOffsets.Length - 3].Index, tokenizer.GetIndexByTokenCountFromEnd(text.AsSpan(), 3, out normalizedText, out tokenCount));
CodeGenTests.cs (84)
255encoding = tokenizer.EncodeToTokens(text.AsSpan(), out _);
263encoding = codeGenTokenizer.EncodeToTokens(text.AsSpan(), addPrefixSpace: codeGenTokenizer.AddPrefixSpace, addBeginningOfSentence: true, addEndOfSentence: false, out _);
271encoding = codeGenTokenizer.EncodeToTokens(text.AsSpan(), addPrefixSpace: codeGenTokenizer.AddPrefixSpace, addBeginningOfSentence: false, addEndOfSentence: true, out _);
279encoding = codeGenTokenizer.EncodeToTokens(text.AsSpan(), addPrefixSpace: codeGenTokenizer.AddPrefixSpace, addBeginningOfSentence: true, addEndOfSentence: true, out _);
287encoding = codeGenTokenizer.EncodeToTokens(text.AsSpan(), addPrefixSpace: true, addBeginningOfSentence: true, addEndOfSentence: true, out _);
296encoding = codeGenTokenizer.EncodeToTokens(text.AsSpan(), addPrefixSpace: true, addBeginningOfSentence: true, addEndOfSentence: true, out _);
350encoding = tokenizer.EncodeToTokens(text.AsSpan(), out _);
356encoding = codeGenTokenizer.EncodeToTokens(text.AsSpan(), addPrefixSpace: false, addBeginningOfSentence: false, addEndOfSentence: false, out _);
362encoding = codeGenTokenizer.EncodeToTokens(text.AsSpan(), addPrefixSpace: true, addBeginningOfSentence: false, addEndOfSentence: false, out _);
372Assert.Equal(ids, tokenizer.EncodeToIds(text.AsSpan()));
375Assert.Equal(expectedIdsWithSpace, codeGenTokenizer.EncodeToIds(text.AsSpan(), addPrefixSpace: true, addBeginningOfSentence: false, addEndOfSentence: false));
377Assert.Equal(expectedIds, codeGenTokenizer.EncodeToIds(text.AsSpan(), addPrefixSpace: false, addBeginningOfSentence: false, addEndOfSentence: false));
382Assert.Equal(ids, codeGenTokenizer.EncodeToIds(text.AsSpan(), ids.Length, out normalizedText, out length));
389Assert.Equal(expectedIds, codeGenTokenizer.EncodeToIds(text.AsSpan(), expectedIds.Length, addPrefixSpace: false, addBeginningOfSentence: false, addEndOfSentence: false, out normalizedText, out length));
396Assert.Equal(expectedIdsWithSpace, codeGenTokenizer.EncodeToIds(text.AsSpan(), expectedIdsWithSpace.Length, addPrefixSpace: true, addBeginningOfSentence: false, addEndOfSentence: false, out normalizedText, out length));
406Assert.Equal(ids.Take(ids.Length - expectedTokensToExclude), codeGenTokenizer.EncodeToIds(text.AsSpan(), ids.Length - 1, out normalizedText, out length));
413Assert.Equal(expectedIds.Take(expectedIds.Length - expectedTokensToExclude), codeGenTokenizer.EncodeToIds(text.AsSpan(), expectedIds.Length - 1, addPrefixSpace: false, addBeginningOfSentence: false, addEndOfSentence: false, out normalizedText, out length));
420Assert.Equal(expectedIdsWithSpace.Take(expectedIdsWithSpace.Length - expectedTokensToExclude), codeGenTokenizer.EncodeToIds(text.AsSpan(), expectedIdsWithSpace.Length - 1, addPrefixSpace: true, addBeginningOfSentence: false, addEndOfSentence: false, out normalizedText, out length));
429Assert.Equal(ids.Length, codeGenTokenizer.CountTokens(text.AsSpan()));
432Assert.Equal(expectedIds.Length, codeGenTokenizer.CountTokens(text.AsSpan(), addPrefixSpace: false, addBeginningOfSentence: false, addEndOfSentence: false));
435Assert.Equal(expectedIdsWithSpace.Length, codeGenTokenizer.CountTokens(text.AsSpan(), addPrefixSpace: true, addBeginningOfSentence: false, addEndOfSentence: false));
446Assert.Equal(offsets[offsets.Length - 1].Index + offsets[offsets.Length - 1].Length, codeGenTokenizer.GetIndexByTokenCount(text.AsSpan(), ids.Length, out normalizedText, out tokenCount));
453Assert.Equal(expectedOffsets[expectedOffsets.Length - 1].Index + expectedOffsets[expectedOffsets.Length - 1].Length, codeGenTokenizer.GetIndexByTokenCount(text.AsSpan(), expectedIds.Length, addPrefixSpace: false, addBeginningOfSentence: false, addEndOfSentence: false, out normalizedText, out tokenCount));
460Assert.Equal(expectedOffsetsWithSpace[expectedOffsetsWithSpace.Length - 1].Index + expectedOffsetsWithSpace[expectedOffsetsWithSpace.Length - 1].Length, codeGenTokenizer.GetIndexByTokenCount(text.AsSpan(), expectedIdsWithSpace.Length, addPrefixSpace: true, addBeginningOfSentence: false, addEndOfSentence: false, out normalizedText, out tokenCount));
473Assert.Equal(expectedIndex, codeGenTokenizer.GetIndexByTokenCountFromEnd(text.AsSpan(), 1, out normalizedText, out tokenCount));
480Assert.Equal(expectedIndex, codeGenTokenizer.GetIndexByTokenCountFromEnd(text.AsSpan(), 1, addPrefixSpace: false, addBeginningOfSentence: false, addEndOfSentence: false, out normalizedText, out tokenCount));
489Assert.Equal(expectedIndex, codeGenTokenizer.GetIndexByTokenCountFromEnd(text.AsSpan(), 1, addPrefixSpace: true, addBeginningOfSentence: false, addEndOfSentence: false, out normalizedText, out tokenCount));
560encoding = codeGenTokenizer.EncodeToTokens(text.AsSpan(), out _);
570encoding = codeGenTokenizer.EncodeToTokens(text.AsSpan(), addPrefixSpace: false, addBeginningOfSentence: true, addEndOfSentence: false, out _);
584encoding = codeGenTokenizer.EncodeToTokens(text.AsSpan(), addPrefixSpace: true, addBeginningOfSentence: true, addEndOfSentence: false, out _);
594encoding = codeGenTokenizer.EncodeToTokens(text.AsSpan(), addPrefixSpace: false, addBeginningOfSentence: false, addEndOfSentence: false, out _);
604encoding = codeGenTokenizer.EncodeToTokens(text.AsSpan(), addPrefixSpace: true, addBeginningOfSentence: false, addEndOfSentence: false, out _);
611ids = codeGenTokenizer.EncodeToIds(text.AsSpan());
615ids = codeGenTokenizer.EncodeToIds(text.AsSpan(), addPrefixSpace: false, addBeginningOfSentence: true, addEndOfSentence: false);
619ids = codeGenTokenizer.EncodeToIds(text.AsSpan(), addPrefixSpace: false, addBeginningOfSentence: false, addEndOfSentence: false);
623ids = codeGenTokenizer.EncodeToIds(text.AsSpan(), maxTokenCount: 5, addPrefixSpace: false, addBeginningOfSentence: true, addEndOfSentence: false, out normalizedText, out charsConsumed);
627int count = codeGenTokenizer.CountTokens(text.AsSpan(), addPrefixSpace: false, addBeginningOfSentence: false, addEndOfSentence: false);
631count = codeGenTokenizer.CountTokens(text.AsSpan());
635count = codeGenTokenizer.CountTokens(text.AsSpan(), addPrefixSpace: false, addBeginningOfSentence: true, addEndOfSentence: false);
641length = codeGenTokenizer.GetIndexByTokenCount(text.AsSpan(), maxTokenCount: 500, out normalizedText, out count);
647length = codeGenTokenizer.GetIndexByTokenCount(text.AsSpan(), maxTokenCount: 500, addPrefixSpace: false, addBeginningOfSentence: true, addEndOfSentence: false, out normalizedText, out count);
653length = codeGenTokenizer.GetIndexByTokenCount(text.AsSpan(), maxTokenCount: 500, addPrefixSpace: false, addBeginningOfSentence: false, addEndOfSentence: false, out normalizedText, out count);
660index = codeGenTokenizer.GetIndexByTokenCountFromEnd(text.AsSpan(), maxTokenCount: 500, out normalizedText, out count);
666index = codeGenTokenizer.GetIndexByTokenCountFromEnd(text.AsSpan(), maxTokenCount: 500, addPrefixSpace: false, addBeginningOfSentence: true, addEndOfSentence: false, out normalizedText, out count);
672index = codeGenTokenizer.GetIndexByTokenCountFromEnd(text.AsSpan(), maxTokenCount: 500, addPrefixSpace: false, addBeginningOfSentence: false, addEndOfSentence: false, out normalizedText, out count);
693encoding = codeGenTokenizer.EncodeToTokens(text.AsSpan(), out _);
703encoding = codeGenTokenizer.EncodeToTokens(text.AsSpan(), addPrefixSpace: false, addBeginningOfSentence: false, addEndOfSentence: true, out _);
717encoding = codeGenTokenizer.EncodeToTokens(text.AsSpan(), addPrefixSpace: true, addBeginningOfSentence: false, addEndOfSentence: true, out _);
727encoding = codeGenTokenizer.EncodeToTokens(text.AsSpan(), addPrefixSpace: false, addBeginningOfSentence: false, addEndOfSentence: false, out _);
737encoding = codeGenTokenizer.EncodeToTokens(text.AsSpan(), addPrefixSpace: true, addBeginningOfSentence: false, addEndOfSentence: false, out _);
744ids = codeGenTokenizer.EncodeToIds(text.AsSpan());
748ids = codeGenTokenizer.EncodeToIds(text.AsSpan(), addPrefixSpace: false, addBeginningOfSentence: false, addEndOfSentence: true);
752ids = codeGenTokenizer.EncodeToIds(text.AsSpan(), addPrefixSpace: false, addBeginningOfSentence: false, addEndOfSentence: false);
756ids = codeGenTokenizer.EncodeToIds(text.AsSpan(), maxTokenCount: 500, addPrefixSpace: false, addBeginningOfSentence: false, addEndOfSentence: true, out normalizedText, out charsConsumed);
760count = codeGenTokenizer.CountTokens(text.AsSpan(), addPrefixSpace: false, addBeginningOfSentence: false, addEndOfSentence: false);
764count = codeGenTokenizer.CountTokens(text.AsSpan());
768count = codeGenTokenizer.CountTokens(text.AsSpan(), addPrefixSpace: false, addBeginningOfSentence: false, addEndOfSentence: true);
774length = codeGenTokenizer.GetIndexByTokenCount(text.AsSpan(), maxTokenCount: 500, out normalizedText, out count);
780length = codeGenTokenizer.GetIndexByTokenCount(text.AsSpan(), maxTokenCount: 500, addPrefixSpace: false, addBeginningOfSentence: false, addEndOfSentence: true, out normalizedText, out count);
786length = codeGenTokenizer.GetIndexByTokenCount(text.AsSpan(), maxTokenCount: 500, addPrefixSpace: false, addBeginningOfSentence: false, addEndOfSentence: false, out normalizedText, out count);
793index = codeGenTokenizer.GetIndexByTokenCountFromEnd(text.AsSpan(), maxTokenCount: 500, out normalizedText, out count);
799index = codeGenTokenizer.GetIndexByTokenCountFromEnd(text.AsSpan(), maxTokenCount: 500, addPrefixSpace: false, addBeginningOfSentence: false, addEndOfSentence: true, out normalizedText, out count);
805index = codeGenTokenizer.GetIndexByTokenCountFromEnd(text.AsSpan(), maxTokenCount: 500, addPrefixSpace: false, addBeginningOfSentence: false, addEndOfSentence: false, out normalizedText, out count);
829encoding = codeGenTokenizer.EncodeToTokens(text.AsSpan(), out _);
841encoding = codeGenTokenizer.EncodeToTokens(text.AsSpan(), addPrefixSpace: false, addBeginningOfSentence: true, addEndOfSentence: true, out _);
859encoding = codeGenTokenizer.EncodeToTokens(text.AsSpan(), addPrefixSpace: true, addBeginningOfSentence: true, addEndOfSentence: true, out _);
871encoding = codeGenTokenizer.EncodeToTokens(text.AsSpan(), addPrefixSpace: false, addBeginningOfSentence: false, addEndOfSentence: false, out _);
883encoding = codeGenTokenizer.EncodeToTokens(text.AsSpan(), addPrefixSpace: true, addBeginningOfSentence: false, addEndOfSentence: false, out _);
892ids = codeGenTokenizer.EncodeToIds(text.AsSpan());
898ids = codeGenTokenizer.EncodeToIds(text.AsSpan(), addPrefixSpace: false, addBeginningOfSentence: true, addEndOfSentence: true);
904ids = codeGenTokenizer.EncodeToIds(text.AsSpan(), addPrefixSpace: false, addBeginningOfSentence: false, addEndOfSentence: false);
910ids = codeGenTokenizer.EncodeToIds(text.AsSpan(), maxTokenCount: 500, addPrefixSpace: false, addBeginningOfSentence: true, addEndOfSentence: true, out normalizedText, out charsConsumed);
915count = codeGenTokenizer.CountTokens(text.AsSpan(), addPrefixSpace: false, addBeginningOfSentence: false, addEndOfSentence: false);
919count = codeGenTokenizer.CountTokens(text.AsSpan());
923count = codeGenTokenizer.CountTokens(text.AsSpan(), addPrefixSpace: false, addBeginningOfSentence: true, addEndOfSentence: true);
928length = codeGenTokenizer.GetIndexByTokenCount(text.AsSpan(), maxTokenCount: 500, out normalizedText, out count);
934length = codeGenTokenizer.GetIndexByTokenCount(text.AsSpan(), maxTokenCount: 500, addPrefixSpace: false, addBeginningOfSentence: true, addEndOfSentence: true, out normalizedText, out count);
940length = codeGenTokenizer.GetIndexByTokenCount(text.AsSpan(), maxTokenCount: 500, addPrefixSpace: false, addBeginningOfSentence: false, addEndOfSentence: false, out normalizedText, out count);
947index = codeGenTokenizer.GetIndexByTokenCountFromEnd(text.AsSpan(), maxTokenCount: 500, out normalizedText, out count);
953index = codeGenTokenizer.GetIndexByTokenCountFromEnd(text.AsSpan(), maxTokenCount: 500, addPrefixSpace: false, addBeginningOfSentence: true, addEndOfSentence: true, out normalizedText, out count);
959index = codeGenTokenizer.GetIndexByTokenCountFromEnd(text.AsSpan(), maxTokenCount: 500, addPrefixSpace: false, addBeginningOfSentence: false, addEndOfSentence: false, out normalizedText, out count);
1002ids = _codegen350MMonoTokenizer.EncodeToIds(input.AsSpan(), maxTokenCount, out _, out charsConsumed);
1012charsConsumed = _codegen350MMonoTokenizer.GetIndexByTokenCount(input.AsSpan(), maxTokenCount, out _, out tokenCount);
1022charsConsumed = _codegen350MMonoTokenizer.GetIndexByTokenCountFromEnd(input.AsSpan(), maxTokenCount, out _, out tokenCount);
EnglishRobertaTests.cs (7)
182IReadOnlyList<EncodedToken> encoding1 = tokenizer.EncodeToTokens(text.AsSpan(), out _);
193Assert.Equal(expectedIds, tokenizer.EncodeToIds(text.AsSpan()));
197Assert.Equal(expectedIds, tokenizer.EncodeToIds(text.AsSpan(), expectedIds.Length, out normalizedText, out length));
205Assert.Equal(expectedIds.Take(expectedIds.Length - 2), tokenizer.EncodeToIds(text.AsSpan(), expectedIds.Length - 2, out normalizedText, out length));
210Assert.Equal(expectedIds.Length, tokenizer.CountTokens(text.AsSpan()));
215Assert.Equal(expectedOffsets[expectedOffsets.Length - 4].Index + expectedOffsets[expectedOffsets.Length - 4].Length, tokenizer.GetIndexByTokenCount(text.AsSpan(), expectedIds.Length - 3, out normalizedText, out tokenCount));
222Assert.Equal(expectedOffsets[expectedOffsets.Length - 3].Index, tokenizer.GetIndexByTokenCountFromEnd(text.AsSpan(), 3, out normalizedText, out tokenCount));
LlamaTests.cs (34)
266IReadOnlyList<EncodedToken> bpeTokens = bpe.EncodeToTokens(normalizedInput.AsSpan(), out _, addBeginningOfSentence: false, addEndOfSentence: false, considerNormalization: false);
272IReadOnlyList<int> encodedIds = bpe.EncodeToIds(normalizedInput.AsSpan(), addBeginningOfSentence: false, addEndOfSentence: false, considerNormalization: false);
274Assert.Equal(isEmptyInput ? 0 : ids.Length - 1, bpe.CountTokens(normalizedInput.AsSpan(), addBeginningOfSentence: false, addEndOfSentence: false, considerNormalization: false));
276bpeTokens = bpe.EncodeToTokens(normalizedInput.AsSpan(), out _, addBeginningOfSentence: false, addEndOfSentence: true, considerNormalization: false);
282encodedIds = bpe.EncodeToIds(normalizedInput.AsSpan(), addBeginningOfSentence: false, addEndOfSentence: true, considerNormalization: false);
284Assert.Equal(isEmptyInput ? 0 : ids.Length, bpe.CountTokens(normalizedInput.AsSpan(), addBeginningOfSentence: false, addEndOfSentence: true, considerNormalization: false));
286bpeTokens = bpe.EncodeToTokens(normalizedInput.AsSpan(), out _, addBeginningOfSentence: true, addEndOfSentence: true, considerNormalization: false);
292encodedIds = bpe.EncodeToIds(normalizedInput.AsSpan(), addBeginningOfSentence: true, addEndOfSentence: true, considerNormalization: false);
294Assert.Equal(isEmptyInput ? 0 : ids.Length + 1, bpe.CountTokens(normalizedInput.AsSpan(), addBeginningOfSentence: true, addEndOfSentence: true, considerNormalization: false));
399Assert.Equal("Hello, World!", normalizer.Normalize("Hello, World!".AsSpan()));
403Assert.Equal("Hello, World!", normalizer.Normalize("Hello, World!".AsSpan()));
407Assert.Equal(" Hello, World!", normalizer.Normalize("Hello, World!".AsSpan()));
411Assert.Equal("▁Hello,▁World!", normalizer.Normalize("Hello, World!".AsSpan()));
415Assert.Equal("▁Hello,▁▁▁▁▁▁World!", normalizer.Normalize("Hello, World!".AsSpan()));
419Assert.Equal("Hello,▁World!▁", normalizer.Normalize("Hello, World!".AsSpan()));
423Assert.Equal("Hello,▁World!", normalizer.Normalize("Hello, World!".AsSpan()));
427Assert.Equal("Hello,▁▁▁▁▁▁World!▁", normalizer.Normalize("Hello, World!".AsSpan()));
431Assert.Equal("Hello, World! ", normalizer.Normalize("Hello, World!".AsSpan()));
501IReadOnlyList<EncodedToken> encoding1 = tokenizer.EncodeToTokens(text.AsSpan(), out _);
525considerNormalization ? text.AsSpan() : normalizedText.AsSpan(),
557Assert.Equal(expectedIds, tokenizer.EncodeToIds(text.AsSpan()));
561Assert.Equal(expectedIds, tokenizer.EncodeToIds(text.AsSpan(), expectedIds.Length, out normalizedString, out length));
585considerNormalization ? text.AsSpan() : normalizedText.AsSpan(),
612considerNormalization ? text.AsSpan() : normalizedText.AsSpan(),
633Assert.Equal(expectedIds.Length, tokenizer.CountTokens(text.AsSpan()));
638Assert.Equal(expectedOffsets[expectedOffsets.Length - 7].Index + expectedOffsets[expectedOffsets.Length - 7].Length, tokenizer.GetIndexByTokenCount(text.AsSpan(), expectedIds.Length - 6, out normalizedString, out tokenCount));
645Assert.Equal(expectedOffsets[expectedOffsets.Length - 7].Index, tokenizer.GetIndexByTokenCountFromEnd(text.AsSpan(), 7, out normalizedString, out tokenCount));
850Assert.True(decodedWithNoSpecialTokens.AsSpan().StartsWith(destinationBuffer.AsSpan().Slice(0, charactersWritten)));
864Assert.True(textWithSpecialTokens.AsSpan().StartsWith(destinationBuffer.AsSpan().Slice(0, charactersWritten)));
889Assert.True(decodedWithNoSpecialTokens.AsSpan().StartsWith(destinationBuffer.AsSpan().Slice(0, charactersWritten)));
908Assert.True(textWithSpecialTokens.AsSpan().StartsWith(sp));
TiktokenTests.cs (7)
588IReadOnlyList<EncodedToken> encoding1 = tokenizer.EncodeToTokens(text.AsSpan(), out _);
599Assert.Equal(expectedIds, tokenizer.EncodeToIds(text.AsSpan()));
603Assert.Equal(expectedIds, tokenizer.EncodeToIds(text.AsSpan(), expectedIds.Length, out normalizedText, out length));
611Assert.Equal(expectedIds.Take(expectedIds.Length - 4), tokenizer.EncodeToIds(text.AsSpan(), expectedIds.Length - 4, out normalizedText, out length));
616Assert.Equal(expectedIds.Length, tokenizer.CountTokens(text.AsSpan()));
621Assert.Equal(expectedOffsets[expectedOffsets.Length - 4].Index + expectedOffsets[expectedOffsets.Length - 4].Length, tokenizer.GetIndexByTokenCount(text.AsSpan(), expectedIds.Length - 3, out normalizedText, out tokenCount));
628Assert.Equal(expectedOffsets[expectedOffsets.Length - 3].Index, tokenizer.GetIndexByTokenCountFromEnd(text.AsSpan(), 3, out normalizedText, out tokenCount));
UnigramTests.cs (16)
333result = _unigramTokenizer.EncodeToTokens(inputText.AsSpan(), out normalized);
341result = _unigramTokenizer.EncodeToTokens(inputText.AsSpan(), out normalized, addBeginningOfSentence: true, addEndOfSentence: false);
349result = _unigramTokenizer.EncodeToTokens(inputText.AsSpan(), out normalized, addBeginningOfSentence: true, addEndOfSentence: true);
384result = _unigramTokenizer.EncodeToIds(inputText.AsSpan(), addBeginningOfSentence: false, addEndOfSentence: false);
395result = _unigramTokenizer.EncodeToIds(inputText.AsSpan(), addBeginningOfSentence: true, addEndOfSentence: false);
412result = _unigramTokenizer.EncodeToIds(inputText.AsSpan(), addBeginningOfSentence: true, addEndOfSentence: true);
427result = _unigramTokenizer.EncodeToIds(inputText.AsSpan(), addBeginningOfSentence: false, addEndOfSentence: false, maxTokenCount: i, out normalized, out charConsumed);
447result = _unigramTokenizer.EncodeToIds(inputText.AsSpan(), addBeginningOfSentence: true, addEndOfSentence: true, maxTokenCount: i, out normalized, out charConsumed);
479result = _unigramTokenizerWithSpecialTokens.EncodeToIds(inputText.AsSpan(), addBeginningOfSentence: false, addEndOfSentence: false, maxTokenCount: i, out normalized, out charConsumed);
503index = _unigramTokenizer.GetIndexByTokenCount(inputText.AsSpan(), addBeginningOfSentence: false, addEndOfSentence: false, maxTokenCount: 1, out normalized, out charConsumed);
505ids1 = _unigramTokenizer.EncodeToIds(normalized!.Substring(0, index).AsSpan(), addBeginningOfSentence: false, addEndOfSentence: false, considerNormalization: false);
506ids2 = index < normalized.Length ? _unigramTokenizer.EncodeToIds(normalized!.Substring(index).AsSpan(), addBeginningOfSentence: false, addEndOfSentence: false, considerNormalization: false) : new List<int>();
515index = _unigramTokenizer.GetIndexByTokenCountFromEnd(inputText.AsSpan(), addBeginningOfSentence: false, addEndOfSentence: false, maxTokenCount: 1, considerNormalization: true, out normalized, out charConsumed);
517ids1 = _unigramTokenizer.EncodeToIds(normalized!.Substring(0, index).AsSpan(), addBeginningOfSentence: false, addEndOfSentence: false, considerNormalization: false);
518ids2 = index < normalized.Length ? _unigramTokenizer.EncodeToIds(normalized!.Substring(index).AsSpan(), addBeginningOfSentence: false, addEndOfSentence: false, considerNormalization: false) : new List<int>();
551Assert.Equal(decodedString.AsSpan().Slice(0, charsWritten).ToString(), buffer.AsSpan().Slice(0, charsWritten).ToString());
Microsoft.ML.Transforms (1)
Microsoft.Net.Http.Headers.Tests (11)
Microsoft.NET.StringTools (3)
Microsoft.NET.StringTools.UnitTests (1)
Microsoft.VisualStudio.LanguageServices (1)
MSBuild (5)
PresentationBuildTasks (6)
PresentationCore (2)
PresentationFramework (17)
ReachFramework (1)
Replay (2)
ResultsOfTGenerator (1)
Roslyn.Diagnostics.Analyzers (3)
SemanticSearch.BuildTask (2)
Shared (8)
System.Collections.Immutable (23)
System.ComponentModel.Annotations (2)
System.ComponentModel.TypeConverter (4)
System.Configuration.ConfigurationManager (1)
System.Data.Common (1)
System.Diagnostics.DiagnosticSource (10)
System.Diagnostics.Process (1)
System.Drawing.Common.Tests (45)
mono\System.Drawing\GraphicsTests.cs (44)
1579g.MeasureString(string.Empty.AsSpan(), _font);
1587g.MeasureString(string.Empty.AsSpan(), null);
1597Assert.Throws<ArgumentNullException>(() => g.MeasureString("a".AsSpan(), null));
1608size = g.MeasureString("a".AsSpan(), _font, SizeF.Empty);
1614size = g.MeasureString(string.Empty.AsSpan(), _font, SizeF.Empty);
1622SizeF size0 = useSpan ? g.MeasureString(s.AsSpan(), _font, 0) : g.MeasureString(s, _font, 0);
1623SizeF sizeN = useSpan ? g.MeasureString(s.AsSpan(), _font, int.MinValue) : g.MeasureString(s, _font, int.MinValue);
1624SizeF sizeP = useSpan ? g.MeasureString(s.AsSpan(), _font, int.MaxValue) : g.MeasureString(s, _font, int.MaxValue);
1657? g.MeasureString(text.AsSpan(), _font, int.MaxValue, string_format)
1662? g.MeasureString(text.AsSpan(), _font, int.MaxValue, string_format)
1667? g.MeasureString(text.AsSpan(), _font, int.MaxValue, string_format)
1690? g.MeasureString(text.AsSpan(), _font, int.MaxValue, string_format)
1695? g.MeasureString(text.AsSpan(), _font, int.MaxValue, string_format)
1700? g.MeasureString(text.AsSpan(), _font, int.MaxValue, string_format)
1721? g.MeasureString(text.AsSpan(), _font, int.MaxValue, string_format)
1726? g.MeasureString(text.AsSpan(), _font, int.MaxValue, string_format)
1731? g.MeasureString(text.AsSpan(), _font, int.MaxValue, string_format)
1754? g.MeasureString(text.AsSpan(), _font, int.MaxValue, string_format)
1759? g.MeasureString(text.AsSpan(), _font, int.MaxValue, string_format)
1764? g.MeasureString(text.AsSpan(), _font, int.MaxValue, string_format)
1782SizeF size = useSpan ? g.MeasureString(s.AsSpan(), _font) : g.MeasureString(s, _font);
1785? g.MeasureString(s.AsSpan(), _font, new SizeF(80, size.Height), null, out int chars, out int lines)
1805SizeF size = useSpan ? g.MeasureString(s.AsSpan(), _font) : g.MeasureString(s, _font);
1810SizeF expected = useSpan ? g.MeasureString(s.AsSpan(), _font) : g.MeasureString(s, _font);
1814size = useSpan ? g.MeasureString(s.AsSpan(), _font) : g.MeasureString(s, _font);
1820expected = useSpan ? g.MeasureString(s.AsSpan(), _font) : g.MeasureString(s, _font);
1822size = useSpan ? g.MeasureString(s.AsSpan(), _font) : g.MeasureString(s, _font);
1826size = useSpan ? g.MeasureString(s.AsSpan(), _font) : g.MeasureString(s, _font);
1833expected = useSpan ? g.MeasureString(s.AsSpan(), _font) : g.MeasureString(s, _font);
1837size = useSpan ? g.MeasureString(s.AsSpan(), _font) : g.MeasureString(s, _font);
1853regions = g.MeasureCharacterRanges(string.Empty.AsSpan(), _font, default, null);
1862regions = g.MeasureCharacterRanges(string.Empty.AsSpan(), null, default, null);
1875g.MeasureCharacterRanges("Mono".AsSpan(), _font, default, new StringFormat());
1885Assert.Throws<ArgumentNullException>(() => g.MeasureCharacterRanges("a".AsSpan(), null, default, null));
1907regions = g.MeasureCharacterRanges(text.AsSpan(), _font, layout_rect, string_format);
1924? g.MeasureString(text.AsSpan(), _font, new Point(0, 0), string_format)
1929g.MeasureCharacterRanges(text.AsSpan(), _font, layout_rect, string_format);
1980regions = g.MeasureCharacterRanges(text.AsSpan(), _font, layout_rect, string_format);
1990g.MeasureCharacterRanges(text.AsSpan(), _font, layout_rect, string_format);
2001Assert.Throws<ArgumentException>(() => g.MeasureCharacterRanges("Mono".AsSpan(), _font, default, null));
2013? gfx.MeasureCharacterRanges("abc".AsSpan(), font, rect, format)
2094g.DrawString("Test String".AsSpan(), _font, Brushes.Black, rect, fmt);
2110g.DrawString("Test String".AsSpan(), _font, Brushes.Black, rect, fmt);
2125sz = g.MeasureString(text.AsSpan(), _font, 80, format);
System.Formats.Asn1 (5)
System.Formats.Cbor (1)
System.Formats.Nrbf (10)
System.Formats.Tar (11)
System.IO.Compression (2)
System.IO.FileSystem.Watcher (8)
System.IO.Packaging (2)
System.IO.Pipes (2)
System.Net.Http (5)
System.Net.HttpListener (3)
System.Net.Mail (5)
System.Net.NetworkInformation (4)
System.Net.Primitives (11)
System.Net.Requests (2)
System.Net.WebHeaderCollection (1)
System.Net.WebSockets (1)
System.Net.WebSockets.Client (2)
System.Private.CoreLib (188)
src\libraries\System.Private.CoreLib\src\System\Convert.cs (11)
2063int r = ParseNumbers.StringToInt(value.AsSpan(), fromBase, ParseNumbers.IsTight | ParseNumbers.TreatAsUnsigned);
2086int r = ParseNumbers.StringToInt(value.AsSpan(), fromBase, ParseNumbers.IsTight | ParseNumbers.TreatAsI1);
2111int r = ParseNumbers.StringToInt(value.AsSpan(), fromBase, ParseNumbers.IsTight | ParseNumbers.TreatAsI2);
2137int r = ParseNumbers.StringToInt(value.AsSpan(), fromBase, ParseNumbers.IsTight | ParseNumbers.TreatAsUnsigned);
2154ParseNumbers.StringToInt(value.AsSpan(), fromBase, ParseNumbers.IsTight) :
2170(uint)ParseNumbers.StringToInt(value.AsSpan(), fromBase, ParseNumbers.TreatAsUnsigned | ParseNumbers.IsTight) :
2185ParseNumbers.StringToLong(value.AsSpan(), fromBase, ParseNumbers.IsTight) :
2201(ulong)ParseNumbers.StringToLong(value.AsSpan(), fromBase, ParseNumbers.TreatAsUnsigned | ParseNumbers.IsTight) :
2666return TryFromBase64Chars(s.AsSpan(), bytes, out bytesWritten);
2933return FromHexString(s.AsSpan());
2975return FromHexString(source.AsSpan(), destination, out charsConsumed, out bytesWritten);
src\libraries\System.Private.CoreLib\src\System\DateOnly.cs (9)
368return Parse(s.AsSpan(), provider, style);
393return ParseExact(s.AsSpan(), format.AsSpan(), provider, style);
417return ParseExact(s.AsSpan(), formats, provider, style);
627return TryParse(s.AsSpan(), provider, style, out result);
658return TryParseExact(s.AsSpan(), format.AsSpan(), provider, style, out result);
687return TryParseExact(s.AsSpan(), formats, provider, style, out result);
772DateTimeFormat.IsValidCustomDateOnlyFormat(format.AsSpan(), throwOnError: true);
src\libraries\System.Private.CoreLib\src\System\Enum.cs (4)
583bool success = TryParse(enumType, value.AsSpan(), ignoreCase, throwOnFailure: true, out object? result);
645bool success = TryParse(value.AsSpan(), ignoreCase, throwOnFailure: true, out TEnum result);
696return TryParse(enumType, value.AsSpan(), ignoreCase, throwOnFailure: false, out result);
859return TryParse(value.AsSpan(), ignoreCase, throwOnFailure: false, out result);
src\libraries\System.Private.CoreLib\src\System\Globalization\CompareInfo.cs (12)
128return IsSortable(text.AsSpan());
278return Compare(string1.AsSpan(), string2.AsSpan(), options);
512return IsPrefix(source.AsSpan(), prefix.AsSpan(), options);
655return IsSuffix(source.AsSpan(), suffix.AsSpan(), options);
822return IndexOf(source.AsSpan(), value.AsSpan(), options);
1155return LastIndexOf(source.AsSpan(), value.AsSpan(), options);
1537return GetHashCode(source.AsSpan(), options);
src\libraries\System.Private.CoreLib\src\System\String.Manipulation.cs (11)
751return JoinCore(separator.AsSpan(), new ReadOnlySpan<string?>(value));
766return JoinCore(separator.AsSpan(), value);
773JoinCore(separator.AsSpan(), value, startIndex, count);
789return JoinCore(separator.AsSpan(), CollectionsMarshal.AsSpan(valuesList));
794return JoinCore(separator.AsSpan(), new ReadOnlySpan<string?>(valuesArray));
863return JoinCore(separator.AsSpan(), (ReadOnlySpan<object?>)values);
877JoinCore(separator.AsSpan(), values);
914JoinCore(separator.AsSpan(), values);
1216return ReplaceCore(this, oldValue.AsSpan(), newValue.AsSpan(), ci ?? CultureInfo.CurrentCulture.CompareInfo, options)
1607int idxOfFirstNewlineChar = this.AsSpan().IndexOfAny(SearchValuesStorage.NewLineCharsExceptLineFeed);
src\libraries\System.Private.CoreLib\src\System\Text\StringBuilder.cs (4)
933public StringBuilder Insert(int index, string? value, int count) => Insert(index, value.AsSpan(), count);
1309public StringBuilder Insert(int index, bool value) => Insert(index, value.ToString().AsSpan(), 1);
2066return Replace(oldValue.AsSpan(), newValue.AsSpan(), startIndex, count);
src\libraries\System.Private.CoreLib\src\System\TimeOnly.cs (9)
536return Parse(s.AsSpan(), provider, style);
561return ParseExact(s.AsSpan(), format.AsSpan(), provider, style);
585return ParseExact(s.AsSpan(), formats, provider, style);
797return TryParse(s.AsSpan(), provider, style, out result);
828return TryParseExact(s.AsSpan(), format.AsSpan(), provider, style, out result);
857return TryParseExact(s.AsSpan(), formats, provider, style, out result);
944DateTimeFormat.IsValidCustomTimeOnlyFormat(format.AsSpan(), throwOnError: true);
System.Private.DataContractSerialization (4)
System.Private.Uri (5)
System.Private.Windows.Core (2)
System.Private.Xml (29)
System.Private.Xml.Linq (2)
System.Reflection.Metadata (8)
System.Reflection.MetadataLoadContext (1)
System.Resources.Extensions (4)
System.Runtime.Numerics (4)
System.Security.Cryptography (39)
System.Security.Cryptography.Pkcs (5)
System.Text.Encodings.Web (7)
System.Text.Json (48)
System.Text.Json.SourceGeneration (2)
System.Text.RegularExpressions (21)
System.Text.RegularExpressions.Generator (21)
System.Web.HttpUtility (5)
System.Windows.Forms (14)
System.Windows.Forms.Design (1)
System.Windows.Forms.Primitives (2)
System.Windows.Forms.Primitives.Tests (2)
System.Windows.Forms.Tests (23)
System.Xaml (1)
Test.Utilities (2)
Text.Analyzers (3)
xunit.assert (2)