1 instantiation of TokenizingByCharactersEstimator
Microsoft.ML.Transforms (1)
Text\TextCatalog.cs (1)
110return new TokenizingByCharactersEstimator(env, useMarkerCharacters, InputOutputColumnPair.ConvertToValueTuples(columns));
15 references to TokenizingByCharactersEstimator
Microsoft.ML.Samples (1)
Dynamic\NgramExtraction.cs (1)
42var charsPipeline = ml.Transforms.Text
Microsoft.ML.Tests (3)
OnnxConversionTest.cs (1)
875var pipeline = new TokenizingByCharactersEstimator(mlContext, useMarkerCharacters: useMarkerCharacters, columns: new[] { ("TokenizedText", "text") });
Transformers\CharTokenizeTests.cs (2)
44var pipe = new TokenizingByCharactersEstimator(Env, columns: new[] { ("TokenizeA", "A"), ("TokenizeB", "B") }); 62var pipe = new TokenizingByCharactersEstimator(Env, columns: new[] { ("TokenizeA", "A"), ("TokenizeB", "B") });
Microsoft.ML.Transforms (11)
Text\TextCatalog.cs (5)
12using CharTokenizingDefaults = TokenizingByCharactersEstimator.Defaults; 70/// Create a <see cref="TokenizingByCharactersEstimator"/>, which tokenizes by splitting text into sequences of characters 89public static TokenizingByCharactersEstimator TokenizeIntoCharactersAsKeys(this TransformsCatalog.TextTransforms catalog, 97/// Create a <see cref="TokenizingByCharactersEstimator"/>, which tokenizes incoming text in input columns and output the tokens as output columns. 104internal static TokenizingByCharactersEstimator TokenizeIntoCharactersAsKeys(this TransformsCatalog.TextTransforms catalog,
Text\TokenizingByCharacters.cs (6)
33/// <see cref="ITransformer"/> resulting from fitting a <see cref="TokenizingByCharactersEstimator"/>. 61public bool UseMarkerChars = TokenizingByCharactersEstimator.Defaults.UseMarkerCharacters; 108internal TokenizingByCharactersTransformer(IHostEnvironment env, bool useMarkerCharacters = TokenizingByCharactersEstimator.Defaults.UseMarkerCharacters, 123if (!TokenizingByCharactersEstimator.IsColumnTypeValid(type)) 124throw Host.ExceptParam(nameof(inputSchema), TokenizingByCharactersEstimator.ExpectedColumnType); 674: base(Contracts.CheckRef(env, nameof(env)).Register(nameof(TokenizingByCharactersEstimator)), new TokenizingByCharactersTransformer(env, useMarkerCharacters, columns))