3 instantiations of WordTokenizingEstimator
Microsoft.ML.Transforms (3)
Text\TextFeaturizingEstimator.cs (1)
487chain = AddToChainAndTransform(chain, new WordTokenizingEstimator(h, xfCols).Fit(view), ref view);
Text\WordBagTransform.cs (1)
168estimator = estimator.Append(new WordTokenizingEstimator(h, tokenizeColumns));
Text\WordHashBagProducingTransform.cs (1)
135ITransformer t1 = new WordTokenizingEstimator(env, tokenizeColumns.ToArray()).Fit(view);
34 references to WordTokenizingEstimator
Microsoft.ML.Samples (1)
Dynamic\Transforms\Text\TokenizeIntoWords.cs (1)
29var textPipeline = mlContext.Transforms.Text.TokenizeIntoWords("Words",
Microsoft.ML.Tests (11)
OnnxConversionTest.cs (1)
1448var pipeline = mlContext.Transforms.Text.TokenizeIntoWords("Tokens", "Text", new[] { ' ' });
Transformers\TextFeaturizerTests.cs (2)
494var est = new WordTokenizingEstimator(Env, "words", "text", separators: new[] { ' ', '?', '!', '.', ',' }); 568new WordTokenizingEstimator.ColumnOptions("Text", "Text")
Transformers\ValueMappingTests.cs (2)
106new WordTokenizingEstimator.ColumnOptions("TokenizeA", "A") 145var estimator = new WordTokenizingEstimator(Env, new[] { new WordTokenizingEstimator.ColumnOptions("TokenizeA", "A") }).
Transformers\WordTokenizeTests.cs (6)
58var pipe = new WordTokenizingEstimator(Env, new[]{ 59new WordTokenizingEstimator.ColumnOptions("TokenizeA", "A"), 60new WordTokenizingEstimator.ColumnOptions("TokenizeB", "B"), 100var pipe = new WordTokenizingEstimator(Env, new[]{ 101new WordTokenizingEstimator.ColumnOptions("TokenizeA", "A"), 102new WordTokenizingEstimator.ColumnOptions("TokenizeB", "B"),
Microsoft.ML.Transforms (22)
Text\TextCatalog.cs (4)
214/// Create a <see cref="WordTokenizingEstimator"/>, which tokenizes input text using <paramref name="separators"/> as separators. 229public static WordTokenizingEstimator TokenizeIntoWords(this TransformsCatalog.TextTransforms catalog, 241internal static WordTokenizingEstimator TokenizeIntoWords(this TransformsCatalog.TextTransforms catalog, 242params WordTokenizingEstimator.ColumnOptions[] columns)
Text\TextFeaturizingEstimator.cs (2)
477var xfCols = new WordTokenizingEstimator.ColumnOptions[textCols.Length]; 481var col = new WordTokenizingEstimator.ColumnOptions(GenerateColumnName(view.Schema, textCols[i], "WordTokenizer"), textCols[i]);
Text\WordBagTransform.cs (2)
128var tokenizeColumns = new WordTokenizingEstimator.ColumnOptions[options.Columns.Length]; 148tokenizeColumns[iinfo] = new WordTokenizingEstimator.ColumnOptions(column.Name, column.Source.Length > 1 ? column.Name : column.Source[0]);
Text\WordHashBagProducingTransform.cs (2)
105var tokenizeColumns = new List<WordTokenizingEstimator.ColumnOptions>(); 116tokenizeColumns.Add(new WordTokenizingEstimator.ColumnOptions(curTmpNames[isrc] = uniqueSourceNames[iinfo][isrc], options.Columns[iinfo].Source[isrc]));
Text\WordTokenizing.cs (12)
36/// <see cref="ITransformer"/> resulting from fitting an <see cref="WordTokenizingEstimator"/>. 107internal IReadOnlyCollection<WordTokenizingEstimator.ColumnOptions> Columns => _columns.AsReadOnly(); 108private readonly WordTokenizingEstimator.ColumnOptions[] _columns; 110private static (string name, string inputColumnName)[] GetColumnPairs(WordTokenizingEstimator.ColumnOptions[] columns) 116internal WordTokenizingTransformer(IHostEnvironment env, params WordTokenizingEstimator.ColumnOptions[] columns) : 125if (!WordTokenizingEstimator.IsColumnTypeValid(type)) 126throw Host.ExceptSchemaMismatch(nameof(inputSchema), "input", ColumnPairs[col].inputColumnName, WordTokenizingEstimator.ExpectedColumnType, type.ToString()); 133_columns = new WordTokenizingEstimator.ColumnOptions[columnsLength]; 142_columns[i] = new WordTokenizingEstimator.ColumnOptions(ColumnPairs[i].outputColumnName, ColumnPairs[i].inputColumnName, separators); 183var cols = new WordTokenizingEstimator.ColumnOptions[options.Columns.Length]; 188cols[i] = new WordTokenizingEstimator.ColumnOptions(item.Name, item.Source ?? item.Name, separators); 489: base(Contracts.CheckRef(env, nameof(env)).Register(nameof(WordTokenizingEstimator)), new WordTokenizingTransformer(env, columns))