53 references to StopWordsRemovingEstimator
Microsoft.ML.Core.Tests (1)
UnitTests\TestEntryPoints.cs (1)
1808StopWordsRemoverOptions = new StopWordsRemovingEstimator.Options(),
Microsoft.ML.PerformanceTests (1)
StochasticDualCoordinateAscentClassifierBench.cs (1)
106StopWordsRemoverOptions = new StopWordsRemovingEstimator.Options(),
Microsoft.ML.Samples (3)
Dynamic\TextTransform.cs (1)
59new StopWordsRemovingEstimator.Options()
Dynamic\Transforms\Text\FeaturizeTextWithOptions.cs (1)
52StopWordsRemoverOptions = new StopWordsRemovingEstimator.Options()
Dynamic\Transforms\Text\RemoveDefaultStopWords.cs (1)
34StopWordsRemovingEstimator.Language.English));
Microsoft.ML.Tests (2)
OnnxConversionTest.cs (1)
1568StopWordsRemovingEstimator.Language.English));
Transformers\TextFeaturizerTests.cs (1)
51var options = new TextFeaturizingEstimator.Options() { StopWordsRemoverOptions = new StopWordsRemovingEstimator.Options(), OutputTokensColumnName = "OutputTokens" };
Microsoft.ML.Transforms (46)
Text\StopWordsRemovingTransformer.cs (37)
60return new StopWordsRemovingTransformer(env, columns.Select(x => new StopWordsRemovingEstimator.ColumnOptions(x.Name, x.Source)).ToArray()); 65/// <see cref="ITransformer"/> resulting from fitting a <see cref="StopWordsRemovingEstimator"/>. 77public StopWordsRemovingEstimator.Language? Language; 110public StopWordsRemovingEstimator.Language Language = StopWordsRemovingEstimator.Defaults.DefaultLanguage; 133internal IReadOnlyCollection<StopWordsRemovingEstimator.ColumnOptions> Columns => _columns.AsReadOnly(); 135private readonly StopWordsRemovingEstimator.ColumnOptions[] _columns; 137private static volatile Dictionary<ReadOnlyMemory<char>, StopWordsRemovingEstimator.Language> _langsDictionary; 149var values = Enum.GetValues(typeof(StopWordsRemovingEstimator.Language)).Cast<int>(); 164private static Dictionary<ReadOnlyMemory<char>, StopWordsRemovingEstimator.Language> LangsDictionary 168Dictionary<ReadOnlyMemory<char>, StopWordsRemovingEstimator.Language> result = _langsDictionary; 171var langsDictionary = Enum.GetValues(typeof(StopWordsRemovingEstimator.Language)).Cast<StopWordsRemovingEstimator.Language>() 181private static (string outputColumnName, string inputColumnName)[] GetColumnPairs(StopWordsRemovingEstimator.ColumnOptions[] columns) 190if (!StopWordsRemovingEstimator.IsColumnTypeValid(type)) 191throw Host.ExceptSchemaMismatch(nameof(inputSchema), "input", ColumnPairs[col].inputColumnName, StopWordsRemovingEstimator.ExpectedColumnType, type.ToString()); 199internal StopWordsRemovingTransformer(IHostEnvironment env, params StopWordsRemovingEstimator.ColumnOptions[] columns) : 233_columns = new StopWordsRemovingEstimator.ColumnOptions[columnsLength]; 236var lang = (StopWordsRemovingEstimator.Language)ctx.Reader.ReadInt32(); 237Contracts.CheckDecode(Enum.IsDefined(typeof(StopWordsRemovingEstimator.Language), lang)); 239_columns[i] = new StopWordsRemovingEstimator.ColumnOptions(ColumnPairs[i].outputColumnName, ColumnPairs[i].inputColumnName, lang, langColName); 261var cols = new StopWordsRemovingEstimator.ColumnOptions[options.Columns.Length]; 265cols[i] = new StopWordsRemovingEstimator.ColumnOptions( 294foreach (StopWordsRemovingEstimator.Language lang in Enum.GetValues(typeof(StopWordsRemovingEstimator.Language))) 319private static void AddResourceIfNotPresent(StopWordsRemovingEstimator.Language lang) 341private static Stream GetResourceFileStreamOrNull(StopWordsRemovingEstimator.Language lang) 374if (!StopWordsRemovingEstimator.IsColumnTypeValid(srcType)) 375throw Host.ExceptSchemaMismatch(nameof(inputSchema), "input", parent._columns[i].InputColumnName, StopWordsRemovingEstimator.ExpectedColumnType, srcType.ToString()); 408StopWordsRemovingEstimator.Language stopWordslang = _parent._columns[iinfo].Language; 483private void UpdateLanguage(ref StopWordsRemovingEstimator.Language langToUse, ValueGetter<ReadOnlyMemory<char>> getLang, ref ReadOnlyMemory<char> langTxt) 488StopWordsRemovingEstimator.Language lang; 494langToUse = StopWordsRemovingEstimator.Defaults.DefaultLanguage; 498private bool ResourceExists(StopWordsRemovingEstimator.Language lang) 661: base(Contracts.CheckRef(env, nameof(env)).Register(nameof(StopWordsRemovingEstimator)), new StopWordsRemovingTransformer(env, columns)) 1077if (!StopWordsRemovingEstimator.IsColumnTypeValid(srcType)) 1078throw Host.ExceptSchemaMismatch(nameof(inputSchema), "input", parent.ColumnPairs[i].inputColumnName, StopWordsRemovingEstimator.ExpectedColumnType, srcType.ToString());
Text\TextCatalog.cs (3)
307public static StopWordsRemovingEstimator RemoveDefaultStopWords(this TransformsCatalog.TextTransforms catalog, 310StopWordsRemovingEstimator.Language language = StopWordsRemovingEstimator.Language.English)
Text\TextFeaturizingEstimator.cs (6)
147/// <description>The <see cref="StopWordsRemovingEstimator.Options"/> removes the language specific list of stop words from the input.</description> 164if (_stopWordsRemoverOptions is StopWordsRemovingEstimator.Options) 167Language = (_stopWordsRemoverOptions as StopWordsRemovingEstimator.Options).Language; 313public StopWordsRemovingEstimator.Language StopwordsLanguage 314=> (StopWordsRemovingEstimator.Language)Enum.Parse(typeof(StopWordsRemovingEstimator.Language), Language.ToString());