5 implementations of IMultiStreamSource
Microsoft.ML.AutoML (1)
ColumnInference\TextFileSample.cs (1)
16
internal sealed class TextFileSample :
IMultiStreamSource
Microsoft.ML.Data (3)
DataLoadSave\MultiFileSource.cs (2)
19
public sealed class MultiFileSource :
IMultiStreamSource
100
public sealed class FileHandleSource :
IMultiStreamSource
Utilities\ModelFileUtils.cs (1)
358
private sealed class RepositoryStreamWrapper :
IMultiStreamSource
Microsoft.ML.TestFramework (1)
BytesStreamSource.cs (1)
15
public sealed class BytesStreamSource :
IMultiStreamSource
120 references to IMultiStreamSource
Microsoft.ML.AutoML (5)
ColumnInference\ColumnTypeInference.cs (2)
240
public static InferenceResult InferTextFileColumnTypes(MLContext context,
IMultiStreamSource
fileSource, Arguments args)
245
private static InferenceResult InferTextFileColumnTypesCore(MLContext context,
IMultiStreamSource
fileSource, Arguments args)
ColumnInference\TextFileContents.cs (2)
51
public static ColumnSplitResult TrySplitColumns(MLContext context,
IMultiStreamSource
source, char[] separatorCandidates)
86
private static bool TryParseFile(MLContext context, TextLoader.Options options,
IMultiStreamSource
source,
ColumnInference\TextFileSample.cs (1)
14
/// This class holds an in-memory sample of the text file, and serves as an <see cref="
IMultiStreamSource
"/> proxy to it.
Microsoft.ML.Data (78)
Commands\DataCommand.cs (5)
27
public IComponentFactory<
IMultiStreamSource
, ILegacyDataLoader> Loader;
215
protected ILegacyDataLoader CreateAndSaveLoader(Func<IHostEnvironment,
IMultiStreamSource
, ILegacyDataLoader> defaultLoaderFactory = null)
319
protected ILegacyDataLoader CreateLoader(Func<IHostEnvironment,
IMultiStreamSource
, ILegacyDataLoader> defaultLoaderFactory = null)
332
Func<IHostEnvironment,
IMultiStreamSource
, ILegacyDataLoader> defaultLoaderFactory = null,
350
IMultiStreamSource
fileSource = new MultiFileSource(dataFile);
Data\ILegacyDataLoader.cs (3)
44
internal delegate void SignatureDataLoader(
IMultiStreamSource
data);
50
internal delegate void SignatureLoadDataLoader(ModelLoadContext ctx,
IMultiStreamSource
data);
54
/// and is instantiatable from arguments and an <see cref="
IMultiStreamSource
"/> .
DataLoadSave\Binary\BinaryLoader.cs (3)
813
public BinaryLoader(IHostEnvironment env, Arguments args,
IMultiStreamSource
file)
883
private static BinaryLoader Create(IHostEnvironment env, ModelLoadContext ctx,
IMultiStreamSource
files)
926
private static Stream OpenStream(
IMultiStreamSource
files)
DataLoadSave\Binary\BinaryLoaderSaverCatalog.cs (2)
19
/// Load a <see cref="IDataView"/> from an <see cref="
IMultiStreamSource
"/> on a binary file.
24
public static IDataView LoadFromBinary(this DataOperationsCatalog catalog,
IMultiStreamSource
fileSource)
DataLoadSave\CompositeDataLoader.cs (3)
10
[assembly: LoadableClass(CompositeDataLoader<
IMultiStreamSource
, ITransformer>.Summary, typeof(CompositeDataLoader<
IMultiStreamSource
, ITransformer>), null, typeof(SignatureLoadModel),
11
"Composite Loader", CompositeDataLoader<
IMultiStreamSource
, ITransformer>.LoaderSignature)]
DataLoadSave\DataLoaderExtensions.cs (1)
27
public static IDataView Load(this IDataLoader<
IMultiStreamSource
> loader, params string[] path)
DataLoadSave\LegacyCompositeDataLoader.cs (3)
36
public IComponentFactory<
IMultiStreamSource
, ILegacyDataLoader> Loader;
94
public static ILegacyDataLoader Create(IHostEnvironment env, Arguments args,
IMultiStreamSource
files)
293
public static ILegacyDataLoader Create(IHostEnvironment env, ModelLoadContext ctx,
IMultiStreamSource
files)
DataLoadSave\Text\TextLoader.cs (16)
30
public sealed partial class TextLoader : IDataLoader<
IMultiStreamSource
>
497
/// <see cref="TextLoader.Load(
IMultiStreamSource
)"/> is called. The sample can be used to infer slot name annotations if present.
704
public Bindings(TextLoader parent, Column[] cols,
IMultiStreamSource
headerFile,
IMultiStreamSource
dataSample)
1148
internal TextLoader(IHostEnvironment env, Options options = null,
IMultiStreamSource
dataSample = null)
1160
IMultiStreamSource
headerFile = null;
1309
private static bool TryParseSchema(IHost host,
IMultiStreamSource
files,
1386
internal static bool FileContainsValidSchema(IHostEnvironment env,
IMultiStreamSource
files, out Options options)
1489
internal static ILegacyDataLoader Create(IHostEnvironment env, ModelLoadContext ctx,
IMultiStreamSource
files)
1491
internal static ILegacyDataLoader Create(IHostEnvironment env, Options options,
IMultiStreamSource
files)
1497
internal static IDataView LoadFile(IHostEnvironment env, Options options,
IMultiStreamSource
fileSource)
1545
public IDataView Load(
IMultiStreamSource
source) => new BoundLoader(this, source);
1553
IMultiStreamSource
dataSample = null)
1569
IMultiStreamSource
dataSample = null)
1641
private readonly
IMultiStreamSource
_files;
1643
public BoundLoader(TextLoader loader,
IMultiStreamSource
files)
DataLoadSave\Text\TextLoaderCursor.cs (6)
138
public static DataViewRowCursor Create(TextLoader parent,
IMultiStreamSource
files, bool[] active)
155
public static DataViewRowCursor[] CreateSet(TextLoader parent,
IMultiStreamSource
files, bool[] active, int n)
209
public static void GetSomeLines(
IMultiStreamSource
source, int count, bool readMultilines, char[] separators, char escapeChar, ref List<ReadOnlyMemory<char>> lines)
240
public static string GetEmbeddedArgs(
IMultiStreamSource
files)
412
private readonly
IMultiStreamSource
_files;
420
public LineReader(
IMultiStreamSource
files, int batchSize, int bufSize, bool hasHeader, bool readMultilines, char[] separators, char escapeChar, long limit, int cref)
DataLoadSave\Text\TextLoaderSaverCatalog.cs (12)
26
/// indicates that the first line in the <see paramref="dataSample"/> will be used for feature names, and that when <see cref="TextLoader.Load(
IMultiStreamSource
)"/>
28
/// skip the first line when <see cref="TextLoader.Load(
IMultiStreamSource
)"/> is called, but columns will not have slot names annotations. This is
29
/// because the output schema is made when the loader is created, and not when <see cref="TextLoader.Load(
IMultiStreamSource
)"/> is called.</param>
56
IMultiStreamSource
dataSample = null,
86
IMultiStreamSource
dataSample = null)
98
/// indicates that the first line in the <see paramref="dataSample"/> will be used for feature names, and that when <see cref="TextLoader.Load(
IMultiStreamSource
)"/>
100
/// skip the first line when <see cref="TextLoader.Load(
IMultiStreamSource
)"/> is called, but columns will not have slot names annotations. This is
101
/// because the output schema is made when the loader is created, and not when <see cref="TextLoader.Load(
IMultiStreamSource
)"/> is called.</param>
116
IMultiStreamSource
dataSample = null,
133
IMultiStreamSource
dataSample = null)
145
/// <see cref="TextLoader.Load(
IMultiStreamSource
)"/> is called.</param>
218
/// <see cref="TextLoader.Load(
IMultiStreamSource
)"/> is called.</param>
DataLoadSave\Transpose\TransposeLoader.cs (4)
326
private readonly
IMultiStreamSource
_file;
395
public TransposeLoader(IHostEnvironment env, Arguments args,
IMultiStreamSource
file)
425
private TransposeLoader(IHost host, ModelLoadContext ctx,
IMultiStreamSource
file)
485
public static TransposeLoader Create(IHostEnvironment env, ModelLoadContext ctx,
IMultiStreamSource
files)
Model\ModelOperationsCatalog.cs (9)
35
/// for an empty transformer chain. Upon loading with <see cref="LoadWithDataLoader(Stream, out IDataLoader{
IMultiStreamSource
})"/>
61
/// for an empty transformer chain. Upon loading with <see cref="LoadWithDataLoader(Stream, out IDataLoader{
IMultiStreamSource
})"/>
168
ModelLoadContext.LoadModelOrNull<IDataLoader<
IMultiStreamSource
>, SignatureLoadModel>(_env, out var dataLoader, rep, null);
224
private ITransformer DecomposeLoader(ref IDataLoader<
IMultiStreamSource
> loader)
228
if (loader is CompositeDataLoader<
IMultiStreamSource
, ITransformer> composite)
258
public ITransformer LoadWithDataLoader(Stream stream, out IDataLoader<
IMultiStreamSource
> loader)
266
ModelLoadContext.LoadModel<IDataLoader<
IMultiStreamSource
>, SignatureLoadModel>(_env, out loader, rep, null);
271
throw _env.Except(ex, "Model does not contain an " + nameof(IDataLoader<
IMultiStreamSource
>) +
286
public ITransformer LoadWithDataLoader(string filePath, out IDataLoader<
IMultiStreamSource
> loader)
Transforms\ValueMapping.cs (2)
423
public IComponentFactory<
IMultiStreamSource
, ILegacyDataLoader> Loader;
613
IMultiStreamSource
fileSource = new MultiFileSource(options.DataFile);
Transforms\ValueToKeyMappingTransformer.cs (3)
113
internal IComponentFactory<
IMultiStreamSource
, ILegacyDataLoader> Loader;
387
string file, string termsColumn, IComponentFactory<
IMultiStreamSource
, ILegacyDataLoader> loaderFactory,
404
IMultiStreamSource
fileSource = new MultiFileSource(file);
Utilities\ComponentCreation.cs (3)
83
internal static ILegacyDataLoader CreateLoader<TArgs>(this IHostEnvironment env, TArgs arguments,
IMultiStreamSource
files)
95
internal static ILegacyDataLoader CreateLoader(this IHostEnvironment env, string settings,
IMultiStreamSource
files)
99
Type factoryType = typeof(IComponentFactory<
IMultiStreamSource
, ILegacyDataLoader>);
Utilities\ModelFileUtils.cs (3)
43
public static IDataView LoadPipeline(IHostEnvironment env, Stream modelStream,
IMultiStreamSource
files, bool extractInnerPipe = false)
63
public static IDataView LoadPipeline(IHostEnvironment env, RepositoryReader rep,
IMultiStreamSource
files, bool extractInnerPipe = false)
176
public static ILegacyDataLoader LoadLoader(IHostEnvironment env, RepositoryReader rep,
IMultiStreamSource
files, bool loadTransforms)
Microsoft.ML.IntegrationTests (5)
ModelFiles.cs (5)
163
IDataLoader<
IMultiStreamSource
> loadedCompositeLoader;
210
Assert.IsType<CompositeDataLoader<
IMultiStreamSource
, ITransformer>>(loadedCompositeLoader).Transformer));
281
IDataLoader<
IMultiStreamSource
> loadedLoader;
414
var compositeLoader = Assert.IsType<CompositeDataLoader<
IMultiStreamSource
, ITransformer>>(loadedLoaderWithTransformer);
477
out ITransformer loadedWithLoader, out IDataLoader<
IMultiStreamSource
> loadedLoaderWithTransformer)
Microsoft.ML.Parquet (12)
ParquetLoader.cs (4)
117
public ParquetLoader(IHostEnvironment env, Arguments args,
IMultiStreamSource
files)
176
private ParquetLoader(IHost host, ModelLoadContext ctx,
IMultiStreamSource
files)
239
public static ParquetLoader Create(IHostEnvironment env, ModelLoadContext ctx,
IMultiStreamSource
files)
371
private static Stream OpenStream(
IMultiStreamSource
files)
PartitionedFileLoader.cs (8)
81
public IComponentFactory<
IMultiStreamSource
, ILegacyDataLoader> Loader;
160
private readonly
IMultiStreamSource
_files;
174
public PartitionedFileLoader(IHostEnvironment env, Arguments args,
IMultiStreamSource
files)
210
private PartitionedFileLoader(IHost host, ModelLoadContext ctx,
IMultiStreamSource
files)
243
public static PartitionedFileLoader Create(IHostEnvironment env, ModelLoadContext ctx,
IMultiStreamSource
files)
353
private ILegacyDataLoader CreateLoaderFromBytes(byte[] loaderBytes,
IMultiStreamSource
files)
388
public Cursor(IChannelProvider provider, PartitionedFileLoader parent,
IMultiStreamSource
files, IEnumerable<DataViewSchema.Column> columnsNeeded, Random rand)
748
private string GetRelativePath(string basepath,
IMultiStreamSource
files)
Microsoft.ML.PerformanceTests (2)
Helpers\EnvironmentFactory.cs (2)
15
where TLoader : IDataLoader<
IMultiStreamSource
>
31
where TLoader : IDataLoader<
IMultiStreamSource
>
Microsoft.ML.Predictor.Tests (1)
TestTransposer.cs (1)
241
IMultiStreamSource
src;
Microsoft.ML.TestFramework (1)
DataPipe\TestDataPipeBase.cs (1)
458
private ILegacyDataLoader LoadPipe(string pathModel, IHostEnvironment env,
IMultiStreamSource
files)
Microsoft.ML.Transforms (16)
SvmLight\SvmLightLoader.cs (9)
77
public sealed class SvmLightLoader : IDataLoader<
IMultiStreamSource
>
392
/// This class creates an <see cref="IDataView"/> from an <see cref="
IMultiStreamSource
"/>, that has a single text column
402
private readonly
IMultiStreamSource
_files;
404
public TextDataView(IHostEnvironment env,
IMultiStreamSource
files)
552
internal SvmLightLoader(IHostEnvironment env, Options options = null,
IMultiStreamSource
dataSample = null)
652
private static IDataView GetData(IHostEnvironment env, long? numRows,
IMultiStreamSource
dataSample)
750
public IDataView Load(
IMultiStreamSource
input)
763
internal static ILegacyDataLoader Create(IHostEnvironment env, ModelLoadContext ctx,
IMultiStreamSource
files)
768
internal static ILegacyDataLoader Create(IHostEnvironment env, Options options,
IMultiStreamSource
files)
SvmLight\SvmLightLoaderSaverCatalog.cs (2)
34
IMultiStreamSource
dataSample = null)
51
IMultiStreamSource
dataSample = null)
Text\StopWordsRemovingTransformer.cs (4)
719
internal IComponentFactory<
IMultiStreamSource
, ILegacyDataLoader> Loader;
779
IComponentFactory<
IMultiStreamSource
, ILegacyDataLoader> loader, ref string stopwordsCol)
832
IComponentFactory<
IMultiStreamSource
, ILegacyDataLoader> loaderFactory, out NormStr.Pool stopWordsMap)
928
string dataFile, string stopwordsColumn, IComponentFactory<
IMultiStreamSource
, ILegacyDataLoader> loader, params (string outputColumnName, string inputColumnName)[] columns) :
Text\WordBagTransform.cs (1)
598
internal IComponentFactory<
IMultiStreamSource
, ILegacyDataLoader> Loader;