14 references to ExtractedIds
Microsoft.ML.Tokenizers.Tests (14)
UnigramTests.cs (14)
405(IEnumerable<int> Ids, IEnumerable<string> Tokens, IEnumerable<Range> Offsets) extracted = ExtractedIds(_unigramTokenizer, result, normalizedText, _unigramTokenizer.AddBeginningOfSentence, _unigramTokenizer.AddEndOfSentence);
409extracted = ExtractedIds(_unigramTokenizerFromJson, result, normalizedText, _unigramTokenizerFromJson.AddBeginningOfSentence, _unigramTokenizerFromJson.AddEndOfSentence);
413extracted = ExtractedIds(_unigramTokenizer, result, normalizedText, _unigramTokenizer.AddBeginningOfSentence, _unigramTokenizer.AddEndOfSentence);
417extracted = ExtractedIds(_unigramTokenizerFromJson, result, normalizedText, _unigramTokenizerFromJson.AddBeginningOfSentence, _unigramTokenizerFromJson.AddEndOfSentence);
421extracted = ExtractedIds(_unigramTokenizer, result, normalizedText, true, false);
425extracted = ExtractedIds(_unigramTokenizerFromJson, result, normalizedText, true, false);
429extracted = ExtractedIds(_unigramTokenizer, result, normalizedText, true, false);
433extracted = ExtractedIds(_unigramTokenizerFromJson, result, normalizedText, true, false);
437extracted = ExtractedIds(_unigramTokenizer, result, normalizedText, true, true);
441extracted = ExtractedIds(_unigramTokenizerFromJson, result, normalizedText, true, true);
445extracted = ExtractedIds(_unigramTokenizer, result, normalizedText, true, true);
449extracted = ExtractedIds(_unigramTokenizerFromJson, result, normalizedText, true, true);
454extracted = ExtractedIds(_unigramTokenizerWithSpecialTokens, result, normalizedText, false, false);
474extracted = ExtractedIds(_unigramTokenizerFromJson, result, normalizedText, false, false);