ExtractedIds

7 references to ExtractedIds

Microsoft.ML.Tokenizers.Tests (7)

UnigramTests.cs (7)

330(IEnumerable<int> Ids, IEnumerable<string> Tokens, IEnumerable<Range> Offsets) extracted = ExtractedIds(_unigramTokenizer, result, normalizedText, _unigramTokenizer.AddBeginningOfSentence, _unigramTokenizer.AddEndOfSentence); 334extracted = ExtractedIds(_unigramTokenizer, result, normalizedText, _unigramTokenizer.AddBeginningOfSentence, _unigramTokenizer.AddEndOfSentence); 338extracted = ExtractedIds(_unigramTokenizer, result, normalizedText, true, false); 342extracted = ExtractedIds(_unigramTokenizer, result, normalizedText, true, false); 346extracted = ExtractedIds(_unigramTokenizer, result, normalizedText, true, true); 350extracted = ExtractedIds(_unigramTokenizer, result, normalizedText, true, true); 355extracted = ExtractedIds(_unigramTokenizerWithSpecialTokens, result, normalizedText, false, false);