Tokenizer

1 write to Tokenizer

Microsoft.ML.TorchSharp (1)

Roberta\QATrainer.cs (1)

215Tokenizer = TokenizerExtensions.GetInstance(ch);

5 references to Tokenizer

Microsoft.ML.TorchSharp (5)

Roberta\QATrainer.cs (5)

352srcTensor = PrepareBatchTensor(ref inputTensors, device: Device, Tokenizer.RobertaModel().PadIndex); 404var contextTokens = Tokenizer.EncodeToTokens(contextString, out string normalized); 406var contextTokenId = Tokenizer.RobertaModel().ConvertIdsToOccurrenceRanks(contextTokens.Select(t => t.Id).ToArray()); 413var questionTokenId = Tokenizer.EncodeToConverted(question.ToString()); 440EnglishRobertaTokenizer robertaModel = Tokenizer as EnglishRobertaTokenizer;