1 write to Tokenizer
Microsoft.ML.TorchSharp (1)
Roberta\QATrainer.cs (1)
585
Tokenizer
= TokenizerExtensions.GetInstance(ch);
6 references to Tokenizer
Microsoft.ML.TorchSharp (6)
Roberta\QATrainer.cs (6)
857
var contextTokenId = _parent.
Tokenizer
.RobertaModel().ConvertIdsToOccurrenceRanks(_parent.
Tokenizer
.EncodeToIds(context.ToString()));
859
var questionTokenId = _parent.
Tokenizer
.RobertaModel().ConvertIdsToOccurrenceRanks(_parent.
Tokenizer
.EncodeToIds(question.ToString()));
927
outputCache.PredictedAnswersBuffer[index] = new ReadOnlyMemory<char>(_parent.
Tokenizer
.Decode(_parent.
Tokenizer
.RobertaModel().ConvertOccurrenceRanksToIds(contextIds).ToArray().AsSpan(predictStart - questionLength - 2, predictEnd - predictStart).ToArray()).Trim().ToCharArray());