Tokenizer

1 write to Tokenizer

Microsoft.ML.TorchSharp (1)

NasBert\NasBertTrainer.cs (1)

203Tokenizer = TokenizerExtensions.GetInstance(ch);

7 references to Tokenizer

Microsoft.ML.TorchSharp (7)

NasBert\NasBertTrainer.cs (5)

204EnglishRobertaTokenizer tokenizerModel = Tokenizer.RobertaModel(); 240return DataUtils.CollateTokens(inputTensors, Tokenizer.RobertaModel().PadIndex, device: Device); 250t = torch.tensor((new[] { 0 /* InitToken */ }).Concat(Tokenizer.EncodeToConverted(sentence1.ToString())).ToList(), device: Device); 258t = torch.tensor((new[] { 0 /* InitToken */ }).Concat(Tokenizer.EncodeToConverted(sentence1.ToString())) 259.Concat(new[] { 2 /* SeparatorToken */ }).Concat(Tokenizer.EncodeToConverted(sentence2.ToString())).ToList(), device: Device);

NasBert\NerTrainer.cs (2)

170IReadOnlyList<EncodedToken> encoding = Tokenizer.EncodeToTokens(sentence, out string normalizedText); 190t = torch.tensor((ZeroArray).Concat(Tokenizer.RobertaModel().ConvertIdsToOccurrenceRanks(encoding.Select(t => t.Id).ToArray())).ToList(), device: Device);