Implemented interface member:
property
Tokenizer
Microsoft.ML.GenAI.Core.ICausalLMPipeline.Tokenizer
2 writes to Tokenizer
Microsoft.ML.GenAI.Core (2)
Pipeline\CausalLMPipeline.cs (2)
101
this.
Tokenizer
= tokenizer;
111
this.
Tokenizer
= default!;
8 references to Tokenizer
Microsoft.ML.GenAI.Core (8)
Pipeline\CausalLMPipeline.cs (8)
80
public TTokenizer TypedTokenizer { get => (TTokenizer)base.
Tokenizer
; }
247
var inputIds = this.
Tokenizer
.EncodeToIds(prompt);
258
var tokens = this.
Tokenizer
.EncodeToTokens(x, out var _, false, false);
273
var duplicateTokenString = this.
Tokenizer
switch
276
_ => this.
Tokenizer
.Decode(tokenIds.Concat(tokenIds)) ?? throw new InvalidOperationException("Failed to decode token ids"),
279
var tokenString = this.
Tokenizer
switch
282
_ => this.
Tokenizer
.Decode(tokenIds) ?? throw new InvalidOperationException("Failed to decode token ids"),
308
var inputIds = this.
Tokenizer
.EncodeToIds(prompt);
