1 instantiation of Word
Microsoft.ML.Tokenizers (1)
Model\Word.cs (1)
28public static Word WithCapacity(int capacity) => new Word(capacity);
14 references to Word
Microsoft.ML.Tokenizers (14)
Model\BPETokenizer.cs (13)
338Cache = new StringSpanOrdinalKeyCache<Word>(); 1117internal StringSpanOrdinalKeyCache<Word>? Cache { get; } 1186internal Word MergeWord(ReadOnlySpan<char> w, ref PriorityQueue<Merge>? priorityQueue) 1188Word word = Word.WithCapacity(w.Length); 1301internal void WordToTokens(ref Word word, List<EncodedToken> tokens, int offset, ReadOnlySpan<int> mapping) => word.ToTokens(VocabReverse, tokens, offset, mapping); 1339Word word; 1370internal int WordToIds(ref Word word, IList<int>? accumulatedIds, out int charsConsumed, int fullTextLength, int maxTokens) 1391internal int WordToIdsFromEnd(ref Word word, IList<int>? accumulatedIds, out int textIndex, int fullTextLength, int maxTokens) 1421Word word; 1453if (Cache.TryGetValue(textSpan, out Word hit)) 1494Word word; 1533if (Cache.TryGetValue(textSpan, out Word hit))
Model\Word.cs (1)
28public static Word WithCapacity(int capacity) => new Word(capacity);