1 instantiation of Word
Microsoft.ML.Tokenizers (1)
14 references to Word
Microsoft.ML.Tokenizers (14)
Model\BPETokenizer.cs (13)
343Cache = new StringSpanOrdinalKeyCache<Word>();
1121internal StringSpanOrdinalKeyCache<Word>? Cache { get; }
1190internal Word MergeWord(ReadOnlySpan<char> w, ref PriorityQueue<Merge>? priorityQueue)
1192Word word = Word.WithCapacity(w.Length);
1305internal void WordToTokens(ref Word word, List<EncodedToken> tokens, int offset, ReadOnlySpan<int> mapping) => word.ToTokens(VocabReverse, tokens, offset, mapping);
1343Word word;
1374internal int WordToIds(ref Word word, IList<int>? accumulatedIds, out int charsConsumed, int fullTextLength, int maxTokens)
1395internal int WordToIdsFromEnd(ref Word word, IList<int>? accumulatedIds, out int textIndex, int fullTextLength, int maxTokens)
1425Word word;
1457if (Cache.TryGetValue(textSpan, out Word hit))
1498Word word;
1537if (Cache.TryGetValue(textSpan, out Word hit))