1 write to _vocabReverse
Microsoft.ML.Tokenizers (1)
Model\SentencePieceUnigramModel.cs (1)
38_vocabReverse = new (string Piece, float Score, ModelProto.Types.SentencePiece.Types.Type Type)[modelProto.Pieces.Count];
17 references to _vocabReverse
Microsoft.ML.Tokenizers (17)
Model\SentencePieceUnigramModel.cs (17)
51_vocabReverse[i] = (piece, score, modelProto.Pieces[i].Type); 63_vocabReverse[i] = (modelProto.Pieces[i].Piece, modelProto.Pieces[i].Score, ModelProto.Types.SentencePiece.Types.Type.Unknown); 85_vocabReverse[modelProto.TrainerSpec.BosId] = (modelProto.TrainerSpec.BosPiece, 0f, ModelProto.Types.SentencePiece.Types.Type.Control); 86_vocabReverse[modelProto.TrainerSpec.EosId] = (modelProto.TrainerSpec.EosPiece, 0f, ModelProto.Types.SentencePiece.Types.Type.Control); 87_vocabReverse[modelProto.TrainerSpec.UnkId] = (modelProto.TrainerSpec.UnkPiece, 0f, ModelProto.Types.SentencePiece.Types.Type.Unknown); 92_vocabReverse[modelProto.TrainerSpec.PadId] = (modelProto.TrainerSpec.PadPiece, 0f, ModelProto.Types.SentencePiece.Types.Type.Control); 134if ((uint)id >= (uint)(_vocabReverse.Length)) 140token = _vocabReverse[id].Piece; 362string stringToken = node.Id == UnknownId ? Helpers.GetString(normalizationSpan.Slice(node.StartsAt, endsAt - node.StartsAt)) : _vocabReverse[node.Id].Piece; 434tokens.Insert(insertionStartPosition++, new EncodedToken(id, _vocabReverse[id].Piece, new Range(offsetStart, offsetStart + charLength))); 439tokens.Insert(insertionStartPosition++, new EncodedToken(id, _vocabReverse[id].Piece, new Range(offsetStart + charLength, offsetStart + charLength))); 506if (_vocabReverse[ret].Type == ModelProto.Types.SentencePiece.Types.Type.Unused) 516float score = _vocabReverse[ret].Type == ModelProto.Types.SentencePiece.Types.Type.UserDefined ? length * _maxScore - 0.1f : _vocabReverse[ret].Score; 903charsConsumed += _vocabReverse[ids[index]].Piece.Length; 1188charsConsumed += _vocabReverse[ids[i].Id].Piece.Length; 1399consumedCharacters += _vocabReverse[node.Id].Piece.Length;