2 writes to _vocabReverse
Microsoft.ML.Tokenizers (2)
Model\SentencePieceUnigramModel.cs (2)
41_vocabReverse = new (string Piece, float Score, ModelProto.Types.SentencePiece.Types.Type Type)[modelProto.Pieces.Count]; 174_vocabReverse = new (string Piece, float Score, ModelProto.Types.SentencePiece.Types.Type Type)[pieces!.Count];
22 references to _vocabReverse
Microsoft.ML.Tokenizers (22)
Model\SentencePieceUnigramModel.cs (22)
54_vocabReverse[i] = (piece, score, modelProto.Pieces[i].Type); 66_vocabReverse[i] = (modelProto.Pieces[i].Piece, modelProto.Pieces[i].Score, ModelProto.Types.SentencePiece.Types.Type.Unknown); 88_vocabReverse[modelProto.TrainerSpec.BosId] = (modelProto.TrainerSpec.BosPiece, 0f, ModelProto.Types.SentencePiece.Types.Type.Control); 89_vocabReverse[modelProto.TrainerSpec.EosId] = (modelProto.TrainerSpec.EosPiece, 0f, ModelProto.Types.SentencePiece.Types.Type.Control); 90_vocabReverse[modelProto.TrainerSpec.UnkId] = (modelProto.TrainerSpec.UnkPiece, 0f, ModelProto.Types.SentencePiece.Types.Type.Unknown); 95_vocabReverse[modelProto.TrainerSpec.PadId] = (modelProto.TrainerSpec.PadPiece, 0f, ModelProto.Types.SentencePiece.Types.Type.Control); 197_vocabReverse[i] = (piece, score, ModelProto.Types.SentencePiece.Types.Type.Unknown); 201_vocabReverse[i] = (piece, score, ModelProto.Types.SentencePiece.Types.Type.Control); 205_vocabReverse[i] = (piece, score, ModelProto.Types.SentencePiece.Types.Type.Normal); 239_vocabReverse[unkId] = (unkToken, 0f, ModelProto.Types.SentencePiece.Types.Type.Unknown); 253_vocabReverse[controlId] = (piece, 0f, ModelProto.Types.SentencePiece.Types.Type.Control); 409if ((uint)id >= (uint)(_vocabReverse.Length)) 415token = _vocabReverse[id].Piece; 637string stringToken = node.Id == UnknownId ? Helpers.GetString(normalizationSpan.Slice(node.StartsAt, endsAt - node.StartsAt)) : _vocabReverse[node.Id].Piece; 709tokens.Insert(insertionStartPosition++, new EncodedToken(id, _vocabReverse[id].Piece, new Range(offsetStart, offsetStart + charLength))); 714tokens.Insert(insertionStartPosition++, new EncodedToken(id, _vocabReverse[id].Piece, new Range(offsetStart + charLength, offsetStart + charLength))); 781if (_vocabReverse[ret].Type == ModelProto.Types.SentencePiece.Types.Type.Unused) 791float score = _vocabReverse[ret].Type == ModelProto.Types.SentencePiece.Types.Type.UserDefined ? length * _maxScore - 0.1f : _vocabReverse[ret].Score; 1189charsConsumed += _vocabReverse[ids[index]].Piece.Length; 1484charsConsumed += _vocabReverse[ids[i].Id].Piece.Length; 1705consumedCharacters += _vocabReverse[node.Id].Piece.Length;