2 writes to _vocabReverse
Microsoft.ML.Tokenizers (2)
Model\SentencePieceUnigramModel.cs (2)
30_vocabReverse = new (string Piece, float Score, ModelProto.Types.SentencePiece.Types.Type Type)[modelProto.Pieces.Count]; 108_vocabReverse = vocabReverse.ToArray();
19 references to _vocabReverse
Microsoft.ML.Tokenizers (19)
Model\SentencePieceUnigramModel.cs (19)
43_vocabReverse[i] = (piece, score, modelProto.Pieces[i].Type); 55_vocabReverse[i] = (modelProto.Pieces[i].Piece, modelProto.Pieces[i].Score, ModelProto.Types.SentencePiece.Types.Type.Unknown); 77_vocabReverse[modelProto.TrainerSpec.BosId] = (modelProto.TrainerSpec.BosPiece, 0f, ModelProto.Types.SentencePiece.Types.Type.Control); 78_vocabReverse[modelProto.TrainerSpec.EosId] = (modelProto.TrainerSpec.EosPiece, 0f, ModelProto.Types.SentencePiece.Types.Type.Control); 79_vocabReverse[modelProto.TrainerSpec.UnkId] = (modelProto.TrainerSpec.UnkPiece, 0f, ModelProto.Types.SentencePiece.Types.Type.Unknown); 84_vocabReverse[modelProto.TrainerSpec.PadId] = (modelProto.TrainerSpec.PadPiece, 0f, ModelProto.Types.SentencePiece.Types.Type.Control); 124_vocabReverse[BeginningOfSentenceId] = (BeginningOfSentenceToken, 0f, 0); 125_vocabReverse[EndOfSentenceId] = (EndOfSentenceToken, 0f, 0); 184if ((uint)id >= (uint)(_vocabReverse.Length)) 190token = _vocabReverse[id].Piece; 412string stringToken = node.Id == UnknownId ? Helpers.GetString(normalizationSpan.Slice(node.StartsAt, endsAt - node.StartsAt)) : _vocabReverse[node.Id].Piece; 484tokens.Insert(insertionStartPosition++, new EncodedToken(id, _vocabReverse[id].Piece, new Range(offsetStart, offsetStart + charLength))); 489tokens.Insert(insertionStartPosition++, new EncodedToken(id, _vocabReverse[id].Piece, new Range(offsetStart + charLength, offsetStart + charLength))); 556if (_vocabReverse[ret].Type == ModelProto.Types.SentencePiece.Types.Type.Unused) 566float score = _vocabReverse[ret].Type == ModelProto.Types.SentencePiece.Types.Type.UserDefined ? length * _maxScore - 0.1f : _vocabReverse[ret].Score; 953charsConsumed += _vocabReverse[ids[index]].Piece.Length; 1238charsConsumed += _vocabReverse[ids[i].Id].Piece.Length; 1449consumedCharacters += _vocabReverse[node.Id].Piece.Length;