1 instantiation of SentencePiece
Microsoft.ML.Tokenizers (1)
SentencepieceModel.cs (1)
238SentencePiece result = new();
29 references to SentencePiece
Microsoft.ML.Tokenizers (29)
Model\SentencePieceBpeModel.cs (11)
30var piece = modelProto.Pieces[i]; 34if (piece.Type == ModelProto.Types.SentencePiece.Types.Type.Byte) 180if (type != (byte)ModelProto.Types.SentencePiece.Types.Type.Unused) 268if (id.Type != (byte)ModelProto.Types.SentencePiece.Types.Type.Unused || 458if (type != (byte)ModelProto.Types.SentencePiece.Types.Type.Unused) 573if (id.Type != (byte)ModelProto.Types.SentencePiece.Types.Type.Unused || 726if (type != (byte)ModelProto.Types.SentencePiece.Types.Type.Unused) 832if (id.Type != (byte)ModelProto.Types.SentencePiece.Types.Type.Unused || 1002if (type != (byte)ModelProto.Types.SentencePiece.Types.Type.Unused) 1108if (id.Type != (byte)ModelProto.Types.SentencePiece.Types.Type.Unused || 1208if (leftId.Type == (byte)ModelProto.Types.SentencePiece.Types.Type.Unused)
Model\SentencePieceUnigramModel.cs (14)
21private readonly (string Piece, float Score, ModelProto.Types.SentencePiece.Types.Type Type)[] _vocabReverse; 38_vocabReverse = new (string Piece, float Score, ModelProto.Types.SentencePiece.Types.Type Type)[modelProto.Pieces.Count]; 45if (modelProto.Pieces[i].Type == ModelProto.Types.SentencePiece.Types.Type.Normal || 46modelProto.Pieces[i].Type == ModelProto.Types.SentencePiece.Types.Type.UserDefined || 47modelProto.Pieces[i].Type == ModelProto.Types.SentencePiece.Types.Type.Unused) 56else if (modelProto.Pieces[i].Type == ModelProto.Types.SentencePiece.Types.Type.Byte) 60else if (modelProto.Pieces[i].Type == ModelProto.Types.SentencePiece.Types.Type.Unknown) 63_vocabReverse[i] = (modelProto.Pieces[i].Piece, modelProto.Pieces[i].Score, ModelProto.Types.SentencePiece.Types.Type.Unknown); 85_vocabReverse[modelProto.TrainerSpec.BosId] = (modelProto.TrainerSpec.BosPiece, 0f, ModelProto.Types.SentencePiece.Types.Type.Control); 86_vocabReverse[modelProto.TrainerSpec.EosId] = (modelProto.TrainerSpec.EosPiece, 0f, ModelProto.Types.SentencePiece.Types.Type.Control); 87_vocabReverse[modelProto.TrainerSpec.UnkId] = (modelProto.TrainerSpec.UnkPiece, 0f, ModelProto.Types.SentencePiece.Types.Type.Unknown); 92_vocabReverse[modelProto.TrainerSpec.PadId] = (modelProto.TrainerSpec.PadPiece, 0f, ModelProto.Types.SentencePiece.Types.Type.Control); 506if (_vocabReverse[ret].Type == ModelProto.Types.SentencePiece.Types.Type.Unused) 516float score = _vocabReverse[ret].Type == ModelProto.Types.SentencePiece.Types.Type.UserDefined ? length * _maxScore - 0.1f : _vocabReverse[ret].Score;
SentencepieceModel.cs (4)
174internal List<Types.SentencePiece> Pieces { get; } = new(); 203result.Pieces.Add(Types.SentencePiece.Parse(data, pos, pos + length)); 236internal static SentencePiece Parse(byte[] data, int start, int end) 238SentencePiece result = new();