30 references to SentencePieceProtobufReader
Microsoft.ML.Tokenizers (30)
SentencepieceModel.cs (30)
185int tag = SentencePieceProtobufReader.ReadRawVarint32(data, end, ref pos);
202length = SentencePieceProtobufReader.ReadLengthPrefix(data, end, ref pos);
208length = SentencePieceProtobufReader.ReadLengthPrefix(data, end, ref pos);
214length = SentencePieceProtobufReader.ReadLengthPrefix(data, end, ref pos);
220SentencePieceProtobufReader.SkipField(data, end, wireType, ref pos);
242int tag = SentencePieceProtobufReader.ReadRawVarint32(data, end, ref pos);
249result.Piece = SentencePieceProtobufReader.ReadString(data, end, ref pos);
253result.Score = SentencePieceProtobufReader.ReadFloat(data, end, ref pos);
257result.Type = (Types.Type)SentencePieceProtobufReader.ReadRawVarint32(data, end, ref pos);
261SentencePieceProtobufReader.SkipField(data, end, wireType, ref pos);
343int tag = SentencePieceProtobufReader.ReadRawVarint32(data, end, ref pos);
350result.ModelType = (Types.ModelType)SentencePieceProtobufReader.ReadRawVarint32(data, end, ref pos);
354result.TreatWhitespaceAsSuffix = SentencePieceProtobufReader.ReadRawVarint32(data, end, ref pos) != 0;
358result.ByteFallback = SentencePieceProtobufReader.ReadRawVarint32(data, end, ref pos) != 0;
362result.UnkId = SentencePieceProtobufReader.ReadRawVarint32(data, end, ref pos);
366result.BosId = SentencePieceProtobufReader.ReadRawVarint32(data, end, ref pos);
370result.EosId = SentencePieceProtobufReader.ReadRawVarint32(data, end, ref pos);
374result.PadId = SentencePieceProtobufReader.ReadRawVarint32(data, end, ref pos);
378result.UnkPiece = SentencePieceProtobufReader.ReadString(data, end, ref pos);
382result.BosPiece = SentencePieceProtobufReader.ReadString(data, end, ref pos);
386result.EosPiece = SentencePieceProtobufReader.ReadString(data, end, ref pos);
390result.PadPiece = SentencePieceProtobufReader.ReadString(data, end, ref pos);
394SentencePieceProtobufReader.SkipField(data, end, wireType, ref pos);
429int tag = SentencePieceProtobufReader.ReadRawVarint32(data, end, ref pos);
436result.Name = SentencePieceProtobufReader.ReadString(data, end, ref pos);
440int length = SentencePieceProtobufReader.ReadLengthPrefix(data, end, ref pos);
446result.AddDummyPrefix = SentencePieceProtobufReader.ReadRawVarint32(data, end, ref pos) != 0;
450result.RemoveExtraWhitespaces = SentencePieceProtobufReader.ReadRawVarint32(data, end, ref pos) != 0;
454result.EscapeWhitespaces = SentencePieceProtobufReader.ReadRawVarint32(data, end, ref pos) != 0;
458SentencePieceProtobufReader.SkipField(data, end, wireType, ref pos);