Fabio/voice_audio_multi
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1
voice_audio_multi/external/sentencepiece/src/sentencepiece_processor.h
Fabio c36510c21e
Some checks failed
Linux Build / Linux cpu (push) Has been cancelled
Details
Linux Build / Linux cpu-clang (push) Has been cancelled
Details
Linux Build / Linux vulkan (push) Has been cancelled
Details
mac-build / macOS CPU (push) Has been cancelled
Details
Nix Build / Nix (cpu) (push) Has been cancelled
Details
Nix Build / Nix (vulkan) (push) Has been cancelled
Details
windows-build / Windows CPU (push) Has been cancelled
Details
Initial import: audio.cpp with Orange Pi documentation
2026-11-04 19:05:22 +01:00

753 lines
28 KiB
C++
Raw Blame History

// Copyright 2016 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.!
#ifndef SENTENCEPIECE_PROCESSOR_H_
#define SENTENCEPIECE_PROCESSOR_H_
#include <cstdint>
#include <cstring>
#include <memory>
#include <string>
#include <string_view>
#include <utility>
#include <vector>
#ifndef SWIG
namespace absl {
using std::string_view;
} // namespace absl
#endif // SWIG
namespace sentencepiece {
namespace util {
enum class StatusCode : int {
kOk = 0,
kCancelled = 1,
kUnknown = 2,
kInvalidArgument = 3,
kDeadlineExceeded = 4,
kNotFound = 5,
kAlreadyExists = 6,
kPermissionDenied = 7,
kResourceExhausted = 8,
kFailedPrecondition = 9,
kAborted = 10,
kOutOfRange = 11,
kUnimplemented = 12,
kInternal = 13,
kUnavailable = 14,
kDataLoss = 15,
kUnauthenticated = 16,
};
class Status {
public:
Status();
~Status();
Status(StatusCode code, absl::string_view error_message);
Status(const Status &s);
void operator=(const Status &s);
bool operator==(const Status &s) const;
bool operator!=(const Status &s) const;
inline bool ok() const { return rep_ == nullptr; }
void set_error_message(const char *str);
const char *error_message() const;
const char *message() const { return error_message(); }
StatusCode code() const;
std::string ToString() const;
void IgnoreError();
private:
struct Rep;
std::unique_ptr<Rep> rep_;
};
} // namespace util
// SentencePieceProcessor:
// Simple and language independent tokenizer and de-tokenizer for
// Neural Network Machine Translation.
//
// SentencePieceProcessor provides Encode() and Decode() methods,
// which correspond to tokenization and de-tokenization respectively.
//
// - Encode:
// Given a raw source sentence, encode it into a sequence
// of pieces or vocabulary ids.
//
// - Decode:
// Given a sequence of pieces or vocabulary ids, decode it
// into a de-tokenized raw sentence.
//
// SentencePieceProcessor provides a lossless data conversion
// that allows the original raw sentence to be perfectly reconstructed
// from the encoded data, i.e., Decode(Encode(input)) == input.
// This characteristics is useful, as we can make the de-tokenization
// completely language independent.
//
// Usage:
// SentencePieceProcessor sp;
// sp.Load("//path/to/model");
//
// vector<string> sps;
// sp.Encode("hello world.", &sps).IgnoreError();
//
// vector<int> ids;
// sp.Encode("hello world.", &ids).IgnoreError();
//
// string detok;
// sp.Decode(sps, &detok);
// CHECK_EQ("hello world.", detok).IgnoreError();
//
// sp.Decode(ids, &detok);
// CHECK_EQ("hello world.", detok).IgnoreError();
//
// We can also use SentencePieceText which manages the byte-offsets
// between user input (output) and internal sentence pieces.
//
// SentencePieceText spt;
// sp.Encode("hello world.", &spt);
// // Emits the byte range of each piece.
// for (const auto &piece : spt.pieces()) {
// LOG(INFO) << piece.begin() << " " << piece.end();
// }
//
// sp.Decode({0, 1, 2, 3..}, &spt);
// for (const auto &piece : spt.pieces()) {
// LOG(INFO) << piece.begin() << " " << piece.end();
// }
//
class NBestSentencePieceText;
class ModelInterface;
class SentencePieceText;
class ModelProto;
class NormalizerSpec;
namespace normalizer {
class Normalizer;
} // namespace normalizer
#ifndef SWIGGO
namespace util {
// Redefine std::string for serialized_proto interface as Python's string is
// a Unicode string. We can enforce the return value to be raw byte sequence
// with SWIG's typemap.
using bytes = std::string;
} // namespace util
#endif // SWIGGO
class NBestSentencePieceText;
class ModelInterface;
class SentencePieceText;
class SentencePieceText_SentencePiece;
// Wrapper class of SentencePieceText
// This wrapper only allows an immutable access to the proto and
// hides the actual implementation of protobuf.
// See sentencepiece.proto for the details of this class.
class ImmutableSentencePieceText_ImmutableSentencePiece {
public:
ImmutableSentencePieceText_ImmutableSentencePiece();
~ImmutableSentencePieceText_ImmutableSentencePiece() = default;
const std::string &piece() const;
const std::string &surface() const;
uint32_t id() const;
uint32_t begin() const;
uint32_t end() const;
friend class ImmutableSentencePieceText;
private:
explicit ImmutableSentencePieceText_ImmutableSentencePiece(
const SentencePieceText_SentencePiece &sp);
const SentencePieceText_SentencePiece *sp_ = nullptr;
};
class ImmutableSentencePieceText {
public:
ImmutableSentencePieceText();
virtual ~ImmutableSentencePieceText();
std::vector<ImmutableSentencePieceText_ImmutableSentencePiece> pieces() const;
size_t pieces_size() const;
ImmutableSentencePieceText_ImmutableSentencePiece pieces(int index) const;
const std::string &text() const;
float score() const;
util::bytes SerializeAsString() const;
// Returns the actual mutable proto.
// Do not use this outside of SentencePieceProcessor, as
// it returns the raw pointer managed by the shared_ptr.
SentencePieceText *mutable_proto();
// Converts the utf8 byte spans into Unicode char span.
void ConvertToUnicodeSpans();
friend class ImmutableNBestSentencePieceText;
private:
explicit ImmutableSentencePieceText(const SentencePieceText &spt);
const SentencePieceText *spt_ = nullptr;
std::shared_ptr<SentencePieceText> rep_;
};
// Wrapper class of SentencePieceText
// This wrapper only allows an immutable access to the proto and
// hides the actual implementation of protobuf.
// See sentencepiece.proto for the details of this class.
class ImmutableNBestSentencePieceText {
public:
ImmutableNBestSentencePieceText();
virtual ~ImmutableNBestSentencePieceText();
std::vector<ImmutableSentencePieceText> nbests() const;
size_t nbests_size() const;
ImmutableSentencePieceText nbests(int index) const;
util::bytes SerializeAsString() const;
// Returns the actual mutable proto.
// Do not use this outside of SentencePieceProcessor, as
// it returns the raw pointer managed by the shared_ptr.
NBestSentencePieceText *mutable_proto();
void ConvertToUnicodeSpans();
private:
std::shared_ptr<NBestSentencePieceText> rep_;
};
class SentencePieceProcessor {
public:
SentencePieceProcessor();
virtual ~SentencePieceProcessor();
// Loads model from `filename`.
// Returns false if `filename` cannot be loaded.
virtual util::Status Load(absl::string_view filename);
// Loads model from `filename`.
// Crash if `filename` cannot be loaded.
virtual void LoadOrDie(absl::string_view filename);
// Loads model from `model_proto`.
// `model_proto` is copied.
virtual util::Status Load(const ModelProto &model_proto);
// Loads model from `model_proto`.
// `model_proto` is moved.
virtual util::Status Load(std::unique_ptr<ModelProto> model_proto);
// Loads model from `serialized`, which is a string-serialized model proto.
// Useful to load the model from a platform independent blob object.
virtual util::Status LoadFromSerializedProto(absl::string_view serialized);
// Returns the status. Encode/Decode methods are valid when status is OK.
virtual util::Status status() const;
// Sets encode extra_option sequence.
virtual util::Status SetEncodeExtraOptions(absl::string_view extra_option);
// Sets decode extra_option sequence.
virtual util::Status SetDecodeExtraOptions(absl::string_view extra_option);
//////////////////////////////////////////////////////////////
// Vocabulary restriction.
// Background:
// https://github.com/rsennrich/subword-nmt#best-practice-advice-for-byte-pair-encoding-in-nmt
// Restricts the vocabulary set.
// The input sentences are encoded into the tokens in `valid_vocab`.
virtual util::Status SetVocabulary(
const std::vector<absl::string_view> &valid_vocab);
// Reverts the vocabulary restriction.
virtual util::Status ResetVocabulary();
// Loads the valid vocabulary set from `filename` in TSV format.
// Format: <token> <tab> <freq>.
// Any token with frequency < threshold will be treated as OOV.
virtual util::Status LoadVocabulary(absl::string_view filename,
int threshold);
//////////////////////////////////////////////////////////////
// Simple Encode and Decode API.
//
// Given a UTF8 input, encodes it into a sequence of sentence pieces.
virtual util::Status Encode(absl::string_view input,
std::vector<std::string> *pieces) const;
// Given a UTF8 input, encodes it into a sequence of ids.
virtual util::Status Encode(absl::string_view input,
std::vector<int> *ids) const;
// Given a sequence of pieces, decodes it into a detokenized output.
virtual util::Status Decode(const std::vector<std::string> &pieces,
std::string *detokenized) const;
// Given a sequence of pieces, decodes it into a detokenized output.
virtual util::Status Decode(const std::vector<absl::string_view> &pieces,
std::string *detokenized) const;
// Given a sequence of ids, decodes it into a detokenized output.
virtual util::Status Decode(const std::vector<int> &ids,
std::string *detokenized) const;
//////////////////////////////////////////////////////////////
// NBest API.
//
// Same as Encode, but returns nbest results.
virtual util::Status NBestEncode(
absl::string_view input, int nbest_size,
std::vector<std::vector<std::string>> *pieces) const;
// Same as Encode, but returns nbest results.
virtual util::Status NBestEncode(absl::string_view input, int nbest_size,
std::vector<std::vector<int>> *ids) const;
//////////////////////////////////////////////////////////////
// Sampling API.
//
// Unigram and BPE support sampling mode.
// - Unigram (--model_type=unigram):
// `nbest_size`: When `nbest_size` is positive value, approximately samples
// one segmentation from nbest candidates. When `nbest_size` is negative
// value, samples one segmentation from the hypotheses (Lattice) according to
// the generation probabilities using forward-filtering and backward-sampling
// algorithm.
// `alpha`: Smoothing parameter (inverse temperature). The best segmentation
// (Viterbi segmentation) is more likely sampled when setting larger alpha.
// When alpha is 0.0, one segmentation is uniformly sampled from the nbest or
// lattice. `nbest_size` and `alpha` correspond to parameters `l` and `alpha`
// in https://arxiv.org/abs/1804.10959 (nbest_size < 0 means l = infinity)
//
// - BPE (--model_type=bpe):
// `alpha`: The dropout probability `p` of bpe merge operations in
// https://arxiv.org/abs/1910.13267 Nbest-based sampling is not supported so
// nbest_size parameter is ignored in BPE.
virtual util::Status SampleEncode(absl::string_view input, int nbest_size,
float alpha,
std::vector<std::string> *pieces) const;
// Same as above, but returns a sequence of ids.
virtual util::Status SampleEncode(absl::string_view input, int nbest_size,
float alpha, std::vector<int> *ids) const;
//////////////////////////////////////////////////////////////
// SampleEncodeAndScore API.
//
// Sample `samples` many tokenisations from the segmentation lattice.
// These methods are only available in model_type=unigram.
//
// `alpha`: smoothing parameter (inverse temperature). The same as `alpha` in
// `Sample` method.
// 'wor`: If `wor` is true, the samples are taken without replacement, and the
// scores are the inclusion probabilities of the elements in the sample;
// otherwise the samples are taken with replacement and the scores are the
// log-probs of sample elements
// `include_best`: If `include_best` is true, the best tokenisation is always
// included in the sample, and the remaining elements are sampled excluding
// the best.
virtual util::Status SampleEncodeAndScore(
absl::string_view input, int num_samples, float alpha, bool wor,
bool include_best,
std::vector<std::pair<std::vector<std::string>, float>> *pieces) const;
// Same as above, but returns a sequence of ids.
virtual util::Status SampleEncodeAndScore(
absl::string_view input, int num_samples, float alpha, bool wor,
bool include_best,
std::vector<std::pair<std::vector<int>, float>> *ids) const;
//////////////////////////////////////////////////////////////
// Entropy API.
//
// This only available in model_type=unigram.
// Calculate entropy of possible tokenisations
virtual util::Status CalculateEntropy(absl::string_view input, float alpha,
float *entropy) const;
//////////////////////////////////////////////////////////////
// Advanced API returning SentencePieceText, which manages
// utf8-byte alignments between user-input/detokenized text
// and internal sentencepiece sequence.
//
// Given a UTF8 input, encodes it into SentencePieceText.
//
// When using these APIs, sentencepiece.pb.h header files must be included.
// We can also use ImutableSentencePieceText as follows.
//
// ImmutableSentencePieceText spt;
// Encode("hello", spt.mutable_proto()).IgnoreError();
// std::cout << spt.pieces_size() << std::endl;
virtual util::Status Encode(absl::string_view input,
SentencePieceText *spt) const;
virtual util::Status NBestEncode(absl::string_view input, int nbest_size,
NBestSentencePieceText *nbest_spt) const;
virtual util::Status SampleEncode(absl::string_view input, int nbest_size,
float alpha, SentencePieceText *spt) const;
virtual util::Status SampleEncodeAndScore(
absl::string_view input, int num_samples, float alpha, bool wor,
bool include_best, NBestSentencePieceText *samples_spt) const;
// DEPRECATED: Remove this API and use std::vector<std::string_view>
virtual util::Status Decode(const std::vector<std::string> &pieces,
SentencePieceText *spt) const;
virtual util::Status Decode(const std::vector<absl::string_view> &pieces,
SentencePieceText *spt) const;
virtual util::Status Decode(const std::vector<int> &ids,
SentencePieceText *spt) const;
#ifdef SWIG
#define SPP_SWIG_CHECK_AND_THROW \
if (!status.ok()) throw status;
#else
#define SPP_SWIG_CHECK_AND_THROW \
if (!status.ok()) { \
}
#endif // SWIG
#define DEFINE_SPP_DIRECT_FUNC_IMPL(FuncName, OutType, ...) \
OutType output; \
const auto status = FuncName(__VA_ARGS__, &output); \
SPP_SWIG_CHECK_AND_THROW; \
return output;
#define DEFINE_SPP_SERIALIZED_PROTO_IMPL(FuncName, OutType, ...) \
OutType output; \
const auto status = FuncName(__VA_ARGS__, output.mutable_proto()); \
SPP_SWIG_CHECK_AND_THROW; \
return output.SerializeAsString();
#define DEFINE_SPP_IMMUTABLE_PROTO_IMPL(FuncName, OutType, ...) \
OutType output; \
const auto status = FuncName(__VA_ARGS__, output.mutable_proto()); \
SPP_SWIG_CHECK_AND_THROW; \
return output;
//////////////////////////////////////////////////////////////
// Handy methods that return the result directly.
// These functions ignore internal errors.
virtual std::vector<std::string> EncodeAsPieces(
absl::string_view input) const {
DEFINE_SPP_DIRECT_FUNC_IMPL(Encode, std::vector<std::string>, input);
}
virtual std::vector<int> EncodeAsIds(absl::string_view input) const {
DEFINE_SPP_DIRECT_FUNC_IMPL(Encode, std::vector<int>, input);
}
virtual std::vector<std::vector<std::string>> NBestEncodeAsPieces(
absl::string_view input, int nbest_size) const {
DEFINE_SPP_DIRECT_FUNC_IMPL(
NBestEncode, std::vector<std::vector<std::string>>, input, nbest_size);
}
virtual std::vector<std::vector<int>> NBestEncodeAsIds(
absl::string_view input, int nbest_size) const {
DEFINE_SPP_DIRECT_FUNC_IMPL(NBestEncode, std::vector<std::vector<int>>,
input, nbest_size);
}
virtual std::vector<std::string> SampleEncodeAsPieces(absl::string_view input,
int nbest_size,
float alpha) const {
DEFINE_SPP_DIRECT_FUNC_IMPL(SampleEncode, std::vector<std::string>, input,
nbest_size, alpha);
}
virtual std::vector<int> SampleEncodeAsIds(absl::string_view input,
int nbest_size,
float alpha) const {
DEFINE_SPP_DIRECT_FUNC_IMPL(SampleEncode, std::vector<int>, input,
nbest_size, alpha);
}
virtual std::vector<std::pair<std::vector<std::string>, float>>
SampleEncodeAndScoreAsPieces(absl::string_view input, int num_samples,
float alpha, bool wor, bool include_best) const {
using _T = std::vector<std::pair<std::vector<std::string>, float>>;
DEFINE_SPP_DIRECT_FUNC_IMPL(SampleEncodeAndScore, _T, input, num_samples,
alpha, wor, include_best);
}
virtual std::vector<std::pair<std::vector<int>, float>>
SampleEncodeAndScoreAsIds(absl::string_view input, int num_samples,
float alpha, bool wor, bool include_best) const {
using _T = std::vector<std::pair<std::vector<int>, float>>;
DEFINE_SPP_DIRECT_FUNC_IMPL(SampleEncodeAndScore, _T, input, num_samples,
alpha, wor, include_best);
}
// DEPRECATED: Remove this API and use std::vector<std::string_view>
virtual std::string DecodePieces(
const std::vector<std::string> &pieces) const {
DEFINE_SPP_DIRECT_FUNC_IMPL(Decode, std::string, pieces);
}
virtual std::string DecodePieces(
const std::vector<absl::string_view> &pieces) const {
DEFINE_SPP_DIRECT_FUNC_IMPL(Decode, std::string, pieces);
}
virtual std::string DecodeIds(const std::vector<int> &ids) const {
DEFINE_SPP_DIRECT_FUNC_IMPL(Decode, std::string, ids);
}
virtual float CalculateEntropy(absl::string_view text, float alpha) const {
DEFINE_SPP_DIRECT_FUNC_IMPL(CalculateEntropy, float, text, alpha);
}
//////////////////////////////////////////////////////////////
// SerializedProto API. (DEPRECATED). Use ImmutableProto API.
// They are used in Python interface. Returns serialized proto.
// In python module, we can get access to the full Proto after
// deserialzing the returned byte sequence.
virtual util::bytes EncodeAsSerializedProto(absl::string_view input) const {
DEFINE_SPP_SERIALIZED_PROTO_IMPL(Encode, ImmutableSentencePieceText, input);
}
virtual util::bytes SampleEncodeAsSerializedProto(absl::string_view input,
int nbest_size,
float alpha) const {
DEFINE_SPP_SERIALIZED_PROTO_IMPL(SampleEncode, ImmutableSentencePieceText,
input, nbest_size, alpha);
}
virtual util::bytes NBestEncodeAsSerializedProto(absl::string_view input,
int nbest_size) const {
DEFINE_SPP_SERIALIZED_PROTO_IMPL(
NBestEncode, ImmutableNBestSentencePieceText, input, nbest_size);
}
virtual util::bytes SampleEncodeAndScoreAsSerializedProto(
absl::string_view input, int num_samples, float alpha, bool wor,
bool include_best) const {
DEFINE_SPP_SERIALIZED_PROTO_IMPL(SampleEncodeAndScore,
ImmutableNBestSentencePieceText, input,
num_samples, alpha, wor, include_best);
}
// TODO(taku): Remove this API and use std::vector<std::string_view>
virtual util::bytes DecodePiecesAsSerializedProto(
const std::vector<std::string> &pieces) const {
DEFINE_SPP_SERIALIZED_PROTO_IMPL(Decode, ImmutableSentencePieceText,
pieces);
}
virtual util::bytes DecodePiecesAsSerializedProto(
const std::vector<absl::string_view> &pieces) const {
DEFINE_SPP_SERIALIZED_PROTO_IMPL(Decode, ImmutableSentencePieceText,
pieces);
}
virtual util::bytes DecodeIdsAsSerializedProto(
const std::vector<int> &ids) const {
DEFINE_SPP_SERIALIZED_PROTO_IMPL(Decode, ImmutableSentencePieceText, ids);
}
//////////////////////////////////////////////////////////////
// ImmutableProto API.
virtual ImmutableSentencePieceText EncodeAsImmutableProto(
absl::string_view input) const {
DEFINE_SPP_IMMUTABLE_PROTO_IMPL(Encode, ImmutableSentencePieceText, input);
}
virtual ImmutableSentencePieceText SampleEncodeAsImmutableProto(
absl::string_view input, int nbest_size, float alpha) const {
DEFINE_SPP_IMMUTABLE_PROTO_IMPL(SampleEncode, ImmutableSentencePieceText,
input, nbest_size, alpha);
}
virtual ImmutableNBestSentencePieceText NBestEncodeAsImmutableProto(
absl::string_view input, int nbest_size) const {
DEFINE_SPP_IMMUTABLE_PROTO_IMPL(
NBestEncode, ImmutableNBestSentencePieceText, input, nbest_size);
}
virtual ImmutableNBestSentencePieceText SampleEncodeAndScoreAsImmutableProto(
absl::string_view input, int num_samples, float alpha, bool wor,
bool include_best) const {
DEFINE_SPP_IMMUTABLE_PROTO_IMPL(SampleEncodeAndScore,
ImmutableNBestSentencePieceText, input,
num_samples, alpha, wor, include_best);
}
// TODO(taku): Remove this API and use std::vector<std::string_view>
virtual ImmutableSentencePieceText DecodePiecesAsImmutableProto(
const std::vector<std::string> &pieces) const {
DEFINE_SPP_IMMUTABLE_PROTO_IMPL(Decode, ImmutableSentencePieceText, pieces);
}
virtual ImmutableSentencePieceText DecodePiecesAsImmutableProto(
const std::vector<absl::string_view> &pieces) const {
DEFINE_SPP_IMMUTABLE_PROTO_IMPL(Decode, ImmutableSentencePieceText, pieces);
}
virtual ImmutableSentencePieceText DecodeIdsAsImmutableProto(
const std::vector<int> &ids) const {
DEFINE_SPP_IMMUTABLE_PROTO_IMPL(Decode, ImmutableSentencePieceText, ids);
}
#undef DEFINE_SPP_DIRECT_FUNC_IMPL
#undef DEFINE_SPP_SERIALIZED_PROTO_IMPL
#undef DEFINE_SPP_IMMUTABLE_PROTO_IMPL
//////////////////////////////////////////////////////////////
// Normalization methods.
// Normalize `input`.
virtual util::Status Normalize(absl::string_view input,
std::string *normalized) const;
// Normalize `input`. Stores the utf8-byte offset from
// the normalized string to the original input.
virtual util::Status Normalize(absl::string_view input,
std::string *normalized,
std::vector<size_t> *norm_to_orig) const;
virtual std::string Normalize(absl::string_view input) const;
//////////////////////////////////////////////////////////////
// Vocabulary management methods.
//
// Returns the size of sentence pieces, which is the same as
// the size of vocabulary for NMT.
virtual int GetPieceSize() const;
// Returns the vocab id of `piece`.
// Returns UNK(0) if `piece` is unknown.
virtual int PieceToId(absl::string_view piece) const;
// Returns the string representation of vocab with `id`.
virtual const std::string &IdToPiece(int id) const;
// Returns the score of `id`.
// Usually score is an emission log probability of unigram language
// model.
virtual float GetScore(int id) const;
// Returns true if `id` is unknown symbol.
virtual bool IsUnknown(int id) const;
// Returns true if `id` is control symbol.
virtual bool IsControl(int id) const;
// Returns true if `id` is unused symbol.
virtual bool IsUnused(int id) const;
// Returns true if `id` is byte symbol.
virtual bool IsByte(int id) const;
// Returns the reserved id.
// Returns -1 if not defined.
// Returns unknown (<unk>) id.
virtual int unk_id() const;
// Returns BOS (<s>) id.
virtual int bos_id() const;
// Returns EOS (</s>) id.
virtual int eos_id() const;
// Returns PAD (<pad>) id.
virtual int pad_id() const;
//////////////////////////////////////////////////////////////
// Model management.
//
// Allows injection of a mock model instance. `model` is moved.
void SetModel(std::unique_ptr<ModelInterface> &&model);
// Allows injection of a normalizer instance. `normalizer` is moved.
void SetNormalizer(std::unique_ptr<normalizer::Normalizer> &&normalizer);
// Returns immutable model proto. Useful to obtain extended
// or experimental parameters encoded in model_proto.
const ModelProto &model_proto() const;
// returns immutable model proto as std::string.
// Useful to save the state of this instance via Python's pickle object.
util::bytes serialized_model_proto() const;
// Returns mutable normalizer_spec.
// Updating the intenral normalization during the encoding/decoding are not
// recommended and may result in unexpected behavior. Use at your own risk.
NormalizerSpec *mutable_normalizer_spec() const;
private:
enum ExtraOption { REVERSE, BOS, EOS, UNK_PIECE };
util::Status ParseExtraOptions(absl::string_view extra_option,
std::vector<ExtraOption> *extra_options) const;
util::Status ApplyExtraOptions(const std::vector<ExtraOption> &extra_options,
SentencePieceText *spt) const;
util::Status PopulateSentencePieceText(
absl::string_view input, absl::string_view normalized,
const std::vector<size_t> &norm_to_orig,
const std::vector<std::pair<absl::string_view, int>> &result,
SentencePieceText *spt) const;
std::unique_ptr<ModelInterface> model_;
std::unique_ptr<normalizer::Normalizer> normalizer_;
std::unique_ptr<normalizer::Normalizer> denormalizer_;
// Underlying model protocol buffer. The same lifetime as model_.
std::unique_ptr<ModelProto> model_proto_;
std::vector<ExtraOption> encode_extra_options_;
std::vector<ExtraOption> decode_extra_options_;
};
// Set seed value of random generator.
// Do not set static_cast<unique_int>(-1),
// as this seed is reserved for initializing from
// std::random_device.
void SetRandomGeneratorSeed(unsigned int seed);
// Set the global log level. The default loglevel is 0.
// The log is emitted only when min_log_level >= output_log_level.
void SetMinLogLevel(int v);
// IO related functions to absorb model formats.
namespace io {
// Loads `model_proto` from `filename`.
// We can instantiate SentencePieceProcessor as follows:
//
// auto model_proto = absl::make_unique<ModelProto>();
// io::LoadModelProto("//path/spm.model", model_proto.get());
// SentencePieceProcessor sp;
// CHECK_OK(sp.Load(std::move(model_proto)));
util::Status LoadModelProto(absl::string_view, ModelProto *model_proto);
// Saves `model_proto` as `filename`.
util::Status SaveModelProto(absl::string_view, const ModelProto &model_proto);
} // namespace io
} // namespace sentencepiece
#endif // SENTENCEPIECE_PROCESSOR_H_
Reference in a new issue View git blame Copy permalink