#pragma once #include "engine/models/muscriptor/types.h" #include #include #include #include #include #include #include namespace engine::models::muscriptor { struct MuScriptorTokenEvent { std::string type; int value = 0; }; class MuScriptorTokenizer { public: MuScriptorTokenizer(); int32_t eos_id() const noexcept; const std::vector & vocab() const noexcept; std::vector condition_instrument_ids(const std::string & csv) const; std::vector forbidden_token_ids(const std::string & csv) const; std::vector tie_section_token_ids(const std::set> & open_notes) const; std::set> open_note_keys(const std::vector & chunks) const; std::string instrument_for_program(int program) const; std::vector decode_chunks(const std::vector & chunks) const; private: int32_t token_id(const std::string & type, int value) const; std::vector vocab_; std::unordered_map group_name_to_id_; std::unordered_map> group_programs_; std::unordered_map program_to_name_; }; std::string muscriptor_events_to_json(const std::vector & events); std::string muscriptor_event_to_json(const MuScriptorEvent & event); std::vector muscriptor_events_to_midi_bytes(const std::vector & events); } // namespace engine::models::muscriptor