#pragma once #include "engine/framework/core/backend_weight_store.h" #include "engine/framework/core/execution_context.h" #include "engine/framework/modules/conv_modules.h" #include "engine/framework/modules/linear_module.h" #include "engine/framework/modules/norm_modules.h" #include #include #include namespace engine::models::latentsync { struct LatentSyncSdVaeConfig { std::array channels{128, 256, 512, 512}; int64_t latent_channels = 4; int64_t layers_per_block = 2; int64_t norm_groups = 32; bool cuda_cudnn = false; bool cuda_f16_activations = false; }; struct SdVaeResnetWeights { modules::NormWeights norm1, norm2; modules::Conv2dWeights conv1, conv2; std::optional shortcut; }; struct SdVaeStageWeights { std::vector resnets; std::optional resample; }; struct SdVaeTowerWeights { modules::Conv2dWeights input, output, quant; modules::NormWeights output_norm, attention_norm; struct AttentionWeights { modules::LinearWeights query, key, value, out; } attention; std::array mid_resnets; std::vector stages; }; SdVaeTowerWeights load_sd_vae_weights(core::BackendWeightStore & store, const assets::TensorSource & source, const LatentSyncSdVaeConfig & config, bool encoder, assets::TensorStorageType type); class LatentSyncSdVae { public: explicit LatentSyncSdVae(LatentSyncSdVaeConfig config); core::TensorValue build_encoder(core::ModuleBuildContext & ctx, const core::TensorValue & image, const SdVaeTowerWeights & weights) const; core::TensorValue build_decoder(core::ModuleBuildContext & ctx, const core::TensorValue & latent, const SdVaeTowerWeights & weights) const; private: core::TensorValue build_resnet(core::ModuleBuildContext & ctx, const core::TensorValue & input, const SdVaeResnetWeights & weights) const; core::TensorValue build_mid(core::ModuleBuildContext & ctx, const core::TensorValue & input, const SdVaeTowerWeights & weights) const; LatentSyncSdVaeConfig config_; }; class LatentSyncSdVaeRuntime { public: LatentSyncSdVaeRuntime(core::ExecutionContext & execution, const assets::TensorSource & source, assets::TensorStorageType weight_type, int64_t resolution = 512, bool cuda_cudnn = false, bool cuda_f16_activations = false, bool share_graph_workspace = false); ~LatentSyncSdVaeRuntime(); std::vector encode_moments(const std::vector & image); std::vector encode(const std::vector & image, const std::vector & posterior_noise); std::vector decode(const std::vector & scaled_latent); private: struct Impl; std::unique_ptr impl_; }; } // namespace engine::models::latentsync