Well, I still cannot make this work, by debugging, I find that the main_export() will take me to optimum.exporters.utils._get_submodels_and_export_configs(), and an error raises here
# When specifying custom export configs for supported transformers architectures, we do
# not force to specify a custom export config for each submodel.
for key, custom_export_config in custom_export_configs.items():
models_and_export_configs[key] = (models_and_export_configs[key][0], custom_export_config)
where the custom_export_configs is the one we passed in with use_past injected, while the models_and_export_configs, generated here
# TODO: this succession of if/else strongly suggests a refactor is needed.
if (
task.startswith(TasksManager._ENCODER_DECODER_TASKS)
and model.config.is_encoder_decoder
and not monolith
):
models_and_export_configs = get_encoder_decoder_models_for_export(model, export_config)
doesn’t contain the key “decoder_with_past”, where the default export_config generated here
export_config_constructor = TasksManager.get_exporter_config_constructor(
model=model, exporter=exporter, task=task, library_name=library_name
)
export_config = export_config_constructor(
model.config,
int_dtype=int_dtype,
float_dtype=float_dtype,
preprocessors=preprocessors,
)
with a default use_past=False, therefore would not generate a config for “decoder_with_past”.
And actually here is what I monkey_patched during the debugging.
I think there is a high dependency between the export config and model config in optimum library, where I although use a customized encoder but still the VisionEncoderDecoder Config as the outermost config, which leads me to the not custom_architecture config processing logic here, which leads to the above error, which may not considered as a normal scenario in design.
if not custom_architecture:
if library_name == "diffusers":
export_config = None
models_and_export_configs = get_diffusion_models_for_export(
model, int_dtype=int_dtype, float_dtype=float_dtype, exporter=exporter
)
else:
export_config_constructor = TasksManager.get_exporter_config_constructor(
model=model, exporter=exporter, task=task, library_name=library_name
)
export_config = export_config_constructor(
model.config,
int_dtype=int_dtype,
float_dtype=float_dtype,
preprocessors=preprocessors,
)
export_config.variant = _variant
all_variants = "\n".join(
[f" - {name}: {description}" for name, description in export_config.VARIANTS.items()]
)
logger.info(f"Using the export variant {export_config.variant}. Available variants are:\n{all_variants}")
# TODO: this succession of if/else strongly suggests a refactor is needed.
if (
task.startswith(TasksManager._ENCODER_DECODER_TASKS)
and model.config.is_encoder_decoder
and not monolith
):
models_and_export_configs = get_encoder_decoder_models_for_export(model, export_config)
elif task.startswith("text-generation") and not monolith:
models_and_export_configs = get_decoder_models_for_export(model, export_config)
elif model.config.model_type == "sam":
models_and_export_configs = get_sam_models_for_export(model, export_config)
elif model.config.model_type == "speecht5":
models_and_export_configs = get_speecht5_models_for_export(model, export_config, model_kwargs)
elif model.config.model_type == "musicgen":
models_and_export_configs = get_musicgen_models_for_export(model, export_config)
else:
models_and_export_configs = {"model": (model, export_config)}
# When specifying custom export configs for supported transformers architectures, we do
# not force to specify a custom export config for each submodel.
for key, custom_export_config in custom_export_configs.items():
models_and_export_configs[key] = (models_and_export_configs[key][0], custom_export_config)