Hello, I am trying to set up an EncoderDecoderModel using PEGASUS encoder and ProphetNet decoder.
First, I initialize a PegasusModel and access its encoder:
from transformers import PegasusModel, PegasusConfig, EncoderDecoderModel
pegasus = PegasusModel(PegasusConfig()).encoder
Then I try to pass that to the EncoderDecoderModel together with decoder from ProphetNet:
pegasus2prophet = EncoderDecoderModel.from_encoder_decoder_pretrained(pegasus, "microsoft/prophetnet-large-uncased")
However running that piece of code results in following errors:
HTTPError Traceback (most recent call last)
~/stanford/xcs224u-project/project_venv/lib/python3.8/site-packages/transformers/configuration_utils.py in get_config_dict(cls, pretrained_model_name_or_path, **kwargs)
416 # Load from URL or cache if already cached
--> 417 resolved_config_file = cached_path(
418 config_file,
~/stanford/xcs224u-project/project_venv/lib/python3.8/site-packages/transformers/file_utils.py in cached_path(url_or_filename, cache_dir, force_download, proxies, resume_download, user_agent, extract_compressed_file, force_extract, use_auth_token, local_files_only)
1077 # URL, so get it from the cache (downloading if necessary)
-> 1078 output_path = get_from_cache(
1079 url_or_filename,
~/stanford/xcs224u-project/project_venv/lib/python3.8/site-packages/transformers/file_utils.py in get_from_cache(url, cache_dir, force_download, proxies, etag_timeout, resume_download, user_agent, use_auth_token, local_files_only)
1215 r = requests.head(url, headers=headers, allow_redirects=False, proxies=proxies, timeout=etag_timeout)
-> 1216 r.raise_for_status()
1217 etag = r.headers.get("X-Linked-Etag") or r.headers.get("ETag")
~/stanford/xcs224u-project/project_venv/lib/python3.8/site-packages/requests/models.py in raise_for_status(self)
942 if http_error_msg:
--> 943 raise HTTPError(http_error_msg, response=self)
944
HTTPError: 414 Client Error: Request-URI Too Large for url: https://huggingface.co/PegasusEncoder(%0A%20%20(embed_tokens):%20Embedding(50265,%201024,%20padding_idx=0)%0A%20%20(embed_positions):%20PegasusSinusoidalPositionalEmbed
(several rows of the URI follow)
During handling of the above exception, another exception occurred:
OSError Traceback (most recent call last)
<ipython-input-3-2cdd15f1864e> in <module>
----> 1 pegasus2prophet = EncoderDecoderModel.from_encoder_decoder_pretrained(pegasus, "microsoft/prophetnet-large-uncased")
~/stanford/xcs224u-project/project_venv/lib/python3.8/site-packages/transformers/models/encoder_decoder/modeling_encoder_decoder.py in from_encoder_decoder_pretrained(cls, encoder_pretrained_model_name_or_path, decoder_pretrained_model_name_or_path, *model_args, **kwargs)
304 from ..auto.configuration_auto import AutoConfig
305
--> 306 encoder_config = AutoConfig.from_pretrained(encoder_pretrained_model_name_or_path)
307 if encoder_config.is_decoder is True or encoder_config.add_cross_attention is True:
308
~/stanford/xcs224u-project/project_venv/lib/python3.8/site-packages/transformers/models/auto/configuration_auto.py in from_pretrained(cls, pretrained_model_name_or_path, **kwargs)
366 {'foo': False}
367 """
--> 368 config_dict, _ = PretrainedConfig.get_config_dict(pretrained_model_name_or_path, **kwargs)
369
370 if "model_type" in config_dict:
~/stanford/xcs224u-project/project_venv/lib/python3.8/site-packages/transformers/configuration_utils.py in get_config_dict(cls, pretrained_model_name_or_path, **kwargs)
434 f"- or '{pretrained_model_name_or_path}' is the correct path to a directory containing a {CONFIG_NAME} file\n\n"
435 )
--> 436 raise EnvironmentError(msg)
437
438 except json.JSONDecodeError:
OSError: Can't load config for 'PegasusEncoder(
(embed_tokens): Embedding(50265, 1024, padding_idx=0)
(embed_positions): PegasusSinusoidalPositionalEmbedding(1024, 1024)
(layers): ModuleList(
(0): PegasusEncoderLayer(
(self_attn): PegasusAttention(
...
Would anyone know if there is any way to bypass this problem?