Hi,
I'm trying to load a pre-trained sentence embedding model but when I enc…ode 1000 sentences it works fine but when I try with all my sentences I'm getting an error. Can someone explain to me what is the problem please ?
`model = SentenceTransformer("dangvantuan/sentence-camembert-large")
sentences = dataset_cleaned['all_text'] # list of String
sentence_embeddings = model.encode(sentences)
cos_sim = util.cos_sim(sentence_embeddings, sentence_embeddings)`
**Error** :
IndexError Traceback (most recent call last)
/var/folders/vk/qdyvc5xx4nv6nmlzhfvc1p4sf_xqvd/T/ipykernel_6448/1081646493.py in <module>
1 sentences = dataset_cleaned['all_text']
2
----> 3 sentence_embeddings = model.encode(sentences)
~/Desktop/projects/venv/lib/python3.7/site-packages/sentence_transformers/SentenceTransformer.py in encode(self, sentences, batch_size, show_progress_bar, output_value, convert_to_numpy, convert_to_tensor, device, normalize_embeddings)
162
163 with torch.no_grad():
--> 164 out_features = self.forward(features)
165
166 if output_value == 'token_embeddings':
~/Desktop/projects/venv/lib/python3.7/site-packages/torch/nn/modules/container.py in forward(self, input)
139 def forward(self, input):
140 for module in self:
--> 141 input = module(input)
142 return input
143
~/Desktop/projects/venv/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
1108 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
1109 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1110 return forward_call(*input, **kwargs)
1111 # Do not call functions when jit is used
1112 full_backward_hooks, non_full_backward_hooks = [], []
~/Desktop/projects/venv/lib/python3.7/site-packages/sentence_transformers/models/Transformer.py in forward(self, features)
64 trans_features['token_type_ids'] = features['token_type_ids']
65
---> 66 output_states = self.auto_model(**trans_features, return_dict=False)
67 output_tokens = output_states[0]
68
~/Desktop/projects/venv/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
1108 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
1109 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1110 return forward_call(*input, **kwargs)
1111 # Do not call functions when jit is used
1112 full_backward_hooks, non_full_backward_hooks = [], []
~/Desktop/projects/venv/lib/python3.7/site-packages/transformers/models/roberta/modeling_roberta.py in forward(self, input_ids, attention_mask, token_type_ids, position_ids, head_mask, inputs_embeds, encoder_hidden_states, encoder_attention_mask, past_key_values, use_cache, output_attentions, output_hidden_states, return_dict)
844 token_type_ids=token_type_ids,
845 inputs_embeds=inputs_embeds,
--> 846 past_key_values_length=past_key_values_length,
847 )
848 encoder_outputs = self.encoder(
~/Desktop/projects/venv/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
1108 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
1109 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1110 return forward_call(*input, **kwargs)
1111 # Do not call functions when jit is used
1112 full_backward_hooks, non_full_backward_hooks = [], []
~/Desktop/projects/venv/lib/python3.7/site-packages/transformers/models/roberta/modeling_roberta.py in forward(self, input_ids, token_type_ids, position_ids, inputs_embeds, past_key_values_length)
131 embeddings = inputs_embeds + token_type_embeddings
132 if self.position_embedding_type == "absolute":
--> 133 position_embeddings = self.position_embeddings(position_ids)
134 embeddings += position_embeddings
135 embeddings = self.LayerNorm(embeddings)
~/Desktop/projects/venv/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
1108 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
1109 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1110 return forward_call(*input, **kwargs)
1111 # Do not call functions when jit is used
1112 full_backward_hooks, non_full_backward_hooks = [], []
~/Desktop/projects/venv/lib/python3.7/site-packages/torch/nn/modules/sparse.py in forward(self, input)
158 return F.embedding(
159 input, self.weight, self.padding_idx, self.max_norm,
--> 160 self.norm_type, self.scale_grad_by_freq, self.sparse)
161
162 def extra_repr(self) -> str:
~/Desktop/projects/venv/lib/python3.7/site-packages/torch/nn/functional.py in embedding(input, weight, padding_idx, max_norm, norm_type, scale_grad_by_freq, sparse)
2181 # remove once script supports set_grad_enabled
2182 _no_grad_embedding_renorm_(weight, input, max_norm, norm_type)
-> 2183 return torch.embedding(weight, input, padding_idx, scale_grad_by_freq, sparse)
2184
2185
IndexError: index out of range in self