NER for multilingual in Tensorflow

Beginners
1

Hi, l am trying to create NER for multilingual in Tensorflow and created the head for model but keep getting below message, please help where l am doing wrong

type# XLM-R for token classification
class XLMRobertaForTokenClassification(TFRobertaPreTrainedModel):
    config_class = XLMRobertaConfig

    def __init__(self, config):
        super().__init__(config)
        self.num_labels = config.num_labels
        #load model body
        self.roberta = TFRobertaModel(config)
        #self.roberta = TFRobertaMainLayer(config, name="roberta")  # Use TFRobertaMainLayer
        #setup token classification head
        self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob)
        self.classifier = tf.keras.layers.Dense(config.num_labels, activation=None)

        #self.init_weights() no need for tensorflow

    def call(self, input_ids=None, attention_mask=None, token_type_ids=None,
             labels=None, **kwargs):
        #kwargs.pop('training', None)



        # Use model body to get encoder representations
        outputs = self.roberta(input_ids, attention_mask=attention_mask,
                               token_type_ids=token_type_ids)


        # Apply classifier to encoder representation
        sequence_output = self.dropout(outputs[0])
        logits = self.classifier(sequence_output)

        # Calculate losses
        loss = None
        if labels is not None:
            loss_fct = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
            loss = loss_fct(labels, logits)

        # Return model output object
        return TFTokenClassifierOutput(loss=loss, logits=logits,
                                     hidden_states=outputs.hidden_states,
                                     attentions= outputs.attentions)


from transformers import AutoConfig

xlmr_config = AutoConfig.from_pretrained(xlmr_model_name,
                                         num_labels=tags.num_classes,
                                         id2label=index2tag, label2id=tag2index)`Preformatted text`

xlmr_model = (XLMRobertaForTokenClassification.from_pretrained(xlmr_model_name, config=xlmr_config))

ValueError: Weights for model ‘tf_roberta_model_3’ have not yet been created. Weights are created when the model is first called on inputs or build() is called with an input_shape.

1 Like
2

self.classifier = tf.keras.layers.Dense(config.num_labels, activation=None)

input_shape is missing?

3

Thanks @John6666 l tried to add input_shape still has same issue, any advice please?

 
        self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob)
        
        self.classifier = tf.keras.layers.Dense(config.num_labels, activation='softmax', name='classifier')
        self.classifier.build(input_shape=(None, config.hidden_size))

ValueError: Weights for model ‘tf_roberta_model_5’ have not yet been created. Weights are created when the model is first called on inputs or build() is called with an input_shape.

1 Like
4

It seems that just getting the import order wrong will cause an error…