Interfacing with Roberta Used In NLP Multitasking

I clearly don’t have a very clear understanding of how to connect inputs and outputs to Roberta. Any suggestions will be much valued.
The following is a summary of the code followed by error messages:

Cell X:

from transformers import BertConfig, TFBertModel
def build_model():
ids = tf.keras.layers.Input((MAX_LEN,),name=“ideez”, dtype=tf.int32)
att = tf.keras.layers.Input((MAX_LEN,),name = “atts”, dtype=tf.int32)
tok = tf.keras.layers.Input((MAX_LEN,), name = “toks”,dtype=tf.int32)
config = BertConfig()
bert_model = TFRobertaModel.from_pretrained(‘bert-base-uncased’,config=config)
x = bert_model(ids,attention_mask=att,token_type_ids=tok)
x1 = tf.keras.layers.Dropout(0.1)(x[0])
x1 = tf.keras.layers.Conv1D(128, 2,padding=‘same’)(x1)
x1 = tf.keras.layers.LeakyReLU()(x1)
x1 = tf.keras.layers.Conv1D(64, 2,padding=‘same’)(x1)
x1 = tf.keras.layers.Dense(1)(x1)
x1 = tf.keras.layers.Flatten()(x1)
x1 = tf.keras.layers.Activation(‘softmax’)(x1)
x2 = tf.keras.layers.Dropout(0.1)(x[0])
x2 = tf.keras.layers.Conv1D(128, 2, padding=‘same’)(x2)
x2 = tf.keras.layers.LeakyReLU()(x2)
x2 = tf.keras.layers.Conv1D(64, 2, padding=‘same’)(x2)
x2 = tf.keras.layers.Dense(1)(x2)
x2 = tf.keras.layers.Flatten()(x2)
x2 = tf.keras.layers.Activation(‘softmax’)(x2)
model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1,x2])
optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5)
model.compile(loss=‘binary_crossentropy’, optimizer=optimizer)
print(model.summary())
return model

dataset_dict = {
“ids”: input_ids,
“token_type_ids”: token_type_ids,
“attention_mask”: attention_mask ,
“x1”: start_tokens,
“x2”: end_tokens }

X = [
dataset_dict[“ids”],
dataset_dict[“token_type_ids”],
dataset_dict[“attention_mask”],
]
Y = [dataset_dict[“x1”], dataset_dict[“x2”]]
model = build_model()
history = model.fit(X,Y,batch_size = 32,epochs=4,verbose=True)

the following are the error messages that I receive

InvalidArgumentError Traceback (most recent call last)

in
5 # batch_size = 32)
6
----> 7 history = model.fit(X,Y,batch_size = 32,epochs=4,verbose=True)
8
9 ‘’'history = model.fit([input_ids,attention_mask,token_type_ids],

~/als_code/tutorials/kaggle_sentiment/venv/lib/python3.6/site-packages/keras/engine/training.py in fit(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)
1182 _r=1):
1183 callbacks.on_train_batch_begin(step)
→ 1184 tmp_logs = self.train_function(iterator)
1185 if data_handler.should_sync:
1186 context.async_wait()

~/als_code/tutorials/kaggle_sentiment/venv/lib/python3.6/site-packages/tensorflow/python/eager/def_function.py in call(self, *args, **kwds)
883
884 with OptionalXlaContext(self._jit_compile):
→ 885 result = self._call(*args, **kwds)
886
887 new_tracing_count = self.experimental_get_tracing_count()

~/als_code/tutorials/kaggle_sentiment/venv/lib/python3.6/site-packages/tensorflow/python/eager/def_function.py in _call(self, *args, **kwds)
915 # In this case we have created variables on the first call, so we run the
916 # defunned version which is guaranteed to never create variables.
→ 917 return self._stateless_fn(*args, **kwds) # pylint: disable=not-callable
918 elif self._stateful_fn is not None:
919 # Release the lock early so that multiple threads can perform the call

~/als_code/tutorials/kaggle_sentiment/venv/lib/python3.6/site-packages/tensorflow/python/eager/function.py in call(self, *args, **kwargs)
3038 filtered_flat_args) = self._maybe_define_function(args, kwargs)
3039 return graph_function._call_flat(
→ 3040 filtered_flat_args, captured_inputs=graph_function.captured_inputs) # pylint: disable=protected-access
3041
3042 @property

~/als_code/tutorials/kaggle_sentiment/venv/lib/python3.6/site-packages/tensorflow/python/eager/function.py in _call_flat(self, args, captured_inputs, cancellation_manager)
1962 # No tape is watching; skip to running the function.
1963 return self._build_call_outputs(self._inference_function.call(
→ 1964 ctx, args, cancellation_manager=cancellation_manager))
1965 forward_backward = self._select_forward_and_backward_functions(
1966 args,

~/als_code/tutorials/kaggle_sentiment/venv/lib/python3.6/site-packages/tensorflow/python/eager/function.py in call(self, ctx, args, cancellation_manager)
594 inputs=args,
595 attrs=attrs,
→ 596 ctx=ctx)
597 else:
598 outputs = execute.execute_with_cancellation(

~/als_code/tutorials/kaggle_sentiment/venv/lib/python3.6/site-packages/tensorflow/python/eager/execute.py in quick_execute(op_name, num_outputs, inputs, attrs, ctx, name)
58 ctx.ensure_initialized()
59 tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,
—> 60 inputs, attrs, num_outputs)
61 except core._NotOkStatusException as e:
62 if name is not None:

InvalidArgumentError: indices[27,4] = 31095 is not in [0, 30522)
[[node model_3/tf_roberta_model_3/roberta/embeddings/Gather (defined at /Users/algh/als_code/tutorials/kaggle_sentiment/venv/lib/python3.6/site-packages/transformers/modeling_tf_roberta.py:198) ]] [Op:__inference_train_function_72773]

Errors may have originated from an input operation.
Input Source operations connected to node model_3/tf_roberta_model_3/roberta/embeddings/Gather:
IteratorGetNext (defined at :7)

Function call stack:
train_function