Add generator to Keras Text Generation

Beginners
1

Hi I am using Keras Text Generation, I want to train it on larger dataset to get accurate result, currently while doing hot encoding it fills the data into memory, leads to high memory utilization, I want to pass the data in the training in batches, I read that it can be done by generator, I tried to create generator function but did not work, can any one help me in this regard, Here is my code:

tokenizer = Tokenizer()
data = open('./data/story_data.txt').read()]
print(len(data))
corpus = data.lower().split("\n")
print(len(corpus))
corpus = [c for count,c in enumerate(corpus) if count<1000]
tokenizer.fit_on_texts(corpus)
total_words = len(tokenizer.word_index) + 1
print(tokenizer.word_index)
print(total_words)
input_sequences = []

for line in corpus:
    token_list = tokenizer.texts_to_sequences([line])[0]
    for i in range(1, len(token_list)):
        n_gram_sequence = token_list[:i+1]
        input_sequences.append(n_gram_sequence)

# pad sequences
max_sequence_len = max([len(x) for x in input_sequences])
print(f"Max sequence length is: {max_sequence_len}")
input_sequences = np.array(pad_sequences(input_sequences, maxlen=max_sequence_len, padding='pre'))

# Existing Predictor and Label methods
# xs, labels = input_sequences[:,:-1],input_sequences[:,-1]
# ys = tf.keras.utils.to_categorical(labels, num_classes=total_words)

## My Generator Function
def generator(batch_size):
    index = batch_size
    print(f'Index: {index}')
    while(True):
        xs, labels = input_sequences[index:, index:-1], input_sequences[index:, -1]
        ys = tf.keras.utils.to_categorical(labels, num_classes=total_words)
        yield xs, ys

#Model build
model = Sequential()
model.add(Embedding(total_words, 100, input_length=max_sequence_len-1))
model.add(Bidirectional(LSTM(150)))
model.add(Dense(total_words, activation='softmax'))
adam = Adam(lr=0.01)
model.compile(loss='categorical_crossentropy', optimizer=adam, metrics=['accuracy'])
#earlystop = EarlyStopping(monitor='val_loss', min_delta=0, patience=5, verbose=0, mode='auto')
# history = model.fit(xs, ys, epochs=1, verbose=1, batch_size=256)
history = model.fit_generator(generator(100), epochs=1, verbose=1)
#print model.summary()
print(model)
model.save('./model/text-gen-model.h5')