i’ve constantly been getting this error :
OSError: Can’t load tokenizer for ‘facebook/wav2vec2-large-xlsr-53’. If you were trying to load it from ‘Models - Hugging Face’, make sure you don’t have a local directory with the same name. Otherwise, make sure ‘facebook/wav2vec2-large-xlsr-53’ is the correct path to a directory containing all relevant files for a Wav2Vec2Tokenizer tokenizer.
I’ve made sure that i dont have a local dir with the same name as the tokenizer and have reinstalled the transformer.
here’s the code :
import torch
from transformers import Wav2Vec2ForCTC, Wav2Vec2Tokenizer
from datasets import load_dataset
from transformers import TrainingArguments, Trainer, default_data_collator
model = Wav2Vec2ForCTC.from_pretrained(“facebook/wav2vec2-large-xlsr-53”)
tokenizer = Wav2Vec2Tokenizer.from_pretrained(“facebook/wav2vec2-large-xlsr-53”)
dataset = load_dataset(“common_voice”, “hi”)
def preprocess(example):
speech, _ = torchaudio.load(example[“path”])
example[“speech”] = speech[0].numpy()
return example
dataset = dataset.map(preprocess)
def tokenize(example):
inputs = tokenizer(example[“speech”], return_tensors=“pt”, padding=“longest”).input_values
example[“input_values”] = inputs.squeeze()
return example
dataset = dataset.map(tokenize, batched=True)
train_dataset = dataset[“train”]
test_dataset = dataset[“test”]
training_args = TrainingArguments(
output_dir=“./mymodel”,
num_train_epochs=10,
per_device_train_batch_size=8,
per_device_eval_batch_size=8,
evaluation_strategy=“epoch”,
logging_steps=100,
save_steps=500,
learning_rate=1e-4,
)
trainer = Trainer(
model=model,
args=training_args,
train_dataset=train_dataset,
eval_dataset=test_dataset,
)
trainer.train()
trainer.save_model(“./mymodel”)