Dataset Generator local files path

🤗Datasets
2

You can pass a relative path to the dl_manager :slight_smile:

e.g.

class MyDataset(datasets.GeneratorBasedBuilder):
    ...
    def _split_generators(self, dl_manager: datasets.DownloadManager) -> List[datasets.SplitGenerator]:
        generator = []
        file_train = dl_manager.download(os.path.join(self.config.name, "train.csv"))
        file_test = dl_manager.download(os.path.join(self.config.name, "test.csv"))
        file_eval = dl_manager.download(os.path.join(self.config.name, "valid.csv"))

        train = datasets.SplitGenerator(
            name=datasets.Split.TRAIN,
            gen_kwargs={
                "filepath": file_train,
                "split": "train",
            },
        )
        generator.append(train)
1 Like