Using datasets to open jsonl

🤗Datasets
3

Another option, albeit a bit rough, is this:

from datasets import load_dataset

def process(example):
    example["term"] = str({k: v for k, v in example["term"].items() if v is not None})
    return example

path = "./testdata.jsonl"
dataset = load_dataset('json', data_files=path, split='train')

print(dataset[1]) # {'src': 'hi', 'term': {'a': None, 'b': 'bb'}}

dataset = dataset.map(process)

print(dataset[1]) # {'src': 'hi', 'term': "{'b': 'bb'}"}