I tried both new_fingerprint and disable_cache(), but all still gave the same bug.
the complete error is as follow:
Map (num_proc=16): 0%| | 0/2116 [00:00<?, ? examples/s]
Traceback (most recent call last):
File "/ws/ifp-54_2/hasegawa/haolong2/AI4EE/CSR4RSR/evaluation.py", line 213, in <module>
main()
File "/ws/ifp-54_2/hasegawa/haolong2/AI4EE/CSR4RSR/evaluation.py", line 178, in main
ds[split] = ds[split].map(
File "/ws/ifp-53_2/hasegawa/haolong2/miniconda3/envs/csr4rsr/lib/python3.10/site-packages/datasets/arrow_dataset.py", line 557, in wrapper
out: Union["Dataset", "DatasetDict"] = func(self, *args, **kwargs)
File "/ws/ifp-53_2/hasegawa/haolong2/miniconda3/envs/csr4rsr/lib/python3.10/site-packages/datasets/arrow_dataset.py", line 3166, in map
for rank, done, content in iflatmap_unordered(
File "/ws/ifp-53_2/hasegawa/haolong2/miniconda3/envs/csr4rsr/lib/python3.10/site-packages/datasets/utils/py_utils.py", line 720, in iflatmap_unordered
[async_result.get(timeout=0.05) for async_result in async_results]
File "/ws/ifp-53_2/hasegawa/haolong2/miniconda3/envs/csr4rsr/lib/python3.10/site-packages/datasets/utils/py_utils.py", line 720, in <listcomp>
[async_result.get(timeout=0.05) for async_result in async_results]
File "/ws/ifp-53_2/hasegawa/haolong2/miniconda3/envs/csr4rsr/lib/python3.10/site-packages/multiprocess/pool.py", line 774, in get
raise self._value
File "/ws/ifp-53_2/hasegawa/haolong2/miniconda3/envs/csr4rsr/lib/python3.10/site-packages/multiprocess/pool.py", line 540, in _handle_tasks
put(task)
File "/ws/ifp-53_2/hasegawa/haolong2/miniconda3/envs/csr4rsr/lib/python3.10/site-packages/multiprocess/connection.py", line 209, in send
self._send_bytes(_ForkingPickler.dumps(obj))
File "/ws/ifp-53_2/hasegawa/haolong2/miniconda3/envs/csr4rsr/lib/python3.10/site-packages/multiprocess/reduction.py", line 54, in dumps
cls(buf, protocol, *args, **kwds).dump(obj)
File "/ws/ifp-53_2/hasegawa/haolong2/miniconda3/envs/csr4rsr/lib/python3.10/site-packages/dill/_dill.py", line 420, in dump
StockPickler.dump(self, obj)
File "/ws/ifp-53_2/hasegawa/haolong2/miniconda3/envs/csr4rsr/lib/python3.10/pickle.py", line 487, in dump
self.save(obj)
File "/ws/ifp-53_2/hasegawa/haolong2/miniconda3/envs/csr4rsr/lib/python3.10/site-packages/dill/_dill.py", line 414, in save
StockPickler.save(self, obj, save_persistent_id)
File "/ws/ifp-53_2/hasegawa/haolong2/miniconda3/envs/csr4rsr/lib/python3.10/pickle.py", line 560, in save
f(self, obj) # Call unbound method with explicit self
File "/ws/ifp-53_2/hasegawa/haolong2/miniconda3/envs/csr4rsr/lib/python3.10/pickle.py", line 902, in save_tuple
save(element)
File "/ws/ifp-53_2/hasegawa/haolong2/miniconda3/envs/csr4rsr/lib/python3.10/site-packages/dill/_dill.py", line 414, in save
StockPickler.save(self, obj, save_persistent_id)
File "/ws/ifp-53_2/hasegawa/haolong2/miniconda3/envs/csr4rsr/lib/python3.10/pickle.py", line 560, in save
f(self, obj) # Call unbound method with explicit self
File "/ws/ifp-53_2/hasegawa/haolong2/miniconda3/envs/csr4rsr/lib/python3.10/pickle.py", line 887, in save_tuple
save(element)
File "/ws/ifp-53_2/hasegawa/haolong2/miniconda3/envs/csr4rsr/lib/python3.10/site-packages/dill/_dill.py", line 414, in save
StockPickler.save(self, obj, save_persistent_id)
File "/ws/ifp-53_2/hasegawa/haolong2/miniconda3/envs/csr4rsr/lib/python3.10/pickle.py", line 560, in save
f(self, obj) # Call unbound method with explicit self
File "/ws/ifp-53_2/hasegawa/haolong2/miniconda3/envs/csr4rsr/lib/python3.10/site-packages/dill/_dill.py", line 1217, in save_module_dict
StockPickler.save_dict(pickler, obj)
File "/ws/ifp-53_2/hasegawa/haolong2/miniconda3/envs/csr4rsr/lib/python3.10/pickle.py", line 972, in save_dict
self._batch_setitems(obj.items())
File "/ws/ifp-53_2/hasegawa/haolong2/miniconda3/envs/csr4rsr/lib/python3.10/pickle.py", line 998, in _batch_setitems
save(v)
File "/ws/ifp-53_2/hasegawa/haolong2/miniconda3/envs/csr4rsr/lib/python3.10/site-packages/dill/_dill.py", line 414, in save
StockPickler.save(self, obj, save_persistent_id)
File "/ws/ifp-53_2/hasegawa/haolong2/miniconda3/envs/csr4rsr/lib/python3.10/pickle.py", line 560, in save
f(self, obj) # Call unbound method with explicit self
File "/ws/ifp-53_2/hasegawa/haolong2/miniconda3/envs/csr4rsr/lib/python3.10/pickle.py", line 902, in save_tuple
save(element)
File "/ws/ifp-53_2/hasegawa/haolong2/miniconda3/envs/csr4rsr/lib/python3.10/site-packages/dill/_dill.py", line 414, in save
StockPickler.save(self, obj, save_persistent_id)
File "/ws/ifp-53_2/hasegawa/haolong2/miniconda3/envs/csr4rsr/lib/python3.10/pickle.py", line 560, in save
f(self, obj) # Call unbound method with explicit self
File "/ws/ifp-53_2/hasegawa/haolong2/miniconda3/envs/csr4rsr/lib/python3.10/pickle.py", line 887, in save_tuple
save(element)
File "/ws/ifp-53_2/hasegawa/haolong2/miniconda3/envs/csr4rsr/lib/python3.10/site-packages/dill/_dill.py", line 414, in save
StockPickler.save(self, obj, save_persistent_id)
File "/ws/ifp-53_2/hasegawa/haolong2/miniconda3/envs/csr4rsr/lib/python3.10/pickle.py", line 560, in save
f(self, obj) # Call unbound method with explicit self
File "/ws/ifp-53_2/hasegawa/haolong2/miniconda3/envs/csr4rsr/lib/python3.10/site-packages/dill/_dill.py", line 1217, in save_module_dict
StockPickler.save_dict(pickler, obj)
File "/ws/ifp-53_2/hasegawa/haolong2/miniconda3/envs/csr4rsr/lib/python3.10/pickle.py", line 972, in save_dict
self._batch_setitems(obj.items())
File "/ws/ifp-53_2/hasegawa/haolong2/miniconda3/envs/csr4rsr/lib/python3.10/pickle.py", line 998, in _batch_setitems
save(v)
File "/ws/ifp-53_2/hasegawa/haolong2/miniconda3/envs/csr4rsr/lib/python3.10/site-packages/dill/_dill.py", line 414, in save
StockPickler.save(self, obj, save_persistent_id)
File "/ws/ifp-53_2/hasegawa/haolong2/miniconda3/envs/csr4rsr/lib/python3.10/pickle.py", line 560, in save
f(self, obj) # Call unbound method with explicit self
File "/ws/ifp-53_2/hasegawa/haolong2/miniconda3/envs/csr4rsr/lib/python3.10/site-packages/dill/_dill.py", line 1985, in save_function
_save_with_postproc(pickler, (_create_function, (
File "/ws/ifp-53_2/hasegawa/haolong2/miniconda3/envs/csr4rsr/lib/python3.10/site-packages/dill/_dill.py", line 1117, in _save_with_postproc
pickler.save_reduce(*reduction)
File "/ws/ifp-53_2/hasegawa/haolong2/miniconda3/envs/csr4rsr/lib/python3.10/pickle.py", line 692, in save_reduce
save(args)
File "/ws/ifp-53_2/hasegawa/haolong2/miniconda3/envs/csr4rsr/lib/python3.10/site-packages/dill/_dill.py", line 414, in save
StockPickler.save(self, obj, save_persistent_id)
File "/ws/ifp-53_2/hasegawa/haolong2/miniconda3/envs/csr4rsr/lib/python3.10/pickle.py", line 560, in save
f(self, obj) # Call unbound method with explicit self
File "/ws/ifp-53_2/hasegawa/haolong2/miniconda3/envs/csr4rsr/lib/python3.10/pickle.py", line 887, in save_tuple
save(element)
File "/ws/ifp-53_2/hasegawa/haolong2/miniconda3/envs/csr4rsr/lib/python3.10/site-packages/dill/_dill.py", line 414, in save
StockPickler.save(self, obj, save_persistent_id)
File "/ws/ifp-53_2/hasegawa/haolong2/miniconda3/envs/csr4rsr/lib/python3.10/pickle.py", line 560, in save
f(self, obj) # Call unbound method with explicit self
File "/ws/ifp-53_2/hasegawa/haolong2/miniconda3/envs/csr4rsr/lib/python3.10/pickle.py", line 932, in save_list
self._batch_appends(obj)
File "/ws/ifp-53_2/hasegawa/haolong2/miniconda3/envs/csr4rsr/lib/python3.10/pickle.py", line 956, in _batch_appends
save(x)
File "/ws/ifp-53_2/hasegawa/haolong2/miniconda3/envs/csr4rsr/lib/python3.10/site-packages/dill/_dill.py", line 414, in save
StockPickler.save(self, obj, save_persistent_id)
File "/ws/ifp-53_2/hasegawa/haolong2/miniconda3/envs/csr4rsr/lib/python3.10/pickle.py", line 603, in save
self.save_reduce(obj=obj, *rv)
File "/ws/ifp-53_2/hasegawa/haolong2/miniconda3/envs/csr4rsr/lib/python3.10/pickle.py", line 717, in save_reduce
save(state)
File "/ws/ifp-53_2/hasegawa/haolong2/miniconda3/envs/csr4rsr/lib/python3.10/site-packages/dill/_dill.py", line 414, in save
StockPickler.save(self, obj, save_persistent_id)
File "/ws/ifp-53_2/hasegawa/haolong2/miniconda3/envs/csr4rsr/lib/python3.10/pickle.py", line 560, in save
f(self, obj) # Call unbound method with explicit self
File "/ws/ifp-53_2/hasegawa/haolong2/miniconda3/envs/csr4rsr/lib/python3.10/site-packages/dill/_dill.py", line 1217, in save_module_dict
StockPickler.save_dict(pickler, obj)
File "/ws/ifp-53_2/hasegawa/haolong2/miniconda3/envs/csr4rsr/lib/python3.10/pickle.py", line 972, in save_dict
self._batch_setitems(obj.items())
File "/ws/ifp-53_2/hasegawa/haolong2/miniconda3/envs/csr4rsr/lib/python3.10/pickle.py", line 998, in _batch_setitems
save(v)
File "/ws/ifp-53_2/hasegawa/haolong2/miniconda3/envs/csr4rsr/lib/python3.10/site-packages/dill/_dill.py", line 414, in save
StockPickler.save(self, obj, save_persistent_id)
File "/ws/ifp-53_2/hasegawa/haolong2/miniconda3/envs/csr4rsr/lib/python3.10/pickle.py", line 560, in save
f(self, obj) # Call unbound method with explicit self
File "/ws/ifp-53_2/hasegawa/haolong2/miniconda3/envs/csr4rsr/lib/python3.10/pickle.py", line 932, in save_list
self._batch_appends(obj)
File "/ws/ifp-53_2/hasegawa/haolong2/miniconda3/envs/csr4rsr/lib/python3.10/pickle.py", line 959, in _batch_appends
save(tmp[0])
File "/ws/ifp-53_2/hasegawa/haolong2/miniconda3/envs/csr4rsr/lib/python3.10/site-packages/dill/_dill.py", line 414, in save
StockPickler.save(self, obj, save_persistent_id)
File "/ws/ifp-53_2/hasegawa/haolong2/miniconda3/envs/csr4rsr/lib/python3.10/pickle.py", line 603, in save
self.save_reduce(obj=obj, *rv)
File "/ws/ifp-53_2/hasegawa/haolong2/miniconda3/envs/csr4rsr/lib/python3.10/pickle.py", line 717, in save_reduce
save(state)
File "/ws/ifp-53_2/hasegawa/haolong2/miniconda3/envs/csr4rsr/lib/python3.10/site-packages/dill/_dill.py", line 414, in save
StockPickler.save(self, obj, save_persistent_id)
File "/ws/ifp-53_2/hasegawa/haolong2/miniconda3/envs/csr4rsr/lib/python3.10/pickle.py", line 560, in save
f(self, obj) # Call unbound method with explicit self
File "/ws/ifp-53_2/hasegawa/haolong2/miniconda3/envs/csr4rsr/lib/python3.10/site-packages/dill/_dill.py", line 1217, in save_module_dict
StockPickler.save_dict(pickler, obj)
File "/ws/ifp-53_2/hasegawa/haolong2/miniconda3/envs/csr4rsr/lib/python3.10/pickle.py", line 972, in save_dict
self._batch_setitems(obj.items())
File "/ws/ifp-53_2/hasegawa/haolong2/miniconda3/envs/csr4rsr/lib/python3.10/pickle.py", line 998, in _batch_setitems
save(v)
File "/ws/ifp-53_2/hasegawa/haolong2/miniconda3/envs/csr4rsr/lib/python3.10/site-packages/dill/_dill.py", line 414, in save
StockPickler.save(self, obj, save_persistent_id)
File "/ws/ifp-53_2/hasegawa/haolong2/miniconda3/envs/csr4rsr/lib/python3.10/pickle.py", line 578, in save
rv = reduce(self.proto)
TypeError: cannot pickle 'ThreadLocalFileContext' object