Getting ERROR serialization.py:406 -- Failed to unpickle serialized exception when trying doing hyperparameter search for finetuning a SetFit model. Unfortunately it’s swallowing the error and I’m unsure if I’m doing something wrong. I’m using transformers==4.42 and ray[tune]~=2.9.
The whole error:
File "/home/lip21jaf/.conda/envs/huggingface_env/lib/python3.8/site-packages/ray/air/execution/_internal/event_manager.py", line 110, in resolve_future
result = ray.get(future)
File "/home/lip21jaf/.conda/envs/huggingface_env/lib/python3.8/site-packages/ray/_private/auto_init_hook.py", line 21, in auto_init_wrapper
return fn(*args, **kwargs)
File "/home/lip21jaf/.conda/envs/huggingface_env/lib/python3.8/site-packages/ray/_private/client_mode_hook.py", line 103, in wrapper
return func(*args, **kwargs)
File "/home/lip21jaf/.conda/envs/huggingface_env/lib/python3.8/site-packages/ray/_private/worker.py", line 2667, in get
values, debugger_breakpoint = worker.get_objects(object_refs, timeout=timeout)
File "/home/lip21jaf/.conda/envs/huggingface_env/lib/python3.8/site-packages/ray/_private/worker.py", line 866, in get_objects
raise value
ray.exceptions.RaySystemError: System error: Failed to unpickle serialized exception
traceback: Traceback (most recent call last):
File "/home/lip21jaf/.conda/envs/huggingface_env/lib/python3.8/site-packages/ray/exceptions.py", line 46, in from_ray_exception
return pickle.loads(ray_exception.serialized_exception)
TypeError: __init__() missing 1 required positional argument: 'inner_exception'
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/home/lip21jaf/.conda/envs/huggingface_env/lib/python3.8/site-packages/ray/_private/serialization.py", line 404, in deserialize_objects
obj = self._deserialize_object(data, metadata, object_ref)
File "/home/lip21jaf/.conda/envs/huggingface_env/lib/python3.8/site-packages/ray/_private/serialization.py", line 293, in _deserialize_object
return RayError.from_bytes(obj)
File "/home/lip21jaf/.conda/envs/huggingface_env/lib/python3.8/site-packages/ray/exceptions.py", line 40, in from_bytes
return RayError.from_ray_exception(ray_exception)
File "/home/lip21jaf/.conda/envs/huggingface_env/lib/python3.8/site-packages/ray/exceptions.py", line 49, in from_ray_exception
raise RuntimeError(msg) from e
RuntimeError: Failed to unpickle serialized exception
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "gridsearch.py", line 136, in <module>
best_trial = trainer.hyperparameter_search(
File "/home/lip21jaf/.conda/envs/huggingface_env/lib/python3.8/site-packages/transformers/trainer.py", line 3206, in hyperparameter_search
best_run = backend_obj.run(self, n_trials, direction, **kwargs)
File "/home/lip21jaf/.conda/envs/huggingface_env/lib/python3.8/site-packages/transformers/hyperparameter_search.py", line 87, in run
return run_hp_search_ray(trainer, n_trials, direction, **kwargs)
File "/home/lip21jaf/.conda/envs/huggingface_env/lib/python3.8/site-packages/transformers/integrations/integration_utils.py", line 370, in run_hp_search_ray
analysis = ray.tune.run(
File "/home/lip21jaf/.conda/envs/huggingface_env/lib/python3.8/site-packages/ray/tune/tune.py", line 1001, in run
runner.step()
File "/home/lip21jaf/.conda/envs/huggingface_env/lib/python3.8/site-packages/ray/tune/execution/tune_controller.py", line 686, in step
if not self._actor_manager.next(timeout=0.1):
File "/home/lip21jaf/.conda/envs/huggingface_env/lib/python3.8/site-packages/ray/air/execution/_internal/actor_manager.py", line 224, in next
self._actor_task_events.resolve_future(future)
File "/home/lip21jaf/.conda/envs/huggingface_env/lib/python3.8/site-packages/ray/air/execution/_internal/event_manager.py", line 113, in resolve_future
on_error(e)
File "/home/lip21jaf/.conda/envs/huggingface_env/lib/python3.8/site-packages/ray/air/execution/_internal/actor_manager.py", line 772, in on_error
self._actor_task_failed(
File "/home/lip21jaf/.conda/envs/huggingface_env/lib/python3.8/site-packages/ray/air/execution/_internal/actor_manager.py", line 291, in _actor_task_failed
raise RuntimeError(
RuntimeError: Caught unexpected exception: System error: Failed to unpickle serialized exception
traceback: Traceback (most recent call last):
File "/home/lip21jaf/.conda/envs/huggingface_env/lib/python3.8/site-packages/ray/exceptions.py", line 46, in from_ray_exception
return pickle.loads(ray_exception.serialized_exception)
TypeError: __init__() missing 1 required positional argument: 'inner_exception'
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/home/lip21jaf/.conda/envs/huggingface_env/lib/python3.8/site-packages/ray/_private/serialization.py", line 404, in deserialize_objects
obj = self._deserialize_object(data, metadata, object_ref)
File "/home/lip21jaf/.conda/envs/huggingface_env/lib/python3.8/site-packages/ray/_private/serialization.py", line 293, in _deserialize_object
return RayError.from_bytes(obj)
File "/home/lip21jaf/.conda/envs/huggingface_env/lib/python3.8/site-packages/ray/exceptions.py", line 40, in from_bytes
return RayError.from_ray_exception(ray_exception)
File "/home/lip21jaf/.conda/envs/huggingface_env/lib/python3.8/site-packages/ray/exceptions.py", line 49, in from_ray_exception
raise RuntimeError(msg) from e
RuntimeError: Failed to unpickle serialized exception