Hello we are new to huggingface and started to get errors on a working space after we received a notification that HF tokens were compromised. I have posted as much detail as I can with regards to our settings and build logs.
Not sure where we can go from here as we are pretty stuck on this error.
We have an organization with private spaces, datasets, and models running on a NVidia A10Gsmall. We use a HF_Token as a secret and we are running a ai space called camille.
title: Camille Trainer
emoji: 
colorFrom: gray
colorTo: gray
sdk: gradio
sdk_version: 4.36.1
app_file: app.py
pinned: true
license: mit
this is from our app.py
import gradio as gr
from docs import title_and_step1
from camille_helpers import fetch_upstream_dataset, fetch_upstream_user_hashes, reindex
from hf_helpers import deploy_model, fetch_hf_dataset, update_hf_dataset
from training_helpers import train_model
…
Camille-trainer
import os
import time
import requests
import gradio as gr
from docs import step2
from hf_helpers import get_inference_endpoint, wake_service
BASE_URL = os.environ.get(
…
hf_helpers.py
import time
import gradio as gr
from typing import List, Dict, Any
from dataclasses import asdict
from datasets import Dataset, load_dataset, DownloadMode
from huggingface_hub import (
InferenceEndpoint,
list_repo_refs,
list_inference_endpoints,
create_inference_endpoint,
)
from huggingface_hub.utils._errors import HfHubHTTPError, RepositoryNotFoundError
from docs import step3, step5
download_mode = DownloadMode.FORCE_REDOWNLOAD
…
Local-requrements.txt
datasets
huggingface-hub>=0.23.3
protobuf<4
click<8.1
pydantic~=1.0
gradio[oauth]==4.36.1
uvicorn>=0.14.0
spaces==0.17.0
model.py
from sentence_transformers import SentenceTransformer, models
import torch
def get_base_model():
device = torch.device(“cuda” if torch.cuda.is_available() else “cpu”)
word_embedding_model = models.Transformer("BAAI/bge-base-en-v1.5", max_seq_length=512)
word_embedding_model.tokenizer.add_tokens(['[TURN]'], special_tokens=True)
word_embedding_model.tokenizer.truncation_side = 'left'
word_embedding_model.auto_model.resize_token_embeddings(len(word_embedding_model.tokenizer))
pooling_model = models.Pooling(
word_embedding_model.get_word_embedding_dimension(), pooling_mode="cls"
)
model = SentenceTransformer(modules=[word_embedding_model, pooling_model], device=device)
return model
…
requirements.txt
–extra-index-url https://download.pytorch.org/whl/cu113
torch
sentence-transformers
huggingface-hub==0.23.3
training_helper.py
import json
import gradio as gr
import pandas as pd
from sentence_transformers import InputExample
from sentence_transformers.losses import OnlineContrastiveLoss
from sentence_transformers.evaluation import EmbeddingSimilarityEvaluator
from datasets import load_dataset, DownloadMode
from torch.utils.data import DataLoader
from huggingface_hub import create_repo, upload_folder, hf_hub_download
from docs import step4
from model import get_base_model
from hf_helpers import get_repo_main_branch
MODEL_PATH = “./model”
BATCH_SIZE = 32
We believe there is a problem with torch . When we get to the train model section we get this error.
raise ImportError(
ImportError: Using the Trainer with PyTorch requires accelerate>=0.20.1: Please run pip install transformers[torch] or pip install accelerate -U
We’ve ran these command with no success
This is the full error
===== Application Startup at 2024-06-17 18:39:58 =====
/home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages/transformers/utils/generic.py:441: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead.
_torch_pytree._register_pytree_node(
/home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages/transformers/utils/generic.py:309: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead.
_torch_pytree._register_pytree_node(
Running on local URL: http://0.0.0.0:7860
To create a public link, set share=True in launch().
/home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages/datasets/dataset_dict.py:1682: FutureWarning: ‘branch’ was deprecated in favor of ‘revision’ in version 2.15.0 and will be removed in 3.0.0.
You can remove this warning by passing ‘revision=main’ instead.
warnings.warn(
/home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages/huggingface_hub/file_download.py:1132: FutureWarning: resume_download is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use force_download=True.
warnings.warn(
/home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages/transformers/utils/generic.py:309: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead.
_torch_pytree._register_pytree_node(
Traceback (most recent call last):
File “/home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages/gradio/queueing.py”, line 532, in process_events
response = await route_utils.call_process_api(
File “/home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages/gradio/route_utils.py”, line 276, in call_process_api
output = await app.get_blocks().process_api(
File “/home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages/gradio/blocks.py”, line 1928, in process_api
result = await self.call_function(
File “/home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages/gradio/blocks.py”, line 1514, in call_function
prediction = await anyio.to_thread.run_sync(
File “/home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages/anyio/to_thread.py”, line 56, in run_sync
return await get_async_backend().run_sync_in_worker_thread(
File “/home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages/anyio/_backends/_asyncio.py”, line 2177, in run_sync_in_worker_thread
return await future
File “/home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages/anyio/_backends/_asyncio.py”, line 859, in run
result = context.run(func, *args)
File “/home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages/gradio/utils.py”, line 832, in wrapper
response = f(*args, **kwargs)
File “/home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages/gradio/utils.py”, line 832, in wrapper
response = f(*args, **kwargs)
File “/home/user/app/training_helpers.py”, line 96, in train_model
model.fit(
File “/home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages/sentence_transformers/fit_mixin.py”, line 304, in fit
args = SentenceTransformerTrainingArguments(
File “”, line 119, in init
File “/home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages/sentence_transformers/training_args.py”, line 73, in post_init
super().post_init()
File “/home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages/transformers/training_args.py”, line 1442, in post_init
and (self.device.type != “cuda”)
File “/home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages/transformers/training_args.py”, line 1887, in device
return self._setup_devices
File “/home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages/transformers/utils/generic.py”, line 54, in get
cached = self.fget(obj)
File “/home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages/transformers/training_args.py”, line 1787, in _setup_devices
raise ImportError(
ImportError: Using the Trainer with PyTorch requires accelerate>=0.20.1: Please run pip install transformers[torch] or pip install accelerate -U
Here is the build log
===== Build Queued at 2024-06-12 19:55:45 / Commit SHA: 6c2158a =====
→ FROM docker.io/nvidia/cuda:12.3.2-cudnn9-devel-ubuntu22.04@sha256:fb1ad20f2552f5b3aafb2c9c478ed57da95e2bb027d15218d7a55b3a0e4b4413
DONE 0.0s
→ COPY --chown=1000:1000 --from=root / /
CACHED
→ RUN apt-get update && apt-get install -y fakeroot && mv /usr/bin/apt-get /usr/bin/.apt-get && echo ‘#!/usr/bin/env sh\nfakeroot /usr/bin/.apt-get $@’ > /usr/bin/apt-get && chmod +x /usr/bin/apt-get && rm -rf /var/lib/apt/lists/* && useradd -m -u 1000 user
CACHED
→ RUN pip install --no-cache-dir pip==22.3.1 && pip install --no-cache-dir datasets “huggingface-hub>=0.19” “hf-transfer>=0.1.4” “protobuf<4” “click<8.1” “pydantic~=1.0”
CACHED
→ RUN curl https://pyenv.run | bash
CACHED
→ WORKDIR /home/user/app
CACHED
→ RUN apt-get update && apt-get install -y git rsync make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev git-lfs ffmpeg libsm6 libxext6 cmake libgl1-mesa-glx && rm -rf /var/lib/apt/lists/* && git lfs install
CACHED
→ RUN pyenv install 3.10 && pyenv global 3.10 && pyenv rehash
CACHED
→ RUN --mount=target=/tmp/requirements.txt,source=requirements.txt pip install --no-cache-dir -r /tmp/requirements.txt
CACHED
→ Restoring cache
DONE 46.0s
→ RUN pip install --no-cache-dir gradio[oauth]==4.36.1 “uvicorn>=0.14.0” spaces
Collecting gradio[oauth]==4.36.1
Downloading gradio-4.36.1-py3-none-any.whl (12.3 MB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 12.3/12.3 MB 193.1 MB/s eta 0:00:00
Collecting uvicorn>=0.14.0
Downloading uvicorn-0.30.1-py3-none-any.whl (62 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 62.4/62.4 kB 308.7 MB/s eta 0:00:00
Collecting spaces
Downloading spaces-0.28.3-py3-none-any.whl (18 kB)
Requirement already satisfied: numpy<3.0,>=1.0 in /home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages (from gradio[oauth]==4.36.1) (1.26.4)
Collecting importlib-resources<7.0,>=1.3
Downloading importlib_resources-6.4.0-py3-none-any.whl (38 kB)
Requirement already satisfied: markupsafe~=2.0 in /home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages (from gradio[oauth]==4.36.1) (2.1.5)
Requirement already satisfied: urllib3~=2.0 in /home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages (from gradio[oauth]==4.36.1) (2.2.1)
Requirement already satisfied: pandas<3.0,>=1.0 in /home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages (from gradio[oauth]==4.36.1) (2.2.2)
Requirement already satisfied: packaging in /home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages (from gradio[oauth]==4.36.1) (24.0)
Collecting altair<6.0,>=4.2.0
Downloading altair-5.3.0-py3-none-any.whl (857 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 857.8/857.8 kB 400.3 MB/s eta 0:00:00
Collecting huggingface-hub>=0.19.3
Downloading huggingface_hub-0.23.3-py3-none-any.whl (401 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 401.7/401.7 kB 431.4 MB/s eta 0:00:00
Requirement already satisfied: pyyaml<7.0,>=5.0 in /home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages (from gradio[oauth]==4.36.1) (6.0.1)
Collecting fastapi
Downloading fastapi-0.111.0-py3-none-any.whl (91 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 92.0/92.0 kB 337.9 MB/s eta 0:00:00
Requirement already satisfied: jinja2<4.0 in /home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages (from gradio[oauth]==4.36.1) (3.1.4)
Collecting httpx>=0.24.1
Downloading httpx-0.27.0-py3-none-any.whl (75 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 75.6/75.6 kB 353.1 MB/s eta 0:00:00
Collecting aiofiles<24.0,>=22.0
Downloading aiofiles-23.2.1-py3-none-any.whl (15 kB)
Collecting ruff>=0.2.2
Downloading ruff-0.4.8-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (8.8 MB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 8.8/8.8 MB 214.5 MB/s eta 0:00:00
Collecting typer<1.0,>=0.12
Downloading typer-0.12.3-py3-none-any.whl (47 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 47.2/47.2 kB 287.9 MB/s eta 0:00:00
Collecting ffmpy
Downloading ffmpy-0.3.2.tar.gz (5.5 kB)
Preparing metadata (setup.py): started
Preparing metadata (setup.py): finished with status ‘done’
Requirement already satisfied: typing-extensions~=4.0 in /home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages (from gradio[oauth]==4.36.1) (4.12.1)
Collecting gradio-client==1.0.1
Downloading gradio_client-1.0.1-py3-none-any.whl (318 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 318.1/318.1 kB 396.0 MB/s eta 0:00:00
Collecting matplotlib~=3.0
Downloading matplotlib-3.9.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (8.3 MB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 8.3/8.3 MB 170.0 MB/s eta 0:00:00
Collecting python-multipart>=0.0.9
Downloading python_multipart-0.0.9-py3-none-any.whl (22 kB)
Collecting tomlkit==0.12.0
Downloading tomlkit-0.12.0-py3-none-any.whl (37 kB)
Requirement already satisfied: pillow<11.0,>=8.0 in /home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages (from gradio[oauth]==4.36.1) (10.3.0)
Collecting semantic-version~=2.0
Downloading semantic_version-2.10.0-py2.py3-none-any.whl (15 kB)
Collecting pydub
Downloading pydub-0.25.1-py2.py3-none-any.whl (32 kB)
Collecting orjson~=3.0
Downloading orjson-3.10.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (142 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 142.7/142.7 kB 376.0 MB/s eta 0:00:00
Collecting pydantic>=2.0
Downloading pydantic-2.7.4-py3-none-any.whl (409 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 409.0/409.0 kB 428.4 MB/s eta 0:00:00
Collecting authlib
Downloading Authlib-1.3.1-py2.py3-none-any.whl (223 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 223.8/223.8 kB 419.3 MB/s eta 0:00:00
Collecting itsdangerous
Downloading itsdangerous-2.2.0-py3-none-any.whl (16 kB)
Collecting websockets<12.0,>=10.0
Downloading websockets-11.0.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (129 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 129.9/129.9 kB 293.8 MB/s eta 0:00:00
Requirement already satisfied: fsspec in /home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages (from gradio-client==1.0.1->gradio[oauth]==4.36.1) (2024.3.1)
Requirement already satisfied: click>=7.0 in /home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages (from uvicorn>=0.14.0) (8.0.4)
Collecting h11>=0.8
Downloading h11-0.14.0-py3-none-any.whl (58 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 58.3/58.3 kB 315.5 MB/s eta 0:00:00
Collecting psutil<6,>=2
Downloading psutil-5.9.8-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (288 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 288.2/288.2 kB 332.1 MB/s eta 0:00:00
Requirement already satisfied: requests<3.0,>=2.19 in /home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages (from spaces) (2.32.3)
Collecting toolz
Downloading toolz-0.12.1-py3-none-any.whl (56 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 56.1/56.1 kB 309.1 MB/s eta 0:00:00
Collecting jsonschema>=3.0
Downloading jsonschema-4.22.0-py3-none-any.whl (88 kB)