Transformers.js Chat Completion on SmolLM2 using

🤗Transformers
1

Hi

I’m new to JS and trying to run smollm2 in my browser. While it seems like I’m successfully able to load the model, I’m facing errors while generating reply after applying chat template and tokenizing the same.
This is the code I’m running.

<!DOCTYPE html>
<html>
<head>
  <title>Test Transformers.js</title>
  <script type="module">
    async function testSummarization() {
      try {
        // Load transformers.js
        const { env, AutoTokenizer, AutoModelForCausalLM, pipeline } = await import('https://cdn.jsdelivr.net/npm/@huggingface/transformers@3.0.2');
        console.log('Transformers.js loaded'); // Debugging statement
        env.allowLocalModels = true;
        // env.allowRemoteModels = true;
        env.useBrowserCache = true;
        let model_name = 'HuggingFaceTB/SmolLM2-135M-Instruct'

        let tokenizer = await AutoTokenizer.from_pretrained(model_name);
        console.log('Tokenizer loaded'); // Debugging statement

        let model = await AutoModelForCausalLM.from_pretrained(model_name, {
          dtype: 'q4f16', device: 'webgpu'
        });
        //'onnx-community/Phi-3.5-mini-instruct-onnx-web', {
            // dtype: 'q4' , use_external_data_format: true
        console.log('Model loaded'); // Debugging statement

        const chat = [{"role": "user", "content": "What is the capital of France."}]

        console.log("chat", chat)
        let input_text = await tokenizer.apply_chat_template(chat , {tokenize: true, return_tensor: true, add_generation_prompt: true});
        console.log("input_text", input_text)
        console.log("input_text.dims",input_text.dims)
        // let inputs = tokenizer.encode(input_text, {return_tensors:true})
        // console.log("inputs", inputs)
        let outputs = await model.generate(input_text) // , { max_new_tokens: 130, max_length: 130, min_length: 3, length_penalty: 2.0, num_beams: 1 });
        console.log(outputs)
        let decoded = tokenizer.decode(outputs[0], { skip_special_tokens: true });
        console.log(decoded)
        
      } catch (error) {
        console.error('Error:', error);
      }
    }

    testSummarization();
  </script>
</head>
<body>
  <h1>Test Transformers.js</h1>
</body>
</html>

I’m running into following error [Error: TypeError: Cannot read properties of null (reading 'dims')] which I’m unable to debug [attaching the stack trace below]. I see there’s dims property in the input_text variable

test_messages.html:10 Transformers.js loaded
test_messages.html:17 Tokenizer loaded
transformers@3.2.0:1 2024-12-17 22:58:04.813899 [W:onnxruntime:, session_state.cc:1168 VerifyEachNodeIsAssignedToAnEp] Some nodes were not assigned to the preferred execution providers which may or may not have an negative impact on performance. e.g. ORT explicitly assigns shape related ops to CPU to improve perf.
lr @ transformers@3.2.0:1
$func11893 @ ort-wasm-simd-threaded.jsep.wasm:0x1033687
$func1968 @ ort-wasm-simd-threaded.jsep.wasm:0x23d5f2
$func841 @ ort-wasm-simd-threaded.jsep.wasm:0xc4693
$func11810 @ ort-wasm-simd-threaded.jsep.wasm:0x102bc5e
$func635 @ ort-wasm-simd-threaded.jsep.wasm:0x90b8e
$func89 @ ort-wasm-simd-threaded.jsep.wasm:0xd8c4
$func14530 @ ort-wasm-simd-threaded.jsep.wasm:0x13cdd12
$func124 @ ort-wasm-simd-threaded.jsep.wasm:0x15ba3
$func2123 @ ort-wasm-simd-threaded.jsep.wasm:0x273389
$func11361 @ ort-wasm-simd-threaded.jsep.wasm:0xfe66eb
$Ma @ ort-wasm-simd-threaded.jsep.wasm:0xaef71e
t.<computed> @ transformers@3.2.0:1
p._OrtCreateSession @ transformers@3.2.0:1
(anonymous) @ transformers@3.2.0:1
xd @ transformers@3.2.0:1
Vd @ transformers@3.2.0:1
loadModel @ transformers@3.2.0:1
createInferenceSessionHandler @ transformers@3.2.0:1
create @ transformers@3.2.0:1
await in create
_ @ transformers@3.2.0:1
(anonymous) @ transformers@3.2.0:1
await in (anonymous)
A @ transformers@3.2.0:1
from_pretrained @ transformers@3.2.0:1
await in from_pretrained
from_pretrained @ transformers@3.2.0:1
await in from_pretrained
testSummarization @ test_messages.html:19
await in testSummarization
(anonymous) @ test_messages.html:45
transformers@3.2.0:1 2024-12-17 22:58:04.815500 [W:onnxruntime:, session_state.cc:1170 VerifyEachNodeIsAssignedToAnEp] Rerunning with verbose output on a non-minimal build will show node assignments.
lr @ transformers@3.2.0:1
$func11893 @ ort-wasm-simd-threaded.jsep.wasm:0x1033687
$func1968 @ ort-wasm-simd-threaded.jsep.wasm:0x23d5f2
$func841 @ ort-wasm-simd-threaded.jsep.wasm:0xc4693
$func11810 @ ort-wasm-simd-threaded.jsep.wasm:0x102bc5e
$func635 @ ort-wasm-simd-threaded.jsep.wasm:0x90b8e
$func89 @ ort-wasm-simd-threaded.jsep.wasm:0xd8c4
$func14530 @ ort-wasm-simd-threaded.jsep.wasm:0x13cdd12
$func124 @ ort-wasm-simd-threaded.jsep.wasm:0x15ba3
$func2123 @ ort-wasm-simd-threaded.jsep.wasm:0x27386e
$func11361 @ ort-wasm-simd-threaded.jsep.wasm:0xfe66eb
$Ma @ ort-wasm-simd-threaded.jsep.wasm:0xaef71e
t.<computed> @ transformers@3.2.0:1
p._OrtCreateSession @ transformers@3.2.0:1
(anonymous) @ transformers@3.2.0:1
xd @ transformers@3.2.0:1
Vd @ transformers@3.2.0:1
loadModel @ transformers@3.2.0:1
createInferenceSessionHandler @ transformers@3.2.0:1
create @ transformers@3.2.0:1
await in create
_ @ transformers@3.2.0:1
(anonymous) @ transformers@3.2.0:1
await in (anonymous)
A @ transformers@3.2.0:1
from_pretrained @ transformers@3.2.0:1
await in from_pretrained
from_pretrained @ transformers@3.2.0:1
await in from_pretrained
testSummarization @ test_messages.html:19
await in testSummarization
(anonymous) @ test_messages.html:45
test_messages.html:24 Model loaded
test_messages.html:28 chat [{…}]
test_messages.html:31 input_text Proxy(o) {ort_tensor: o}
test_messages.html:32 input_text.dims (2) [1, 37]
test_messages.html:41 Error: TypeError: Cannot read properties of null (reading 'dims')
    at Function.generate (transformers@3.2.0:1:509425)
    at testSummarization (test_messages.html:35:35)
testSummarization @ test_messages.html:41
await in testSummarization
(anonymous) @ test_messages.html:45

please help.

1 Like