{"users":[{"id":121789,"username":"ursbijju","name":"Raghavendra Reddy","avatar_template":"/user_avatar/discuss.huggingface.co/ursbijju/{size}/62760_2.png","trust_level":0},{"id":52272,"username":"John6666","name":"John Smith","avatar_template":"/user_avatar/discuss.huggingface.co/john6666/{size}/27664_2.png","trust_level":3},{"id":121471,"username":"thordata","name":"Greta","avatar_template":"/user_avatar/discuss.huggingface.co/thordata/{size}/62596_2.png","trust_level":1},{"id":6790,"username":"bogolese","name":"Rich Bergmann","avatar_template":"/user_avatar/discuss.huggingface.co/bogolese/{size}/53040_2.png","trust_level":1},{"id":111468,"username":"cyr0930","name":"Young-rok Cha","avatar_template":"https://avatars.discourse-cdn.com/v4/letter/c/34f0e0/{size}.png","trust_level":0},{"id":-1,"username":"system","name":"system","avatar_template":"https://us1.discourse-cdn.com/hellohellohello/original/2X/d/de4155eb4aa4108ecb32a1389d7cc37ae69f88b7.png","admin":true,"moderator":true,"trust_level":4},{"id":110512,"username":"JuliaGecko","name":"Julia","avatar_template":"/user_avatar/discuss.huggingface.co/juliagecko/{size}/57398_2.png","trust_level":0},{"id":67580,"username":"ali-issa","name":"Ali issa","avatar_template":"https://avatars.discourse-cdn.com/v4/letter/a/a698b9/{size}.png","trust_level":1},{"id":4044,"username":"ccfeidao","name":"chen chen","avatar_template":"https://avatars.discourse-cdn.com/v4/letter/c/f08c70/{size}.png","trust_level":1},{"id":3725,"username":"mariosasko","name":"Mario Šaško","avatar_template":"/user_avatar/discuss.huggingface.co/mariosasko/{size}/31548_2.png","trust_level":4},{"id":42613,"username":"msinghy","name":"Manu Yadav","avatar_template":"/user_avatar/discuss.huggingface.co/msinghy/{size}/24386_2.png","trust_level":1},{"id":107882,"username":"hamingsi","name":"Liangyu","avatar_template":"https://avatars.discourse-cdn.com/v4/letter/h/eb8c5e/{size}.png","trust_level":0},{"id":32083,"username":"BoltzmachineQ","name":"Weikang Qiu","avatar_template":"https://avatars.discourse-cdn.com/v4/letter/b/e47c2d/{size}.png","trust_level":1},{"id":107652,"username":"Pandores","name":"-----","avatar_template":"https://avatars.discourse-cdn.com/v4/letter/p/dbc845/{size}.png","trust_level":0},{"id":105374,"username":"iamTHOPE","name":"Praval Pattam","avatar_template":"/user_avatar/discuss.huggingface.co/iamthope/{size}/54712_2.png","trust_level":0},{"id":105019,"username":"DanielMielke","name":"Daniel Mielke","avatar_template":"https://avatars.discourse-cdn.com/v4/letter/d/f08c70/{size}.png","trust_level":0},{"id":103787,"username":"ggunio","name":"Jinhyun Woo","avatar_template":"/user_avatar/discuss.huggingface.co/ggunio/{size}/60658_2.png","trust_level":1},{"id":2084,"username":"hfnlpmb","name":"t k","avatar_template":"https://avatars.discourse-cdn.com/v4/letter/h/ecd19e/{size}.png","trust_level":1},{"id":98595,"username":"gursi26","name":"Gursimar Singh","avatar_template":"https://avatars.discourse-cdn.com/v4/letter/g/41988e/{size}.png","trust_level":0},{"id":98658,"username":"AD0021","name":"Andrzej Domański","avatar_template":"https://avatars.discourse-cdn.com/v4/letter/a/51bf81/{size}.png","trust_level":0},{"id":79053,"username":"GuyShur","name":"Guy Shur","avatar_template":"https://avatars.discourse-cdn.com/v4/letter/g/ce73a5/{size}.png","trust_level":1},{"id":97008,"username":"Felicitywood","name":"Felicity Wood","avatar_template":"/user_avatar/discuss.huggingface.co/felicitywood/{size}/49463_2.png","trust_level":1},{"id":98435,"username":"NuralNexus","name":"David Riggs","avatar_template":"/user_avatar/discuss.huggingface.co/nuralnexus/{size}/56230_2.png","trust_level":2},{"id":98025,"username":"torch-nn","name":"t-sneeze","avatar_template":"https://avatars.discourse-cdn.com/v4/letter/t/e9bcb4/{size}.png","trust_level":0},{"id":89779,"username":"e950280","name":"HanSheng-Lai","avatar_template":"https://avatars.discourse-cdn.com/v4/letter/e/3be4f8/{size}.png","trust_level":1},{"id":88738,"username":"ak400400","name":"אברהם כהן","avatar_template":"https://avatars.discourse-cdn.com/v4/letter/a/9e8a1a/{size}.png","trust_level":0},{"id":87934,"username":"springk","name":"Kim Bomm","avatar_template":"/user_avatar/discuss.huggingface.co/springk/{size}/43759_2.png","trust_level":0},{"id":60617,"username":"Jason-Hwang1","name":"Jason-Hwang1","avatar_template":"https://avatars.discourse-cdn.com/v4/letter/j/ccd318/{size}.png","trust_level":0},{"id":87679,"username":"gweidart","name":"gweidart","avatar_template":"/user_avatar/discuss.huggingface.co/gweidart/{size}/43593_2.png","trust_level":0},{"id":1864,"username":"Boltzmachine","name":"Weikang Qiu","avatar_template":"https://avatars.discourse-cdn.com/v4/letter/b/46a35a/{size}.png","trust_level":1},{"id":760,"username":"wjs20","name":"William Steele ","avatar_template":"https://avatars.discourse-cdn.com/v4/letter/w/bc79bd/{size}.png","trust_level":1},{"id":87482,"username":"vinayakdev","name":"Vinayak Dev","avatar_template":"https://avatars.discourse-cdn.com/v4/letter/v/6f9a4e/{size}.png","trust_level":0},{"id":87173,"username":"aaaksenova","name":"Anna Aksenova","avatar_template":"https://avatars.discourse-cdn.com/v4/letter/a/a6a055/{size}.png","trust_level":0},{"id":21384,"username":"steventrouble","name":"Steven Weiss","avatar_template":"/user_avatar/discuss.huggingface.co/steventrouble/{size}/16596_2.png","trust_level":0},{"id":7005,"username":"ArthurZ","name":"Arthur Zucker","avatar_template":"/user_avatar/discuss.huggingface.co/arthurz/{size}/26972_2.png","moderator":true,"trust_level":4},{"id":86817,"username":"lone17","name":"ian","avatar_template":"https://avatars.discourse-cdn.com/v4/letter/l/ccd318/{size}.png","trust_level":1},{"id":48690,"username":"Aurelien-Morgan","name":"Aurélien-Morgan CLAUDON","avatar_template":"/user_avatar/discuss.huggingface.co/aurelien-morgan/{size}/26438_2.png","trust_level":1},{"id":76958,"username":"Alanturner2","name":"Alan turner","avatar_template":"/user_avatar/discuss.huggingface.co/alanturner2/{size}/37542_2.png","trust_level":2},{"id":58120,"username":"sleeping4cat","name":"TAWSIF AHMED","avatar_template":"/user_avatar/discuss.huggingface.co/sleeping4cat/{size}/29463_2.png","trust_level":1},{"id":46798,"username":"timdadum","name":"Tim Rood","avatar_template":"https://avatars.discourse-cdn.com/v4/letter/t/8edcca/{size}.png","trust_level":1},{"id":77485,"username":"dristi","name":"Dristi Sigdel","avatar_template":"https://avatars.discourse-cdn.com/v4/letter/d/f17d59/{size}.png","trust_level":0},{"id":82734,"username":"Invicto69","name":"Dinesh","avatar_template":"/user_avatar/discuss.huggingface.co/invicto69/{size}/40711_2.png","trust_level":0},{"id":80944,"username":"samirchar","name":"Samir Char","avatar_template":"https://avatars.discourse-cdn.com/v4/letter/s/c2a13f/{size}.png","trust_level":1}],"primary_groups":[],"flair_groups":[],"topic_list":{"can_create_topic":false,"filter":"latest","more_topics_url":"/c/tokenizers/11?page=1","per_page":30,"topics":[{"fancy_title":"STEM image/text based PDF textbooks for extraction, indexing and embeddings","id":179503,"title":"STEM image/text based PDF textbooks for extraction, indexing and embeddings","slug":"stem-image-text-based-pdf-textbooks-for-extraction-indexing-and-embeddings","posts_count":3,"reply_count":0,"highest_post_number":3,"image_url":null,"created_at":"2026-08-31T12:05:40.698Z","last_posted_at":"2026-09-02T07:35:15.426Z","bumped":true,"bumped_at":"2026-09-02T07:35:15.426Z","archetype":"regular","unseen":false,"pinned":false,"unpinned":null,"visible":true,"closed":false,"archived":false,"bookmarked":null,"liked":null,"views":155,"like_count":3,"has_summary":false,"last_poster_username":"thordata","category_id":11,"op_like_count":1,"pinned_globally":false,"featured_link":null,"has_accepted_answer":false,"can_vote":false,"posters":[{"extras":null,"description":"Original Poster","user_id":121789,"primary_group_id":null,"flair_group_id":null},{"extras":null,"description":"Recent Poster","user_id":52272,"primary_group_id":null,"flair_group_id":null},{"extras":"latest","description":"Most Recent Poster","user_id":121471,"primary_group_id":null,"flair_group_id":null}]},{"fancy_title":"ValueError loading Helsinki-NLP tokenizers","id":174192,"title":"ValueError loading Helsinki-NLP tokenizers","slug":"valueerror-loading-helsinki-nlp-tokenizers","posts_count":4,"reply_count":0,"highest_post_number":4,"image_url":null,"created_at":"2026-03-11T15:43:55.060Z","last_posted_at":"2026-03-12T13:02:44.728Z","bumped":true,"bumped_at":"2026-03-12T13:02:44.728Z","archetype":"regular","unseen":false,"pinned":false,"unpinned":null,"visible":true,"closed":false,"archived":false,"bookmarked":null,"liked":null,"views":151,"like_count":3,"has_summary":false,"last_poster_username":"bogolese","category_id":11,"op_like_count":1,"pinned_globally":false,"featured_link":null,"has_accepted_answer":false,"can_vote":false,"posters":[{"extras":"latest","description":"Original Poster, Most Recent Poster","user_id":6790,"primary_group_id":null,"flair_group_id":null},{"extras":null,"description":"Recent Poster","user_id":52272,"primary_group_id":null,"flair_group_id":null}]},{"fancy_title":"Doesn’t gpt-oss use tiktoken?","id":173215,"title":"Doesn't gpt-oss use tiktoken?","slug":"doesnt-gpt-oss-use-tiktoken","posts_count":3,"reply_count":1,"highest_post_number":3,"image_url":null,"created_at":"2026-02-09T08:36:01.551Z","last_posted_at":"2026-02-10T05:25:19.129Z","bumped":true,"bumped_at":"2026-02-10T05:25:19.129Z","archetype":"regular","unseen":false,"pinned":false,"unpinned":null,"visible":true,"closed":true,"archived":false,"bookmarked":null,"liked":null,"views":466,"like_count":3,"has_summary":false,"last_poster_username":"cyr0930","category_id":11,"op_like_count":1,"pinned_globally":false,"featured_link":null,"has_accepted_answer":true,"can_vote":false,"posters":[{"extras":"latest","description":"Original Poster, Most Recent Poster","user_id":111468,"primary_group_id":null,"flair_group_id":null},{"extras":null,"description":"Recent Poster, Accepted Answer","user_id":52272,"primary_group_id":null,"flair_group_id":null},{"extras":null,"description":"Recent Poster","user_id":-1,"primary_group_id":null,"flair_group_id":null}]},{"fancy_title":"XLM-R vs llama-7b tokenization","id":172649,"title":"XLM-R vs llama-7b tokenization","slug":"xlm-r-vs-llama-7b-tokenization","posts_count":6,"reply_count":4,"highest_post_number":6,"image_url":null,"created_at":"2026-01-20T21:03:32.584Z","last_posted_at":"2026-02-06T04:26:05.342Z","bumped":true,"bumped_at":"2026-02-06T04:26:05.342Z","archetype":"regular","unseen":false,"pinned":false,"unpinned":null,"visible":true,"closed":false,"archived":false,"bookmarked":null,"liked":null,"views":286,"like_count":3,"has_summary":false,"last_poster_username":"John6666","category_id":11,"op_like_count":1,"pinned_globally":false,"featured_link":null,"has_accepted_answer":false,"can_vote":false,"posters":[{"extras":null,"description":"Original Poster","user_id":110512,"primary_group_id":null,"flair_group_id":null},{"extras":"latest","description":"Most Recent Poster","user_id":52272,"primary_group_id":null,"flair_group_id":null}]},{"fancy_title":"Training a tokenizer clarification question","id":171140,"title":"Training a tokenizer clarification question","slug":"training-a-tokenizer-clarification-question","posts_count":2,"reply_count":0,"highest_post_number":4,"image_url":null,"created_at":"2025-12-04T07:57:07.803Z","last_posted_at":"2025-12-04T11:18:23.674Z","bumped":true,"bumped_at":"2025-12-04T11:18:23.674Z","archetype":"regular","unseen":false,"pinned":false,"unpinned":null,"visible":true,"closed":false,"archived":false,"bookmarked":null,"liked":null,"visibility_reason_id":1,"views":143,"like_count":1,"has_summary":false,"last_poster_username":"John6666","category_id":11,"op_like_count":1,"pinned_globally":false,"featured_link":null,"has_accepted_answer":false,"can_vote":false,"posters":[{"extras":null,"description":"Original Poster","user_id":67580,"primary_group_id":null,"flair_group_id":null},{"extras":null,"description":"Recent Poster","user_id":-1,"primary_group_id":null,"flair_group_id":null},{"extras":"latest","description":"Most Recent Poster","user_id":52272,"primary_group_id":null,"flair_group_id":null}]},{"fancy_title":"Tokenizer dataset is very slow","id":19722,"title":"Tokenizer dataset is very slow","slug":"tokenizer-dataset-is-very-slow","posts_count":5,"reply_count":3,"highest_post_number":6,"image_url":null,"created_at":"2022-06-28T12:35:02.887Z","last_posted_at":"2025-11-25T09:30:45.508Z","bumped":true,"bumped_at":"2025-11-25T09:30:45.508Z","archetype":"regular","unseen":false,"pinned":false,"unpinned":null,"visible":true,"closed":false,"archived":false,"bookmarked":null,"liked":null,"views":4773,"like_count":3,"has_summary":false,"last_poster_username":"hamingsi","category_id":11,"op_like_count":0,"pinned_globally":false,"featured_link":null,"has_accepted_answer":true,"can_vote":false,"posters":[{"extras":null,"description":"Original Poster","user_id":4044,"primary_group_id":null,"flair_group_id":null},{"extras":null,"description":"Recent Poster, Accepted Answer","user_id":3725,"primary_group_id":null,"flair_group_id":null},{"extras":null,"description":"Frequent Poster","user_id":42613,"primary_group_id":null,"flair_group_id":null},{"extras":"latest","description":"Most Recent Poster","user_id":107882,"primary_group_id":null,"flair_group_id":null}]},{"fancy_title":"Return_offsets_mapping when decoding","id":152215,"title":"Return_offsets_mapping when decoding","slug":"return-offsets-mapping-when-decoding","posts_count":5,"reply_count":1,"highest_post_number":5,"image_url":null,"created_at":"2025-04-24T23:52:58.346Z","last_posted_at":"2025-11-20T18:40:24.863Z","bumped":true,"bumped_at":"2025-11-20T18:40:24.863Z","archetype":"regular","unseen":false,"pinned":false,"unpinned":null,"visible":true,"closed":false,"archived":false,"bookmarked":null,"liked":null,"views":451,"like_count":3,"has_summary":false,"last_poster_username":"Pandores","category_id":11,"op_like_count":1,"pinned_globally":false,"featured_link":null,"has_accepted_answer":false,"can_vote":false,"posters":[{"extras":null,"description":"Original Poster","user_id":32083,"primary_group_id":null,"flair_group_id":null},{"extras":null,"description":"Recent Poster","user_id":52272,"primary_group_id":null,"flair_group_id":null},{"extras":"latest","description":"Most Recent Poster","user_id":107652,"primary_group_id":null,"flair_group_id":null}]},{"fancy_title":"Training a Vision Language Model on Text Only Data","id":169089,"title":"Training a Vision Language Model on Text Only Data","slug":"training-a-vision-language-model-on-text-only-data","posts_count":3,"reply_count":1,"highest_post_number":3,"image_url":null,"created_at":"2025-10-10T22:28:35.481Z","last_posted_at":"2025-10-11T11:07:37.138Z","bumped":true,"bumped_at":"2025-10-11T11:07:37.138Z","archetype":"regular","unseen":false,"pinned":false,"unpinned":null,"visible":true,"closed":false,"archived":false,"bookmarked":null,"liked":null,"views":117,"like_count":2,"has_summary":false,"last_poster_username":"iamTHOPE","category_id":11,"op_like_count":1,"pinned_globally":false,"featured_link":null,"has_accepted_answer":false,"can_vote":false,"posters":[{"extras":"latest","description":"Original Poster, Most Recent Poster","user_id":105374,"primary_group_id":null,"flair_group_id":null},{"extras":null,"description":"Recent Poster","user_id":52272,"primary_group_id":null,"flair_group_id":null}]},{"fancy_title":"Advice for Singing Synthesis and Tokenization","id":168913,"title":"Advice for Singing Synthesis and Tokenization","slug":"advice-for-singing-synthesis-and-tokenization","posts_count":3,"reply_count":1,"highest_post_number":3,"image_url":null,"created_at":"2025-10-04T21:20:46.539Z","last_posted_at":"2025-10-08T20:50:34.696Z","bumped":true,"bumped_at":"2025-10-04T23:46:30.581Z","archetype":"regular","unseen":false,"pinned":false,"unpinned":null,"visible":true,"closed":false,"archived":false,"bookmarked":null,"liked":null,"views":88,"like_count":3,"has_summary":false,"last_poster_username":"DanielMielke","category_id":11,"op_like_count":1,"pinned_globally":false,"featured_link":null,"has_accepted_answer":false,"can_vote":false,"posters":[{"extras":"latest","description":"Original Poster, Most Recent Poster","user_id":105019,"primary_group_id":null,"flair_group_id":null},{"extras":null,"description":"Recent Poster","user_id":52272,"primary_group_id":null,"flair_group_id":null}]},{"fancy_title":"Intelligent Tokenizer: Attention Needs No Vocabulary (Demo + Paper)","id":168393,"title":"Intelligent Tokenizer: Attention Needs No Vocabulary (Demo + Paper)","slug":"intelligent-tokenizer-attention-needs-no-vocabulary-demo-paper","posts_count":1,"reply_count":0,"highest_post_number":1,"image_url":null,"created_at":"2025-09-14T17:37:32.992Z","last_posted_at":"2025-09-14T17:37:33.106Z","bumped":true,"bumped_at":"2025-09-14T17:55:30.221Z","archetype":"regular","unseen":false,"pinned":false,"unpinned":null,"visible":true,"closed":false,"archived":false,"bookmarked":null,"liked":null,"views":81,"like_count":1,"has_summary":false,"last_poster_username":"ggunio","category_id":11,"op_like_count":1,"pinned_globally":false,"featured_link":null,"has_accepted_answer":false,"can_vote":false,"posters":[{"extras":"latest single","description":"Original Poster, Most Recent Poster","user_id":103787,"primary_group_id":null,"flair_group_id":null}]},{"fancy_title":"Qwen tokenizer, omit Ġ from offset_mapping","id":163495,"title":"Qwen tokenizer, omit Ġ from offset_mapping","slug":"qwen-tokenizer-omit-g-from-offset-mapping","posts_count":4,"reply_count":1,"highest_post_number":4,"image_url":null,"created_at":"2025-07-17T17:19:03.177Z","last_posted_at":"2025-07-18T10:39:53.831Z","bumped":true,"bumped_at":"2025-07-18T10:39:53.831Z","archetype":"regular","unseen":false,"pinned":false,"unpinned":null,"visible":true,"closed":false,"archived":false,"bookmarked":null,"liked":null,"views":240,"like_count":4,"has_summary":false,"last_poster_username":"John6666","category_id":11,"op_like_count":1,"pinned_globally":false,"featured_link":null,"has_accepted_answer":false,"can_vote":false,"posters":[{"extras":null,"description":"Original Poster","user_id":2084,"primary_group_id":null,"flair_group_id":null},{"extras":"latest","description":"Most Recent Poster","user_id":52272,"primary_group_id":null,"flair_group_id":null}]},{"fancy_title":"Word level tokenizer pulls special tokens out of pretokenized strings","id":161708,"title":"Word level tokenizer pulls special tokens out of pretokenized strings","slug":"word-level-tokenizer-pulls-special-tokens-out-of-pretokenized-strings","posts_count":4,"reply_count":1,"highest_post_number":4,"image_url":null,"created_at":"2025-07-03T18:00:11.860Z","last_posted_at":"2025-07-04T15:26:54.995Z","bumped":true,"bumped_at":"2025-07-04T15:26:54.995Z","archetype":"regular","unseen":false,"pinned":false,"unpinned":null,"visible":true,"closed":false,"archived":false,"bookmarked":null,"liked":null,"views":95,"like_count":3,"has_summary":false,"last_poster_username":"AD0021","category_id":11,"op_like_count":1,"pinned_globally":false,"featured_link":null,"has_accepted_answer":false,"can_vote":false,"posters":[{"extras":null,"description":"Original Poster","user_id":98595,"primary_group_id":null,"flair_group_id":null},{"extras":null,"description":"Recent Poster","user_id":52272,"primary_group_id":null,"flair_group_id":null},{"extras":"latest","description":"Most Recent Poster","user_id":98658,"primary_group_id":null,"flair_group_id":null}]},{"fancy_title":"Adding atomic / indivisible tokens to BPE tokenizer","id":161486,"title":"Adding atomic / indivisible tokens to BPE tokenizer","slug":"adding-atomic-indivisible-tokens-to-bpe-tokenizer","posts_count":9,"reply_count":1,"highest_post_number":9,"image_url":null,"created_at":"2025-07-02T07:55:53.466Z","last_posted_at":"2025-07-03T14:17:13.694Z","bumped":true,"bumped_at":"2025-07-03T14:17:13.694Z","archetype":"regular","unseen":false,"pinned":false,"unpinned":null,"visible":true,"closed":false,"archived":false,"bookmarked":null,"liked":null,"views":358,"like_count":6,"has_summary":false,"last_poster_username":"John6666","category_id":11,"op_like_count":1,"pinned_globally":false,"featured_link":null,"has_accepted_answer":false,"can_vote":false,"posters":[{"extras":null,"description":"Original Poster","user_id":79053,"primary_group_id":null,"flair_group_id":null},{"extras":null,"description":"Frequent Poster","user_id":97008,"primary_group_id":null,"flair_group_id":null},{"extras":null,"description":"Recent Poster","user_id":98435,"primary_group_id":null,"flair_group_id":null},{"extras":"latest","description":"Most Recent Poster","user_id":52272,"primary_group_id":null,"flair_group_id":null}]},{"fancy_title":"Tokenizer is splitting special token","id":161147,"title":"Tokenizer is splitting special token","slug":"tokenizer-is-splitting-special-token","posts_count":4,"reply_count":1,"highest_post_number":4,"image_url":null,"created_at":"2025-06-30T02:43:18.082Z","last_posted_at":"2025-06-30T05:29:37.127Z","bumped":true,"bumped_at":"2025-06-30T05:29:37.127Z","archetype":"regular","unseen":false,"pinned":false,"unpinned":null,"visible":true,"closed":false,"archived":false,"bookmarked":null,"liked":null,"views":123,"like_count":2,"has_summary":false,"last_poster_username":"John6666","category_id":11,"op_like_count":1,"pinned_globally":false,"featured_link":null,"has_accepted_answer":false,"can_vote":false,"posters":[{"extras":null,"description":"Original Poster","user_id":98025,"primary_group_id":null,"flair_group_id":null},{"extras":"latest","description":"Most Recent Poster","user_id":52272,"primary_group_id":null,"flair_group_id":null}]},{"fancy_title":"How to determine if a token is special","id":152329,"title":"How to determine if a token is special","slug":"how-to-determine-if-a-token-is-special","posts_count":3,"reply_count":0,"highest_post_number":3,"image_url":null,"created_at":"2025-04-25T16:54:56.919Z","last_posted_at":"2025-04-29T02:53:42.518Z","bumped":true,"bumped_at":"2025-04-29T02:53:42.518Z","archetype":"regular","unseen":false,"pinned":false,"unpinned":null,"visible":true,"closed":false,"archived":false,"bookmarked":null,"liked":null,"views":275,"like_count":2,"has_summary":false,"last_poster_username":"e950280","category_id":11,"op_like_count":1,"pinned_globally":false,"featured_link":null,"has_accepted_answer":false,"can_vote":false,"posters":[{"extras":null,"description":"Original Poster","user_id":32083,"primary_group_id":null,"flair_group_id":null},{"extras":null,"description":"Recent Poster","user_id":52272,"primary_group_id":null,"flair_group_id":null},{"extras":"latest","description":"Most Recent Poster","user_id":89779,"primary_group_id":null,"flair_group_id":null}]},{"fancy_title":"Saving tokens by system prompt","id":151783,"title":"Saving tokens by system prompt","slug":"saving-tokens-by-system-prompt","posts_count":2,"reply_count":0,"highest_post_number":2,"image_url":null,"created_at":"2025-04-22T10:21:26.249Z","last_posted_at":"2025-04-22T11:07:49.125Z","bumped":true,"bumped_at":"2025-04-22T11:07:49.125Z","archetype":"regular","unseen":false,"pinned":false,"unpinned":null,"visible":true,"closed":false,"archived":false,"bookmarked":null,"liked":null,"views":281,"like_count":2,"has_summary":false,"last_poster_username":"John6666","category_id":11,"op_like_count":1,"pinned_globally":false,"featured_link":null,"has_accepted_answer":false,"can_vote":false,"posters":[{"extras":null,"description":"Original Poster","user_id":88738,"primary_group_id":null,"flair_group_id":null},{"extras":"latest","description":"Most Recent Poster","user_id":52272,"primary_group_id":null,"flair_group_id":null}]},{"fancy_title":"Introducing FlashTokenizer: The World’s Fastest Tokenizer Library for LLM Inference","id":146746,"title":"Introducing FlashTokenizer: The World's Fastest Tokenizer Library for LLM Inference","slug":"introducing-flashtokenizer-the-worlds-fastest-tokenizer-library-for-llm-inference","posts_count":1,"reply_count":0,"highest_post_number":1,"image_url":null,"created_at":"2025-03-21T06:02:03.956Z","last_posted_at":"2025-03-21T06:02:04.018Z","bumped":true,"bumped_at":"2025-03-21T06:02:04.018Z","archetype":"regular","unseen":false,"pinned":false,"unpinned":null,"visible":true,"closed":false,"archived":false,"bookmarked":null,"liked":null,"visibility_reason_id":1,"views":97,"like_count":1,"has_summary":false,"last_poster_username":"springk","category_id":11,"op_like_count":1,"pinned_globally":false,"featured_link":null,"has_accepted_answer":false,"can_vote":false,"posters":[{"extras":"latest","description":"Original Poster, Most Recent Poster","user_id":87934,"primary_group_id":null,"flair_group_id":null},{"extras":null,"description":"Recent Poster","user_id":-1,"primary_group_id":null,"flair_group_id":null}]},{"fancy_title":"Why does tokenization take so long?","id":102098,"title":"Why does tokenization take so long?","slug":"why-does-tokenization-take-so-long","posts_count":2,"reply_count":0,"highest_post_number":2,"image_url":null,"created_at":"2024-08-13T04:13:42.732Z","last_posted_at":"2025-03-19T09:12:31.489Z","bumped":true,"bumped_at":"2025-03-19T09:12:31.489Z","archetype":"regular","unseen":false,"pinned":false,"unpinned":null,"visible":true,"closed":false,"archived":false,"bookmarked":null,"liked":null,"views":745,"like_count":4,"has_summary":false,"last_poster_username":"gweidart","category_id":11,"op_like_count":3,"pinned_globally":false,"featured_link":null,"has_accepted_answer":false,"can_vote":false,"posters":[{"extras":null,"description":"Original Poster","user_id":60617,"primary_group_id":null,"flair_group_id":null},{"extras":"latest","description":"Most Recent Poster","user_id":87679,"primary_group_id":null,"flair_group_id":null}]},{"fancy_title":"Call rust function in python","id":103446,"title":"Call rust function in python","slug":"call-rust-function-in-python","posts_count":2,"reply_count":0,"highest_post_number":2,"image_url":null,"created_at":"2024-08-22T06:01:41.370Z","last_posted_at":"2025-03-19T09:07:53.057Z","bumped":true,"bumped_at":"2025-03-19T09:07:53.057Z","archetype":"regular","unseen":false,"pinned":false,"unpinned":null,"visible":true,"closed":false,"archived":false,"bookmarked":null,"liked":null,"views":99,"like_count":3,"has_summary":false,"last_poster_username":"gweidart","category_id":11,"op_like_count":2,"pinned_globally":false,"featured_link":null,"has_accepted_answer":false,"can_vote":false,"posters":[{"extras":null,"description":"Original Poster","user_id":1864,"primary_group_id":null,"flair_group_id":null},{"extras":"latest","description":"Most Recent Poster","user_id":87679,"primary_group_id":null,"flair_group_id":null}]},{"fancy_title":"Tokenizer taking extremely long time to train","id":1875,"title":"Tokenizer taking extremely long time to train","slug":"tokenizer-taking-extremely-long-time-to-train","posts_count":2,"reply_count":0,"highest_post_number":2,"image_url":null,"created_at":"2020-11-03T16:29:20.356Z","last_posted_at":"2025-03-19T08:49:15.429Z","bumped":true,"bumped_at":"2025-03-19T08:49:15.429Z","archetype":"regular","unseen":false,"pinned":false,"unpinned":null,"visible":true,"closed":false,"archived":false,"bookmarked":null,"liked":null,"views":1065,"like_count":3,"has_summary":false,"last_poster_username":"gweidart","category_id":11,"op_like_count":2,"pinned_globally":false,"featured_link":null,"has_accepted_answer":false,"can_vote":false,"posters":[{"extras":null,"description":"Original Poster","user_id":760,"primary_group_id":null,"flair_group_id":null},{"extras":"latest","description":"Most Recent Poster","user_id":87679,"primary_group_id":null,"flair_group_id":null}]},{"fancy_title":"Rs-bpe tokenizer [PyPI | Python] - Outperforms tiktoken & tokenizers","id":146410,"title":"Rs-bpe tokenizer [PyPI | Python] - Outperforms tiktoken & tokenizers","slug":"rs-bpe-tokenizer-pypi-python-outperforms-tiktoken-tokenizers","posts_count":1,"reply_count":0,"highest_post_number":1,"image_url":"https://us1.discourse-cdn.com/hellohellohello/optimized/3X/6/7/676892508822063f2aea934d7b6b8dfefb97ea18_2_1024x495.webp","created_at":"2025-03-19T08:43:24.777Z","last_posted_at":"2025-03-19T08:43:24.895Z","bumped":true,"bumped_at":"2025-03-19T08:43:24.895Z","archetype":"regular","unseen":false,"pinned":false,"unpinned":null,"visible":true,"closed":false,"archived":false,"bookmarked":null,"liked":null,"visibility_reason_id":1,"views":250,"like_count":0,"has_summary":false,"last_poster_username":"gweidart","category_id":11,"op_like_count":0,"pinned_globally":false,"featured_link":null,"has_accepted_answer":false,"can_vote":false,"posters":[{"extras":"latest","description":"Original Poster, Most Recent Poster","user_id":87679,"primary_group_id":null,"flair_group_id":null},{"extras":null,"description":"Recent Poster","user_id":-1,"primary_group_id":null,"flair_group_id":null}]},{"fancy_title":"Build failure with NoGIL Python on main","id":146168,"title":"Build failure with NoGIL Python on main","slug":"build-failure-with-nogil-python-on-main","posts_count":1,"reply_count":0,"highest_post_number":1,"image_url":null,"created_at":"2025-03-17T16:45:36.734Z","last_posted_at":"2025-03-17T16:45:36.789Z","bumped":true,"bumped_at":"2025-03-17T16:45:36.789Z","archetype":"regular","unseen":false,"pinned":false,"unpinned":null,"visible":true,"closed":false,"archived":false,"bookmarked":null,"liked":null,"views":62,"like_count":1,"has_summary":false,"last_poster_username":"vinayakdev","category_id":11,"op_like_count":1,"pinned_globally":false,"featured_link":null,"has_accepted_answer":false,"can_vote":false,"posters":[{"extras":"latest single","description":"Original Poster, Most Recent Poster","user_id":87482,"primary_group_id":null,"flair_group_id":null}]},{"fancy_title":"SentencePiece to Tokenizers conversion","id":145756,"title":"SentencePiece to Tokenizers conversion","slug":"sentencepiece-to-tokenizers-conversion","posts_count":1,"reply_count":0,"highest_post_number":1,"image_url":"https://us1.discourse-cdn.com/hellohellohello/optimized/3X/b/e/be31d21f02a0592b0d6c917591b117b34dd23532_2_1024x466.png","created_at":"2025-03-14T10:18:40.968Z","last_posted_at":"2025-03-14T10:18:41.032Z","bumped":true,"bumped_at":"2025-03-14T10:18:41.032Z","archetype":"regular","unseen":false,"pinned":false,"unpinned":null,"visible":true,"closed":false,"archived":false,"bookmarked":null,"liked":null,"views":312,"like_count":2,"has_summary":false,"last_poster_username":"aaaksenova","category_id":11,"op_like_count":2,"pinned_globally":false,"featured_link":null,"has_accepted_answer":false,"can_vote":false,"posters":[{"extras":"latest single","description":"Original Poster, Most Recent Poster","user_id":87173,"primary_group_id":null,"flair_group_id":null}]},{"fancy_title":"Decode token IDs into a list (not a single string)","id":42991,"title":"Decode token IDs into a list (not a single string)","slug":"decode-token-ids-into-a-list-not-a-single-string","posts_count":5,"reply_count":2,"highest_post_number":5,"image_url":null,"created_at":"2023-06-12T22:58:16.552Z","last_posted_at":"2025-03-11T20:53:56.448Z","bumped":true,"bumped_at":"2025-03-11T20:53:56.448Z","archetype":"regular","unseen":false,"pinned":false,"unpinned":null,"visible":true,"closed":false,"archived":false,"bookmarked":null,"liked":null,"views":5092,"like_count":5,"has_summary":false,"last_poster_username":"lone17","category_id":11,"op_like_count":1,"pinned_globally":false,"featured_link":null,"has_accepted_answer":true,"can_vote":false,"posters":[{"extras":null,"description":"Original Poster, Accepted Answer","user_id":21384,"primary_group_id":null,"flair_group_id":null},{"extras":null,"description":"Recent Poster","user_id":7005,"primary_group_id":null,"flair_group_id":null},{"extras":"latest","description":"Most Recent Poster","user_id":86817,"primary_group_id":null,"flair_group_id":null}]},{"fancy_title":"Adapter-aware chat_template","id":136535,"title":"Adapter-aware chat_template","slug":"adapter-aware-chat-template","posts_count":4,"reply_count":1,"highest_post_number":4,"image_url":null,"created_at":"2025-01-20T13:39:47.132Z","last_posted_at":"2025-02-21T11:51:54.900Z","bumped":true,"bumped_at":"2025-02-21T13:11:22.843Z","archetype":"regular","unseen":false,"pinned":false,"unpinned":null,"visible":true,"closed":false,"archived":false,"bookmarked":null,"liked":null,"views":234,"like_count":1,"has_summary":false,"last_poster_username":"Aurelien-Morgan","category_id":11,"op_like_count":0,"pinned_globally":false,"featured_link":null,"has_accepted_answer":false,"can_vote":false,"posters":[{"extras":"latest","description":"Original Poster, Most Recent Poster","user_id":48690,"primary_group_id":null,"flair_group_id":null},{"extras":null,"description":"Recent Poster","user_id":76958,"primary_group_id":null,"flair_group_id":null}]},{"fancy_title":"Fine-tuning whsiper on custom special tokens","id":141139,"title":"Fine-tuning whsiper on custom special tokens","slug":"fine-tuning-whsiper-on-custom-special-tokens","posts_count":1,"reply_count":0,"highest_post_number":1,"image_url":null,"created_at":"2025-02-16T18:00:44.837Z","last_posted_at":"2025-02-16T18:00:44.899Z","bumped":true,"bumped_at":"2025-02-16T18:00:44.899Z","archetype":"regular","unseen":false,"pinned":false,"unpinned":null,"visible":true,"closed":false,"archived":false,"bookmarked":null,"liked":null,"views":227,"like_count":1,"has_summary":false,"last_poster_username":"sleeping4cat","category_id":11,"op_like_count":1,"pinned_globally":false,"featured_link":null,"has_accepted_answer":false,"can_vote":false,"posters":[{"extras":"latest single","description":"Original Poster, Most Recent Poster","user_id":58120,"primary_group_id":null,"flair_group_id":null}]},{"fancy_title":"Inconsistent, mojibaked tokenization in some but not all Huggingface tokenizers","id":140439,"title":"Inconsistent, mojibaked tokenization in some but not all Huggingface tokenizers","slug":"inconsistent-mojibaked-tokenization-in-some-but-not-all-huggingface-tokenizers","posts_count":2,"reply_count":0,"highest_post_number":2,"image_url":null,"created_at":"2025-02-12T11:22:22.828Z","last_posted_at":"2025-02-12T13:41:13.510Z","bumped":true,"bumped_at":"2025-02-12T13:41:13.510Z","archetype":"regular","unseen":false,"pinned":false,"unpinned":null,"visible":true,"closed":false,"archived":false,"bookmarked":null,"liked":null,"views":86,"like_count":1,"has_summary":false,"last_poster_username":"John6666","category_id":11,"op_like_count":1,"pinned_globally":false,"featured_link":null,"has_accepted_answer":false,"can_vote":false,"posters":[{"extras":null,"description":"Original Poster","user_id":46798,"primary_group_id":null,"flair_group_id":null},{"extras":"latest","description":"Most Recent Poster","user_id":52272,"primary_group_id":null,"flair_group_id":null}]},{"fancy_title":"Errors with Tokenizers on Llama","id":132264,"title":"Errors with Tokenizers on Llama","slug":"errors-with-tokenizers-on-llama","posts_count":3,"reply_count":0,"highest_post_number":3,"image_url":null,"created_at":"2024-12-19T11:52:38.742Z","last_posted_at":"2025-02-08T17:11:47.832Z","bumped":true,"bumped_at":"2025-02-08T17:11:47.832Z","archetype":"regular","unseen":false,"pinned":false,"unpinned":null,"visible":true,"closed":false,"archived":false,"bookmarked":null,"liked":null,"views":453,"like_count":2,"has_summary":false,"last_poster_username":"Invicto69","category_id":11,"op_like_count":1,"pinned_globally":false,"featured_link":null,"has_accepted_answer":false,"can_vote":false,"posters":[{"extras":null,"description":"Original Poster","user_id":77485,"primary_group_id":null,"flair_group_id":null},{"extras":null,"description":"Recent Poster","user_id":52272,"primary_group_id":null,"flair_group_id":null},{"extras":"latest","description":"Most Recent Poster","user_id":82734,"primary_group_id":null,"flair_group_id":null}]},{"fancy_title":"Character level tokenizer with specific order","id":139150,"title":"Character level tokenizer with specific order","slug":"character-level-tokenizer-with-specific-order","posts_count":6,"reply_count":1,"highest_post_number":6,"image_url":null,"created_at":"2025-02-05T14:54:36.673Z","last_posted_at":"2025-02-07T16:49:09.310Z","bumped":true,"bumped_at":"2025-02-07T18:39:39.825Z","archetype":"regular","unseen":false,"pinned":false,"unpinned":null,"visible":true,"closed":false,"archived":false,"bookmarked":null,"liked":null,"views":211,"like_count":4,"has_summary":false,"last_poster_username":"samirchar","category_id":11,"op_like_count":1,"pinned_globally":false,"featured_link":null,"has_accepted_answer":false,"can_vote":false,"posters":[{"extras":"latest","description":"Original Poster, Most Recent Poster","user_id":80944,"primary_group_id":null,"flair_group_id":null},{"extras":null,"description":"Recent Poster","user_id":52272,"primary_group_id":null,"flair_group_id":null}]},{"fancy_title":"Custom Tokenizer Error - Please Help!","id":139539,"title":"Custom Tokenizer Error - Please Help!","slug":"custom-tokenizer-error-please-help","posts_count":1,"reply_count":0,"highest_post_number":1,"image_url":"https://us1.discourse-cdn.com/hellohellohello/optimized/3X/e/9/e95dde95bd50bdc498ecadc238c96574c65fe3f8_2_1024x120.png","created_at":"2025-02-07T16:08:23.114Z","last_posted_at":"2025-02-07T16:08:23.190Z","bumped":true,"bumped_at":"2025-02-07T16:08:23.190Z","archetype":"regular","unseen":false,"pinned":false,"unpinned":null,"visible":true,"closed":false,"archived":false,"bookmarked":null,"liked":null,"views":71,"like_count":0,"has_summary":false,"last_poster_username":"samirchar","category_id":11,"op_like_count":0,"pinned_globally":false,"featured_link":null,"has_accepted_answer":false,"can_vote":false,"posters":[{"extras":"latest single","description":"Original Poster, Most Recent Poster","user_id":80944,"primary_group_id":null,"flair_group_id":null}]}]}}