Reproduce a detokenization error observed when using vLLM for inference

20250520_vllm_detokenize_issue.py

# Related to issue at https://github.com/vllm-project/vllm/issues/17448
import os
from vllm import LLM, SamplingParams
from vllm.inputs import TokensPrompt

# Get your Huging face access token at https://huggingface.co/settings/tokens
HF_TOKEN = os.getenv("HF_TOKEN")

llm = LLM(model="google/gemma-3-1b-it")
token_ids = [2, 236772, 35559, 236772, 107, 3048, 659, 496, 9550, 7820, 34799, 236761, 5180, 4209, 563, 531, 18840, 496, 2238, 1816, 2354, 236764, 14846, 7798, 531, 18698, 14050, 236764, 532, 8430, 784, 3629, 9994, 1534, 20880, 6950, 2351, 506, 1816, 236761, 30235, 22525, 580, 21558, 532, 22454, 1239, 9994, 236764, 2440, 4889, 1003, 506, 6638, 20880, 236761, 108, 236772, 59881, 236772, 107, 236770, 236761, 8847, 506, 2744, 1816, 13058, 236761, 49644, 14491, 529, 20880, 568, 6265, 236764, 3328, 236768, 600, 659, 808, 181670, 4596, 236829, 2721, 580, 506, 1938, 6212, 528, 506, 1816, 236761, 41152, 1800, 20880, 528, 496, 6727, 8386, 531, 886, 529, 506, 9155, 2165, 11128, 236779, 9846, 21233, 107, 236778, 236761, 1701, 1546, 8385, 4191, 236764, 8430, 506, 2269, 1938, 19753, 236787, 107, 140, 236772, 3738, 236787, 7343, 529, 506, 3738, 13186, 236764, 135351, 236761, 107, 140, 236772, 3738, 236779, 2084, 236787, 669, 1722, 236786, 10618, 529, 506, 3738, 13186, 236761, 20360, 577, 886, 529, 236787, 870, 113543, 4012, 7193, 236764, 43720, 236764, 69911, 236764, 119055, 236764, 216990, 236764, 211881, 1619, 107, 140, 236772, 3738, 236779, 7777, 236787, 562, 63510, 236764, 1265, 236772, 66436, 13315, 600, 8207, 23669, 1015, 236786, 14070, 506, 3738, 13186, 563, 236764, 2440, 1061, 2307, 13963, 236764, 3853, 236764, 532, 17141, 618, 6950, 528, 506, 1816, 236761, 669, 6492, 1374, 1975, 580, 1061, 1852, 2180, 20226, 4767, 4403, 236761, 107, 140, 236772, 3328, 236787, 7343, 529, 506, 3328, 13186, 236764, 135351, 236761, 107, 140, 236772, 3328, 236779, 2084, 236787, 669, 1722, 236786, 10618, 529, 506, 3328, 13186, 236761, 20360, 577, 886, 529, 236787, 870, 113543, 4012, 7193, 236764, 43720, 236764, 69911, 236764, 119055, 236764, 216990, 236764, 211881, 1619, 107, 140, 236772, 3328, 236779, 7777, 236787, 562, 63510, 236764, 1265, 236772, 66436, 13315, 600, 8207, 23669, 1015, 236786, 14070, 506, 3328, 13186, 563, 236764, 2440, 1061, 2307, 13963, 236764, 3853, 236764, 532, 17141, 618, 6950, 528, 506, 1816, 236761, 669, 6492, 1374, 1975, 580, 1061, 1852, 2180, 20226, 4767, 4403, 236761, 107, 140, 236772, 4191, 236787, 562, 3582, 6492, 26804, 808, 7843, 236829, 506, 3738, 532, 3328, 20880, 659, 4596, 3894, 531, 506, 1816, 236761, 107, 140, 236772, 4191, 236779, 38651, 236787, 1455, 11995, 6317, 1534, 236743, 236771, 532, 236743, 236770, 236771, 568, 83950, 779, 17674, 506, 6332, 653, 10805, 529, 506, 4191, 8385, 528, 506, 1816, 236761, 236743, 236771, 2820, 1401, 7209, 236786, 1571, 21375, 236764, 236743, 236770, 236771, 2820, 1401, 3188, 236786, 103617, 586, 8249, 236761, 108, 139, 12703, 29141, 236787, 107, 139, 236772, 44125, 236787, 623, 97421, 524, 528, 15649, 236743, 236778, 1781, 107, 139, 236772, 27026, 236787, 623, 236776, 9203, 7820, 29226, 600, 56017, 528, 22477, 6675, 532, 815, 1010, 5158, 528, 17792, 3879, 2338, 236743, 236778, 236771, 236770, 236810, 1781, 108, 236800, 236761, 36353, 808, 17136, 236829, 8385, 4191, 618, 496, 3161, 10434, 2495, 98469, 20976, 531, 506, 2269, 3904, 236787, 107, 236782, 107, 140, 236775, 6265, 1083, 19019, 6265, 236779, 11128, 236779, 1201, 67010, 107, 140, 236775, 6265, 236779, 2084, 1083, 19019, 6265, 236779, 11128, 236779, 2084, 67010, 107, 140, 236775, 6265, 236779, 7777, 1083, 19019, 6265, 236779, 11128, 236779, 7777, 67010, 107, 140, 236775, 5654, 1083, 19019, 5654, 236779, 11128, 236779, 1201, 67010, 107, 140, 236775, 5654, 236779, 2084, 1083, 19019, 5654, 236779, 11128, 236779, 2084, 67010, 107, 140, 236775, 5654, 236779, 7777, 1083, 19019, 5654, 236779, 11128, 236779, 7777, 67010, 107, 140, 236775, 77984, 1083, 19019, 77984, 236779, 7777, 67010, 107, 140, 236775, 77984, 236779, 38651, 1083, 655, 77984, 236779, 38651, 236779, 28018, 236813, 107, 236783, 108, 236812, 236761, 9657, 506, 1626, 3938, 528, 7830, 618, 496, 3161, 10434, 1694, 7906, 808, 8960, 236829, 506, 8385, 4191, 7751, 699, 14938, 236743, 236800, 236761, 41152, 506, 3938, 563, 496, 4341, 10434, 1694, 568, 37798, 607, 2165, 34104, 532, 10842, 607, 2165, 236842, 8347, 607, 7751, 15914, 684, 162760, 769, 1637, 951, 9994, 659, 1765, 236764, 994, 614, 7738, 1694, 2165, 3805, 21233, 108, 236810, 236761, 1637, 13959, 1131, 7830, 563, 3149, 236764, 17866, 808, 8960, 236829, 506, 3004, 529, 506, 6492, 6192, 20442, 6265, 236779, 7777, 8347, 2165, 5654, 236779, 7777, 8347, 2165, 77984, 59395, 3574, 711, 17866, 13186, 5618, 20442, 6265, 8347, 2165, 5654, 18833, 653, 4458, 20442, 6265, 236779, 2084, 8347, 2165, 5654, 236779, 2084, 59740, 653, 506, 3904, 529, 506, 10434, 236761, 108, 236772, 38408, 236772, 108, 12703, 236743, 236770, 236787, 107, 4661, 1816, 236787, 107, 236775, 4967, 118669, 569, 38433, 1008, 569, 110395, 44053, 568, 4502, 12622, 236768, 569, 537, 236789, 40740, 97239, 503, 236789, 598, 218494, 3190, 10053, 60699, 33779, 236764, 759, 7001, 18353, 638, 723, 54777, 569, 16086, 2731, 236761, 639, 236789, 160896, 569, 2372, 70540, 2456, 164325, 72168, 568, 210396, 236768, 569, 537, 236789, 4708, 236764, 44443, 3190, 696, 236743, 236778, 236771, 236770, 236832, 236764, 79717, 1608, 123610, 583, 759, 195425, 696, 50503, 569, 110395, 4154, 1640, 51588, 51718, 236761, 86832, 2534, 14567, 236764, 537, 236789, 2586, 1536, 4102, 513, 236789, 8328, 759, 98978, 2916, 31308, 569, 759, 110395, 19399, 97239, 236764, 759, 7001, 45849, 2369, 15154, 161449, 537, 236789, 184577, 9966, 696, 73573, 638, 2637, 3037, 2915, 11754, 19228, 703, 72124, 2026, 97239, 1781, 108, 46456, 3938, 236787, 107, 236840, 107, 138, 236782, 107, 140, 236775, 6265, 1083, 623, 40740, 18153, 236859, 17087, 827, 107, 140, 236775, 6265, 236779, 2084, 1083, 623, 136955, 56731, 827, 107, 140, 236775, 6265, 236779, 7777, 1083, 623, 40740, 48502, 1008, 87273, 569, 236743, 236778, 236832, 51588, 51718, 236764, 13548, 3190, 513, 236789, 2253, 118669, 569, 38433, 1008, 569, 110395, 44053, 568, 4502, 12622, 22730, 107, 140, 236775, 5654, 1083, 623, 210396, 827, 107, 140, 236775, 5654, 236779, 2084, 1083, 623, 203292, 827, 107, 140, 236775, 5654, 236779, 7777, 1083, 623, 6654, 70540, 2456, 164325, 72168, 236764, 3037, 17358, 569, 537, 236789, 4708, 44443, 3190, 696, 236743, 236778, 236771, 236770, 236832, 2637, 123610, 583, 759, 195425, 696, 50503, 569, 110395, 4154, 1640, 51588, 51718, 827, 107, 140, 236775, 77984, 1083, 623, 236798, 236789, 4708, 496, 102644, 759, 112184, 2880, 674, 42432, 569, 890, 118669, 569, 38433, 1008, 569, 110395, 44053, 827, 107, 140, 236775, 77984, 236779, 38651, 1083, 236743, 236819, 107, 138, 1263, 107, 138, 236782, 107, 140, 236775, 6265, 1083, 623, 31756, 827, 107, 140, 236775, 6265, 236779, 2084, 1083, 623, 136955, 56731, 827, 107, 140, 236775, 6265, 236779, 7777, 1083, 623, 194776, 139592, 221581, 18353, 638, 723, 54777, 569, 16086, 2731, 2880, 1640, 19898, 569, 110395, 569, 537, 236789, 4708, 1008, 73573, 638, 2637, 537, 236789, 1474, 142378, 72124, 2026, 827, 107, 140, 236775, 5654, 1083, 623, 40740, 18153, 236859, 17087, 827, 107, 140, 236775, 5654, 236779, 2084, 1083, 623, 136955, 56731, 827, 107, 140, 236775, 5654, 236779, 7777, 1083, 623, 40740, 48502, 1008, 87273, 569, 236743, 236778, 236832, 51588, 51718, 236764, 13548, 3190, 513, 236789, 2253, 118669, 569, 38433, 1008, 569, 110395, 44053, 568, 4502, 12622, 22730, 107, 140, 236775, 77984, 1083, 623, 4967, 7001, 97778, 723, 54777, 569, 16086, 2731, 2880, 1640, 19898, 569, 110395, 569, 537, 236789, 4708, 827, 107, 140, 236775, 77984, 236779, 38651, 1083, 236743, 236828, 107, 138, 1263, 107, 138, 236782, 107, 140, 236775, 6265, 1083, 623, 31756, 827, 107, 140, 236775, 6265, 236779, 2084, 1083, 623, 136955, 56731, 827, 107, 140, 236775, 6265, 236779, 7777, 1083, 623, 194776, 139592, 221581, 18353, 638, 723, 54777, 569, 16086, 2731, 2880, 1640, 19898, 569, 110395, 569, 537, 236789, 4708, 1008, 73573, 638, 2637, 537, 236789, 1474, 142378, 72124, 2026, 827, 107, 140, 236775, 5654, 1083, 623, 2586, 1536, 827, 107, 140, 236775, 5654, 236779, 2084, 1083, 623, 136955, 56731, 827, 107, 140, 236775, 5654, 236779, 7777, 1083, 623, 151374, 1747, 145112, 569, 537, 236789, 149878, 2026, 30709, 236764, 3037, 37734, 109134, 2947, 63253, 569, 98978, 2916, 31308, 1608, 759, 110395, 19399, 97239, 827, 107, 140, 236775, 77984, 1083, 623, 4967, 7001, 77819, 970, 2369, 15154, 161449, 537, 236789, 2586, 1536, 9966, 696, 73573, 638, 2637, 537, 236789, 1474, 142378, 72124, 2026, 97239, 827, 107, 140, 236775, 77984, 236779, 38651, 1083, 236743, 236832, 107, 138, 1263, 107, 138, 236782, 107, 140, 236775, 6265, 1083, 623, 2586, 1536, 827, 107, 140, 236775, 6265, 236779, 2084, 1083, 623, 136955, 56731, 827, 107, 140, 236775, 6265, 236779, 7777, 1083, 623, 151374, 1747, 145112, 569, 537, 236789, 149878, 2026, 30709, 236764, 3037, 37734, 109134, 2947, 63253, 569, 98978, 2916, 31308, 1608, 759, 110395, 19399, 97239, 827, 107, 140, 236775, 5654, 1083, 623, 40740, 18153, 236859, 17087, 827, 107, 140, 236775, 5654, 236779, 2084, 1083, 623, 136955, 56731, 827, 107, 140, 236775, 5654, 236779, 7777, 1083, 623, 40740, 48502, 1008, 87273, 569, 236743, 236778, 236832, 51588, 51718, 236764, 13548, 3190, 513, 236789, 2253, 118669, 569, 38433, 1008, 569, 110395, 44053, 568, 4502, 12622, 22730, 107, 140, 236775, 77984, 1083, 623, 236798, 236789, 2586, 1536, 63253, 569, 98978, 2916, 31308, 1608, 759, 110395, 19399, 97239, 236764, 100274, 638, 1640, 7880, 569, 110395, 569, 537, 236789, 4708, 827, 107, 140, 236775, 77984, 236779, 38651, 1083, 236743, 236828, 107, 138, 1263, 107, 138, 236782, 107, 140, 236775, 6265, 1083, 623, 31756, 827, 107, 140, 236775, 6265, 236779, 2084, 1083, 623, 136955, 56731, 827, 107, 140, 236775, 6265, 236779, 7777, 1083, 623, 194776, 139592, 221581, 18353, 638, 723, 54777, 569, 16086, 2731, 2880, 1640, 19898, 569, 110395, 569, 537, 236789, 4708, 1008, 73573, 638, 2637, 537, 236789, 1474, 142378, 72124, 2026, 827, 107, 140, 236775, 5654, 1083, 623, 3713, 142378, 72124, 2026, 97239, 827, 107, 140, 236775, 5654, 236779, 2084, 1083, 623, 203292, 827, 107, 140, 236775, 5654, 236779, 7777, 1083, 623, 32258, 180268, 944, 759, 7001, 159194, 1608, 152187, 30049, 537, 236789, 724, 93300, 831, 569, 537, 236789, 26179, 696, 50503, 569, 110395, 1008, 569, 38433, 827, 107, 140, 236775, 77984, 1083, 623, 4967, 7001, 15587, 754, 2637, 3037, 2915, 11754, 19228, 703, 72124, 2026, 97239, 827, 107, 140, 236775, 77984, 236779, 38651, 1083, 236743, 236819, 107, 138, 1263, 107, 138, 236782, 107, 140, 236775, 6265, 1083, 623, 34630, 13601, 569, 759, 79931, 236780, 827, 107, 140, 236775, 6265, 236779, 2084, 1083, 623, 54779, 827, 107, 140, 236775, 6265, 236779, 7777, 1083, 623, 11242, 605, 569, 9019, 13601, 569, 759, 118669, 569, 38433, 1008, 569, 110395, 44053, 569, 537, 236789, 4708, 10053, 60699, 33779, 827, 107, 140, 236775, 5654, 1083, 623, 40740, 18153, 236859, 17087, 827, 107, 140, 236775, 5654, 236779, 2084, 1083, 623, 136955, 56731, 827, 107, 140, 236775, 5654, 236779, 7777, 1083, 623, 40740, 48502, 1008, 87273, 569, 236743, 236778, 236832, 51588, 51718, 236764, 13548, 3190, 513, 236789, 2253, 118669, 569, 38433, 1008, 569, 110395, 44053, 568, 4502, 12622, 22730, 107, 140, 236775, 77984, 1083, 623, 11242, 605, 696, 28817, 3229, 12590, 569, 537, 236789, 4708, 827, 107, 140, 236775, 77984, 236779, 38651, 1083, 236743, 236770, 236771, 107, 138, 1263, 107, 138, 236782, 107, 140, 236775, 6265, 1083, 623, 236798, 16296, 569, 759, 112184, 827, 107, 140, 236775, 6265, 236779, 2084, 1083, 623, 54779, 827, 107, 140, 236775, 6265, 236779, 7777, 1083, 623, 237642, 140462, 1155, 2556, 13142, 674, 37271, 569, 537, 236789, 160896, 569, 2372, 70540, 2456, 164325, 72168, 569, 537, 236789, 4708, 696, 236743, 236778, 236771, 236770, 236832, 827, 107, 140, 236775, 5654, 1083, 623, 210396, 827, 107, 140, 236775, 5654, 236779, 2084, 1083, 623, 203292, 827, 107, 140, 236775, 5654, 236779, 7777, 1083, 623, 6654, 70540, 2456, 164325, 72168, 236764, 3037, 17358, 569, 537, 236789, 4708, 44443, 3190, 696, 236743, 236778, 236771, 236770, 236832, 2637, 123610, 583, 759, 195425, 696, 50503, 569, 110395, 4154, 1640, 51588, 51718, 827, 107, 140, 236775, 77984, 1083, 623, 237642, 140462, 1155, 2556, 13142, 674, 37271, 569, 8416, 17358, 827, 107, 140, 236775, 77984, 236779, 38651, 1083, 236743, 236770, 236771, 107, 138, 236783, 107, 236842, 108, 12703, 236743, 236778, 236787, 107, 4661, 1816, 236787, 107, 236775, 818, 4855, 12991, 11127, 815, 137992, 618, 640, 2567, 236747, 73853, 122580, 13120, 580, 6635, 12631, 236761, 799, 3072, 236764, 506, 2590, 532, 6322, 735, 12073, 29646, 2342, 640, 2567, 236747, 14989, 528, 70921, 532, 6697, 32454, 171683, 43112, 531, 3856, 49094, 8827, 236761, 669, 12160, 75583, 506, 14971, 21401, 862, 236772, 79572, 1641, 86196, 236764, 13832, 573, 4185, 5754, 236761, 26916, 32216, 532, 506, 44696, 1894, 5251, 7880, 2342, 506, 18337, 1580, 236764, 1651, 7001, 815, 29866, 506, 1685, 11870, 20410, 2370, 629, 531, 506, 4128, 236761, 669, 5371, 29933, 10928, 1003, 10969, 11646, 532, 506, 3435, 573, 3342, 96998, 236764, 2238, 16704, 236789, 236751, 1894, 573, 506, 18337, 1580, 532, 506, 3996, 4108, 529, 16953, 2302, 528, 506, 2433, 1781, 108, 46456, 3938, 236787, 107, 236840, 107, 138, 236782, 107, 140, 236775, 6265, 1083, 623, 236814, 2567, 236747, 128307, 827, 107, 140, 236775, 6265, 236779, 2084, 1083, 623, 136955, 56731, 827, 107, 140, 236775, 6265, 236779, 7777, 1083, 623, 236874, 18110, 236772, 5140, 49791, 2299, 207222, 13120, 580, 6635, 12631, 528, 506, 4855, 12991, 827, 107, 140, 236775, 5654, 1083, 623, 74253, 12631, 827, 107, 140, 236775, 5654, 236779, 2084, 1083, 623, 183462, 827, 107, 140, 236775, 5654, 236779, 7777, 1083, 623, 28061, 5754, 4588, 26751, 684, 640, 2567, 236747, 13120, 528, 506, 4855, 12991, 827, 107, 140, 236775, 77984, 1083, 623, 5466, 8003, 827, 107, 140, 236775, 77984, 236779, 38651, 1083, 236743, 236770, 236771, 107, 138, 1263, 107, 138, 236782, 107, 140, 236775, 6265, 1083, 623, 27211, 4184, 827, 107, 140, 236775, 6265, 236779, 2084, 1083, 623, 136955, 56731, 827, 107, 140, 236775, 6265, 236779, 7777, 1083, 623, 27969, 25479, 29646, 2342, 640, 2567, 236747, 14989, 532, 5830, 72878, 7880, 531, 3856, 49094, 8827, 827, 107, 140, 236775, 5654, 1083, 623, 236814, 2567, 236747, 128307, 827, 107, 140, 236775, 5654, 236779, 2084, 1083, 623, 136955, 56731, 827, 107, 140, 236775, 5654, 236779, 7777, 1083, 623, 236874, 18110, 236772, 5140, 49791, 2299, 207222, 13120, 580, 6635, 12631, 528, 506, 4855, 12991, 827, 107, 140, 236775, 77984, 1083, 623, 147935, 236751, 29646, 2342, 827, 107, 140, 236775, 77984, 236779, 38651, 1083, 236743, 236819, 107, 138, 1263, 107, 138, 236782, 107, 140, 236775, 6265, 1083, 623, 27211, 15305, 827, 107, 140, 236775, 6265, 236779, 2084, 1083, 623, 136955, 56731, 827, 107, 140, 236775, 6265, 236779, 7777, 1083, 623, 27969, 22765, 528, 29646, 2342, 640, 2567, 236747, 14989, 532, 11584, 49094, 5052, 7880, 827, 107, 140, 236775, 5654, 1083, 623, 236814, 2567, 236747, 128307, 827, 107, 140, 236775, 5654, 236779, 2084, 1083, 623, 136955, 56731, 827, 107, 140, 236775, 5654, 236779, 7777, 1083, 623, 236874, 18110, 236772, 5140, 49791, 2299, 207222, 13120, 580, 6635, 12631, 528, 506, 4855, 12991, 827, 107, 140, 236775, 77984, 1083, 623, 50473, 1090, 528, 29646, 2342, 827, 107, 140, 236775, 77984, 236779, 38651, 1083, 236743, 236828, 107, 138, 1263, 107, 138, 236782, 107, 140, 236775, 6265, 1083, 623, 16004, 171683, 43112, 827, 107, 140, 236775, 6265, 236779, 2084, 1083, 623, 203292, 827, 107, 140, 236775, 6265, 236779, 7777, 1083, 623, 13666, 495, 1353, 44704, 5585, 6697, 531, 3856, 49094, 8827, 528, 506, 4855, 12991, 827, 107, 140, 236775, 5654, 1083, 623, 74253, 12631, 827, 107, 140, 236775, 5654, 236779, 2084, 1083, 623, 183462, 827, 107, 140, 236775, 5654, 236779, 7777, 1083, 623, 28061, 5754, 4588, 26751, 684, 640, 2567, 236747, 13120, 528, 506, 4855, 12991, 827, 107, 140, 236775, 77984, 1083, 623, 25151, 101210, 827, 107, 140, 236775, 77984, 236779, 38651, 1083, 236743, 236819, 107, 138, 1263, 107, 138, 236782, 107, 140, 236775, 6265, 1083, 623, 133727, 32216, 827, 107, 140, 236775, 6265, 236779, 2084, 1083, 623, 136955, 56731, 827, 107, 140, 236775, 6265, 236779, 7777, 1083, 623, 27969, 11584, 5251, 7880, 2342, 640, 2567, 236747, 57698, 827, 107, 140, 236775, 5654, 1083, 623, 28061, 7880, 827, 107, 140, 236775, 5654, 236779, 2084, 1083, 623, 203292, 827, 107, 140, 236775, 5654, 236779, 7777, 1083, 623, 13666, 495, 1353, 7419, 531, 6651, 640, 2567, 236747, 57698, 528, 506, 4855, 12991, 4128, 827, 107, 140, 236775, 77984, 1083, 623, 195251, 827, 107, 140, 236775, 77984, 236779, 38651, 1083, 236743, 236828, 107, 138, 1263, 107, 138, 236782, 107, 140, 236775, 6265, 1083, 623, 31756, 827, 107, 140, 236775, 6265, 236779, 2084, 1083, 623, 136955, 56731, 827, 107, 140, 236775, 6265, 236779, 7777, 1083, 623, 27969, 76702, 44704, 10091, 531, 506, 4128, 531, 1894, 49094, 5052, 827, 107, 140, 236775, 5654, 1083, 623, 57643, 2370, 629, 827, 107, 140, 236775, 5654, 236779, 2084, 1083, 623, 113543, 4012, 7193, 827, 107, 140, 236775, 5654, 236779, 7777, 1083, 623, 42150, 1685, 11870, 29866, 531, 506, 4855, 12991, 4128, 827, 107, 140, 236775, 77984, 1083, 623, 3321, 675, 21334, 827, 107, 140, 236775, 77984, 236779, 38651, 1083, 236743, 236832, 107, 138, 236783, 107, 236842, 108, 236772, 20235, 5774, 236772, 107, 8826, 48279, 107, 11128, 236779, 9846, 236787, 85992, 236764, 43720, 236764, 69911, 236764, 119055, 236764, 216990, 236764, 211881, 107, 4661, 1816, 236787, 26390, 45337, 8179, 236764, 107, 3120, 498, 12848, 236764, 55827, 1608, 207220, 513, 236789, 115746, 1608, 890, 215817, 4758, 1925, 1117, 2789, 432, 398, 602, 31118, 569, 124866, 134772, 509, 19478, 1640, 33779, 236743, 236770, 236819, 236825, 236771, 432, 398, 237167, 107, 236761, 21745, 723, 107, 9692, 100020, 696, 236743, 236778, 236771, 236770, 236770, 236764, 537, 236789, 39421, 1843, 3990, 2806, 72799, 929, 84744, 853, 115693, 3229, 1300, 2972, 9181, 236755, 1875, 9298, 2789, 432, 398, 236752, 7407, 1008, 34040, 530, 3705, 432, 398, 11611, 759, 107, 185844, 39195, 61352, 1747, 11565, 508, 1009, 2789, 432, 398, 2253, 11754, 138767, 2637, 207220, 432, 398, 11611, 711, 638, 690, 236789, 236937, 537, 236789, 27368, 569, 759, 61352, 236764, 2789, 432, 398, 8630, 8461, 759, 3628, 236772, 4279, 27726, 236743, 236778, 236771, 236771, 236825, 236764, 580, 107, 236749, 236858, 236746, 2982, 130256, 723, 28622, 569, 32471, 25792, 236859, 3229, 11565, 508, 236772, 94063, 432, 398, 237167, 107, 8826, 48279, 107, 4606, 236787, 107]

# Create TokensPrompt input
prompt = TokensPrompt(prompt_token_ids=token_ids)

# Configure sampling parameters
sampling_params = SamplingParams(
    max_tokens=1024,
    temperature=0.2,
    top_p=0.4,
    top_k=64,
    n=1,
    presence_penalty=0.0,
    frequency_penalty=0.0,
    repetition_penalty=1.0
)

# Generate predictions
outputs = llm.generate(prompt, sampling_params)

# Print results
for output in outputs:
    print(f"Input tokens: {output.prompt_token_ids}")
    print(f"Generated: {output.outputs[0].text}")
    print(f"Generated len token IDs: {len(output.outputs[0].token_ids)}\n")
    print("-" * 50)