Files
LLM-from-scratch/tokenizer_file/dolly_tokenizer_spm/tokenizer_en.vocab
T
2025-04-29 18:21:17 +08:00

1001 lines
9.6 KiB
Plaintext
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<unk> 0
<s> 0
</s> 0
<0x00> 0
<0x01> 0
<0x02> 0
<0x03> 0
<0x04> 0
<0x05> 0
<0x06> 0
<0x07> 0
<0x08> 0
<0x09> 0
<0x0A> 0
<0x0B> 0
<0x0C> 0
<0x0D> 0
<0x0E> 0
<0x0F> 0
<0x10> 0
<0x11> 0
<0x12> 0
<0x13> 0
<0x14> 0
<0x15> 0
<0x16> 0
<0x17> 0
<0x18> 0
<0x19> 0
<0x1A> 0
<0x1B> 0
<0x1C> 0
<0x1D> 0
<0x1E> 0
<0x1F> 0
<0x20> 0
<0x21> 0
<0x22> 0
<0x23> 0
<0x24> 0
<0x25> 0
<0x26> 0
<0x27> 0
<0x28> 0
<0x29> 0
<0x2A> 0
<0x2B> 0
<0x2C> 0
<0x2D> 0
<0x2E> 0
<0x2F> 0
<0x30> 0
<0x31> 0
<0x32> 0
<0x33> 0
<0x34> 0
<0x35> 0
<0x36> 0
<0x37> 0
<0x38> 0
<0x39> 0
<0x3A> 0
<0x3B> 0
<0x3C> 0
<0x3D> 0
<0x3E> 0
<0x3F> 0
<0x40> 0
<0x41> 0
<0x42> 0
<0x43> 0
<0x44> 0
<0x45> 0
<0x46> 0
<0x47> 0
<0x48> 0
<0x49> 0
<0x4A> 0
<0x4B> 0
<0x4C> 0
<0x4D> 0
<0x4E> 0
<0x4F> 0
<0x50> 0
<0x51> 0
<0x52> 0
<0x53> 0
<0x54> 0
<0x55> 0
<0x56> 0
<0x57> 0
<0x58> 0
<0x59> 0
<0x5A> 0
<0x5B> 0
<0x5C> 0
<0x5D> 0
<0x5E> 0
<0x5F> 0
<0x60> 0
<0x61> 0
<0x62> 0
<0x63> 0
<0x64> 0
<0x65> 0
<0x66> 0
<0x67> 0
<0x68> 0
<0x69> 0
<0x6A> 0
<0x6B> 0
<0x6C> 0
<0x6D> 0
<0x6E> 0
<0x6F> 0
<0x70> 0
<0x71> 0
<0x72> 0
<0x73> 0
<0x74> 0
<0x75> 0
<0x76> 0
<0x77> 0
<0x78> 0
<0x79> 0
<0x7A> 0
<0x7B> 0
<0x7C> 0
<0x7D> 0
<0x7E> 0
<0x7F> 0
<0x80> 0
<0x81> 0
<0x82> 0
<0x83> 0
<0x84> 0
<0x85> 0
<0x86> 0
<0x87> 0
<0x88> 0
<0x89> 0
<0x8A> 0
<0x8B> 0
<0x8C> 0
<0x8D> 0
<0x8E> 0
<0x8F> 0
<0x90> 0
<0x91> 0
<0x92> 0
<0x93> 0
<0x94> 0
<0x95> 0
<0x96> 0
<0x97> 0
<0x98> 0
<0x99> 0
<0x9A> 0
<0x9B> 0
<0x9C> 0
<0x9D> 0
<0x9E> 0
<0x9F> 0
<0xA0> 0
<0xA1> 0
<0xA2> 0
<0xA3> 0
<0xA4> 0
<0xA5> 0
<0xA6> 0
<0xA7> 0
<0xA8> 0
<0xA9> 0
<0xAA> 0
<0xAB> 0
<0xAC> 0
<0xAD> 0
<0xAE> 0
<0xAF> 0
<0xB0> 0
<0xB1> 0
<0xB2> 0
<0xB3> 0
<0xB4> 0
<0xB5> 0
<0xB6> 0
<0xB7> 0
<0xB8> 0
<0xB9> 0
<0xBA> 0
<0xBB> 0
<0xBC> 0
<0xBD> 0
<0xBE> 0
<0xBF> 0
<0xC0> 0
<0xC1> 0
<0xC2> 0
<0xC3> 0
<0xC4> 0
<0xC5> 0
<0xC6> 0
<0xC7> 0
<0xC8> 0
<0xC9> 0
<0xCA> 0
<0xCB> 0
<0xCC> 0
<0xCD> 0
<0xCE> 0
<0xCF> 0
<0xD0> 0
<0xD1> 0
<0xD2> 0
<0xD3> 0
<0xD4> 0
<0xD5> 0
<0xD6> 0
<0xD7> 0
<0xD8> 0
<0xD9> 0
<0xDA> 0
<0xDB> 0
<0xDC> 0
<0xDD> 0
<0xDE> 0
<0xDF> 0
<0xE0> 0
<0xE1> 0
<0xE2> 0
<0xE3> 0
<0xE4> 0
<0xE5> 0
<0xE6> 0
<0xE7> 0
<0xE8> 0
<0xE9> 0
<0xEA> 0
<0xEB> 0
<0xEC> 0
<0xED> 0
<0xEE> 0
<0xEF> 0
<0xF0> 0
<0xF1> 0
<0xF2> 0
<0xF3> 0
<0xF4> 0
<0xF5> 0
<0xF6> 0
<0xF7> 0
<0xF8> 0
<0xF9> 0
<0xFA> 0
<0xFB> 0
<0xFC> 0
<0xFD> 0
<0xFE> 0
<0xFF> 0
▁t -0
he -1
▁a -2
▁the -3
in -4
▁o -5
re -6
▁s -7
▁w -8
er -9
on -10
en -11
at -12
ou -13
an -14
▁b -15
▁c -16
it -17
ed -18
▁of -19
ar -20
▁f -21
ing -22
is -23
▁m -24
▁h -25
es -26
ll -27
▁p -28
▁d -29
▁an -30
▁to -31
or -32
▁in -33
▁th -34
▁l -35
as -36
ic -37
▁n -38
ly -39
▁e -40
le -41
▁" -42
▁and -43
ro -44
▁be -45
ct -46
ion -47
st -48
▁re -49
▁g -50
ve -51
ent -52
ut -53
ad -54
gh -55
ot -56
id -57
ow -58
ce -59
▁it -60
om -61
se -62
ver -63
▁on -64
ur -65
ld -66
▁he -67
al -68
im -69
▁T -70
▁as -71
ri -72
un -73
ain -74
et -75
▁that -76
ir -77
▁M -78
▁A -79
▁B -80
▁at -81
▁we -82
▁wh -83
ra -84
ect -85
ould -86
ght -87
▁I -88
ith -89
ch -90
▁st -91
▁was -92
▁for -93
▁with -94
▁y -95
th -96
▁The -97
▁is -98
▁not -99
ay -100
ore -101
all -102
▁by -103
us -104
oun -105
▁_ -106
tain -107
▁C -108
ul -109
ap -110
!" -111
▁ex -112
▁con -113
▁P -114
her -115
if -116
ble -117
▁sh -118
▁you -119
ter -120
oon -121
il -122
▁his -123
ight -124
ican -125
▁had -126
arb -127
▁se -128
▁S -129
qu -130
ject -131
ation -132
▁su -133
arbican -134
ess -135
▁or -136
ich -137
ke -138
▁Barbican -139
," -140
ast -141
ill -142
▁ha -143
red -144
dan -145
▁Ar -146
▁de -147
▁they -148
▁ne -149
est -150
▁Ardan -151
ous -152
▁all -153
rom -154
▁v -155
ol -156
os -157
▁Moon -158
um -159
▁their -160
▁dis -161
am -162
ds -163
roject -164
ile -165
very -166
▁E -167
▁com -168
ck -169
▁from -170
▁have -171
ers -172
res -173
▁un -174
." -175
ate -176
▁Project -177
▁them -178
ity -179
-- -180
ant -181
ust -182
el -183
▁r -184
▁H -185
pp -186
▁al -187
ough -188
▁could -189
▁this -190
lo -191
and -192
▁but -193
▁so -194
▁W -195
▁our -196
▁sp -197
up -198
ven -199
art -200
our -201
▁en -202
▁pro -203
ge -204
▁were -205
out -206
▁fe -207
ak -208
▁li -209
▁no -210
▁which -211
?" -212
▁sa -213
▁would -214
▁Projectile -215
ound -216
ace -217
▁its -218
oo -219
▁than -220
sel -221
reat -222
der -223
▁ab -224
ther -225
▁me -226
▁wor -227
▁can -228
▁ob -229
aptain -230
▁qu -231
ome -232
▁more -233
▁F -234
oll -235
are -236
ong -237
▁Captain -238
ive -239
ions -240
here -241
able -242
ost -243
em -244
ard -245
arth -246
cc -247
▁been -248
ure -249
▁N -250
▁are -251
pl -252
▁her -253
ab -254
fore -255
▁k -256
▁Earth -257
hen -258
hat -259
▁tim -260
ide -261
itt -262
▁int -263
▁now -264
end -265
▁us -266
▁there -267
_, -268
▁great -269
act -270
▁one -271
ried -272
ally -273
▁sur -274
ort -275
▁comp -276
ull -277
▁like -278
ite -279
▁po -280
ind -281
ved -282
ist -283
ser -284
▁out -285
▁j -286
▁pl -287
▁very -288
▁im -289
▁O -290
▁G -291
ci -292
▁who -293
▁ch -294
ig -295
▁L -296
▁up -297
▁any -298
▁even -299
▁said -300
hed -301
▁ar -302
▁le -303
ence -304
▁only -305
▁him -306
op -307
▁1 -308
ked -309
int -310
▁do -311
ance -312
▁other -313
ac -314
ell -315
▁per -316
▁cried -317
ie -318
ber -319
ving -320
▁app -321
now -322
▁tra -323
▁time -324
▁gl -325
▁some -326
ass -327
ren -328
ated -329
hing -330
▁what -331
▁fri -332
ittle -333
▁cont -334
▁about -335
▁obser -336
ire -337
▁if -338
ible -339
self -340
ach -341
▁little -342
pt -343
Nich -344
Nicholl -345
ied -346
▁see -347
▁ra -348
▁must -349
▁pre -350
▁dist -351
ine -352
▁ag -353
unar -354
ag -355
per -356
▁D -357
▁surf -358
iles -359
▁long -360
▁when -361
▁surface -362
cl -363
▁such -364
▁point -365
▁tw -366
00 -367
ered -368
od -369
ary -370
▁every -371
ose -372
ves -373
▁In -374
▁It -375
▁man -376
▁But -377
oss -378
urn -379
▁tr -380
▁miles -381
vell -382
▁att -383
ud -384
ake -385
ame -386
ree -387
▁ad -388
ction -389
right -390
▁know -391
ily -392
▁my -393
▁still -394
ff -395
▁bl -396
ations -397
▁travell -398
ark -399
ertain -400
cul -401
▁part -402
ountain -403
▁should -404
▁ey -405
▁your -406
one -407
▁day -408
▁how -409
ful -410
▁over -411
con -412
▁into -413
▁pres -414
ans -415
▁travellers -416
esc -417
uch -418
wn -419
▁light -420
ear -421
ards -422
ater -423
▁most -424
au -425
ty -426
ep -427
▁too -428
▁ever -429
▁never -430
og -431
▁acc -432
▁off -433
▁rem -434
▁lunar -435
ious -436
ment -437
▁fir -438
▁these -439
less -440
▁reg -441
▁Th -442
▁gr -443
▁before -444
▁sl -445
▁str -446
▁mountain -447
ack -448
way -449
▁mom -450
▁disc -451
▁moment -452
ies -453
led -454
▁ter -455
▁co -456
▁say -457
▁work -458
▁being -459
▁inter -460
▁ro -461
ling -462
velo -463
ely -464
iff -465
▁cr -466
ident -467
▁last -468
▁much -469
▁under -470
▁few -471
▁has -472
▁rep -473
▁res -474
▁thr -475
▁cons -476
▁bec -477
▁inst -478
▁poss -479
ink -480
ood -481
ted -482
ne -483
▁Sun -484
▁loo -485
▁through -486
ron -487
▁asked -488
▁R -489
▁will -490
ared -491
ness -492
▁first -493
ense -494
ying -495
▁They -496
▁repl -497
av -498
ully -499
fter -500
ount -501
ress -502
▁two -503
phere -504
iz -505
ends -506
ating -507
▁form -508
▁soon -509
▁enough -510
ph -511
▁mo -512
uten -513
▁hour -514
▁read -515
▁certain -516
ub -517
ature -518
▁present -519
ever -520
▁dire -521
az -522
ys -523
icul -524
▁diff -525
ens -526
▁pe -527
arly -528
ided -529
▁cal -530
▁replied -531
ade -532
ang -533
▁fl -534
ical -535
▁far -536
wards -537
▁He -538
▁exp -539
▁hard -540
ced -541
▁eas -542
▁nor -543
▁though -544
we -545
▁2 -546
▁she -547
man -548
ning -549
rest -550
selves -551
▁where -552
▁contin -553
ice -554
ting -555
▁imp -556
▁observed -557
HE -558
ip -559
berg -560
▁bet -561
The -562
▁may -563
veloc -564
▁This -565
▁veloc -566
ably -567
ourse -568
▁friends -569
▁velocity -570
▁Y -571
ise -572
▁bot -573
▁min -574
▁ref -575
▁then -576
▁well -577
▁again -578
utenberg -579
ific -580
▁therefore -581
ish -582
rib -583
▁br -584
▁course -585
ry -586
ue -587
orm -588
▁after -589
▁secon -590
sw -591
▁perf -592
▁fall -593
▁just -594
▁quest -595
ath -596
ces -597
mos -598
read -599
ston -600
▁beg -601
aking -602
▁right -603
▁hardly -604
▁himself -605
ase -606
▁ke -607
▁sm -608
▁dep -609
atter -610
thing -611
▁eyes -612
▁once -613
▁atmos -614
igh -615
ord -616
eter -617
▁ext -618
▁care -619
--" -620
ick -621
▁dec -622
▁night -623
▁those -624
▁Gutenberg -625
▁Mar -626
▁occ -627
ory -628
▁THE -629
▁What -630
▁answ -631
▁start -632
▁compan -633
▁nothing -634
▁without -635
ft -636
age -637
ail -638
▁ -639
e -640
t -641
a -642
o -643
n -644
i -645
r -646
s -647
h -648
l -649
d -650
c -651
u -652
f -653
m -654
y -655
g -656
w -657
, -658
p -659
b -660
v -661
" -662
. -663
k -664
T -665
I -666
_ -667
A -668
M -669
' -670
! -671
B -672
- -673
x -674
E -675
S -676
C -677
j -678
N -679
P -680
H -681
? -682
W -683
q -684
; -685
O -686
L -687
F -688
0 -689
R -690
z -691
D -692
G -693
1 -694
Y -695
: -696
U -697
2 -698
5 -699
3 -700
V -701
4 -702
J -703
6 -704
8 -705
7 -706
( -707
) -708
X -709
K -710
° -711
9 -712
Q -713
[ -714
] -715
/ -716
{ -717
* -718
} -719
é -720
“ -721
” -722
Z -723
’ -724
ê -725
• -726
= -727
Æ -728
à -729
$ -730
+ -731
æ -732
è -733
ö -734
— -735
# -736
% -737
â -738
ô -739
‘ -740