Tokenization

The Idea

Before a model sees text, the text is split into tokens: subword chunks from a fixed vocabulary learned from data. Frequent words become one token; rare names and odd spellings break into several pieces.

Real-World Example

A stenographer's shorthand does the same trade. Everyday words get a single stroke, while an unfamiliar surname has to be spelled out letter by letter — same page, very different effort.

The Code

vocab = ["token", "ization", "un", "believ", "able"]

def tokenize(text):
    out, i = [], 0
    while i < len(text):
        hits = [v for v in vocab if text.startswith(v, i)]
        if not hits:                       # no entry fits: emit one character
            out.append(text[i]); i += 1
            continue
        piece = max(hits, key=len)         # greedy longest match
        out.append(piece); i += len(piece)
    return out

print(tokenize("tokenization"))    # ['token', 'ization']
print(tokenize("unbelievable"))    # ['un', 'believ', 'able']

vocab = ["low", "est", "lowest"]
text = "lowest"
out, i = [], 0
while i < len(text):
  hits = [v for v in vocab if text.startswith(v, i)]
  piece = ___                  # take the longest matching entry
  out.append(piece)
  i += len(piece)

print(out)     # want ['lowest']