tokenizer.py

from nltk.tokenize import word_tokenize
from tokenize import tokenize
from io import BytesIO

rename_globals = {}

def tokenize_python(code):
    g = tokenize(BytesIO(code.encode('utf-8')).readline)
    try:
        tokens = [c[1] for c in g if c[1]!='' and c[1]!='\n'][1:]
    except:
        tokens = tokenize_if_block(code)
    
    clean_tokens = []
    
    for t in tokens:
        if ' ' in t:
            for rg in rename_globals:
                t = t.replace(rg, rename_globals[rg])
            clean_tokens += tokenize_if_block(t.replace('"', '').replace("'", ''))
        else:
            clean_tokens.append(t)
    
    return clean_tokens