mex-atividade-4.1.py

from transformers import AutoTokenizer

bert_tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")
gpt2_tokenizer = AutoTokenizer.from_pretrained("gpt2")

sentences = [
    "Today is a beautiful day.",
    "Today is a beatiful day.",
    "That beat is so dope!"
]

for s in sentences:
    print(f"Frase: {s}")
    print("BERT:", bert_tokenizer.tokenize(s))
    print("GPT-2:", gpt2_tokenizer.tokenize(s))
    print("-" * 40)