thomwolf · May 23, 2025 06:32 · oltip · May 16, 2019 · mandar1010 · May 20, 2019
diff --git a/persona-chat.py b/persona-chat.py
 import json
 from pytorch_pretrained_bert import cached_path

 url = "https://s3.amazonaws.com/datasets.huggingface.co/personachat/personachat_self_original.json"

 # Download and load JSON dataset
 personachat_file = cached_path(url)
 with open(personachat_file, "r", encoding="utf-8") as f:
    dataset = json.loads(f.read())

 # Tokenize and encode the dataset using our loaded GPT tokenizer
 def tokenize(obj):
    if isinstance(obj, str):
        return tokenizer.convert_tokens_to_ids(tokenizer.tokenize(obj))
    if isinstance(obj, dict):
        return dict((n, tokenize(o)) for n, o in obj.items())
    return list(tokenize(o) for o in obj)
 
 dataset = tokenize(dataset)
	import json
	from pytorch_pretrained_bert import cached_path

	url = "https://s3.amazonaws.com/datasets.huggingface.co/personachat/personachat_self_original.json"

	# Download and load JSON dataset
	personachat_file = cached_path(url)
	with open(personachat_file, "r", encoding="utf-8") as f:
	dataset = json.loads(f.read())

	# Tokenize and encode the dataset using our loaded GPT tokenizer
	def tokenize(obj):
	if isinstance(obj, str):
	return tokenizer.convert_tokens_to_ids(tokenizer.tokenize(obj))
	if isinstance(obj, dict):
	return dict((n, tokenize(o)) for n, o in obj.items())
	return list(tokenize(o) for o in obj)

	dataset = tokenize(dataset)