Lab Meeting Dataset upload Code

upload_csv.py

# See https://huggingface.co/docs/datasets/upload_dataset for more details

from datasets import load_dataset

dataset_name = "PUT_YOUR_NAME_HERE"
data_files = {"train": "train.csv", "dev": "dev.csv", "test": "test.csv"}
dataset = load_dataset("namespace/your_dataset_name", data_files=data_files)
datasets.push_to_hub(f"SALT-NLP/{dataset_name}", private=True)

upload_dataset_processed.py

from datasets import load_dataset

dataset_name = "PUT_YOUR_NAME_HERE"
transform = #PUT SOME PREPROCESSING FUNCTION HERE

raw_datasets = load_dataset(
  "glue",
  data_args.task_name,
  cache_dir=cache_name,
  use_auth_token=True if model_args.use_auth_token else None,
)

dialect_datasets = raw_datasets.map(
  transform,
  batched=True,
  num_proc=24,
  desc="Transform Dataset",
)

datasets.push_to_hub(f"SALT-NLP/{dataset_name}", private=True)