Skip to content

Instantly share code, notes, and snippets.

@ducnh1022
Created August 1, 2024 04:28
Show Gist options
  • Save ducnh1022/028db59c5a9cbf3ce7a89ad9f07649bd to your computer and use it in GitHub Desktop.
Save ducnh1022/028db59c5a9cbf3ce7a89ad9f07649bd to your computer and use it in GitHub Desktop.
csvtobqdnh
import pandas as pd
from google.cloud import bigquery
# Set up BigQuery client
client = bigquery.Client()
# Function to create temporary table from CSV
def create_temp_table_from_csv(dataset_id, table_name, csv_path):
# Read CSV into pandas DataFrame
df = pd.read_csv(csv_path)
# Generate schema from DataFrame
schema = []
for column in df.columns:
dtype = df[column].dtype
if dtype == 'int64':
schema.append(f"{column} INT64")
elif dtype == 'float64':
schema.append(f"{column} FLOAT64")
elif dtype == 'bool':
schema.append(f"{column} BOOL")
else:
schema.append(f"{column} STRING")
# Create temporary table query
table_ref = client.dataset(dataset_id).table(table_name)
table = bigquery.Table(table_ref, schema=schema)
table.expires = 3600 # Set expiry time for the temporary table (optional)
client.create_table(table) # Create the temporary table
print(f"Temporary table {dataset_id}.{table_name} created successfully.")
# Example usage
if __name__ == "__main__":
dataset_id = "your_dataset" # Replace with your BigQuery dataset ID
table_name = "temp_table" # Replace with your desired temporary table name
csv_path = "path/to/your/file.csv" # Replace with the path to your CSV file
create_temp_table_from_csv(dataset_id, table_name, csv_path)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment