Skip to content

Instantly share code, notes, and snippets.

@lucascheung
Created April 6, 2020 11:40
Show Gist options
  • Save lucascheung/f4918ce836adcf3e4443db0ead9a29c5 to your computer and use it in GitHub Desktop.
Save lucascheung/f4918ce836adcf3e4443db0ead9a29c5 to your computer and use it in GitHub Desktop.
pandas dataframe to various output
from bigquery_schema_generator.generate_schema import SchemaGenerator
import json
from datetime import datetime
import gspread
from oauth2client.service_account import ServiceAccountCredentials
from df2gspread import df2gspread as d2g
def report_output_helper(report_name, df, output):
report_name = report_name.lower().replace(" ", "")
date = datetime.today().strftime('%Y%m%d')
if output == 'pandas':
return df
elif output == 'csv':
filename = f'{report_name}.csv'
df.to_csv(filename)
elif output == 'json':
return df.to_json(orient='records')
elif output == 'json-file':
filename = f'{report_name}_{date}.json'
return df.to_json(filename, orient='records')
elif output == 'json-bq':
filename = f'{report_name}_bq_{date}.json'
df.to_json(filename, orient='records', lines=True)
#Create BQ schema
generator = SchemaGenerator(keep_nulls=True)
input_file = open(filename, 'rb')
output_file = open(f'{report_name}_bq_schema_{date}.json', 'w')
generator.run(input_file, output_file)
else:
return df
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment