Import Mixpanel events to Google BigQuery

.env.sample

MIXPANEL_AUTH=my.mp-service-account:MYTOKEN

import.sh

#!/bin/bash
set -e
source .env
today=$(date --utc -I)
echo $today
yesterday=$(date --utc -I -d "-1 day")
echo $yesterday
yesterday_timestamp=$(date --utc --date=$yesterday +%s)
echo $yesterday_timestamp
url="https://data.mixpanel.com/api/2.0/export?from_date=$yesterday&to_date=$today&project_id=2459223&where=properties[%22mp_country_code%22]%20%3D%3D%20%22AU%22"
echo $url
curl --fail --no-progress-meter --location --globoff $url -o events-raw.ndjson -u $MIXPANEL_AUTH
jq --compact-output -f query.jq --arg day $yesterday_timestamp events-raw.ndjson > events-fixed.ndjson
table="mixpanel_app_events.events\$${yesterday//-/}"
bq load --replace=true --source_format=NEWLINE_DELIMITED_JSON $table events-fixed.ndjson

query.jq

.+= { time:(.properties.time-36000), distinct_id:.properties.distinct_id, insert_id:.properties."$insert_id" } |
  del(.properties.time,.properties.distinct_id,.properties."$insert_id") |
  select(.time >= ($day|tonumber) and .time < ($day|tonumber+86400)) |
  .time |= todateiso8601