TheLinuxGuy · April 5, 2025 22:10 · JGLord · Apr 5, 2025
diff --git a/thelinuxguy-teslafi-data-exporter.py b/thelinuxguy-teslafi-data-exporter.py
 # Author: Giovanni Mazzeo (github.com/thelinuxguy)
 # Script fetches your TeslaFi.com user data to allow importing into TeslaMate.

 # Updated 04/05/2025 to include more fields to normalize based on comments in gist.

 # My script fixes a couple bugs and issues seen by other people running the older script:
 # 1) "Invalid CSV delimiter" issue: https://github.com/teslamate-org/teslamate/issues/4569
 # 2) "battery_level" column integeter data change in 2024. https://github.com/teslamate-org/teslamate/issues/4477

 # You can thank me by buying me a coffee :) 
 # https://buymeacoffee.com/thelinuxguy

 import requests
 import csv
 from io import StringIO
 from lxml.html import fromstring

 username = 'username'
 password = 'password'
 years = [2020, 2021, 2022, 2023, 2024, 2025] # array of years you want to export
 months = [1,2,3,4,5,6,7,8,9,10,11,12] # I assume all the months, up to you
 cookie = ''

 # Set the proper delimiter that validate_csv.py expects
 CSV_DELIMITER = ',' # Change this if your validator expects a different delimiter

 def login():
    url = "https://teslafi.com/userlogin.php"
    response = requests.request("GET", url, headers={}, data={})
    
    cookies = ""
    for key in response.cookies.keys():
        this_cookie = key + "=" + response.cookies.get(key)
        if cookies == "":
            cookies = this_cookie
        else:
            cookies += "; " + this_cookie
    
    token = fromstring(response.text).forms[0].fields['token']
    global cookie
    cookie = cookies
    payload = {'username': username,'password': password,'remember': '1','submit': 'Login','token': token}
    headers = {"Cookie": cookies}
    l = requests.request("POST", url, headers=headers, data=payload)
    return True

 def getdata(m,y):
    url = "https://teslafi.com/exportMonth.php"
    headers = {'Content-Type': 'application/x-www-form-urlencoded','Cookie': cookie}
    response = requests.request("POST", url, headers=headers, data=pl(m,y))
    return response

 def detect_delimiter(text):
    """Detects the most likely delimiter in the CSV data"""
    if not text or '\n' not in text:
        return ','
        
    # Sample the first line to detect delimiter
    first_line = text.split('\n', 1)[0]
    delimiters = [(',', first_line.count(',')), 
                 (';', first_line.count(';')), 
                 ('\t', first_line.count('\t'))]
    
    # Sort by frequency, highest first
    delimiters.sort(key=lambda x: x[1], reverse=True)
    
    # Return the most common delimiter, or comma if none found
    return delimiters[0][0] if delimiters[0][1] > 0 else ','

 def normalize_field(rows, header, field_name):
    """
    Normalize field values to integers without decimal points:
    - Always convert to integer representation
    - If decimal part < 0.50, round down 
    - If decimal part >= 0.50, round up
    
    Args:
        rows: List of CSV rows (lists)
        header: List of column names
        field_name: Name of the field to normalize
        
    Returns:
        Tuple of (modified_rows, normalization_count)
    """
    # Find the index of the specified field column
    try:
        field_index = header.index(field_name)
    except ValueError:
        # If field column doesn't exist, return original rows
        return rows, 0
    
    normalization_count = 0
    
    # Iterate through all rows
    for i, row in enumerate(rows):
        # Skip if row is too short or field is empty
        if len(row) <= field_index or not row[field_index].strip():
            continue
            
        try:
            # Try to convert the field to a float
            value = float(row[field_index])
            
            # Get the integer value (either rounded up or down based on decimal part)
            if value - int(value) < 0.5:
                new_value = int(value)  # Round down
            else:
                new_value = int(value) + 1  # Round up
                
            # Convert to string representation of integer
            new_value_str = str(new_value)
            
            # Only count as normalization if we actually changed the value
            if row[field_index] != new_value_str:
                row[field_index] = new_value_str
                normalization_count += 1
            
        except (ValueError, TypeError):
            # Skip if conversion fails
            continue
    
    return rows, normalization_count

 def normalize_battery_level(rows, header):
    """
    Normalize battery_level values to integers without decimal points.
    This is a wrapper around normalize_field for backward compatibility.
    
    Args:
        rows: List of CSV rows (lists)
        header: List of column names
        
    Returns:
        Tuple of (modified_rows, normalization_count)
    """
    return normalize_field(rows, header, 'battery_level')

 def savefile(response, m, y):
    try:
        # Detect what delimiter the API is using
        input_delimiter = detect_delimiter(response.text)
        
        # Read the CSV data with the detected delimiter
        csv_data = StringIO(response.text)
        reader = csv.reader(csv_data, delimiter=input_delimiter)
        rows = list(reader)
        
        # Extract the header and data rows
        if not rows:
            print(f"Skipped creating {fname(m,y)} for year {y} and month number {m} due to lack of data from TeslaFi.")
            return
            
        header = rows[0]
        data_rows = rows[1:]
        
        # Check if there are any data rows
        if not data_rows:
            print(f"Skipped creating {fname(m,y)} for year {y} and month number {m} due to lack of data from TeslaFi.")
            return
        
        # Normalize fields
        fields_to_normalize = ['battery_level', 'charger_actual_current', 'charger_voltage']
        for field in fields_to_normalize:
            data_rows, normalization_count = normalize_field(data_rows, header, field)
            if normalization_count > 0:
                print(f"Detected `{field}` column malformed, {normalization_count} rows of data have been autocorrected")
        
        # Write the standardized CSV with the correct delimiter
        with open(fname(m,y), "w", newline='', encoding='utf-8') as file:
            writer = csv.writer(file, delimiter=CSV_DELIMITER, quoting=csv.QUOTE_MINIMAL)
            writer.writerow(header)
            writer.writerows(data_rows)
        
        print(f"Saved: {fname(m,y)}")
        
    except Exception as e:
        print(f"Error processing CSV: {str(e)}")
    return 

 def fname(m,y):
    return("TeslaFi" + str(m) + str(y) + ".csv")

 def pl(m,y):
    url = 'https://teslafi.com/export2.php'
    response = requests.request("GET", url, headers={"Cookie": cookie})
    magic = fromstring(response.text).forms[0].fields['__csrf_magic']
    return('__csrf_magic=' + magic + '&Month=' + str(m) + '&Year=' + str(y))

 def go():
    login()
    for year in years:
        for month in months:
            print(f"Processing: {month}/{year}")
            d = getdata(month, year)
            savefile(d, month, year)

 go()
	# Author: Giovanni Mazzeo (github.com/thelinuxguy)
	# Script fetches your TeslaFi.com user data to allow importing into TeslaMate.

	# Updated 04/05/2025 to include more fields to normalize based on comments in gist.

	# My script fixes a couple bugs and issues seen by other people running the older script:
	# 1) "Invalid CSV delimiter" issue: https://github.com/teslamate-org/teslamate/issues/4569
	# 2) "battery_level" column integeter data change in 2024. https://github.com/teslamate-org/teslamate/issues/4477

	# You can thank me by buying me a coffee :)
	# https://buymeacoffee.com/thelinuxguy

	import requests
	import csv
	from io import StringIO
	from lxml.html import fromstring

	username = 'username'
	password = 'password'
	years = [2020, 2021, 2022, 2023, 2024, 2025] # array of years you want to export
	months = [1,2,3,4,5,6,7,8,9,10,11,12] # I assume all the months, up to you
	cookie = ''

	# Set the proper delimiter that validate_csv.py expects
	CSV_DELIMITER = ',' # Change this if your validator expects a different delimiter

	def login():
	url = "https://teslafi.com/userlogin.php"
	response = requests.request("GET", url, headers={}, data={})

	cookies = ""
	for key in response.cookies.keys():
	this_cookie = key + "=" + response.cookies.get(key)
	if cookies == "":
	cookies = this_cookie
	else:
	cookies += "; " + this_cookie

	token = fromstring(response.text).forms[0].fields['token']
	global cookie
	cookie = cookies
	payload = {'username': username,'password': password,'remember': '1','submit': 'Login','token': token}
	headers = {"Cookie": cookies}
	l = requests.request("POST", url, headers=headers, data=payload)
	return True

	def getdata(m,y):
	url = "https://teslafi.com/exportMonth.php"
	headers = {'Content-Type': 'application/x-www-form-urlencoded','Cookie': cookie}
	response = requests.request("POST", url, headers=headers, data=pl(m,y))
	return response

	def detect_delimiter(text):
	"""Detects the most likely delimiter in the CSV data"""
	if not text or '\n' not in text:
	return ','

	# Sample the first line to detect delimiter
	first_line = text.split('\n', 1)[0]
	delimiters = [(',', first_line.count(',')),
	(';', first_line.count(';')),
	('\t', first_line.count('\t'))]

	# Sort by frequency, highest first
	delimiters.sort(key=lambda x: x[1], reverse=True)

	# Return the most common delimiter, or comma if none found
	return delimiters[0][0] if delimiters[0][1] > 0 else ','

	def normalize_field(rows, header, field_name):
	"""
	Normalize field values to integers without decimal points:
	- Always convert to integer representation
	- If decimal part < 0.50, round down
	- If decimal part >= 0.50, round up

	Args:
	rows: List of CSV rows (lists)
	header: List of column names
	field_name: Name of the field to normalize

	Returns:
	Tuple of (modified_rows, normalization_count)
	"""
	# Find the index of the specified field column
	try:
	field_index = header.index(field_name)
	except ValueError:
	# If field column doesn't exist, return original rows
	return rows, 0

	normalization_count = 0

	# Iterate through all rows
	for i, row in enumerate(rows):
	# Skip if row is too short or field is empty
	if len(row) <= field_index or not row[field_index].strip():
	continue

	try:
	# Try to convert the field to a float
	value = float(row[field_index])

	# Get the integer value (either rounded up or down based on decimal part)
	if value - int(value) < 0.5:
	new_value = int(value) # Round down
	else:
	new_value = int(value) + 1 # Round up

	# Convert to string representation of integer
	new_value_str = str(new_value)

	# Only count as normalization if we actually changed the value
	if row[field_index] != new_value_str:
	row[field_index] = new_value_str
	normalization_count += 1

	except (ValueError, TypeError):
	# Skip if conversion fails
	continue

	return rows, normalization_count

	def normalize_battery_level(rows, header):
	"""
	Normalize battery_level values to integers without decimal points.
	This is a wrapper around normalize_field for backward compatibility.

	Args:
	rows: List of CSV rows (lists)
	header: List of column names

	Returns:
	Tuple of (modified_rows, normalization_count)
	"""
	return normalize_field(rows, header, 'battery_level')

	def savefile(response, m, y):
	try:
	# Detect what delimiter the API is using
	input_delimiter = detect_delimiter(response.text)

	# Read the CSV data with the detected delimiter
	csv_data = StringIO(response.text)
	reader = csv.reader(csv_data, delimiter=input_delimiter)
	rows = list(reader)

	# Extract the header and data rows
	if not rows:
	print(f"Skipped creating {fname(m,y)} for year {y} and month number {m} due to lack of data from TeslaFi.")
	return

	header = rows[0]
	data_rows = rows[1:]

	# Check if there are any data rows
	if not data_rows:
	print(f"Skipped creating {fname(m,y)} for year {y} and month number {m} due to lack of data from TeslaFi.")
	return

	# Normalize fields
	fields_to_normalize = ['battery_level', 'charger_actual_current', 'charger_voltage']
	for field in fields_to_normalize:
	data_rows, normalization_count = normalize_field(data_rows, header, field)
	if normalization_count > 0:
	print(f"Detected `{field}` column malformed, {normalization_count} rows of data have been autocorrected")

	# Write the standardized CSV with the correct delimiter
	with open(fname(m,y), "w", newline='', encoding='utf-8') as file:
	writer = csv.writer(file, delimiter=CSV_DELIMITER, quoting=csv.QUOTE_MINIMAL)
	writer.writerow(header)
	writer.writerows(data_rows)

	print(f"Saved: {fname(m,y)}")

	except Exception as e:
	print(f"Error processing CSV: {str(e)}")
	return

	def fname(m,y):
	return("TeslaFi" + str(m) + str(y) + ".csv")

	def pl(m,y):
	url = 'https://teslafi.com/export2.php'
	response = requests.request("GET", url, headers={"Cookie": cookie})
	magic = fromstring(response.text).forms[0].fields['__csrf_magic']
	return('__csrf_magic=' + magic + '&Month=' + str(m) + '&Year=' + str(y))

	def go():
	login()
	for year in years:
	for month in months:
	print(f"Processing: {month}/{year}")
	d = getdata(month, year)
	savefile(d, month, year)

	go()