simple script to parse a .har file and spit out a CSV with a list of URLs,* for scraping or some quick and dirty web archival.

parse_har_1.py

# python 3.12.3+


# simple script to parse a .har file and spit out a CSV with a list of URLs,*
# for scraping or some quick and dirty web archival.
# (plug the output url list txt file into wget)

# *: Right now, it spits out all the URLs for which the response was bigger than 1 MiB, or was a "partial-content" response type.

# note: for PROPER archival of webpages, consider using something else, like wget,
# or just the wayback machine ( https://web.archive.org/save/ ).


# includes a bunch of code copied from my friend ethan's repo (with permission), 
# cuz i don't feel like writing a bunch of boilerplate to interact with json 👍
# 
# https://github.com/Eiim/NuTek-Tools
# NuTek-Tools is Copyright (c) 2026 Ethan Chapman,
# and is available under the MIT License.


# MIT License
# 
# Copyright (c) 2026 ChainSwordCS
# 
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# 
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
# 
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.


import json
import sys

if len(sys.argv) < 2:
	print(
	'USAGE:' + '\n' +
	'First argument: .har file')
	quit()

with open(sys.argv[1], "rb") as f:
	har_file = json.load(f)
	entries = har_file["log"]["entries"]

csv_filename = sys.argv[1]
if len(csv_filename) >= 5:
	l = len(csv_filename)
	if (csv_filename[l-5:l] == '.json'):
		csv_filename = csv_filename[0:l-5]
		l = len(csv_filename)
	if (csv_filename[l-4:l] == '.har'):
		csv_filename = csv_filename[0:l-4]
		l = len(csv_filename)
urllist_filename = csv_filename + '_out_urllist.txt'
csv_filename = csv_filename + '_out.csv'

csv_out = "url,status,size\n"
urllist_out = ""

print('test')
print(entries[0]["request"]["url"])

for e in entries:
	addToCsv = False
	status = 0
	bodySize = 0
	
	if "response" in e:
		if "bodySize" in e["response"]:
			bodySize = e["response"]["bodySize"]
			# if (approx.) file size is greater than 1 MiB
			# NOTE: This is here because, last I checked, in HAR files exported from Firefox,
			# for a given file (encoded in base64), if it's bigger than 1 MiB,
			# it will be truncated in the HAR file.
			if bodySize > 1048576:
				addToCsv = True
		if "status" in e["response"]:
			status = e["response"]["status"]
			if status == 206:
				addToCsv = True
			elif status == 0:
				if "time" not in e:
					print('\"time\" key not in entry for url ' + e["request"]["url"] + ' ???')
					quit()
				if e["time"] == 0:
					# Blocked (?)
					addToCsv = True
				else:
					# Cached version was loaded
					addToCsv = False
					print('[Info] cached(?): ' + e["request"]["url"])
					continue
			elif status == 404:
				addToCsv = False
				continue
			elif status != 200:
				print('[Info] URL returned unusual status ' + str(status) + ': ' + e["request"]["url"])
	if addToCsv:
		csv_out += e["request"]["url"] + "," + str(status) + "," + str(bodySize) + "\n"
		urllist_out += e["request"]["url"] + "\n"

with open(csv_filename, "w") as f:
	f.write(csv_out)
with open(urllist_filename, "w") as f:
	f.write(urllist_out)