Feed checker script for IDing feeders with issues

feedcheck.py

#!/usr/bin/python3
# OARC ADS-B Feed Checker v4 - Mark 2M0IIG with additional help from Roger M7TEE
# This script reads in the readsb and mlat-server clients.json files and presents the information in a clearer manner
# to identify users with issues

import json
import argparse

colour_red = "\033[1m\033[91m***"
colour_yellow = "\033[1m\033[93m***"
colour_magenta = "\033[1m\033[95m***"
colour_green = "\033[1m\033[92m***"
colour_blue = "\033[1m\033[94m***"
colour_cyan = "\033[1m\033[96m***"
colour_darkcyan = "\033[1m\033[36m***"
colour_bold = "\033[1m"
colour_underline = "\033[4m"
colour_end = "\033[0m"


# This opens the two JSON files
def parseJSONfile():
    with open('/run/readsb/clients.json', 'r') as readsb_client_json:
        readsb_data = json.load(readsb_client_json)

    with open('/run/mlat-server/clients.json', 'r') as mlat_client_json:
        mlat_data = json.load(mlat_client_json)

    return readsb_data, mlat_data


# Set up some variables for later
readsb_data, mlat_data = parseJSONfile()

all_ip_addresses = []
readsb_ip_count = dict()
mlat_ip_count = dict()

readsb_ip_uuids = dict()
mlat_ip_uuids = dict()

mlat_ip_names = dict()


for readsb_entry in readsb_data['clients']:
    client_ip = readsb_entry[1].split("port", 1)[0].strip()
    all_ip_addresses.append(client_ip)
    # update count of readsb ip_count
    if client_ip in readsb_ip_count:
        readsb_ip_count[client_ip] += 1
    else:
        readsb_ip_count[client_ip] = 1

    uuid = readsb_entry[0]
    # add uuid to readsb_ip_uuids
    if client_ip in readsb_ip_uuids:
        readsb_ip_uuids[client_ip].append(uuid)
    else:
        readsb_ip_uuids[client_ip] = [uuid]


mlat_ip_msgs_per_second = dict()
mlat_ip_outlier_percentages = dict()
mlat_ip_peer_counts = dict()


for mlat_entry in mlat_data:
    client_ip = mlat_data[mlat_entry]['source_ip']

    all_ip_addresses.append(client_ip)
    if client_ip in mlat_ip_count:
        mlat_ip_count[client_ip] += 1
    else:
        mlat_ip_count[client_ip] = 1

    mlat_username = mlat_data[mlat_entry]['user']
    if client_ip in mlat_ip_names:
        mlat_ip_names[client_ip].append(mlat_username)
    else:
        mlat_ip_names[client_ip] = [mlat_username]

    uuid = mlat_data[mlat_entry]['uuid']
    if client_ip in mlat_ip_uuids:
        mlat_ip_uuids[client_ip].append(uuid)
    else:
        mlat_ip_uuids[client_ip] = [uuid]

    msgs_per_second = mlat_data[mlat_entry]['message_rate']
    if client_ip in mlat_ip_msgs_per_second:
        mlat_ip_msgs_per_second[client_ip].append(msgs_per_second)
    else:
        mlat_ip_msgs_per_second[client_ip] = [msgs_per_second]

    outlier_percentage = mlat_data[mlat_entry]['outlier_percent']
    if client_ip in mlat_ip_outlier_percentages:
        mlat_ip_outlier_percentages[client_ip].append(outlier_percentage)
    else:
        mlat_ip_outlier_percentages[client_ip] = [outlier_percentage]

    peer_count = mlat_data[mlat_entry]['peer_count']
    if client_ip in mlat_ip_peer_counts:
        mlat_ip_peer_counts[client_ip].append(peer_count)
    else:
        mlat_ip_peer_counts[client_ip] = [peer_count]


# get command line args
parser = argparse.ArgumentParser(description="OARC ADS-B Feed Checker", formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('-w', '--skip_working', action='store_true', help='Skip working feeders')
parser.add_argument('-u', '--show_uuid_breakdown', action='store_true', help='Show UUID breakdown')
args = parser.parse_args()

# remove duplicates
unique_ip_addresses = list(dict.fromkeys(all_ip_addresses))
# unique_ip_addresses.sort()

# sort the list

# print count of IP addresses
print("Total IP addresses: " + str(len(all_ip_addresses)))
print("Unique IP addresses: " + str(len(unique_ip_addresses)))

if(args.skip_working): print("\nSkipping working feeders")
if not(args.show_uuid_breakdown): print("\nSkipping UUID breakdown")
print("\n")

table_format = "{:3} {:16} {:<6} {:<6} {:<6} {:<6} {:<20} {:}"

num = 0

print(table_format.format("Num", "IP Address ", "READSB", "MLAT", "READSB", "MLAT", "MLAT", "Notes"))
print(table_format.format(" ", " ", "Count", "Count", "UUIDs", "UUIDs", "Username ", " "))
for ip in unique_ip_addresses:
    num += 1
    notes = ''

    readsb_count = readsb_ip_count.get(ip, 0)
    mlat_count = mlat_ip_count.get(ip, 0)

    if(readsb_count != mlat_count):
        if(len(notes) > 0):
            notes += ", "
        notes += "" + colour_red + " beast/mlat counts do not match ***\033[0m"

    if(mlat_count < readsb_count):
        if(len(notes) > 0):
            notes += ", "
        notes += "" + colour_red + " MLAT not enabled on at least one of these feeds ***\033[0m"

    readsb_uuids = readsb_ip_uuids.get(ip, [])
    mlat_uuids = mlat_ip_uuids.get(ip, [])

    mlat_usernames = mlat_ip_names.get(ip, ['NO NAME'])

    if(len(readsb_uuids) != len(mlat_uuids)):
        if(len(notes) > 0):
            notes += ", "
        notes += "" + colour_red + " readsb/mlat uuid counts do not match ***\033[0m"

    all_uuids = []
    all_uuids = all_uuids + readsb_uuids
    all_uuids = all_uuids + mlat_uuids
    # all_unique_uuids = list(dict.fromkeys(all_uuids))
    # alternative, which avoids TypeError: unhashable type: 'list'
    all_unique_uuids = []
    for uuid in all_uuids:
        if uuid not in all_unique_uuids:
            all_unique_uuids.append(uuid)

    if(len(all_unique_uuids) > 1):
        if(len(notes) > 0):
            notes += ", "
        notes += "" + colour_yellow + " multiple uuids found (" + str(len(all_unique_uuids)) + ") ***\033[0m"

    uuid_none_count = all_uuids.count(None)
    if(uuid_none_count > 0):
        if(len(notes) > 0):
            notes += ", "
        notes += "" + colour_yellow + " at least one entry of a null UUID (" + str(uuid_none_count) + ") ***\033[0m"

    mlat_messages_per_second = mlat_ip_msgs_per_second.get(ip, [])
    if(0 in mlat_messages_per_second):
        if(len(notes) > 0):
            notes += ", "
        notes += "" + colour_yellow + " at least one entry of 0 MLAT messages per second ***\033[0m"

#    mlat_outlier_percentages = mlat_ip_outlier_percentages.get(ip, [])
#    outlier_pc_threshold = 0.1
#    if(any(outlier_percentage > outlier_pc_threshold for outlier_percentage in mlat_outlier_percentages)):
#        if(len(notes) > 0):
#            notes += ", "
#        notes += "" + colour_red + " at least one entry of greater than 0.1 outlier percentage ***\033[0m"

#    mlat_peer_count = mlat_ip_peer_counts.get(ip, [])
#    if(any(peer_count < 2 for peer_count in mlat_peer_count)):
#        if(len(notes) > 0):
#            notes += ", "
#        notes += "" + colour_red + " at least one entry of less than 2 peers ***\033[0m"

    if(ip == "::1"):
        notes = '' # We don't care about the mlat-server feed

    if (notes == ''):
        if not(args.skip_working):
            print(table_format.format(str(num), ip, readsb_count, mlat_count, len(readsb_uuids), len(mlat_uuids), mlat_usernames[0], notes))
    else:
        print(table_format.format(str(num), ip, readsb_count, mlat_count, len(readsb_uuids), len(mlat_uuids), mlat_usernames[0], notes))

    if (notes == ''):
        if not(args.skip_working):
            if(args.show_uuid_breakdown):
                sub_num = 0
                for uuid in all_unique_uuids:
                    sub_num += 1
                    # print(table_format.format( "{:}-{:}".format(num, sub_num), str(uuid), "", "", "", "", ""))
                    print(table_format.format( "    {:}".format(sub_num), str(uuid), "", "", "", "", "", ""))
    else:
        if(args.show_uuid_breakdown):
            sub_num = 0
            for uuid in all_unique_uuids:
                sub_num += 1
                # print(table_format.format( "{:}-{:}".format(num, sub_num), str(uuid), "", "", "", "", ""))
                print(table_format.format( "    {:}".format(sub_num), str(uuid), "", "", "", "", "", ""))