Function to parse and process output from HTTPX

brevity-process-httpx.py

def processHTTPXCrawl(programName, rawBucketPath, refinedBucketPath, inputBucketPath, presentationBucketPath):
    filePath = refinedBucketPath + programName + '/' + programName + '-httpx-crawl.json'
    df = pd.read_json(filePath, lines=True)
    df['program'] = programName
    
    from urllib.parse import urlparse

    def _parseUrlRoot(urlvalue):
        cleanurl = urlparse(urlvalue).netloc
        return cleanurl

    def _parseUrlBase(urlvalue):
        baseurl = urlparse(urlvalue)#.netloc
        baseurl = baseurl.scheme + '://' + baseurl.netloc + baseurl.path
        return baseurl

    df['domain'] = df['url'].apply(_parseUrlRoot)
    df['baseurl'] = df['url'].apply(_parseUrlBase)
    
    storePath = presentationBucketPath + 'urls/' + programName + '-httpx-crawl.csv'
    df.to_csv(storePath, index=False, columns=['program','domain','baseurl','url','content-type','content-length','status-code'])
    
    storePathUrl = inputBucketPath + programName + '/' + programName + '-urls-crawl.csv'
    df['url'].to_csv(storePathUrl, header=False, index=False, sep='\n')
    return df