Extract links from a URL or a local file system path

getlinks.py

#!/usr/bin/env python3
import codecs
import fnmatch
from pathlib import Path
from urllib.parse import urlparse

import lxml.html
import requests
import urllib3
from lxml.etree import ElementTree

urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)


DEFAULT_TYPES = (
    '*.mp3', '*.acc', '*.flac', '*.ogg', '*.oga', '*.ogm', '*.wma', '*.wav', 
    '*.mp4', '*.m4a', '*.ape', '*.mkv', '*.webm', '*.mpg', '*.mpeg', '*.mov',
    '*.avi', '*.zip')


def get_host(url):
    p = urlparse(url)
    return "%s://%s" % (p.scheme, p.netloc)
  
  
if __name__ == '__main__':

    import sys
    import argparse

    parser = argparse.ArgumentParser(description='Extract links from a web page OR local path')
    parser.add_argument('datasource',
                        help='URL or local file to get links from',
                        action='store')
    parser.add_argument('--filterby', '-f',
                        nargs='*',
                        metavar='filterstring',
                        help='Only include specific file extensions (always lowercase). By default: %(default)s',
                        default=DEFAULT_TYPES)
    parser.add_argument('--encoding', '-e',
                        dest='custom_encoding',
                        help='Force web page encoding')
    parser.add_argument('--output-file', '-o',
                        metavar='filename',
                        help='Output filename')

    args = parser.parse_args(sys.argv[1:])

    if args.output_file:
        output = codecs.open(args.output_file, 'w', 'utf-8')
    else:
        output = sys.stdout

    if isinstance(args.filterby, list):
        args.filterby = tuple(args.filterby)

    if args.datasource.startswith(('http://', 'https://')):
        url = args.datasource
        host = get_host(url)

        headers = {
            'Connection': 'keep-alive',
            'Cache-Control': 'max-age=0',
            'Upgrade-Insecure-Requests': '1',
            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.135 Safari/537.36',
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
        }

        resp = requests.get(url, headers=headers, verify=False)

        if args.custom_encoding:
            resp.encoding = args.custom_encoding

        doc: lxml.html.HtmlElement | None = lxml.html.fromstring(resp.text)
        doc.make_links_absolute(base_url=url, resolve_base_href=True)
    else:
        path: Path = Path(args.datasource)
        tree: ElementTree = lxml.html.parse(str(path.absolute()))
        doc: lxml.html.HtmlElement = tree.getroot()

    for element, attr_name, href, pos in doc.iterlinks():
        if [href for ext in args.filterby if fnmatch.fnmatch(href.lower(), ext.lower())]:
            output.write(href + '\n')

    if args.output_file:
        output.close()