meetingscraper.py

import requests
from bs4 import BeautifulSoup
import xml.etree.ElementTree as ET
import ollama


def fetch_rss_feed(url):
    response = requests.get(url)
    response.raise_for_status()
    return response.content


def parse_rss_feed(content):
    root = ET.fromstring(content)
    items = []
    for item in root.findall(".//item"):
        title = item.find("title").text
        link = item.find("link").text
        items.append({"title": title, "link": link})
    return items


def fetch_agenda_items(meeting_url):
    response = requests.get(meeting_url)
    response.raise_for_status()
    return response.content


def resolve_redirect(url):
    response = requests.head(url, allow_redirects=True)
    return response.url


def parse_agenda_items(content):
    root = ET.fromstring(content)
    items = []
    for item in root.findall(".//item"):
        title = item.find("title").text
        link = resolve_redirect(item.find("link").text) + "&FullText=1"
        print(link)
        description = item.find("description").text
        category = item.find("category").text
        items.append(
            {
                "title": title,
                "link": link,
                "description": description,
                "category": category,
            }
        )
    return items


def fetch_full_text(item_url):
    response = requests.get(item_url)
    response.raise_for_status()
    soup = BeautifulSoup(response.content, "html.parser")

    # Find the div with "margin: 10px;" in its style attribute
    doctype_div = soup.find(
        "div", style=lambda value: value and "margin: 10px;" in value
    )
    full_text = doctype_div.get_text() if doctype_div else ""

    attachments = []
    attachments_table = soup.find(
        "table", id="ctl00_ContentPlaceHolder1_tblAttachments"
    )
    if attachments_table:
        for a in attachments_table.find_all("a"):
            attachments.append("https://alameda.legistar.com/" + a["href"])
    return full_text, attachments


def summarize_meeting(meeting_text):
    stream = ollama.generate(
        model="llama3.2",
        prompt=f"Summerize the interesting parts of this upcoming meeting. Consent items are less interesting unless they discuss housing, land use (except for rental agreements), and the police." +
            f"The summary is going in an email to let me know as someone interested in local politics will want to know. Anything that deals with land use, police, zoning, etc." + 
            f"We want to alert on anything hot button or will bring out a lot of folks to public comment.  : \n\n{meeting_text}",
        stream=True,
    )

    for chunk in stream:
        print(chunk['response'], end='', flush=True)


def main():
    root_rss_url = "https://alameda.legistar.com/Feed.ashx?M=Calendar&ID=27225317&GUID=7f18aa65-2321-4ec1-81f4-c47a74100b2d&Mode=Next%20Week&Title=City+of+Alameda+-+Calendar+(Next+Week)"
    root_rss_content = fetch_rss_feed(root_rss_url)
    print(root_rss_url)
    meetings = parse_rss_feed(root_rss_content)

    for meeting in meetings:
        meeting_rss_url = meeting["link"].replace(
            "Gateway.aspx?M=MD", "Feed.ashx?M=CalendarDetail"
        )
        print(meeting_rss_url)
        meeting_rss_content = fetch_agenda_items(meeting_rss_url)
        agenda_items = parse_agenda_items(meeting_rss_content)

        meeting_text = ""

        for item in agenda_items:
            full_text, attachments = fetch_full_text(item["link"])
            meeting_text += (
                item["category"] + ": " + item["title"] + ": " + item["description"] + "\n\n"
            )
            meeting_text += full_text + "\n\n-\n\n"
            print(f"Agenda Item: {item['title']}")
            print(f"Description: {item['description']}")
        
        with open("Meeting.txt", "w") as text_file:
            text_file.write(meeting_text)
        summarize_meeting(meeting_text)

if __name__ == "__main__":
    main()