Created
August 10, 2026 05:02
-
-
Save AnythingLinux/204626fb97868cf2f0777d3afbccc529 to your computer and use it in GitHub Desktop.
Create Prompt Markdown Instructions (.md Files)
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/usr/bin/env python3 | |
| """ | |
| html_to_ai_markdown.py | |
| Convert every index.html/index.htm in a static HTML website to index.md. | |
| Rules: | |
| - HTML is the source of truth. | |
| - Each generated Markdown file gets YAML front matter: | |
| title: | |
| author: "Karim (Masum)" | |
| date: 2026-08-10 | |
| description: | |
| - Exactly one H1. | |
| - Remove website chrome such as header/nav/footer/scripts/styles. | |
| - Preserve useful headings, paragraphs, lists, links, images, tables and code. | |
| - Validate the GENERATED Markdown before writing it. | |
| - --dry-run never modifies files. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import html | |
| import re | |
| import shutil | |
| import sys | |
| import tempfile | |
| from dataclasses import dataclass, field | |
| from html.parser import HTMLParser | |
| from pathlib import Path | |
| # ============================================================================ | |
| # CONFIGURATION | |
| # ============================================================================ | |
| AUTHOR = "Karim (Masum)" | |
| DATE = "2026-08-10" | |
| HTML_NAMES = {"index.html", "index.htm"} | |
| # HTML elements that should never appear in the generated AI Markdown. | |
| REMOVE_TAGS = { | |
| "script", | |
| "style", | |
| "noscript", | |
| "template", | |
| "svg", | |
| "canvas", | |
| "nav", | |
| "footer", | |
| "header", | |
| "form", | |
| "iframe", | |
| "video", | |
| "audio", | |
| "source", | |
| "object", | |
| "embed", | |
| } | |
| # UI/container classes that normally represent website chrome. | |
| REMOVE_CLASS_PATTERNS = ( | |
| "footer", | |
| "navbar", | |
| "navigation", | |
| "breadcrumb", | |
| "cookie", | |
| "modal", | |
| "popup", | |
| "sidebar", | |
| "menu", | |
| ) | |
| # ============================================================================ | |
| # HTML TREE | |
| # ============================================================================ | |
| @dataclass | |
| class Node: | |
| tag: str | None = None | |
| attrs: dict[str, str] = field(default_factory=dict) | |
| children: list["Node"] = field(default_factory=list) | |
| text: str | None = None | |
| class SiteParser(HTMLParser): | |
| VOID = { | |
| "area", | |
| "base", | |
| "br", | |
| "col", | |
| "embed", | |
| "hr", | |
| "img", | |
| "input", | |
| "link", | |
| "meta", | |
| "param", | |
| "source", | |
| "track", | |
| "wbr", | |
| } | |
| def __init__(self) -> None: | |
| super().__init__(convert_charrefs=True) | |
| self.root = Node("root") | |
| self.stack = [self.root] | |
| @property | |
| def current(self) -> Node: | |
| return self.stack[-1] | |
| def handle_starttag(self, tag: str, attrs) -> None: | |
| node = Node( | |
| tag=tag.lower(), | |
| attrs={ | |
| str(key).lower(): str(value or "") | |
| for key, value in attrs | |
| }, | |
| ) | |
| self.current.children.append(node) | |
| if node.tag not in self.VOID: | |
| self.stack.append(node) | |
| def handle_startendtag(self, tag: str, attrs) -> None: | |
| node = Node( | |
| tag=tag.lower(), | |
| attrs={ | |
| str(key).lower(): str(value or "") | |
| for key, value in attrs | |
| }, | |
| ) | |
| self.current.children.append(node) | |
| def handle_endtag(self, tag: str) -> None: | |
| tag = tag.lower() | |
| for index in range(len(self.stack) - 1, 0, -1): | |
| if self.stack[index].tag == tag: | |
| del self.stack[index:] | |
| return | |
| def handle_data(self, data: str) -> None: | |
| if data: | |
| self.current.children.append(Node(text=data)) | |
| def parse_html(source: str) -> Node: | |
| parser = SiteParser() | |
| parser.feed(source) | |
| parser.close() | |
| return parser.root | |
| # ============================================================================ | |
| # HTML HELPERS | |
| # ============================================================================ | |
| def clean_text(value: str) -> str: | |
| value = html.unescape(value or "") | |
| value = value.replace("\xa0", " ") | |
| return re.sub( | |
| r"[ \t\r\f\v]+", | |
| " ", | |
| value, | |
| ).strip() | |
| def attr(node: Node, name: str) -> str: | |
| return node.attrs.get(name.lower(), "") | |
| def classes(node: Node) -> set[str]: | |
| return set(attr(node, "class").lower().split()) | |
| def node_text(node: Node | None) -> str: | |
| if node is None: | |
| return "" | |
| if node.text is not None: | |
| return clean_text(node.text) | |
| return clean_text( | |
| " ".join( | |
| node_text(child) | |
| for child in node.children | |
| ) | |
| ) | |
| def find_first(node: Node, tag: str) -> Node | None: | |
| for child in node.children: | |
| if child.text is not None: | |
| continue | |
| if child.tag == tag: | |
| return child | |
| found = find_first(child, tag) | |
| if found: | |
| return found | |
| return None | |
| def find_all(node: Node, tag: str) -> list[Node]: | |
| result = [] | |
| for child in node.children: | |
| if child.text is not None: | |
| continue | |
| if child.tag == tag: | |
| result.append(child) | |
| result.extend( | |
| find_all(child, tag) | |
| ) | |
| return result | |
| def is_removed(node: Node) -> bool: | |
| if node.tag in REMOVE_TAGS: | |
| return True | |
| class_text = " ".join(classes(node)) | |
| if any( | |
| pattern in class_text | |
| for pattern in REMOVE_CLASS_PATTERNS | |
| ): | |
| return True | |
| role = attr(node, "role").lower() | |
| if role in { | |
| "navigation", | |
| "banner", | |
| "contentinfo", | |
| }: | |
| return True | |
| return False | |
| def clone_content(node: Node) -> Node: | |
| copied = Node( | |
| tag=node.tag, | |
| attrs=dict(node.attrs), | |
| text=node.text, | |
| ) | |
| if node.text is not None: | |
| return copied | |
| for child in node.children: | |
| if is_removed(child): | |
| continue | |
| copied.children.append( | |
| clone_content(child) | |
| ) | |
| return copied | |
| # ============================================================================ | |
| # PAGE METADATA | |
| # ============================================================================ | |
| def get_content_root(root: Node) -> Node: | |
| """ | |
| Prefer <main>, then <article>, then <body>. | |
| This prevents website-wide header/footer/navigation content | |
| from becoming part of the AI Markdown document. | |
| """ | |
| return ( | |
| find_first(root, "main") | |
| or find_first(root, "article") | |
| or find_first(root, "body") | |
| or root | |
| ) | |
| def get_title(root: Node) -> str: | |
| title = node_text( | |
| find_first(root, "title") | |
| ) | |
| if title: | |
| return title | |
| h1 = node_text( | |
| find_first(root, "h1") | |
| ) | |
| if h1: | |
| return h1 | |
| return "Untitled Page" | |
| def get_description( | |
| root: Node, | |
| content: Node, | |
| ) -> str: | |
| # First preference: standard meta description. | |
| for meta in find_all(root, "meta"): | |
| name = attr(meta, "name").lower() | |
| prop = attr(meta, "property").lower() | |
| if ( | |
| name == "description" | |
| or prop == "og:description" | |
| ): | |
| value = clean_text( | |
| attr(meta, "content") | |
| ) | |
| if value: | |
| return value | |
| # Second preference: first meaningful paragraph. | |
| for paragraph in find_all( | |
| content, | |
| "p", | |
| ): | |
| value = node_text(paragraph) | |
| if len(value) >= 30: | |
| return value[:240].rstrip() | |
| return ( | |
| "Content and information provided on this page." | |
| ) | |
| # ============================================================================ | |
| # YAML | |
| # ============================================================================ | |
| def yaml_quote(value: str) -> str: | |
| value = value.replace( | |
| "\\", | |
| "\\\\", | |
| ) | |
| value = value.replace( | |
| '"', | |
| '\\"', | |
| ) | |
| return f'"{value}"' | |
| def front_matter( | |
| title: str, | |
| description: str, | |
| ) -> str: | |
| return ( | |
| "---\n" | |
| f"title: {yaml_quote(title)}\n" | |
| f"author: {yaml_quote(AUTHOR)}\n" | |
| f"date: {DATE}\n" | |
| f"description: {yaml_quote(description)}\n" | |
| "---\n" | |
| ) | |
| # ============================================================================ | |
| # MARKDOWN INLINE CONVERSION | |
| # ============================================================================ | |
| def inline(node: Node) -> str: | |
| if node.text is not None: | |
| return clean_text(node.text) | |
| tag = node.tag or "" | |
| if tag == "br": | |
| return "\n" | |
| # Links. | |
| if tag == "a": | |
| text = inline_children(node) | |
| href = attr( | |
| node, | |
| "href", | |
| ).strip() | |
| if text and href: | |
| return f"[{text}]({href})" | |
| return text | |
| # Images. | |
| if tag == "img": | |
| src = attr( | |
| node, | |
| "src", | |
| ).strip() | |
| alt = clean_text( | |
| attr( | |
| node, | |
| "alt", | |
| ) | |
| ) | |
| if src: | |
| return f"" | |
| return "" | |
| # Bold. | |
| if tag in { | |
| "strong", | |
| "b", | |
| }: | |
| text = inline_children(node) | |
| if text: | |
| return f"**{text}**" | |
| return "" | |
| # Italic. | |
| if tag in { | |
| "em", | |
| "i", | |
| }: | |
| node_classes = classes(node) | |
| # Ignore Font Awesome icons. | |
| if ( | |
| tag == "i" | |
| and ( | |
| attr( | |
| node, | |
| "aria-hidden", | |
| ).lower() == "true" | |
| or "fa" in node_classes | |
| or any( | |
| item.startswith("fa-") | |
| for item in node_classes | |
| ) | |
| ) | |
| ): | |
| return "" | |
| text = inline_children(node) | |
| if text: | |
| return f"*{text}*" | |
| return "" | |
| # Inline code. | |
| if tag == "code": | |
| text = inline_children(node) | |
| if text: | |
| text = text.replace( | |
| "`", | |
| r"\`", | |
| ) | |
| return f"`{text}`" | |
| return "" | |
| # Strikethrough. | |
| if tag in { | |
| "del", | |
| "s", | |
| }: | |
| text = inline_children(node) | |
| if text: | |
| return f"~~{text}~~" | |
| return "" | |
| return inline_children(node) | |
| def inline_children(node: Node) -> str: | |
| parts = [ | |
| inline(child) | |
| for child in node.children | |
| ] | |
| value = " ".join( | |
| part | |
| for part in parts | |
| if part | |
| ) | |
| value = re.sub( | |
| r"[ \t]+", | |
| " ", | |
| value, | |
| ) | |
| value = re.sub( | |
| r" *\n *", | |
| "\n", | |
| value, | |
| ) | |
| return value.strip() | |
| # ============================================================================ | |
| # MARKDOWN BLOCK RENDERER | |
| # ============================================================================ | |
| class Renderer: | |
| def __init__(self) -> None: | |
| self.lines: list[str] = [] | |
| def blank(self) -> None: | |
| while ( | |
| self.lines | |
| and self.lines[-1] == "" | |
| ): | |
| self.lines.pop() | |
| self.lines.append("") | |
| def add( | |
| self, | |
| text: str = "", | |
| ) -> None: | |
| self.lines.append( | |
| text.rstrip() | |
| ) | |
| def render( | |
| self, | |
| node: Node, | |
| ) -> str: | |
| for child in node.children: | |
| self.render_node(child) | |
| while ( | |
| self.lines | |
| and not self.lines[-1].strip() | |
| ): | |
| self.lines.pop() | |
| return "\n".join( | |
| self.lines | |
| ) | |
| def render_node( | |
| self, | |
| node: Node, | |
| ) -> None: | |
| if node.text is not None: | |
| return | |
| tag = node.tag or "" | |
| # Headings. | |
| if re.fullmatch( | |
| r"h[1-6]", | |
| tag, | |
| ): | |
| text = inline_children(node) | |
| if text: | |
| self.blank() | |
| self.add( | |
| "#" * int(tag[1]) | |
| + " " | |
| + text | |
| ) | |
| self.blank() | |
| return | |
| # Paragraph. | |
| if tag == "p": | |
| text = inline_children(node) | |
| if text: | |
| self.blank() | |
| self.add(text) | |
| self.blank() | |
| return | |
| # Code blocks. | |
| if tag == "pre": | |
| code = find_first( | |
| node, | |
| "code", | |
| ) | |
| source = node_text( | |
| code or node | |
| ).rstrip() | |
| language = "" | |
| if code: | |
| match = re.search( | |
| r"(?:language|lang)-([\w+-]+)", | |
| attr( | |
| code, | |
| "class", | |
| ), | |
| ) | |
| if match: | |
| language = match.group(1) | |
| if source: | |
| self.blank() | |
| self.add( | |
| "```" + language | |
| ) | |
| self.lines.extend( | |
| source.splitlines() | |
| ) | |
| self.add("```") | |
| self.blank() | |
| return | |
| # Blockquotes. | |
| if tag == "blockquote": | |
| text = inline_children(node) | |
| if text: | |
| self.blank() | |
| for line in text.splitlines(): | |
| self.add( | |
| "> " + line | |
| ) | |
| self.blank() | |
| return | |
| # Lists. | |
| if tag in { | |
| "ul", | |
| "ol", | |
| }: | |
| self.render_list( | |
| node, | |
| tag == "ol", | |
| ) | |
| return | |
| # Tables. | |
| if tag == "table": | |
| self.render_table(node) | |
| return | |
| # Horizontal rule. | |
| if tag == "hr": | |
| self.blank() | |
| self.add("---") | |
| self.blank() | |
| return | |
| # Standalone image. | |
| if tag == "img": | |
| text = inline(node) | |
| if text: | |
| self.blank() | |
| self.add(text) | |
| self.blank() | |
| return | |
| # Details/summary. | |
| if tag == "details": | |
| summary = find_first( | |
| node, | |
| "summary", | |
| ) | |
| if summary: | |
| text = inline_children( | |
| summary | |
| ) | |
| if text: | |
| self.blank() | |
| self.add( | |
| "### " + text | |
| ) | |
| self.blank() | |
| for child in node.children: | |
| if child is not summary: | |
| self.render_node(child) | |
| return | |
| # Generic container. | |
| for child in node.children: | |
| self.render_node(child) | |
| def render_list( | |
| self, | |
| node: Node, | |
| ordered: bool, | |
| ) -> None: | |
| items = [ | |
| child | |
| for child in node.children | |
| if ( | |
| child.text is None | |
| and child.tag == "li" | |
| ) | |
| ] | |
| if not items: | |
| return | |
| self.blank() | |
| number = 1 | |
| for item in items: | |
| parts = [] | |
| nested = [] | |
| for child in item.children: | |
| if child.text is not None: | |
| value = clean_text( | |
| child.text | |
| ) | |
| if value: | |
| parts.append(value) | |
| elif child.tag in { | |
| "ul", | |
| "ol", | |
| }: | |
| nested.append(child) | |
| else: | |
| value = inline(child) | |
| if value: | |
| parts.append(value) | |
| text = clean_text( | |
| " ".join(parts) | |
| ) | |
| if not text: | |
| continue | |
| if ordered: | |
| prefix = f"{number}. " | |
| else: | |
| prefix = "- " | |
| self.add( | |
| prefix + text | |
| ) | |
| for sublist in nested: | |
| nested_renderer = Renderer() | |
| nested_renderer.render_list( | |
| sublist, | |
| sublist.tag == "ol", | |
| ) | |
| for line in nested_renderer.lines: | |
| if line.strip(): | |
| self.add( | |
| " " + line | |
| ) | |
| number += 1 | |
| self.blank() | |
| def render_table( | |
| self, | |
| node: Node, | |
| ) -> None: | |
| rows = [] | |
| for tr in find_all( | |
| node, | |
| "tr", | |
| ): | |
| cells = [] | |
| for child in tr.children: | |
| if ( | |
| child.text is None | |
| and child.tag in { | |
| "th", | |
| "td", | |
| } | |
| ): | |
| cells.append( | |
| inline_children( | |
| child | |
| ).replace( | |
| "|", | |
| r"\|", | |
| ) | |
| ) | |
| if cells: | |
| rows.append(cells) | |
| if not rows: | |
| return | |
| width = max( | |
| len(row) | |
| for row in rows | |
| ) | |
| rows = [ | |
| row + [""] * ( | |
| width - len(row) | |
| ) | |
| for row in rows | |
| ] | |
| self.blank() | |
| self.add( | |
| "| " | |
| + " | ".join(rows[0]) | |
| + " |" | |
| ) | |
| self.add( | |
| "| " | |
| + " | ".join( | |
| ["---"] * width | |
| ) | |
| + " |" | |
| ) | |
| for row in rows[1:]: | |
| self.add( | |
| "| " | |
| + " | ".join(row) | |
| + " |" | |
| ) | |
| self.blank() | |
| # ============================================================================ | |
| # MARKDOWN POST-PROCESSING | |
| # ============================================================================ | |
| def normalize_headings( | |
| body: str, | |
| title: str, | |
| ) -> str: | |
| result = [] | |
| h1_seen = False | |
| for line in body.splitlines(): | |
| match = re.match( | |
| r"^(#{1,6})\s+(.+?)\s*$", | |
| line, | |
| ) | |
| if not match: | |
| result.append(line) | |
| continue | |
| level = len( | |
| match.group(1) | |
| ) | |
| text = match.group(2).strip() | |
| if level == 1: | |
| if not h1_seen: | |
| result.append( | |
| "# " + title | |
| ) | |
| h1_seen = True | |
| else: | |
| result.append( | |
| "## " + text | |
| ) | |
| else: | |
| result.append( | |
| "#" * level | |
| + " " | |
| + text | |
| ) | |
| if not h1_seen: | |
| result.insert( | |
| 0, | |
| "# " + title, | |
| ) | |
| return "\n".join(result) | |
| def add_toc(body: str) -> str: | |
| # Only add TOC when there are enough sections | |
| # to make it useful. | |
| if len( | |
| re.findall( | |
| r"^##\s+", | |
| body, | |
| re.MULTILINE, | |
| ) | |
| ) < 4: | |
| return body | |
| lines = body.splitlines() | |
| if "[TOC]" in lines: | |
| return body | |
| h1_index = next( | |
| ( | |
| index | |
| for index, line | |
| in enumerate(lines) | |
| if line.startswith("# ") | |
| ), | |
| None, | |
| ) | |
| if h1_index is None: | |
| return body | |
| insert = h1_index + 1 | |
| while ( | |
| insert < len(lines) | |
| and not lines[insert].strip() | |
| ): | |
| insert += 1 | |
| # Move past the first paragraph. | |
| while insert < len(lines): | |
| if not lines[insert].strip(): | |
| break | |
| if lines[insert].startswith("#"): | |
| break | |
| insert += 1 | |
| lines[ | |
| insert:insert | |
| ] = [ | |
| "", | |
| "[TOC]", | |
| ] | |
| return "\n".join(lines) | |
| def wrap_prose( | |
| body: str, | |
| width: int = 80, | |
| ) -> str: | |
| output = [] | |
| in_code = False | |
| for line in body.splitlines(): | |
| stripped = line.strip() | |
| # Track fenced code blocks. | |
| if stripped.startswith("```"): | |
| in_code = not in_code | |
| output.append(line) | |
| continue | |
| if in_code: | |
| output.append(line) | |
| continue | |
| if not stripped: | |
| output.append(line) | |
| continue | |
| # Don't wrap Markdown structural lines. | |
| if ( | |
| stripped.startswith("#") | |
| or stripped.startswith("- ") | |
| or re.match( | |
| r"^\d+\.\s", | |
| stripped, | |
| ) | |
| or stripped.startswith("|") | |
| or stripped == "[TOC]" | |
| or stripped.startswith("![") | |
| or stripped.startswith("> ") | |
| or stripped == "---" | |
| ): | |
| output.append(line) | |
| continue | |
| words = stripped.split() | |
| current = "" | |
| for word in words: | |
| candidate = ( | |
| word | |
| if not current | |
| else current + " " + word | |
| ) | |
| if len(candidate) <= width: | |
| current = candidate | |
| else: | |
| if current: | |
| output.append( | |
| current | |
| ) | |
| current = word | |
| if current: | |
| output.append(current) | |
| return "\n".join(output) | |
| def clean_body(body: str) -> str: | |
| body = re.sub( | |
| r"\n{3,}", | |
| "\n\n", | |
| body, | |
| ) | |
| lines = [] | |
| for line in body.splitlines(): | |
| if line.strip(): | |
| lines.append( | |
| line.rstrip() | |
| ) | |
| elif ( | |
| lines | |
| and lines[-1] != "" | |
| ): | |
| lines.append("") | |
| while ( | |
| lines | |
| and not lines[-1].strip() | |
| ): | |
| lines.pop() | |
| return "\n".join(lines) | |
| # ============================================================================ | |
| # GENERATION | |
| # ============================================================================ | |
| def generate_markdown( | |
| html_file: Path, | |
| ) -> str: | |
| source = html_file.read_text( | |
| encoding="utf-8", | |
| errors="replace", | |
| ) | |
| root = parse_html(source) | |
| title = get_title(root) | |
| original_content = get_content_root( | |
| root | |
| ) | |
| description = get_description( | |
| root, | |
| original_content, | |
| ) | |
| # Remove website chrome. | |
| content = clone_content( | |
| original_content | |
| ) | |
| renderer = Renderer() | |
| body = renderer.render( | |
| content | |
| ) | |
| body = normalize_headings( | |
| body, | |
| title, | |
| ) | |
| body = add_toc(body) | |
| body = clean_body(body) | |
| body = wrap_prose(body) | |
| body = clean_body(body) | |
| return ( | |
| front_matter( | |
| title, | |
| description, | |
| ) | |
| + "\n" | |
| + body | |
| + "\n" | |
| ) | |
| # ============================================================================ | |
| # VALIDATION | |
| # ============================================================================ | |
| def parse_front_matter( | |
| markdown: str, | |
| ) -> tuple[dict[str, str], str]: | |
| match = re.match( | |
| r"\A---\r?\n" | |
| r"(.*?)" | |
| r"\r?\n---\r?\n?", | |
| markdown, | |
| re.DOTALL, | |
| ) | |
| if not match: | |
| return {}, markdown | |
| fields = {} | |
| for line in match.group(1).splitlines(): | |
| field_match = re.match( | |
| r"^([A-Za-z][A-Za-z0-9_-]*):[ \t]*(.*)$", | |
| line, | |
| ) | |
| if not field_match: | |
| continue | |
| key = field_match.group(1) | |
| value = field_match.group(2).strip() | |
| # Parse simple quoted YAML scalar. | |
| if ( | |
| len(value) >= 2 | |
| and value[0] == '"' | |
| and value[-1] == '"' | |
| ): | |
| value = ( | |
| value[1:-1] | |
| .replace( | |
| '\\"', | |
| '"', | |
| ) | |
| .replace( | |
| "\\\\", | |
| "\\", | |
| ) | |
| ) | |
| fields[key] = value | |
| return ( | |
| fields, | |
| markdown[match.end():], | |
| ) | |
| def validate_markdown( | |
| markdown: str, | |
| html_file: Path, | |
| ) -> list[str]: | |
| errors = [] | |
| fields, body = parse_front_matter( | |
| markdown | |
| ) | |
| # ------------------------------------------------------------------------ | |
| # FRONT MATTER | |
| # ------------------------------------------------------------------------ | |
| if not fields: | |
| errors.append( | |
| "Missing or invalid YAML front matter." | |
| ) | |
| return errors | |
| required_fields = ( | |
| "title", | |
| "author", | |
| "date", | |
| "description", | |
| ) | |
| for field_name in required_fields: | |
| if not fields.get( | |
| field_name, | |
| "", | |
| ).strip(): | |
| errors.append( | |
| "Invalid or missing front " | |
| f"matter field: {field_name}" | |
| ) | |
| # Author must be exact. | |
| if fields.get("author") != AUTHOR: | |
| errors.append( | |
| f'Invalid author: expected ' | |
| f'"{AUTHOR}", got ' | |
| f'"{fields.get("author", "")}".' | |
| ) | |
| # Date must be exact. | |
| if fields.get("date") != DATE: | |
| errors.append( | |
| f"Invalid date: expected " | |
| f"{DATE}, got " | |
| f'"{fields.get("date", "")}".' | |
| ) | |
| # ------------------------------------------------------------------------ | |
| # H1 VALIDATION | |
| # ------------------------------------------------------------------------ | |
| h1s = re.findall( | |
| r"^#\s+.+$", | |
| body, | |
| re.MULTILINE, | |
| ) | |
| if len(h1s) != 1: | |
| errors.append( | |
| "Expected exactly one H1; " | |
| f"found {len(h1s)}." | |
| ) | |
| # ------------------------------------------------------------------------ | |
| # BODY VALIDATION | |
| # ------------------------------------------------------------------------ | |
| if not body.strip(): | |
| errors.append( | |
| "Markdown body is empty." | |
| ) | |
| # Empty Markdown links. | |
| if re.search( | |
| r"\]\(\s*\)", | |
| body, | |
| ): | |
| errors.append( | |
| "Empty Markdown link found." | |
| ) | |
| # Empty Markdown images. | |
| if re.search( | |
| r"!\[[^\]]*\]\(\s*\)", | |
| body, | |
| ): | |
| errors.append( | |
| "Empty Markdown image found." | |
| ) | |
| # Images should have alt text. | |
| for alt in re.findall( | |
| r"!\[([^\]]*)\]\([^)]+\)", | |
| body, | |
| ): | |
| if not alt.strip(): | |
| errors.append( | |
| "Image without descriptive " | |
| "alt text found." | |
| ) | |
| # Fenced code blocks must be balanced. | |
| if ( | |
| len( | |
| re.findall( | |
| r"^```", | |
| body, | |
| re.MULTILINE, | |
| ) | |
| ) | |
| % 2 | |
| ): | |
| errors.append( | |
| "Unclosed fenced code block." | |
| ) | |
| # ------------------------------------------------------------------------ | |
| # FORBIDDEN HTML | |
| # ------------------------------------------------------------------------ | |
| # | |
| # IMPORTANT: | |
| # HTML examples inside fenced code blocks are valid Markdown. | |
| # Therefore remove fenced blocks before checking for forbidden HTML. | |
| # ------------------------------------------------------------------------ | |
| body_without_code = re.sub( | |
| r"```.*?```", | |
| "", | |
| body, | |
| flags=re.DOTALL, | |
| ) | |
| for tag in ( | |
| "script", | |
| "style", | |
| "iframe", | |
| "svg", | |
| "canvas", | |
| ): | |
| if re.search( | |
| rf"<\s*{tag}(?:\s|>)", | |
| body_without_code, | |
| re.IGNORECASE, | |
| ): | |
| errors.append( | |
| f"Forbidden HTML element found: " | |
| f"<{tag}>." | |
| ) | |
| # ------------------------------------------------------------------------ | |
| # SOURCE HTML VALIDATION | |
| # ------------------------------------------------------------------------ | |
| source = html_file.read_text( | |
| encoding="utf-8", | |
| errors="replace", | |
| ) | |
| if not re.search( | |
| r"<\s*(main|article|body)\b", | |
| source, | |
| re.IGNORECASE, | |
| ): | |
| errors.append( | |
| "Source HTML has no " | |
| "main/article/body container." | |
| ) | |
| return errors | |
| # ============================================================================ | |
| # FILE DISCOVERY | |
| # ============================================================================ | |
| def find_html_files( | |
| root: Path, | |
| ) -> list[Path]: | |
| files = [] | |
| for path in root.rglob("*"): | |
| if not path.is_file(): | |
| continue | |
| if path.name.lower() not in HTML_NAMES: | |
| continue | |
| relative = path.relative_to(root) | |
| # Ignore hidden directories. | |
| if any( | |
| part.startswith(".") | |
| for part in relative.parts | |
| ): | |
| continue | |
| files.append(path) | |
| return sorted(files) | |
| # ============================================================================ | |
| # SAFE FILE WRITING | |
| # ============================================================================ | |
| def atomic_write( | |
| path: Path, | |
| content: str, | |
| ) -> None: | |
| with tempfile.NamedTemporaryFile( | |
| "w", | |
| encoding="utf-8", | |
| dir=path.parent, | |
| prefix=f".{path.name}.", | |
| suffix=".tmp", | |
| delete=False, | |
| ) as temp: | |
| temp.write(content) | |
| temp.flush() | |
| temp_path = Path( | |
| temp.name | |
| ) | |
| try: | |
| temp_path.replace(path) | |
| except Exception: | |
| temp_path.unlink( | |
| missing_ok=True | |
| ) | |
| raise | |
| # ============================================================================ | |
| # PROCESS ONE HTML PAGE | |
| # ============================================================================ | |
| def process_one( | |
| html_file: Path, | |
| dry_run: bool, | |
| no_backup: bool, | |
| ) -> tuple[bool, list[str]]: | |
| # IMPORTANT: | |
| # Generate Markdown directly from HTML. | |
| # | |
| # We validate the newly generated Markdown. | |
| # We do NOT validate an existing index.md. | |
| markdown = generate_markdown( | |
| html_file | |
| ) | |
| errors = validate_markdown( | |
| markdown, | |
| html_file, | |
| ) | |
| # Never write invalid Markdown. | |
| if errors: | |
| return False, errors | |
| output = ( | |
| html_file.parent | |
| / "index.md" | |
| ) | |
| # Dry-run never modifies anything. | |
| if dry_run: | |
| return True, [] | |
| # Backup existing Markdown. | |
| if ( | |
| output.exists() | |
| and not no_backup | |
| ): | |
| backup = ( | |
| output.with_name( | |
| "index.md.bak" | |
| ) | |
| ) | |
| shutil.copy2( | |
| output, | |
| backup, | |
| ) | |
| # Safe atomic replacement. | |
| atomic_write( | |
| output, | |
| markdown, | |
| ) | |
| return True, [] | |
| # ============================================================================ | |
| # MAIN | |
| # ============================================================================ | |
| def main() -> int: | |
| parser = argparse.ArgumentParser( | |
| description=( | |
| "Convert HTML index pages to " | |
| "AI-friendly Markdown." | |
| ) | |
| ) | |
| parser.add_argument( | |
| "directory", | |
| nargs="?", | |
| default="/var/www/html", | |
| help=( | |
| "Website root. " | |
| "Default: /var/www/html" | |
| ), | |
| ) | |
| parser.add_argument( | |
| "--dry-run", | |
| action="store_true", | |
| help=( | |
| "Generate and validate in memory; " | |
| "do not write files." | |
| ), | |
| ) | |
| parser.add_argument( | |
| "--no-backup", | |
| action="store_true", | |
| help=( | |
| "Do not create index.md.bak " | |
| "before replacing index.md." | |
| ), | |
| ) | |
| args = parser.parse_args() | |
| root = Path( | |
| args.directory | |
| ).resolve() | |
| if not root.is_dir(): | |
| print( | |
| f"ERROR: directory does not exist: " | |
| f"{root}" | |
| ) | |
| return 2 | |
| html_files = find_html_files( | |
| root | |
| ) | |
| print() | |
| print("=" * 78) | |
| print( | |
| "HTML -> AI MARKDOWN GENERATOR" | |
| ) | |
| print("=" * 78) | |
| print( | |
| f"Website root: {root}" | |
| ) | |
| print( | |
| f"HTML files found: " | |
| f"{len(html_files)}" | |
| ) | |
| print() | |
| passed = 0 | |
| failed = 0 | |
| for html_file in html_files: | |
| relative = html_file.relative_to( | |
| root | |
| ) | |
| try: | |
| ok, errors = process_one( | |
| html_file, | |
| dry_run=args.dry_run, | |
| no_backup=args.no_backup, | |
| ) | |
| except Exception as exc: | |
| ok = False | |
| errors = [ | |
| "Unexpected error: " | |
| f"{type(exc).__name__}: {exc}" | |
| ] | |
| if ok: | |
| passed += 1 | |
| print( | |
| f"PASS {relative}" | |
| ) | |
| else: | |
| failed += 1 | |
| print( | |
| f"FAIL {relative}" | |
| ) | |
| for error in errors: | |
| print( | |
| f" - {error}" | |
| ) | |
| # ------------------------------------------------------------------------ | |
| # SUMMARY | |
| # ------------------------------------------------------------------------ | |
| print() | |
| print("=" * 78) | |
| print( | |
| "VALIDATION SUMMARY" | |
| ) | |
| print("=" * 78) | |
| print( | |
| f"HTML files scanned : " | |
| f"{len(html_files)}" | |
| ) | |
| print( | |
| f"Validation passed : " | |
| f"{passed}" | |
| ) | |
| print( | |
| f"Validation failed : " | |
| f"{failed}" | |
| ) | |
| print("=" * 78) | |
| if failed: | |
| print() | |
| print( | |
| "RESULT: FAILED" | |
| ) | |
| if args.dry_run: | |
| print( | |
| "No files were modified." | |
| ) | |
| else: | |
| print( | |
| "Invalid pages were not written." | |
| ) | |
| print( | |
| "Do not deploy until all " | |
| "validation errors are resolved." | |
| ) | |
| return 1 | |
| print() | |
| if args.dry_run: | |
| print( | |
| "RESULT: DRY RUN SUCCESS" | |
| ) | |
| print( | |
| "No files were modified." | |
| ) | |
| else: | |
| print( | |
| "RESULT: SUCCESS" | |
| ) | |
| print( | |
| "All generated Markdown files " | |
| "passed validation." | |
| ) | |
| return 0 | |
| # ============================================================================ | |
| # ENTRY POINT | |
| # ============================================================================ | |
| if __name__ == "__main__": | |
| sys.exit(main()) | |
Author
Author
delete all .bak file then restart apache:
#!/usr/bin/env python3
import subprocess
import sys
from pathlib import Path
WEB_ROOT = Path("/var/www/html")
def run_command(command):
return subprocess.run(
command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
)
def main():
print("=" * 70)
print("DELETE .BAK FILES + RESTART APACHE")
print("=" * 70)
print(f"Website root: {WEB_ROOT}")
print()
if not WEB_ROOT.is_dir():
print(f"ERROR: {WEB_ROOT} does not exist.")
return 1
# Find all .bak files recursively.
bak_files = sorted(
path
for path in WEB_ROOT.rglob("*")
if path.is_file() and path.name.lower().endswith(".bak")
)
print(f".bak files found: {len(bak_files)}")
print()
if bak_files:
print("Files to be deleted:")
print("-" * 70)
for path in bak_files:
print(path)
print("-" * 70)
print()
answer = input(
"Delete ALL listed .bak files? [y/N]: "
).strip().lower()
if answer != "y":
print()
print("Operation cancelled.")
return 0
deleted = 0
failed = 0
print()
print("Deleting .bak files...")
for path in bak_files:
try:
path.unlink()
deleted += 1
print(f"DELETED: {path}")
except Exception as exc:
failed += 1
print(
f"ERROR: Could not delete {path}: {exc}"
)
print()
print(f"Deleted : {deleted}")
print(f"Failed : {failed}")
if failed:
print()
print(
"ERROR: Some .bak files could not be deleted."
)
print(
"Apache restart has been cancelled."
)
return 1
else:
print("No .bak files found.")
print("Nothing to delete.")
# Verify no .bak files remain.
remaining = sorted(
path
for path in WEB_ROOT.rglob("*")
if path.is_file() and path.name.lower().endswith(".bak")
)
print()
print(f"Remaining .bak files: {len(remaining)}")
if remaining:
print()
print("ERROR: .bak files still remain:")
for path in remaining:
print(path)
print()
print("Apache restart cancelled.")
return 1
# Validate Apache configuration before restart.
print()
print("=" * 70)
print("VALIDATING APACHE CONFIGURATION")
print("=" * 70)
config_test = run_command(
["apachectl", "configtest"]
)
print(config_test.stdout.strip())
if config_test.returncode != 0:
print(config_test.stderr.strip())
print()
print(
"ERROR: Apache configuration test failed."
)
print(
"Apache was NOT restarted."
)
return 1
# Restart Apache.
print()
print("=" * 70)
print("RESTARTING APACHE")
print("=" * 70)
restart = run_command(
["systemctl", "restart", "apache2"]
)
if restart.returncode != 0:
print(
"ERROR: Apache restart failed."
)
if restart.stderr.strip():
print(restart.stderr.strip())
return 1
print("Apache restart command completed.")
# Verify Apache status.
print()
print("=" * 70)
print("CHECKING APACHE STATUS")
print("=" * 70)
status = run_command(
["systemctl", "is-active", "apache2"]
)
apache_status = status.stdout.strip()
print(f"Apache status: {apache_status}")
if apache_status != "active":
print()
print(
"ERROR: Apache is not active after restart."
)
return 1
print()
print("=" * 70)
print("RESULT: SUCCESS")
print("=" * 70)
print(f".bak files deleted : {len(bak_files)}")
print("Apache config : VALID")
print("Apache status : ACTIVE")
print("=" * 70)
return 0
if name == "main":
sys.exit(main())
Author
nano delete_bak_restart_apache.py
chmod +x delete_bak_restart_apache.py
python3 delete_bak_restart_apache.py
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
nano html_to_ai_markdown.py
chmod +x html_to_ai_markdown.py
python3 ./html_to_ai_markdown.py /var/www/html --dry-run
python3 ./html_to_ai_markdown.py /var/www/html