Utility helper that takes raw markdown scraped from arbitrary websites and strips out URLs, tables, data URIs, and noisy tokens so it’s safe to feed into an LLM prompt

sanitize-crawled-markdown.ts

function sanitizeCrawledMarkdown(markdown: string): string {
	const urlRegex = /https?:\/\/\S+/g;
	const dataUriRegex =
		/data:(?:image|application)\/[a-z0-9.+-]+;base64,[^\s)"]+/gi;
	const imageRegex = /!\[[^\]]*\]\([^)]*\)/g;

	const looksLikeTableRow = (line: string) => {
		const trimmed = line.trim();
		if (trimmed.startsWith("|") && trimmed.endsWith("|")) {
			return true;
		}
		// rows composed primarily of table punctuation
		return /^[-+|:*._\s]+$/.test(trimmed);
	};

	const vowelRegex = /[aeiouıöüâêîôû]/i;

	const isRepeatingChunk = (text: string) => {
		const maxChunkLength = Math.floor(text.length / 3);
		for (let size = 2; size <= maxChunkLength; size += 1) {
			if (text.length % size !== 0) {
				continue;
			}
			const chunk = text.slice(0, size);
			if (chunk.repeat(text.length / size) === text) {
				return true;
			}
		}
		return false;
	};

	const looksLikeGarbageToken = (token: string) => {
		const plain = token.replace(/[^\p{L}\p{N}]/gu, "");

		if (plain.length === 0) {
			return true;
		}

		if (plain.length >= 20 && !vowelRegex.test(plain)) {
			return true;
		}

		if (plain.length >= 6 && isRepeatingChunk(plain.toLowerCase())) {
			return true;
		}

		const nonAlphaNumericRatio =
			token.replace(/[\p{L}\p{N}]/gu, "").length / token.length;
		if (token.length >= 12 && nonAlphaNumericRatio > 0.6) {
			return true;
		}

		return false;
	};

	const cleanedLines = markdown
		.split("\n")
		.map((line) => {
			let sanitized = line
				.replace(dataUriRegex, "")
				.replace(imageRegex, "")
				.replace(urlRegex, "");

			// Handle markdown links: [text](url) -> text, [text]( -> text, [text] -> text
			// First, handle complete links: [text](url)
			sanitized = sanitized.replace(/\[([^\]]+)\]\([^)]*\)/g, "$1");
			// Then, handle incomplete links: [text](
			sanitized = sanitized.replace(/\[([^\]]+)\]\(/g, "$1");
			// Finally, handle standalone brackets: [text]
			sanitized = sanitized.replace(/\[([^\]]+)\]/g, "$1");

			const tokens = sanitized
				.split(/\s+/)
				.filter((token: string) => token && !looksLikeGarbageToken(token));

			return tokens.join(" ").trim();
		})
		.filter((line) => {
			if (!line) {
				return false;
			}
			if (looksLikeTableRow(line)) {
				return false;
			}

			// Drop lines that are only bullets or punctuation (e.g. "+ + +", "***")
			const alphaNumericChars = line.replace(/[\p{L}\p{N}]/gu, "");
			if (alphaNumericChars.length === line.length) {
				return false;
			}

			return true;
		});

	return cleanedLines.join("\n\n");
}