Import UpNote notes into Obsidian

UpNote_to_Obsidian.md

upnote_to_obsidian.py

Converts an UpNote markdown export of web clippings into Obsidian notes with YAML frontmatter.

To obtain the backup folder in UpNote, Enable "Export backup file to Markdown" under Settings -> Backup, and "Backup now".

UpNote exports each clipping as a UUID-named .md file with no frontmatter — just a title line, the source URL, and the article body — and records notebook membership only as symlinks under notebooks//. This script reads that structure and, for each note, derives frontmatter (title, url, publication, published, description, author, clipped, and tags from the notebooks), renames the file to its title, and writes the result to a new folder — leaving your original export untouched.

Usage:

# Point it at your UpNote export folder (the one containing the .md files + notebooks/)
python upnote_to_obsidian.py path/to/export

# Preview what it'll do without writing anything
python upnote_to_obsidian.py path/to/export --dry-run

# Choose where the converted notes go (default: <export>/Clippings)
python upnote_to_obsidian.py path/to/export -o ~/MyVault/Clippings

No dependencies — standard library only (Python 3.10+). Then copy the output folder into your vault.

Notes

• Notebooks become Obsidian topics (kebab-cased, e.g. Social Media → social-media). Notes in no notebook get no topics property.
• publication is inferred from the URL's domain (extend the DOMAIN_PUBLICATION map for your sources) with the article-title suffix as a fallback.
• description/published/author are best-effort extractions from the article text and may be missing or imperfect — easy to touch up in Obsidian.
• clipped date uses the file's modified time (the export date), since UpNote doesn't preserve the original clip date here.
• Trashed notes are skipped. If a notebook still links to a note that lives in trash/, that notebook's tag is applied instead to the kept (non-trashed) copy of the same article, matched by URL. Exports without a trash/ folder are unaffected.
• --category NAME sets the categories: [[NAME]] wikilink (default Clippings); pass --category "" to drop it if your vault doesn't use that convention.
• Tested with UpNote v9.14.20's markdown export

upnote_to_obsidian.py

#!/usr/bin/env python3
"""Convert an UpNote markdown export into Obsidian "Clippings" notes.

UpNote exports each web clipping as a UUID-named .md file with no frontmatter:
a `#`/`##` title line, the source URL as a markdown link, then the article body.
Notebook membership is recorded only as symlinks under `notebooks/<Name>/`.

This script derives YAML frontmatter (categories/title/url/publication/published/
clipped/description/author/tags) from the file content plus the symlink map, renames
each note to its (cleaned) title, and writes the result to an output `Clippings/`
directory, leaving the original UUID export untouched.

Run:  uv run upnote_to_obsidian.py [INPUT_DIR] [-o OUTPUT_DIR] [--dry-run]
Default INPUT_DIR is ./upnote-markdown, OUTPUT_DIR is INPUT_DIR/Clippings.
"""

from __future__ import annotations

import argparse
import os
import re
import sys
from collections import defaultdict
from datetime import date
from pathlib import Path
from urllib.parse import urlparse

# --- Domain -> human publication name. A seed list: extend it with your own
# sources. Unknown domains fall back to the title's trailing " - <source>"
# segment, then to the bare domain. ---
DOMAIN_PUBLICATION = {
    "washingtonpost.com": "The Washington Post",
    "nytimes.com": "The New York Times",
    "reuters.com": "Reuters",
    "propublica.org": "ProPublica",
    "theguardian.com": "The Guardian",
    "politico.eu": "POLITICO",
    "politico.com": "POLITICO",
    "theatlantic.com": "The Atlantic",
    "time.com": "TIME",
    "nature.com": "Nature",
    "inquirer.com": "The Philadelphia Inquirer",
    "academic.oup.com": "Oxford Academic",
    "nybooks.com": "The New York Review of Books",
}

# Title/publication separators, longest/most-specific first.
TITLE_SEPARATORS = [" — ", " – ", " - ", " | "]

# A standalone date line, e.g. "November 26, 2025" or "2025-04-25".
MONTHS = {m.lower(): i for i, m in enumerate(
    ["January", "February", "March", "April", "May", "June", "July",
     "August", "September", "October", "November", "December"], start=1)}
RE_TEXT_DATE = re.compile(
    r"^\s*(January|February|March|April|May|June|July|August|September|"
    r"October|November|December)\s+(\d{1,2}),\s+(\d{4})\s*$", re.IGNORECASE)
RE_ISO_DATE = re.compile(r"^\s*(\d{4})-(\d{2})-(\d{2})\s*$")

# First markdown link on a line: [label](target)
RE_MD_LINK = re.compile(r"\[[^\]]*\]\(([^)]+)\)")
# Author credit line, e.g. "By PETER EISLER, NED PARKER and JOSEPH TANFANI"
RE_BY_LINE = re.compile(r"^\s*By\s+(?P<author>[^.]+?)\s*$", re.IGNORECASE)

ILLEGAL_FS_CHARS = re.compile(r'[/\\:*?"<>|]')


def slugify_tag(name: str) -> str:
    """Notebook name -> kebab-case Obsidian tag (no spaces allowed in tags)."""
    s = name.strip().lower()
    s = re.sub(r"[^\w\s-]", "", s)          # drop punctuation
    s = re.sub(r"[\s_]+", "-", s)            # spaces/underscores -> hyphen
    return re.sub(r"-{2,}", "-", s).strip("-")


def sanitize_filename(title: str, maxlen: int = 120) -> str:
    """Title -> safe filename stem (illegal chars removed, whitespace collapsed)."""
    s = ILLEGAL_FS_CHARS.sub("", title)
    s = re.sub(r"\s+", " ", s).strip().strip(".")
    return s[:maxlen].strip() or "untitled"


def build_notebook_map(input_dir: Path, trash_url_to_root: dict[str, str]) -> dict[str, set[str]]:
    """Map root-file basename -> set of notebook names, following symlinks.

    Symlinks resolving into trash/ are remapped to the root file sharing the
    trashed note's URL (so a tag on a trashed copy lands on the kept article).
    """
    notebooks_dir = input_dir / "notebooks"
    mapping: dict[str, set[str]] = defaultdict(set)
    if not notebooks_dir.is_dir():
        return mapping

    for nb in sorted(notebooks_dir.iterdir()):
        if not nb.is_dir():
            continue
        for link in sorted(nb.iterdir()):
            if not link.name.endswith(".md"):
                continue
            target = Path(os.path.realpath(link))
            if not target.exists():
                continue
            if target.parent.name == "trash":
                # Redirect tag to the root copy with the same URL, if any.
                url = extract_url(target.read_text(encoding="utf-8"))
                root_base = trash_url_to_root.get(url) if url else None
                if root_base:
                    mapping[root_base].add(nb.name)
                # else: orphaned trash link -> dropped (no root copy to tag)
            else:
                mapping[target.name].add(nb.name)
    return mapping


def extract_url(text: str) -> str | None:
    """First markdown-link *target* in the note (the clipped source URL)."""
    for line in text.splitlines():
        m = RE_MD_LINK.search(line)
        if m:
            # Unescape markdown backslash-escapes (e.g. \_ in NYT share links).
            return re.sub(r"\\([_*()\[\]])", r"\1", m.group(1)).strip()
    return None


def split_title_publication(title: str, url: str | None):
    """Return (clean_title, publication, author_from_title). Prefer the domain
    map for publication; strip a recognized publication (or "by <author>")
    suffix from the title when present."""
    domain = ""
    if url:
        domain = urlparse(url).netloc.lower().removeprefix("www.")
    pub_from_domain = DOMAIN_PUBLICATION.get(domain)

    # Look for a trailing " - / | / — / – <segment>" in the title.
    for sep in TITLE_SEPARATORS:
        if sep in title:
            head, _, tail = title.rpartition(sep)
            tail = tail.strip()
            head = head.strip()
            # A "by <name>" suffix is an author credit, not a publication.
            m = re.match(r"(?i)^by\s+(.+)$", tail)
            if m:
                return head, pub_from_domain, m.group(1).strip()
            # Strip the suffix when it matches the domain publication, or when we
            # have no domain publication and the tail looks like a source name.
            if pub_from_domain and tail.lower() == pub_from_domain.lower():
                return head, pub_from_domain, None
            if not pub_from_domain and tail:
                return head, tail, None
            # tail didn't match a known publication; keep scanning shorter seps
    return title.strip(), (pub_from_domain or domain or None), None


def normalize_date(line: str) -> str | None:
    m = RE_ISO_DATE.match(line)
    if m:
        return f"{m.group(1)}-{m.group(2)}-{m.group(3)}"
    m = RE_TEXT_DATE.match(line)
    if m:
        month = MONTHS[m.group(1).lower()]
        return f"{int(m.group(3)):04d}-{month:02d}-{int(m.group(2)):02d}"
    return None


def is_noise_line(line: str) -> bool:
    """Lines that are not article prose (headings, images, links, breaks,
    captions, bylines, dates) -- used only to find the description."""
    s = line.strip()
    if not s or s in ("<br>", "By") or s.startswith("#"):
        return True
    if RE_BY_LINE.match(s):                       # byline, e.g. "By JANE DOE"
        return True
    if normalize_date(s):                         # date line
        return True
    if re.fullmatch(r"\(.*\)", s):                # caption like "(iStock)"
        return True
    # Lone link/image line: strip link/image markup (including a dangling
    # `](url)` or `[` from a link split across lines) and bare URLs; if no
    # letters remain the line carried no prose (covers the bare-URL line).
    residual = re.sub(r"!?\[[^\]]*\]\([^)]*\)", "", s)
    residual = re.sub(r"\]\([^)]*\)", "", residual)
    residual = re.sub(r"https?://\S+", "", residual).strip(" []()")
    if not re.search(r"[A-Za-z]", residual):
        return True
    return False


def parse_note(text: str):
    """Extract metadata + body lines from a raw UpNote note."""
    lines = text.splitlines()
    # Title: first non-empty line, minus leading #'s.
    title_idx, title = None, None
    for i, line in enumerate(lines):
        if line.strip():
            title_idx = i
            title = re.sub(r"^#{1,6}\s*", "", line).strip()
            break

    url = extract_url(text)

    # Body = everything after the title, dropping the title's duplicate H1/H2
    # and the bare source-URL line (now captured in frontmatter).
    body_start = (title_idx + 1) if title_idx is not None else 0
    body_lines = lines[body_start:]

    # Find description (first prose paragraph) and published date in the body.
    description, published, author = None, None, None
    for idx, line in enumerate(body_lines):
        s = line.strip()
        if published is None:
            d = normalize_date(s)
            if d:
                published = d
        if author is None:
            m = RE_BY_LINE.match(s)
            # Require it to look like a real byline (has a letter, not too long).
            if m and re.search(r"[A-Za-z]", m.group("author")) and len(s) < 120:
                author = m.group("author").strip()
        if description is None and not is_noise_line(line):
            description = s

    # Drop the redundant duplicate-title line and the bare-URL line from the body.
    cleaned = []
    dropped_dup_title = False
    dropped_url = False
    for line in body_lines:
        s = line.strip()
        if not dropped_dup_title and title and re.sub(r"^#{1,6}\s*", "", s).strip() == title:
            dropped_dup_title = True
            continue
        if not dropped_url and url and RE_MD_LINK.search(s) and url in s.replace("\\", ""):
            dropped_url = True
            continue
        cleaned.append(line)
    # Trim leading blank/`<br>` lines left where the title/URL were removed.
    while cleaned and cleaned[0].strip() in ("", "<br>"):
        cleaned.pop(0)
    body = "\n".join(cleaned).strip("\n")

    return {
        "title": title or "Untitled",
        "url": url,
        "description": description,
        "published": published,
        "author": author,
        "body": body,
    }


def yaml_str(value: str) -> str:
    """Quote a scalar for YAML when needed; escape embedded double quotes."""
    if value == "" or re.search(r'[:#\[\]{}",\'`]|^[\s>|*&!%@-]|^\d', value) or value != value.strip():
        return '"' + value.replace("\\", "\\\\").replace('"', '\\"') + '"'
    return value


def dump_frontmatter(meta: dict, tags: list[str], category: str = "Clippings") -> str:
    """Emit YAML frontmatter, omitting empty fields, matching the vault style.

    `category` is added as a `categories: [[<category>]]` wikilink (the kepano
    vault convention); pass an empty string to omit the categories block.
    """
    out = ["---"]
    if category:
        out.append("categories:")
        out.append(f'  - "[[{category}]]"')
    out.append(f"title: {yaml_str(meta['title'])}")
    if meta.get("url"):
        out.append(f"url: {meta['url']}")
    if meta.get("publication"):
        out.append(f"publication: {yaml_str(meta['publication'])}")
    if meta.get("author"):
        out.append("author:")
        out.append(f'  - "[[{meta["author"]}]]"')
    if meta.get("published"):
        out.append(f"published: {meta['published']}")
    out.append(f"clipped: {meta['clipped']}")
    if meta.get("description"):
        out.append(f"description: {yaml_str(meta['description'])}")
    if tags:
        out.append("topics:")
        for t in tags:
            out.append(f"  - {t}")
    out.append('imported-from: "UpNote"')
    out.append("---")
    return "\n".join(out)


def convert(input_dir: Path, output_dir: Path, dry_run: bool, category: str = "Clippings") -> int:
    root_files = sorted(p for p in input_dir.glob("*.md") if p.is_file())
    if not root_files:
        print(f"No .md files found in {input_dir}", file=sys.stderr)
        return 1

    # URL -> root basename, so trashed notebook links can be remapped to root copies.
    trash_url_to_root: dict[str, str] = {}
    for p in root_files:
        url = extract_url(p.read_text(encoding="utf-8"))
        if url:
            trash_url_to_root.setdefault(url, p.name)

    notebook_map = build_notebook_map(input_dir, trash_url_to_root)

    if not dry_run:
        output_dir.mkdir(parents=True, exist_ok=True)

    used_names: dict[str, int] = {}
    rows = []
    for p in root_files:
        text = p.read_text(encoding="utf-8")
        meta = parse_note(text)
        clean_title, publication, title_author = split_title_publication(meta["title"], meta["url"])
        meta["title"] = clean_title
        meta["publication"] = publication
        # Prefer a byline found in the body; fall back to a "by ..." title suffix.
        meta["author"] = meta.get("author") or title_author
        meta["clipped"] = date.fromtimestamp(p.stat().st_mtime).isoformat()

        tags = sorted(slugify_tag(n) for n in notebook_map.get(p.name, set()))

        # Filename = title, de-duplicated with " (N)".
        stem = sanitize_filename(clean_title)
        n = used_names.get(stem.lower(), 0) + 1
        used_names[stem.lower()] = n
        out_name = f"{stem}.md" if n == 1 else f"{stem} ({n}).md"

        content = dump_frontmatter(meta, tags, category) + "\n\n" + meta["body"] + "\n"
        if not dry_run:
            (output_dir / out_name).write_text(content, encoding="utf-8")

        rows.append((p.name[:12], out_name, publication or "-", ",".join(tags) or "-",
                     "y" if meta["description"] else "-",
                     "y" if meta["published"] else "-",
                     "y" if meta["author"] else "-"))

    # Summary table.
    print(f"{'uuid':12}  {'output file':50}  {'publication':22}  {'tags':24} desc pub auth")
    print("-" * 130)
    for uuid, name, pub, tags, d, pub_d, auth in rows:
        print(f"{uuid:12}  {name[:50]:50}  {pub[:22]:22}  {tags[:24]:24}  {d}    {pub_d}   {auth}")
    print("-" * 130)
    action = "Would write" if dry_run else "Wrote"
    print(f"{action} {len(rows)} files to {output_dir}")
    return 0


def main() -> int:
    ap = argparse.ArgumentParser(description=__doc__,
                                 formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("input_dir", nargs="?", default="upnote-markdown",
                    help="UpNote export dir (default: upnote-markdown)")
    ap.add_argument("-o", "--output-dir", default=None,
                    help="Output dir (default: INPUT_DIR/Clippings)")
    ap.add_argument("--category", default="Clippings",
                    help='Wikilink category added to frontmatter (default: '
                         '"Clippings"); pass "" to omit the categories block')
    ap.add_argument("--dry-run", action="store_true",
                    help="Print the summary table without writing files")
    args = ap.parse_args()

    input_dir = Path(args.input_dir).resolve()
    output_dir = Path(args.output_dir).resolve() if args.output_dir else input_dir / "Clippings"
    return convert(input_dir, output_dir, args.dry_run, args.category)


if __name__ == "__main__":
    raise SystemExit(main())