Simple Patreon Attachment Downloader

patreondownloader.js

// ==UserScript==
// @name         Simple Patreon Attachment Downloader
// @namespace    Ouraker.SimplePatreonAttachmentDowloader
// @version      0.2
// @description  A simple downloader that download patreon posts.
// @author       Ouraker
// @match        https://www.patreon.com/*/posts
// @icon         https://www.google.com/s2/favicons?sz=64&domain=patreon.com
// @require      https://raw.githubusercontent.com/Stuk/jszip/main/dist/jszip.min.js
// @require      https://raw.githubusercontent.com/eligrey/FileSaver.js/master/dist/FileSaver.js
// @grant        none
// ==/UserScript==

(function() {
    'use strict';
    const originalFetch = window.fetch;

    // Create a deep clone of the original fetch function
    const clonedFetch = function(...args) {
        return originalFetch.apply(this, args);
    };

    // ===== User Config =====
    var use_folder = true; // true = post will be saved into folder individually; false = files will be save in root folder and with pid added into prefix
    var use_title_for_naming_folder = true; // true = folder name will look like "87947807_Hot_officer_sheperd", false = folder name will look like "87947807"
    var pid_postfix = "_";
    var attachment_prefix = "a_";
    var post_file_prefix = "p_";

    var manifest_fetch_interval = 700 // in ms, affects how fast we can fetch post manifest. increase this if http 429 occurs.
    var file_fetch_interval = 700; // in ms, affects how fast we can download post. increase this if http 429 occurs.

    // ===== Vars for Program =====
    var zip = new JSZip();
    var author = window.location.href
        .slice(0,window.location.href.search(/\/posts/))
        .split('/')
        .pop()
    var author_id = document.getElementById("avatar-image").getAttribute("src")
    author_id = author_id.slice(author_id.search(/campaign/)+'campaign/'.length).split("/")[0]
    var author_name = window.location.href.split('/').reverse()[1];

    var requests = [
        'https://www.patreon.com/api/posts?json-api-version=1.0',
        'include=campaign%2Caccess_rules%2Cattachments',
        'fields[campaign]=currency%2Cshow_audio_post_download_links%2Cavatar_photo_url%2Cavatar_photo_image_urls%2Cearnings_visibility%2Cis_nsfw%2Cis_monthly%2Cname%2Curl',
        'fields[post]=content%2Cpost_file%2Cpost_metadata%2Cpublished_at%2Cpatreon_url%2Ctitle%2Curl',
        'filter[contains_exclusive_posts]=true&sort=-published_at',
        'filter[campaign_id]=' + author_id
    ]

    var r_cursor = '&page[cursor]=';

    // ===== Functions =====
    const Sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms));
    const Random32B = () => (crypto.getRandomValues(new Uint32Array(1))[0]).toString(16);

    async function Download(url){
        try {
            return await (await clonedFetch(url, {"access-control-request-headers":"*"})).blob()
        } catch (err){
            console.log(err)
        }
    }

    async function SaveZip(resetZip, filename){
        var fn = `${author}.zip`;
        if (filename != null) {
            fn = filename + ".zip";
        }
        // Save file

        console.log(`Saving file as ${fn}`);
        zip.generateAsync({type:"blob"})
        .then(function (blob) {
            saveAs(blob, fn);
        });

        if (resetZip == true){
            zip = new JSZip();
        }
    }

    async function Download_Post(post){
        var Func = async (pid)=>{
            let current_post = post[pid];
            console.log(`Downloading: ${pid}-${current_post.title}`);

            let fetch_memo = [];
            let file_prefix = "";
            if (!use_folder){
                file_prefix = pid + pid_postfix;
            }

            // create zip dir
            var dir = zip.folder();
            if (use_folder && use_title_for_naming_folder){
                dir = zip.folder(pid + pid_postfix + current_post.title);
            } else if (use_folder){
                dir = zip.folder(pid + pid_postfix);
            }

            // save text content
            dir.file(`${file_prefix}post_message.txt`, current_post.text);
            ///console.log(current_post.bag);
            // download and save large content
            current_post.bag.forEach((file) => {
                var fn = file_prefix;
                switch (file.type){
                    case "attachment":
                        fn += attachment_prefix + file.filename;
                        break;
                    case "post_file":
                        fn += post_file_prefix + (file.filename || (Random32B() + '.png'));
                        break;
                    default:
                        fn += Random32B() + file.filename;
                        break;
                }
                fetch_memo.push(
                    Download(file.url).then(res => {
                        dir.file(fn, res);
                    })
                );
            })
            await Promise.allSettled(fetch_memo) // Wait for all fetches
        };

        var keys = Object.keys(post);
        for(var i in keys){
            await Func(keys[i]);
            await Sleep(file_fetch_interval);
        }
    }

    function Fetch_Posts(post){
        // mark every attachments for later use
        var payload = post.included;
        var payload_map = {};
        payload.forEach((i, index) => {
            if (i.type = "attachment"){
                payload_map[i.id] = index;
            }
        })

        // dive in post data
        var parsed_post = {};
        post.data.forEach((d)=> {
            let data_bag = [];
            let metadata = d.attributes;
            let manifest = d.relationships;

            // filters <>?:"|*/ and whitespace to to compatible with windows filesystem
            let title = metadata.title.replaceAll(/[<>?:"|*\s]!?/g,"_").replaceAll("/","&");
            if (metadata.content == null){
                metadata.content = ''; // idk why post could be empty
            }

            // save post text
            let text = `${metadata.title}\n${metadata.url}\n\n${metadata.content.replaceAll(/<\/?p>/g,"\n").replaceAll("<br>","\n")}\n\n${metadata.published_at}`

            if (manifest.attachments == undefined){
                console.warn(`Title has no attachments: ${metadata.title} - ${metadata.url}`);
                return;
            }

            // look for attachment
            if (manifest.attachments.data != null && manifest.attachments.data.length > 0){
                manifest.attachments.data.forEach((i)=>{
                    // search in payload map
                    let lookfor = payload_map[i.id];

                    if (lookfor != null){
                        data_bag.push({
                            "filename": payload[lookfor].attributes.name,
                            "type": "attachment",
                            "url": payload[lookfor].attributes.url
                        });
                    }
                })
            }

            // look for post image
            if (metadata.post_file != null){
                data_bag.push({
                    "filename": metadata.post_file.name,
                    "type": "post_file",
                    "url": metadata.post_file.url
                });
            }

            Object.assign(parsed_post, {[d.id]: { "title": title, "text": text, "bag": data_bag }})
        })

        return {"content": parsed_post,
                "pagination": post.meta.pagination}
    }

    const Get_Posts = (cursor) => {
        let endpoint = requests.join("&");
        if (cursor != null || cursor != ""){
            endpoint += r_cursor + cursor;
        }

        return fetch(endpoint,
            {
                headers: {
                    "Content-Type": "application/vnd.api+json",
                    "Referer": window.location.href
                }
            }
        ).then(res => res.json()
        ).catch(err => {
            console.error("Failed to obtain data.");
            console.error(err);
        });
    }

    // ===== HTML Injection =====
    var inject_timeout = 1500;
    var helper_container_name = "Download-Helper";
    var post_container_name = "dh-post-container";
    var status_info_name = "dh-status-info";
    var html_element = `
<style>
    #${helper_container_name} {
        height: 300px;
        border: solid 1px whitesmoke;
        display: flex;
        flex-wrap: wrap;
        flex-direction: row;
        overflow: scroll;
        overflow-x: hidden;
        max-width: inherit;
        align-content: flex-start;
        resize: both;
        background: var(--global-bg-base-default);
    }

    .${post_container_name} {
        height: 2rem;
        width: calc(100% - 0.6rem);
        margin: 0.3rem;
        background: var(--component-button-onAction-default);
        border-radius: 0.6rem;
        cursor: pointer;
        font-size: small;
        line-break: anywhere;
        padding: 0.1rem 0.5rem 0.1rem 0.5rem;
    }

    .${post_container_name}.selected {
        background: royalblue;
    }

    .control-panel {
        margin-top: 0.5rem;
        display: flex;
        flex-wrap: wrap;
        flex-direction: row;
        justify-content: flex-start;
        align-content: center;
        gap: 0.25rem;
        padding: 0 0.5rem 0 0.5rem;
    }

    .${status_info_name} {
        margin: 0.5rem;
        padding: 0.5rem;
        background: var(--global-bg-base-default);
        border: 1px solid var(--component-button-onAction-default);
        border-radius: 0.25rem;
        font-size: small;
    }
</style>

<div id="${helper_container_name}"></div>
<div class="control-panel">
    <button id="fetch-data">Fetch</button>
    <button id="fetch-all">Fetch All</button>
    <button id="download-selected">Download Selected</button>
    <button id="download-all">Download All</button>
    <button id="reset-size">Reset Size</button>
</div>
<div class="${status_info_name}">
    <div>Fetch Status: <span id="fetch-status">Idle</span></div>
    <div>Download Status: <span id="download-status">Idle</span></div>
</div>
`;

    const post_container = (datatag, title) => `
<div class="${post_container_name}" data-postinfo="${datatag}">${title}</div>
`;

    const Inject_HTML = () => {
        var html_entry_point = document.querySelectorAll("nav")[0].lastChild;
        html_entry_point.insertAdjacentHTML("beforeBegin", html_element);

        document.getElementById("fetch-data").addEventListener("click", Main);
        document.getElementById("fetch-all").addEventListener("click", Fetch_All);
        document.getElementById("download-selected").addEventListener("click", Download_Selected);
        document.getElementById("download-all").addEventListener("click", Download_All);
        document.getElementById("reset-size").addEventListener("click", Reset_Size);
    };

    // ===== Render Posts =====
    const Render_Posts = () => {
        var container = document.getElementById(helper_container_name);
        container.innerHTML = "";

        var keys = Object.keys(post_collection);
        keys.sort((a, b) => b - a); // Sort keys in descending order

        for (let i = 0; i < keys.length; i++) {
            var key = keys[i];
            var post = post_collection[key];
            container.insertAdjacentHTML("beforeend", post_container(key, post.title));
        }

        var post_elements = document.getElementsByClassName(post_container_name);
        for (let i = 0; i < post_elements.length; i++) {
            post_elements[i].addEventListener("click", function() {
                this.classList.toggle("selected");
            });
        }
    };

    // ===== Download Functions =====
    const Download_Selected = () => {
        Update_Status("Done", "Downloading Selected...");
        var selected_posts = {};
        var selected_elements = document.getElementsByClassName("selected");
        for (var i = 0; i < selected_elements.length; i++) {
            var post_id = selected_elements[i].getAttribute("data-postinfo");
            selected_posts[post_id] = post_collection[post_id];
        }
        Download_Post(selected_posts).then(() => {
            SaveZip(true, `${author_name}_selected_${Math.floor((new Date()).getTime() / 1000)}`);
            Update_Status("Done", "Download Selected Done");
        });
    };

    const Download_All = () => {
        Update_Status("Done", "Downloading All...");
        Download_Post(post_collection).then(() => {

            SaveZip(true, `${author_name}_all_attachments_${Math.floor((new Date()).getTime() / 1000)}`);
            Update_Status("Done", "Download All Done");
        });
    };

    // ===== Reset Size Function =====
    const Reset_Size = () => {
        var container = document.getElementById(helper_container_name);
        container.style = null;
    };

    // ===== Fetch All Function =====
    const Fetch_All = async () => {
        while (next_cursor != null) {
            await Main();
            await Sleep(manifest_fetch_interval); // Delay between fetches to lower the possibility of triggering HTTP 429
        }
        console.log("Done fetching.");
        Update_Status("Done", "Idle");
    };

    // ===== Update Status Info =====
    const Update_Status = (fetch_status, download_status) => {
        document.getElementById("fetch-status").textContent = fetch_status;
        document.getElementById("download-status").textContent = download_status;
    };


    // ================
    // ===== Main =====
    // ================

    var next_cursor = '';
    var post_count = 0;
    var current_fetch = 0;

    var post_collection = {}

    // ===== Main =====
    const Main = async () => {
        if (next_cursor == null) {
            console.log(post_collection);
            console.log("Cursor empty, nothing left to fetch.");
            Update_Status("Done", "Idle");
            return;
        }
        Update_Status("Fetching...", "Idle");
        await Get_Posts(next_cursor).then(res => {
            var post = Fetch_Posts(res);
            next_cursor = post.pagination.cursors.next;
            post_count = post.pagination.total;
            current_fetch += Object.keys(post.content).length;
            Object.assign(post_collection, post.content);
        });
        console.log(`Fetch progress:\t ${current_fetch}/${post_count}, next cursor is ${next_cursor}`);
        Update_Status(`Fetched ${current_fetch}/${post_count}`, "Idle");
        Render_Posts();
    };

    Sleep(inject_timeout).then(() => {
        Inject_HTML();
    });

})();