Twitter archive thing NodeJS

archive.js

const fetch = require("node-fetch");
const { readFileSync, writeFileSync } = require('fs');

const getAccess = async () => {
    const refresh_token = readFileSync('last_refresh', { encoding: 'utf-8' });

    const data = new URLSearchParams();
    data.append('refresh_token', refresh_token);
    data.append('grant_type', 'refresh_token');

    const res = await fetch('https://api.twitter.com/2/oauth2/token', {
        body: data,
        method: 'POST',
        headers: {
            'Authorization': 'Basic ' + Buffer.from('CLIENT_ID:CLIENT_SECRET').toString('base64')
        }
    });

    const { access_token, refresh_token: new_refresh_token } = await res.json();

    writeFileSync('last_refresh', new_refresh_token);

    return access_token;
};

const getAllTweetsAndMedia = async () => {
    let last_result_count = -1, next_token = null;

    const defOpts = {
        max_results: 100,
        expansions: 'attachments.media_keys',
        'tweet.fields': 'attachments,public_metrics,created_at',
        'media.fields': 'media_key,url,variants,preview_image_url'
    };

    const tweets = [];
    const media = [];

    while (last_result_count !== 0) {
        const data = new URLSearchParams();
        for (const [key, value] of Object.entries(defOpts))
            data.append(key, value);

        if (next_token !== null)
            data.append('pagination_token', next_token);

        // console.log(data);

        const auth = await getAccess();

        const res = await fetch(`https://api.twitter.com/2/users/{{USER_ID}}/tweets?${data.toString()}`, {
            headers: {
                Authorization: 'Bearer ' + auth
            }
        });

        const resData = await res.json();

        // console.log(JSON.stringify(resData, null, 2));

        if (!resData.data) {
            console.log(JSON.stringify(resData, null, 2));
            break;
        }

        tweets.push(...resData.data);
        console.log(`Processed ${resData.data.length} tweets. (Total: ${tweets.length})`);

        if (resData.includes.media && Array.isArray(resData.includes.media)) {
            media.push(...resData.includes.media);
            
            for (const medium of resData.includes.media) {
                let downloadUrl = '';

                switch (medium.type) {
                    case "photo": {
                        downloadUrl = medium.url;
                        break;
                    }
                    case "video": {
                        console.log("VIDEO DOWNLOAD");

                        if (!medium.variants || !Array.isArray(medium.variants)) {
                            console.log("Video with no variants " + medium.media_key);
                            continue;
                        }
                        
                        let maxBitRate = -1, maxBitRateUrl;

                        for (const variant of medium.variants) {
                            if (!variant.bit_rate)
                                continue;
                            
                            if (variant.bit_rate > maxBitRate)
                                maxBitRateUrl = variant.url;

                            maxBitRate = Math.max(maxBitRate, variant.bit_rate);
                        }

                        downloadUrl = maxBitRateUrl;

                        break;
                    }
                    default: {
                        console.log("Unknown media type " + medium.type);
                        continue;
                    }
                }

                const chunks = downloadUrl.split('.');
                const ext = chunks[chunks.length - 1];

                const res = await fetch(downloadUrl);
                const buf = await res.buffer();

                writeFileSync(`${medium.media_key}.${ext}`, buf);

                console.log(`Downloaded media ${medium.media_key}.${ext}`);
            }

            console.log('Processed ' + resData.includes.media.length + ' (Total ' + media.length + ')');
        }
        
        next_token = resData.meta.next_token;
        if (!next_token)
            break;

        last_result_count = resData.meta.result_count;

        writeFileSync('tweets.json', JSON.stringify(tweets, null, 2));
        writeFileSync('media.json', JSON.stringify(media, null, 2));
    }

    writeFileSync('tweets.json', JSON.stringify(tweets, null, 2));
    writeFileSync('media.json', JSON.stringify(media, null, 2));

    console.log('DONE: Tweets ' + tweets.length + ' Media ' + media.length);
};

getAllTweetsAndMedia();
// getAccess().then(console.log);

Wanted to get an archive of my Twitter but Twitter isn't doing that right now! I was worried it won't survive the weekend (though I admit such worries are pretty overblown) and so I wrote a script using the Twitter V2 API to fetch an archive. You're on your own for figuring out the OAuth 2.0 parts...