Better upload for box-cli

upload.js

// Better upload for box-cli (https://github.com/box/boxcli)
// - skip existing folders/files to let you resume interrupted uploads
// - parallel uploads
//
// Use as a drop-in replacement of src/commands/folders/upload.js

'use strict';

const BoxCommand = require('../../box-command');
const { flags } = require('@oclif/command');
const fs = require('fs');
const path = require('path');
const BoxCLIError = require('../../cli-error');
const utils = require('../../util');

const CHUNKED_UPLOAD_FILE_SIZE = 1024 * 1024 * 100; // 100 MiB
const { Semaphore } = require('await-semaphore');  // install await-semaphore if not already available
const MAX_CONCURRENT_UPLOADS = 5;
const uploadSemaphore = new Semaphore(MAX_CONCURRENT_UPLOADS);

class FoldersUploadCommand extends BoxCommand {
    async run() {
        const { flags, args } = this.parse(FoldersUploadCommand);

        let folderId = await this.uploadFolder(args.path, flags['parent-folder'], flags['folder-name']);
        let folder = await this.client.folders.get(folderId);
        await this.output(folder);
    }

    async uploadFolder(folderPath, parentFolderId, folderName) {
        folderName = folderName || path.basename(folderPath);

        let folderItems;
        try {
            folderItems = await fs.promises.readdir(folderPath);
            console.log(`Reading directory ${folderPath}`);
        } catch (ex) {
            throw new Error(`Could not read directory ${folderPath}: ${ex}`);
        }
        folderItems = folderItems.filter(item => item[0] !== '.');

        let folderId;
        try {
            const iterator = await this.client.folders.getItems(parentFolderId, {
                usemarker: false,
                fields: ['type', 'name'],
                limit: 100
            });

            let result;
            let existingFolder = null;

            do {
                result = await iterator.next();
                if (result.value && result.value.type === 'folder' && result.value.name === folderName) {
                    existingFolder = result.value;
                    break;
                }
            } while (!result.done);

            if (existingFolder) {
                folderId = existingFolder.id;
                console.log(`Using existing folder with ID ${folderId}`);
            } else {
                let folder = await this.client.folders.create(parentFolderId, folderName);
                folderId = folder.id;
                console.log(`Created new folder with ID ${folderId}`);
            }
        } catch (ex) {
            if (ex.statusCode === 409) {
                console.log(`Folder creation skipped: Item with name ${folderName} already exists`);
            } else {
                throw new Error(`Could not check or create folder in Box: ${ex}`);
            }
        }

        const uploadTasks = folderItems.map(item => async () => {
            let itemPath = path.join(folderPath, item);
            let itemStat = await fs.promises.stat(itemPath);
            if (itemStat.isDirectory()) {
                await this.uploadFolder(itemPath, folderId);  // Recursive call for subdirectories
            } else {
                const release = await uploadSemaphore.acquire();
                try {
                    let fileStream = fs.createReadStream(itemPath);
                    let size = itemStat.size;
                    if (size < CHUNKED_UPLOAD_FILE_SIZE) {
                        await this.client.files.uploadFile(folderId, item, fileStream);
                        console.log(`Uploaded file ${item} to folder ID ${folderId}`);
                    } else {
                        let uploader = await this.client.files.getChunkedUploader(folderId, size, item, fileStream);
                        await uploader.start();
                        console.log(`Uploaded large file ${item} to folder ID ${folderId} via chunked upload`);
                    }
                } catch (ex) {
                    if (ex.statusCode === 409) {
                        console.log(`File upload skipped: Item with name ${item} already exists`);
                    } else {
                        console.log(`Error uploading file ${itemPath}: ${ex}`);
                        throw new Error(`Could not upload file ${itemPath}: ${ex}`);
                    }
                } finally {
                    release();  // Release the semaphore lock
                }
            }
        });

        // Execute all upload tasks
        await Promise.all(uploadTasks.map(task => task()));

        return folderId;
    }

}

FoldersUploadCommand.description = 'Upload a folder';
FoldersUploadCommand.examples = ['box folders:upload /path/to/folder'];

FoldersUploadCommand.flags = {
    ...BoxCommand.flags,
    'folder-name': flags.string({ description: 'Name to use for folder if not using local folder name' }),
    'id-only': flags.boolean({
        description: 'Return only an ID to output from this command',
    }),
    'parent-folder': flags.string({
        char: 'p',
        description: 'Folder to upload this folder into; defaults to the root folder',
        default: '0',
    })
};

FoldersUploadCommand.args = [
    {
        name: 'path',
        required: true,
        hidden: false,
        description: 'Local path to the folder to upload',
    }
];

module.exports = FoldersUploadCommand;