A repository migration script for moving repos in multiple organization from GitHub to BitBucket

repoMove.js

/* 
  Instructions:
  1) Download this script to a folder
  2) Install axios
    2.1) npm install axios
    2.2) yarn add axios
  3) Run: "node repoMove.js" to see usage
 */

const fs = require('fs');
const path = require('path');
const axios = require('axios');
const { execSync } = require('child_process');

const baseDir = path.resolve('./');
const languages = [
  'c',
  'c#',
  'c++',
  'go',
  'html/css',
  'java',
  'javascript',
  'objective-c',
  'perl',
  'php',
  'python',
  'ruby',
  'swift',
  'abap',
  'actionscript',
  'ada',
  'arc',
  'apex',
  'asciidoc',
  'android',
  'asp',
  'arduino',
  'assembly',
  'autoit',
  'blitzmax',
  'boo',
  'ceylon',
  'clojure',
  'coco',
  'coffeescript',
  'coldfusion',
  'common lisp',
  'componentpascal',
  'css',
  'cuda',
  'd',
  'dart',
  'delphi',
  'duby',
  'dylan',
  'eiffel',
  'elixir',
  'emacs lisp',
  'erlang',
  'euphoria',
  'f#',
  'fantom',
  'forth',
  'fortran',
  'foxpro',
  'gambas',
  'groovy',
  'hack',
  'haskell',
  'haxe',
  'igorpro',
  'inform',
  'io',
  'julia',
  'kotlin',
  'labview',
  'lasso',
  'latex',
  'limbo',
  'livescript',
  'lua',
  'lilypond',
  'm',
  'markdown',
  'mathematica',
  'matlab',
  'max/msp',
  'mercury',
  'nemerle',
  'nimrod',
  'nodejs',
  'nu',
  'object pascal',
  'objective-j',
  'ocaml',
  'occam',
  'occam-pi',
  'octave',
  'ooc',
  'other',
  'oxygene',
  'pl/sql',
  'powerbasic',
  'powershell',
  'processing',
  'prolog',
  'puppet',
  'purebasic',
  'pure data',
  'qml',
  'quorum',
  'r',
  'racket',
  'realbasic',
  'restructuredtext',
  'rust',
  'sass',
  'scala',
  'scheme',
  'scilab',
  'sclang',
  'self',
  'shell',
  'smalltalk',
  'sourcepawn',
  'sql',
  'standardml',
  'supercollider',
  'tcl',
  'tex',
  'typescript',
  'unityscript',
  'unrealscript',
  'vala',
  'verilog',
  'vhdl',
  'viml',
  'visualbasic',
  'vb.net',
  'xml',
  'xojo',
  'xpages',
  'xquery',
  'xtend',
  'zsh',
];

function findBestLanguage(lang) {
  if (!lang) return '';
  return languages.find((l) => l.toUpperCase().indexOf(lang.toUpperCase()) >= 0);
}

async function getData(url, username, password) {
  const response = await axios.get(url, {
    auth: { username, password },
  });
  return response.data;
}

async function getGitHubRepositories(orgNames, userName, apiKey) {
  const allRepos = {};
  for (const orgName of orgNames) {
    let pageNum = 1;
    let hasMore = true;
    const baseReposUrl = `https://api.github.com/orgs/${orgName}/repos`;
    do {
      console.log(`Fetching GitHub organization ${orgName} repositories page #${pageNum}`);
      const repoPageUrl = `${baseReposUrl}${!!pageNum ? `?page=${pageNum}` : ''}`;
      const repos = await getData(repoPageUrl, userName, apiKey);

      pageNum++;
      hasMore = !!repos && repos.length > 0;
      allRepos[orgName] = allRepos[orgName] || [];
      allRepos[orgName].push(...repos);
    } while (hasMore);
  }

  return allRepos;
}

async function getBitBucketProject(ownerName, projectKey, userName, apiKey) {
  console.log(`Fetching BitBucket project ${ownerName} / ${projectKey}`);
  const projectUrl = `https://api.bitbucket.org/2.0/teams/${ownerName}/projects/${projectKey}`;
  const project = await getData(projectUrl, userName, apiKey);

  let pageNum = 1;
  let hasMore = true;
  const baseReposUrl = project.links.repositories.href.replace(/'/g, '');
  do {
    console.log(
      `Fetching BitBucket project ${ownerName} / ${projectKey} repositories page #${pageNum}`
    );
    const repoPageUrl = `${baseReposUrl}${!!pageNum ? `&page=${pageNum}` : ''}`;
    const result = await getData(repoPageUrl, userName, apiKey);
    const repos = result.values;

    pageNum++;
    hasMore = !!result.next;
    project.repositories = project.repositories || [];
    project.repositories.push(...repos);
  } while (hasMore);

  return project;
}

async function getGitHubPullRequests(orgNames, userName, apiKey) {
  const repoPulls = {};
  const allRepos = await getGitHubRepositories(orgNames, userName, apiKey);

  for (const org of Object.keys(allRepos)) {
    for (const repo of allRepos[org]) {
      const pullsUrl = repo.pulls_url.replace(/\{.+?\}/i, '');
      const repoPRs = ((await getData(pullsUrl, userName, apiKey)) || []).filter(
        (pr) => pr.state === 'open'
      );
      if (!!repoPRs && repoPRs.length > 0) {
        repoPulls[repo.full_name] = repoPRs.map((pr) => {
          return {
            title: pr.title,
            url: pr.html_url,
            user: pr.user.login,
          };
        });
      }
    }
  }

  return repoPulls;
}

async function cloneAllRepos(
  bbOwnerName,
  bbProjectKey,
  bbUsername,
  bbApiKey,
  ghUsername,
  ghApiKey,
  ghOrgs,
  filterPrefix
) {
  const crossOrgDups = [];
  const repoGroups = await getGitHubRepositories(ghOrgs, ghUsername, ghApiKey);
  const project = await getBitBucketProject(bbOwnerName, bbProjectKey, bbUsername, bbApiKey);
  const activeRepoKeys = Object.keys(repoGroups);
  const activeRepos = activeRepoKeys.reduce((p, c) => {
    const newSet = p.concat(
      repoGroups[c].map((r) => {
        return {
          org: c,
          name: r.name,
          language: findBestLanguage(r.language),
          description: r.description,
          mainBranch: r.default_branch,
          fullName: r.full_name,
          cloneUrl: r.clone_url,
        };
      })
    );
    const dupes = newSet
      .filter((v, i, a) => {
        const matches = a.filter(
          (v2, i2) => i !== i2 && v2.name.toUpperCase() === v.name.toUpperCase()
        );
        return !!matches && matches.length > 0;
      })
      .map((v) => {
        return { index: newSet.indexOf(v), value: v };
      })
      .sort((a, b) => b.index - a.index)
      .map((v) => newSet.splice(v.index, 1))
      .reduce((p2, c2) => [...p2, ...c2], []);
    crossOrgDups.push(...dupes);
    return newSet;
  }, []);

  const repoCount = activeRepos.length;
  console.log(
    `Operating in: ${baseDir}\nFound ${repoCount} repos\nSkipping dupes: ${JSON.stringify(
      crossOrgDups.map((d) => d.fullName),
      undefined,
      2
    )}`
  );

  const cmds = activeRepos
    .sort((a, b) => {
      a = a.name.toUpperCase();
      b = b.name.toUpperCase();
      if (a > b) return 1;
      if (a < b) return -1;
      return 0;
    })
    .map((r) => {
      if (!!filterPrefix & !r.name.startsWith(filterPrefix)) return undefined;

      const repoPath = path.join(baseDir, r.fullName);
      const gitHubOriginUri = `https://github.com/${r.fullName}.git`;
      const bitBucketOriginUri = `[email protected]:${bbOwnerName}/${r.name}.git`;
      const repoCmds = [];
      const bitBucketMatch = project.repositories.find(
        (er) => er.name.toUpperCase() === r.name.toUpperCase()
      );

      const bitBucketData = {
        scm: 'git',
        is_private: true,
        name: !!bitBucketMatch ? bitBucketMatch.name : r.name,
        description: r.description || '',
        language: (r.language || '').toLowerCase(),
        project: { key: bbProjectKey },
      };
      repoCmds.push(
        `curl --silent --fail -X ${
          !!bitBucketMatch ? 'PUT' : 'POST'
        } -u ${bbUsername}:${bbApiKey} -H "Content-Type: application/json" -d '${JSON.stringify(
          bitBucketData
        )}' https://api.bitbucket.org/2.0/repositories/${bbOwnerName}/${r.name.toLowerCase()}`
      );

      if (!fs.existsSync(repoPath)) {
        repoCmds.push(
          `git clone --bare ${r.cloneUrl} '${repoPath}'`,
          `cd ${repoPath}`,
          `git remote add downstream ${bitBucketOriginUri}`,
          `git remote add upstream ${gitHubOriginUri}`
        );
      } else {
        repoCmds.push(`cd ${repoPath}`);
      }

      return [...repoCmds, `git fetch upstream`, `git push --mirror downstream`];
    })
    .filter((a) => !!a && a.length > 0)
    .reduce((p, c) => p.concat(c), []);

  while (cmds.length > 0) {
    /* console.log(`?>\t${cmds.shift()}`);
    continue; */

    let attempts = 0;
    const maxAttempts = 5;
    const cmd = cmds.shift();
    console.log(cmd);
    do {
      attempts++;
      try {
        if (cmd.startsWith('cd')) {
          const newDir = cmd.substr(3);
          process.chdir(newDir);
          console.log(`The current directory is now ${process.cwd()}`);
        } else {
          execSync(cmd);
        }
        break;
      } catch (e) {
        if (attempts > maxAttempts - 1) {
          console.error(`Failed to execute command!\n\n${cmd}\n\n${e.stack || e.toString()}`);
          return;
        } else console.log('Retrying...');
      }
    } while (attempts < maxAttempts);
  }
}

const operation = process.argv[2];
const a = process.argv.slice(3).reduce((p, c) => {
  const parts = c.split('=').map((v, i) => (i === 0 ? v.replace(/[^a-zA-Z]/i, '') : v));
  p[parts[0]] = parts[1];
  return p;
}, {});

if (operation === 'move') {
  // This is the primary purpose of the script, move the repos.
  cloneAllRepos(a.o, a.p, a.u, a.k, a.u2, a.k2, a.orgs.split(','), a.f);
} else if (operation === 'prs') {
  // This will output all open pull requests in GitHub because PRs are meta t othe host and not part of the repo.
  getGitHubPullRequests(a.orgs.split(','), a.u2, a.k2).then((v) =>
    console.log(JSON.stringify(v, undefined, 2))
  );
} else {
  console.log(`
Usage:

  node repoMove.js <prs|move> -o=<bitbucket_team_name> -p=<bitbucket_project_key> -u=<bitbucket_username> -k=<bitbucket_api_key> -u2=<github_username> -k2=<github_api_key> -orgs=<github_org1,org2...> -f=[repo_filter_string]

  <bitbucket_team_name>:
  https://bitbucket.org/a_name/a_repo/src/a_branch/
                        ^^^^^^ Team Name
  
  <bitbucket_project_key>:
  https://bitbucket.org/account/user/a_name/projects/A_KEY
                                         Project Key ^^^^^
  
  <bitbucket_username>:
  https://bitbucket.org/account/user/your_name
                            Username ^^^^^^^^^

  <bitbucket_api_key>:
  https://bitbucket.org/account/user/your_name/app-passwords
  - "Create app password" / "Repositories" read + write + admin

  <github_username>:
  https://github.com/a_name
            Username ^^^^^^

  <github_api_key>:
  https://github.com/settings/tokens
  - "Generate new token" / "repo" - Full control of private repositories

  <github_org1,org2...>:
  https://github.com/settings/profile
  - Let Menu Bottom

  [repo_filter_string]:
  - Evaluated by repo_name.startsWith(<repo_filter_string>)
  `);
}