Bulk Import/Migrate Github Repos

migrate.js

const { spawn } = require('child_process')
const fs = require('fs')
const path = require('path')
const os = require('os')

// Replace with your GitHub usernames and personal access tokens
const sourceOrg = 'sourceOrg'
const sourceUser = 'sourceUser'
const sourceToken = 'sourceUserToken'
const targetOrg = 'targetOrg'
const targetUser = 'targetUser'
const targetToken = 'targetUserToken'

// Helper function to perform fetch requests with retries
async function fetchWithRetries(url, options, retries = 3, backoff = 3000) {
  for (let i = 0; i < retries; i++) {
    try {
      const response = await fetch(url, options)
      if (!response.ok && response.status >= 500) {
        throw new Error(`Server error: ${response.statusText}`)
      }
      return response
    } catch (error) {
      if (i < retries - 1) {
        console.log(
          `Fetch failed (attempt ${i + 1}): ${error.message}. Retrying in ${
            backoff / 1000
          } seconds...`
        )
        await new Promise(resolve => setTimeout(resolve, backoff))
      } else {
        console.log(`Fetch failed after ${retries} attempts: ${error.message}`)
        throw error
      }
    }
  }
}

// Function to get all repositories from the source organisation
async function getRepos(org, user, token) {
  let repos = []
  let page = 1
  while (true) {
    const apiUrl = `https://api.github.com/orgs/${org}/repos?page=${page}&per_page=100`
    const response = await fetchWithRetries(apiUrl, {
      headers: {
        Authorization: `Basic ${Buffer.from(`${user}:${token}`).toString(
          'base64'
        )}`,
      },
    })
    if (response.status !== 200) {
      console.log(
        `Failed to fetch repositories: ${
          response.status
        } ${await response.text()}`
      )
      break
    }
    const reposPage = await response.json()
    if (reposPage.length === 0) {
      break
    }
    repos = repos.concat(reposPage)
    page += 1
  }
  return repos
}

// Function to create a new repository in the target organisation
async function createRepo(repoName, targetOrg, targetUser, targetToken) {
  const apiUrl = `https://api.github.com/orgs/${targetOrg}/repos`
  const payload = {
    name: repoName,
    private: true,
  }
  const response = await fetchWithRetries(apiUrl, {
    method: 'POST',
    headers: {
      Authorization: `Basic ${Buffer.from(
        `${targetUser}:${targetToken}`
      ).toString('base64')}`,
      'Content-Type': 'application/json',
    },
    body: JSON.stringify(payload),
  })
  if (response.status === 201) {
    console.log(`Successfully created repository ${repoName} in ${targetOrg}`)
  } else {
    console.log(
      `Failed to create repository ${repoName}: ${
        response.status
      } ${await response.text()}`
    )
  }
}

// Function to execute git commands with retries and timeout
function execGitCommand(
  command,
  args,
  options = {},
  retries = 3,
  timeout = 20000
) {
  return new Promise((resolve, reject) => {
    const attempt = count => {
      const child = spawn(command, args, { ...options, timeout })

      child.on('close', code => {
        if (code === 0) {
          resolve()
        } else if (count < retries) {
          console.log(`Git command failed (attempt ${count + 1}). Retrying...`)
          setTimeout(() => attempt(count + 1), 3000)
        } else {
          reject(
            new Error(
              `Git command failed after ${retries} attempts: ${command} ${args.join(
                ' '
              )}`
            )
          )
        }
      })

      child.on('error', error => {
        reject(error)
      })
    }

    attempt(0)
  })
}

// Semaphore class to limit concurrent tasks
class Semaphore {
  constructor(maxConcurrency) {
    this.tasks = []
    this.maxConcurrency = maxConcurrency
    this.currentConcurrency = 0
  }

  async acquire() {
    if (this.currentConcurrency >= this.maxConcurrency) {
      await new Promise(resolve => this.tasks.push(resolve))
    }
    this.currentConcurrency++
  }

  release() {
    this.currentConcurrency--
    if (this.tasks.length > 0) {
      const nextTask = this.tasks.shift()
      nextTask()
    }
  }
}

// Function to clone the source repository and push it to the target organisation
async function cloneAndPushRepo(
  repoName,
  sourceOrg,
  sourceUser,
  sourceToken,
  targetOrg,
  targetUser,
  targetToken
) {
  const tempDir = path.join(os.tmpdir(), 'temp_repos', repoName)
  const sourceRepoUrl = `https://${sourceUser}:${sourceToken}@github.com/${sourceOrg}/${repoName}.git`
  const targetRepoUrl = `https://${targetUser}:${targetToken}@github.com/${targetOrg}/${repoName}.git`

  // Check if the directory already exists
  if (fs.existsSync(tempDir)) {
    console.log(`Directory ${tempDir} already exists. Deleting it.`)
    fs.rmSync(tempDir, { recursive: true, force: true })
  }

  // Clone the source repository
  try {
    await execGitCommand('git', ['clone', sourceRepoUrl, tempDir], {
      stdio: 'inherit',
    })
  } catch (error) {
    console.log(`Failed to clone repository ${repoName}: ${error.message}`)
    return
  }

  // Change directory to the cloned repository
  process.chdir(tempDir)

  // Add the target repository as a remote and push
  try {
    await execGitCommand('git', ['remote', 'add', 'target', targetRepoUrl], {
      stdio: 'inherit',
    })
    await execGitCommand('git', ['push', 'target', '--all'], {
      stdio: 'inherit',
    })
    await execGitCommand('git', ['push', 'target', '--tags'], {
      stdio: 'inherit',
    })
  } catch (error) {
    console.log(
      `Failed to push repository ${repoName} to target: ${error.message}`
    )
  }

  // Cleanup: remove the temporary directory
  process.chdir(__dirname)
  fs.rmSync(tempDir, { recursive: true, force: true })
}

// Function to check if a repository exists in the target organisation
async function repoExists(repoName, targetOrg, targetUser, targetToken) {
  const apiUrl = `https://api.github.com/repos/${targetOrg}/${repoName}`
  const response = await fetchWithRetries(apiUrl, {
    headers: {
      Authorization: `Basic ${Buffer.from(
        `${targetUser}:${targetToken}`
      ).toString('base64')}`,
    },
  })
  return response.status === 200
}

// Function to fetch the latest commit hash from a repository
async function getLatestCommit(org, repoName, user, token) {
  const apiUrl = `https://api.github.com/repos/${org}/${repoName}/commits`
  const response = await fetchWithRetries(apiUrl, {
    headers: {
      Authorization: `Basic ${Buffer.from(`${user}:${token}`).toString(
        'base64'
      )}`,
    },
  })
  if (response.status === 200) {
    const commits = await response.json()
    return commits[0].sha
  } else {
    console.log(
      `Failed to fetch commits for ${repoName}: ${
        response.status
      } ${await response.text()}`
    )
    return null
  }
}

// Main function to fetch and import repositories
async function main() {
  const sourceRepos = await getRepos(sourceOrg, sourceUser, sourceToken)
  const targetRepos = await getRepos(targetOrg, targetUser, targetToken)
  const targetRepoNames = targetRepos.map(repo => repo.name)

  const semaphore = new Semaphore(3) // Limit to 3 concurrent tasks

  const tasks = sourceRepos.map(async repo => {
    await semaphore.acquire()

    try {
      if (targetRepoNames.includes(repo.name)) {
        console.log(
          `Repository ${repo.name} already exists in the target organisation, checking for new commits.`
        )

        const sourceLatestCommit = await getLatestCommit(
          sourceOrg,
          repo.name,
          sourceUser,
          sourceToken
        )
        const targetLatestCommit = await getLatestCommit(
          targetOrg,
          repo.name,
          targetUser,
          targetToken
        )

        if (sourceLatestCommit !== targetLatestCommit) {
          console.log(
            `New commits found in repository ${repo.name}, updating target repository.`
          )
          await cloneAndPushRepo(
            repo.name,
            sourceOrg,
            sourceUser,
            sourceToken,
            targetOrg,
            targetUser,
            targetToken
          )
        } else {
          console.log(
            `No new commits found in repository ${repo.name}, skipping update.`
          )
        }
      } else {
        await createRepo(repo.name, targetOrg, targetUser, targetToken)
        await cloneAndPushRepo(
          repo.name,
          sourceOrg,
          sourceUser,
          sourceToken,
          targetOrg,
          targetUser,
          targetToken
        )
      }
    } finally {
      semaphore.release()
    }
  })

  await Promise.all(tasks)
}

main()