Get all contributors for all repos of a GitHub account

contributors.kt

import arrow.core.Either
import arrow.core.getOrElse
import arrow.core.raise.catch
import arrow.core.raise.either
import arrow.fx.coroutines.autoCloseable
import arrow.fx.coroutines.continuations.ResourceScope
import arrow.fx.coroutines.parMap
import arrow.fx.coroutines.resourceScope
import io.ktor.client.HttpClient
import io.ktor.client.call.body
import io.ktor.client.plugins.auth.Auth
import io.ktor.client.plugins.auth.providers.BearerTokens
import io.ktor.client.plugins.auth.providers.bearer
import io.ktor.client.plugins.cache.HttpCache
import io.ktor.client.plugins.contentnegotiation.ContentNegotiation
import io.ktor.client.request.get
import io.ktor.client.request.prepareGet
import io.ktor.client.statement.bodyAsText
import io.ktor.http.HttpStatusCode
import io.ktor.serialization.kotlinx.json.json
import io.ktor.utils.io.ByteReadChannel
import io.ktor.utils.io.core.isEmpty
import io.ktor.utils.io.core.readBytes
import kotlinx.serialization.ExperimentalSerializationApi
import kotlinx.serialization.SerialName
import kotlinx.serialization.Serializable
import kotlinx.serialization.json.Json
import okio.FileSystem
import okio.Path.Companion.toPath
import okio.Sink
import okio.buffer

const val githubAccessToken = "ghp_<<github_token_here>>"
const val githubApiBaseUrl = "https://api.github.com"
const val githubAccount = "joreilly"

suspend fun main() {
  resourceScope {
    val httpClient = HttpClient()
    val client: GithubClient = DefaultGithubClient(httpClient)
    val repositories = client.repositories(githubAccount).getOrThrow()
    val contributors = repositories
      .parMap { githubRepository ->
        client.contributors(githubRepository).getOrThrow()
      }
      .flatten()
      .associateBy { it.login }
      .toList()

    val fileSystem = FileSystem.SYSTEM
    val savedImagesDir = fileSystem.canonicalize("./src/main/resources".toPath())
    contributors.parMap { (name, contributor) ->
      contributor.avatarUrl ?: return@parMap println("Skipping contributor:$name as they have no avatar url")
      httpClient.prepareGet(contributor.avatarUrl).execute { httpResponse ->
        httpResponse.body<ByteReadChannel>().readFully(fileSystem.sink(savedImagesDir / "${contributor.login}.jpeg"))
      }
    }

    println("Number of contributors: ${contributors.size}")
    println("Images saved at $savedImagesDir")
  }
}

interface GithubClient {
  suspend fun repositories(owner: String): Either<MyError, List<GithubRepository>>
  suspend fun contributors(repository: GithubRepository): Either<MyError, List<Contributor>>
}

@OptIn(ExperimentalSerializationApi::class)
suspend fun ResourceScope.HttpClient(): HttpClient {
  return autoCloseable {
    HttpClient {
      install(HttpCache)
      install(Auth) {
        bearer {
          loadTokens {
            BearerTokens(
              githubAccessToken,
              githubAccessToken,
            )
          }
        }
      }
      install(ContentNegotiation) {
        json(
          Json {
            prettyPrint = true
            isLenient = true
            ignoreUnknownKeys = true
            explicitNulls = false
          },
        )
      }
    }
  }
}

class DefaultGithubClient(
  private val client: HttpClient,
) : GithubClient {
  override suspend fun repositories(owner: String): Either<MyError, List<GithubRepository>> {
    return client.getBody<List<GithubRepository>>("$githubApiBaseUrl/users/$owner/repos")
  }

  override suspend fun contributors(repository: GithubRepository): Either<MyError, List<Contributor>> {
    return client.getBody<List<Contributor>>(
      "$githubApiBaseUrl/repos/${repository.owner.login}/${repository.name}/contributors",
    )
  }
}

@Serializable
data class GithubRepository(
  val owner: Contributor,
  val name: String,
)

@Serializable
data class Contributor(
  val id: String,
  val login: String,
  @SerialName("avatar_url")
  val avatarUrl: String?,
)

suspend inline fun <reified T> HttpClient.getBody(
  urlString: String,
): Either<MyError, T> {
  return either {
    val httpResponse = catch(
      { get(urlString) },
      { raise(MyError.Unknown(it)) },
    )
    with(httpResponse) {
      when (status) {
        HttpStatusCode.OK -> body<T>()
        else -> raise(MyError.Http(status, this.bodyAsText()))
      }
    }
  }
}

fun <T> Either<Any, T>.getOrThrow(): T {
  return getOrElse { error(it) }
}

sealed interface MyError {
  data class Http(val errorCode: HttpStatusCode, val message: String) : MyError
  data class Unknown(val e: Throwable) : MyError
}

private const val OKIO_RECOMMENDED_BUFFER_SIZE: Int = 8192

@Suppress("NAME_SHADOWING")
suspend fun ByteReadChannel.readFully(sink: Sink) {
  val channel = this
  sink.buffer().use { sink ->
    while (!channel.isClosedForRead) {
      val packet = channel.readRemaining(OKIO_RECOMMENDED_BUFFER_SIZE.toLong())
      while (!packet.isEmpty) {
        sink.write(packet.readBytes())
      }
    }
  }
}

Results as of 2023-10-07

Number of contributors: 192