Summarize GitHub Projects

Summarize-GitHubProjects.ps1

param(
    [string]$OutDir = ".\gh-project-summaries",
    [string]$ReposDir = ".\cloned-repos",
    [int]$Limit = 1000,
    [string]$Model = "qwen2.5-coder:32b",
    [switch]$IncludeForks,

    [int]$MaxSelectedFiles = 220,
    [int]$MaxPackedChars = 450000,
    [int]$ChunkChars = 120000
)

$ErrorActionPreference = "Stop"

function Require-Command {
    param([string]$Name)
    if (-not (Get-Command $Name -ErrorAction SilentlyContinue)) {
        throw "Missing required command: $Name"
    }
}

function Safe-Name {
    param([string]$Name)
    return $Name.Replace("/", "__").Replace("\", "__").Replace(":", "_")
}

function Normalize-PathForGlob {
    param([string]$Path)
    return $Path.Replace("\", "/")
}

function Invoke-OllamaPrompt {
    param(
        [string]$Model,
        [string]$Prompt,
        [string]$OutputFile
    )

    $tmpPrompt = [System.IO.Path]::GetTempFileName()
    $Prompt | Set-Content -Encoding UTF8 $tmpPrompt

    Get-Content -Raw $tmpPrompt | ollama run $Model | Set-Content -Encoding UTF8 $OutputFile

    Remove-Item $tmpPrompt -Force
}

function Is-SkippedPath {
    param([string]$Rel)

    $p = $Rel.Replace("\", "/").ToLowerInvariant()

    $skipParts = @(
        "/.git/",
        "/node_modules/",
        "/vendor/",
        "/dist/",
        "/build/",
        "/target/",
        "/.next/",
        "/.nuxt/",
        "/.turbo/",
        "/.cache/",
        "/__pycache__/",
        "/.venv/",
        "/venv/",
        "/env/",
        "/coverage/",
        "/.pytest_cache/",
        "/.mypy_cache/",
        "/.idea/",
        "/.vscode/"
    )

    $wrapped = "/" + $p

    foreach ($s in $skipParts) {
        if ($wrapped.Contains($s)) {
            return $true
        }
    }

    $skipExtensions = @(
        ".png", ".jpg", ".jpeg", ".gif", ".webp", ".ico", ".pdf",
        ".zip", ".tar", ".gz", ".rar", ".7z",
        ".mp4", ".mov", ".mp3", ".wav",
        ".ttf", ".otf", ".woff", ".woff2",
        ".exe", ".dll", ".so", ".dylib",
        ".jar", ".class", ".pyc", ".pyo",
        ".sqlite", ".db", ".lock"
    )

    foreach ($ext in $skipExtensions) {
        if ($p.EndsWith($ext)) {
            return $true
        }
    }

    $skipExact = @(
        "package-lock.json",
        "yarn.lock",
        "pnpm-lock.yaml",
        "poetry.lock",
        "pipfile.lock",
        "cargo.lock",
        "go.sum",
        "gemfile.lock"
    )

    $name = Split-Path $p -Leaf
    if ($skipExact -contains $name) {
        return $true
    }

    return $false
}

function Get-FileScore {
    param([string]$Rel)

    $p = $Rel.Replace("\", "/")
    $lower = $p.ToLowerInvariant()
    $name = Split-Path $lower -Leaf

    $score = 0

    $manifestNames = @(
        "package.json",
        "pyproject.toml",
        "requirements.txt",
        "pipfile",
        "cargo.toml",
        "go.mod",
        "pom.xml",
        "build.gradle",
        "settings.gradle",
        "gemfile",
        "composer.json",
        "pubspec.yaml",
        "mix.exs",
        "deno.json",
        "bunfig.toml",
        "turbo.json",
        "nx.json",
        "tsconfig.json",
        "vite.config.ts",
        "vite.config.js",
        "next.config.js",
        "next.config.mjs",
        "nuxt.config.ts",
        "svelte.config.js"
    )

    if ($manifestNames -contains $name) {
        $score += 1000
    }

    if ($name -eq "dockerfile" -or $lower.EndsWith("docker-compose.yml") -or $lower.EndsWith("docker-compose.yaml")) {
        $score += 850
    }

    if ($lower.StartsWith(".github/workflows/")) {
        $score += 650
    }

    if ($name -in @("makefile", "justfile", "taskfile.yml", "taskfile.yaml")) {
        $score += 600
    }

    if ($lower.Contains("terraform") -or $lower.EndsWith(".tf") -or $lower.Contains("k8s") -or $lower.Contains("kubernetes") -or $lower.Contains("helm")) {
        $score += 500
    }

    $entryNames = @(
        "main.py", "app.py", "server.py", "manage.py",
        "index.js", "index.ts", "index.tsx",
        "main.js", "main.ts", "main.tsx",
        "server.js", "server.ts",
        "main.go",
        "main.rs",
        "program.cs"
    )

    if ($entryNames -contains $name) {
        $score += 800
    }

    if ($lower.StartsWith("src/")) { $score += 350 }
    if ($lower.StartsWith("app/")) { $score += 350 }
    if ($lower.StartsWith("lib/")) { $score += 300 }
    if ($lower.StartsWith("server/")) { $score += 350 }
    if ($lower.StartsWith("client/")) { $score += 250 }
    if ($lower.StartsWith("api/")) { $score += 400 }
    if ($lower.StartsWith("routes/")) { $score += 400 }
    if ($lower.StartsWith("pages/")) { $score += 300 }
    if ($lower.StartsWith("components/")) { $score += 150 }
    if ($lower.StartsWith("cmd/")) { $score += 400 }
    if ($lower.StartsWith("internal/")) { $score += 350 }
    if ($lower.StartsWith("pkg/")) { $score += 250 }

    if ($lower.Contains("/route") -or $lower.Contains("/routes") -or $lower.Contains("/api/")) {
        $score += 300
    }

    if ($lower.Contains("controller") -or $lower.Contains("service") -or $lower.Contains("resolver") -or $lower.Contains("handler")) {
        $score += 250
    }

    if ($lower.Contains("model") -or $lower.Contains("schema") -or $lower.Contains("entity") -or $lower.Contains("migration")) {
        $score += 300
    }

    if ($lower.EndsWith(".sql") -or $lower.Contains("prisma/schema.prisma")) {
        $score += 500
    }

    if ($lower.Contains("config") -or $lower.EndsWith(".env.example")) {
        $score += 250
    }

    if ($lower.Contains("test") -or $lower.Contains("spec") -or $lower.Contains("__tests__")) {
        $score += 120
    }

    if ($name -in @("readme.md", "architecture.md", "overview.md", "contributing.md", "changelog.md")) {
        $score += 300
    }

    if ($lower.StartsWith("docs/")) {
        $score += 120
    }

    $sourceExts = @(
        ".py", ".js", ".jsx", ".ts", ".tsx", ".go", ".rs", ".java", ".kt",
        ".swift", ".c", ".cpp", ".h", ".hpp", ".cs", ".php", ".rb",
        ".scala", ".dart", ".lua", ".sh", ".sql", ".graphql", ".proto"
    )

    foreach ($ext in $sourceExts) {
        if ($lower.EndsWith($ext)) {
            $score += 100
            break
        }
    }

    if ($lower.Contains("generated") -or $lower.Contains(".min.") -or $lower.Contains("/fixtures/") -or $lower.Contains("/mock/")) {
        $score -= 300
    }

    $depth = ($p.ToCharArray() | Where-Object { $_ -eq "/" }).Count
    $score -= [Math]::Min($depth * 8, 80)

    return $score
}

function Get-SmartSelectedFiles {
    param(
        [string]$RepoPath,
        [int]$MaxFiles
    )

    $all = Get-ChildItem -Path $RepoPath -Recurse -File -Force |
        ForEach-Object {
            $rel = Resolve-Path -Relative $_.FullName
            $rel = $rel.TrimStart(".", "\", "/")
            [PSCustomObject]@{
                Path = Normalize-PathForGlob $rel
                Size = $_.Length
            }
        } |
        Where-Object {
            -not (Is-SkippedPath $_.Path) -and $_.Size -lt 500000
        }

    $scored = foreach ($f in $all) {
        $score = Get-FileScore $f.Path

        if ($score -gt 0) {
            [PSCustomObject]@{
                Path = $f.Path
                Size = $f.Size
                Score = $score
            }
        }
    }

    $selected = New-Object System.Collections.Generic.List[object]
    $dirCounts = @{}

    $ordered = $scored | Sort-Object -Property Score, Size -Descending

    foreach ($f in $ordered) {
        if ($selected.Count -ge $MaxFiles) {
            break
        }

        $top = $f.Path.Split("/")[0]
        if (-not $dirCounts.ContainsKey($top)) {
            $dirCounts[$top] = 0
        }

        $cap = 25
        if ($top -in @("src", "app", "lib", "server", "api", "cmd", "internal", "pkg")) {
            $cap = 60
        }
        if ($top -in @("docs", "test", "tests", "__tests__")) {
            $cap = 12
        }

        if ($dirCounts[$top] -lt $cap) {
            $selected.Add($f)
            $dirCounts[$top] += 1
        }
    }

    return $selected
}

function Write-SelectionReport {
    param(
        [array]$Selected,
        [string]$OutputFile
    )

    $lines = @()
    $lines += "# Selected files"
    $lines += ""
    $lines += "| Score | Size | Path |"
    $lines += "|---:|---:|---|"

    foreach ($f in $Selected) {
        $lines += "| $($f.Score) | $($f.Size) | `$($f.Path)` |"
    }

    $lines | Set-Content -Encoding UTF8 $OutputFile
}

function Split-TextChunks {
    param(
        [string]$Text,
        [int]$ChunkSize
    )

    $chunks = @()
    $i = 0

    while ($i -lt $Text.Length) {
        $len = [Math]::Min($ChunkSize, $Text.Length - $i)
        $chunks += $Text.Substring($i, $len)
        $i += $len
    }

    return $chunks
}

Require-Command gh
Require-Command git
Require-Command ollama
Require-Command repomix

New-Item -ItemType Directory -Force -Path $OutDir | Out-Null
New-Item -ItemType Directory -Force -Path $ReposDir | Out-Null

$OutDir = (Resolve-Path $OutDir).Path
$ReposDir = (Resolve-Path $ReposDir).Path

Write-Host "Getting repos from GitHub..." -ForegroundColor Cyan

$repoJson = gh repo list `
    --limit $Limit `
    --json nameWithOwner,url,description,isPrivate,isFork,primaryLanguage,pushedAt

$repos = $repoJson | ConvertFrom-Json

if (-not $IncludeForks) {
    $repos = $repos | Where-Object { -not $_.isFork }
}

$allSummaries = @()

foreach ($repo in $repos) {
    $fullName = $repo.nameWithOwner
    $safe = Safe-Name $fullName
    $repoPath = Join-Path $ReposDir $safe

    Write-Host ""
    Write-Host "=== $fullName ===" -ForegroundColor Green

    if (Test-Path (Join-Path $repoPath ".git")) {
        Write-Host "Updating repo..."
        git -C $repoPath pull --ff-only
    }
    else {
        if (Test-Path $repoPath) {
            Remove-Item -Recurse -Force $repoPath
        }

        Write-Host "Cloning repo..."
        gh repo clone $fullName $repoPath
    }

    $repoOutDir = Join-Path $OutDir $safe
    New-Item -ItemType Directory -Force -Path $repoOutDir | Out-Null

    $metadataFile = Join-Path $repoOutDir "metadata.json"
    $selectedFile = Join-Path $repoOutDir "selected-files.txt"
    $selectionReportFile = Join-Path $repoOutDir "selected-files-report.md"
    $repomixFile = Join-Path $repoOutDir "repomix-selected.md"
    $summaryFile = Join-Path $repoOutDir "summary.md"
    $chunkDir = Join-Path $repoOutDir "chunks"

    New-Item -ItemType Directory -Force -Path $chunkDir | Out-Null

    $metadata = [ordered]@{
        nameWithOwner = $repo.nameWithOwner
        url = $repo.url
        description = $repo.description
        isPrivate = $repo.isPrivate
        isFork = $repo.isFork
        primaryLanguage = $repo.primaryLanguage.name
        pushedAt = $repo.pushedAt
    }

    $metadata | ConvertTo-Json -Depth 10 | Set-Content -Encoding UTF8 $metadataFile

    Write-Host "Selecting high-signal files..."
    Push-Location $repoPath
    $selected = Get-SmartSelectedFiles -RepoPath "." -MaxFiles $MaxSelectedFiles
    Pop-Location

    if (-not $selected -or $selected.Count -eq 0) {
        Write-Host "No useful files selected, skipping." -ForegroundColor Yellow
        continue
    }

    $selectedPaths = $selected | ForEach-Object { $_.Path }
    $selectedPaths | Set-Content -Encoding UTF8 $selectedFile
    Write-SelectionReport -Selected $selected -OutputFile $selectionReportFile

    $includeArg = ($selectedPaths -join ",")

    Write-Host "Packing selected files with Repomix: $($selected.Count) files"

    Push-Location $repoPath

    repomix `
        --output $repomixFile `
        --style markdown `
        --remove-comments `
        --remove-empty-lines `
        --compress `
        --include $includeArg

    Pop-Location

    $metaText = Get-Content -Raw $metadataFile
    $selectionText = Get-Content -Raw $selectionReportFile
    $packedText = Get-Content -Raw $repomixFile

    if ($packedText.Length -le $MaxPackedChars) {
        Write-Host "Summarizing selected pack directly with Ollama..."

        $prompt = @"
You are summarizing a large GitHub repository using a smart-selected subset of high-signal files.

Rules:
- Use only the provided repository metadata, selected-file list, and packed contents.
- Do not invent features.
- If selected files are insufficient, say what is missing.
- This repository may be huge, so infer carefully from manifests, entrypoints, configs, source files, tests, and infra files.
- Be specific and developer-useful.

Repository metadata:
$metaText

Selected file report:
$selectionText

Packed selected repository contents:
$packedText

Produce this Markdown exactly:

# $fullName

## One-liner
A single sentence.

## What it does
A grounded explanation of the project.

## Evidence used
Briefly describe what files made you conclude this.

## Tech stack
- Languages
- Frameworks
- Libraries
- Tools
- Databases or storage
- Infra

## Architecture
Explain major modules/directories and how they fit together.

## Important files and directories
- `path`: why it matters

## Entry points and commands
List run/build/test/deploy commands and the files they came from.

## Data, storage, and external services
Mention DBs, schemas, API clients, cloud services, auth, queues, files, etc.

## Tests and quality
Mention test setup, lint, CI, formatting, or lack of evidence.

## Deployment / infrastructure
Mention Docker, workflows, IaC, env vars, hosting clues.

## Risks, gaps, and unknowns
- Missing context or uncertain conclusions.

## Resume bullet
One strong bullet point.

## Portfolio description
3-5 polished sentences.
"@

        Invoke-OllamaPrompt -Model $Model -Prompt $prompt -OutputFile $summaryFile
    }
    else {
        Write-Host "Packed content is huge. Chunking before final summary..."

        $chunks = Split-TextChunks -Text $packedText -ChunkSize $ChunkChars
        $chunkSummaryFiles = @()

        for ($i = 0; $i -lt $chunks.Count; $i++) {
            $chunkNumber = $i + 1
            $chunkSummaryFile = Join-Path $chunkDir ("chunk-{0:D3}.summary.md" -f $chunkNumber)
            $chunkSummaryFiles += $chunkSummaryFile

            Write-Host "Summarizing chunk $chunkNumber / $($chunks.Count)..."

            $chunkPrompt = @"
You are summarizing one chunk of a large GitHub repository.

Rules:
- Use only this chunk.
- Extract concrete facts: files, functions/modules/classes, APIs, configs, commands, DB/storage, tests, infra.
- Do not write a final project summary yet.
- Focus on evidence and useful details.

Repository metadata:
$metaText

Selected file report:
$selectionText

Chunk $chunkNumber of $($chunks.Count):
$($chunks[$i])

Produce Markdown:

# Chunk $chunkNumber notes

## Files covered
List important paths visible in this chunk.

## Concrete findings
Bullet list of facts grounded in this chunk.

## Tech stack clues
Bullet list.

## Architecture clues
Bullet list.

## Entry point / command clues
Bullet list.

## Data / external service clues
Bullet list.

## Test / infra clues
Bullet list.

## Unknowns
Bullet list.
"@

            Invoke-OllamaPrompt -Model $Model -Prompt $chunkPrompt -OutputFile $chunkSummaryFile
        }

        $combinedChunkSummaries = ""
        foreach ($f in $chunkSummaryFiles) {
            $combinedChunkSummaries += "`n`n--- CHUNK SUMMARY: $f ---`n"
            $combinedChunkSummaries += Get-Content -Raw $f
        }

        Write-Host "Creating final repo summary from chunk summaries..."

        $finalPrompt = @"
You are writing the final summary of a large GitHub repository.

Rules:
- Use only the metadata, selected-file report, and chunk summaries.
- Do not invent features.
- Prefer concrete details over vague claims.
- Mention uncertainty where chunk summaries are incomplete.

Repository metadata:
$metaText

Selected file report:
$selectionText

Chunk summaries:
$combinedChunkSummaries

Produce this Markdown exactly:

# $fullName

## One-liner
A single sentence.

## What it does
A grounded explanation of the project.

## Evidence used
Briefly describe what files/chunks made you conclude this.

## Tech stack
- Languages
- Frameworks
- Libraries
- Tools
- Databases or storage
- Infra

## Architecture
Explain major modules/directories and how they fit together.

## Important files and directories
- `path`: why it matters

## Entry points and commands
List run/build/test/deploy commands and the files they came from.

## Data, storage, and external services
Mention DBs, schemas, API clients, cloud services, auth, queues, files, etc.

## Tests and quality
Mention test setup, lint, CI, formatting, or lack of evidence.

## Deployment / infrastructure
Mention Docker, workflows, IaC, env vars, hosting clues.

## Risks, gaps, and unknowns
- Missing context or uncertain conclusions.

## Resume bullet
One strong bullet point.

## Portfolio description
3-5 polished sentences.
"@

        Invoke-OllamaPrompt -Model $Model -Prompt $finalPrompt -OutputFile $summaryFile
    }

    $allSummaries += [ordered]@{
        repo = $fullName
        url = $repo.url
        summaryFile = $summaryFile
        selectedFiles = $selectedFile
        selectionReport = $selectionReportFile
        repomixFile = $repomixFile
        metadataFile = $metadataFile
    }
}

$indexJson = Join-Path $OutDir "index.json"
$indexMd = Join-Path $OutDir "ALL_PROJECT_SUMMARIES.md"

$allSummaries | ConvertTo-Json -Depth 10 | Set-Content -Encoding UTF8 $indexJson

"# All GitHub Project Summaries`n" | Set-Content -Encoding UTF8 $indexMd

foreach ($item in $allSummaries) {
    Add-Content -Encoding UTF8 $indexMd "`n---`n"
    Add-Content -Encoding UTF8 $indexMd "# $($item.repo)`n"
    Add-Content -Encoding UTF8 $indexMd "**Repo:** $($item.url)`n"
    Add-Content -Encoding UTF8 $indexMd "**Selected files:** $($item.selectedFiles)`n"
    Add-Content -Encoding UTF8 $indexMd "**Selection report:** $($item.selectionReport)`n"
    Add-Content -Encoding UTF8 $indexMd "**Repomix pack:** $($item.repomixFile)`n"
    Add-Content -Encoding UTF8 $indexMd "`n"
    Add-Content -Encoding UTF8 $indexMd (Get-Content -Raw $item.summaryFile)
}

Write-Host ""
Write-Host "Done." -ForegroundColor Green
Write-Host "Combined report: $indexMd"
Write-Host "Index JSON: $indexJson"