Use bulk npgsql to export data pretty fast

readme.md

Make sure you have installed the powershell module ImportDependency and Pwsh

Create a lib subdirectory like in this path C:\faststats\scripts\npgbulk and put the unzipped nuget packages npgsql.10.0.1 and microsoft.extensions.logging.abstractions.10.0.3 in there.

This script exports like 4m rows just in a few seconds.

SL "C:\faststats\scripts\npgbulk"

Import-Dependency -LoadWholePackageFolder -verbose

$connStr = 'database=xxx;username=xxx;host=127.0.0.1;port=1111;password=xxx'
$conn = New-Object Npgsql.NpgsqlConnection($connStr)
$conn.Open()

$exportQuery = @"
COPY (
    SELECT *
    FROM schema.transactions
    ORDER BY customer_id
    --LIMIT 100
) TO STDOUT (FORMAT CSV, HEADER)
"@

$copyReader = $conn.BeginTextExport($exportQuery)  # FORMAT CSV in query
#$streamReader = [System.IO.StreamReader]::new($copyReader)
$fileStream = [System.IO.File]::OpenWrite("C:\faststats\scripts\npgbulk\big_table.csv")
$writer = [System.IO.StreamWriter]::new($fileStream)

$i = 0
$line = $copyReader.ReadLine()
while ($line -ne $null) {
    $writer.WriteLine($line)
    $line = $copyReader.ReadLine()
    $i += 1
    if ($i % 100000 -eq 0) {
        "Done $i rows"
    }
}

$writer.Flush()
$writer.Close()
$copyReader.Close()
$conn.Close()

"Done $( $i - 1) rows"