benchmark: compare the memory efficiency of the streaming and inefficient methods directly in the benchmark results.

main.cs

using System;
using System.IO;
using System.Linq;
using BenchmarkDotNet.Attributes;
using BenchmarkDotNet.Running;

[MemoryDiagnoser] // Enables memory usage tracking
public class FileProcessingBenchmark
{
    private const string FilePath = "largeDataset.txt";

    [GlobalSetup]
    public void Setup()
    {
        if (!File.Exists(FilePath))
        {
            Console.WriteLine("Generating test data...");
            using (var writer = new StreamWriter(FilePath))
            {
                var random = new Random();
                for (int i = 0; i < 10_000_00; i++) // 1 million lines
                {
                    writer.WriteLine(random.Next(1, 100));
                }
            }
            Console.WriteLine("Test data generated.");
        }
    }

    [Benchmark]
    public long StreamingMethod()
    {
        long sum = 0;
        using (var streamReader = new StreamReader(FilePath))
        {
            string line;
            while ((line = streamReader.ReadLine()) != null)
            {
                if (long.TryParse(line, out long number))
                {
                    sum += number;
                }
            }
        }
        return sum;
    }

    [Benchmark]
    public long InefficientMethod()
    {
        var allLines = File.ReadAllLines(FilePath);
        return allLines
            .Select(line => long.TryParse(line, out long number) ? number : 0)
            .Sum();
    }
}

class Program
{
    static void Main(string[] args)
    {
        var summary = BenchmarkRunner.Run<FileProcessingBenchmark>();
    }
}

Run

dotnet run -c Release

Result

BenchmarkDotNet v0.14.0, Ubuntu 20.04.2 LTS (Focal Fossa)
AMD EPYC 7B13, 1 CPU, 8 logical and 4 physical cores
.NET SDK 7.0.410
  [Host]     : .NET 7.0.20 (7.0.2024.26716), X64 RyuJIT AVX2
  DefaultJob : .NET 7.0.20 (7.0.2024.26716), X64 RyuJIT AVX2

Method	Mean	Error	StdDev	Gen0	Gen1	Gen2	Allocated
StreamingMethod	32.76 ms	0.681 ms	1.987 ms	1875.0000	-	-	30.26 MB
InefficientMethod	122.73 ms	3.295 ms	9.347 ms	3600.0000	3600.0000	1800.0000	53.9 MB

Streaming Method:

Faster for large datasets due to reduced memory usage.

Inefficient Method:

Slower and consumes significantly more memory.

.net 8

BenchmarkDotNet v0.14.0, Ubuntu 20.04.2 LTS (Focal Fossa)
AMD EPYC 7B13, 1 CPU, 8 logical and 4 physical cores
.NET SDK 8.0.404
  [Host]     : .NET 8.0.11 (8.0.1124.51707), X64 RyuJIT AVX2
  DefaultJob : .NET 8.0.11 (8.0.1124.51707), X64 RyuJIT AVX2

Method	Mean	Error	StdDev	Gen0	Gen1	Gen2	Allocated
StreamingMethod	39.28 ms	1.631 ms	4.706 ms	1818.1818	-	-	29.83 MB
InefficientMethod	109.94 ms	3.555 ms	10.369 ms	2800.0000	2800.0000	1200.0000	53.46 MB