Multi GPU INT4/INT8/FP16 benchmark from long gemini session

mi50_multi_precision_bench.cpp

#include <hip/hip_runtime.h>
#include <hip/hip_fp16.h>
#include <iostream>
#include <vector>
#include <thread>
#include <chrono>
#include <iomanip>
#include <atomic>
#include <string>

enum PrecisionMode {
    PREC_INT4,
    PREC_INT8,
    PREC_FP16
};

// ---------------------------------------------------------------------------
// Compute Kernels
// ---------------------------------------------------------------------------

__global__ void __launch_bounds__(256, 2) mi50_int4_kernel(uint64_t iterations, int* dummy_out) {
    int src0 = 0x12345678;
    int src1 = 0x87654321;
    int acc0 = 0, acc1 = 0, acc2 = 0, acc3 = 0;
    int acc4 = 0, acc5 = 0, acc6 = 0, acc7 = 0;

    #pragma unroll 1
    for (uint64_t i = 0; i < iterations; ++i) {
        #pragma unroll
        for (int k = 0; k < 16; ++k) {
            acc0 = __builtin_amdgcn_sdot8(src0, src1, acc0, false);
            acc1 = __builtin_amdgcn_sdot8(src0, src1, acc1, false);
            acc2 = __builtin_amdgcn_sdot8(src0, src1, acc2, false);
            acc3 = __builtin_amdgcn_sdot8(src0, src1, acc3, false);
            acc4 = __builtin_amdgcn_sdot8(src0, src1, acc4, false);
            acc5 = __builtin_amdgcn_sdot8(src0, src1, acc5, false);
            acc6 = __builtin_amdgcn_sdot8(src0, src1, acc6, false);
            acc7 = __builtin_amdgcn_sdot8(src0, src1, acc7, false);
        }
    }

    if (threadIdx.x == 0 && blockIdx.x == 0) {
        *dummy_out = acc0 + acc1 + acc2 + acc3 + acc4 + acc5 + acc6 + acc7;
    }
}

__global__ void __launch_bounds__(256, 2) mi50_int8_kernel(uint64_t iterations, int* dummy_out) {
    int src0 = 0x12345678;
    int src1 = 0x87654321;
    int acc0 = 0, acc1 = 0, acc2 = 0, acc3 = 0;
    int acc4 = 0, acc5 = 0, acc6 = 0, acc7 = 0;

    #pragma unroll 1
    for (uint64_t i = 0; i < iterations; ++i) {
        #pragma unroll
        for (int k = 0; k < 16; ++k) {
            acc0 = __builtin_amdgcn_sdot4(src0, src1, acc0, false);
            acc1 = __builtin_amdgcn_sdot4(src0, src1, acc1, false);
            acc2 = __builtin_amdgcn_sdot4(src0, src1, acc2, false);
            acc3 = __builtin_amdgcn_sdot4(src0, src1, acc3, false);
            acc4 = __builtin_amdgcn_sdot4(src0, src1, acc4, false);
            acc5 = __builtin_amdgcn_sdot4(src0, src1, acc5, false);
            acc6 = __builtin_amdgcn_sdot4(src0, src1, acc6, false);
            acc7 = __builtin_amdgcn_sdot4(src0, src1, acc7, false);
        }
    }

    if (threadIdx.x == 0 && blockIdx.x == 0) {
        *dummy_out = acc0 + acc1 + acc2 + acc3 + acc4 + acc5 + acc6 + acc7;
    }
}

__global__ void __launch_bounds__(256, 2) mi50_fp16_kernel(uint64_t iterations, half2* dummy_out) {
    half2 src0 = __float2half2_rn(1.001f);
    half2 src1 = __float2half2_rn(0.999f);
    half2 acc0 = __float2half2_rn(0.1f);
    half2 acc1 = __float2half2_rn(0.2f);
    half2 acc2 = __float2half2_rn(0.3f);
    half2 acc3 = __float2half2_rn(0.4f);
    half2 acc4 = __float2half2_rn(0.5f);
    half2 acc5 = __float2half2_rn(0.6f);
    half2 acc6 = __float2half2_rn(0.7f);
    half2 acc7 = __float2half2_rn(0.8f);

    #pragma unroll 1
    for (uint64_t i = 0; i < iterations; ++i) {
        #pragma unroll
        for (int k = 0; k < 16; ++k) {
            acc0 = __hfma2(src0, src1, acc0);
            acc1 = __hfma2(src0, src1, acc1);
            acc2 = __hfma2(src0, src1, acc2);
            acc3 = __hfma2(src0, src1, acc3);
            acc4 = __hfma2(src0, src1, acc4);
            acc5 = __hfma2(src0, src1, acc5);
            acc6 = __hfma2(src0, src1, acc6);
            acc7 = __hfma2(src0, src1, acc7);
        }
    }

    if (threadIdx.x == 0 && blockIdx.x == 0) {
        *dummy_out = acc0 + acc1 + acc2 + acc3 + acc4 + acc5 + acc6 + acc7;
    }
}

// ---------------------------------------------------------------------------
// Worker Infrastructure
// ---------------------------------------------------------------------------

struct GpuResult {
    int dev_id;
    std::string name;
    float kernel_ms;
    double tops;
    double total_ops;
    bool success;
};

void worker_thread(int dev_id, PrecisionMode mode, uint64_t iterations, GpuResult* result,
                   std::atomic<int>& ready_count, std::atomic<bool>& start_flag) {
    result->dev_id = dev_id;
    result->success = false;

    if (hipSetDevice(dev_id) != hipSuccess) return;

    hipDeviceProp_t prop;
    if (hipGetDeviceProperties(&prop, dev_id) != hipSuccess) return;
    result->name = prop.name;

    const int num_blocks = 240; // 60 CUs * 4 blocks
    const int threads_per_block = 256;
    const double total_threads = (double)num_blocks * threads_per_block;

    double ops_per_thread_per_iter = 0.0;
    if (mode == PREC_INT4)      ops_per_thread_per_iter = 128.0 * 16.0; // 2048 ops
    else if (mode == PREC_INT8) ops_per_thread_per_iter = 128.0 * 8.0;  // 1024 ops
    else if (mode == PREC_FP16) ops_per_thread_per_iter = 128.0 * 4.0;  // 512 FLOPs

    result->total_ops = ops_per_thread_per_iter * (double)iterations * total_threads;

    void* d_out = nullptr;
    if (hipMalloc(&d_out, sizeof(double)) != hipSuccess) return;

    // Warmup pass
    if (mode == PREC_INT4) {
        mi50_int4_kernel<<<num_blocks, threads_per_block>>>(10000ULL, (int*)d_out);
    } else if (mode == PREC_INT8) {
        mi50_int8_kernel<<<num_blocks, threads_per_block>>>(10000ULL, (int*)d_out);
    } else if (mode == PREC_FP16) {
        mi50_fp16_kernel<<<num_blocks, threads_per_block>>>(10000ULL, (half2*)d_out);
    }
    
    if (hipDeviceSynchronize() != hipSuccess) {
        hipFree(d_out);
        return;
    }

    hipEvent_t start, stop;
    hipEventCreate(&start);
    hipEventCreate(&stop);

    ready_count++;

    while (!start_flag.load(std::memory_order_acquire)) {
        std::this_thread::yield();
    }

    // Synchronized launch
    hipEventRecord(start);
    if (mode == PREC_INT4) {
        mi50_int4_kernel<<<num_blocks, threads_per_block>>>(iterations, (int*)d_out);
    } else if (mode == PREC_INT8) {
        mi50_int8_kernel<<<num_blocks, threads_per_block>>>(iterations, (int*)d_out);
    } else if (mode == PREC_FP16) {
        mi50_fp16_kernel<<<num_blocks, threads_per_block>>>(iterations, (half2*)d_out);
    }
    hipEventRecord(stop);
    hipDeviceSynchronize();

    float milliseconds = 0;
    hipEventElapsedTime(&milliseconds, start, stop);

    double seconds = milliseconds / 1000.0;
    result->kernel_ms = milliseconds;
    result->tops = (result->total_ops / seconds) / 1e12;
    result->success = true;

    hipEventDestroy(start);
    hipEventDestroy(stop);
    hipFree(d_out);
}

double run_benchmark_pass(int device_count, PrecisionMode mode, const std::string& label, uint64_t iterations) {
    std::cout << "=============================================================================\n";
    std::cout << " RUNNING " << label << " BENCHMARK PASS (" << iterations << " iterations)\n";
    std::cout << "=============================================================================\n";

    std::vector<GpuResult> results(device_count);
    std::vector<std::thread> threads;
    std::atomic<int> ready_count(0);
    std::atomic<bool> start_flag(false);

    for (int i = 0; i < device_count; ++i) {
        threads.emplace_back(worker_thread, i, mode, iterations, &results[i], 
                             std::ref(ready_count), std::ref(start_flag));
    }

    while (ready_count.load(std::memory_order_relaxed) < device_count) {
        std::this_thread::yield();
    }

    auto wall_start = std::chrono::high_resolution_clock::now();
    start_flag.store(true, std::memory_order_release);

    for (auto& t : threads) {
        t.join();
    }

    auto wall_stop = std::chrono::high_resolution_clock::now();
    double wall_seconds = std::chrono::duration<double>(wall_stop - wall_start).count();

    std::cout << std::left << std::setw(10) << "Logical ID"
              << std::setw(30) << "Device Name"
              << std::setw(16) << "Time (ms)"
              << std::setw(16) << (mode == PREC_FP16 ? "Per-GPU TFLOPS" : "Per-GPU TOPS") << "\n";
    std::cout << std::string(72, '-') << "\n";

    double sum_ops = 0.0;
    int active_gpus = 0;

    for (const auto& r : results) {
        if (r.success) {
            std::cout << std::left << std::setw(10) << r.dev_id
                      << std::setw(30) << r.name
                      << std::setw(16) << std::fixed << std::setprecision(2) << r.kernel_ms
                      << std::setw(16) << std::setprecision(2) << r.tops << "\n";
            sum_ops += r.total_ops;
            active_gpus++;
        }
    }

    std::cout << std::string(72, '-') << "\n";
    double aggregate_throughput = (sum_ops / wall_seconds) / 1e12;

    std::cout << "Active GPUs:            " << active_gpus << " / " << device_count << "\n";
    std::cout << "Concurrent Wall Time:   " << std::fixed << std::setprecision(3) << wall_seconds * 1000.0 << " ms\n";
    std::cout << "AGGREGATE SYSTEM " << label << ": " << std::setprecision(2) << aggregate_throughput 
              << (mode == PREC_FP16 ? " TFLOPS" : " TOPS") << "\n\n";

    return aggregate_throughput;
}

int main() {
    int device_count = 0;
    if (hipGetDeviceCount(&device_count) != hipSuccess || device_count == 0) {
        std::cerr << "No valid HIP devices found." << std::endl;
        return 1;
    }

    std::cout << "Dispatched multi-precision benchmark across " << device_count << " visible GPU device(s)...\n\n";

    const uint64_t iterations = 1000000ULL;

    double agg_int4 = run_benchmark_pass(device_count, PREC_INT4, "INT4", iterations);
    double agg_int8 = run_benchmark_pass(device_count, PREC_INT8, "INT8", iterations);
    double agg_fp16 = run_benchmark_pass(device_count, PREC_FP16, "FP16", iterations);

    std::cout << "=============================================================================\n";
    std::cout << "                       SYSTEM SUMMARY PERFORMANCE                            \n";
    std::cout << "=============================================================================\n";
    std::cout << " INT4 Aggregate Compute:  " << std::fixed << std::setprecision(2) << agg_int4 << " TOPS\n";
    std::cout << " INT8 Aggregate Compute:  " << std::fixed << std::setprecision(2) << agg_int8 << " TOPS\n";
    std::cout << " FP16 Aggregate Compute:  " << std::fixed << std::setprecision(2) << agg_fp16 << " TFLOPS\n";
    std::cout << "=============================================================================\n";

    return 0;
}

Including two more gfx906 type GPUs (MI50 is gfx906 as well) via OpenMP did show 1.0003 POPS (peta operations per second)!
https://github.com/Hermann-SW/1.0003-POPS

=============================================================================
                   TOTAL 10-GPU CLUSTER PERFORMANCE                          
=============================================================================
 Combined INT4 Compute:  1000.37 TOPS
 Combined INT8 Compute:  500.05 TOPS
 Combined FP16 Compute:  260.13 TFLOPS
=============================================================================