Demonstrate 77 TFLOPS FP16 (__half) on NVIDIA RTX 5060 GPU

benchmark_fp16.cu

#include <iostream>
#include <vector>
#include <cuda_runtime.h>
#include <cublas_v2.h>

#define CUDA_CHECK(status) \
    if (status != cudaSuccess) { \
        std::cerr << "CUDA Error: " << cudaGetErrorString(status) \
                  << " at line " << __LINE__ << std::endl; \
        exit(EXIT_FAILURE); \
    }

#define CUBLAS_CHECK(status) \
    if (status != CUBLAS_STATUS_SUCCESS) { \
        std::cerr << "cuBLAS Error: " << status \
                  << " at line " << __LINE__ << std::endl; \
        exit(EXIT_FAILURE); \
    }

void run_fp16_benchmark(cublasHandle_t handle, int M, int N, int K, int warmup = 10, int bench = 100) {
    size_t size_A = (size_t)M * K * sizeof(__half);
    size_t size_B = (size_t)K * N * sizeof(__half);
    size_t size_C = (size_t)M * N * sizeof(__half);

    __half *d_A, *d_B, *d_C;
    CUDA_CHECK(cudaMalloc(&d_A, size_A));
    CUDA_CHECK(cudaMalloc(&d_B, size_B));
    CUDA_CHECK(cudaMalloc(&d_C, size_C));

    CUDA_CHECK(cudaMemset(d_A, 0x3C, size_A));
    CUDA_CHECK(cudaMemset(d_B, 0x3C, size_B));
    CUDA_CHECK(cudaMemset(d_C, 0, size_C));

    // For CUBLAS_COMPUTE_16F, alpha and beta must be __half
    __half alpha = __float2half(1.0f);
    __half beta  = __float2half(0.0f);

    // Warmup
    for (int i = 0; i < warmup; ++i) {
        CUBLAS_CHECK(cublasGemmEx(
            handle, 
            CUBLAS_OP_N, CUBLAS_OP_N,
            M, N, K,
            &alpha, 
            d_A, CUDA_R_16F, M,
            d_B, CUDA_R_16F, K,
            &beta, 
            d_C, CUDA_R_16F, M,
            CUBLAS_COMPUTE_16F,
            CUBLAS_GEMM_DEFAULT_TENSOR_OP
        ));
    }
    CUDA_CHECK(cudaDeviceSynchronize());

    cudaEvent_t start, stop;
    CUDA_CHECK(cudaEventCreate(&start));
    CUDA_CHECK(cudaEventCreate(&stop));

    // Timed Loop
    CUDA_CHECK(cudaEventRecord(start));
    for (int i = 0; i < bench; ++i) {
        CUBLAS_CHECK(cublasGemmEx(
            handle, 
            CUBLAS_OP_N, CUBLAS_OP_N,
            M, N, K,
            &alpha, 
            d_A, CUDA_R_16F, M,
            d_B, CUDA_R_16F, K,
            &beta, 
            d_C, CUDA_R_16F, M,
            CUBLAS_COMPUTE_16F,
            CUBLAS_GEMM_DEFAULT_TENSOR_OP
        ));
    }
    CUDA_CHECK(cudaEventRecord(stop));
    CUDA_CHECK(cudaEventSynchronize(stop));

    float total_ms = 0;
    CUDA_CHECK(cudaEventElapsedTime(&total_ms, start, stop));

    double avg_ms = total_ms / bench;
    double total_flops = 2.0 * (double)M * (double)N * (double)K;
    double tflops = (total_flops / (avg_ms * 1e-3)) / 1e12;

    std::cout << "--------------------------------------------------------\n";
    std::cout << " Matrix Size (M x N x K): " << M << " x " << N << " x " << K << "\n";
    std::cout << " Average Time per GEMM  : " << avg_ms << " ms\n";
    std::cout << " Sustained FP16 TFLOPS  : " << tflops << " TFLOPS\n";
    std::cout << "--------------------------------------------------------\n";

    CUDA_CHECK(cudaFree(d_A));
    CUDA_CHECK(cudaFree(d_B));
    CUDA_CHECK(cudaFree(d_C));
}

int main() {
    int deviceCount = 0;
    CUDA_CHECK(cudaGetDeviceCount(&deviceCount));

    cudaDeviceProp prop;
    CUDA_CHECK(cudaGetDeviceProperties(&prop, 0));
    std::cout << "Running FP16 Benchmark on GPU 0: " << prop.name
              << " (Compute Capability: " << prop.major << "." << prop.minor << ")\n";

    cublasHandle_t handle;
    CUBLAS_CHECK(cublasCreate(&handle));

    std::vector<int> dimensions = {4096, 8192, 16384};
    for (int dim : dimensions) {
        run_fp16_benchmark(handle, dim, dim, dim);
    }

    cublasDestroy(handle);
    return 0;
}

More than 1 POPS (1000 TOPS) shown here (10 old server GPUs):
https://github.com/Hermann-SW/1.0003-POPS/blob/main/README.md#10003-pops