biefly compare pandas/polars load speed

run.py

"""
vibration wave data, for a full (100000, 3) file, the size is 1,070,89KB
"""

if __name__ == "__main__":
    
    time_spent = []

    for file in os.listdir(WAVE_DATA_DIR):
        start = time.time()

        df = pl.read_csv(os.path.join(WAVE_DATA_DIR, file))

        print(df.shape)

        # get the end timestamp in seconds
        end = time.time()

        diff = end - start
        print(f"Time spent: {diff} seconds")

        time_spent.append(diff)

    print(f"Polars Average time spent: {sum(time_spent)/len(time_spent)} seconds")

    
    time_spent = []

    for file in os.listdir(WAVE_DATA_DIR):
        start = time.time()

        df = pd.read_csv(os.path.join(WAVE_DATA_DIR, file))

        print(df.shape)

        # get the end timestamp in seconds
        end = time.time()

        diff = end - start
        print(f"Time spent: {diff} seconds")

        time_spent.append(diff)

    print(f"Pands Average time spent: {sum(time_spent)/len(time_spent)} seconds")
    
    
    
""""
(100000, 3)
Time spent: 2.0111846923828125 seconds
(100000, 3)
Time spent: 2.2304136753082275 seconds
(100000, 3)
Time spent: 2.110846757888794 seconds
(100000, 3)
Time spent: 2.1382851600646973 seconds
(100000, 3)
Time spent: 2.308527946472168 seconds
(100000, 3)
Time spent: 2.185439348220825 seconds
(100000, 3)
Time spent: 2.284050703048706 seconds
(100000, 3)
Time spent: 2.1947689056396484 seconds
(100000, 3)
Time spent: 2.4297051429748535 seconds
(100000, 3)
Time spent: 2.2954442501068115 seconds
(100000, 3)
Time spent: 2.3191771507263184 seconds
(100000, 3)
Time spent: 2.443420171737671 seconds
(100000, 3)
Time spent: 2.5716891288757324 seconds
(100000, 3)
Time spent: 2.475531578063965 seconds
(100000, 3)
Time spent: 2.692166566848755 seconds
(100000, 3)
Time spent: 2.4687774181365967 seconds
(100000, 3)
Time spent: 2.3710458278656006 seconds
(100000, 3)
Time spent: 2.3846435546875 seconds
(100000, 3)
Time spent: 2.297212600708008 seconds
(100000, 3)
Time spent: 2.176015853881836 seconds
(56748, 3)
Time spent: 1.385786771774292 seconds
Polars Average time spent: 2.2749587240673246 seconds
(100000, 3)
Time spent: 9.235079765319824 seconds
(100000, 3)
Time spent: 9.712808847427368 seconds
(100000, 3)
Time spent: 9.46042799949646 seconds
(100000, 3)
Time spent: 9.677271366119385 seconds
(100000, 3)
Time spent: 9.790785312652588 seconds
(100000, 3)
Time spent: 9.804165840148926 seconds
(100000, 3)
Time spent: 9.894776821136475 seconds
(100000, 3)
Time spent: 9.980101108551025 seconds
(100000, 3)
Time spent: 10.19819712638855 seconds
(100000, 3)
Time spent: 10.110341548919678 seconds
(100000, 3)
Time spent: 10.195951223373413 seconds
(100000, 3)
Time spent: 10.533878564834595 seconds
(100000, 3)
Time spent: 12.463850498199463 seconds
(100000, 3)
Time spent: 9.985363245010376 seconds
(100000, 3)
Time spent: 10.521904468536377 seconds
(100000, 3)
Time spent: 10.794193267822266 seconds
(100000, 3)
Time spent: 10.94998574256897 seconds
(100000, 3)
Time spent: 11.566393613815308 seconds
(100000, 3)
Time spent: 11.53919243812561 seconds
(100000, 3)
Time spent: 11.943904876708984 seconds
(56748, 3)
Time spent: 7.477388620376587 seconds
Pands Average time spent: 10.277902966453915 seconds
"""