getline

#!/bin/bash

echo "Generating files"
(seq 1 1000000 ; seq 10000000 19000000) | shuf > 20M.a.txt
(seq 1 1000000 ; seq 20000000 29000000) | shuf > 20M.b.txt

echo "Test Python"
cat > main.py <<EOF
import sys

print(
    len(
        set(
            open(sys.argv[1]).readlines()
        ).intersection(
            set(
                open(sys.argv[2]).readlines()
            )
        )
    )
)
EOF
time python main.py 20M.a.txt 20M.b.txt

echo "Test CPP"
cat > main.cpp <<EOF
#include <fstream>
#include <iostream>
#include <set>
#include <unordered_set>
#include <vector>

int main(int argc, char **argv)
{
    std::string line;
    std::unordered_set<std::string> set_a;
    std::unordered_set<std::string> set_b;

    std::ifstream file_a(argv[1]);
    while (std::getline(file_a, line))
    {
        set_a.insert(line);
    }
    file_a.close();

    std::ifstream file_b(argv[2]);
    while (std::getline(file_b, line))
    {
        set_b.insert(line);
    }
    file_b.close();

    int common = 0;
    for (std::unordered_set<std::string>::iterator it = set_a.begin(); it != set_a.end(); ++it)
    {
        if (set_b.find(*it) != set_b.end())
        {
            ++common;
        }
    }

    std::cout << common << std::endl;
}
EOF

g++ -O2 -o main main.cpp
time ./main 20M.a.txt 20M.b.txt