I drafted the code in here

bq-case-03352234

#!/bin/bash

# Case #03352234
# BQ load

# Riccardo modifications:
# 1. I added a sort/uniq to make sure the lines are unique.
# 2. I removed parallelism in bq commands.

#composed_files_gs_path='/path/to/gs/files'
#total_files=1000
#bqtable='nameoftable'

counter=0

gsutil ls $composed_files_gs_path > load.txt 
cat load.txt |sort|uniq| while read line ; do 
    echo "Loading file " $line "(" $current_file "out of "$total_files")" 
    if [ $counter -eq 5 ]; then 
        sleep 30 
        /usr/local/bin/bq load --nosync --max_bad_records=10000 --field_delimiter="\t" $bqtable $line 
        counter=1 
    else 
        /usr/local/bin/bq load --nosync --max_bad_records=10000 --field_delimiter="\t" $bqtable $line
        let counter=counter+1
    fi 
    let current_file=current_file+1
done