remove correlated columns in pandas DataFrame

rm_corr.py

# remove low variance columns
rna_data = rna_data.loc[:, (rna_data.var() > 0.5)]

# remove correlated columns
vars = rna_data.var()

tbd = []

# from each pair of highly correlated columns, remove columns with lower variance
for i, j in zip(*np.where(np.corrcoef(rna_data.values.T) > 0.5)):
    if i < j:
        if vars.iloc[i] < vars.iloc[j]:
            tbd.append(i)
        else:
            tbd.append(j)

rna_data = rna_data.drop(rna_data.columns[tbd], axis=1)

# scale and center values
rna_data -= rna_data.mean()
rna_data /= rna_data.std()

data = pd.concat((clinical_data.loc[:,'tumor_site'], rna_data), axis=1, join='inner')