statistical significance filter

stat_sig_filter.m

% small script to show how stat significance filters results and leads to
% overestimation of effect size if only positive findings are considered


clear;
clc;

STD = 1; % Standard deviation of the effect (true std of the population we are modelling)
UES = 0.1:.1:1; % Unstandardized Effect size (true mean of the population we are modelling)
SES = UES/STD; % Standardized effect size (for info)

SS = [8:6:32 40:10:70]; % Sample size we wanna try
NbSample = 20000; % Number fo samples for each sample size

% Run the simulation
for iSS=1:numel(SS)
    for iES=1:numel(UES)
        Sample = UES(iES) + randn(SS(iSS),NbSample)*STD; %Model the sample
        H_05(:,iES,iSS) = ttest(Sample,0,'alpha',0.05); %See if different from 0
        H_005(:,iES,iSS) = ttest(Sample,0,'alpha',0.005); %See if different from 0
        EmpES(:,iES,iSS) = mean(Sample)./std(Sample);
    end
end

%%
close all

COLOR = linspace(0.75,0,numel(UES));

figure(1)

subplot(153)
hold on
for iES = 1:numel(UES)
    for iSS=1:numel(SS)
        tmp(iSS)= mean(EmpES(H_005(:,iES,iSS)==1,iES,iSS));
    end
    plot( SS , tmp, 'color', repmat(COLOR(iES),3,1))
end
set(gca,'xtick', SS, 'xticklabel', SS)
xlabel('Sample size')
ylabel('Empirical effect size when p<.005')
axis([7  71    0.2   2]);
set(gca, 'fontsize', 8)

subplot(151)
hold on
for iES = 1:numel(UES)
    plot( SS , mean(squeeze(EmpES(:,iES,:))), 'color', repmat(COLOR(iES),3,1))
end
set(gca,'xtick', SS, 'xticklabel', SS)
xlabel('Sample size')
ylabel('Empirical effect size')
axis([7  71    0.2   2]);
set(gca, 'fontsize', 8)

subplot(152)
hold on
for iES = 1:numel(UES)
    for iSS=1:numel(SS)
        tmp(iSS)= mean(EmpES(H_05(:,iES,iSS)==1,iES,iSS));
    end
    plot( SS , tmp, 'color', repmat(COLOR(iES),3,1))
end
set(gca,'xtick', SS, 'xticklabel', SS)
xlabel('Sample size')
ylabel('Empirical effect size when p<.05')
axis([7  71    0.2   2]);
set(gca, 'fontsize', 8)



subplot(154)
hold on
for iES = 1:numel(UES)
    plot( SS , mean(squeeze(H_05(:,iES,:))), 'color', repmat(COLOR(iES),3,1))
end
set(gca,'xtick', SS, 'xticklabel', SS)
xlabel('Sample size')
ylabel('Proportion of p<.05')
set(gca, 'fontsize', 8)


subplot(155)
hold on
for iES = 1:numel(UES)
    plot( SS , mean(squeeze(H_005(:,iES,:))), 'color', repmat(COLOR(iES),3,1))
end
set(gca,'xtick', SS, 'xticklabel', SS)
xlabel('Sample size')
ylabel('Proportion of p<.005')
set(gca, 'fontsize', 8)