aibs-data-harvest

#! /usr/bin/env perl
#
# http://help.brain-map.org/display/api/Downloading+an+Image
#
# sudo apt-get install libxml-simple-perl


use warnings "all";
use Data::Dumper;

use XML::Simple qw(:strict);


$opt{'fake'} = 0;
$opt{'verbose'} = 1;

$ds = 2;
$api_url = "http://api.brain-map.org/api/v2";

# get ids 
# wget -O all.xml "http://api.brain-map.org/api/v2/data/query.xml?criteria=model::SectionDataSet,rma::criteria,products%5Bid$eq5%5D,rma::include,specimen(injections(primary_injection_structure,structure))"
#
# start_row=0&num_rows=1
$total_rows = 1390;
$step = 50;
@xmls = ();
for($i=0; $i<=$total_rows; $i+=$step){
   $fname = sprintf("xmls/all-data_%05d.xml", $i);
   
   if(!-e $fname){
   
      &do_cmd('wget', '-O', $fname,
	     'http://api.brain-map.org/api/v2/data/query.xml?' . 
	     'criteria=model::SectionDataSet,rma::criteria,products%5Bid$eq5%5D,' .
	     'rma::include,specimen(injections(primary_injection_structure,structure))' .
	     "&start_row=$i&num_rows=50");
	  }
   push(@xmls, $fname);
   }

@ids = ();
foreach $xml (@xmls){
   push(@ids, split(/\n/, `grep "<id>" $xml`));
   }

foreach (@ids){
   $_ =~ s/.*\<id\>//g;
   $_ =~ s/\<\/id\>.*//g;
   }

print "IDS[$#ids]: " . join(" ", @ids) . "\n";

open(FH, ">all-ids.txt");
print FH join("\n", @ids);
close(FH);

#$id = 126862385;
#$id = 100141599;
#$id=100141273;
#$id=264076081;

# shuf --head 200 all-ids.txt > ids-toget.txt

@ids = split(/\n/, `cat ids-toget.txt`);


foreach $id (@ids){
   $mncfile = "$ds-$id.r.mnc";

   print "+++ Working on $id - $mncfile +++\n";

   if(-e $mncfile){
	  print "  e $mncfile exists -- skippingn";
	  next;
	  }

   $fname = "$id.xml";
   if(!-e $fname){
      &do_cmd('wget', '--continue', '-O', $fname,
         "$api_url/data/SectionDataSet/$id.xml?include=equalization,section_images");
      }
   
   $xml = `cat $id.xml`;
   my $config = XMLin($xml, ForceArray => 0, KeyAttr => []);
   
   
   # print Dumper($config);
   
   &do_cmd('mkdir', '-p', $id);
   
   # get normalisation
   $r_lower = $config->{'section-data-sets'}{'section-data-set'}{'equalization'}{'red-lower'}{'content'};
   $r_upper = $config->{'section-data-sets'}{'section-data-set'}{'equalization'}{'red-upper'}{'content'};
   $g_lower = $config->{'section-data-sets'}{'section-data-set'}{'equalization'}{'green-lower'}{'content'};
   $g_upper = $config->{'section-data-sets'}{'section-data-set'}{'equalization'}{'green-upper'}{'content'};
   $b_lower = $config->{'section-data-sets'}{'section-data-set'}{'equalization'}{'blue-lower'}{'content'};
   $b_upper = $config->{'section-data-sets'}{'section-data-set'}{'equalization'}{'blue-upper'}{'content'};
   
   print "   [$id] NORM: [$r_lower:$r_upper|$g_lower:$g_upper|$b_lower:$b_upper]\n";
   
   
   # images

   $n_subimages = scalar(@{$config->{'section-data-sets'}{'section-data-set'}{'section-images'}{'section-image'}});
   print "   [$id] Found $n_subimages sub images\n";

   $n = 0;
   $xyres = undef;
   @r_rawfiles = ();
   @g_rawfiles = ();
   @b_rawfiles = ();
   foreach $si (@{$config->{'section-data-sets'}{'section-data-set'}{'section-images'}{'section-image'}}){
      
      $w = $si->{'width'}{'content'};
      $h = $si->{'height'}{'content'};
      $sid = $si->{'id'}{'content'};
      $n = $si->{'section-number'}{'content'};
      print "   [$id]   WxH [$w x $h] -- $sid  ($n)\n";
      
      # get and check resolution
      $xyres = $si->{'resolution'}{'content'} if !defined($xyres);
      
      if($xyres != $si->{'resolution'}{'content'}){
         warn "\n!!! Resolution does not match $xyres !!!\n\n"
         } 
      
      $fname = sprintf("$id/$ds-$id-%04d.jpg", $n);
      $fname_r = sprintf("$id/$ds-$id-%04d.r.raw", $n);
      $fname_g = sprintf("$id/$ds-$id-%04d.g.raw", $n);
      $fname_b = sprintf("$id/$ds-$id-%04d.b.raw", $n);
      
      if(!-e $fname){
   	  
   	 $url = "$api_url/section_image_download/$sid?" .
   	    "downsample=$ds&" .
   	    "quality=100&" . 
   	    "range=$r_lower,$r_upper,$g_lower,$g_upper,$b_lower,$b_upper";
   	 
        &do_cmd('wget', '--continue', '-O', $fname, $url);
        }
      
      #if(!-e $fname_r || !-e $fname_g || !-e $fname_b){
      if(!-e $fname_r){
   	   
         # convert to RAW
         #&do_cmd('convert', '-type', 'Grayscale', $fname, "GRAY:$fname_r");
         &do_cmd('convert', '-channel', 'R', '-separate', $fname, "GRAY:$fname_r");
         #&do_cmd('convert', '-channel', 'G', '-separate', $fname, "GRAY:$fname_g");
         #&do_cmd('convert', '-channel', 'B', '-separate', $fname, "GRAY:$fname_b");
         }
   	
      push(@r_rawfiles, $fname_r);
      push(@g_rawfiles, $fname_g);
      push(@b_rawfiles, $fname_b);
      $n++;
      }
   
   
   
   print "   [$id] INF W H: [$#r_rawfiles][$w][$h] -- " . (2 ** $ds) . "\n";
   
   $ny = $#r_rawfiles + 1;
   $nz = int($h / (2 ** $ds));
   $nx = int($w / (2 ** $ds));
   print "   [$id] ZYX: [$nz][$ny][$nx]\n";
   
   $ystep = -1 * 0.1;
   $zstep = -1 * $xyres / 1000 * (2 ** $ds);
   $xstep = -1 * $xyres / 1000 * (2 ** $ds);
   print "   [$id] SZYX: [$zstep][$ystep][$xstep]\n";
   
   $ystart = -1 * $ny * $ystep / 2;
   $zstart = -1 * $nz * $zstep / 2;
   $xstart = -1 * $nx * $xstep / 2;
   print "   [$id] STZYX: [$zstart][$ystart][$xstart]\n";
   
   
   # convert to MINC
   if(!-e $mncfile){
      $inf = join(" ", sort(@r_rawfiles));
      &do_cmd("cat $inf > all_r.raw");
      #$inf = join(" ", sort(@g_rawfiles));
      #&do_cmd("cat $inf > all_g.raw");
      #$inf = join(" ", sort(@b_rawfiles));
      #&do_cmd("cat $inf > all_b.raw");
   
      &do_cmd('rawtominc', '-2', '-clobber',
         '-yzx', 
         '-input', "all_r.raw",
         '-xstep', $xstep, '-ystep', $ystep, '-zstep', $zstep,
         '-xstart', $xstart, '-ystart', $ystart, '-zstart', $zstart,
         '-byte', $mncfile, $ny, $nz, $nx);
      }
   
   #&do_cmd('rawtominc', '-2', '-clobber',
   #   '-yzx', 
   #   '-input', "all_g.raw",
   #   '-xstep', $xstep, '-ystep', $ystep, '-zstep', $zstep,
   #   '-xstart', $xstart, '-ystart', $ystart, '-zstart', $zstart,
   #   '-byte', "$ds-$id.g.mnc", $ny, $nz, $nx);
   
   #&do_cmd('rawtominc', '-2', '-clobber',
   #   '-yzx', 
   #   '-input', "all_b.raw",
   #   '-xstep', $xstep, '-ystep', $ystep, '-zstep', $zstep,
   #   '-xstart', $xstart, '-ystart', $ystart, '-zstart', $zstart,
   #   '-byte', "$ds-$id.b.mnc", $ny, $nz, $nx);

   }


sub do_cmd { 
   print STDOUT "@_\n" if $opt{'verbose'};
   if(!$opt{'fake'}){
      system(@_) == 0 or die;
   }
}