ScrollThingi

scroll.coffee

query =
  #hidden


Q = require('q')
qhttp = require 'q-io/http'

get_scroll_id = ->
  request = {
    method: "POST"
    body: [JSON.stringify(query)]
    url: 'http://localhost:19201/_search?search_type=scan&scroll=10m&size=50'
  }

  qhttp.request(request)
  .then((response) -> response.body.read())
  .then((resp) -> JSON.parse(resp.toString()))
  .then((json) -> console.log "TOTAL:", json.hits.total; json._scroll_id)

get_next_set = (scroll_id) ->
  request = {
    method: "GET"
    url: "http://localhost:19201/_search/scroll/#{scroll_id}?scroll=10m"
  }
  qhttp.request(request)
  .then((response) -> response.body.read())
  .then((resp) -> JSON.parse(resp.toString()))
  .then((json) -> json.hits.hits)
  .then((hits) -> {scroll_id: scroll_id, hits: hits})

process_hit = (hit) ->
  #hidden

process_and_check_for_end = (hits, scroll_id) ->
  nhits = (process_hit(hit) for hit in hits)
  if hits.length > 0
    return get_next_set(scroll_id).delay(1000).then((cont) -> process_and_check_for_end(cont.hits, cont.scroll_id))
  else 
    console.log("FINISHED")

Q.fcall(-> console.log "Starting to download stuff")
.then( -> get_scroll_id())
.then( (scroll_id) -> get_next_set(scroll_id))
.then( (cont) -> process_and_check_for_end(cont.hits, cont.scroll_id))
.fail(console.log)