hanzlfs

from scrapy.spider import  Spider
from scrapy.contrib.spiders import CrawlSpider, Rule
from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
from scrapy.selector import Selector
from scrapy.item import Item, Field
import urllib

class Question(Item):
    tags = Field()
    answers = Field()

hanzlfs

Fractional Max-Pooling (FMP)

Introduction

• Link to Paper
• Spatial pooling layers are building blocks for Convolutional Neural Networks (CNNs).
• Input to pooling operation is a N_in x N_in matrix and output is a smaller matrix N_out x N_out.
• Pooling operation divides N_in x N_in square into N²_out pooling regions P_{i, j}.
• P_{i, j} ⊂ {1, 2, . . . , N_in} ∀ (i, j) ∈ {1, . . . , N_out}²

hanzlfs

pip install https://github.com/fchollet/keras/tarball/$version$

hanzlfs

create bash

$ docker exec -i -t [Container Id or Name] bash

spark jobserver link

run with sudo

docker run -e GRANT_SUDO=yes -d -p 8888:8888 jupyter/all-spark-notebook start-notebook.sh 

hanzlfs

• reference
  • https://www.dataquest.io/blog/installing-pyspark/
  • http://npatta01.github.io/2015/08/01/pyspark_jupyter/
• steps
  • test with pyspark : run $SPARK_HOME$/bin/pyspark
  • Environment variables :
    • export SPARK_HOME="$HOME/spark-1.6xx"
    • export PYSPARK_SUBMIT_ARGS="--master local[2]"
    • export PATH=$SPARK_HOME/bin:$PATH
    • export PYTHONPATH=$SPARK_HOME/python/:$PYTHONPATH

hanzlfs

"""Information Retrieval metrics

Useful Resources:
http://www.cs.utexas.edu/~mooney/ir-course/slides/Evaluation.ppt
http://www.nii.ac.jp/TechReports/05-014E.pdf
http://www.stanford.edu/class/cs276/handouts/EvaluationNew-handout-6-per.pdf
http://hal.archives-ouvertes.fr/docs/00/72/67/60/PDF/07-busa-fekete.pdf
Learning to Rank for Information Retrieval (Tie-Yan Liu)
"""
import numpy as np