running Stanford Classifier on web request data

logs.prop

#
# Features
#
useClassFeature=true
#
# generate features on url
10.useSplitNGrams=true
10.usePrefixSuffixNGrams=true
#
# generate features on mimetype
12.useSplitNGrams=true
12.usePrefixSuffixNGrams=true
#
# generate features on UA
15.useSplitNGrams=true
15.usePrefixSuffixNGrams=true
#
# Printing
#
printClassifier=HighWeight
printClassifierParam=200
#
# Mapping
#
goldAnswerColumn=0
#
# Optimization
#
intern=true
sigma=3
useQN=true
QNsize=15
tolerance=1e-4
#
# Training input
#
trainFile=logs.train
testFile=logs.test

run.sh

zcat /a/squid/archive/sampled/sampled-1000.log-20121101.gz | head -n 10000 | sed 's/\s/\t/g' | awk 'NF==14{print}{}' | head -n 90000 | sed 's/\(.*\)/pre\t\1/' > logs.train
zcat /a/squid/archive/sampled/sampled-1000.log-20121201.gz | head -n 10000 | sed 's/\s/\t/g' | awk 'NF==14{print}{}' | head -n 90000 | sed 's/\(.*\)/post\t\1/' > logs.train

zcat /a/squid/archive/sampled/sampled-1000.log-20121102.gz | head -n 10000 | sed 's/\s/\t/g' | awk 'NF==14{print}{}' | head -n 90000 | sed 's/\(.*\)/pre\t\1/' > logs.test
zcat /a/squid/archive/sampled/sampled-1000.log-20121202.gz | head -n 10000 | sed 's/\s/\t/g' | awk 'NF==14{print}{}' | head -n 90000 | sed 's/\(.*\)/post\t\1/' > logs.test

java -jar stanford-classifier.jar -prop logs.prop -printClassifier HighWeight