Simple script to count word frequency and line length for a given set of text files.

core-vocab-analysis.sh

#!/bin/bash
cd $1
# CONFIG VARIABLES
ignoreChars=',.?!'

for filename in ./*.txt; do
	outputFile=$(basename $filename .txt)'_'"$(date '+%Y-%m-%d-%H:%M:%S')"'.txt'
	touch $outputFile

	echo '****************************' >> $outputFile
	echo '*** Gloss word frequency ***' >> $outputFile
	echo '****************************' >> $outputFile
	echo ' ' >> $outputFile
	cat $filename | tr '/' ' ' | sed 's|['$ignoreChars']||gI' | sed 's|\s\s|\s|g' | sed 's|\s$||g' | tr ' ' '\n' | sort | uniq -ic | sort -nr >> $outputFile

	echo ' ' >> $outputFile
	echo ' ' >> $outputFile
	echo '**********************' >> $outputFile
	echo '*** Words per line ***' >> $outputFile
	echo '**********************' >> $outputFile
	echo ' ' >> $outputFile
	lineNum=1
	cat $filename | while read line;do
	  echo $line | wc -w >> $outputFile
	  ((lineNum++))
	done
done