#!/usr/sh
to_dir="dostoj"
from="http://www.zeno.org/Literatur/M/Dostoevskij,+F%C3%ABdor+Michajlovi%C4%8D/Romane/Der+Idiot/Erster+Teil"
data="Kapitel"
if [ ! -d "$to_dir" ]; then
  mkdir $to_dir
fi
for i in {1..16}; do
	wget -O "$i.$data.htm" "$from/$i."
	lynx -dump "$i.$data.htm" >> $to_dir/erstesBuch.txt
done



#!/usr/sh
to_dir="dostoj"
file=$to_dir/erstesBuch.txt
file_freq="$to_dir/cleanbook.txt"
echo "cat $file | tr -s ' ' '\n' | tr -d [[:punct:]] | sort -f | uniq -c | sort -nr > $file_freq"
cat $file | tr -s " " "\n" | tr -d [[:punct:]] | sort -f | uniq -c | sort -nr > $file_freq



#!/usr/sh
to_dir="dostoj"
file=$to_dir/erstesBuch.txt
file_freq="$to_dir/freq.txt"
freq=5
# AWK
echo " awk Number 1 "
awk '$2 ~ /^[A-ZÜÖÄ].*$/' < $file_freq | head -$freq
echo " awk Number 2 "
awk '$2 ~ /^[a-zöüä].*$/' < $file_freq | head -$freq
echo " awk Number 3 "
awk '$2 ~ /^[0-9]+$/' < $file_freq | head -$freq

# python oneliners
echo "Python Number 1"
python3 -m oneliner -ne "_ if W[1].istitle() else print('', end='')" < $file_freq > out.txt; head -$freq out.txt
echo "Python Number 2"
python3 -m oneliner -ne '_ if W[1].islower() else print("",end="")' < $file_freq >> out.txt; head -$freq out.txt
echo "Python Number 3"
python3 -m oneliner -ne '_ if W[1].isdigit() else print("",end="")' < $file_freq >> out.txt; head -$freq out.txt
#!/usr/sh
to_dir="dostoj"
from="http://www.zeno.org/Literatur/M/Dostoevskij,+F%C3%ABdor+Michajlovi%C4%8D/Romane/Der+Idiot/Erster+Teil"
data="Kapitel"
result=cleanbook.txt

if [ ! -d "$to_dir" ]; then
  echo "mkdir $to_dir"
  mkdir $to_dir
fi

if [ -e "$to_dir/$result" ]; then
 echo "rm $to_dir/$result"
 rm $to_dir/$result
fi

for i in {1..5}; do
	echo "wget -O ${to_dir}/$i.$data.htm $from/$i."
	wget -O ${to_dir}/${i}.${data}.htm "$from/$i."
	echo "iconv -f ISO-8859-1 -t UTF-8 ${to_dir}/${i}.${data}.htm > ${to_dir}/out.htm"
	iconv -f ISO-8859-1 -t UTF-8 ${to_dir}/${i}.${data}.htm > ${to_dir}/out.htm
	cat ${to_dir}/out.htm | python3 -m oneliner -m re -ne '_ if re.search(r"<\/p>", _) else print("", end="")' \
	| python3 -m oneliner -m re -ne '_ if re.search("^(?!<div class=).*", _) else print("", end="")' \
	| python3 -m oneliner -m re -ne 're.sub("<a href.*?<\/a>","",_)' \
	| python3 -m oneliner -m re -ne 're.sub("»","",_)' \
	| lynx -assume_charset=UTF8 -stdin -dump >> $to_dir/$result
done
 

 #!/usr/sh
to_dir="dostoj"
file=$to_dir/cleanbook.txt
file_freq="$to_dir/freq.txt"
echo "cat $file | tr -s ' ' '\n' | tr -d [[:punct:]] | sort -f | uniq -c | sort -nr > $file_freq"
cat $file | tr -s " " "\n" | tr -d [[:punct:]] | tr -d "»" | sort -f | uniq -c | sort -nr > $file_freq
#!/usr/sh
### Define Variables
to_dir="dostoj"
file=$to_dir/erstesBuch.txt
file_freq="$to_dir/cleanbook.txt"
freq=5
####


# AWK
echo " awk Number 1 "
awk '$2 ~ /^[A-ZÜÖÄ].*$/' < $file_freq | head -$freq
echo " awk Number 2 "
awk '$2 ~ /^[a-zöüä].*$/' < $file_freq | head -$freq
echo " awk Number 3 "
awk '$2 ~ /^[0-9]+$/' < $file_freq | head -$freq

# python oneliners
echo "Python Number 1"
python3 -m oneliner -ne "_ if W[1].istitle() else print('', end='')" < $file_freq > out.txt; head -$freq out.txt
echo "Python Number 2"
python3 -m oneliner -ne '_ if W[1].islower() else print("",end="")' < $file_freq >> out.txt; head -$freq out.txt
echo "Python Number 3"
python3 -m oneliner -ne '_ if W[1].isdigit() else print("",end="")' < $file_freq >> out.txt; head -$freq out.txt