svn/view/stage1-iprev.bash

#!/bin/bash
set -e

#
# input: massp25.og
#
# output:
#       massp25.og.ip.sort
#       01-splitted/*.ptr
#       01-splitted/*.weird
#       ptr.unique
#

#(( debug = 1 ))

# requires $ip
# echoes stdout and stderr e.g.
#       217.92.231.121  mail.wessendorf.com     iprev
#       217.92.232.41   familie-nissen.eu       noiprev
# cannot use debug mode coz stdout gets redirected
function checkiprev {
        [[ -z $ip ]] && echo function $0 requires \$ip && exit 1

        # separate the commands to get the first return code
        ptrstr=`host -W3 $ip` || return
        #{ echo DEBUG: ip is $ip; return; }

        # we need to avoid backslash escaped foreign chars e.g.
        # \233\155\187\232\133\166\229\174\164003.ktnp.gov.tw.
        # we also don't want weird PTRs that would eventually validate iprev
        # ^R9d^O9^R8^A91^K8^R9^A89~-a.ktnp.gov.tw
        # *.highrockhosting7.com
        ptr=`echo $ptrstr | awk '{print $NF}' | grep -E '^[[:alnum:]._-]+$'`

        [[ -z $ptr ]] && echo weird Z ptr on ip $ip >> $piece.weird && return
        [[ $ptr = "" ]] && echo weird EMPTY ptr on ip $ip >> $piece.weird && return

        hostipstr=`host -W3 $ptr`
        #{ echo DEBUG: ptr is $ptr; return; }
        hostip=`echo $hostipstr | awk '{print $NF}'`
        
        if [[ $ip = $hostip ]]; then
                printf "$ip\t${ptr%\.}\tiprev\n"
        else
                printf "$ip\t${ptr%\.}\tnoiprev\n"
        fi
}

# requires $piece
# writes stdout to $piece.ptr and stderr to terminal
function processpiece {
        for ip in `cat $piece`; do
                if (( debug == 1 )); then
                        checkiprev
                        sleep 1
                else
                        checkiprev
                fi
        done > $piece.ptr && echo wrote to $piece.ptr || echo $piece.ptr FAIL; unset ip
}

[[ ! -f massp25.og ]] && echo missing file massp25.og && exit 1

# hmm masscan delivered duplicates
# 11370506 massp25.og.ip
# 11328069 massp25.og.ip.sort

if [[ ! -f massp25.og.ip.sort && ! -s massp25.og.ip.sort ]]; then
        echo writing unique ip addresses to massp25.og.ip.sort
        time grep -v ^# massp25.og | awk '{print $4}' | sort --version-sort -u > massp25.og.ip.sort && echo done
        #sort -V -k4
fi
wc -l massp25.og.ip.sort
(( debug == 1 )) && echo everything fine?
(( debug == 1 )) && read

echo -n entering 01-splitted/ ...
mkdir -p 01-splitted/
cd 01-splitted/ && echo done

if [[ ! -f ip109 ]]; then
        echo -n splitting massp25.og.ip.sort into 110 pieces...
        split -a3 -d -nl/110 ../massp25.og.ip.sort ip && echo done
fi
wc -l ip109
(( debug == 1 )) && echo everything fine?
(( debug == 1 )) && read

echo cleaning-up previous attempt
rm -f ip*.ptr
rm -f ip*.weird

echo
echo $0 starts `date`
echo

if (( debug == 1 )); then
        echo DEBUG MODE - PROCESSING ONLY splitted/ip109
        piece=ip109
        processpiece &
        unset piece
else
        for piece in ip[0-9][0-9][0-9]; do
                processpiece &
        done; unset piece
        #for x in `seq -w 000 050`; do
        #       piece=ip$x
        #       processpiece &
        #done; unset x
fi

echo -n "processes: "
pgrep stage1 | wc -l

cat <<EOF

check with

        ps auxfww | grep stage1 | grep -v grep
        pgrep stage1 | wc -l
        tail -F 01-splitted/ip109.ptr

EOF
time wait

# if this prints, we passed through without any fatal error (set -e is there)
echo
echo $0 ends `date`
echo

# all PTRs in one place, sorting and sanitizing
echo -n writing to ptr.unique ...
#sed 's/\.$//'
#grep -E '^[[:alnum:]._-]+$'
cut -f2 ../*.ptr | sort --version-sort -u > ptr.unique && echo done

#TODO GETTING ALL IPREV ONLY

#echo "writing to $iprev $noiprev $weird"
#rm -f $iprev $noiprev $weird
#iprev=massp25.iprev
#noiprev=massp25.noiprev
#weird=massp25.weird
