svn/view/checkdomains.bash

#!/bin/bash
set -e

#tmp=`pgrep checkdomain | wc -l`
#(( tmp > 1 )) && echo something is already going on && pgrep -a checkdomain && exit 1
#unset tmp
echo
pgrep -a checkdomain
echo
echo press enter if you do not see more than one of those
read

#I just home multiple background process have their different env and the variables
#are not overlapping from one another.  The background processes here are showing
#up as checkdomain.bash so it seems the fork is good enough
function secondlevel {
        [[ -z $1 ]] && echo function $0 requires file argument && exit 1

        echo writing to $1.domains
        #assuming there is no leading dot hence . means it is second-level
        #we keep the original FQDN in $1 and do NOT want to appear again
        #be it a domain.tld PTR or whatever
        #TODO strip out diffs so there's no duplicates
        for domain in `cat $1`; do
                level=`echo $domain | tr -dc [.]`
                [[ -z $level ]] && echo Error domain $domain level noexist && exit 1
                [[ $level = "" ]] && echo Error domain $domain level is empty && exit 1

                #we're starting at one level down already, nothing happens thereafter
                #in case we're already on a domain.tld PTR, and we got those in $1
                #anyways.  in case we're on something.domain.tld, then domain.tld will
                #appear -- that is why we need a 2nd pass to eventually strip it back out
                [[ $level = . ]] && echo $domain
                until [[ $level = . ]]; do
                        domain=${domain#*\.}
                        level=`echo $domain | tr -dc [.]`

                        #avoid re-writing the same parent-domain over and over again
                        #this is only possible since we've got an ordered list already
                        #dealing with domain levels 4, 3, 2
                        if [[ $level = ... ]]; then
                                [[ $domain != $previousc ]] && echo $domain
                                previousc=$domain
                        fi
                        if [[ $level = .. ]]; then
                                [[ $domain != $previousb ]] && echo $domain
                                previousb=$domain
                        fi
                        if [[ $level = . ]]; then
                                [[ $domain != $previous ]] && echo $domain
                                previous=$domain
                        fi
                        #expected output:
                        #181-183-130-205.genericrev.telcel.net.ve
                        #genericrev.telcel.net.ve
                        #telcel.net.ve
                        #net.ve
                        #181-183-130-206.genericrev.telcel.net.ve
                        #181-183-130-207.genericrev.telcel.net.ve
                        #...
                done
                unset level
        done > $1.domains && echo wrote to $1.domains; unset domain
        unset previous previousb previousc
}

#[[ ! -f public_suffix_list.dat ]] && wget https://publicsuffix.org/list/public_suffix_list.dat
#[[ ! -f public_suffix_list.dat ]] && echo Error could not fetch public_suffix_list.dat && exit 1

echo -n entering domains/ and cleaning-up...
mkdir -p domains/
rm -f domains/*
cd domains/ && echo done

#getting rid of rev.sfr.net and others who have this terrible manner of dealing
#with reverse dns naming
echo -n writing to ptr.unique.nomadness while avoiding ip.subdomain.madness...
sed -r '/^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+\./d' ../ptr.unique > ptr.unique.nomadness && echo done

#preparing for domain identification
echo -n splitting ptr.unique.nomadness into 7 files...
split -a2 -d -nl/7 ptr.unique.nomadness ptr.unique.nomadness && echo done
#ls -lhF ptr.unique.nomadness[0-9][0-9]

#let's get second-level domains and deference anything that's above second-level
#note that second-level public suffixes will be dealt with as an aftermath
for f in ptr.unique.nomadness[0-9][0-9]; do
        secondlevel $f &
done; unset f
jobs
cat <<EOF

you may want to watch it live

        ps auxfww | grep checkdomain
        watch ls -lF domains/

this program will proceed once all processes are done

EOF

time wait

echo writing to ptr.unique.nomadness.sld
for sld in `cat ~/masspie/SLDs`; do
        grep -E "^[^.]+\.$sld$" ptr.unique.nomadness
done > ptr.unique.nomadness.sld && echo done; unset sld

#no need to uniq this one as we're doing one more shot of this sorting hereby
#echo writing to ptr.unique.nomadness.sld.unique
#sort -u ptr.unique.nomadness.sld > ptr.unique.nomadness.sld.unique && echo done

echo -n merging back and sorting into domains.unique ...
sort --version-sort -u ptr.unique.nomadness[0-9][0-9].domains ptr.unique.nomadness.sld > domains.unique && echo done

#we don't need the following as we're going to deal with second-level+fix only
#echo -n getting rid of duplicates between PTR FQDNs and last results into ptr.unique.nomadness.unique.wodupl ...
#grep -vf ptr.unique ptr.unique.nomadness.unique > ptr.unique.nomadness.unique.wodupl && echo done
#--> grep: memory exhausted
#wc -l ptr.unique.nomadness.unique
#wc -l ptr.unique.nomadness.unique.wodupl
