svn/view/count.ksh

#!/bin/ksh
#
# KSH93 only (floating point)
#
set -e

debug=0

[[ -z $1 ]] && echo want \$shot && exit 1
shot=$1

LC_NUMERIC=en_US
#sep="sed ':a;s/\B[0-9]\{3\}\>/,&/;ta'"
#| eval $sep

echo -n entering ~/$shot/splitted/ ...
cd ~/$shot/splitted/ && echo done

#based on exclude.conf
internet=3970693888

range=`seq -w 00 89; seq 9000 9447`

(( debug == 1 )) && counting inbound smtp hosts
#smtp=0
#for x in $range; do
#       (( smtp = smtp + `grep -v ^# x$x | wc -l` )) || echo FAIL:x$x
#done; unset x
smtp=`grep -v ^# ../massp25.og | wc -l`

(( debug == 1 )) && echo counting hosts which iprev resolve x\*.ptr
(( iprev = `cat x*.ptr | wc -l` ))

(( debug == 1 )) && echo counting unique iprev hosts
iprevu=`wc -l < ptr.unique`

(( debug == 1 )) && echo counting those which do not talk SSL x\*.ptr.nossl
nossl=0
for x in $range; do
        (( nossl = nossl + `wc -l < x$x.ptr.nossl` )) || echo FAIL:x$x
done; unset x

(( debug == 1 )) && echo counting those which have wrong SAN x\*.ptr.wrongcn
wrongcn=0
for x in $range; do
        (( wrongcn = wrongcn + `wc -l < x$x.ptr.wrongcn` )) || echo FAIL:x$x
done; unset x

(( debug == 1 )) &&  echo counting those which have valid SAN x\*.ptr.validcn
validcn=0
for x in $range; do
        (( validcn = validcn + `wc -l < x$x.ptr.validcn` )) || echo FAIL:x$x
done; unset x

(( debug == 1 )) && echo counting those which validate x\*.ptr.validcn.return
(( validate = `grep 'Verify return code: 0 (ok)' x*.ptr.validcn.return | wc -l` )) || echo FAIL

echo
printf "internet is\\t\\t%'.f\n" $internet
printf "inbound smtp hosts are\\t%'.f\n" $smtp
printf "iprev are\\t\\t%'.f\n" $iprev
printf "unique iprev are\\t%'.f\n" $iprevu
printf "no ssl are\\t\\t%'.f\n" $nossl
printf "wrong CN/SAN are\\t%'.f\n" $wrongcn
printf "iprev CN/SAN are\\t%'.f\n" $validcn
printf "valid chains are\\t%'.f\n" $validate

typeset -F2 internet smtp iprev iprevu nossl wrongcn validcn validate
typeset -F2 ssldiff iprevdiff

(( ssldiff = iprev - nossl ))
(( iprevdiff = iprev - iprevu ))

typeset -F2 result

echo

(( result = smtp * 100 / internet ))
echo $result% of the public network listens on port 25/tcp - ${smtp%\.*} out of ${internet%\.*}

(( result = iprev * 100 / smtp ))
echo $result% of the smtp servers are full-circle reverse DNS - ${iprev%\.*} out of ${smtp%\.*}

(( result = iprevdiff * 100 / iprev ))
echo $result% of those iprev hosts are multi-homed \(round-robin\) - ${iprevdiff%\.*} out of ${iprev%\.*}

(( result = ssldiff * 100 / iprev ))
echo $result% of full-circle hosts talk SSL/STARTTLS - ${ssldiff%\.*} out of ${iprev%\.*}

#echo $(( validcn * 100 / iprev ))% of full-circle hosts advertise a valid subject alternative - ${validcn%\.*} out of ${iprev%\.*}

(( result = validcn * 100 / ssldiff ))
echo $result% of SSL-enabled hosts advertise an iprev subject alternative name - ${validcn%\.*} out of ${ssldiff%\.*}

(( result = validate * 100 / validcn ))
echo $result% of SAN hosts have a valid certificate chain - ${validate%\.*} out of ${validcn%\.*}

echo

#
# here comes stats based on MX records
#

ptrs=`wc -l < ptr.unique`

cd domains/

domains=`wc -l < domains.unique`

cd mx/

mx=`wc -l < mx.unique`

cd dane/

(( debug == 1 )) && echo counting trueok
trueok=`grep '0 (ok)$' *.ssl | grep -v 'New, (NONE), Cipher is (NONE)' | wc -l`

(( debug == 1 )) && echo counting fakeok
fakeok=`grep '0 (ok)$' *.ssl | grep 'New, (NONE), Cipher is (NONE)' | wc -l`

#grep 'Verify return code: ' mx.unique37.ssl | cut -f2 -d: | sort -u
(( debug == 1 )) && echo counting notyet
notyet=`grep '9 (certificate is not yet valid)$' *.ssl | wc -l`

(( debug == 1 )) && echo counting expired
expired=`grep '10 (certificate has expired)$' *.ssl | wc -l`

(( debug == 1 )) && echo counting selfsigned
selfsigned=`grep '18 (self signed certificate)$' *.ssl | wc -l`

(( debug == 1 )) && echo counting selfchain
selfchain=`grep '19 (self signed certificate in certificate chain)$' *.ssl | wc -l`

(( debug == 1 )) && echo counting untrusted
untrusted=`grep '20 (unable to get local issuer certificate)$' *.ssl | wc -l`

(( debug == 1 )) && echo counting firstinvalid
invalid=`grep '21 (unable to verify the first certificate)$' *.ssl | wc -l`

(( debug == 1 )) && echo counting purpose
purpose=`grep '26 (unsupported certificate purpose)$' *.ssl | wc -l`

(( debug == 1 )) && echo counting dane
dane=`grep THIS-LOOKS-LIKE-DANE$ *.dane | wc -l`
#notlsa=`grep notlsa$ *.dane | wc -l`
#servfail=`grep servfail$ *.dane | wc -l`
#timeout=`grep timeout$ *.dane | wc -l`

(( debug == 1 )) && echo counting daneee and daneta
daneee=`grep ^DANE *.dane.results | grep 'matched EE' | wc -l`
daneta=`grep ^DANE *.dane.results | grep 'matched TA' | wc -l`
#grep ^DANE *.results | grep -vE 'matched EE|matched TA'

(( debug == 1 )) && echo counting STARTTLS
#TOFIX
#enforcetotal=`cat mx.unique*.starttls.enforce.dist | wc -l`
enforcetotal2=`cat mx.unique*.starttls.enforce | wc -l`
connectbaddns=`grep connect-bad-dns$ mx.unique*.starttls.enforce | wc -l`
connectfailed=`grep connect-failed$ mx.unique*.starttls.enforce | wc -l`
connect4xx=`grep connect-4xx$ mx.unique*.starttls.enforce | wc -l`
connect5xx=`grep connect-5xx$ mx.unique*.starttls.enforce | wc -l`
connectrefused=`grep connect-refused$ mx.unique*.starttls.enforce | wc -l`
connectclosed=`grep connect-closed$ mx.unique*.starttls.enforce | wc -l`
connecttimeout=`grep connect-timeout$ mx.unique*.starttls.enforce | wc -l`
connectunknown=`grep connect-unknown$ mx.unique*.starttls.enforce | wc -l`
ehlo2xxnostarttls=`grep ehlo-2xx-no-starttls$ mx.unique*.starttls.enforce | wc -l`
ehlo5xx=`grep ehlo-5xx$ mx.unique*.starttls.enforce | wc -l`
ehlo4xx=`grep ehlo-4xx$ mx.unique*.starttls.enforce | wc -l`
ehlotimeout=`grep ehlo-timeout$ mx.unique*.starttls.enforce | wc -l`
ehlounknown=`grep ehlo-unknown$ mx.unique*.starttls.enforce | wc -l`
#sender2xx=`grep sender-2xx$ mx.unique*.starttls.enforce | wc -l`
#sender5xx=`grep sender-5xx$ mx.unique*.starttls.enforce | wc -l`
#sender4xx=`grep sender-4xx$ mx.unique*.starttls.enforce | wc -l`
hastls=`grep -E '^250-STARTTLS|^250 STARTTLS' mx.unique*.starttls | wc -l`

#we want
#530 5.7.0 Must issue a STARTTLS command first
#530 5.5.1 Invalid command: Must issue a STARTTLS command first
#530 Must issue STARTTLS first.
#430 4.7.0 Must issue a STARTTLS command first
#grep -E '[[:digit:]]{3} 5\.7\.3 ' mx.unique*.starttls

#exceptions
#530 5.7.1 Client was not authenticated
#530 5.7.3 Client was not authenticated
#530 aws.besteffort.com ESMTP MailEnable Service, Version: 9.76-9.76- denied access at 01/22/20 13:32:04

must=`grep -E '^575 |^[[:digit:]]{3} 5\.7\.3 |^530 |^451 5\.7\.3 |^430 4\.7\.0 |^451 .*TLS.*|^550 AUTH TLS |^550 TLS ' mx.unique*.starttls | grep -vE ' 5\.7\.1 |530 5\.7\.3 |MailEnable' | wc -l`

(( mxdown = connectbaddns + connectfailed + connect4xx + connect5xx + connectrefused + connectclosed + connecttimeout + connectunknown ))

cd ../../../

(( debug == 1 )) && echo

printf "IPREV PTRs\t\t\t%'.f\n" $ptrs
#next versions will have 2nd-3rd-level domains: $domains
printf "Deferenced domains\t\t%'.f\n" $domains
printf "MX records\t\t\t%'.f\n" $mx
echo

printf "Total results for enforce check\t%'.f\n" $enforcetotal2
printf "Unreachable MXen\t\t%'.f\n" $mxdown
printf "No STARTTLS\t\t\t%'.f\n" $ehlo2xxnostarttls
#printf "Opportunistic STARTTLS\t\t%'.f\n" $sender2xx
#printf "Enforced STARTTLS or 5xx\t%'.f\n" $sender5xx
#printf "Enforced STARTTLS or 4xx\t%'.f\n" $sender4xx
printf "Offers STARTTLS\t\t\t%'.f\n" $hastls
printf "Enforces STARTTLS\t\t%'.f\n" $must
echo

printf "Trusted certificate chain\t%'.f (ok)\n" $trueok
printf "Cipher is (NONE)\t\t%'.f (ok)\n" $fakeok
printf "From the future\t\t\t%'.f (certificate is not yet valid)\n" $notyet
printf "Expired\t\t\t\t%'.f (certificate has expired)\n" $expired
printf "Self-signed\t\t\t%'.f (self signed certificate)\n" $selfsigned
printf "Self-signed CA\t\t\t%'.f (self signed certificate in certificate chain)\n" $selfchain
printf "Untrusted certificate chain\t%'.f (unable to get local issuer certificate)\n" $untrusted
printf "Invalid certificate\t\t%'.f (unable to verify the first certificate)\n" $invalid
printf "Wrong purpose certificate\t%'.f (unsupported certificate purpose)\n" $purpose
printf "TLSA records\t\t\t%'.f\n" $dane
printf "Valid PKIX/DANE-EE\t\t%'.f\n" $daneee
printf "Valid PKIX/DANE-TA\t\t%'.f\n" $daneta
echo

typeset -F2 result
(( result = mx * 100 / domains ))
echo $result% of deferenced domains have an MX record
unset result

typeset -F2 result
(( result = trueok * 100 / mx ))
echo $result% of MX certificates are valid
unset result

typeset -F2 result
(( result = fakeok * 100 / mx ))
echo $result% of MX end-points do not offer STARTTLS
unset result

typeset -F2 result
(( result = expired * 100 / mx ))
echo $result% of MX certificates are expired
unset result

typeset -F2 result
(( result = selfsigned * 100 / mx ))
echo $result% of MX certificates are self-signed
unset result

typeset -F2 result
(( result = untrusted * 100 / mx ))
echo $result% of MX certificates are private
unset result

typeset -F2 result
(( result = invalid * 100 / mx ))
echo $result% of MX certificates are invalid
unset result

typeset -F2 result
(( result = dane * 100 / mx ))
echo $result% of MX end-points have a DANE record
unset result

typeset -F2 result
(( result = ( daneee + daneta ) * 100 / mx ))
echo $result% of MX end-points validating DANE \(trusted, private and self-signed\)
unset result

echo
