forked from leebaird/discover
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathdiscover.sh
executable file
·3797 lines (3147 loc) · 165 KB
/
discover.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/bin/bash
#
# By Lee Baird
# Feel free to contact me via chat or email with any feedback or suggestions that you may have:
#
# Special thanks to the following people:
#
# Ben Wood - regex master
# Dave Klug - planning, testing and bug reports
# Jay Townsend - conversion of discover.sh to Kali
# Jason Arnold - planning original concept, author of ssl-check and co-author of crack-wifi
# Eric Milam - total re-write using functions
# Martin Bos - IDS evasion techniques
# Matt Banick - original development
# Numerous people on freenode IRC - #bash and #sed (e36freak)
# Rob Dixon - report framework idea
# Robert Clowser - all things
# Saviour Emmanuel - convert nmap xlm to csv
# Steve Copland - report framework design
##############################################################################################################
# Catch ctrl+c from user
trap f_terminate INT
# Global variables
distro=$(uname -n)
interface=$(ifconfig | grep -B1 'inet addr' | egrep -v '(-|inet addr|Loopback)' | cut -d ' ' -f1)
ip=$(ifconfig | grep 'Bcast' | awk '{print$2}' | cut -d ':' -f2)
long='============================================================================================================================='
medium='====================================================================================='
short='========================================'
sip='sort -n -u -t . -k 1,1 -k 2,2 -k 3,3 -k 4,4'
user=$(whoami)
##############################################################################################################
f_banner(){
echo
echo "
______ ___ ______ ______ _____ _ _ ______ _____
| \ | |____ | | | \ / |_____ |____/
|_____/ _|_ _____| |_____ |_____| \/ |_____ | \_
By Lee Baird"
echo
echo
}
##############################################################################################################
f_error(){
echo
echo -e "\e[1;31m$medium\e[0m"
echo
echo -e "\e[1;31m *** Invalid choice or entry. ***\e[0m"
echo
echo -e "\e[1;31m$medium\e[0m"
sleep 2
f_main
}
##############################################################################################################
f_location(){
echo
echo -n "Enter the location of your list: "
read location
# Check for no answer
if [ -z $location ]; then
f_error
fi
# Check for wrong answer
if [ ! -f $location ]; then
f_error
fi
}
##############################################################################################################
f_runlocally(){
if [[ -z $DISPLAY ]]; then
clear
f_banner
echo
echo -e "\e[1;31m$medium\e[0m"
echo
echo -e "\e[1;31m *** This option must be run locally, in an X-Windows environment. ***\e[0m"
echo
echo -e "\e[1;31m$medium\e[0m"
sleep 4
f_main
fi
}
##############################################################################################################
f_terminate(){
mkdir /$user/data/cancelled-`date +%m-%d-%y-%H%M`
# Nmap and Metasploit scans
mv $name/ /$user/data/cancelled-`date +%m-%d-%y-%H%M` 2>/dev/null
# Recon files
mv emails* names records squatting whois* sub* doc pdf ppt txt xls tmp* z* /$user/data/cancelled-`date +%m-%d-%y-%H%M` 2>/dev/null
#pid=$(ps -ef | grep 'discover.sh' | grep -v 'grep' | awk '{print $2}')
#kill -9 $pid
}
##############################################################################################################
f_domain(){
clear
f_banner
echo -e "\e[1;34mRECON\e[0m"
echo
echo "1. Passive"
echo "2. Active"
echo "3. Previous menu"
echo
echo -n "Choice: "
read choice
case $choice in
1)
echo
echo $medium
echo
echo "Usage: target.com"
echo
echo -n "Domain: "
read domain
# Check for no answer
if [ -z $domain ]; then
f_error
fi
# If folder doesn't exist, create it
if [ ! -d /$user/data/$domain ]; then
cp -R /opt/discover/report/ /$user/data/$domain
sed 's/REPLACEDOMAIN/'$domain'/g' /$user/data/$domain/index.htm > tmp
mv tmp /$user/data/$domain/index.htm
fi
# Number of tests
total=23
echo
echo $medium
echo
echo "dnsrecon (1/$total)"
dnsrecon -d $domain -t goo > tmp
grep $domain tmp | egrep -v '(Performing Google|Records Found)' > tmp2
# Remove first 6 characters from each line
sed 's/^......//' tmp2 > tmp3
sed 's/A //g' tmp3 | sed 's/CNAME //g' | awk '$2 !~ /[a-z]/' | column -t | sort -u > sub1
echo
echo "goofile (2/$total)"
goofile -d $domain -f doc > tmp
goofile -d $domain -f docx >> tmp
goofile -d $domain -f pdf >> tmp
goofile -d $domain -f ppt >> tmp
goofile -d $domain -f pptx >> tmp
goofile -d $domain -f txt >> tmp
goofile -d $domain -f xls >> tmp
goofile -d $domain -f xlsx >> tmp
grep $domain tmp | grep -v 'Searching in' | grep -Fv '...' | sort > tmp2
grep '.doc' tmp2 | egrep -v '(.pdf|.ppt|.xls)' > doc
grep '.pdf' tmp2 > pdf
grep '.ppt' tmp2 > ppt
grep '.txt' tmp2 | grep -v 'robots.txt' > txt
grep '.xls' tmp2 > xls
echo
echo "goog-mail (3/$total)"
/opt/discover/mods/goog-mail.py $domain | sort -u > tmp
grep -Fv '..' tmp > tmp2
# Remove lines that start with a number
sed '/^[0-9]/d' tmp2 > tmp3
# Change to lower case
cat tmp3 | tr '[A-Z]' '[a-z]' > tmp4
# Remove blank lines
sed '/^$/d' tmp4 > zgoog-mail
echo
echo "goohost"
echo " IP (4/$total)"
/opt/discover/mods/goohost.sh -t $domain -m ip >/dev/null
echo " Email (5/$total)"
/opt/discover/mods/goohost.sh -t $domain -m mail >/dev/null
cat report-* > tmp
# Move the second column to the first position
grep $domain tmp | awk '{ print $2 " " $1 }' > tmp2
column -t tmp2 > zgoohost
rm *-$domain.txt
echo
echo "theharvester"
echo " Ask-mod (6/$total)"
/opt/discover/mods/theHarvester2.py -d $domain -b ask > zask-mod
echo " Bing (7/$total)"
theharvester -d $domain -b bing > zbing
echo " Google (8/$total)"
theharvester -d $domain -b google > zgoogle
echo " Google Profiles (9/$total)"
theharvester -d $domain -b google-profiles > zgoogle-profiles
echo " Jigsaw (10/$total)"
theharvester -d $domain -b jigsaw > zjigsaw
echo " LinkedIn (11/$total)"
theharvester -d $domain -b linkedin > zlinkedin
echo " Login-mod (12/$total)"
/opt/discover/mods/theHarvester2.py -d $domain -b login > zlogin-mod
echo " PGP (13/$total)"
theharvester -d $domain -b pgp > zpgp
echo " Yahoo-mod (14/$total)"
/opt/discover/mods/theHarvester2.py -d $domain -b yahoo > zyahoo-mod
echo " All (15/$total)"
theharvester -d $domain -b all > zall
echo
echo "Metasploit (16/$total)"
/opt/metasploit/msf3/msfcli gather/search_email_collector DOMAIN=$domain E > tmp 2>/dev/null
grep @$domain tmp | awk '{print $2}' | grep -v '%' | grep -Fv '...@' | sort -u > tmp2
# Change to lower case
cat tmp2 | tr '[A-Z]' '[a-z]' > tmp3
# Remove blank lines
sed '/^$/d' tmp3 > zmsf
echo
echo "URLCrazy (17/$total)"
urlcrazy $domain -o tmp > /dev/null
# Clean up
egrep -v '(#|:|\?|RESERVED|Typo Type|URLCrazy)' tmp | sed 's/[A-Z]\{2\},//g' > tmp2
# Remove lines that start with -
grep -v '^-' tmp2 > tmp3
# Remove blank lines
sed '/^$/d' tmp3 > tmp4
sed 's/AUSTRALIA/Australia/g; s/AUSTRIA/Austria/g; s/BAHAMAS/Bahamas/g; s/BANGLADESH/Bangladesh/g; s/BELGIUM/Belgium/g; s/CANADA/Canada/g; s/CAYMAN ISLANDS/Cayman Islands/g;
s/CHILE/Chile/g; s/CHINA/China/g; s/COSTA RICA/Costa Rica/g; s/CZECH REPUBLIC/Czech Republic/g; s/DENMARK/Denmark/g; s/EUROPEAN UNION/European Union/g; s/FINLAND/Finland/g;
s/FRANCE/France/g; s/GERMANY/Germany/g; s/HONG KONG/Hong Kong/g; s/HUNGARY/Hungary/g; s/INDIA/India/g; s/IRELAND/Ireland/g; s/ISRAEL/Israel/g; s/ITALY/Italy/g; s/JAPAN/Japan/g;
s/KOREA REPUBLIC OF/Republic of Korea/g; s/LUXEMBOURG/Luxembourg/g; s/NETHERLANDS/Netherlands/g; s/NORWAY/Norway/g; s/POLAND/Poland/g; s/RUSSIAN FEDERATION/Russia/g;
s/SAUDI ARABIA/Saudi Arabia/g; s/SPAIN/Spain/g; s/SWEDEN/Sweden/g; s/SWITZERLAND/Switzerland/g; s/TAIWAN; REPUBLIC OF China (ROC)/Taiwan/g; s/THAILAND/Thailand/g; s/TURKEY/Turkey/g;
s/UKRAINE/Ukraine/g; s/UNITED KINGDOM/United Kingdom/g; s/UNITED STATES/United States/g; s/VIRGIN ISLANDS (BRITISH)/Virgin Islands/g' tmp4 > squatting
##############################################################
cat z* | egrep -v '(@|\*|-|=|\||;|:|"|<|>|/|\?)' > tmp
# Remove blank lines
sed '/^$/d' tmp > tmp2
# Remove lines that contain a number
sed '/[0-9]/d' tmp2 > tmp3
# Remove lines that start with @ or .
sed '/^\@\./d' tmp3 > tmp4
# Remove trailing white space from each line
sed 's/[ \t]*$//' tmp4 > tmp5
# Substitute a space for a plus sign
sed 's/+/ /g' tmp5 > tmp6
# Change to lower case
cat tmp6 | tr '[A-Z]' '[a-z]' > tmp7
# Clean up
egrep -v '(academy|account|achievement|active|administrator|administrative|advanced|adventure|advertising|america|american|analysis|analyst|antivirus|apple seems|application|applications|architect|article|asian|assistant|associate|association|attorney|auditor|australia|automation|automotive|balance|bank|bbc|beginning|berlin|beta theta|between|big game|billion|bioimages|biometrics|bizspark|breaches|broker|business|buyer|buying|california|cannot|capital|career|carrying|cashing|certified|challenger|championship|change|chapter|charge|china|chinese|clearance|cloud|code|college|columbia|communications|community|company pages|competition|competitive|compliance|computer|concept|conference|config|connections|connect|construction|consultant|contractor|contributor|controllang|cooperation|coordinator|corporation|creative|critical|croatia|crm|dallas|day care|death toll|delta|department|description|designer|design|detection|developer|develop|development|devine|digital|diploma|director|disability|disaster|disclosure|dispute|division|document|dos poc|download|drivers|during|economy|ecovillage|editor|education|effect|electronic|else|emails|embargo|emerging|empower|employment|end user|energy|engineer|enterprise|entertainment|entreprises|entrepreneur|entry|environmental|error page|ethical|example|excellence|executive|expertzone|exploit|facebook|faculty|failure|fall edition|fast track|fatherhood|fbi|federal|filmmaker|finance|financial|forensic|found|freelance|from|frontiers in tax|full|function|fuzzing|germany|get control|global|google|government|graphic|greater|group|guardian|hackers|hacking|harden|harder|hawaii|hazing|headquarters|health|help|history|homepage|hospital|house|how to|hurricane|icmp|idc|in the news|index|informatics|information|innovation|installation|insurers|integrated|international|internet|instructor|insurance|interested|investigation|investment|investor|israel|items|japan|job|justice|kelowna|knowing|laptops|leadership|letter|licensing|lighting|limitless|liveedu|llp|local|looking|ltd|lsu|luscous|malware|managed|management|manager|managing|manufacturing|marketplace|mastering|md|media|medical|medicine|member|meta tags|methane|metro|microsoft|middle east|mitigation|money|monitor|more coming|mortgage|museums|negative|network|network|new user|newspaper|new york|next page|nitrogen|nyc|obtain|occupied|offers|office|online|operations|organizational|outbreak|owners|page|partner|pathology|peace|people|perceptions|philippines|photo|picture|places|planning|portfolio|potential|preassigned|preparatory|president|principal|print|private|process|producer|product|professional|professor|profile|project|program|publichealth|published|pyramid|questions|recruiter|redeem|redirect|region|register|registry|regulation|rehab|remote|report|republic|research|resolving|revised|rising|rural health|sales|satellite|save the date|school|scheduling|science|search|searc|sections|secured|security|secretary|secrets|see more|selection|senior|server|service|services|social|software|solutions|source|special|station home|statistics|strategy|student|successful|superheroines|supervisor|support|switch|system|systems|talent|targeted|tax|tcp|technical|technology|tester|textoverflow|theater|time in|tit for tat|title|toolbook|tools|traditions|trafficking|transfer|treasury|trojan|twitter|training|ts|tylenol|types of scams|unclaimed|underground|university|united states|untitled|verification|vietnam|view|Violent|virginia bar|voice|volkswagen|volume|wanted|web search|web site|website|welcome|west virginia|when the|whiskey|window|worker|world|www|xbox)' tmp7 > tmp8
# Remove leading and trailing whitespace from each line
sed 's/^[ \t]*//;s/[ \t]*$//' tmp8 > tmp9
# Remove lines that contain a single word
sed '/[[:blank:]]/!d' tmp9 > tmp10
# Clean up
sed 's/\..../ /g' tmp10 | sed 's/\.../ /g; s/iii/III/g; s/ii/II/g' > tmp11
# Capitalize the first letter of every word, print last name then first name
sed 's/\b\(.\)/\u\1/g' tmp11 | awk '{print $2", "$1}' | sort -u > names
##############################################################
cat z* | grep @$domain | grep -vF '...' | egrep -v '(%|\*|=|\+|\[|\||;|:|"|<|>|/|\?)' > tmp
# Remove trailing whitespace from each line
sed 's/[ \t]*$//' tmp > tmp2
# Change to lower case
cat tmp2 | tr '[A-Z]' '[a-z]' > tmp3
# Clean up
egrep -v '(web search|www|xxx)' tmp3 | cut -d ' ' -f2 | sort -u > emails
##############################################################
cat z* | sed '/^[0-9]/!d' | grep -v '@' > tmp
# Substitute a space for a colon
sed 's/:/ /g' tmp > tmp2
# Move the second column to the first position
awk '{ print $2 " " $1 }' tmp2 > tmp3
column -t tmp3 > tmp4
# Change to lower case
cat tmp4 | tr '[A-Z]' '[a-z]' > tmp5
sed 's/<strong>//g; s/<//g' tmp5 | grep $domain | column -t | sort -u > sub2
##############################################################
echo
echo "Whois"
echo " Domain (18/$total)"
whois -H $domain > tmp 2>/dev/null
# Remove leading whitespace
sed 's/^[ \t]*//' tmp > tmp2
# Clean up
egrep -v '(%|<a|=-=-=-=|Access may be|Additionally|Afilias except|and DNS Hosting|and limitations of|any use of|Be sure to|By submitting an|by the terms|can easily change|circumstances will|clientDeleteProhibited|clientTransferProhibited|clientUpdateProhibited|company may be|complaint will|contact information|Contact us|Copy and paste|currently set|database|data contained in|data presented in|date of|dissemination|Domaininfo AB|Domain Management|Domain names in|Domain status: ok|enable high|except as reasonably|failure to|facsimile of|for commercial purpose|for detailed information|For information for|for information purposes|for the sole|Get Noticed|Get a FREE|guarantee its|HREF|In Europe|In most cases|in obtaining|in the address|includes restrictions|including spam|information is provided|is not the|is providing|Learn how|Learn more|makes this information|MarkMonitor|mining this data|minute and one|modify existing|modify these terms|must be sent|name cannot|NamesBeyond|not to use|Note: This|NOTICE|obtaining information about|of Moniker|of this data|or hiding any|or otherwise support|other use of|own existing customers|Please be advised|Please note|policy|prior written consent|privacy is|Problem Reporting System|Professional and|prohibited without|Promote your|protect the|Public Interest|queries or|Register your|Registrars|registration record|repackaging,|responsible for|See Business Registration|server at|solicitations via|sponsorship|Status|support questions|support the transmission|telephone, or facsimile|that apply to|that you will|the right| The data is|The fact that|the transmission|The Trusted Partner|This listing is|This feature is|This information|This service is|to collect or|to entities|to report any|transmission of mass|UNITED STATES|United States|unsolicited advertising|Users may|Version 6|via e-mail|Visit AboutUs.org|while believed|will use this|with many different|with no guarantee|We reserve the|Whois|you agree|You may not)' tmp2 > tmp3
# Remove lines starting with "*"
sed '/^*/d' tmp3 > tmp4
# Remove lines starting with "-"
sed '/^-/d' tmp4 > tmp5
# Remove lines starting with http
sed '/^http/d' tmp5 > tmp6
# Remove lines starting with US
sed '/^US/d' tmp6 > tmp7
# Clean up phone numbers
sed 's/+1.//g' tmp7 > tmp8
# Remove leading whitespace from file
awk '!d && NF {sub(/^[[:blank:]]*/,""); d=1} d' tmp8 > tmp9
# Remove trailing whitespace from each line
sed 's/[ \t]*$//' tmp9 > tmp10
# Compress blank lines
cat -s tmp10 > tmp11
# Remove lines that end with various words then a colon or period(s)
egrep -v '(2:$|3:$|Address.$|Address........$|Address.........$|Ext.:$|FAX:$|Fax............$|Fax.............$|Province:$|Server:$)' tmp11 > tmp12
# Remove line after "Domain Servers:"
sed -i '/^Domain Servers:/{n; /.*/d}' tmp12
# Remove line after "Domain servers"
sed -i '/^Domain servers/{n; /.*/d}' tmp12
# Remove blank lines from end of file
awk '/^[[:space:]]*$/{p++;next} {for(i=0;i<p;i++){printf "\n"}; p=0; print}' tmp12 > tmp13
while IFS=$': \t'
read first rest; do
if [[ $first$rest ]]; then
printf '%-20s %s\n' "$first:" "$rest"
else
echo
fi
done < tmp13 > whois-domain
echo " IP (19/$total)"
y=$(ping -c1 -w2 $domain | grep 'PING' | cut -d ')' -f1 | cut -d '(' -f2) ; whois -H $y > tmp
# Remove leading whitespace
sed 's/^[ \t]*//' tmp > tmp2
# Remove trailing whitespace from each line
sed 's/[ \t]*$//' tmp2 > tmp3
# Clean up
egrep -v '(\#|\%|\*|All reports|Comment|dynamic hosting|For fastest|For more|Found a referral|http|OriginAS:$|Parent:$|point in|RegDate:$|remarks:|The activity|the correct|Without these)' tmp3 > tmp4
# Remove leading whitespace from file
awk '!d && NF {sub(/^[[:blank:]]*/,""); d=1} d' tmp4 > tmp5
# Remove blank lines from end of file
awk '/^[[:space:]]*$/{p++;next} {for(i=0;i<p;i++){printf "\n"}; p=0; print}' tmp5 > tmp6
# Compress blank lines
cat -s tmp6 > tmp7
# Clean up
sed 's/+1-//g' tmp7 > tmp8
while IFS=$': \t'
read first rest; do
if [[ $first$rest ]]; then
printf '%-20s %s\n' "$first:" "$rest"
else
echo
fi
done < tmp8 > whois-ip
echo
# Remove all empty files
find -type f -empty -exec rm {} +
echo "dnssy.com (20/$total)"
wget -q http://www.dnssy.com/report.php?q=$domain -O tmp
sed -n '/Results for/,/\/table/p' tmp > tmp2
echo "<html>" > tmp3
cat tmp2 | grep -v 'Results for' >> tmp3
echo "</html>" >> tmp3
sed 's/Pass/<center><img src="..\/images\/icons\/green.png" height="50" width="50"><\/center>/g;
s/Warning/<center><img src="..\/images\/icons\/yellow.png" height="50" width="50"><\/center>/g;
s/Fail/<center><img src="..\/images\/icons\/red.png" height="50" width="50"><\/center>/g;
s/ class="info"//g; s/ class="rfail"//g; s/ class="rinfo"//g; s/ class="rpass"//g; s/ class="rsecu"//g; s/ class="rwarn"//g;
s/All of the glue/Glue/g; s/All of your MX/All MX/g; s/All of your nameservers/Nameservers/g; s/Checking domain format/Domain format/g;
s/Checking for parent nameservers/Parent nameservers/g; s/Checking for parent glue/Parent glue/g; s/Each of your nameservers/Each nameserver/g;
s/I expected/Expected/g; s/I found the following MX records://g; s/I got an error response to my/Received an error response to/g;
s/I was unable/Unable/g; s/None of your MX/No MX/g; s/This is all of the MX servers I found.//g; s/WWW/www/g;
s/Your nameservers/Nameservers/g; s/Your NS records at your nameservers are://g; s/Your NS records at your parent nameserver are://g;
s/Your SOA/SOA/g; s/Your web server/The web server/g; s/Your web server says it is://g' tmp3 > /$user/data/$domain/data/config.htm
echo "ewhois.com (21/$total)"
wget -q http://www.ewhois.com/$domain/ -O tmp
cat tmp | grep 'visitors' | cut -d '(' -f1 | cut -d '>' -f2 | grep -v 'OTHER' | column -t | sort -u > sub3
echo "myipneighbors.net (22/$total)"
wget -q http://www.myipneighbors.net/?s=$domain -O tmp
grep 'Domains' tmp | sed 's/<\/tr>/\\\n/g' | cut -d '=' -f3,6 | sed 's/" rel=/ /g' | sed 's/" rel//g' | grep -v '/' | column -t | sort -u > sub4
cat sub* | grep -v "$domain\." | sed 's/www\.//g' | column -t | sort -u > tmp
# Remove lines that contain a single word
sed '/[[:blank:]]/!d' tmp > subdomains
echo "urlvoid.com (23/$total)"
wget -q http://www.urlvoid.com/scan/$domain -O tmp
sed -n '/Safety Scan Report/,/<\/table>/p' tmp | grep -v 'Safety Scan Report' | sed 's/View more details.../Details/g' > /$user/data/$domain/data/black-listed.htm
awk '{print $2}' subdomains > tmp
grep -E '([0-9]{1,3})\.([0-9]{1,3})\.([0-9]{1,3})\.([0-9]{1,3})' tmp | egrep -v '(-|=|:)' | $sip > hosts
cat hosts >> /$user/data/$domain/data/hosts.htm; echo "</pre>" >> /$user/data/$domain/data/hosts.htm
##############################################################
echo > zreport
echo >> zreport
echo "Summary" >> zreport
echo $short >> zreport
echo > tmp
if [ -e emails ]; then
emailcount=$(wc -l emails | cut -d ' ' -f1)
echo "Emails $emailcount" >> zreport
echo "Emails ($emailcount)" >> tmp
echo $short >> tmp
cat emails >> tmp
echo >> tmp
cat emails >> /$user/data/$domain/data/emails.htm
fi
if [ -e names ]; then
namecount=$(wc -l names | cut -d ' ' -f1)
echo "Names $namecount" >> zreport
echo "Names ($namecount)" >> tmp
echo $short >> tmp
cat names >> tmp
echo >> tmp
cat names >> /$user/data/$domain/data/names.htm
fi
if [ -e hosts ]; then
hostcount=$(wc -l hosts | cut -d ' ' -f1)
echo "Hosts $hostcount" >> zreport
echo "Hosts ($hostcount)" >> tmp
echo $short >> tmp
cat hosts >> tmp
echo >> tmp
fi
if [ -e squatting ]; then
urlcount2=$(wc -l squatting | cut -d ' ' -f1)
echo "Squatting $urlcount2" >> zreport
echo "Squatting ($urlcount2)" >> tmp
echo $long >> tmp
cat squatting >> tmp
echo >> tmp
cat squatting >> /$user/data/$domain/data/squatting.htm
fi
if [ -e subdomains ]; then
urlcount=$(wc -l subdomains | cut -d ' ' -f1)
echo "Subdomains $urlcount" >> zreport
echo "Subdomains ($urlcount)" >> tmp
echo $long >> tmp
cat subdomains >> tmp
echo >> tmp
cat subdomains >> /$user/data/$domain/data/subdomains.htm
fi
if [ -e xls ]; then
xlscount=$(wc -l xls | cut -d ' ' -f1)
echo "Excel $xlscount" >> zreport
echo "Excel Files ($xlscount)" >> tmp
echo $long >> tmp
cat xls >> tmp
echo >> tmp
cat xls >> /$user/data/$domain/data/xls.htm; echo "</pre>" >> /$user/data/$domain/data/xls.htm
fi
if [ -e pdf ]; then
pdfcount=$(wc -l pdf | cut -d ' ' -f1)
echo "PDF $pdfcount" >> zreport
echo "PDF Files ($pdfcount)" >> tmp
echo $long >> tmp
cat pdf >> tmp
echo >> tmp
cat pdf >> /$user/data/$domain/data/pdf.htm; echo "</pre>" >> /$user/data/$domain/data/pdf.htm
fi
if [ -e ppt ]; then
pptcount=$(wc -l ppt | cut -d ' ' -f1)
echo "PowerPoint $pptcount" >> zreport
echo "PowerPoint Files ($pptcount)" >> tmp
echo $long >> tmp
cat ppt >> tmp
echo >> tmp
cat ppt >> /$user/data/$domain/data/ppt.htm; echo "</pre>" >> /$user/data/$domain/data/ppt.htm
fi
if [ -e txt ]; then
txtcount=$(wc -l txt | cut -d ' ' -f1)
echo "Text $txtcount" >> zreport
echo "Text Files ($txtcount)" >> tmp
echo $long >> tmp
cat txt >> tmp
echo >> tmp
cat txt >> /$user/data/$domain/data/txt.htm; echo "</pre>" >> /$user/data/$domain/data/txt.htm
fi
if [ -e doc ]; then
doccount=$(wc -l doc | cut -d ' ' -f1)
echo "Word $doccount" >> zreport
echo "Word Files ($doccount)" >> tmp
echo $long >> tmp
cat doc >> tmp
echo >> tmp
cat doc >> /$user/data/$domain/data/doc.htm; echo "</pre>" >> /$user/data/$domain/data/doc.htm
fi
cat tmp >> zreport
echo "Whois Domain" >> zreport
echo $long >> zreport
cat whois-domain >> zreport
echo "Whois IP" >> zreport
echo $long >> zreport
cat whois-ip >> zreport
echo "</pre>" >> /$user/data/$domain/data/emails.htm
echo "</pre>" >> /$user/data/$domain/data/names.htm
echo "</pre>" >> /$user/data/$domain/data/squatting.htm
echo "</pre>" >> /$user/data/$domain/data/subdomains.htm
cat whois-domain >> /$user/data/$domain/data/whois-domain.htm; echo "</pre>" >> /$user/data/$domain/data/whois-domain.htm
cat whois-ip >> /$user/data/$domain/data/whois-ip.htm; echo "</pre>" >> /$user/data/$domain/data/whois-ip.htm
cat zreport >> /$user/data/$domain/data/passive-recon.htm; echo "</pre>" >> /$user/data/$domain/data/passive-recon.htm
rm emails* hosts names squatting sub* tmp* whois* z* doc pdf ppt txt xls 2>/dev/null
echo
echo $medium
echo
echo "***Scan complete.***"
echo
echo
printf 'The supporting data folder is located at \e[1;33m%s\e[0m\n' /$user/data/$domain/
echo
read -p "Press <return> to continue."
##############################################################
f_runlocally
firefox &
sleep 4
firefox -new-tab images.google.com &
sleep 1
firefox -new-tab arin.net &
sleep 1
firefox -new-tab toolbar.netcraft.com/site_report?url=http://www.$domain &
sleep 1
firefox -new-tab shodanhq.com/search?q=$domain &
sleep 1
firefox -new-tab connect.data.com/login/ &
sleep 1
firefox -new-tab pastebin.com/ &
sleep 1
firefox -new-tab google.com/#q=filetype%3Axls+OR+filetype%3Axlsx+site%3A$domain &
sleep 1
firefox -new-tab google.com/#q=filetype%3Appt+OR+filetype%3Apptx+site%3A$domain &
sleep 1
firefox -new-tab google.com/#q=filetype%3Adoc+OR+filetype%3Adocx+site%3A$domain &
sleep 1
firefox -new-tab google.com/#q=filetype%3Apdf+site%3A$domain &
sleep 1
firefox -new-tab google.com/#q=filetype%3Atxt+site%3A$domain &
sleep 1
firefox -new-tab http://www.urlvoid.com/scan/$domain &
sleep 1
firefox -new-tab sec.gov/edgar/searchedgar/companysearch.html &
sleep 1
firefox -new-tab reuters.com/finance/stocks
echo
echo
exit
;;
2)
echo
echo $medium
echo
echo "Usage: target.com"
echo
echo -n "Domain: "
read domain
# Check for no answer
if [ -z $domain ]; then
f_error
fi
# If folder doesn't exist, create it
if [ ! -d /$user/data/$domain ]; then
cp -R /opt/discover/report/ /$user/data/$domain
sed 's/REPLACEDOMAIN/'$domain'/' /$user/data/$domain/index.htm > tmp
mv tmp /$user/data/$domain/index.htm
fi
# Number of tests
total=11
echo
echo $medium
echo
echo "Nmap"
echo " Email (1/$total)"
nmap -Pn -n --open -p80 --script=http-email-harvest --script-args=http-email-harvest.maxpagecount=100,http-email-harvest.maxdepth=10 $domain > tmp
grep @$domain tmp | grep -v '%20' | grep -v 'jpg' | awk '{print $2}' > tmp2
# Change to lower case
cat tmp2 | tr '[A-Z]' '[a-z]' | sort -u > emails1
echo
echo "dnsrecon"
echo " DNS Records (2/$total)"
dnsrecon -d $domain -t std > tmp
egrep -v '(All queries|Bind Version for|Could not|Enumerating SRV|It is resolving|not configured|Performing|Records Found|Recursion|Resolving|TXT|Wildcard)' tmp > tmp2
# Remove first 6 characters from each line
sed 's/^......//' tmp2 | awk '{print $2,$1,$3,$4,$5,$6,$7,$8,$9,$10}' | column -t | sort -u -k2 -k1 > tmp3
grep 'TXT' tmp | sed 's/^......//' | awk '{print $2,$1,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15}' >> tmp3
egrep -v '(SEC3|SKEYs|SSEC)' tmp3 > records
cat /$user/data/$domain/data/records.htm records | grep -v '<' | column -t | sort -u -k2 -k1 > tmp3
echo '<pre style="font-size:14px;">' > /$user/data/$domain/data/records.htm
cat tmp3 | column -t >> /$user/data/$domain/data/records.htm; echo "</pre>" >> /$user/data/$domain/data/records.htm
echo " Zone Transfer (3/$total)"
dnsrecon -d $domain -t axfr > tmp
egrep -v '(Checking for|Failed|filtered|NS Servers|Removing|TCP Open|Testing NS)' tmp | sed 's/^....//' | sed /^$/d > zonetransfer
echo " Sub-domains (~5 min) (4/$total)"
dnsrecon -d $domain -t brt -D /usr/share/dnsrecon/namelist.txt --iw -f > tmp
grep $domain tmp | grep -v "$domain\." | egrep -v '(Performing|Records Found)' | sed 's/\[\*\] //g; s/^[ \t]*//' | awk '{print $2,$3}' | column -t | sort -u > subdomains-dnsrecon
echo
echo "Fierce (~5 min) (5/$total)"
fierce -dns $domain -wordlist /usr/share/fierce/hosts.txt -suppress -file tmp4
sed -n '/Now performing/,/Subnets found/p' tmp4 | grep $domain | awk '{print $2 " " $1}' | column -t | sort -u > subdomains-fierce
cat subdomains-dnsrecon subdomains-fierce | egrep -v '(.nat.|1.1.1.1|6.9.6.9|127.0.0.1)' | column -t | sort -u | awk '$2 !~ /[a-z]/' > subdomains
if [ -e /$user/data/$domain/data/subdomains.htm ]; then
cat /$user/data/$domain/data/subdomains.htm subdomains | grep -v "<" | grep -v "$domain\." | column -t | sort -u > subdomains-combined
echo '<pre style="font-size:14px;">' > /$user/data/$domain/data/subdomains.htm
cat subdomains-combined >> /$user/data/$domain/data/subdomains.htm
echo "</pre>" >> /$user/data/$domain/data/subdomains.htm
fi
awk '{print $3}' records > tmp
awk '{print $2}' subdomains-dnsrecon subdomains-fierce >> tmp
grep -E '[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}' tmp | egrep -v '(-|=|:|1.1.1.1|6.9.6.9|127.0.0.1)' | $sip > hosts
echo
echo "Loadbalancing (6/$total)"
lbd $domain > tmp 2>/dev/null
egrep -v '(Checks if a given|Written by|Proof-of-concept)' tmp > tmp2
# Remove leading whitespace from file
awk '!d && NF {sub(/^[[:blank:]]*/,""); d=1} d' tmp2 > tmp3
# Remove leading whitespace from each line
sed 's/^[ \t]*//' tmp3 > tmp4
egrep -v '(does Load-balancing|does NOT use Load-balancing)' tmp4 | sed 's/Checking for //g' > tmp5
# Remove blank lines from end of file
awk '/^[[:space:]]*$/{p++;next} {for(i=0;i<p;i++){printf "\n"}; p=0; print}' tmp5 > tmp6
# Clean up
cat -s tmp6 | grep -v 'P3P' > loadbalancing
echo
echo "Web Application Firewall (7/$total)"
wafw00f -a http://www.$domain > tmp
cat tmp | egrep -v '(By Sandro|Checking http://www.|Generic Detection|requests|WAFW00F)' > tmp2
sed "s/ http:\/\/www.$domain//g" tmp2 | egrep -v "(\_|\^|\||<|')" | sed '1,4d' > waf
echo
echo "Traceroute"
echo " UDP (8/$total)"
echo "UDP" > tmp
traceroute $domain | awk -F" " '{print $1,$2,$3}' >> tmp
echo >> tmp
echo "ICMP ECHO" >> tmp
echo " ICMP ECHO (9/$total)"
traceroute -I $domain | awk -F" " '{print $1,$2,$3}' >> tmp
echo >> tmp
echo "TCP SYN" >> tmp
echo " TCP SYN (10/$total)"
traceroute -T $domain | awk -F" " '{print $1,$2,$3}' >> tmp
grep -v 'traceroute' tmp > tmp2
# Remove blank lines from end of file
awk '/^[[:space:]]*$/{p++;next} {for(i=0;i<p;i++){printf "\n"}; p=0; print}' tmp2 > ztraceroute
echo
echo "Whatweb (11/$total)"
grep -v '<' /$user/data/$domain/data/subdomains.htm | awk '{print $1}' > tmp
whatweb -i tmp --color=never --no-errors -t 255 > tmp2
# Find lines that start with http, and insert a line after
sort tmp2 | sed '/^http/a\ ' > tmp3
# Cleanup
sed 's/,/\n/g' tmp3 | sed 's/^[ \t]*//' | sed 's/\(\[[0-9][0-9][0-9]\]\)/\n\1/g; s/http:\/\///g' | grep -v 'Country' > whatweb
grep '@' whatweb | sed 's/Email//g; s/\[//g; s/\]//g' > tmp
# Change to lower case
cat tmp | tr '[A-Z]' '[a-z]' > emails2
cat emails1 emails2 | grep "@$domain" | grep -v 'hosting' | cut -d ' ' -f2 | sort -u > emails
# If this file is empty, delete it
if [ ! -s emails ]; then rm emails; fi
if [ ! -s hosts ]; then rm hosts; fi
if [ ! -s records ]; then rm records; fi
if [ ! -s subdomains ]; then rm subdomains; fi
##############################################################
echo > zreport
echo >> zreport
echo "Summary" >> zreport
echo $short >> zreport
echo > tmp
if [ -e emails ]; then
emailcount=$(wc -l emails | cut -d ' ' -f1)
echo "Emails $emailcount" >> zreport
echo "Emails ($emailcount)" >> tmp
echo $short >> tmp
cat emails >> tmp
echo >> tmp
fi
if [ -e hosts ]; then
hostcount=$(wc -l hosts | cut -d ' ' -f1)
echo "Hosts $hostcount" >> zreport
echo "Hosts ($hostcount)" >> tmp
echo $short >> tmp
cat hosts >> tmp
echo >> tmp
fi
if [ -e records ]; then
recordcount=$(wc -l records | cut -d ' ' -f1)
echo "DNS Records $recordcount" >> zreport
echo "DNS Records ($recordcount)" >> tmp
echo $long >> tmp
cat records >> tmp
echo >> tmp
fi
if [ -e subdomains ]; then
subdomaincount=$(wc -l subdomains | cut -d ' ' -f1)
echo "Subdomains $subdomaincount" >> zreport
echo "Subdomains ($subdomaincount)" >> tmp
echo $long >> tmp
cat subdomains >> tmp
echo >> tmp
fi
cat tmp >> zreport
echo "Loadbalancing" >> zreport
echo $long >> zreport
cat loadbalancing >> zreport
echo "Web Application Firewall" >> zreport
echo $long >> zreport
cat waf >> zreport
echo >> zreport
echo "Traceroute" >> zreport
echo $long >> zreport
cat ztraceroute >> zreport
echo >> zreport
echo "Zone Transfer" >> zreport
echo $long >> zreport
cat zonetransfer >> zreport
echo >> zreport
echo "Whatweb" >> zreport
echo $long >> zreport
cat whatweb >> zreport
cat loadbalancing >> /$user/data/$domain/data/loadbalancing.htm; echo "</pre>" >> /$user/data/$domain/data/loadbalancing.htm
cat zreport >> /$user/data/$domain/data/active-recon.htm; echo "</pre>" >> /$user/data/$domain/data/active-recon.htm
cat ztraceroute >> /$user/data/$domain/data/traceroute.htm; echo "</pre>" >> /$user/data/$domain/data/traceroute.htm
cat waf >> /$user/data/$domain/data/waf.htm; echo "</pre>" >> /$user/data/$domain/data/waf.htm
cat whatweb >> /$user/data/$domain/data/whatweb.htm; echo "</pre>" >> /$user/data/$domain/data/whatweb.htm
cat zonetransfer >> /$user/data/$domain/data/zonetransfer.htm; echo "</pre>" >> /$user/data/$domain/data/zonetransfer.htm
if [[ -e /$user/data/$domain/data/emails.htm && -e emails ]]; then
cat /$user/data/$domain/data/emails.htm emails | grep -v '<' | sort -u > tmp
echo '<pre style="font-size:14px;">' > /$user/data/$domain/data/emails.htm
cat tmp >> /$user/data/$domain/data/emails.htm; echo "</pre>" >> /$user/data/$domain/data/emails.htm
fi
cat hosts /$user/data/$domain/data/hosts.htm | grep -v '<' | $sip > tmp
echo '<pre style="font-size:14px;">' > /$user/data/$domain/data/hosts.htm
cat tmp >> /$user/data/$domain/data/hosts.htm; echo "</pre>" >> /$user/data/$domain/data/hosts.htm
rm emails* hosts loadbalancing records sub* tmp* waf whatweb z*
echo
echo $medium
echo
echo "***Scan complete.***"
echo
echo
printf 'The supporting data folder is located at \e[1;33m%s\e[0m\n' /$user/data/$domain/
echo
echo
firefox /$user/data/$domain/index.htm &
exit
;;
3) f_main;;
*) f_error;;
esac
}
##############################################################################################################
f_person(){
f_runlocally
clear
f_banner
echo -e "\e[1;34mRECON\e[0m"
echo
echo -n "First name: "
read firstName
# Check for no answer
if [ -z $firstName ]; then
f_error
fi
echo -n "Last name: "
read lastName
# Check for no answer
if [ -z $lastName ]; then
f_error
fi
firefox &
sleep 2
firefox -new-tab http://www.411.com/name/$firstName-$lastName/ &
sleep 1
firefox -new-tab http://www.advancedbackgroundchecks.com/search/searchpreview.aspx?type=&fn=$firstName&mi=&ln=$lastName&age=&city=&state=
sleep 1
firefox -new-tab http://www.cvgadget.com/person/$firstName/$lastName &
sleep 1
firefox -new-tab http://www.peekyou.com/$fireName_$lastName &
sleep 1
firefox -new-tab http://phonenumbers.addresses.com/people/$firstName+$lastName &
sleep 1
firefox -new-tab https://pipl.com/search/?q=$firstName+$lastName&l=&sloc=&in=10 &
sleep 1
firefox -new-tab http://www.spokeo.com/search?q=$firstName+$lastName&s3=t24 &
sleep 1
firefox -new-tab http://www.zabasearch.com/query1_zaba.php?sname=$firstName%20$lastName&state=ALL&ref=$ref&se=$se&doby=&city=&name_style=1&tm=&tmr=
f_main
}
##############################################################################################################
f_salesforce(){
clear
f_banner
echo -e "\e[1;34mCreate a free account at salesforce (https://connect.data.com/login).\e[0m"
echo -e "\e[1;34mPerform a search on your target company > select the company name > see all.\e[0m"
echo -e "\e[1;34mCopy the results into a new file.\e[0m"
f_location
echo
echo
sed 's/Direct Dial Available//g' $location | sed 's/\[\]//g; s/\.//g; s/,,//g; s/,`//g; s/`,//g; s/-cpg//g; s/3d/3D/g; s/Aberdeen Pr//g; s/ACADEMIC/Academic/g; s/account/Account/g;
s/ACTING/Acting/g; s/3administrator/Administrator/g; s/Europe and Africa//g; s/Sub Saharan Africa//g; s/South Africa//g; s/Agoura Hills//g; s/New Albany//g; s/Albion QL//g;
s/Allison Park//g; s/Altamonte S//g; s/Am-east,//g; s/Am-west,//g; s/Head of Americas//g; s/The Americas//g; s/Amst-north America//g; s/ANALYSIST/Analysist/g; s/Analyst\//Analyst, /g;
s/analytics/Analytics/g; s/and New England//g; s/and Central Us//g; s/North Andover//g; s/Andrews Air//g; s/android/Android/g; s/Annapolis J//g; s/Ann Arbor//g; s/Apple Valley//g;
s/applications/Applications/g; s/Arlington H//g; s/Asia-Pacific//g; s/Asia and India//g; s/asia Pacific Region//g; s/Asia Pacific//g; s/assistant/Assistant/g;
s/AssistantChiefPatrolAgent/Assistant Chief Patrol Agent/g; s/associate/Associate/g; s/at Google//g; s/Atlantic City//g; s/Atm/ATM/g; s/attorney/Attorney/g; s/Australia S//g;
s/automated/Automated/g; s/Ballston Spa//g; s/Bangalore S//g; s/banking/Banking/g; s/Basking Ridge//g; s/Baton Rouge//g; s/Battle Creek//g; s/Battle Ground//g; s/Bay City//g;
s/Bay Shore//g; s/BC//g; s/Bd/BD/g; s/Beaver Falls//g; s/Bel Air//g; s/Bella Vista//g; s/Berkeley He//g; s/Berwyn Hts//g; s/Bethel Park//g; s/Beverly Hills//g; s/billing/Billing/g;
s/Black Belt//g; s/Boca Raton//g; s/BORDER/Border/g; s/Bowling Green//g; s/Boynton Beach//g; s/branch/Branch/g; s/\/Branch/, Branch/g; s/branch/Branch/g; s/Buffalo Grove//g;
s/business/Business/g; s/buyer/Buyer/g; s/By The//g; s/Calabasas Hls//g; s/Camp Hill//g; s/Camp Springs//g; s/Canoga Park//g; s/Canyon Country//g; s/Cape Canaveral//g; s/Cape Coral//g;
s/Cape May//g; s/Capitol Hei//g; s/cargo/Cargo/g; s/Carol Stream//g; s/Carol Stream//g; s/cascade/Cascade/g; s/Castle Rock//g; s/Cedar Hill//g; s/Cedar Rapids//g; s/census/Census/g;
s/Center Line//g; s/CENTER/Center/g; s/Central California//g; s/Central Region//g; s/central Region//g; s/Chagrin Falls//g; s/Charles Town//g; s/Cherry Hill//g; s/Chester Le //g;
s/East Chicago//g; s/\/Chief/, Chief/g; s/China //g; s/Chino Hills//g; s/chromecast/Chromecast/g; s/Chula Vista//g; s/Cissp/CISSP/g; s/CITRIX/Citrix/g; s/clean/Clean/g;
s/Clifton Park//g; s/cms/CMS/g; s/Cms/CMS/g; s/CNN News Group Cable News Network//g; s/Cmms/CMMS/g; s/Cocoa Beach//g; s/Cold Spring//g; s/Colorado Sp//g; s/Commerce City//g;
s/CommitteemanagementOfficer/Committee Management Officer/g; s/compliance/Compliance/g; s/commercial/Commercial/g; s/connected/Connected/g; s/CONSULTANT/Consultant/g;
s/consumer/Consumer/g; s/contact/Contact/g; s/content/Content/g; s/corporate/Corporate/g; s/Corpus Christi//g; s/Council Bluffs//g; s/COUNSEL/Counsel/g; s/counsel/Counsel/g;
s/Cranberry T//g; s/Cranberry Twp//g; s/credit/Credit/g; s/CREDIT/Credit/g; s/Crm/CRM/g; s/Croton On H//g; s/Cross Junction//g; s/Crum Lynne//g; s/Crystal Lake//g; s/Ctr/Center/g;
s/Culver City//g; s/Cuyahoga Falls//g; s/Daly City//g; s/database/Database/g; s/dealer/Dealer/g; s/defense/Defense/g; s/DELIVERY/Delivery/g; s/Del Mar//g; s/Delray Beach//g;
s/Deer Park//g; s/Del Rio//g; s/DEPUTY/Deputy/g; s/West Des Mo//g; s/Des Moines//g; s/Des Plaines//g; s/DesignatedFederalOfficial/Designated Federal Official/g; s/DESIGNER/Designer/g;
s/DESIGN/Design/g; s/development/Development/g; s/DEVICES/Devices/g; s/Diamond Bar//g; s/director/Director/g; s/DISCIPLINED/Disciplined/g; s/discovery/Discovery/g; s/display/Display/g;
s/Dns/DNS/g; s/Downers Grove//g; s/Drexel Hill//g; s/Du Bois//g; s/East Brunswick//g; s/East Central//g; s/East Coast//g; s/East Douglas//g; s/East Greenbush//g; s/East Hanover//g;
s/East Hartford//g; s/East Lansing//g; s/East Peters//g; s/East Stroud//g; s/East Syracuse//g; s/eastern Region//g; s/Eau Claire//g; s/Eden Prairie//g; s/education/Education/g;
s/Egg Harbor//g; s/Egg Harbor//g; s/El Cajon//g; s/El Centro//g; s/El Monte//g; s/El Paso//g; s/El Segundo//g; s/ELECTRIC/Electric /g; s/ELECTRONICS/Electronics/g; s/Port Elizabeth//g;
s/Elk Grove V//g; s/Elk Grove//g; s/Ellicott City//g; s/Elk Grove V//g; s/Elkhart//g; s/Elm Grove//g; s/emerging/Emerging/g; s/endocrinology/Endocrinology/g; s/energy/Energy/g;
s/engineer/Engineer/g; s/enterprise/Enterprise/g; s/ETHICS/Ethics/g; s/Northern Europe//g; s/EVENT/Event/g; s/executive/Executive/g; s/Fairfax Sta//g; s/Fairview He//g;
s/Fall River//g; s/Falls Church//g; s/Farmington Hls//g; s/fashion/Fashion/g; s/federal/Federal/g; s/FELLOW/Fellow/g; s/Fha/FHA/g; s/FIELD/Field/g; s/fillmore/Fillmore/g;
s/financial/Financial/g; s/Flat Rock//g; s/FLIGHT/Flight/g; s/Florham Park//g; s/Flower Mound//g; s/Floyds Knobs//g; s/for Asia and//g; s/Forest Hills//g; s/Forest Hill//g;
s/Forest Park//g; s/Forked River//g; s/foreign/Foreign/g; s/Fort Belvoir//g; s/Fort Bliss//g; s/Fort Collins//g; s/Fort Dodge//g; s/Fort Fairfield//g; s/Fort George//g;
s/Fort Huachuca//g; s/Fort Knox//g; s/Fort Lauder//g; s/Fort Leaven//g; s/Fort Mill//g; s/Fort Monmouth//g; s/Fort Monroe//g; s/Fort Myers//g; s/Fort Pierce//g; s/Fort Rucker//g;
s/Fort Walton//g; s/Fort Washin//g; s/Fort Wayne//g; s/Fort Worth//g; s/Fountain Va//g; s/Franklin Park//g; s/Fremont//g; s/Los Fresnos//g; s/Front Royal//g; s/Fsa/FSA/g; s/Fso/FSO/g;
s/Ft Mitchell//g; s/Ft Worth//g; s/Ft Wright//g; s/FUNCTIONLead/Function Lead/g; s/Gaap/GAAP/g; s/Galway G//g; s/Garden City//g; s/Gig Harbor//g; s/Glen Burnie//g;
s/Glen Ellyn//g; s/Glen Ridge//g; s/Glen Rock//g; s/global/Global/g; s/A Google Company//g; s/Google Access//g; s/Google Adwords//g; s/Google Analytics//g; s/Google Books//g;
s/Google Brand//g; s/Google Checkout//g; s/Google Earth//g; s/Google Enterprise//g; s/Google Federal//g; s/Google Fiber//g; s/Google Finance//g; s/Google Geospatial Services//g;
s/Google Glass//g; s/Google Health//g; s/Google Maps//g; s/Google Media Sales//g; s/Google Offers//g; s/Google Payments//g; s/Google Payment//g; s/Google Plus//g; s/Google Print//g;
s/Google Shopping Express//g; s/Google Shopping//g; s/Google Street View//g; s/Google Talk Team//g; s/Google Travel//g; s/Google Ventures//g; s/Google Voice//g; s/Google Wallet//g;
s/Google X//g; s/Goose Creek//g; s/Granbury//g; s/Grand Forks//g; s/Grand Haven//g; s/Grand Island//g; s/Grand Junction//g; s/Grand Prairie//g; s/Grand Rapids//g; s/Granite City//g;
s/Grants Pass//g; s/Grayslake//g; s/Great Falls//g; s/Green Bay//g; s/Green Belt//g; s/Greenwood Vlg//g; s/Grosse Ile//g; s/Grosse Poin//g; s/group/Group/g; s/Grove City//g;
s/Grp/Group/g; s/Gsa/GSA/g; s/Gsm/GSM/g; s/Gulf Breeze//g; s/Gulf Coast//g; s/Gwynn Oak//g; s/Hampton Cove//g; s/Hampton Roads//g; s/Harbor City//g; s/Harpers Ferry//g;
s/Harrison City//g; s/New Hartford//g; s/West Hartford//g; s/Hanscom Afb//g; s/hazard/Hazard/g; s/Hazel Park//g; s/Hd/HD/g; s/\/Head/, Head/g; s/Hermosa Beach//g; s/Highland Hls//g;
s/Highland Park//g; s/Hilton Head//g; s/Hoffman Est//g; s/West Hollywood//g; s/Homer Glen//g; s/Hot Springs//g; s/Hq/HQ/g; s/Huntingtn Bch//g; s/Hurlburt Field//g; s/Idaho Falls//g;
s/Iii/III/g; s/Ii/II/g; s/IMPORT/Import/g; s/Indian Harb//g; s/information/Information/g; s/institutional/Institutional/g; s/INSTRUMENT/Instrument/g; s/insurance/Insurance/g;
s/intelligence/Intelligence/g; s/international/International/g; s/Inver Grove//g; s/Iselin//g; s/Italy//g; s/Jefferson City//g; s/Jersey City//g; s/Johnson City//g; s/Kansas City//g;
s/KANSS CITY//g; s/Keego Harbor//g; s/Kennett Square//g; s/King George//g; s/King Of Pru//g; s/King Of Pru//g; s/Kings Bay//g; s/Kings Park//g; s/La Follette//g; s/La Grange Park//g;
s/La Grange//g; s/La Jolla//g; s/La Mesa//g; s/La Palma//g; s/La Plata//g; s/La Pocatiere//g; s/Laguna Hills//g; s/Laguna Niguel//g; s/Lake Charles//g; s/Salt Lake City//g;
s/Lake City//g; s/Lake Geneva//g; s/Lake Mary//g; s/Lake Montezuma//g; s/Lake Oswego//g; s/landowner/Landowner/g; s/Las Cruces//g; s/North Las V//g; s/Las Vegas//g;
s/Latin America North//g; s/Latin America//g; s/Mount Laurel//g; s/League City//g; s/LEARNING/Learning/g; s/legal/Legal/g; s/lending/Lending/g; s/Lexington Park//g; s/Linthicum H//g;
s/Little Rock//g; s/Llc/LLC/g; s/New London//g; s/Lone Tree//g; s/Long Beach//g; s/Long Valley//g; s/Logan Township//g; s/Los Angeles//g; s/Los Lunas//g; s/Loves Park//g;
s/Lvl/Level/g; s/Macquarie Park//g; s/MAINFRAME/ Mainframe/g; s/MANAGER/Manager/g; s/Manager\//Manager, /g; s/Mangr/Manager/g; s/manager/Manager/g; s/mangr/Manager/g; s/Manhattan B//g;
s/manufacturing/Manufacturing/g; s/MANUFACTURING/Manufacturing/g; s/Maple Grove//g; s/Maple Shade//g; s/March Air R//g; s/MarketingProductionManager/Marketing Production Manager/g;
s/Marina Del Rey//g; s/market/Market/g; s/master/Master/g; s/materials/Materials/g; s/Mayfield West//g; s/Mays Landing//g; s/Mba/MBA/g; s/Mc Lean//g; s/Mc Coll//g; s/Mc Cordsville//g; s/Mc Kees Rocks//g; s/Mcse/MCSE/g; s/MECHANIC/Mechanic/g; s/medical/Medical/g; s/Melbourne B//g; s/Menlo Park//g; s/Merritt Island//g; s/Metro Jersey District//g; s/Miami Beach//g;
s/Mid-Atlantic//g; s/Middle East//g; s/Middle River//g; s/Upper Midwest//g; s/Millstone T//g; s/Mira Loma//g; s/Mississauga//g; s/MOBILITY/Mobility/g; s/model/Model/g;
s/Moncks Corner//g; s/Monroe Town//g; s/Moor Row//g; s/Moreno Valley//g; s/mortgage/Mortgage/g; s/Morgan Hill//g; s/Morris Plains//g; s/Moss Point//g; s/MOTOROLA/Motorola/g;
s/motorola/Motorola/g; s/Mound City//g; s/Mount Airy//g; s/Mount Holly//g; s/Mount Laurel//g; s/Mount Morrs//g; s/Mount Pleasant//g; s/Mount Pocono//g; s/Mount Prospect//g;
s/Mount Vernon//g; s/Mount Weather//g; s/mountain Region//g; s/Mountain States//g; s/Mountain View//g; s/Mount Waverley//g; s/Muscle Shoals//g; s/Mullica Hill//g; s/MULTI/Multi/g;
s/music/Music/g; s/MyHR/HR/g; s/Myrtle Beach//g; s/National City//g; s/Naval Anaco//g; s/Needham Hei//g; s/negotiator/Negotiator/g; s/New Castle//g; s/New Church//g;
s/New Cumberland//g; s/New Delhi//g; s/New Haven//g; s/New Malden//g; s/New Market//g; s/New Martins//g; s/New Orleans//g; s/New Port Ri//g; s/New Stanton//g; s/New Town//g;
s/New York//g; s/New Zealand//g; s/Newbury Park//g; s/Newport Beach//g; s/Newport News//g; s/Niagara Falls//g; s/North America //g; s/North and Central//g; s/North Baldwin//g;
s/North Bergen//g; s/North Charl//g; s/North East//g; s/North Highl//g; s/North Holly//g; s/North Kings//g; s/North Myrtl//g; s/North Olmsted//g; s/North Royalton//g;
s/North Vernon//g; s/North Wales//g; s/North York//g; s/northern/Northern/g; s/Nsa/NSA/g; s/Nso/NSO/g; s/O Fallon//g; s/Oak Brook//g; s/Oak Creek//g; s/Oak Hill//g; s/Oak Park//g;
s/Oak Ridge//g; s/Oak View//g; s/Oakbrook Te//g; s/Ocean City//g; s/Ocean Grove//g; s/Ocean Springs//g; s/officer/Officer/g; s/Officer\//Officer, /g; s/OFFICE/Office/g;
s/office/Office/g; s/Offutt A F B//g; s/Oklahoma City//g; s/Old Bridge//g; s/Olmsted Falls//g; s/Onited States//g; s/online/Online/g; s/operations/Operations/g; s/Orange Park//g;
s/oriented/Oriented/g; s/Orland Park//g; s/Overland Park//g; s/Owings Mills//g; s/Oxon Hill//g; s/PACKAGING/Packaging/g; s/PACIFIC NORTHWEST//g; s/Pacific Southwest Region //g;
s/Palm Bay//g; s/Palm Beach//g; s/Palm Coast//g; s/Palm Harbor//g; s/Palo Alto//g; s/Palos Hills//g; s/Pompano Beach//g; s/Panama City//g; s/paralegal/Paralegal/g; s/parent/Parent/g;
s/Park Forest//g; s/Park Ridge//g; s/PATROL/Patrol/g; s/Patuxent River//g; s/payments/Payments/g; s/Pc/PC/g; s/Pearl City//g; s/Peachtree City//g; s/Pell City//g; s/Pembroke Pines//g;
s/Perry Hall//g; s/physical/Physical/g; s/Pico Rivera//g; s/Pinellas Park//g; s/PLANNER/Planner/g; s/PLANNING/Planning/g; s/platform/Platform/g; s/PMo/PMO/g; s/PMp//g; s/PMP, //g;
s/Pmp/PMP/g; s/Pm/PM/g; s/Point Pleasant//g; s/PMo/PMO/g; s/Ponca City//g; s/Ponte Vedra//g; s/Poplar Branch//g; s/PortDirector/Port Director/g; s/Port Allen//g; s/Port Deposit//g;
s/Port Orange//g; s/Port Orchard//g; s/PortDirector/Port Directorg/g; s/portfolio/Portfolio/g; s/Powder Springs//g; s/premium/Premium/g; s/Prescott Va//g; s/President -/President, /g;
s/President-/President, /g; s/President\//President, /g; s/president/President/g; s/Princess Anne//g; s/principal/Principal/g; s/Prineville//g; s/private/Private/g;
s/PROCESS/Process/g; s/procurement/Procurement/g; s/PROCUREMENT/Procurement/g; s/producer/Producer/g; s/PRODUCER/Producer/g; s/PROGRAMMING/Programming/g; s/program/Program/g;
s/project/Project/g; s/Prospect Park//g; s/R and D/R&D/g; s/RADIOLOGY/Radiology/g; s/Rancho Palo//g; s/Ransom Canyon//g; s/Rapid City//g; s/real/Real/g; s/receives/Receives/g;
s/recreation/Recreation/g; s/Recruiter\//Recruiter, /g; s/Red Bank//g; s/Redondo Beach//g; s/Redwood City//g; s/regional/Regional/g; s/relationship/Relationship/g;
s/reliability/Reliability/g; s/retail/Retail/g; s/retirement/Retirement/g; s/RFid/RFID/g; s/Rf/RF/g; s/New Richmond//g; s/River Edge//g; s/Rllng Hls Est//g; s/Rochester Hls//g;
s/Rocky Hill//g; s/Rocky Mount//g; s/Rocky River//g; s/Rock Springs//g; s/Rohnert Park//g; s/Rolling Mea//g; s/Round Lk Bch//g; s/Round Rock//g; s/Royal Oak//g; s/SAFETY/Safety/g;
s/Saint-laurent//g; s/Saint Albans//g; s/Saint Ann//g; s/Saint Augus//g; s/Saint Charles//g; s/Saint Clair//g; s/Saint Cloud//g; s/Saint Joseph//g; s/Saint Louis//g; s/Saint Paul//g;
s/Saint Peter//g; s/Saint Rose//g; s/Saint Simon//g; s/sales/Sales/g; s/Salt Lake City//g; s/San Antonio//g; s/San Bernardino//g; s/San Bruno//g; s/San Carlos//g; s/San Clemente//g;
s/San Diego//g; s/San Dimas//g; s/san Francisco Bay//g; s/San Francisco//g; s/San Jose//g; s/San Juan//g; s/San Marcos//g; s/San Mateo//g; s/San Pedro//g; s/San Ramon//g;
s/Santa Ana//g; s/Santa Barbara//g; s/Santa Clara//g; s/Santa Clarita//g; s/Santa Fe//g; s/Santa Isabel//g; s/Santa Maria//g; s/Santa Monica//g; s/Santa Rosa//g; s/Sao Paulo//g;
s/Saratoga Sp//g; s/Schiller Park//g; s/scholar/Scholar/g; s/scientist/Scientist/g; s/SCIENTIST/Scientist/g; s/SCONSUTANT/Consultant/g; s/Scotch Plains//g; s/Scott Afb//g;
s/Scott Air F//g; s/Scotts Valley//g; s/Seal Beach//g; s/SECURITY/Security/g; s/security/Security/g; s/\/Senior/, Senior/g; s/senior/Senior/g; s/SerVices/Services/g;
s/service/Service/g; s/Severna Park//g; s/Sftwr/Software/g; s/Sheffield Vlg//g; s/Shelby Town//g; s/Sherman Oaks//g; s/Show Low//g; s/Sierra Vista//g; s/Silver Spring//g;
s/Sioux City//g; s/Snr/Senior/g; s/Sioux Falls//g; s/smart/Smart/g; s/Smb/SMB/g; s/Sms/SMS/g; s/social/Social/g; s/Solana Beach//g; s/Southeast Region//g; s/Southern and , ,//g;
s/Southern Pines//g; s/South Africa//g; s/South Bend//g; s/South Burli//g; s/South Central//g; s/South Dakota//g; s/South East//g; s/South-east//g; s/South Orange//g; s/South San F//g;
s/South Lake//g; s/South Ozone//g; s/South Plain//g; s/South River//g; s/South East Asia//g; s/South-east Asia//g; s/space/Space/g; s/spain/Spain/g; s/Spring City//g; s/Sql/SQL/g;
s/SrBranch/Senior Branch/g; s/SrSales/Senior Sales/g; s/Ssl/SSL/g; s/St. Asaph//g; s/St Augustine//g; s/St Charles//g; s/St Johnsbury//g; s/St Leonards//g; s/St Petersburg//g;
s/St Thomas//g; s/State College//g; s/Stennis Spa//g; s/Stephens City//g; s/Sterling He//g; s/Stevens Point//g; s/Stf/Staff/g; s/STOCK/Stock/g; s/Stone Harbor//g; s/Stone Mountain//g;
s/strategic/Strategic/g; s/subsidiary/Subsidiary/g; s/Sugar Land//g; s/Sugar Grove//g; s/supply/Supply/g; s/support/Support/g; s/Takoma Park//g; s/Tall Timbers//g; s/teacher/Teacher/g;
s/TEAM/Team/g; s/Teaneck//g; s/technical/Technical/g; s/technology/Technology/g; s/TELECOMMUNICATIONS/Telecommunications/g; s/television/Television/g; s/TEST/Test/g;
s/Thailand and Philippines//g; s/The Dalles//g; s/Thousand Oaks//g; s/Timber Lake//g; s/Tipp City//g; s/Township Of//g; s/Trabuco Canyon//g; s/TRADEMARKS/Trademarks/g;
s/trainer/Trainer/g; s/TRANSPORTATION/Transportation/g; s/treasury/Treasury/g; s/Tunbridge W//g; s/Twin Falls//g; s/UK//g; s/U.S.//g; s/UNDERWRITER/Underwriter/g; s/Union Ban//g;
s/Union City//g; s/Union Office//g; s/United Kingdom//g; s/United States//g; s/Universal City//g; s/university/University/g; s/Upper Chich//g; s/Upper Marlboro//g; s/Uscg/USCG/g;
s/valve/Valve/g; s/Valley Stream//g; s/Van Nuys//g; s/vendor/Vendor/g; s/Vernon Hills//g; s/Vero Beach//g; s/Vii/VII/g; s/Vi /VI/g; s/Vice-President/Vice President/g;
s/Vicepresident/Vice President/g; s/Virginia Beach//g; s/La Vista//g; s/Voip/VoIP/g; s/Walled Lake//g; s/Wallops Island//g; s/Walnut Creek//g; s/Warner Robins//g; s/wealth/Wealth/g;
s/West Bloomf//g; s/West Chester//g; s/West Columbia//g; s/West Dundee//g; s/West Harrison//g; s/West Linn//g; s/West Mifflin//g; s/West Nyack//g; s/West Orange//g; s/West Palm B//g;
s/West Paterson//g; s/west Region//g; s/West Sacram//g; s/West Spring//g; s/Western Spr//g; s/West Orange//g; s/White Lake//g; s/White Plains//g; s/White River//g; s/Whiteman Ai//g;
s/Whitmore Lake//g; s/Williston Park//g; s/Willow Grove//g; s/South Windsor//g; s/Windsor Locks//g; s/Windsor Mill//g; s/Winston Salem//g; s/Winter Park//g; s/Winter Springs//g;
s/Woodland Hills//g; s/Woodland Park//g; s/worldwide/Worldwide/g;
s/AK //g; s/AL //g; s/AR //g; s/AZ //g; s/CA //g; s/CO //g; s/CT //g; s/DC //g; s/DE //g; s/FL //g; s/GA //g; s/HI //g; s/IA //g; s/ID //g; s/IL //g; s/IN //g; s/KA //g; s/KS //g;
s/KY //g; s/LA //g; s/MA //g; s/ME //g; s/MD //g; s/MI //g; s/MO //g; s/MN //g; s/MS //g; s/MT //g; s/NC //g; s/NE //g; s/ND //g; s/NH //g; s/NJ //g; s/NM //g; s/NV //g; s/NY //g;
s/OH //g; s/OK //g; s/ON //g; s/OR //g; s/PA //g; s/PR //g; s/QC //g; s/RI //g; s/SC //g; s/SD //g; s/TN //g; s/TX //g; s/Uk //g; s/UP //g; s/UT //g; s/VA //g; s/VT //g; s/WA //g;
s/WI //g; s/WV //g; s/WY //g; s/AP //g; s/DL //g; s/MH //g; s/[0-9]\{2\}\/[0-9]\{2\}\/[0-9]\{2\}//g; s/^[ \tmp]*//g' > tmp
# Author: Ben Wood
perl -ne 'if ($_ =~ /(.*?)\t\s*(.*)/) {printf("%-40s%s\n",$1,$2);}' tmp | sed 's/[ \t]*$//g' | sort> tmp2
cat tmp2 | sed 's/ -/ -/g; s/, /, /g; s/, , , , //g; s/, , , //g; s/, , /, /g; s/,$//g; s/\/$//g; s/-$//g; s/Aberdeen$//g; s/Abilene$//g; s/Abingdon$//g; s/Abington$//g;
s/Acworth$//g; s/Adamstown$//g; s/Addison$//g; s/Adena$//g; s/AdSense$//g; s/Adwords$//g; s/Africa$//g; s/Aguadilla$//g; s/Ainsworth$//g; s/Akron$//g; s/Alabaster$//g; s/Albany$//g;
s/Albuquerque$//g; s/Aldershot$//g; s/Alexandria$//g; s/Allegan$//g; s/Allentown$//g; s/Alma$//g; s/Alpena$//g; s/Alpharetta$//g; s/Americas$//g; s/Americus$//g; s/Ambler$//g;