diff options
| -rwxr-xr-x | perl/crawlink.pl | 42 |
1 files changed, 32 insertions, 10 deletions
diff --git a/perl/crawlink.pl b/perl/crawlink.pl index d7855c383..53be500cc 100755 --- a/perl/crawlink.pl +++ b/perl/crawlink.pl @@ -9,10 +9,14 @@ # Written to use 'curl' for URL checking. # # Author: Daniel Stenberg <daniel@haxx.se> -# Version: 0.2 Dec 19, 2000 +# Version: 0.3 Jan 3, 2001 # # HISTORY # +# 0.3 - The -i now adds regexes that if a full URL link matches one of those, +# it is not followed. This can then be used to prevent this script from +# following '.*\.cgi', specific pages or whatever. +# # 0.2 - Made it only HEAD non html files (i.e skip the GET). Makes it a lot # faster to skip large non HTML files such as pdfs or big RFCs! ;-) # Added a -c option that allows me to pass options to curl. @@ -32,6 +36,8 @@ my $help; my $external; my $curlopts; +my @ignorelist; + argv: if($ARGV[0] eq "-v" ) { $verbose++; @@ -44,6 +50,12 @@ elsif($ARGV[0] eq "-c" ) { shift @ARGV; goto argv; } +elsif($ARGV[0] eq "-i" ) { + push @ignorelist, $ARGV[1]; + shift @ARGV; + shift @ARGV; + goto argv; +} elsif($ARGV[0] eq "-l" ) { $linenumber = 1; shift @ARGV; @@ -72,10 +84,12 @@ $rooturls{$ARGV[0]}=1; if(($geturl eq "") || $help) { print "Usage: $0 [-hilvx] <full URL>\n", " Use a traling slash for directory URLs!\n", - " -h This help text\n", - " -l Line number report for BAD links\n", - " -v Verbose mode\n", - " -x Check non-local (external?) links only\n"; + " -c [data] Pass [data] as argument to every curl invoke\n", + " -h This help text\n", + " -i [regex] Ignore root links that match this pattern\n", + " -l Line number report for BAD links\n", + " -v Verbose mode\n", + " -x Check non-local (external?) links only\n"; exit; } @@ -303,9 +317,6 @@ while(1) { if($geturl == -1) { last; } - if($verbose) { - print "ROOT: $geturl\n"; - } # # Splits the URL in its different parts @@ -332,6 +343,8 @@ while(1) { next; } + print " ==== $geturl ====\n"; + if($verbose == 2) { printf("Error code $error, Content-Type: $ctype, got %d bytes\n", length($in)); @@ -405,8 +418,17 @@ while(1) { } } else { - # the link works, add it! - $rooturls{$link}++; # check this if not checked already + # the link works, add it if it isn't in the ingore list + my $ignore=0; + for(@ignorelist) { + if($link =~ /$_/) { + $ignore=1; + } + } + if(!$ignore) { + # not ignored, add + $rooturls{$link}++; # check this if not checked already + } } } |