3 # Get the TOR exit and relay lists from the net, extract the exit and relay
4 # node ip addresses and store them, one per line, in the standard places
5 # in /spider/local_data.
10 # search local then perl directories
14 # root of directory tree for this system
15 our $root = "/spider";
16 $root = $ENV{'DXSPIDER_ROOT'} if $ENV{'DXSPIDER_ROOT'};
18 mkdir "$root/local_data", 02777 unless -d "$root/local_data";
20 unshift @INC, "$root/perl"; # this IS the right way round!
21 unshift @INC, "$root/local";
22 our $data = "$root/data";
39 $ENV{PERL_JSON_BACKEND} = "JSON::XS,JSON::PP";
44 if (@ARGV && $ARGV[0] eq '-x') {
48 my $url = "https://onionoo.torproject.org/details";
49 my $relayfn = localdata('badip.torrelay');
50 my $exitfn = localdata('badip.torexit');
52 my $last_seen_window = 10800;
58 open IN, $fn or die "$0 cannot open file $fn, $!";
62 $content = get($url) or die "$0: connect error on $url, $!\n";
65 die "No TOR content available $!\n" unless $content;
67 my $l = length $content;
68 my $data = decode_json($content);
74 open RELAY, ">$relayfn.$rand" or die "$0: cannot open $relayfn $!";
75 open EXIT, ">$exitfn.$rand" or die "$0: cannot open $exitfn $1";
77 foreach my $e (@{$data->{relays}}) {
79 my $seen = str2time($e->{last_seen});
80 next unless $seen >= $now - $last_seen_window;
82 my @or = clean_addr(@{$e->{or_addresses}}) if exists $e->{or_addresses};
83 my @exit = clean_addr(@{$e->{exit_addresses}}) if exists $e->{exit_addresses} ;
84 my $ors = join ', ', @or;
85 my $es = join ', ', @exit;
86 dbg "$0: $e->{nickname} $e->{last_seen} relays: [$ors] exits: [$es]" if $debug;
100 dbg("$0: $rcount relays $ecount exits found");
101 move "$relayfn.$rand", $relayfn if $rcount;
102 move "$exitfn.$rand", $exitfn if $ecount;
103 unlink "$relayfn.$rand";
104 unlink "$exitfn.$rand";
112 my ($ipv4) = /^((?:\d+\.){3}\d+)/;
117 my ($ipv6) = /^\[([:a-f\d]+)\]/;
118 push @out, $ipv6 if $ipv6;