Clean the contrib directory with torch and machete.

We've accumulated a lot of cruft in this directory over the years: so
much, that it passed the point of being so disorganized that we no
longer browsed through it to see how bad it had gotten.

This patch (based on changes by rl1987) tries to remove the most
useless items, and split the others into reasonable directories.  It
creates a new scripts/ directory for maint and test scripts.

This patch was generated with the script below.  No other changes are made in
this patch.

#############
# new directories
mkdir -p contrib/test-tools
mkdir -p contrib/or-tools
mkdir -p contrib/dirauth-tools
mkdir -p contrib/operator-tools
mkdir -p contrib/client-tools
mkdir -p contrib/test-tools
mkdir -p contrib/dist
mkdir -p contrib/dist/suse
mkdir -p contrib/win32build

mkdir -p scripts/maint
mkdir -p scripts/test

############
# Deleted -- nobody who wants this is going to be looking for it here any
# longer.  Also, nobody wants it.
git rm contrib/auto-naming/README

# Deleted: We no longer do polipo.
git rm contrib/polipo/Makefile.mingw
git rm contrib/polipo/README
git rm contrib/polipo/polipo-mingw.nsi

# We haven't even tried to run this for ages. It is a relic of a bygone era
git rm contrib/mdd.py

# contrib/dir-tools/directory-archive/
# Tools for running a directory archive. No longer used - deleting them.
git rm contrib/directory-archive/crontab.sample
git rm contrib/directory-archive/fetch-all
git rm contrib/directory-archive/fetch-all-v3
git rm contrib/directory-archive/tar-them-up
git rm contrib/directory-archive/fetch-all-functions
git rm contrib/directory-archive/sort-into-month-folder

# This appears to be related to very old windows packaging stuff.
git rm contrib/bundle.nsi
git rm contrib/package_nsis-weasel.sh
git rm contrib/package_nsis.sh
git rm contrib/netinst.nsi
git rm contrib/torinst32.ico
git rm contrib/xenobite.ico

# This should not be needed for cross-compilation any more, should it?
git rm contrib/cross.sh

# I don't think anyone ever used this.
git rm contrib/make-signature.sh

# These are attempts to send tor controller commands from the command-line.
# They don't support modern authentication.
git rm contrib/tor-ctrl.sh

# this is for fetching about a tor server from a dirauth. But it
# doesn't authenticate the dirauth: yuck.
git rm contrib/sd

# wow, such unused, very perl4.
git rm contrib/tor-stress

####### contrib/dirauth-tools/
# Tools for running a directory authority

git mv contrib/add-tor contrib/dirauth-tools/
git mv contrib/nagios-check-tor-authority-cert contrib/dirauth-tools/

#######
# contrib/or-tools/
# Tools for examining relays
git mv contrib/check-tor contrib/or-tools/check-tor
git mv contrib/checksocks.pl contrib/or-tools/checksocks.pl
git mv contrib/exitlist contrib/or-tools/exitlist

#######
# contrib/operator-tools

# Tools for running a relay.
git mv contrib/linux-tor-prio.sh contrib/operator-tools/linux-tor-prio.sh
git mv contrib/tor-exit-notice.html contrib/operator-tools/tor-exit-notice.html
git mv contrib/tor.logrotate.in contrib/operator-tools/

######
# contrib/dist

git mv contrib/rc.subr contrib/dist/
git mv contrib/tor.sh.in contrib/dist/
git mv contrib/torctl.in contrib/dist/
git mv contrib/suse/* contrib/dist/suse/

######
# client-tools
git mv contrib/torify contrib/client-tools/torify
git mv contrib/tor-resolve.py contrib/client-tools/

######
# win32build

git mv contrib/package_nsis-mingw.sh contrib/win32build/
git mv contrib/tor.nsi.in contrib/win32build/
# Erinn didn't ask for this...
git mv contrib/tor-mingw.nsi.in contrib/win32build/
git mv contrib/tor.ico contrib/win32build/

######
# scripts/test
git mv contrib/cov-blame scripts/test/cov-blame
git mv contrib/cov-diff scripts/test/cov-diff
git mv contrib/coverage scripts/test/coverage
git mv contrib/scan-build.sh scripts/test/

######## scripts/maint
# Maintainance scripts
#
# These are scripts for developers to use when hacking on Tor.  They mostly
# look at the Tor source in one way or another.
git mv contrib/findMergedChanges.pl scripts/maint/findMergedChanges.pl
git mv contrib/checkOptionDocs.pl scripts/maint/checkOptionDocs.pl
git mv contrib/checkSpace.pl scripts/maint/checkSpace.pl
git mv contrib/redox.py scripts/maint/redox.py
git mv contrib/updateVersions.pl scripts/maint/updateVersions.pl
git mv contrib/checkLogs.pl scripts/maint/checkLogs.pl
git mv contrib/format_changelog.py scripts/maint/

3 files changed, 447 insertions, 0 deletions

diff --git a/contrib/or-tools/check-tor b/contrib/or-tools/check-tor
new file mode 100755
index 0000000000..e981a35fcc
--- /dev/null
+++ b/contrib/or-tools/check-tor
@@ -0,0 +1,41 @@
+#!/bin/sh
+
+## Originally written by Peter Palfrader.
+
+## This script lets you quickly check if a given router (by nickname)
+## will let you do a TLS handshake, or will let you download a directory.
+
+## Usage: check-tor nickname
+
+#set -x
+
+router="$1"
+dirserver="http://belegost.seul.org:80/tor/"
+
+lines=$( wget -q $dirserver --proxy=off -O - | grep -A5 '^router '"$router"' ' )
+line=$( echo "$lines" | head -n1 )
+
+if [ -z "$line" ]; then
+	echo "Not found" >&2
+	exit 1
+fi
+
+echo "$lines"
+echo
+
+ipor=$( echo "$line" | awk '{printf "%s:%s", $3, $4}' )
+
+op=$( echo "$line" | awk '{printf $6}' )
+ipop=$( echo "$line" | awk '{printf "%s:%s", $3, $6}' )
+
+echo
+echo ">>" openssl s_client -connect "$ipor"
+timeout 5 openssl s_client -connect "$ipor" < /dev/null
+if [ "$op" != "0" ]; then
+	echo
+	echo ">>" wget --proxy=off -O - http://$ipop/tor/
+	timeout 5 wget --proxy=off -O - http://$ipop/tor/ | head -n3
+fi
+
+echo
+echo -n "$router "; echo "$lines" | grep 'fingerprint' | sed -e 's/^opt //' -e 's/^fingerprint //';
diff --git a/contrib/or-tools/checksocks.pl b/contrib/or-tools/checksocks.pl
new file mode 100755
index 0000000000..3fcc0df14a
--- /dev/null
+++ b/contrib/or-tools/checksocks.pl
@@ -0,0 +1,83 @@
+#!/usr/bin/perl -w
+
+require 5.005;
+use strict;
+use IO::Socket;
+use Getopt::Std;
+
+# Checks routers for open socks-ports and socks5
+# Successful connects go to STDOUT, failed ones to STDERR.
+# We only do one check per loop in -d mode, so it takes some time.
+
+# Contributed by Peter Kornherr <peter at wuschelpuschel dot org>, and
+# cleaned up by Peter Palfrader <peter at palfrader dot org>.
+
+our($opt_i,$opt_p,$opt_d,$opt_h,$opt_l);
+getopts('i:p:dhl:');
+
+if ($opt_h || !($opt_d||$opt_i||$opt_l)) {
+    print "Usage: $0 -d < file_with_routers_in_it\n";
+    print "or:    $0 -i IP -p Port\n";
+    print "or:    $0 -l IP:Port\n";
+    exit;
+}
+
+if ($opt_d) {
+    open (IN,"<-") or die $!;
+    while (<IN>) {
+        next unless /^router /;
+        (my $routername,my $checkip,my $checkport) = (split(" "))[1,2,4];
+        &do_check($checkip,$checkport,$routername);
+    }
+} elsif ($opt_i && $opt_p) {
+    &do_check($opt_i,$opt_p);
+} elsif ($opt_l) {
+    &do_check(split(":",$opt_l));
+}
+
+sub do_check {
+    (my $checkip, my $checkport,my $routername) = @_;
+    # as socksports may not be published (therefore "0") here,
+    # let's try 9050, the default port:
+    if ($checkport == 0) { $checkport = 9050; }
+    # print "Checking $checkip:$checkport\n";
+    my $s5socket = IO::Socket::INET->new(PeerAddr => $checkip,
+        PeerPort => $checkport, Proto => "tcp", Type => SOCK_STREAM,
+        Timeout => "20");
+    if ($s5socket) {
+        my @got;
+        print $s5socket pack("CCC",'5','1','0');
+        eval {
+            local $SIG{ALRM} = sub { die "alarm\n" };
+            alarm 10;
+            read ($s5socket,$got[0],1);
+            read ($s5socket,$got[1],1);
+            alarm 0;
+        };
+        if ($@) {
+            return; # die unless $@ eq "alarm\n";
+        }
+        if ($got[0] eq pack('C','5')) {
+            if(defined($routername)) {
+                print "Found SOCKS5 at $routername ($checkip:$checkport)\n";
+            } else {
+                print "Found SOCKS5 at $checkip:$checkport\n";
+            }
+        } else {
+            if(defined($routername)) {
+                print "$routername ($checkip:$checkport) answers - " .
+                      "but not SOCKS5.\n";
+            } else {
+                print "$checkip:$checkport answers - but not SOCKS5.\n";
+            }
+        }
+    } else {
+        if(defined($routername)) {
+            print STDERR "Can't connect to $routername " .
+                         "($checkip:$checkport) ($!)\n";
+        } else {
+            print STDERR "Can't connect to $checkip:$checkport ($!)\n";
+        }
+    }
+}
+
diff --git a/contrib/or-tools/exitlist b/contrib/or-tools/exitlist
new file mode 100755
index 0000000000..3fd26b5166
--- /dev/null
+++ b/contrib/or-tools/exitlist
@@ -0,0 +1,323 @@
+#!/usr/bin/python
+# Copyright 2005-2006 Nick Mathewson
+# See the LICENSE file in the Tor distribution for licensing information.
+
+# Requires Python 2.2 or later.
+
+"""
+ exitlist -- Given a Tor directory on stdin, lists the Tor servers
+ that accept connections to given addreses.
+
+ example usage:
+
+    cat ~/.tor/cached-descriptors* | python exitlist 18.244.0.188:80
+
+ You should look at the "FetchUselessDescriptors" and "FetchDirInfoEarly"
+ config options in the man page.
+
+ Note that this script won't give you a perfect list of IP addresses
+ that might connect to you using Tor.
+ False negatives:
+   - Some Tor servers might exit from other addresses than the one they
+     publish in their descriptor.
+ False positives:
+   - This script just looks at the descriptor lists, so it counts relays
+     that were running a day in the past and aren't running now (or are
+     now running at a different address).
+
+ See https://check.torproject.org/ for an alternative (more accurate!)
+  approach.
+
+"""
+
+#
+# Change this to True if you want more verbose output.  By default, we
+# only print the IPs of the servers that accept any the listed
+# addresses, one per line.
+#
+VERBOSE = False
+
+#
+# Change this to True if you want to reverse the output, and list the
+# servers that accept *none* of the listed addresses.
+#
+INVERSE = False
+
+#
+# Change this list to contain all of the target services you are interested
+# in.  It must contain one entry per line, each consisting of an IPv4 address,
+# a colon, and a port number. This default is only used if we don't learn
+# about any addresses from the command-line.
+#
+ADDRESSES_OF_INTEREST = """
+    1.2.3.4:80
+"""
+
+
+#
+# YOU DO NOT NEED TO EDIT AFTER THIS POINT.
+#
+
+import sys
+import re
+import getopt
+import socket
+import struct
+import time
+
+assert sys.version_info >= (2,2)
+
+
+def maskIP(ip,mask):
+    return "".join([chr(ord(a) & ord(b)) for a,b in zip(ip,mask)])
+
+def maskFromLong(lng):
+    return struct.pack("!L", lng)
+
+def maskByBits(n):
+    return maskFromLong(0xffffffffl ^ ((1L<<(32-n))-1))
+
+class Pattern:
+    """
+    >>> import socket
+    >>> ip1 = socket.inet_aton("192.169.64.11")
+    >>> ip2 = socket.inet_aton("192.168.64.11")
+    >>> ip3 = socket.inet_aton("18.244.0.188")
+
+    >>> print Pattern.parse("18.244.0.188")
+    18.244.0.188/255.255.255.255:1-65535
+    >>> print Pattern.parse("18.244.0.188/16:*")
+    18.244.0.0/255.255.0.0:1-65535
+    >>> print Pattern.parse("18.244.0.188/2.2.2.2:80")
+    2.0.0.0/2.2.2.2:80-80
+    >>> print Pattern.parse("192.168.0.1/255.255.00.0:22-25")
+    192.168.0.0/255.255.0.0:22-25
+    >>> p1 = Pattern.parse("192.168.0.1/255.255.00.0:22-25")
+    >>> import socket
+    >>> p1.appliesTo(ip1, 22)
+    False
+    >>> p1.appliesTo(ip2, 22)
+    True
+    >>> p1.appliesTo(ip2, 25)
+    True
+    >>> p1.appliesTo(ip2, 26)
+    False
+    """
+    def __init__(self, ip, mask, portMin, portMax):
+        self.ip = maskIP(ip,mask)
+        self.mask = mask
+        self.portMin = portMin
+        self.portMax = portMax
+
+    def __str__(self):
+        return "%s/%s:%s-%s"%(socket.inet_ntoa(self.ip),
+                              socket.inet_ntoa(self.mask),
+                              self.portMin,
+                              self.portMax)
+
+    def parse(s):
+        if ":" in s:
+            addrspec, portspec = s.split(":",1)
+        else:
+            addrspec, portspec = s, "*"
+
+        if addrspec == '*':
+            ip,mask = "\x00\x00\x00\x00","\x00\x00\x00\x00"
+        elif '/' not in addrspec:
+            ip = socket.inet_aton(addrspec)
+            mask = "\xff\xff\xff\xff"
+        else:
+            ip,mask = addrspec.split("/",1)
+            ip = socket.inet_aton(ip)
+            if "." in mask:
+                mask = socket.inet_aton(mask)
+            else:
+                mask = maskByBits(int(mask))
+
+        if portspec == '*':
+            portMin = 1
+            portMax = 65535
+        elif '-' not in portspec:
+            portMin = portMax = int(portspec)
+        else:
+            portMin, portMax = map(int,portspec.split("-",1))
+
+        return Pattern(ip,mask,portMin,portMax)
+
+    parse = staticmethod(parse)
+
+    def appliesTo(self, ip, port):
+        return ((maskIP(ip,self.mask) == self.ip) and
+                (self.portMin <= port <= self.portMax))
+
+class Policy:
+    """
+    >>> import socket
+    >>> ip1 = socket.inet_aton("192.169.64.11")
+    >>> ip2 = socket.inet_aton("192.168.64.11")
+    >>> ip3 = socket.inet_aton("18.244.0.188")
+
+    >>> pol = Policy.parseLines(["reject *:80","accept 18.244.0.188:*"])
+    >>> print str(pol).strip()
+    reject 0.0.0.0/0.0.0.0:80-80
+    accept 18.244.0.188/255.255.255.255:1-65535
+    >>> pol.accepts(ip1,80)
+    False
+    >>> pol.accepts(ip3,80)
+    False
+    >>> pol.accepts(ip3,81)
+    True
+    """
+
+    def __init__(self, lst):
+        self.lst = lst
+
+    def parseLines(lines):
+        r = []
+        for item in lines:
+            a,p=item.split(" ",1)
+            if a == 'accept':
+                a = True
+            elif a == 'reject':
+                a = False
+            else:
+                raise ValueError("Unrecognized action %r",a)
+            p = Pattern.parse(p)
+            r.append((p,a))
+        return Policy(r)
+
+    parseLines = staticmethod(parseLines)
+
+    def __str__(self):
+        r = []
+        for pat, accept in self.lst:
+            rule = accept and "accept" or "reject"
+            r.append("%s %s\n"%(rule,pat))
+        return "".join(r)
+
+    def accepts(self, ip, port):
+        for pattern,accept in self.lst:
+            if pattern.appliesTo(ip,port):
+                return accept
+        return True
+
+class Server:
+    def __init__(self, name, ip, policy, published, fingerprint):
+        self.name = name
+        self.ip = ip
+        self.policy = policy
+        self.published = published
+        self.fingerprint = fingerprint
+
+def uniq_sort(lst):
+    d = {}
+    for item in lst: d[item] = 1
+    lst = d.keys()
+    lst.sort()
+    return lst
+
+def run():
+    global VERBOSE
+    global INVERSE
+    global ADDRESSES_OF_INTEREST
+
+    if len(sys.argv) > 1:
+        try:
+            opts, pargs = getopt.getopt(sys.argv[1:], "vx")
+        except getopt.GetoptError, e:
+            print """
+usage: cat ~/.tor/cached-routers* | %s [-v] [-x] [host:port [host:port [...]]]
+    -v  verbose output
+    -x  invert results
+""" % sys.argv[0]
+            sys.exit(0)
+
+        for o, a in opts:
+            if o == "-v":
+                VERBOSE = True
+            if o == "-x":
+                INVERSE = True
+        if len(pargs):
+            ADDRESSES_OF_INTEREST = "\n".join(pargs)
+
+    servers = []
+    policy = []
+    name = ip = None
+    published = 0
+    fp = ""
+    for line in sys.stdin.xreadlines():
+        if line.startswith('router '):
+            if name:
+                servers.append(Server(name, ip, Policy.parseLines(policy),
+                                      published, fp))
+            _, name, ip, rest = line.split(" ", 3)
+            policy = []
+            published = 0
+            fp = ""
+        elif line.startswith('fingerprint') or \
+                 line.startswith('opt fingerprint'):
+            elts = line.strip().split()
+            if elts[0] == 'opt': del elts[0]
+            assert elts[0] == 'fingerprint'
+            del elts[0]
+            fp = "".join(elts)
+        elif line.startswith('accept ') or line.startswith('reject '):
+            policy.append(line.strip())
+        elif line.startswith('published '):
+            date = time.strptime(line[len('published '):].strip(),
+                                 "%Y-%m-%d %H:%M:%S")
+            published = time.mktime(date)
+
+    if name:
+        servers.append(Server(name, ip, Policy.parseLines(policy), published,
+                              fp))
+
+    targets = []
+    for line in ADDRESSES_OF_INTEREST.split("\n"):
+        line = line.strip()
+        if not line: continue
+        p = Pattern.parse(line)
+        targets.append((p.ip, p.portMin))
+
+    # remove all but the latest server of each IP/Nickname pair.
+    latest = {}
+    for s in servers:
+        if (not latest.has_key((s.fingerprint))
+            or s.published > latest[(s.fingerprint)]):
+            latest[s.fingerprint] = s
+    servers = latest.values()
+
+    accepters, rejecters = {}, {}
+    for s in servers:
+        for ip,port in targets:
+            if s.policy.accepts(ip,port):
+                accepters[s.ip] = s
+                break
+        else:
+            rejecters[s.ip] = s
+
+    # If any server at IP foo accepts, the IP does not reject.
+    for k in accepters.keys():
+        if rejecters.has_key(k):
+            del rejecters[k]
+
+    if INVERSE:
+        printlist = rejecters.values()
+    else:
+        printlist = accepters.values()
+
+    ents = []
+    if VERBOSE:
+        ents = uniq_sort([ "%s\t%s"%(s.ip,s.name) for s in printlist ])
+    else:
+        ents = uniq_sort([ s.ip for s in printlist ])
+    for e in ents:
+        print e
+
+def _test():
+    import doctest, exitparse
+    return doctest.testmod(exitparse)
+#_test()
+
+run()
+