Migrate the reindex script to python, since adding features to the Perl was getting too ugly. Make it handle line-continuations, and include target versions in output.

svn:r17531

4 files changed, 165 insertions, 147 deletions

diff --git a/doc/spec/proposals/000-index.txt b/doc/spec/proposals/000-index.txt
index 8fca91b324..63ee4701f3 100644
--- a/doc/spec/proposals/000-index.txt
+++ b/doc/spec/proposals/000-index.txt
@@ -13,7 +13,7 @@ Overview:
    This is an informational document.
 
    Everything in this document below the line of '=' signs is automatically
-   generated by reindex.pl; do not edit by hand.
+   generated by reindex.py; do not edit by hand.
 
 ============================================================
 Proposals by number:
@@ -66,7 +66,7 @@ Proposals by number:
 141  Download server descriptors on demand [DRAFT]
 142  Combine Introduction and Rendezvous Points [OPEN]
 143  Improvements of Distributed Storage for Tor Hidden Service Descriptors [OPEN]
-144  Increase the diversity of circuits by detecting nodes belonging the [DRAFT]
+144  Increase the diversity of circuits by detecting nodes belonging the same provider [DRAFT]
 145  Separate "suitable as a guard" from "suitable as a new guard" [OPEN]
 146  Add new flag to reflect long-term stability [OPEN]
 147  Eliminate the need for v2 directories in generating v3 directories [ACCEPTED]
@@ -74,7 +74,7 @@ Proposals by number:
 149  Using data from NETINFO cells [OPEN]
 150  Exclude Exit Nodes from a circuit [CLOSED]
 151  Improving Tor Path Selection [DRAFT]
-152  Optionally allow exit from single-hop circuits  [CLOSED]
+152  Optionally allow exit from single-hop circuits [CLOSED]
 153  Automatic software update protocol [SUPERSEDED]
 154  Automatic Software Update Protocol [SUPERSEDED]
 155  Four Improvements of Hidden Service Performance [OPEN]
@@ -89,62 +89,61 @@ Proposals by status:
    132  A Tor Web Service For Verifying Correct Browser Configuration
    133  Incorporate Unreachable ORs into the Tor Network
    141  Download server descriptors on demand
-   144  Increase the diversity of circuits by detecting nodes belonging the
+   144  Increase the diversity of circuits by detecting nodes belonging the same provider
    151  Improving Tor Path Selection
- OPEN:
-   121  Hidden Service Authentication
-   142  Combine Introduction and Rendezvous Points
-   143  Improvements of Distributed Storage for Tor Hidden Service Descriptors
-   145  Separate "suitable as a guard" from "suitable as a new guard"
-   146  Add new flag to reflect long-term stability
-   149  Using data from NETINFO cells
-   155  Four Improvements of Hidden Service Performance
-   156  Tracking blocked ports on the client side
-   157  Make certificate downloads specific
  NEEDS-REVISION:
    131  Help users to verify they are using Tor
- NEEDS-RESEARCH:
+ OPEN:
+   121  Hidden Service Authentication [for 0.2.1.x]
+   142  Combine Introduction and Rendezvous Points [for 0.2.2.x]
+   143  Improvements of Distributed Storage for Tor Hidden Service Descriptors [for 0.2.1.x]
+   145  Separate "suitable as a guard" from "suitable as a new guard" [for 0.2.1.x]
+   146  Add new flag to reflect long-term stability [for 0.2.1.x]
+   149  Using data from NETINFO cells [for 0.2.1.x]
+   155  Four Improvements of Hidden Service Performance [for 0.2.1.x]
+   156  Tracking blocked ports on the client side [for 0.2.?]
+   157  Make certificate downloads specific [for 0.2.1.x]
  ACCEPTED:
-   110  Avoiding infinite length circuits
-   117  IPv6 exits
-   118  Advertising multiple ORPorts at once
-   134  More robust consensus voting with diverse authority sets
-   140  Provide diffs between consensuses
-   147  Eliminate the need for v2 directories in generating v3 directories
-   148  Stream end reasons from the client side should be uniform
+   110  Avoiding infinite length circuits [for 0.2.1.x] [in 0.2.1.3-alpha]
+   117  IPv6 exits [for 0.2.1.x]
+   118  Advertising multiple ORPorts at once [for 0.2.1.x]
+   134  More robust consensus voting with diverse authority sets [for 0.2.1.x]
+   140  Provide diffs between consensuses [for 0.2.1.x]
+   147  Eliminate the need for v2 directories in generating v3 directories [for 0.2.1.x]
+   148  Stream end reasons from the client side should be uniform [for 0.2.1.x]
  META:
    000  Index of Tor Proposals
    001  The Tor Proposal Process
    098  Proposals that should be written
    099  Miscellaneous proposals
  FINISHED:
-   111  Prioritizing local traffic over relayed traffic
-   128  Families of private bridges
+   111  Prioritizing local traffic over relayed traffic [in 0.2.0.x]
+   128  Families of private bridges [in 0.2.0.x]
  CLOSED:
-   101  Voting on the Tor Directory System
-   102  Dropping "opt" from the directory format
-   103  Splitting identity key from regularly used signing key
-   104  Long and Short Router Descriptors
-   105  Version negotiation for the Tor protocol
-   106  Checking fewer things during TLS handshakes
-   107  Uptime Sanity Checking
-   108  Base "Stable" Flag on Mean Time Between Failures
-   109  No more than one server per IP address
-   114  Distributed Storage for Tor Hidden Service Descriptors
-   119  New PROTOCOLINFO command for controllers
-   122  Network status entries need a new Unnamed flag
-   123  Naming authorities automatically create bindings
-   125  Behavior for bridge users, bridge relays, and bridge authorities
-   126  Getting GeoIP data and publishing usage summaries
-   129  Block Insecure Protocols by Default
-   130  Version 2 Tor connection protocol
-   135  Simplify Configuration of Private Tor Networks
-   136  Mass authority migration with legacy keys
-   137  Keep controllers informed as Tor bootstraps
-   138  Remove routers that are not Running from consensus documents
-   139  Download consensus documents only when it will be trusted
-   150  Exclude Exit Nodes from a circuit
-   152  Optionally allow exit from single-hop circuits 
+   101  Voting on the Tor Directory System [in 0.2.0.x]
+   102  Dropping "opt" from the directory format [in 0.2.0.x]
+   103  Splitting identity key from regularly used signing key [in 0.2.0.x]
+   104  Long and Short Router Descriptors [in 0.2.0.x]
+   105  Version negotiation for the Tor protocol [in 0.2.0.x]
+   106  Checking fewer things during TLS handshakes [in 0.2.0.x]
+   107  Uptime Sanity Checking [in 0.2.0.x]
+   108  Base "Stable" Flag on Mean Time Between Failures [in 0.2.0.x]
+   109  No more than one server per IP address [in 0.2.0.x]
+   114  Distributed Storage for Tor Hidden Service Descriptors [in 0.2.0.x]
+   119  New PROTOCOLINFO command for controllers [in 0.2.0.x]
+   122  Network status entries need a new Unnamed flag [in 0.2.0.x]
+   123  Naming authorities automatically create bindings [in 0.2.0.x]
+   125  Behavior for bridge users, bridge relays, and bridge authorities [in 0.2.0.x]
+   126  Getting GeoIP data and publishing usage summaries [in 0.2.0.x]
+   129  Block Insecure Protocols by Default [in 0.2.0.x]
+   130  Version 2 Tor connection protocol [in 0.2.0.x]
+   135  Simplify Configuration of Private Tor Networks [for 0.2.1.x] [in 0.2.1.2-alpha]
+   136  Mass authority migration with legacy keys [in 0.2.0.x]
+   137  Keep controllers informed as Tor bootstraps [in 0.2.1.x]
+   138  Remove routers that are not Running from consensus documents [in 0.2.1.2-alpha]
+   139  Download consensus documents only when it will be trusted [in 0.2.1.x]
+   150  Exclude Exit Nodes from a circuit [in 0.2.1.3-alpha]
+   152  Optionally allow exit from single-hop circuits [in 0.2.1.6-alpha]
  SUPERSEDED:
    112  Bring Back Pathlen Coin Weight
    113  Simplifying directory authority administration
diff --git a/doc/spec/proposals/144-enforce-distinct-providers.txt b/doc/spec/proposals/144-enforce-distinct-providers.txt
index 8d1cb70cc6..aa460482f1 100644
--- a/doc/spec/proposals/144-enforce-distinct-providers.txt
+++ b/doc/spec/proposals/144-enforce-distinct-providers.txt
@@ -1,6 +1,6 @@
 Filename: 144-enforce-distinct-providers.txt
 Title: Increase the diversity of circuits by detecting nodes belonging the
-same provider
+   same provider
 Author: Mfr
 Created: 2008-06-15
 Status: Draft
diff --git a/doc/spec/proposals/reindex.pl b/doc/spec/proposals/reindex.pl
deleted file mode 100755
index 2727351628..0000000000
--- a/doc/spec/proposals/reindex.pl
+++ /dev/null
@@ -1,98 +0,0 @@
-#!/usr/bin/perl -w
-# Copyright 2007 Nick Mathewson.  See LICENSE for licensing information.
-
-use strict;
-
-my $propdir = ".";
-local *DIR;
-local *F;
-
-opendir(DIR, $propdir) || die "Can't open $propdir";
-my @files = sort grep { /^\d\d\d-.*[^\~]$/ } readdir(DIR);
-closedir DIR;
-
-my %title = ();
-my %status = ();
-
-my @KNOWN_STATUSES = qw{
-    DRAFT OPEN NEEDS-REVISION NEEDS-RESEARCH ACCEPTED META FINISHED CLOSED
-    SUPERSEDED DEAD};
-
-for my $f (@files) {
-    my $num = substr($f, 0, 3);
-    my $status = undef;
-    my $title = undef;
-    my $implemented_in = undef;
-    my $target = undef;
-    my $alleged_fname = undef;
-    if ($f !~ /\.txt/) { print "$f doesn't end with .txt\n"; }
-    open(F, "$f");
-    while (<F>) {
-	last if (/^\s*$/);
-	if (/^Status: (.*)/) {
-	    $status = uc $1;
-	    chomp $status;
-	}
-	if (/^Filename: (.*)/) {
-	    $alleged_fname = $1;
-	    chomp $alleged_fname;
-	}
-	if (/^Title: (.*)/) {
-	    $title = $1;
-	    $title =~ s/\.$//;
-	    chomp $title;
-	}
-	if (/^Implemented-In: (.*)/) {
-	    $implemented_in = $1;
-	    chomp $implemented_in;
-	}
-	if (/^Target: (.*)/) {
-	    $target = $1;
-	    chomp $target;
-	}
-    }
-    close F;
-    die "Proposal $num has no status line" if (!defined $status);
-    die "I've never heard of status $status in proposal $num"
-	unless (grep(/$status/, @KNOWN_STATUSES) == 1);
-    die "Proposal $num has no title line" if (!defined $title);
-    die "Proposal $num has no Filename line" unless (defined $alleged_fname);
-    die "Proposal $num says its fname is $alleged_fname, but it's really $f"
-	if ($alleged_fname ne $f);
-    print "No Target for proposal $num\n" if (($status eq 'OPEN' or
-					       $status eq 'ACCEPTED')
-					      and !defined $target);
-    print "No Implemented-In for proposal $num\n"
-	if (($status eq 'CLOSED' or $status eq 'FINISHED')
-	    and !defined $implemented_in);
-
-    $title{$num} = $title;
-    $status{$num} = $status;
-}
-
-local *OUT;
-open(OUT, ">000-index.txt.tmp");
-
-open(F, "000-index.txt") or die "Can't open index file.";
-while (<F>) {
-    print OUT;
-    last if (/^={3,}/);
-}
-close(F);
-
-print OUT "Proposals by number:\n\n";
-
-for my $num (sort keys %title) {
-    print OUT "$num  $title{$num} [$status{$num}]\n";
-}
-
-print OUT "\n\nProposals by status:\n\n";
-for my $status (@KNOWN_STATUSES) {
-    print OUT " $status:\n";
-    for my $num (sort keys %status) {
-	next unless ($status{$num} eq $status);
-	print OUT "   $num  $title{$num}\n";
-    }
-}
-
-rename('000-index.txt.tmp', '000-index.txt');
diff --git a/doc/spec/proposals/reindex.py b/doc/spec/proposals/reindex.py
new file mode 100755
index 0000000000..2b4c02516b
--- /dev/null
+++ b/doc/spec/proposals/reindex.py
@@ -0,0 +1,117 @@
+#!/usr/bin/python
+
+import re, os
+class Error(Exception): pass
+
+STATUSES = """DRAFT NEEDS-REVISION NEEDS-RESEARCH OPEN ACCEPTED META FINISHED
+   CLOSED SUPERSEDED DEAD""".split()
+REQUIRED_FIELDS = [ "Filename", "Status", "Title" ]
+CONDITIONAL_FIELDS = { "OPEN" : [ "Target" ],
+                       "ACCEPTED" : [ "Target "],
+                       "CLOSED" : [ "Implemented-In" ],
+                       "FINISHED" : [ "Implemented-In" ] }
+FNAME_RE = re.compile(r'^(\d\d\d)-.*[^\~]$')
+DIR = "."
+OUTFILE = "000-index.txt"
+TMPFILE = OUTFILE+".tmp"
+
+def indexed(seq):
+    n = 0
+    for i in seq:
+        yield n, i
+        n += 1
+
+def readProposal(fn):
+    fields = { }
+    f = open(fn, 'r')
+    lastField = None
+    try:
+        for lineno, line in indexed(f):
+            line = line.rstrip()
+            if not line:
+                return fields
+            if line[0].isspace():
+                fields[lastField] += " %s"%(line.strip())
+            else:
+                parts = line.split(":", 1)
+                if len(parts) != 2:
+                    raise Error("%s:%s:  Neither field nor continuation"%
+                                (fn,lineno))
+                else:
+                    fields[parts[0]] = parts[1].strip()
+                    lastField = parts[0]
+
+        return fields
+    finally:
+        f.close()
+
+def checkProposal(fn, fields):
+    status = fields.get("Status")
+    need_fields = REQUIRED_FIELDS + CONDITIONAL_FIELDS.get(status, [])
+    for f in need_fields:
+        if not fields.has_key(f):
+            raise Error("%s has no %s field"%(fn, f))
+    if fn != fields['Filename']:
+        print `fn`, `fields['Filename']`
+        raise Error("Mismatched Filename field in %s"%fn)
+    if fields['Title'][-1] == '.':
+        fields['Title'] = fields['Title'][:-1]
+
+    status = fields['Status'] = status.upper()
+    if status not in STATUSES:
+        raise Error("I've never heard of status %s in %s"%(status,fn))
+    if status in [ "SUPERSEDED", "DEAD" ]:
+        for f in [ 'Implemented-In', 'Target' ]:
+            if fields.has_key(f): del fields[f]
+
+def readProposals():
+    res = []
+    for fn in os.listdir(DIR):
+        m = FNAME_RE.match(fn)
+        if not m: continue
+        if not fn.endswith(".txt"):
+            raise Error("%s doesn't end with .txt"%fn)
+        num = m.group(1)
+        fields = readProposal(fn)
+        checkProposal(fn, fields)
+        fields['num'] = num
+        res.append(fields)
+    return res
+
+def writeIndexFile(proposals):
+    proposals.sort(key=lambda f:f['num'])
+    seenStatuses = set()
+    for p in proposals:
+        seenStatuses.add(p['Status'])
+
+    out = open(TMPFILE, 'w')
+    inf = open(OUTFILE, 'r')
+    for line in inf:
+        out.write(line)
+        if line.startswith("====="): break
+    inf.close()
+
+    out.write("Proposals by number:\n\n")
+    for prop in proposals:
+        out.write("%(num)s  %(Title)s [%(Status)s]\n"%prop)
+    out.write("\n\nProposals by status:\n\n")
+    for s in STATUSES:
+        if s not in seenStatuses: continue
+        out.write(" %s:\n"%s)
+        for prop in proposals:
+            if s == prop['Status']:
+                out.write("   %(num)s  %(Title)s"%prop)
+                if prop.has_key('Target'):
+                    out.write(" [for %(Target)s]"%prop)
+                if prop.has_key('Implemented-In'):
+                    out.write(" [in %(Implemented-In)s]"%prop)
+                out.write("\n")
+    out.close()
+    os.rename(TMPFILE, OUTFILE)
+
+try:
+    os.unlink(TMPFILE)
+except OSError:
+    pass
+
+writeIndexFile(readProposals())