diff options
Diffstat (limited to 'scripts')
| -rw-r--r-- | scripts/README | 5 | ||||
| -rw-r--r-- | scripts/coccinelle/calloc.cocci | 23 | ||||
| -rw-r--r-- | scripts/coccinelle/malloc_cast.cocci | 38 | ||||
| -rw-r--r-- | scripts/coccinelle/uncalloc.cocci | 13 | ||||
| -rwxr-xr-x | scripts/codegen/gen_server_ciphers.py | 2 | ||||
| -rw-r--r-- | scripts/codegen/get_mozilla_ciphers.py | 2 | ||||
| -rw-r--r-- | scripts/codegen/makedesc.py | 218 | ||||
| -rwxr-xr-x | scripts/codegen/run_trunnel.sh | 11 | ||||
| -rwxr-xr-x | scripts/maint/checkSpace.pl | 33 | ||||
| -rwxr-xr-x | scripts/maint/format_changelog.py | 298 | ||||
| -rwxr-xr-x | scripts/maint/lintChanges.py | 55 | ||||
| -rwxr-xr-x | scripts/maint/redox.py | 2 | ||||
| -rwxr-xr-x | scripts/maint/sortChanges.py | 8 | ||||
| -rwxr-xr-x | scripts/maint/updateCopyright.pl | 7 | ||||
| -rwxr-xr-x | scripts/test/cov-diff | 4 |
15 files changed, 685 insertions, 34 deletions
diff --git a/scripts/README b/scripts/README index 70c763923c..02faabe06b 100644 --- a/scripts/README +++ b/scripts/README @@ -56,3 +56,8 @@ for servers to choose from. codegen/get_mozilla_ciphers.py -- Generate a list of TLS ciphersuites for clients to use in order to look like Firefox. +Code transformation scripts +--------------------------- + +coccinelle/calloc.cocci -- Transform code to replace variants of +malloc(a*b) with calloc(a,b) diff --git a/scripts/coccinelle/calloc.cocci b/scripts/coccinelle/calloc.cocci new file mode 100644 index 0000000000..fbda88e538 --- /dev/null +++ b/scripts/coccinelle/calloc.cocci @@ -0,0 +1,23 @@ +// Use calloc or realloc as appropriate instead of multiply-and-alloc + +@malloc_to_calloc@ +identifier f =~ "(tor_malloc|tor_malloc_zero)"; +expression a; +constant b; +@@ +- f(a * b) ++ tor_calloc(a, b) + +@calloc_arg_order@ +expression a; +type t; +@@ +- tor_calloc(sizeof(t), a) ++ tor_calloc(a, sizeof(t)) + +@realloc_to_reallocarray@ +expression a, b; +expression p; +@@ +- tor_realloc(p, a * b) ++ tor_reallocarray(p, a, b) diff --git a/scripts/coccinelle/malloc_cast.cocci b/scripts/coccinelle/malloc_cast.cocci new file mode 100644 index 0000000000..20321d4fd0 --- /dev/null +++ b/scripts/coccinelle/malloc_cast.cocci @@ -0,0 +1,38 @@ +@cast_malloc@ +expression e; +type T; +@@ +- (T *)tor_malloc(e) ++ tor_malloc(e) + +@cast_malloc_zero@ +expression e; +type T; +identifier func; +@@ +- (T *)tor_malloc_zero(e) ++ tor_malloc_zero(e) + +@cast_calloc@ +expression a, b; +type T; +identifier func; +@@ +- (T *)tor_calloc(a, b) ++ tor_calloc(a, b) + +@cast_realloc@ +expression e; +expression p; +type T; +@@ +- (T *)tor_realloc(p, e) ++ tor_realloc(p, e) + +@cast_reallocarray@ +expression a,b; +expression p; +type T; +@@ +- (T *)tor_reallocarray(p, a, b) ++ tor_reallocarray(p, a, b) diff --git a/scripts/coccinelle/uncalloc.cocci b/scripts/coccinelle/uncalloc.cocci new file mode 100644 index 0000000000..bf3f74165a --- /dev/null +++ b/scripts/coccinelle/uncalloc.cocci @@ -0,0 +1,13 @@ + +@@ +expression a; +@@ +- tor_calloc(1, a) ++ tor_malloc_zero(a) + +@@ +expression a; +@@ +- tor_calloc(a, 1) ++ tor_malloc_zero(a) + diff --git a/scripts/codegen/gen_server_ciphers.py b/scripts/codegen/gen_server_ciphers.py index 97ed9d0469..0dca8a6734 100755 --- a/scripts/codegen/gen_server_ciphers.py +++ b/scripts/codegen/gen_server_ciphers.py @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright 2014, The Tor Project, Inc +# Copyright 2014-2015, The Tor Project, Inc # See LICENSE for licensing information # This script parses openssl headers to find ciphersuite names, determines diff --git a/scripts/codegen/get_mozilla_ciphers.py b/scripts/codegen/get_mozilla_ciphers.py index 0636eb3658..e0a662bea0 100644 --- a/scripts/codegen/get_mozilla_ciphers.py +++ b/scripts/codegen/get_mozilla_ciphers.py @@ -1,6 +1,6 @@ #!/usr/bin/python # coding=utf-8 -# Copyright 2011, The Tor Project, Inc +# Copyright 2011-2015, The Tor Project, Inc # original version by Arturo Filastò # See LICENSE for licensing information diff --git a/scripts/codegen/makedesc.py b/scripts/codegen/makedesc.py new file mode 100644 index 0000000000..833951945b --- /dev/null +++ b/scripts/codegen/makedesc.py @@ -0,0 +1,218 @@ +#!/usr/bin/python +# Copyright 2014-2015, The Tor Project, Inc. +# See LICENSE for license information + +# This is a kludgey python script that uses ctypes and openssl to sign +# router descriptors and extrainfo documents and put all the keys in +# the right places. There are examples at the end of the file. + +# I've used this to make inputs for unit tests. I wouldn't suggest +# using it for anything else. + +import base64 +import binascii +import ctypes +import ctypes.util +import hashlib + +crypt = ctypes.CDLL(ctypes.util.find_library('crypto')) +BIO_s_mem = crypt.BIO_s_mem +BIO_s_mem.argtypes = [] +BIO_s_mem.restype = ctypes.c_void_p + +BIO_new = crypt.BIO_new +BIO_new.argtypes = [ctypes.c_void_p] +BIO_new.restype = ctypes.c_void_p + +RSA_generate_key = crypt.RSA_generate_key +RSA_generate_key.argtypes = [ctypes.c_int, ctypes.c_ulong, ctypes.c_void_p, ctypes.c_void_p] +RSA_generate_key.restype = ctypes.c_void_p + +RSA_private_encrypt = crypt.RSA_private_encrypt +RSA_private_encrypt.argtypes = [ + ctypes.c_int, ctypes.c_char_p, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int ] +RSA_private_encrypt.restype = ctypes.c_int + +i2d_RSAPublicKey = crypt.i2d_RSAPublicKey +i2d_RSAPublicKey.argtypes = [ + ctypes.c_void_p, ctypes.POINTER(ctypes.c_char_p) +] +i2d_RSAPublicKey.restype = ctypes.c_int + +def b64(x): + x = base64.b64encode(x) + res = [] + for i in xrange(0, len(x), 64): + res.append(x[i:i+64]+"\n") + return "".join(res) + +def bio_extract(bio): + buf = ctypes.c_char_p() + length = crypt.BIO_ctrl(bio, 3, 0, ctypes.byref(buf)) + return ctypes.string_at(buf, length) + +def make_key(e=65537): + rsa = crypt.RSA_generate_key(1024, e, None, None) + bio = BIO_new(BIO_s_mem()) + crypt.PEM_write_bio_RSAPublicKey(bio, rsa) + pem = bio_extract(bio).rstrip() + crypt.BIO_free(bio) + + buf = ctypes.create_string_buffer(1024) + pBuf = ctypes.c_char_p(ctypes.addressof(buf)) + n = crypt.i2d_RSAPublicKey(rsa, ctypes.byref(pBuf)) + s = buf.raw[:n] + digest = hashlib.sha1(s).digest() + + return (rsa,pem,digest) + +def signdesc(body, args_out=None): + rsa, ident_pem, id_digest = make_key() + _, onion_pem, _ = make_key() + + hexdigest = binascii.b2a_hex(id_digest).upper() + fingerprint = " ".join(hexdigest[i:i+4] for i in range(0,len(hexdigest),4)) + + MAGIC = "<<<<<<MAGIC>>>>>>" + args = { + "RSA-IDENTITY" : ident_pem, + "ONION-KEY" : onion_pem, + "FINGERPRINT" : fingerprint, + "FINGERPRINT-NOSPACE" : hexdigest, + "RSA-SIGNATURE" : MAGIC + } + if args_out: + args_out.update(args) + body = body.format(**args) + + idx = body.rindex("\nrouter-signature") + end_of_sig = body.index("\n", idx+1) + + signed_part = body[:end_of_sig+1] + + digest = hashlib.sha1(signed_part).digest() + assert len(digest) == 20 + + buf = ctypes.create_string_buffer(1024) + n = RSA_private_encrypt(20, digest, buf, rsa, 1) + sig = buf.raw[:n] + + sig = """-----BEGIN SIGNATURE----- +%s +-----END SIGNATURE-----""" % b64(sig).rstrip() + body = body.replace(MAGIC, sig) + + return body.rstrip() + +def emit_ri(name, body, args_out=None): + print "const char %s[] ="%name + body = "\n".join(line.rstrip() for line in body.split("\n"))+"\n" + b = signdesc(body, args_out) + for line in b.split("\n"): + print ' "%s\\n"'%line + print " ;" + +def emit_ei(name, body): + args = { 'NAME' : name } + emit_ri(name, body, args) + args['key'] = "\n".join( + ' "%s\\n"'%line for line in args['RSA-IDENTITY'].split("\n")) + print """ +const char {NAME}_fp[] = "{FINGERPRINT-NOSPACE}"; +const char {NAME}_key[] = +{key};""".format(**args) + +if 0: + emit_ri("minimal", + """\ +router fred 127.0.0.1 9001 0 9002 +signing-key +{RSA-IDENTITY} +onion-key +{ONION-KEY} +published 2014-10-05 12:00:00 +bandwidth 1000 1000 1000 +reject *:* +router-signature +{RSA-SIGNATURE} +""") + +if 0: + emit_ri("maximal", + """\ +router fred 127.0.0.1 9001 0 9002 +signing-key +{RSA-IDENTITY} +onion-key +{ONION-KEY} +published 2014-10-05 12:00:00 +bandwidth 1000 1000 1000 +reject 127.0.0.1:* +accept *:80 +reject *:* +ipv6-policy accept 80,100,101 +ntor-onion-key s7rSohmz9SXn8WWh1EefTHIsWePthsEntQi0WL+ScVw +uptime 1000 +hibernating 0 +unrecognized-keywords are just dandy in this format +platform Tor 0.2.4.23 on a Banana PC Jr 6000 Series +contact O.W.Jones +fingerprint {FINGERPRINT} +read-history 900 1,2,3,4 +write-history 900 1,2,3,4 +extra-info-digest AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +hidden-service-dir +allow-single-hop-exits +family $AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA $BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB +caches-extra-info +or-address [::1:2:3:4]:9999 +or-address 127.0.0.99:10000 +opt fred is a fine router +router-signature +{RSA-SIGNATURE} +""") + +if 0: + emit_ei("maximal", +"""\ +extra-info bob {FINGERPRINT-NOSPACE} +published 2014-10-05 20:07:00 +opt foobarbaz +read-history 900 1,2,3 +write-history 900 1,2,3 +dirreq-v2-ips 1 +dirreq-v3-ips 100 +dirreq-v3-reqs blahblah +dirreq-v2-share blahblah +dirreq-v3-share blahblah +dirreq-v2-resp djfkdj +dirreq-v3-resp djfkdj +dirreq-v2-direct-dl djfkdj +dirreq-v3-direct-dl djfkdj +dirreq-v2-tunneled-dl djfkdj +dirreq-v3-tunneled-dl djfkdj +dirreq-stats-end foobar +entry-ips jfsdfds +entry-stats-end ksdflkjfdkf +cell-stats-end FOO +cell-processed-cells FOO +cell-queued-cells FOO +cell-time-in-queue FOO +cell-circuits-per-decile FOO +exit-stats-end FOO +exit-kibibytes-written FOO +exit-kibibytes-read FOO +exit-streams-opened FOO +router-signature +{RSA-SIGNATURE} +""") + +if 0: + emit_ei("minimal", +"""\ +extra-info bob {FINGERPRINT-NOSPACE} +published 2014-10-05 20:07:00 +router-signature +{RSA-SIGNATURE} +""") + diff --git a/scripts/codegen/run_trunnel.sh b/scripts/codegen/run_trunnel.sh new file mode 100755 index 0000000000..5f694ce6c9 --- /dev/null +++ b/scripts/codegen/run_trunnel.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +if test "x$TRUNNEL_PATH" != "x"; then + PYTHONPATH="${TRUNNEL_PATH}:${PYTHONPATH}" + export PYTHONPATH +fi + +python -m trunnel --require-version=1.2 ./src/trunnel/*.trunnel + +python -m trunnel --require-version=1.2 --write-c-files --target-dir=./src/ext/trunnel/ + diff --git a/scripts/maint/checkSpace.pl b/scripts/maint/checkSpace.pl index 682dbced00..c785d89567 100755 --- a/scripts/maint/checkSpace.pl +++ b/scripts/maint/checkSpace.pl @@ -13,30 +13,45 @@ for $fn (@ARGV) { $incomment = 0; while (<F>) { ## Warn about windows-style newlines. + # (We insist on lines that end with a single LF character, not + # CR LF.) if (/\r/) { print " CR:$fn:$.\n"; } ## Warn about tabs. + # (We only use spaces) if (/\t/) { print " TAB:$fn:$.\n"; } - ## Warn about markers that don't have a space in front of them + ## Warn about labels that don't have a space in front of them + # (We indent every label at least one space) if (/^[a-zA-Z_][a-zA-Z_0-9]*:/) { print "nosplabel:$fn:$.\n"; } ## Warn about trailing whitespace. + # (We don't allow whitespace at the end of the line; make your + # editor highlight it for you so you can stop adding it in.) if (/ +$/) { print "Space\@EOL:$fn:$.\n"; } ## Warn about control keywords without following space. + # (We put a space after every 'if', 'while', 'for', 'switch', etc) if ($C && /\s(?:if|while|for|switch)\(/) { print " KW(:$fn:$.\n"; } ## Warn about #else #if instead of #elif. + # (We only allow #elif) if (($lastline =~ /^\# *else/) and ($_ =~ /^\# *if/)) { print " #else#if:$fn:$.\n"; } ## Warn about some K&R violations + # (We use K&R-style C, where open braces go on the same line as + # the statement that introduces them. In other words: + # if (a) { + # stuff; + # } else { + # other stuff; + # } if (/^\s+\{/ and $lastline =~ /^\s*(if|while|for|else if)/ and $lastline !~ /\{$/) { print "non-K&R {:$fn:$.\n"; @@ -46,10 +61,13 @@ for $fn (@ARGV) { } $lastline = $_; ## Warn about unnecessary empty lines. + # (Don't put an empty line before a line that contains nothing + # but a closing brace.) if ($lastnil && /^\s*}\n/) { print " UnnecNL:$fn:$.\n"; } ## Warn about multiple empty lines. + # (At most one blank line in a row.) if ($lastnil && /^$/) { print " DoubleNL:$fn:$.\n"; } elsif (/^$/) { @@ -59,6 +77,7 @@ for $fn (@ARGV) { } ## Terminals are still 80 columns wide in my world. I refuse to ## accept double-line lines. + # (Don't make lines wider than 80 characters, including newline.) if (/^.{80}/) { print " Wide:$fn:$.\n"; } @@ -83,11 +102,13 @@ for $fn (@ARGV) { s!"(?:[^\"]+|\\.)*"!"X"!g; next if /^\#/; ## Warn about C++-style comments. + # (Use C style comments only.) if (m!//!) { # print " //:$fn:$.\n"; s!//.*!!; } ## Warn about unquoted braces preceded by non-space. + # (No character except a space should come before a {) if (/([^\s'])\{/) { print " $1\{:$fn:$.\n"; } @@ -101,15 +122,21 @@ for $fn (@ARGV) { # print " {X:$fn:$.\n"; #} ## Warn about function calls with space before parens. + # (Don't put a space between the name of a function and its + # arguments.) if (/(\w+)\s\(([A-Z]*)/) { if ($1 ne "if" and $1 ne "while" and $1 ne "for" and $1 ne "switch" and $1 ne "return" and $1 ne "int" and $1 ne "elsif" and $1 ne "WINAPI" and $2 ne "WINAPI" and - $1 ne "void" and $1 ne "__attribute__" and $1 ne "op") { + $1 ne "void" and $1 ne "__attribute__" and $1 ne "op" and + $1 ne "size_t" and $1 ne "double") { print " fn ():$fn:$.\n"; } } ## Warn about functions not declared at start of line. + # (When you're declaring functions, put "static" and "const" + # and the return type on one line, and the function name at + # the start of a new line.) if ($in_func_head || ($fn !~ /\.h$/ && /^[a-zA-Z0-9_]/ && ! /^(?:const |static )*(?:typedef|struct|union)[^\(]*$/ && @@ -130,6 +157,8 @@ for $fn (@ARGV) { } } } + ## Warn if the file doesn't end with a blank line. + # (End each file with a single blank line.) if (! $lastnil) { print " EOL\@EOF:$fn:$.\n"; } diff --git a/scripts/maint/format_changelog.py b/scripts/maint/format_changelog.py index f67e89b602..d1b4a3dff3 100755 --- a/scripts/maint/format_changelog.py +++ b/scripts/maint/format_changelog.py @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (c) 2014, The Tor Project, Inc. +# Copyright (c) 2014-2015, The Tor Project, Inc. # See LICENSE for licensing information # # This script reformats a section of the changelog to wrap everything to @@ -12,6 +12,7 @@ import os import re import sys +import optparse # ============================== # Oh, look! It's a cruddy approximation to Knuth's elegant text wrapping @@ -33,6 +34,9 @@ import sys NO_HYPHENATE=set(""" pf-divert +tor-resolve +tor-gencert +tor-fw-helper """.split()) LASTLINE_UNDERFLOW_EXPONENT = 1 @@ -115,7 +119,10 @@ def wrap_graf(words, prefix_len1=0, prefix_len2=0, width=72): return lines -def hyphenateable(word): +def hyphenatable(word): + if "--" in word: + return False + if re.match(r'^[^\d\-]\D*-', word): stripped = re.sub(r'^\W+','',word) stripped = re.sub(r'\W+$','',word) @@ -128,7 +135,7 @@ def split_paragraph(s): r = [] for word in s.split(): - if hyphenateable(word): + if hyphenatable(word): while "-" in word: a,word = word.split("-",1) r.append(a+"\xff") @@ -156,10 +163,13 @@ TP_SECHEAD = 3 TP_ITEMFIRST = 4 TP_ITEMBODY = 5 TP_END = 6 +TP_PREHEAD = 7 def head_parser(line): - if re.match(r'^[A-Z]', line): + if re.match(r'^Changes in', line): return TP_MAINHEAD + elif re.match(r'^[A-Za-z]', line): + return TP_PREHEAD elif re.match(r'^ o ', line): return TP_SECHEAD elif re.match(r'^\s*$', line): @@ -178,17 +188,67 @@ def body_parser(line): return TP_BLANK elif re.match(r'^Changes in', line): return TP_END + elif re.match(r'^\s+\S', line): + return TP_HEADTEXT else: print "Weird line %r"%line +def clean_head(head): + return head + +def head_score(s): + m = re.match(r'^ +o (.*)', s) + if not m: + print >>sys.stderr, "Can't score %r"%s + return 99999 + lw = m.group(1).lower() + if lw.startswith("security") and "feature" not in lw: + score = -300 + elif lw.startswith("deprecated version"): + score = -200 + elif (('new' in lw and 'requirement' in lw) or + ('new' in lw and 'dependenc' in lw) or + ('build' in lw and 'requirement' in lw) or + ('removed' in lw and 'platform' in lw)): + score = -100 + elif lw.startswith("major feature"): + score = 00 + elif lw.startswith("major bug"): + score = 50 + elif lw.startswith("major"): + score = 70 + elif lw.startswith("minor feature"): + score = 200 + elif lw.startswith("minor bug"): + score = 250 + elif lw.startswith("minor"): + score = 270 + else: + score = 1000 + + if 'secur' in lw: + score -= 2 + + if "(other)" in lw: + score += 2 + + if '(' not in lw: + score -= 1 + + return score + class ChangeLog(object): - def __init__(self): + def __init__(self, wrapText=True, blogOrder=True, drupalBreak=False): + self.prehead = [] self.mainhead = None self.headtext = [] self.curgraf = None self.sections = [] self.cursection = None self.lineno = 0 + self.wrapText = wrapText + self.blogOrder = blogOrder + self.drupalBreak = drupalBreak def addLine(self, tp, line): self.lineno += 1 @@ -197,6 +257,9 @@ class ChangeLog(object): assert not self.mainhead self.mainhead = line + elif tp == TP_PREHEAD: + self.prehead.append(line) + elif tp == TP_HEADTEXT: if self.curgraf is None: self.curgraf = [] @@ -240,6 +303,11 @@ class ChangeLog(object): self.lint_item(item_line, grafs, head_type) def dumpGraf(self,par,indent1,indent2=-1): + if not self.wrapText: + for line in par: + print line + return + if indent2 == -1: indent2 = indent1 text = " ".join(re.sub(r'\s+', ' ', line.strip()) for line in par) @@ -249,38 +317,210 @@ class ChangeLog(object): initial_indent=" "*indent1, subsequent_indent=" "*indent2)) + def dumpPreheader(self, graf): + self.dumpGraf(graf, 0) + print + + def dumpMainhead(self, head): + print head + + def dumpHeadGraf(self, graf): + self.dumpGraf(graf, 2) + print + + def dumpSectionHeader(self, header): + print header + + def dumpStartOfSections(self): + pass + + def dumpEndOfSections(self): + pass + + def dumpEndOfSection(self): + print + + def dumpEndOfChangelog(self): + print + + def dumpDrupalBreak(self): + pass + + def dumpItem(self, grafs): + self.dumpGraf(grafs[0],4,6) + for par in grafs[1:]: + print + self.dumpGraf(par,6,6) + + def collateAndSortSections(self): + heads = [] + sectionsByHead = { } + for _, head, items in self.sections: + head = clean_head(head) + try: + s = sectionsByHead[head] + except KeyError: + s = sectionsByHead[head] = [] + heads.append( (head_score(head), head.lower(), head, s) ) + + s.extend(items) + + heads.sort() + self.sections = [ (0, head, items) for _1,_2,head,items in heads ] + def dump(self): - print self.mainhead + if self.prehead: + self.dumpPreheader(self.prehead) + + if not self.blogOrder: + self.dumpMainhead(self.mainhead) + for par in self.headtext: - self.dumpGraf(par, 2) - print + self.dumpHeadGraf(par) + + if self.blogOrder: + self.dumpMainhead(self.mainhead) + + drupalBreakAfter = None + if self.drupalBreak and len(self.sections) > 4: + drupalBreakAfter = self.sections[1][2] + + self.dumpStartOfSections() for _,head,items in self.sections: if not head.endswith(':'): print >>sys.stderr, "adding : to %r"%head head = head + ":" - print head + self.dumpSectionHeader(head) for _,grafs in items: - self.dumpGraf(grafs[0],4,6) - for par in grafs[1:]: - print - self.dumpGraf(par,6,6) - print - print + self.dumpItem(grafs) + self.dumpEndOfSection() + if items is drupalBreakAfter: + self.dumpDrupalBreak() + self.dumpEndOfSections() + self.dumpEndOfChangelog() + +class HTMLChangeLog(ChangeLog): + def __init__(self, *args, **kwargs): + ChangeLog.__init__(self, *args, **kwargs) + + def htmlText(self, graf): + for line in graf: + line = line.rstrip().replace("&","&") + line = line.rstrip().replace("<","<").replace(">",">") + sys.stdout.write(line.strip()) + sys.stdout.write(" ") + + def htmlPar(self, graf): + sys.stdout.write("<p>") + self.htmlText(graf) + sys.stdout.write("</p>\n") + + def dumpPreheader(self, graf): + self.htmlPar(graf) + + def dumpMainhead(self, head): + sys.stdout.write("<h2>%s</h2>"%head) + + def dumpHeadGraf(self, graf): + self.htmlPar(graf) + + def dumpSectionHeader(self, header): + header = header.replace(" o ", "", 1).lstrip() + sys.stdout.write(" <li>%s\n"%header) + sys.stdout.write(" <ul>\n") + + def dumpEndOfSection(self): + sys.stdout.write(" </ul>\n\n") + + def dumpEndOfChangelog(self): + pass -CL = ChangeLog() -parser = head_parser + def dumpStartOfSections(self): + print "<ul>\n" + + def dumpEndOfSections(self): + print "</ul>\n" -if len(sys.argv) == 1: + def dumpDrupalBreak(self): + print "\n</ul>\n" + print "<p> </p>" + print "\n<!--break-->\n\n" + print "<ul>" + + def dumpItem(self, grafs): + grafs[0][0] = grafs[0][0].replace(" - ", "", 1).lstrip() + sys.stdout.write(" <li>") + if len(grafs) > 1: + for par in grafs: + self.htmlPar(par) + else: + self.htmlText(grafs[0]) + print + +op = optparse.OptionParser(usage="usage: %prog [options] [filename]") +op.add_option('-W', '--no-wrap', action='store_false', + dest='wrapText', default=True, + help='Do not re-wrap paragraphs') +op.add_option('-S', '--no-sort', action='store_false', + dest='sort', default=True, + help='Do not sort or collate sections') +op.add_option('-o', '--output', dest='output', + default='-', metavar='FILE', help="write output to FILE") +op.add_option('-H', '--html', action='store_true', + dest='html', default=False, + help="generate an HTML fragment") +op.add_option('-1', '--first', action='store_true', + dest='firstOnly', default=False, + help="write only the first section") +op.add_option('-b', '--blog-header', action='store_true', + dest='blogOrder', default=False, + help="Write the header in blog order") +op.add_option('-B', '--blog', action='store_true', + dest='blogFormat', default=False, + help="Set all other options as appropriate for a blog post") +op.add_option('--inplace', action='store_true', + dest='inplace', default=False, + help="Alter the ChangeLog in place") +op.add_option('--drupal-break', action='store_true', + dest='drupalBreak', default=False, + help='Insert a drupal-friendly <!--break--> as needed') + +options,args = op.parse_args() + +if options.blogFormat: + options.blogOrder = True + options.html = True + options.sort = False + options.wrapText = False + options.firstOnly = True + options.drupalBreak = True + +if len(args) > 1: + op.error("Too many arguments") +elif len(args) == 0: fname = 'ChangeLog' else: - fname = sys.argv[1] + fname = args[0] -fname_new = fname+".new" +if options.inplace: + assert options.output == '-' + options.output = fname -sys.stdin = open(fname, 'r') +if fname != '-': + sys.stdin = open(fname, 'r') nextline = None +if options.html: + ChangeLogClass = HTMLChangeLog +else: + ChangeLogClass = ChangeLog + +CL = ChangeLogClass(wrapText=options.wrapText, + blogOrder=options.blogOrder, + drupalBreak=options.drupalBreak) +parser = head_parser + for line in sys.stdin: line = line.rstrip() tp = parser(line) @@ -295,14 +535,26 @@ for line in sys.stdin: CL.lint() -sys.stdout = open(fname_new, 'w') +if options.output != '-': + fname_new = options.output+".new" + fname_out = options.output + sys.stdout = open(fname_new, 'w') +else: + fname_new = fname_out = None + +if options.sort: + CL.collateAndSortSections() CL.dump() +if options.firstOnly: + sys.exit(0) + if nextline is not None: print nextline for line in sys.stdin: sys.stdout.write(line) -os.rename(fname_new, fname) +if fname_new is not None: + os.rename(fname_new, fname_out) diff --git a/scripts/maint/lintChanges.py b/scripts/maint/lintChanges.py new file mode 100755 index 0000000000..69963aea28 --- /dev/null +++ b/scripts/maint/lintChanges.py @@ -0,0 +1,55 @@ +#!/usr/bin/python + +import sys +import re + + + +def lintfile(fname): + have_warned = [] + def warn(s): + if not have_warned: + have_warned.append(1) + print fname,":" + print "\t",s + + m = re.search(r'(\d{3,})', fname) + if m: + bugnum = m.group(1) + else: + bugnum = None + + with open(fname) as f: + contents = f.read() + + if bugnum and bugnum not in contents: + warn("bug number %s does not appear"%bugnum) + + lines = contents.split("\n") + isBug = ("bug" in lines[0] or "fix" in lines[0]) + + if not re.match(r'^ +o (.*)', contents): + warn("header not in format expected") + + contents = " ".join(contents.split()) + + if re.search(r'\#\d{2,}', contents): + warn("don't use a # before ticket numbers") + + if isBug and not re.search(r'(\d+)', contents): + warn("bugfix does not mention a number") + elif isBug and not re.search(r'Fixes ([a-z ]*)bug (\d+)', contents): + warn("bugfix does not say 'Fixes bug XXX'") + + if re.search(r'[bB]ug (\d+)', contents): + if not re.search(r'[Bb]ugfix on ', contents): + warn("bugfix does not say 'bugfix on X.Y.Z'") + elif not re.search('[fF]ixes ([a-z ]*)bug (\d+); bugfix on ', contents): + warn("bugfix incant is not semicoloned") + + +if __name__=='__main__': + for fname in sys.argv[1:]: + if fname.endswith("~"): + continue + lintfile(fname) diff --git a/scripts/maint/redox.py b/scripts/maint/redox.py index fa816a7267..5933d49773 100755 --- a/scripts/maint/redox.py +++ b/scripts/maint/redox.py @@ -1,6 +1,6 @@ #!/usr/bin/python # -# Copyright (c) 2008-2013, The Tor Project, Inc. +# Copyright (c) 2008-2015, The Tor Project, Inc. # See LICENSE for licensing information. # # Hi! diff --git a/scripts/maint/sortChanges.py b/scripts/maint/sortChanges.py index 726a723f93..ad28c79d9d 100755 --- a/scripts/maint/sortChanges.py +++ b/scripts/maint/sortChanges.py @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (c) 2014, The Tor Project, Inc. +# Copyright (c) 2014-2015, The Tor Project, Inc. # See LICENSE for licensing information """This script sorts a bunch of changes files listed on its command @@ -18,10 +18,10 @@ def fetch(fn): s = "%s\n" % s.rstrip() return s -def score(s): +def score(s,fname=None): m = re.match(r'^ +o (.*)', s) if not m: - print >>sys.stderr, "Can't score %r"%s + print >>sys.stderr, "Can't score %r from %s"%(s,fname) lw = m.group(1).lower() if lw.startswith("major feature"): score = 0 @@ -41,7 +41,7 @@ def score(s): return (score, lw, s) -changes = [ score(fetch(fn)) for fn in sys.argv[1:] if not fn.endswith('~') ] +changes = [ score(fetch(fn),fn) for fn in sys.argv[1:] if not fn.endswith('~') ] changes.sort() diff --git a/scripts/maint/updateCopyright.pl b/scripts/maint/updateCopyright.pl new file mode 100755 index 0000000000..ec82616a19 --- /dev/null +++ b/scripts/maint/updateCopyright.pl @@ -0,0 +1,7 @@ +#!/usr/bin/perl -i -w -p + +$NEWYEAR=2015; + +s/Copyright(.*) (201[^5]), The Tor Project/Copyright$1 $2-${NEWYEAR}, The Tor Project/; + +s/Copyright(.*)-(20..), The Tor Project/Copyright$1-${NEWYEAR}, The Tor Project/; diff --git a/scripts/test/cov-diff b/scripts/test/cov-diff index 33a54802b6..48dbec9d54 100755 --- a/scripts/test/cov-diff +++ b/scripts/test/cov-diff @@ -9,8 +9,8 @@ DIRB="$2" for A in $DIRA/*; do B=$DIRB/`basename $A` - perl -pe 's/^\s*\d+:/ 1:/; s/^([^:]+:)[\d\s]+:/$1/;' "$A" > "$A.tmp" - perl -pe 's/^\s*\d+:/ 1:/; s/^([^:]+:)[\d\s]+:/$1/;' "$B" > "$B.tmp" + perl -pe 's/^\s*\d+:/ 1:/; s/^([^:]+:)[\d\s]+:/$1/; s/^ *-:(Runs|Programs):.*//;' "$A" > "$A.tmp" + perl -pe 's/^\s*\d+:/ 1:/; s/^([^:]+:)[\d\s]+:/$1/; s/^ *-:(Runs|Programs):.*//;' "$B" > "$B.tmp" diff -u "$A.tmp" "$B.tmp" rm "$A.tmp" "$B.tmp" done |