#!/usr/bin/env python # Copyright (c) 2014-2019, The Tor Project, Inc. # See LICENSE for licensing information # # This script reformats a section of the changelog to wrap everything to # the right width and put blank lines in the right places. Eventually, # it might include a linter. # # To run it, pipe a section of the changelog (starting with "Changes # in Tor 0.x.y.z-alpha" through the script.) # Future imports for Python 2.7, mandatory in 3.0 from __future__ import division from __future__ import print_function from __future__ import unicode_literals import os import re import sys import optparse # ============================== # Oh, look! It's a cruddy approximation to Knuth's elegant text wrapping # algorithm, with totally ad hoc parameters! # # We're trying to minimize: # The total of the cubes of ragged space on underflowed intermediate lines, # PLUS # 100 * the fourth power of overflowed characters # PLUS # .1 * a bit more than the cube of ragged space on the last line. # PLUS # OPENPAREN_PENALTY for each line that starts with ( # # We use an obvious dynamic programming algorithm to sorta approximate this. # It's not coded right or optimally, but it's fast enough for changelogs # # (Code found in an old directory of mine, lightly cleaned. -NM) NO_HYPHENATE=set(""" pf-divert tor-resolve tor-gencert """.split()) LASTLINE_UNDERFLOW_EXPONENT = 1 LASTLINE_UNDERFLOW_PENALTY = 1 UNDERFLOW_EXPONENT = 3 UNDERFLOW_PENALTY = 1 OVERFLOW_EXPONENT = 4 OVERFLOW_PENALTY = 2000 ORPHAN_PENALTY = 10000 OPENPAREN_PENALTY = 200 def generate_wrapping(words, divisions): lines = [] last = 0 for i in divisions: w = words[last:i] last = i line = " ".join(w).replace("\xff ","-").replace("\xff","-") lines.append(line.strip()) return lines def wrapping_quality(words, divisions, width1, width2): total = 0.0 lines = generate_wrapping(words, divisions) for line in lines: length = len(line) if line is lines[0]: width = width1 else: width = width2 if line[0:1] == '(': total += OPENPAREN_PENALTY if length > width: total += OVERFLOW_PENALTY * ( (length - width) ** OVERFLOW_EXPONENT ) else: if line is lines[-1]: e,p = (LASTLINE_UNDERFLOW_EXPONENT, LASTLINE_UNDERFLOW_PENALTY) if " " not in line: total += ORPHAN_PENALTY else: e,p = (UNDERFLOW_EXPONENT, UNDERFLOW_PENALTY) total += p * ((width - length) ** e) return total def wrap_graf(words, prefix_len1=0, prefix_len2=0, width=72): wrapping_after = [ (0,), ] w1 = width - prefix_len1 w2 = width - prefix_len2 for i in range(1, len(words)+1): best_so_far = None best_score = 1e300 for j in range(i): t = wrapping_after[j] t1 = t[:-1] + (i,) t2 = t + (i,) wq1 = wrapping_quality(words, t1, w1, w2) wq2 = wrapping_quality(words, t2, w1, w2) if wq1 < best_score: best_so_far = t1 best_score = wq1 if wq2 < best_score: best_so_far = t2 best_score = wq2 wrapping_after.append( best_so_far ) lines = generate_wrapping(words, wrapping_after[-1]) return lines def hyphenatable(word): if "--" in word: return False if re.match(r'^[^\d\-]\D*-', word): stripped = re.sub(r'^\W+','',word) stripped = re.sub(r'\W+$','',word) return stripped not in NO_HYPHENATE else: return False def split_paragraph(s): "Split paragraph into words; tuned for Tor." r = [] for word in s.split(): if hyphenatable(word): while "-" in word: a,word = word.split("-",1) r.append(a+"\xff") r.append(word) return r def fill(text, width, initial_indent, subsequent_indent): words = split_paragraph(text) lines = wrap_graf(words, len(initial_indent), len(subsequent_indent), width) res = [ initial_indent, lines[0], "\n" ] for line in lines[1:]: res.append(subsequent_indent) res.append(line) res.append("\n") return "".join(res) # ============================== TP_MAINHEAD = 0 TP_HEADTEXT = 1 TP_BLANK = 2 TP_SECHEAD = 3 TP_ITEMFIRST = 4 TP_ITEMBODY = 5 TP_END = 6 TP_PREHEAD = 7 def head_parser(line): if re.match(r'^Changes in', line): return TP_MAINHEAD elif re.match(r'^[A-Za-z]', line): return TP_PREHEAD elif re.match(r'^ o ', line): return TP_SECHEAD elif re.match(r'^\s*$', line): return TP_BLANK else: return TP_HEADTEXT def body_parser(line): if re.match(r'^ o ', line): return TP_SECHEAD elif re.match(r'^ -',line): return TP_ITEMFIRST elif re.match(r'^ \S', line): return TP_ITEMBODY elif re.match(r'^\s*$', line): return TP_BLANK elif re.match(r'^Changes in', line): return TP_END elif re.match(r'^\s+\S', line): return TP_HEADTEXT else: print("Weird line %r"%line, file=sys.stderr) def clean_head(head): return head def head_score(s): m = re.match(r'^ +o (.*)', s) if not m: print("Can't score %r"%s, file=sys.stderr) return 99999 lw = m.group(1).lower() if lw.startswith("security") and "feature" not in lw: score = -300 elif lw.startswith("deprecated version"): score = -200 elif lw.startswith("directory auth"): score = -150 elif (('new' in lw and 'requirement' in lw) or ('new' in lw and 'dependenc' in lw) or ('build' in lw and 'requirement' in lw) or ('removed' in lw and 'platform' in lw)): score = -100 elif lw.startswith("major feature"): score = 00 elif lw.startswith("major bug"): score = 50 elif lw.startswith("major"): score = 70 elif lw.startswith("minor feature"): score = 200 elif lw.startswith("minor bug"): score = 250 elif lw.startswith("minor"): score = 270 else: score = 1000 if 'secur' in lw: score -= 2 if "(other)" in lw: score += 2 if '(' not in lw: score -= 1 return score class ChangeLog(object): def __init__(self, wrapText=True, blogOrder=True, drupalBreak=False): self.prehead = [] self.mainhead = None self.headtext = [] self.curgraf = None self.sections = [] self.cursection = None self.lineno = 0 self.wrapText = wrapText self.blogOrder = blogOrder self.drupalBreak = drupalBreak def addLine(self, tp, line): self.lineno += 1 if tp == TP_MAINHEAD: assert not self.mainhead self.mainhead = line elif tp == TP_PREHEAD: self.prehead.append(line) elif tp == TP_HEADTEXT: if self.curgraf is None: self.curgraf = [] self.headtext.append(self.curgraf) self.curgraf.append(line) elif tp == TP_BLANK: self.curgraf = None elif tp == TP_SECHEAD: self.cursection = [ self.lineno, line, [] ] self.sections.append(self.cursection) elif tp == TP_ITEMFIRST: item = ( self.lineno, [ [line] ]) self.curgraf = item[1][0] self.cursection[2].append(item) elif tp == TP_ITEMBODY: if self.curgraf is None: self.curgraf = [] self.cursection[2][-1][1].append(self.curgraf) self.curgraf.append(line) else: assert False # This should be unreachable. def lint_head(self, line, head): m = re.match(r'^ *o ([^\(]+)((?:\([^\)]+\))?):', head) if not m: print("Weird header format on line %s"%line, file=sys.stderr) def lint_item(self, line, grafs, head_type): pass def lint(self): self.head_lines = {} for sec_line, sec_head, items in self.sections: head_type = self.lint_head(sec_line, sec_head) for item_line, grafs in items: self.lint_item(item_line, grafs, head_type) def dumpGraf(self,par,indent1,indent2=-1): if not self.wrapText: for line in par: print(line) return if indent2 == -1: indent2 = indent1 text = " ".join(re.sub(r'\s+', ' ', line.strip()) for line in par) sys.stdout.write(fill(text, width=72, initial_indent=" "*indent1, subsequent_indent=" "*indent2)) def dumpPreheader(self, graf): self.dumpGraf(graf, 0) print() def dumpMainhead(self, head): print(head) def dumpHeadGraf(self, graf): self.dumpGraf(graf, 2) print() def dumpSectionHeader(self, header): print(header) def dumpStartOfSections(self): pass def dumpEndOfSections(self): pass def dumpEndOfSection(self): print() def dumpEndOfChangelog(self): print() def dumpDrupalBreak(self): pass def dumpItem(self, grafs): self.dumpGraf(grafs[0],4,6) for par in grafs[1:]: print() self.dumpGraf(par,6,6) def collateAndSortSections(self): heads = [] sectionsByHead = { } for _, head, items in self.sections: head = clean_head(head) try: s = sectionsByHead[head] except KeyError: s = sectionsByHead[head] = [] heads.append( (head_score(head), head.lower(), head, s) ) s.extend(items) heads.sort() self.sections = [ (0, head, items) for _1,_2,head,items in heads ] def dump(self): if self.prehead: self.dumpPreheader(self.prehead) if not self.blogOrder: self.dumpMainhead(self.mainhead) for par in self.headtext: self.dumpHeadGraf(par) if self.blogOrder: self.dumpMainhead(self.mainhead) drupalBreakAfter = None if self.drupalBreak and len(self.sections) > 4: drupalBreakAfter = self.sections[1][2] self.dumpStartOfSections() for _,head,items in self.sections: if not head.endswith(':'): print("adding : to %r"%head, file=sys.stderr) head = head + ":" self.dumpSectionHeader(head) for _,grafs in items: self.dumpItem(grafs) self.dumpEndOfSection() if items is drupalBreakAfter: self.dumpDrupalBreak() self.dumpEndOfSections() self.dumpEndOfChangelog() # Map from issue prefix to pair of (visible prefix, url prefix) ISSUE_PREFIX_MAP = { "" : ( "", "tpo/core/tor" ), "tor#" : ( "", "tpo/core/tor" ), "chutney#" : ( "chutney#", "tpo/core/chutney" ), "torspec#" : ( "torspec#", "tpo/core/torspec" ), "trunnel#" : ( "trunnel#", "tpo/core/trunnel" ), "torsocks#" : ( "torsocks#", "tpo/core/torsocks"), } # Let's turn bugs to html. BUG_PAT = re.compile(r'(bug|ticket|issue|feature)\s+([\w/]+#)?(\d{4,6})', re.I) def bug_html(m): kind = m.group(1) prefix = m.group(2) or "" bugno = m.group(3) try: disp_prefix, url_prefix = ISSUE_PREFIX_MAP[prefix] except KeyError: print("Can't figure out URL for {}{}".format(prefix,bugno), file=sys.stderr) return "{} {}{}".format(kind, prefix, bugno) return "{} {}{}".format( kind, url_prefix, bugno, disp_prefix, bugno) class HTMLChangeLog(ChangeLog): def __init__(self, *args, **kwargs): ChangeLog.__init__(self, *args, **kwargs) def htmlText(self, graf): output = [] for line in graf: line = line.rstrip().replace("&","&") line = line.rstrip().replace("<","<").replace(">",">") output.append(line.strip()) output = " ".join(output) output = BUG_PAT.sub(bug_html, output) sys.stdout.write(output) def htmlPar(self, graf): sys.stdout.write("

") self.htmlText(graf) sys.stdout.write("

\n") def dumpPreheader(self, graf): self.htmlPar(graf) def dumpMainhead(self, head): sys.stdout.write("

%s

"%head) def dumpHeadGraf(self, graf): self.htmlPar(graf) def dumpSectionHeader(self, header): header = header.replace(" o ", "", 1).lstrip() sys.stdout.write("
  • %s\n"%header) sys.stdout.write(" \n\n") def dumpEndOfChangelog(self): pass def dumpStartOfSections(self): print("\n") def dumpDrupalBreak(self): print("\n\n") print("

     

    ") print("\n\n\n") print("