18 files changed, 116 insertions, 47 deletions
diff --git a/scripts/codegen/fuzzing_include_am.py b/scripts/codegen/fuzzing_include_am.py
index aa3ba49a73..ae50563074 100755
--- a/scripts/codegen/fuzzing_include_am.py
+++ b/scripts/codegen/fuzzing_include_am.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 
 # Future imports for Python 2.7, mandatory in 3.0
 from __future__ import division
diff --git a/scripts/codegen/gen_server_ciphers.py b/scripts/codegen/gen_server_ciphers.py
index 3b77952243..8c88e54a13 100755
--- a/scripts/codegen/gen_server_ciphers.py
+++ b/scripts/codegen/gen_server_ciphers.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 # Copyright 2014-2019, The Tor Project, Inc
 # See LICENSE for licensing information
 
diff --git a/scripts/codegen/get_mozilla_ciphers.py b/scripts/codegen/get_mozilla_ciphers.py
index 165105736a..ff01dd8719 100755
--- a/scripts/codegen/get_mozilla_ciphers.py
+++ b/scripts/codegen/get_mozilla_ciphers.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 # coding=utf-8
 # Copyright 2011-2019, The Tor Project, Inc
 # original version by Arturo Filastò
diff --git a/scripts/codegen/makedesc.py b/scripts/codegen/makedesc.py
index af926a6438..48d1d31a02 100644
--- a/scripts/codegen/makedesc.py
+++ b/scripts/codegen/makedesc.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 # Copyright 2014-2019, The Tor Project, Inc.
 # See LICENSE for license information
 
diff --git a/scripts/maint/annotate_ifdef_directives.py b/scripts/maint/annotate_ifdef_directives.py
index cd70b55c8c..9ca090d595 100755
--- a/scripts/maint/annotate_ifdef_directives.py
+++ b/scripts/maint/annotate_ifdef_directives.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 # Copyright (c) 2017-2019, The Tor Project, Inc.
 # See LICENSE for licensing information
 
diff --git a/scripts/maint/checkIncludes.py b/scripts/maint/checkIncludes.py
index 2ca46347f0..ae0ccb9e12 100755
--- a/scripts/maint/checkIncludes.py
+++ b/scripts/maint/checkIncludes.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 # Copyright 2018 The Tor Project, Inc.  See LICENSE file for licensing info.
 
 # This file is no longer here; see practracker/includes.py for this
diff --git a/scripts/maint/checkManpageAlpha.py b/scripts/maint/checkManpageAlpha.py
new file mode 100755
index 0000000000..70421c2fd1
--- /dev/null
+++ b/scripts/maint/checkManpageAlpha.py
@@ -0,0 +1,72 @@
+#!/usr/bin/python
+
+import difflib
+import re
+import sys
+
+# Assume we only use the "== Section Name" section title syntax
+sectionheader_re = re.compile(r'^==+\s(.*)\s*$')
+
+# Assume we only use the "[[ItemName]]" anchor syntax
+anchor_re = re.compile(r'^\[\[([^]]+)\]\]')
+
+class Reader(object):
+    def __init__(self):
+        self.d = {}
+        # Initial state is to gather section headers
+        self.getline = self._getsec
+        self.section = None
+
+    def _getsec(self, line):
+        """Read a section header
+
+        Prepare to gather anchors from subsequent lines.  Don't change
+        state if the line isn't a section header.
+        """
+        m = sectionheader_re.match(line)
+        if not m:
+            return
+        self.anchors = anchors = []
+        self.d[m.group(1)] = anchors
+        self.getline = self._getanchor
+
+    def _getanchor(self, line):
+        """Read an anchor for an item definition
+
+        Append the anchor names to the list of items in the current
+        section.
+        """
+        m = anchor_re.match(line)
+        if not m:
+            return self._getsec(line)
+        self.anchors.append(m.group(1))
+
+    def diffsort(self, key):
+        """Unified diff of unsorted and sorted item lists
+        """
+        # Append newlines because difflib works better with them
+        a = [s + '\n' for s in self.d[key]]
+        b = sorted(a, key=str.lower)
+        return difflib.unified_diff(a, b, fromfile=key+' unsorted',
+                                    tofile=key+' sorted')
+
+def main():
+    """Diff unsorted and sorted lists of option names in a manpage
+
+    Use the file named by the first argument, or standard input if
+    there is none.
+    """
+    try:
+        fname = sys.argv[1]
+        f = open(fname, 'r')
+    except IndexError:
+        f = sys.stdin
+
+    reader = Reader()
+    for line in f:
+        reader.getline(line)
+    for key in sorted(reader.d.keys(), key=str.lower):
+        sys.stdout.writelines(reader.diffsort(key))
+
+if __name__ == '__main__':
+    main()
diff --git a/scripts/maint/format_changelog.py b/scripts/maint/format_changelog.py
index 32085c3602..7cf55a0d96 100755
--- a/scripts/maint/format_changelog.py
+++ b/scripts/maint/format_changelog.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 # Copyright (c) 2014-2019, The Tor Project, Inc.
 # See LICENSE for licensing information
 #
diff --git a/scripts/maint/lintChanges.py b/scripts/maint/lintChanges.py
index 88a865a572..cf7b09fcc3 100755
--- a/scripts/maint/lintChanges.py
+++ b/scripts/maint/lintChanges.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 
 # Future imports for Python 2.7, mandatory in 3.0
 from __future__ import division
@@ -25,7 +25,12 @@ KNOWN_GROUPS = set([
     "Code simplification and refactoring",
     "Removed features",
     "Deprecated features",
-    "Directory authority changes"])
+    "Directory authority changes",
+
+    # These aren't preferred, but sortChanges knows how to clean them up.
+    "Code simplifications and refactoring",
+    "Code simplification and refactorings",
+    "Code simplifications and refactorings"])
 
 NEEDS_SUBCATEGORIES = set([
     "Minor bugfix",
diff --git a/scripts/maint/locatemissingdoxygen.py b/scripts/maint/locatemissingdoxygen.py
index 7733977359..a2844346d6 100755
--- a/scripts/maint/locatemissingdoxygen.py
+++ b/scripts/maint/locatemissingdoxygen.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 
 """
   This script parses the stderr output of doxygen and looks for undocumented
diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt
index 70e6a55199..b4ef1e51aa 100644
--- a/scripts/maint/practracker/exceptions.txt
+++ b/scripts/maint/practracker/exceptions.txt
@@ -35,7 +35,6 @@
 
 problem file-size /src/app/config/config.c 7400
 problem include-count /src/app/config/config.c 80
-problem function-size /src/app/config/config.c:options_act_reversible() 298
 problem function-size /src/app/config/config.c:options_act() 381
 problem function-size /src/app/config/config.c:resolve_my_address() 190
 problem function-size /src/app/config/config.c:options_validate_cb() 780
@@ -48,8 +47,8 @@ problem function-size /src/app/config/config.c:parse_dir_authority_line() 150
 problem function-size /src/app/config/config.c:parse_dir_fallback_line() 101
 problem function-size /src/app/config/config.c:port_parse_config() 450
 problem function-size /src/app/config/config.c:parse_ports() 132
-problem file-size /src/app/config/or_options_st.h 1115
-problem include-count /src/app/main/main.c 69
+problem file-size /src/app/config/or_options_st.h 1053
+problem include-count /src/app/main/main.c 68
 problem function-size /src/app/main/main.c:dumpstats() 102
 problem function-size /src/app/main/main.c:tor_init() 101
 problem function-size /src/app/main/main.c:sandbox_init_filter() 291
@@ -63,17 +62,16 @@ problem dependency-violation /src/core/crypto/onion_tap.c 3
 problem dependency-violation /src/core/crypto/relay_crypto.c 9
 problem file-size /src/core/mainloop/connection.c 5577
 problem include-count /src/core/mainloop/connection.c 62
-problem function-size /src/core/mainloop/connection.c:connection_free_minimal() 185
+problem function-size /src/core/mainloop/connection.c:connection_free_minimal() 181
 problem function-size /src/core/mainloop/connection.c:connection_listener_new() 324
 problem function-size /src/core/mainloop/connection.c:connection_handle_listener_read() 161
-problem function-size /src/core/mainloop/connection.c:connection_proxy_connect() 148
 problem function-size /src/core/mainloop/connection.c:connection_read_proxy_handshake() 153
 problem function-size /src/core/mainloop/connection.c:retry_listener_ports() 112
 problem function-size /src/core/mainloop/connection.c:connection_handle_read_impl() 111
 problem function-size /src/core/mainloop/connection.c:connection_buf_read_from_socket() 180
 problem function-size /src/core/mainloop/connection.c:connection_handle_write_impl() 241
 problem function-size /src/core/mainloop/connection.c:assert_connection_ok() 143
-problem dependency-violation /src/core/mainloop/connection.c 44
+problem dependency-violation /src/core/mainloop/connection.c 47
 problem dependency-violation /src/core/mainloop/cpuworker.c 12
 problem include-count /src/core/mainloop/mainloop.c 64
 problem function-size /src/core/mainloop/mainloop.c:conn_close_if_marked() 108
@@ -85,9 +83,9 @@ problem dependency-violation /src/core/mainloop/netstatus.c 4
 problem dependency-violation /src/core/mainloop/periodic.c 2
 problem dependency-violation /src/core/or/address_set.c 1
 problem dependency-violation /src/core/or/cell_queue_st.h 1
-problem file-size /src/core/or/channel.c 3487
+problem file-size /src/core/or/channel.c 3464
 problem dependency-violation /src/core/or/channel.c 9
-problem file-size /src/core/or/channel.h 781
+problem file-size /src/core/or/channel.h 775
 problem dependency-violation /src/core/or/channel.h 1
 problem dependency-violation /src/core/or/channelpadding.c 6
 problem function-size /src/core/or/channeltls.c:channel_tls_handle_var_cell() 160
@@ -105,7 +103,6 @@ problem include-count /src/core/or/circuitlist.c 55
 problem function-size /src/core/or/circuitlist.c:HT_PROTOTYPE() 109
 problem function-size /src/core/or/circuitlist.c:circuit_free_() 146
 problem function-size /src/core/or/circuitlist.c:circuit_find_to_cannibalize() 101
-problem function-size /src/core/or/circuitlist.c:circuit_about_to_free() 120
 problem function-size /src/core/or/circuitlist.c:circuits_handle_oom() 117
 problem dependency-violation /src/core/or/circuitlist.c 19
 problem dependency-violation /src/core/or/circuitlist.h 1
@@ -123,7 +120,7 @@ problem function-size /src/core/or/circuitstats.c:circuit_build_times_parse_stat
 problem dependency-violation /src/core/or/circuitstats.c 11
 problem file-size /src/core/or/circuituse.c 3162
 problem function-size /src/core/or/circuituse.c:circuit_is_acceptable() 128
-problem function-size /src/core/or/circuituse.c:circuit_expire_building() 394
+problem function-size /src/core/or/circuituse.c:circuit_expire_building() 389
 problem function-size /src/core/or/circuituse.c:circuit_log_ancient_one_hop_circuits() 126
 problem function-size /src/core/or/circuituse.c:circuit_build_failed() 149
 problem function-size /src/core/or/circuituse.c:circuit_launch_by_extend_info() 108
@@ -146,23 +143,22 @@ problem function-size /src/core/or/connection_edge.c:connection_exit_connect() 1
 problem dependency-violation /src/core/or/connection_edge.c 27
 problem dependency-violation /src/core/or/connection_edge.h 1
 problem file-size /src/core/or/connection_or.c 3122
-problem include-count /src/core/or/connection_or.c 51
 problem function-size /src/core/or/connection_or.c:connection_or_group_set_badness_() 105
 problem function-size /src/core/or/connection_or.c:connection_or_client_learned_peer_id() 142
-problem function-size /src/core/or/connection_or.c:connection_or_compute_authenticate_cell_body() 231
 problem dependency-violation /src/core/or/connection_or.c 20
 problem dependency-violation /src/core/or/dos.c 6
 problem dependency-violation /src/core/or/onion.c 2
 problem file-size /src/core/or/or.h 1107
-problem include-count /src/core/or/or.h 49
+problem include-count /src/core/or/or.h 48
 problem dependency-violation /src/core/or/or.h 1
 problem dependency-violation /src/core/or/or_periodic.c 1
-problem file-size /src/core/or/policies.c 3249
+problem file-size /src/core/or/policies.c 3195
 problem function-size /src/core/or/policies.c:policy_summarize() 107
 problem dependency-violation /src/core/or/policies.c 14
 problem function-size /src/core/or/protover.c:protover_all_supported() 117
 problem dependency-violation /src/core/or/reasons.c 2
 problem file-size /src/core/or/relay.c 3264
+problem dependency-violation /src/core/or/relay_handshake.c 5
 problem function-size /src/core/or/relay.c:circuit_receive_relay_cell() 127
 problem function-size /src/core/or/relay.c:relay_send_command_from_edge_() 109
 problem function-size /src/core/or/relay.c:connection_ap_process_end_not_open() 192
@@ -207,15 +203,15 @@ problem function-size /src/feature/control/control_getinfo.c:getinfo_helper_misc
 problem function-size /src/feature/control/control_getinfo.c:getinfo_helper_dir() 297
 problem function-size /src/feature/control/control_getinfo.c:getinfo_helper_events() 234
 problem function-size /src/feature/dirauth/bwauth.c:dirserv_read_measured_bandwidths() 121
-problem file-size /src/feature/dirauth/dirvote.c 4700
+problem file-size /src/feature/dirauth/dirvote.c 4687
 problem include-count /src/feature/dirauth/dirvote.c 53
-problem function-size /src/feature/dirauth/dirvote.c:format_networkstatus_vote() 231
+problem function-size /src/feature/dirauth/dirvote.c:format_networkstatus_vote() 230
 problem function-size /src/feature/dirauth/dirvote.c:networkstatus_compute_bw_weights_v10() 233
-problem function-size /src/feature/dirauth/dirvote.c:networkstatus_compute_consensus() 956
+problem function-size /src/feature/dirauth/dirvote.c:networkstatus_compute_consensus() 952
 problem function-size /src/feature/dirauth/dirvote.c:networkstatus_add_detached_signatures() 119
 problem function-size /src/feature/dirauth/dirvote.c:dirvote_add_vote() 162
 problem function-size /src/feature/dirauth/dirvote.c:dirvote_compute_consensuses() 164
-problem function-size /src/feature/dirauth/dirvote.c:dirserv_generate_networkstatus_vote_obj() 283
+problem function-size /src/feature/dirauth/dirvote.c:dirserv_generate_networkstatus_vote_obj() 281
 problem function-size /src/feature/dirauth/dsigs_parse.c:networkstatus_parse_detached_signatures() 196
 problem function-size /src/feature/dirauth/guardfraction.c:dirserv_read_guardfraction_file_from_str() 109
 problem function-size /src/feature/dirauth/process_descs.c:dirserv_add_descriptor() 125
@@ -226,7 +222,7 @@ problem function-size /src/feature/dircache/consdiffmgr.c:consdiffmgr_rescan_fla
 problem function-size /src/feature/dircache/consdiffmgr.c:consensus_diff_worker_threadfn() 132
 problem function-size /src/feature/dircache/dircache.c:handle_get_current_consensus() 165
 problem function-size /src/feature/dircache/dircache.c:directory_handle_command_post() 124
-problem file-size /src/feature/dirclient/dirclient.c 3165
+problem file-size /src/feature/dirclient/dirclient.c 3156
 problem include-count /src/feature/dirclient/dirclient.c 51
 problem function-size /src/feature/dirclient/dirclient.c:directory_get_from_dirserver() 126
 problem function-size /src/feature/dirclient/dirclient.c:directory_initiate_request() 201
@@ -248,20 +244,19 @@ problem function-size /src/feature/hibernate/hibernate.c:accounting_parse_option
 problem function-size /src/feature/hs/hs_cell.c:hs_cell_build_establish_intro() 115
 problem function-size /src/feature/hs/hs_cell.c:hs_cell_parse_introduce2() 152
 problem function-size /src/feature/hs/hs_client.c:send_introduce1() 103
-problem function-size /src/feature/hs/hs_client.c:hs_config_client_authorization() 107
 problem function-size /src/feature/hs/hs_common.c:hs_get_responsible_hsdirs() 102
-problem function-size /src/feature/hs/hs_config.c:config_service_v3() 107
+problem function-size /src/feature/hs/hs_config.c:config_service_v3() 128
 problem function-size /src/feature/hs/hs_config.c:config_generic_service() 138
 problem function-size /src/feature/hs/hs_descriptor.c:desc_encode_v3() 101
 problem function-size /src/feature/hs/hs_descriptor.c:decrypt_desc_layer() 111
 problem function-size /src/feature/hs/hs_descriptor.c:decode_introduction_point() 122
 problem function-size /src/feature/hs/hs_descriptor.c:desc_decode_superencrypted_v3() 107
 problem function-size /src/feature/hs/hs_descriptor.c:desc_decode_encrypted_v3() 109
-problem file-size /src/feature/hs/hs_service.c 4172
+problem file-size /src/feature/hs/hs_service.c 4131
 problem function-size /src/feature/keymgt/loadkey.c:ed_key_init_from_file() 326
 problem function-size /src/feature/nodelist/authcert.c:trusted_dirs_load_certs_from_string() 123
 problem function-size /src/feature/nodelist/authcert.c:authority_certs_fetch_missing() 295
-problem function-size /src/feature/nodelist/fmt_routerstatus.c:routerstatus_format_entry() 162
+problem function-size /src/feature/nodelist/fmt_routerstatus.c:routerstatus_format_entry() 158
 problem function-size /src/feature/nodelist/microdesc.c:microdesc_cache_rebuild() 134
 problem include-count /src/feature/nodelist/networkstatus.c 63
 problem function-size /src/feature/nodelist/networkstatus.c:networkstatus_check_consensus_signature() 175
@@ -279,6 +274,7 @@ problem function-size /src/feature/nodelist/routerlist.c:update_extrainfo_downlo
 problem function-size /src/feature/relay/dns.c:dns_resolve_impl() 131
 problem function-size /src/feature/relay/dns.c:configure_nameservers() 161
 problem function-size /src/feature/relay/dns.c:evdns_callback() 108
+problem function-size /src/feature/relay/relay_handshake.c:connection_or_compute_authenticate_cell_body() 231
 problem file-size /src/feature/relay/router.c 3520
 problem include-count /src/feature/relay/router.c 57
 problem function-size /src/feature/relay/router.c:init_keys() 252
diff --git a/scripts/maint/practracker/includes.py b/scripts/maint/practracker/includes.py
index fe0f32e253..e9b02c35b0 100755
--- a/scripts/maint/practracker/includes.py
+++ b/scripts/maint/practracker/includes.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 # Copyright 2018 The Tor Project, Inc.  See LICENSE file for licensing info.
 
 """This script looks through all the directories for files matching *.c or
diff --git a/scripts/maint/practracker/metrics.py b/scripts/maint/practracker/metrics.py
index ae88b84f31..300a4501a9 100644
--- a/scripts/maint/practracker/metrics.py
+++ b/scripts/maint/practracker/metrics.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 
 # Implementation of various source code metrics.
 # These are currently ad-hoc string operations and regexps.
diff --git a/scripts/maint/practracker/practracker.py b/scripts/maint/practracker/practracker.py
index 6149fb79cb..76ffd64cfb 100755
--- a/scripts/maint/practracker/practracker.py
+++ b/scripts/maint/practracker/practracker.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 
 """
 Best-practices tracker for Tor source code.
@@ -24,7 +24,7 @@ from __future__ import division
 from __future__ import print_function
 from __future__ import unicode_literals
 
-import os, sys
+import codecs, os, sys
 
 import metrics
 import util
@@ -63,12 +63,8 @@ TOR_TOPDIR = None
 
 #######################################################
 
-if sys.version_info[0] <= 2:
-    def open_file(fname):
-        return open(fname, 'r')
-else:
-    def open_file(fname):
-        return open(fname, 'r', encoding='utf-8')
+def open_file(fname):
+    return codecs.open(fname, 'r', encoding='utf-8')
 
 def consider_file_size(fname, f):
     """Consider the size of 'f' and yield an FileSizeItem for it.
diff --git a/scripts/maint/practracker/practracker_tests.py b/scripts/maint/practracker/practracker_tests.py
index 8d0418880c..e03c9e05ae 100755
--- a/scripts/maint/practracker/practracker_tests.py
+++ b/scripts/maint/practracker/practracker_tests.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 
 """Some simple tests for practracker metrics"""
 
diff --git a/scripts/maint/rectify_include_paths.py b/scripts/maint/rectify_include_paths.py
index 111cf816ce..c6c5026711 100755
--- a/scripts/maint/rectify_include_paths.py
+++ b/scripts/maint/rectify_include_paths.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 
 # Future imports for Python 2.7, mandatory in 3.0
 from __future__ import division
diff --git a/scripts/maint/redox.py b/scripts/maint/redox.py
index 3ad3e3f1b8..12b02c8a44 100755
--- a/scripts/maint/redox.py
+++ b/scripts/maint/redox.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 #
 #  Copyright (c) 2008-2019, The Tor Project, Inc.
 #  See LICENSE for licensing information.
diff --git a/scripts/maint/sortChanges.py b/scripts/maint/sortChanges.py
index 2e049b1e53..5f6324e387 100755
--- a/scripts/maint/sortChanges.py
+++ b/scripts/maint/sortChanges.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 # Copyright (c) 2014-2019, The Tor Project, Inc.
 # See LICENSE for licensing information