Merge pull request #1478 from mig5/fetch-builtin-bridges

Fetch the built-in bridges from Tor's Censorship Circumvention API, rather than hardcode them

8 files changed, 186 insertions, 67 deletions

diff --git a/cli/onionshare_cli/censorship.py b/cli/onionshare_cli/censorship.py
index f84b1058..9f41d61c 100644
--- a/cli/onionshare_cli/censorship.py
+++ b/cli/onionshare_cli/censorship.py
@@ -25,21 +25,46 @@ from .meek import MeekNotRunning
 class CensorshipCircumvention(object):
     """
     Connect to the Tor Moat APIs to retrieve censorship
-    circumvention recommendations, over the Meek client.
+    circumvention recommendations or the latest bridges.
+
+    We support reaching this API over Tor, or Meek
+    (domain fronting) if Tor is not connected.
     """
 
-    def __init__(self, common, meek, domain_fronting=True):
+    def __init__(self, common, meek=None, onion=None):
         """
         Set up the CensorshipCircumvention object to hold
         common and meek objects.
         """
         self.common = common
-        self.meek = meek
         self.common.log("CensorshipCircumvention", "__init__")
-
-        # Bail out if we requested domain fronting but we can't use meek
-        if domain_fronting and not self.meek.meek_proxies:
-            raise MeekNotRunning()
+        self.api_proxies = {}
+        if meek:
+            self.meek = meek
+            if not self.meek.meek_proxies:
+                raise MeekNotRunning()
+            else:
+                self.common.log(
+                    "CensorshipCircumvention",
+                    "__init__",
+                    "Using Meek with CensorShipCircumvention API",
+                )
+                self.api_proxies = self.meek.meek_proxies
+        if onion:
+            self.onion = onion
+            if not self.onion.is_authenticated:
+                return False
+            else:
+                self.common.log(
+                    "CensorshipCircumvention",
+                    "__init__",
+                    "Using Tor with CensorShipCircumvention API",
+                )
+                (socks_address, socks_port) = self.onion.get_tor_socks_port()
+                self.api_proxies = {
+                    "http": f"socks5h://{socks_address}:{socks_port}",
+                    "https": f"socks5h://{socks_address}:{socks_port}",
+                }
 
     def request_map(self, country=False):
         """
@@ -52,6 +77,8 @@ class CensorshipCircumvention(object):
         Note that this API endpoint doesn't return actual bridges,
         it just returns the recommended bridge type countries.
         """
+        if not self.api_proxies:
+            return False
         endpoint = "https://bridges.torproject.org/moat/circumvention/map"
         data = {}
         if country:
@@ -61,7 +88,7 @@ class CensorshipCircumvention(object):
             endpoint,
             json=data,
             headers={"Content-Type": "application/vnd.api+json"},
-            proxies=self.meek.meek_proxies,
+            proxies=self.api_proxies,
         )
         if r.status_code != 200:
             self.common.log(
@@ -95,6 +122,8 @@ class CensorshipCircumvention(object):
         Optionally, a list of transports can be specified in order to
         return recommended settings for just that transport type.
         """
+        if not self.api_proxies:
+            return False
         endpoint = "https://bridges.torproject.org/moat/circumvention/settings"
         data = {}
         if country:
@@ -105,7 +134,7 @@ class CensorshipCircumvention(object):
             endpoint,
             json=data,
             headers={"Content-Type": "application/vnd.api+json"},
-            proxies=self.meek.meek_proxies,
+            proxies=self.api_proxies,
         )
         if r.status_code != 200:
             self.common.log(
@@ -142,11 +171,13 @@ class CensorshipCircumvention(object):
         """
         Retrieves the list of built-in bridges from the Tor Project.
         """
+        if not self.api_proxies:
+            return False
         endpoint = "https://bridges.torproject.org/moat/circumvention/builtin"
         r = requests.post(
             endpoint,
             headers={"Content-Type": "application/vnd.api+json"},
-            proxies=self.meek.meek_proxies,
+            proxies=self.api_proxies,
         )
         if r.status_code != 200:
             self.common.log(
diff --git a/cli/onionshare_cli/onion.py b/cli/onionshare_cli/onion.py
index 5ac669b8..ba9e2e77 100644
--- a/cli/onionshare_cli/onion.py
+++ b/cli/onionshare_cli/onion.py
@@ -18,17 +18,19 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 
+from .censorship import CensorshipCircumvention
+from .meek import Meek
 from stem.control import Controller
 from stem import ProtocolError, SocketClosed
 from stem.connection import MissingPassword, UnreadableCookieFile, AuthenticationFailure
 import base64
 import nacl.public
 import os
-import tempfile
+import psutil
+import shlex
 import subprocess
+import tempfile
 import time
-import shlex
-import psutil
 import traceback
 
 from distutils.version import LooseVersion as Version
@@ -258,9 +260,7 @@ class Onion(object):
                         and cmdline[2] == self.tor_torrc
                     ):
                         self.common.log(
-                            "Onion",
-                            "connect",
-                            "found a stale tor process, killing it",
+                            "Onion", "connect", "found a stale tor process, killing it"
                         )
                         proc.terminate()
                         proc.wait()
@@ -317,49 +317,75 @@ class Onion(object):
             )
 
             with open(self.tor_torrc, "w") as f:
+                self.common.log("Onion", "connect", "Writing torrc template file")
                 f.write(torrc_template)
 
                 # Bridge support
                 if self.settings.get("bridges_enabled"):
+                    f.write("\nUseBridges 1\n")
                     if self.settings.get("bridges_type") == "built-in":
-                        if self.settings.get("bridges_builtin_pt") == "obfs4":
-                            with open(
-                                self.common.get_resource_path("torrc_template-obfs4")
-                            ) as o:
-                                f.write(o.read())
-                        elif self.settings.get("bridges_builtin_pt") == "meek-azure":
-                            with open(
-                                self.common.get_resource_path(
-                                    "torrc_template-meek_lite_azure"
-                                )
-                            ) as o:
-                                f.write(o.read())
-                        elif self.settings.get("bridges_builtin_pt") == "snowflake":
-                            with open(
-                                self.common.get_resource_path(
-                                    "torrc_template-snowflake"
+                        use_torrc_bridge_templates = False
+                        builtin_bridge_type = self.settings.get("bridges_builtin_pt")
+                        # Use built-inbridges stored in settings, if they are there already.
+                        # They are probably newer than that of our hardcoded copies.
+                        if self.settings.get("bridges_builtin"):
+                            try:
+                                for line in self.settings.get("bridges_builtin")[
+                                    builtin_bridge_type
+                                ]:
+                                    if line.strip() != "":
+                                        f.write(f"Bridge {line}\n")
+                                self.common.log(
+                                    "Onion",
+                                    "connect",
+                                    "Wrote in the built-in bridges from OnionShare settings",
                                 )
-                            ) as o:
-                                f.write(o.read())
-
+                            except KeyError:
+                                # Somehow we had built-in bridges in our settings, but
+                                # not for this bridge type. Fall back to using the hard-
+                                # coded templates.
+                                use_torrc_bridge_templates = True
+                        else:
+                            use_torrc_bridge_templates = True
+                        if use_torrc_bridge_templates:
+                            if builtin_bridge_type == "obfs4":
+                                with open(
+                                    self.common.get_resource_path(
+                                        "torrc_template-obfs4"
+                                    )
+                                ) as o:
+                                    f.write(o.read())
+                            elif builtin_bridge_type == "meek-azure":
+                                with open(
+                                    self.common.get_resource_path(
+                                        "torrc_template-meek_lite_azure"
+                                    )
+                                ) as o:
+                                    f.write(o.read())
+                            elif builtin_bridge_type == "snowflake":
+                                with open(
+                                    self.common.get_resource_path(
+                                        "torrc_template-snowflake"
+                                    )
+                                ) as o:
+                                    f.write(o.read())
+                            self.common.log(
+                                "Onion",
+                                "connect",
+                                "Wrote in the built-in bridges from torrc templates",
+                            )
                     elif self.settings.get("bridges_type") == "moat":
                         for line in self.settings.get("bridges_moat").split("\n"):
                             if line.strip() != "":
                                 f.write(f"Bridge {line}\n")
-                        f.write("\nUseBridges 1\n")
 
                     elif self.settings.get("bridges_type") == "custom":
                         for line in self.settings.get("bridges_custom").split("\n"):
                             if line.strip() != "":
                                 f.write(f"Bridge {line}\n")
-                        f.write("\nUseBridges 1\n")
 
             # Execute a tor subprocess
-            self.common.log(
-                "Onion",
-                "connect",
-                f"starting {self.tor_path} subprocess",
-            )
+            self.common.log("Onion", "connect", f"starting {self.tor_path} subprocess")
             start_ts = time.time()
             if self.common.platform == "Windows":
                 # In Windows, hide console window when opening tor.exe subprocess
@@ -385,19 +411,11 @@ class Onion(object):
                 )
 
             # Wait for the tor controller to start
-            self.common.log(
-                "Onion",
-                "connect",
-                f"tor pid: {self.tor_proc.pid}",
-            )
+            self.common.log("Onion", "connect", f"tor pid: {self.tor_proc.pid}")
             time.sleep(2)
 
             # Connect to the controller
-            self.common.log(
-                "Onion",
-                "connect",
-                "authenticating to tor controller",
-            )
+            self.common.log("Onion", "connect", "authenticating to tor controller")
             try:
                 if (
                     self.common.platform == "Windows"
@@ -638,6 +656,14 @@ class Onion(object):
         # https://trac.torproject.org/projects/tor/ticket/28619
         self.supports_v3_onions = self.tor_version >= Version("0.3.5.7")
 
+        # Now that we are connected to Tor, if we are using built-in bridges,
+        # update them with the latest copy available from the Tor API
+        if (
+            self.settings.get("bridges_enabled")
+            and self.settings.get("bridges_type") == "built-in"
+        ):
+            self.update_builtin_bridges()
+
     def is_authenticated(self):
         """
         Returns True if the Tor connection is still working, or False otherwise.
@@ -881,3 +907,68 @@ class Onion(object):
             return ("127.0.0.1", 9150)
         else:
             return (self.settings.get("socks_address"), self.settings.get("socks_port"))
+
+    def update_builtin_bridges(self):
+        """
+        Use the CensorshipCircumvention API to fetch the latest built-in bridges
+        and update them in settings.
+        """
+        builtin_bridges = False
+        meek = None
+        # Try obtaining bridges over Tor, if we're connected to it.
+        if self.is_authenticated:
+            self.common.log(
+                "Onion",
+                "update_builtin_bridges",
+                "Updating the built-in bridges. Trying over Tor first",
+            )
+            self.censorship_circumvention = CensorshipCircumvention(
+                self.common, None, self
+            )
+            builtin_bridges = self.censorship_circumvention.request_builtin_bridges()
+
+        if not builtin_bridges:
+            # Tor was not running or it failed to hit the Tor API.
+            # Fall back to using Meek (domain-fronting).
+            self.common.log(
+                "Onion",
+                "update_builtin_bridges",
+                "Updating the built-in bridges. Trying via Meek (no Tor)",
+            )
+            meek = Meek(self.common)
+            meek.start()
+            self.censorship_circumvention = CensorshipCircumvention(
+                self.common, meek, None
+            )
+            builtin_bridges = self.censorship_circumvention.request_builtin_bridges()
+            meek.cleanup()
+
+        if builtin_bridges:
+            # If we got to this point, we have bridges
+            self.common.log(
+                "Onion",
+                "update_builtin_bridges",
+                f"Obtained bridges: {builtin_bridges}",
+            )
+            if builtin_bridges["meek"]:
+                # Meek bridge needs to be defined as "meek_lite", not "meek",
+                # for it to work with obfs4proxy.
+                # We also refer to this bridge type as 'meek-azure' in our settings.
+                # So first, rename the key in the dict
+                builtin_bridges["meek-azure"] = builtin_bridges.pop("meek")
+                new_meek_bridges = []
+                # Now replace the values. They also need the url/front params appended
+                for item in builtin_bridges["meek-azure"]:
+                    newline = item.replace("meek", "meek_lite")
+                    new_meek_bridges.append(
+                        f"{newline} url=https://meek.azureedge.net/ front=ajax.aspnetcdn.com"
+                    )
+                builtin_bridges["meek-azure"] = new_meek_bridges
+            # Save the new settings
+            self.settings.set("bridges_builtin", builtin_bridges)
+            self.settings.save()
+        else:
+            self.common.log(
+                "Onion", "update_builtin_bridges", "Error getting built-in bridges"
+            )
+            return False
diff --git a/cli/onionshare_cli/resources/torrc_template-meek_lite_azure b/cli/onionshare_cli/resources/torrc_template-meek_lite_azure
index 6f601681..cbc5a9ee 100644
--- a/cli/onionshare_cli/resources/torrc_template-meek_lite_azure
+++ b/cli/onionshare_cli/resources/torrc_template-meek_lite_azure
@@ -1,3 +1,2 @@
 # Enable built-in meek-azure bridge
 Bridge meek_lite 0.0.2.0:3 97700DFE9F483596DDA6264C4D7DF7641E1E39CE url=https://meek.azureedge.net/ front=ajax.aspnetcdn.com
-UseBridges 1
diff --git a/cli/onionshare_cli/resources/torrc_template-obfs4 b/cli/onionshare_cli/resources/torrc_template-obfs4
index 720cc28c..4ea90134 100644
--- a/cli/onionshare_cli/resources/torrc_template-obfs4
+++ b/cli/onionshare_cli/resources/torrc_template-obfs4
@@ -1,17 +1,16 @@
 # Enable built-in obfs4-bridge
-Bridge obfs4 192.95.36.142:443 CDF2E852BF539B82BD10E27E9115A31734E378C2 cert=qUVQ0srL1JI/vO6V6m/24anYXiJD3QP2HgzUKQtQ7GRqqUvs7P+tG43RtAqdhLOALP7DJQ iat-mode=1
-Bridge obfs4 38.229.1.78:80 C8CBDB2464FC9804A69531437BCF2BE31FDD2EE4 cert=Hmyfd2ev46gGY7NoVxA9ngrPF2zCZtzskRTzoWXbxNkzeVnGFPWmrTtILRyqCTjHR+s9dg iat-mode=1
 Bridge obfs4 38.229.33.83:80 0BAC39417268B96B9F514E7F63FA6FBA1A788955 cert=VwEFpk9F/UN9JED7XpG1XOjm/O8ZCXK80oPecgWnNDZDv5pdkhq1OpbAH0wNqOT6H6BmRQ iat-mode=1
-Bridge obfs4 37.218.245.14:38224 D9A82D2F9C2F65A18407B1D2B764F130847F8B5D cert=bjRaMrr1BRiAW8IE9U5z27fQaYgOhX1UCmOpg2pFpoMvo6ZgQMzLsaTzzQNTlm7hNcb+Sg iat-mode=0
+Bridge obfs4 193.11.166.194:27015 2D82C2E354D531A68469ADF7F878FA6060C6BACA cert=4TLQPJrTSaDffMK7Nbao6LC7G9OW/NHkUwIdjLSS3KYf0Nv4/nQiiI8dY2TcsQx01NniOg iat-mode=0
 Bridge obfs4 85.31.186.98:443 011F2599C0E9B27EE74B353155E244813763C3E5 cert=ayq0XzCwhpdysn5o0EyDUbmSOx3X/oTEbzDMvczHOdBJKlvIdHHLJGkZARtT4dcBFArPPg iat-mode=0
+Bridge obfs4 146.57.248.225:22 10A6CD36A537FCE513A322361547444B393989F0 cert=K1gDtDAIcUfeLqbstggjIw2rtgIKqdIhUlHp82XRqNSq/mtAjp1BIC9vHKJ2FAEpGssTPw iat-mode=0
 Bridge obfs4 85.31.186.26:443 91A6354697E6B02A386312F68D82CF86824D3606 cert=PBwr+S8JTVZo6MPdHnkTwXJPILWADLqfMGoVvhZClMq/Urndyd42BwX9YFJHZnBB3H0XCw iat-mode=0
-Bridge obfs4 144.217.20.138:80 FB70B257C162BF1038CA669D568D76F5B7F0BABB cert=vYIV5MgrghGQvZPIi1tJwnzorMgqgmlKaB77Y3Z9Q/v94wZBOAXkW+fdx4aSxLVnKO+xNw iat-mode=0
-Bridge obfs4 193.11.166.194:27015 2D82C2E354D531A68469ADF7F878FA6060C6BACA cert=4TLQPJrTSaDffMK7Nbao6LC7G9OW/NHkUwIdjLSS3KYf0Nv4/nQiiI8dY2TcsQx01NniOg iat-mode=0
-Bridge obfs4 193.11.166.194:27020 86AC7B8D430DAC4117E9F42C9EAED18133863AAF cert=0LDeJH4JzMDtkJJrFphJCiPqKx7loozKN7VNfuukMGfHO0Z8OGdzHVkhVAOfo1mUdv9cMg iat-mode=0
-Bridge obfs4 193.11.166.194:27025 1AE2C08904527FEA90C4C4F8C1083EA59FBC6FAF cert=ItvYZzW5tn6v3G4UnQa6Qz04Npro6e81AP70YujmK/KXwDFPTs3aHXcHp4n8Vt6w/bv8cA iat-mode=0
+Bridge obfs4 51.222.13.177:80 5EDAC3B810E12B01F6FD8050D2FD3E277B289A08 cert=2uplIpLQ0q9+0qMFrK5pkaYRDOe460LL9WHBvatgkuRr/SL31wBOEupaMMJ6koRE6Ld0ew iat-mode=0
 Bridge obfs4 209.148.46.65:443 74FAD13168806246602538555B5521A0383A1875 cert=ssH+9rP8dG2NLDN2XuFw63hIO/9MNNinLmxQDpVa+7kTOa9/m+tGWT1SmSYpQ9uTBGa6Hw iat-mode=0
-Bridge obfs4 146.57.248.225:22 10A6CD36A537FCE513A322361547444B393989F0 cert=K1gDtDAIcUfeLqbstggjIw2rtgIKqdIhUlHp82XRqNSq/mtAjp1BIC9vHKJ2FAEpGssTPw iat-mode=0
 Bridge obfs4 45.145.95.6:27015 C5B7CD6946FF10C5B3E89691A7D3F2C122D2117C cert=TD7PbUO0/0k6xYHMPW3vJxICfkMZNdkRrb63Zhl5j9dW3iRGiCx0A7mPhe5T2EDzQ35+Zw iat-mode=0
+Bridge obfs4 38.229.1.78:80 C8CBDB2464FC9804A69531437BCF2BE31FDD2EE4 cert=Hmyfd2ev46gGY7NoVxA9ngrPF2zCZtzskRTzoWXbxNkzeVnGFPWmrTtILRyqCTjHR+s9dg iat-mode=1
+Bridge obfs4 193.11.166.194:27025 1AE2C08904527FEA90C4C4F8C1083EA59FBC6FAF cert=ItvYZzW5tn6v3G4UnQa6Qz04Npro6e81AP70YujmK/KXwDFPTs3aHXcHp4n8Vt6w/bv8cA iat-mode=0
+Bridge obfs4 193.11.166.194:27020 86AC7B8D430DAC4117E9F42C9EAED18133863AAF cert=0LDeJH4JzMDtkJJrFphJCiPqKx7loozKN7VNfuukMGfHO0Z8OGdzHVkhVAOfo1mUdv9cMg iat-mode=0
+Bridge obfs4 37.218.245.14:38224 D9A82D2F9C2F65A18407B1D2B764F130847F8B5D cert=bjRaMrr1BRiAW8IE9U5z27fQaYgOhX1UCmOpg2pFpoMvo6ZgQMzLsaTzzQNTlm7hNcb+Sg iat-mode=0
+Bridge obfs4 144.217.20.138:80 FB70B257C162BF1038CA669D568D76F5B7F0BABB cert=vYIV5MgrghGQvZPIi1tJwnzorMgqgmlKaB77Y3Z9Q/v94wZBOAXkW+fdx4aSxLVnKO+xNw iat-mode=0
+Bridge obfs4 192.95.36.142:443 CDF2E852BF539B82BD10E27E9115A31734E378C2 cert=qUVQ0srL1JI/vO6V6m/24anYXiJD3QP2HgzUKQtQ7GRqqUvs7P+tG43RtAqdhLOALP7DJQ iat-mode=1
 Bridge obfs4 [2a0c:4d80:42:702::1]:27015 C5B7CD6946FF10C5B3E89691A7D3F2C122D2117C cert=TD7PbUO0/0k6xYHMPW3vJxICfkMZNdkRrb63Zhl5j9dW3iRGiCx0A7mPhe5T2EDzQ35+Zw iat-mode=0
-Bridge obfs4 51.222.13.177:80 5EDAC3B810E12B01F6FD8050D2FD3E277B289A08 cert=2uplIpLQ0q9+0qMFrK5pkaYRDOe460LL9WHBvatgkuRr/SL31wBOEupaMMJ6koRE6Ld0ew iat-mode=0
-UseBridges 1
diff --git a/cli/onionshare_cli/resources/torrc_template-snowflake b/cli/onionshare_cli/resources/torrc_template-snowflake
index 4100d3be..20efe28c 100644
--- a/cli/onionshare_cli/resources/torrc_template-snowflake
+++ b/cli/onionshare_cli/resources/torrc_template-snowflake
@@ -1,3 +1 @@
-# Enable built-in snowflake bridge
-Bridge snowflake 192.0.2.3:1 2B280B23E1107BB62ABFC40DDCC8824814F80A72
-UseBridges 1
+Bridge snowflake 0.0.3.0:1 2B280B23E1107BB62ABFC40DDCC8824814F80A72
diff --git a/cli/onionshare_cli/settings.py b/cli/onionshare_cli/settings.py
index c7d74a70..8a4a9939 100644
--- a/cli/onionshare_cli/settings.py
+++ b/cli/onionshare_cli/settings.py
@@ -110,6 +110,7 @@ class Settings(object):
             "bridges_builtin_pt": "obfs4",  # "obfs4", "meek-azure", or "snowflake"
             "bridges_moat": "",
             "bridges_custom": "",
+            "bridges_builtin": {},
             "persistent_tabs": [],
             "locale": None,  # this gets defined in fill_in_defaults()
             "theme": 0,
diff --git a/cli/tests/test_cli_settings.py b/cli/tests/test_cli_settings.py
index 4a4e6fd0..f370a674 100644
--- a/cli/tests/test_cli_settings.py
+++ b/cli/tests/test_cli_settings.py
@@ -34,6 +34,7 @@ class TestSettings:
             "bridges_builtin_pt": "obfs4",
             "bridges_moat": "",
             "bridges_custom": "",
+            "bridges_builtin": {},
             "persistent_tabs": [],
             "theme": 0,
         }
diff --git a/desktop/src/onionshare/gui_common.py b/desktop/src/onionshare/gui_common.py
index d0ad249b..2a0bae4d 100644
--- a/desktop/src/onionshare/gui_common.py
+++ b/desktop/src/onionshare/gui_common.py
@@ -507,5 +507,4 @@ class GuiCommon:
             return strings._("error_stealth_not_supported")
         elif type(e) is PortNotAvailable:
             return strings._("error_port_not_available")
-
         return None