Merge branch 'censorship' into 1442_settings_tabs

6 files changed, 401 insertions, 13 deletions

diff --git a/cli/onionshare_cli/__init__.py b/cli/onionshare_cli/__init__.py
index 4bc00929..4e34a508 100644
--- a/cli/onionshare_cli/__init__.py
+++ b/cli/onionshare_cli/__init__.py
@@ -27,13 +27,10 @@ from datetime import datetime
 from datetime import timedelta
 
 from .common import Common, CannotFindTor
+from .censorship import CensorshipCircumvention
+from .meek import Meek, MeekNotRunning
 from .web import Web
-from .onion import (
-    TorErrorProtocolError,
-    TorTooOldEphemeral,
-    TorTooOldStealth,
-    Onion,
-)
+from .onion import TorErrorProtocolError, TorTooOldEphemeral, TorTooOldStealth, Onion
 from .onionshare import OnionShare
 from .mode_settings import ModeSettings
 
@@ -94,12 +91,7 @@ def main(cwd=None):
         help="Filename of persistent session",
     )
     # General args
-    parser.add_argument(
-        "--title",
-        metavar="TITLE",
-        default=None,
-        help="Set a title",
-    )
+    parser.add_argument("--title", metavar="TITLE", default=None, help="Set a title")
     parser.add_argument(
         "--public",
         action="store_true",
@@ -293,6 +285,20 @@ def main(cwd=None):
     # Create the Web object
     web = Web(common, False, mode_settings, mode)
 
+    # Create the Meek object and start the meek client
+    # meek = Meek(common)
+    # meek.start()
+
+    # Create the CensorshipCircumvention object to make
+    # API calls to Tor over Meek
+    censorship = CensorshipCircumvention(common, meek)
+    # Example: request recommended bridges, pretending to be from China, using
+    # domain fronting.
+    # censorship_recommended_settings = censorship.request_settings(country="cn")
+    # print(censorship_recommended_settings)
+    # Clean up the meek subprocess once we're done working with the censorship circumvention API
+    # meek.cleanup()
+
     # Start the Onion object
     try:
         onion = Onion(common, use_tmp_dir=True)
@@ -409,7 +415,7 @@ def main(cwd=None):
             sys.exit(1)
 
         # Warn about sending large files over Tor
-        if web.share_mode.download_filesize >= 157286400:  # 150mb
+        if web.share_mode.download_filesize >= 157_286_400:  # 150mb
             print("")
             print("Warning: Sending a large share could take hours")
             print("")
diff --git a/cli/onionshare_cli/censorship.py b/cli/onionshare_cli/censorship.py
new file mode 100644
index 00000000..f84b1058
--- /dev/null
+++ b/cli/onionshare_cli/censorship.py
@@ -0,0 +1,169 @@
+# -*- coding: utf-8 -*-
+"""
+OnionShare | https://onionshare.org/
+
+Copyright (C) 2014-2021 Micah Lee, et al. <micah@micahflee.com>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+import requests
+
+from .meek import MeekNotRunning
+
+
+class CensorshipCircumvention(object):
+    """
+    Connect to the Tor Moat APIs to retrieve censorship
+    circumvention recommendations, over the Meek client.
+    """
+
+    def __init__(self, common, meek, domain_fronting=True):
+        """
+        Set up the CensorshipCircumvention object to hold
+        common and meek objects.
+        """
+        self.common = common
+        self.meek = meek
+        self.common.log("CensorshipCircumvention", "__init__")
+
+        # Bail out if we requested domain fronting but we can't use meek
+        if domain_fronting and not self.meek.meek_proxies:
+            raise MeekNotRunning()
+
+    def request_map(self, country=False):
+        """
+        Retrieves the Circumvention map from Tor Project and store it
+        locally for further look-ups if required.
+
+        Optionally pass a country code in order to get recommended settings
+        just for that country.
+
+        Note that this API endpoint doesn't return actual bridges,
+        it just returns the recommended bridge type countries.
+        """
+        endpoint = "https://bridges.torproject.org/moat/circumvention/map"
+        data = {}
+        if country:
+            data = {"country": country}
+
+        r = requests.post(
+            endpoint,
+            json=data,
+            headers={"Content-Type": "application/vnd.api+json"},
+            proxies=self.meek.meek_proxies,
+        )
+        if r.status_code != 200:
+            self.common.log(
+                "CensorshipCircumvention",
+                "censorship_obtain_map",
+                f"status_code={r.status_code}",
+            )
+            return False
+
+        result = r.json()
+
+        if "errors" in result:
+            self.common.log(
+                "CensorshipCircumvention",
+                "censorship_obtain_map",
+                f"errors={result['errors']}",
+            )
+            return False
+
+        return result
+
+    def request_settings(self, country=False, transports=False):
+        """
+        Retrieves the Circumvention Settings from Tor Project, which
+        will return recommended settings based on the country code of
+        the requesting IP.
+
+        Optionally, a country code can be specified in order to override
+        the IP detection.
+
+        Optionally, a list of transports can be specified in order to
+        return recommended settings for just that transport type.
+        """
+        endpoint = "https://bridges.torproject.org/moat/circumvention/settings"
+        data = {}
+        if country:
+            data = {"country": country}
+        if transports:
+            data.append({"transports": transports})
+        r = requests.post(
+            endpoint,
+            json=data,
+            headers={"Content-Type": "application/vnd.api+json"},
+            proxies=self.meek.meek_proxies,
+        )
+        if r.status_code != 200:
+            self.common.log(
+                "CensorshipCircumvention",
+                "censorship_obtain_settings",
+                f"status_code={r.status_code}",
+            )
+            return False
+
+        result = r.json()
+
+        if "errors" in result:
+            self.common.log(
+                "CensorshipCircumvention",
+                "censorship_obtain_settings",
+                f"errors={result['errors']}",
+            )
+            return False
+
+        # There are no settings - perhaps this country doesn't require censorship circumvention?
+        # This is not really an error, so we can just check if False and assume direct Tor
+        # connection will work.
+        if not "settings" in result:
+            self.common.log(
+                "CensorshipCircumvention",
+                "censorship_obtain_settings",
+                "No settings found for this country",
+            )
+            return False
+
+        return result
+
+    def request_builtin_bridges(self):
+        """
+        Retrieves the list of built-in bridges from the Tor Project.
+        """
+        endpoint = "https://bridges.torproject.org/moat/circumvention/builtin"
+        r = requests.post(
+            endpoint,
+            headers={"Content-Type": "application/vnd.api+json"},
+            proxies=self.meek.meek_proxies,
+        )
+        if r.status_code != 200:
+            self.common.log(
+                "CensorshipCircumvention",
+                "censorship_obtain_builtin_bridges",
+                f"status_code={r.status_code}",
+            )
+            return False
+
+        result = r.json()
+
+        if "errors" in result:
+            self.common.log(
+                "CensorshipCircumvention",
+                "censorship_obtain_builtin_bridges",
+                f"errors={result['errors']}",
+            )
+            return False
+
+        return result
diff --git a/cli/onionshare_cli/common.py b/cli/onionshare_cli/common.py
index 07e0aa0a..b76e72b2 100644
--- a/cli/onionshare_cli/common.py
+++ b/cli/onionshare_cli/common.py
@@ -22,6 +22,7 @@ import hashlib
 import os
 import platform
 import random
+import requests
 import socket
 import sys
 import threading
@@ -314,6 +315,7 @@ class Common:
                 raise CannotFindTor()
             obfs4proxy_file_path = shutil.which("obfs4proxy")
             snowflake_file_path = shutil.which("snowflake-client")
+            meek_client_file_path = os.path.join(base_path, "meek-client")
             prefix = os.path.dirname(os.path.dirname(tor_path))
             tor_geo_ip_file_path = os.path.join(prefix, "share/tor/geoip")
             tor_geo_ipv6_file_path = os.path.join(prefix, "share/tor/geoip6")
@@ -322,6 +324,7 @@ class Common:
             tor_path = os.path.join(base_path, "Tor", "tor.exe")
             obfs4proxy_file_path = os.path.join(base_path, "Tor", "obfs4proxy.exe")
             snowflake_file_path = os.path.join(base_path, "Tor", "snowflake-client.exe")
+            meek_client_file_path = os.path.join(base_path, "Tor", "meek-client.exe")
             tor_geo_ip_file_path = os.path.join(base_path, "Data", "Tor", "geoip")
             tor_geo_ipv6_file_path = os.path.join(base_path, "Data", "Tor", "geoip6")
         elif self.platform == "Darwin":
@@ -330,6 +333,7 @@ class Common:
                 raise CannotFindTor()
             obfs4proxy_file_path = shutil.which("obfs4proxy")
             snowflake_file_path = shutil.which("snowflake-client")
+            meek_client_file_path = shutil.which("meek-client")
             prefix = os.path.dirname(os.path.dirname(tor_path))
             tor_geo_ip_file_path = os.path.join(prefix, "share/tor/geoip")
             tor_geo_ipv6_file_path = os.path.join(prefix, "share/tor/geoip6")
@@ -339,6 +343,7 @@ class Common:
             tor_geo_ipv6_file_path = "/usr/local/share/tor/geoip6"
             obfs4proxy_file_path = "/usr/local/bin/obfs4proxy"
             snowflake_file_path = "/usr/local/bin/snowflake-client"
+            meek_client_file_path = "/usr/local/bin/meek-client"
 
         return (
             tor_path,
@@ -346,6 +351,7 @@ class Common:
             tor_geo_ipv6_file_path,
             obfs4proxy_file_path,
             snowflake_file_path,
+            meek_client_file_path,
         )
 
     def build_data_dir(self):
diff --git a/cli/onionshare_cli/meek.py b/cli/onionshare_cli/meek.py
new file mode 100644
index 00000000..6b31a584
--- /dev/null
+++ b/cli/onionshare_cli/meek.py
@@ -0,0 +1,197 @@
+# -*- coding: utf-8 -*-
+"""
+OnionShare | https://onionshare.org/
+
+Copyright (C) 2014-2021 Micah Lee, et al. <micah@micahflee.com>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+import os
+import subprocess
+import time
+from queue import Queue, Empty
+from threading import Thread
+
+
+class Meek(object):
+    """
+    The Meek object starts the meek-client as a subprocess.
+    This process is used to do domain-fronting to connect to
+    the Tor APIs for censorship circumvention and retrieving
+    bridges, before connecting to Tor.
+    """
+
+    def __init__(self, common, get_tor_paths=None):
+        """
+        Set up the Meek object
+        """
+
+        self.common = common
+        self.common.log("Meek", "__init__")
+
+        # Set the path of the meek binary
+        if not get_tor_paths:
+            get_tor_paths = self.common.get_tor_paths
+        (
+            self.tor_path,
+            self.tor_geo_ip_file_path,
+            self.tor_geo_ipv6_file_path,
+            self.obfs4proxy_file_path,
+            self.snowflake_file_path,
+            self.meek_client_file_path,
+        ) = get_tor_paths()
+
+        self.meek_proxies = {}
+        self.meek_url = "https://moat.torproject.org.global.prod.fastly.net/"
+        self.meek_front = "cdn.sstatic.net"
+        self.meek_env = {
+            "TOR_PT_MANAGED_TRANSPORT_VER": "1",
+            "TOR_PT_CLIENT_TRANSPORTS": "meek",
+        }
+        self.meek_host = "127.0.0.1"
+        self.meek_port = None
+
+    def start(self):
+        """
+        Start the Meek Client and populate the SOCKS proxies dict
+        for use with requests to the Tor Moat API.
+        """
+        # Small method to read stdout from the subprocess.
+        # We use this to obtain the random port that Meek
+        # started on
+        def enqueue_output(out, queue):
+            for line in iter(out.readline, b""):
+                queue.put(line)
+            out.close()
+
+        # Abort early if we can't find the Meek client
+        if self.meek_client_file_path is None or not os.path.exists(
+            self.meek_client_file_path
+        ):
+            raise MeekNotFound()
+
+        # Start the Meek Client as a subprocess.
+        self.common.log("Meek", "start", "Starting meek client")
+
+        if self.common.platform == "Windows":
+            # In Windows, hide console window when opening meek-client.exe subprocess
+            startupinfo = subprocess.STARTUPINFO()
+            startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
+            self.meek_proc = subprocess.Popen(
+                [
+                    self.meek_client_file_path,
+                    "--url",
+                    self.meek_url,
+                    "--front",
+                    self.meek_front,
+                ],
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                startupinfo=startupinfo,
+                bufsize=1,
+                env=self.meek_env,
+                text=True,
+            )
+        else:
+            self.meek_proc = subprocess.Popen(
+                [
+                    self.meek_client_file_path,
+                    "--url",
+                    self.meek_url,
+                    "--front",
+                    self.meek_front,
+                ],
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                bufsize=1,
+                env=self.meek_env,
+                text=True,
+            )
+
+        # Queue up the stdout from the subprocess for polling later
+        q = Queue()
+        t = Thread(target=enqueue_output, args=(self.meek_proc.stdout, q))
+        t.daemon = True  # thread dies with the program
+        t.start()
+
+        while True:
+            # read stdout without blocking
+            try:
+                line = q.get_nowait()
+            except Empty:
+                # no stdout yet?
+                pass
+            else:  # we got stdout
+                if "CMETHOD meek socks5" in line:
+                    self.meek_host = line.split(" ")[3].split(":")[0]
+                    self.meek_port = line.split(" ")[3].split(":")[1]
+                    self.common.log("Meek", "start", f"Meek host is {self.meek_host}")
+                    self.common.log("Meek", "start", f"Meek port is {self.meek_port}")
+                    break
+
+        if self.meek_port:
+            self.meek_proxies = {
+                "http": f"socks5h://{self.meek_host}:{self.meek_port}",
+                "https": f"socks5h://{self.meek_host}:{self.meek_port}",
+            }
+        else:
+            self.common.log("Meek", "start", "Could not obtain the meek port")
+            raise MeekNotRunning()
+
+    def cleanup(self):
+        """
+        Kill any meek subprocesses.
+        """
+        self.common.log("Meek", "cleanup")
+
+        if self.meek_proc:
+            self.meek_proc.terminate()
+            time.sleep(0.2)
+            if self.meek_proc.poll() is None:
+                self.common.log(
+                    "Meek",
+                    "cleanup",
+                    "Tried to terminate meek-client process but it's still running",
+                )
+                try:
+                    self.meek_proc.kill()
+                    time.sleep(0.2)
+                    if self.meek_proc.poll() is None:
+                        self.common.log(
+                            "Meek",
+                            "cleanup",
+                            "Tried to kill meek-client process but it's still running",
+                        )
+                except Exception:
+                    self.common.log(
+                        "Meek", "cleanup", "Exception while killing meek-client process"
+                    )
+            self.meek_proc = None
+
+            # Reset other Meek settings
+            self.meek_proxies = {}
+            self.meek_port = None
+
+
+class MeekNotRunning(Exception):
+    """
+    We were unable to start Meek or obtain the port
+    number it started on, in order to do domain fronting.
+    """
+
+
+class MeekNotFound(Exception):
+    """
+    We were unable to find the Meek Client binary.
+    """
diff --git a/cli/onionshare_cli/onion.py b/cli/onionshare_cli/onion.py
index aa2344db..e8fcc12a 100644
--- a/cli/onionshare_cli/onion.py
+++ b/cli/onionshare_cli/onion.py
@@ -154,6 +154,7 @@ class Onion(object):
             self.tor_geo_ipv6_file_path,
             self.obfs4proxy_file_path,
             self.snowflake_file_path,
+            self.meek_client_file_path,
         ) = get_tor_paths()
 
         # The tor process
diff --git a/cli/tests/test_cli_common.py b/cli/tests/test_cli_common.py
index a4798d1b..9a64d762 100644
--- a/cli/tests/test_cli_common.py
+++ b/cli/tests/test_cli_common.py
@@ -162,6 +162,9 @@ class TestGetTorPaths:
         tor_geo_ip_file_path = os.path.join(base_path, "Resources", "Tor", "geoip")
         tor_geo_ipv6_file_path = os.path.join(base_path, "Resources", "Tor", "geoip6")
         obfs4proxy_file_path = os.path.join(base_path, "Resources", "Tor", "obfs4proxy")
+        meek_client_file_path = os.path.join(
+            base_path, "Resources", "Tor", "meek-client"
+        )
         snowflake_file_path = os.path.join(
             base_path, "Resources", "Tor", "snowflake-client"
         )
@@ -171,6 +174,7 @@ class TestGetTorPaths:
             tor_geo_ipv6_file_path,
             obfs4proxy_file_path,
             snowflake_file_path,
+            meek_client_file_path,
         )
 
     @pytest.mark.skipif(sys.platform != "linux", reason="requires Linux")
@@ -181,6 +185,7 @@ class TestGetTorPaths:
             tor_geo_ipv6_file_path,
             _,  # obfs4proxy is optional
             _,  # snowflake-client is optional
+            _,  # meek-client is optional
         ) = common_obj.get_tor_paths()
 
         assert os.path.basename(tor_path) == "tor"
@@ -207,6 +212,9 @@ class TestGetTorPaths:
         snowflake_file_path = os.path.join(
             os.path.join(base_path, "Tor"), "snowflake-client.exe"
         )
+        meek_client_file_path = os.path.join(
+            os.path.join(base_path, "Tor"), "meek-client.exe"
+        )
         tor_geo_ip_file_path = os.path.join(
             os.path.join(os.path.join(base_path, "Data"), "Tor"), "geoip"
         )
@@ -219,6 +227,7 @@ class TestGetTorPaths:
             tor_geo_ipv6_file_path,
             obfs4proxy_file_path,
             snowflake_file_path,
+            meek_client_file_path,
         )