From c0a55c1f755c51cf37ce26431db98bf42631631e Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Wed, 7 Apr 2021 23:34:58 +0300 Subject: block subdomains of blocked hosts --- qutebrowser/components/hostblock.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/qutebrowser/components/hostblock.py b/qutebrowser/components/hostblock.py index 8a0174584..e19536744 100644 --- a/qutebrowser/components/hostblock.py +++ b/qutebrowser/components/hostblock.py @@ -37,6 +37,7 @@ from qutebrowser.api import ( qtutils, ) from qutebrowser.components.utils import blockutils +from qutebrowser.config.configutils import _widened_hostnames from qutebrowser.utils import version # FIXME: Move needed parts into api namespace? @@ -124,10 +125,16 @@ class HostBlocker: if not config.get("content.blocking.enabled", url=first_party_url): return False + if blockutils.is_whitelisted_url(request_url): + return False + host = request_url.host() - return ( - host in self._blocked_hosts or host in self._config_blocked_hosts - ) and not blockutils.is_whitelisted_url(request_url) + + for hostname in _widened_hostnames(host): + if hostname in self._blocked_hosts or hostname in self._config_blocked_hosts: + return True + + return False def filter_request(self, info: interceptor.Request) -> None: """Block the given request if necessary.""" -- cgit v1.2.3-54-g00ecf From 2b752a4929fe8a38cd4c7bb6f1f66b5bc4759f20 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Fri, 9 Apr 2021 00:19:48 +0300 Subject: make _widened_hostnames public, move to urlutils --- qutebrowser/components/hostblock.py | 5 ++--- qutebrowser/config/configutils.py | 13 ++----------- qutebrowser/utils/urlutils.py | 11 ++++++++++- tests/unit/config/test_configutils.py | 23 ----------------------- tests/unit/utils/test_urlutils.py | 23 +++++++++++++++++++++++ 5 files changed, 37 insertions(+), 38 deletions(-) diff --git a/qutebrowser/components/hostblock.py b/qutebrowser/components/hostblock.py index e19536744..64c23da3e 100644 --- a/qutebrowser/components/hostblock.py +++ b/qutebrowser/components/hostblock.py @@ -37,8 +37,7 @@ from qutebrowser.api import ( qtutils, ) from qutebrowser.components.utils import blockutils -from qutebrowser.config.configutils import _widened_hostnames -from qutebrowser.utils import version # FIXME: Move needed parts into api namespace? +from qutebrowser.utils import urlutils, version # FIXME: Move needed parts into api namespace? logger = logging.getLogger("network") @@ -130,7 +129,7 @@ class HostBlocker: host = request_url.host() - for hostname in _widened_hostnames(host): + for hostname in urlutils.widened_hostnames(host): if hostname in self._blocked_hosts or hostname in self._config_blocked_hosts: return True diff --git a/qutebrowser/config/configutils.py b/qutebrowser/config/configutils.py index d619eb21f..28d4a3fe3 100644 --- a/qutebrowser/config/configutils.py +++ b/qutebrowser/config/configutils.py @@ -32,22 +32,13 @@ from PyQt5.QtCore import QUrl from PyQt5.QtGui import QFontDatabase from PyQt5.QtWidgets import QApplication -from qutebrowser.utils import utils, urlmatch, usertypes, qtutils +from qutebrowser.utils import utils, urlmatch, urlutils, usertypes, qtutils from qutebrowser.config import configexc if TYPE_CHECKING: from qutebrowser.config import configdata -def _widened_hostnames(hostname: str) -> Iterable[str]: - """A generator for widening string hostnames. - - Ex: a.c.foo -> [a.c.foo, c.foo, foo]""" - while hostname: - yield hostname - hostname = hostname.partition(".")[-1] - - class ScopedValue: """A configuration value which is valid for a UrlPattern. @@ -231,7 +222,7 @@ class Values: candidates: List[ScopedValue] = [] # Urls trailing with '.' are equivalent to non-trailing types. # urlutils strips them, so in order to match we will need to as well. - widened_hosts = _widened_hostnames(url.host().rstrip('.')) + widened_hosts = urlutils.widened_hostnames(url.host().rstrip('.')) # We must check the 'None' key as well, in case any patterns that # did not have a domain match. for host in itertools.chain(widened_hosts, [None]): diff --git a/qutebrowser/utils/urlutils.py b/qutebrowser/utils/urlutils.py index 045981680..002f10411 100644 --- a/qutebrowser/utils/urlutils.py +++ b/qutebrowser/utils/urlutils.py @@ -26,7 +26,7 @@ import ipaddress import posixpath import urllib.parse import mimetypes -from typing import Optional, Tuple, Union +from typing import Optional, Tuple, Union, Iterable from PyQt5.QtCore import QUrl from PyQt5.QtNetwork import QHostInfo, QHostAddress, QNetworkProxy @@ -619,3 +619,12 @@ def parse_javascript_url(url: QUrl) -> str: raise Error("Resulted in empty JavaScript code") return code + + +def widened_hostnames(hostname: str) -> Iterable[str]: + """A generator for widening string hostnames. + + Ex: a.c.foo -> [a.c.foo, c.foo, foo]""" + while hostname: + yield hostname + hostname = hostname.partition(".")[-1] diff --git a/tests/unit/config/test_configutils.py b/tests/unit/config/test_configutils.py index 4d3082a92..725a38c28 100644 --- a/tests/unit/config/test_configutils.py +++ b/tests/unit/config/test_configutils.py @@ -300,29 +300,6 @@ def test_domain_lookup_sparse_benchmark(url, values, benchmark): benchmark(lambda: values.get_for_url(url)) -class TestWiden: - - @pytest.mark.parametrize('hostname, expected', [ - ('a.b.c', ['a.b.c', 'b.c', 'c']), - ('foobarbaz', ['foobarbaz']), - ('', []), - ('.c', ['.c', 'c']), - ('c.', ['c.']), - ('.c.', ['.c.', 'c.']), - (None, []), - ]) - def test_widen_hostnames(self, hostname, expected): - assert list(configutils._widened_hostnames(hostname)) == expected - - @pytest.mark.parametrize('hostname', [ - 'test.qutebrowser.org', - 'a.b.c.d.e.f.g.h.i.j.k.l.m.n.o.p.q.r.s.t.u.v.w.z.y.z', - 'qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq.c', - ]) - def test_bench_widen_hostnames(self, hostname, benchmark): - benchmark(lambda: list(configutils._widened_hostnames(hostname))) - - class TestFontFamilies: @pytest.mark.parametrize('family_str, expected', [ diff --git a/tests/unit/utils/test_urlutils.py b/tests/unit/utils/test_urlutils.py index a5599c6c9..97ff268ca 100644 --- a/tests/unit/utils/test_urlutils.py +++ b/tests/unit/utils/test_urlutils.py @@ -778,3 +778,26 @@ class TestParseJavascriptUrl: pass else: assert parsed == source + + +class TestWiden: + + @pytest.mark.parametrize('hostname, expected', [ + ('a.b.c', ['a.b.c', 'b.c', 'c']), + ('foobarbaz', ['foobarbaz']), + ('', []), + ('.c', ['.c', 'c']), + ('c.', ['c.']), + ('.c.', ['.c.', 'c.']), + (None, []), + ]) + def test_widen_hostnames(self, hostname, expected): + assert list(urlutils.widened_hostnames(hostname)) == expected + + @pytest.mark.parametrize('hostname', [ + 'test.qutebrowser.org', + 'a.b.c.d.e.f.g.h.i.j.k.l.m.n.o.p.q.r.s.t.u.v.w.z.y.z', + 'qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq.c', + ]) + def test_bench_widen_hostnames(self, hostname, benchmark): + benchmark(lambda: list(urlutils.widened_hostnames(hostname))) -- cgit v1.2.3-54-g00ecf From e3eb603f798a4151d32690a6ec4f18bf0906daad Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Fri, 9 Apr 2021 00:28:53 +0300 Subject: fix pylint --- qutebrowser/components/hostblock.py | 8 ++++++-- qutebrowser/config/configutils.py | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/qutebrowser/components/hostblock.py b/qutebrowser/components/hostblock.py index 64c23da3e..0e7278d1b 100644 --- a/qutebrowser/components/hostblock.py +++ b/qutebrowser/components/hostblock.py @@ -37,7 +37,10 @@ from qutebrowser.api import ( qtutils, ) from qutebrowser.components.utils import blockutils -from qutebrowser.utils import urlutils, version # FIXME: Move needed parts into api namespace? +from qutebrowser.utils import ( # FIXME: Move needed parts into api namespace? + urlutils, + version +) logger = logging.getLogger("network") @@ -130,7 +133,8 @@ class HostBlocker: host = request_url.host() for hostname in urlutils.widened_hostnames(host): - if hostname in self._blocked_hosts or hostname in self._config_blocked_hosts: + if hostname in self._blocked_hosts \ + or hostname in self._config_blocked_hosts: return True return False diff --git a/qutebrowser/config/configutils.py b/qutebrowser/config/configutils.py index 28d4a3fe3..480bbd85f 100644 --- a/qutebrowser/config/configutils.py +++ b/qutebrowser/config/configutils.py @@ -25,7 +25,7 @@ import collections import itertools import operator from typing import ( - TYPE_CHECKING, Any, Dict, Iterable, Iterator, List, Optional, Sequence, Set, Union, + TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Sequence, Set, Union, MutableMapping) from PyQt5.QtCore import QUrl -- cgit v1.2.3-54-g00ecf From 4449fe914c93d95788fffa1073c9e862a18f2d4d Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Fri, 9 Apr 2021 01:22:19 +0300 Subject: fix test --- tests/unit/components/test_hostblock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/components/test_hostblock.py b/tests/unit/components/test_hostblock.py index 8dd8d6dda..9344e1160 100644 --- a/tests/unit/components/test_hostblock.py +++ b/tests/unit/components/test_hostblock.py @@ -279,7 +279,7 @@ def test_disabled_blocking_per_url(config_stub, host_blocker_factory): pattern = urlmatch.UrlPattern(example_com) config_stub.set_obj("content.blocking.enabled", False, pattern=pattern) - url = QUrl("blocked.example.com") + url = QUrl("https://blocked.example.com") host_blocker = host_blocker_factory() host_blocker._blocked_hosts.add(url.host()) -- cgit v1.2.3-54-g00ecf From 26b84e1c9502d4178c0d0dbcf07a8f8eca8606cb Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Fri, 9 Apr 2021 22:57:41 +0300 Subject: add subdomain blocking test --- tests/unit/components/test_hostblock.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/unit/components/test_hostblock.py b/tests/unit/components/test_hostblock.py index 9344e1160..00a7a5f8f 100644 --- a/tests/unit/components/test_hostblock.py +++ b/tests/unit/components/test_hostblock.py @@ -563,3 +563,11 @@ def test_adblock_benchmark(data_tmpdir, benchmark, host_blocker_factory): assert blocker._blocked_hosts benchmark(lambda: blocker._is_blocked(url)) + + +def test_subdomain_blocking(config_stub, host_blocker_factory): + config_stub.val.content.blocking.method = "hosts" + config_stub.val.content.blocking.hosts.lists = None + host_blocker = host_blocker_factory() + host_blocker._blocked_hosts.add("example.com") + assert host_blocker._is_blocked(QUrl("https://subdomain.example.com")) -- cgit v1.2.3-54-g00ecf