diff options
97 files changed, 3235 insertions, 1150 deletions
@@ -37,6 +37,7 @@ exclude = .*,__pycache__,resources.py # (numpy-style) # D413: Missing blank line after last section (not in pep257?) # A003: Builtin name for class attribute (needed for overridden methods) +# W503: like break before binary operator # W504: line break after binary operator # FI15: __future__ import "generator_stop" missing ignore = @@ -47,7 +48,7 @@ ignore = P101,P102,P103, D102,D103,D106,D107,D104,D105,D209,D211,D401,D402,D403,D412,D413, A003, - W504, + W503, W504 FI15 min-version = 3.6.0 max-complexity = 12 @@ -46,6 +46,7 @@ disable=locally-disabled, too-many-statements, too-few-public-methods, import-outside-toplevel, + bad-continuation # This lint disagrees with Black [BASIC] function-rgx=[a-z_][a-z0-9_]{2,50}$ diff --git a/README.asciidoc b/README.asciidoc index 5c254959e..42013368c 100644 --- a/README.asciidoc +++ b/README.asciidoc @@ -44,46 +44,11 @@ See the https://github.com/qutebrowser/qutebrowser/releases[github releases page] for available downloads and the link:doc/install.asciidoc[INSTALL] file for detailed instructions on how to get qutebrowser running on various platforms. -Documentation -------------- - -In addition to the topics mentioned in this README, the following documents are -available: - -* https://raw.githubusercontent.com/qutebrowser/qutebrowser/master/doc/img/cheatsheet-big.png[Key binding cheatsheet]: + -image:https://raw.githubusercontent.com/qutebrowser/qutebrowser/master/doc/img/cheatsheet-small.png["qutebrowser key binding cheatsheet",link="https://raw.githubusercontent.com/qutebrowser/qutebrowser/master/doc/img/cheatsheet-big.png"] -* link:doc/quickstart.asciidoc[Quick start guide] -* https://www.shortcutfoo.com/app/dojos/qutebrowser[Free training course] to remember those key bindings -* link:doc/faq.asciidoc[Frequently asked questions] -* link:doc/help/configuring.asciidoc[Configuring qutebrowser] -* link:doc/contributing.asciidoc[Contributing to qutebrowser] -* link:doc/install.asciidoc[Installing qutebrowser] -* link:doc/changelog.asciidoc[Change Log] -* link:doc/stacktrace.asciidoc[Reporting segfaults] -* link:doc/userscripts.asciidoc[How to write userscripts] - -Getting help ------------- - -You can get help in the IRC channel -irc://irc.freenode.org/#qutebrowser[`#qutebrowser`] on -https://freenode.net/[Freenode] -(https://webchat.freenode.net/?channels=#qutebrowser[webchat]), or by writing a -message to the -https://lists.schokokeks.org/mailman/listinfo.cgi/qutebrowser[mailinglist] at -mailto:qutebrowser@lists.qutebrowser.org[]. - -There's also an https://lists.schokokeks.org/mailman/listinfo.cgi/qutebrowser-announce[announce-only mailinglist] -at mailto:qutebrowser-announce@lists.qutebrowser.org[] (the announcements also -get sent to the general qutebrowser@ list). - -If you're a reddit user, there's a -https://www.reddit.com/r/qutebrowser/[/r/qutebrowser] subreddit there. +Documentation and getting help +------------------------------ -Finally, qutebrowser is participating in the Beta for GitHub's new Discussions -feature, so you can also use the -https://github.com/qutebrowser/qutebrowser/discussions[discussions tab] on -GitHub to get in touch. +Please see the link:doc/help/index.asciidoc[help page] for available documentation +pages and support channels. Contributions / Bugs -------------------- @@ -137,6 +102,7 @@ The following software and libraries are required to run qutebrowser: The following libraries are optional: +* https://pypi.org/project/adblock/[adblock] (for improved adblocking using ABP syntax) * On Windows, https://pypi.python.org/pypi/colorama/[colorama] for colored log output. * http://asciidoc.org/[asciidoc] to generate the documentation for the `:help` diff --git a/doc/changelog.asciidoc b/doc/changelog.asciidoc index 5a0ce3938..5bd8778f7 100644 --- a/doc/changelog.asciidoc +++ b/doc/changelog.asciidoc @@ -35,6 +35,8 @@ Major changes at the time of writing, it's recommended to https://github.com/qutebrowser/qutebrowser/blob/master/doc/install.asciidoc#installing-qutebrowser-with-virtualenv[install qutebrowser in a virtualenv] with a newer version of Qt/PyQt. +- New optional dependency on the Python `adblock` library, which is now used to + integrate Brave's Rust adblocker library, if the `adblock` module is available. - Windows 7 is not supported anymore by the Windows binaries. - The (formerly optional) `cssutils` dependency is now removed. It was only needed for improved behavior in corner cases when using `:download --mhtml` @@ -66,6 +68,9 @@ Removed Added ~~~~~ +- New settings for the ABP-based adblocker: + * `content.blocking.method` to decide which blocker(s) should be used. + * `content.blocking.adblock.lists` to configure ABP-like lists to use. - When QtWebEngine has been updated but PyQtWebEngine hasn't yet, the dark mode settings might stop working. As a (currently undocumented) escape hatch, this version adds a `QUTE_DARKMODE_VARIANT=qt_515_2` environment variable which can @@ -76,6 +81,8 @@ Added - New userscripts: - `kodi` to play videos in Kodi - `qr` to generate a QR code of the current URL + - `add-nextcloud-bookmarks` to create bookmarks in Nextcloud's Bookmarks app + - `add-nextcloud-cookbook` to add recipes to Nextcloud's Cookbook app Changed ~~~~~~~ @@ -105,14 +112,29 @@ Changed `~/.local/share/qutebrowser/`). - The `:later` command now understands a time specification like `5m` or `1h5m2s`, rather than just taking milliseconds. +- The `importer.py` script doesn't use a browser argument anymore; instead its + `--input-format` switch can be used to configure the input format. The help also + was expanded to explain how to use it properly. +- If `tabs.tabs_are_windows` is set, the `tabs.last_close` setting is now + ignored and the window is always closed when using `:close` (`d`). +- Various host-blocking settings have been renamed to accomodate the new ABP-like + adblocker: + * `content.host_blocking.enabled` -> `content.blocking.enabled` (controlling both blockers) + * `content.host_blocking.whitelist` -> `content.blocking.whitelist` (controlling both blockers) + * `content.host_blocking.lists` -> `content.blocking.hosts.lists` Fixed ~~~~~ - With interpolated color settings (`colors.tabs.indicator.*` and `colors.downloads.*`), the alpha channel is now handled correctly. -- The `format_json` userscript now uses `env` in its shebang, making it work - correctly on systems where `bash` isn't located in `/bin`. +- Fixes to userscripts: + * `format_json` now uses `env` in its shebang, making it work + correctly on systems where `bash` isn't located in `/bin`. + * `qute-pass` now handles the MIME output format introduced in gopass 1.10.0. + * `qute-lastpass` now types multiple `<` or `>` characters correctly. +- The `:undo` completion now sorts its entries correctly (by the numerical index + rather than lexicographically). - TODO: Due to a long-standing bug in the `pkg_resources` dependency, it caused qutebrowser's startup to slow down by around 150ms-1s (heavily depending on the system). Since the dependency is now removed, qutebrowser's startup time diff --git a/doc/faq.asciidoc b/doc/faq.asciidoc index 39df56faa..16a791975 100644 --- a/doc/faq.asciidoc +++ b/doc/faq.asciidoc @@ -474,7 +474,7 @@ Can you share details on the swag?:: + image:https://qutebrowser.org/img/sponsors/swag.jpg["swag",width=300,link="https://qutebrowser.org/img/sponsors/swag.jpg"] + -It's planned to order more swag, depending on the exact demand. Possibilites +It's planned to order more swag, depending on the exact demand. Possibilities would include: + - qutebrowser pens (refillable) diff --git a/doc/help/commands.asciidoc b/doc/help/commands.asciidoc index 5e3395931..eb8e4925d 100644 --- a/doc/help/commands.asciidoc +++ b/doc/help/commands.asciidoc @@ -29,7 +29,7 @@ possible to run or bind multiple commands by separating them with `;;`. [options="header",width="75%",cols="25%,75%"] |============== |Command|Description -|<<adblock-update,adblock-update>>|Update the adblock block lists. +|<<adblock-update,adblock-update>>|Update block lists for both the host- and the Brave ad blocker. |<<back,back>>|Go back in the history of the current tab. |<<bind,bind>>|Bind a key to a command. |<<bookmark-add,bookmark-add>>|Save the current page as a bookmark, or a specific url. @@ -139,9 +139,7 @@ possible to run or bind multiple commands by separating them with `;;`. |============== [[adblock-update]] === adblock-update -Update the adblock block lists. - -This updates `~/.local/share/qutebrowser/blocked-hosts` with downloaded host lists and re-reads `~/.config/qutebrowser/blocked-hosts`. +Update block lists for both the host- and the Brave ad blocker. [[back]] === back @@ -290,7 +288,7 @@ Set all settings back to their default. [[config-cycle]] === config-cycle -Syntax: +:config-cycle [*--pattern* 'pattern'] [*--temp*] [*--print*] 'option' ['values' ['values' ...]]+ +Syntax: +:config-cycle [*--pattern* 'pattern'] [*--temp*] [*--print*] 'option' ['values' ...]+ Cycle an option between multiple values. @@ -625,7 +623,7 @@ Show help about a command or setting. [[hint]] === hint -Syntax: +:hint [*--mode* 'mode'] [*--add-history*] [*--rapid*] [*--first*] ['group'] ['target'] ['args' ['args' ...]]+ +Syntax: +:hint [*--mode* 'mode'] [*--add-history*] [*--rapid*] [*--first*] ['group'] ['target'] ['args' ...]+ Start hinting. @@ -1088,7 +1086,7 @@ The count that run_with_count itself received. [[save]] === save -Syntax: +:save ['what' ['what' ...]]+ +Syntax: +:save ['what' ...]+ Save configs and state. diff --git a/doc/help/configuring.asciidoc b/doc/help/configuring.asciidoc index 89866ccce..35a0fbb62 100644 --- a/doc/help/configuring.asciidoc +++ b/doc/help/configuring.asciidoc @@ -116,7 +116,7 @@ accepted values depend on the type of the option. Commonly used are: - Booleans: `c.completion.shrink = True` - Integers: `c.messages.timeout = 5000` - Dictionaries: - * `c.headers.custom = {'X-Hello': 'World', 'X-Awesome': 'yes'}` to override + * `c.content.headers.custom = {'X-Hello': 'World', 'X-Awesome': 'yes'}` to override any other values in the dictionary. * `c.aliases['foo'] = 'message-info foo'` to add a single value. - Lists: diff --git a/doc/help/index.asciidoc b/doc/help/index.asciidoc index 7b6efa490..7424e1a65 100644 --- a/doc/help/index.asciidoc +++ b/doc/help/index.asciidoc @@ -6,6 +6,7 @@ Documentation The following help pages are currently available: +* https://raw.githubusercontent.com/qutebrowser/qutebrowser/master/doc/img/cheatsheet-big.png[Key binding cheatsheet (hosted on GitHub)] * link:../quickstart{outfilesuffix}[Quick start guide] * link:../faq{outfilesuffix}[Frequently asked questions] * link:../changelog{outfilesuffix}[Change Log] @@ -14,6 +15,8 @@ The following help pages are currently available: * link:settings{outfilesuffix}[Documentation of settings] * link:../userscripts{outfilesuffix}[How to write userscripts] * link:../contributing{outfilesuffix}[Contributing to qutebrowser] +* link:../install{outfilesuffix}[Installing qutebrowser] +* link:../stacktrace{outfilesuffix}[Reporting segfaults] Getting help ------------ @@ -26,6 +29,17 @@ message to the https://lists.schokokeks.org/mailman/listinfo.cgi/qutebrowser[mailinglist] at mailto:qutebrowser@lists.qutebrowser.org[]. +There's also an https://lists.schokokeks.org/mailman/listinfo.cgi/qutebrowser-announce[announce-only mailinglist] +at mailto:qutebrowser-announce@lists.qutebrowser.org[] (the announcements also +get sent to the general qutebrowser@ list). + +If you're a reddit user, there's a +https://www.reddit.com/r/qutebrowser/[/r/qutebrowser] subreddit there. + +Finally, qutebrowser is using GitHub's new Discussions feature, so you can also use the +https://github.com/qutebrowser/qutebrowser/discussions[discussions tab] on GitHub to get +in touch. + Bugs ---- diff --git a/doc/help/settings.asciidoc b/doc/help/settings.asciidoc index 309f1ab1d..b1666b2c2 100644 --- a/doc/help/settings.asciidoc +++ b/doc/help/settings.asciidoc @@ -137,6 +137,11 @@ |<<completion.web_history.max_items,completion.web_history.max_items>>|Number of URLs to show in the web history. |<<confirm_quit,confirm_quit>>|Require a confirmation before quitting the application. |<<content.autoplay,content.autoplay>>|Automatically start playing `<video>` elements. +|<<content.blocking.adblock.lists,content.blocking.adblock.lists>>|List of URLs to ABP-style adblocking rulesets. +|<<content.blocking.enabled,content.blocking.enabled>>|Enable the ad/host blocker +|<<content.blocking.hosts.lists,content.blocking.hosts.lists>>|List of URLs to host blocklists for the host blocker. +|<<content.blocking.method,content.blocking.method>>|Which method of blocking ads should be used. +|<<content.blocking.whitelist,content.blocking.whitelist>>|A list of patterns that should always be loaded, despite being blocked by the ad-/host-blocker. |<<content.cache.appcache,content.cache.appcache>>|Enable support for the HTML 5 web application cache feature. |<<content.cache.maximum_pages,content.cache.maximum_pages>>|Maximum number of pages to hold in the global memory page cache. |<<content.cache.size,content.cache.size>>|Size (in bytes) of the HTTP network cache. Null to use the default value. @@ -155,9 +160,6 @@ |<<content.headers.do_not_track,content.headers.do_not_track>>|Value to send in the `DNT` header. |<<content.headers.referer,content.headers.referer>>|When to send the Referer header. |<<content.headers.user_agent,content.headers.user_agent>>|User agent to send. -|<<content.host_blocking.enabled,content.host_blocking.enabled>>|Enable host blocking. -|<<content.host_blocking.lists,content.host_blocking.lists>>|List of URLs of lists which contain hosts to block. -|<<content.host_blocking.whitelist,content.host_blocking.whitelist>>|A list of patterns that should always be loaded, despite being ad-blocked. |<<content.hyperlink_auditing,content.hyperlink_auditing>>|Enable hyperlink auditing (`<a ping>`). |<<content.images,content.images>>|Load images automatically in web pages. |<<content.javascript.alert,content.javascript.alert>>|Show javascript alerts. @@ -1903,6 +1905,85 @@ Default: +pass:[true]+ This setting is only available with the QtWebEngine backend. +[[content.blocking.adblock.lists]] +=== content.blocking.adblock.lists +List of URLs to ABP-style adblocking rulesets. + +Only used when Brave's ABP-style adblocker is used (see `content.blocking.method`). + + +Type: <<types,List of Url>> + +Default: + +- +pass:[https://easylist.to/easylist/easylist.txt]+ +- +pass:[https://easylist.to/easylist/easyprivacy.txt]+ + +[[content.blocking.enabled]] +=== content.blocking.enabled +Enable the ad/host blocker + +This setting supports URL patterns. + +Type: <<types,Bool>> + +Default: +pass:[true]+ + +[[content.blocking.hosts.lists]] +=== content.blocking.hosts.lists +List of URLs to host blocklists for the host blocker. + +Only used when the simple host-blocker is used (see `content.blocking.method`). + +The file can be in one of the following formats: + +- An `/etc/hosts`-like file +- One host per line +- A zip-file of any of the above, with either only one file, or a file + named `hosts` (with any extension). + +It's also possible to add a local file or directory via a `file://` URL. In +case of a directory, all files in the directory are read as adblock lists. + +The file `~/.config/qutebrowser/blocked-hosts` is always read if it exists. + + +Type: <<types,List of Url>> + +Default: + +- +pass:[https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts]+ + +[[content.blocking.method]] +=== content.blocking.method +Which method of blocking ads should be used. + +Support for Adblock Plus (ABP) syntax blocklists using Brave's Rust library requires +the `adblock` Python package to be installed, which is an optional dependency of +qutebrowser. It is required when either `adblock` or `both` are selected. + + +Type: <<types,String>> + +Valid values: + + * +auto+: Use Brave's ABP-style adblocker if available, host blocking otherwise + * +adblock+: Use Brave's ABP-style adblocker + * +hosts+: Use hosts blocking + * +both+: Use both hosts blocking and Brave's ABP-style adblocker + +Default: +pass:[auto]+ + +[[content.blocking.whitelist]] +=== content.blocking.whitelist +A list of patterns that should always be loaded, despite being blocked by the ad-/host-blocker. +Local domains are always exempt from adblocking. +Note this whitelists otherwise blocked requests, not first-party URLs. As an example, if `example.org` loads an ad from `ads.example.org`, the whitelist entry could be `https://ads.example.org/*`. If you want to disable the adblocker on a given page, use the `content.blocking.enabled` setting with a URL pattern instead. + +Type: <<types,List of UrlPattern>> + +Default: empty + [[content.cache.appcache]] === content.cache.appcache Enable support for the HTML 5 web application cache feature. @@ -2141,49 +2222,6 @@ Type: <<types,FormatString>> Default: +pass:[Mozilla/5.0 ({os_info}) AppleWebKit/{webkit_version} (KHTML, like Gecko) {qt_key}/{qt_version} {upstream_browser_key}/{upstream_browser_version} Safari/{webkit_version}]+ -[[content.host_blocking.enabled]] -=== content.host_blocking.enabled -Enable host blocking. - -This setting supports URL patterns. - -Type: <<types,Bool>> - -Default: +pass:[true]+ - -[[content.host_blocking.lists]] -=== content.host_blocking.lists -List of URLs of lists which contain hosts to block. - -The file can be in one of the following formats: - -- An `/etc/hosts`-like file -- One host per line -- A zip-file of any of the above, with either only one file, or a file - named `hosts` (with any extension). - -It's also possible to add a local file or directory via a `file://` URL. In -case of a directory, all files in the directory are read as adblock lists. - -The file `~/.config/qutebrowser/blocked-hosts` is always read if it exists. - - -Type: <<types,List of Url>> - -Default: - -- +pass:[https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts]+ - -[[content.host_blocking.whitelist]] -=== content.host_blocking.whitelist -A list of patterns that should always be loaded, despite being ad-blocked. -Note this whitelists blocked hosts, not first-party URLs. As an example, if `example.org` loads an ad from `ads.example.org`, the whitelisted host should be `ads.example.org`. If you want to disable the adblocker on a given page, use the `content.host_blocking.enabled` setting with a URL pattern instead. -Local domains are always exempt from hostblocking. - -Type: <<types,List of UrlPattern>> - -Default: empty - [[content.hyperlink_auditing]] === content.hyperlink_auditing Enable hyperlink auditing (`<a ping>`). @@ -3812,6 +3850,7 @@ Default: +pass:[3]+ [[tabs.last_close]] === tabs.last_close How to behave when the last tab is closed. +If the `tabs.tabs_are_windows` setting is set, this is ignored and the behavior is always identical to the `close` value. Type: <<types,String>> diff --git a/doc/install.asciidoc b/doc/install.asciidoc index f7a3d8a60..74b18a6cf 100644 --- a/doc/install.asciidoc +++ b/doc/install.asciidoc @@ -432,11 +432,10 @@ You can specify a Qt/PyQt version with the `--pyqt-version` flag, see `mkenv.py --help` for a list of available versions. By default, the latest version which plays well with qutebrowser is used. -NOTE: If qutebrowser fails to start with a _"This application failed to start +NOTE: If the Qt smoke test fails with a _"This application failed to start because no Qt platform plugin could be initialized."_ message, most likely a -system-wide library is missing. Run qutebrowser again after -`export QT_DEBUG_PLUGINS=1` and keep attention to a -_QLibraryPrivate::loadPlugin failed on ..._ line for details. +system-wide library is missing. Pay attention to a _QLibraryPrivate::loadPlugin +failed on ..._ line for details. Installing dependencies (system-wide Qt) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -487,5 +486,6 @@ Updating When you updated your local copy of the code (e.g. by pulling the git repo, or extracting a new version), the virtualenv should automatically use the updated -code. However, dependencies won't be updated that way. Re-running `mkvenv.py` -will recreate the virtualenv with updated dependencies. +code. However, dependencies won't be updated that way. Thus, it's recommended +to run `mkvenv.py --update` instead, which will run `git pull` and recreate the +virtualenv with updated dependencies. diff --git a/doc/userscripts.asciidoc b/doc/userscripts.asciidoc index 9bbc68ce0..e97a951c4 100644 --- a/doc/userscripts.asciidoc +++ b/doc/userscripts.asciidoc @@ -38,7 +38,11 @@ The following environment variables will be set when a userscript is launched: - `QUTE_CONFIG_DIR`: Path of the directory containing qutebrowser's configuration. - `QUTE_DATA_DIR`: Path of the directory containing qutebrowser's data. - `QUTE_DOWNLOAD_DIR`: Path of the downloads directory. -- `QUTE_COMMANDLINE_TEXT`: Text currently in qutebrowser's command line. +- `QUTE_COMMANDLINE_TEXT`: Text currently in qutebrowser's command line. Note + this is only useful for userscripts spawned (e.g. via a keybinding) when + qutebrowser is still in command mode. If you want to receive arguments passed + to your userscript via `:spawn`, use the normal way of getting commandline + arguments (e.g. `$@` in bash or `sys.argv` / `argparse` / ... in Python). In `command` mode: diff --git a/misc/requirements/requirements-dev.txt b/misc/requirements/requirements-dev.txt index f58af8072..2dd1e96b9 100644 --- a/misc/requirements/requirements-dev.txt +++ b/misc/requirements/requirements-dev.txt @@ -3,7 +3,7 @@ bump2version==1.0.1 certifi==2020.12.5 cffi==1.14.4 -chardet==3.0.4 +chardet==4.0.0 colorama==0.4.4 cryptography==3.3.1 github3.py==1.3.0 @@ -17,7 +17,7 @@ Pympler==0.9 pyparsing==2.4.7 PyQt-builder==1.6.0 python-dateutil==2.8.1 -requests==2.25.0 +requests==2.25.1 sip==5.5.0 six==1.15.0 toml==0.10.2 diff --git a/misc/requirements/requirements-flake8.txt b/misc/requirements/requirements-flake8.txt index 6ed02ad61..d795b98c3 100644 --- a/misc/requirements/requirements-flake8.txt +++ b/misc/requirements/requirements-flake8.txt @@ -4,7 +4,7 @@ attrs==20.3.0 flake8==3.8.4 flake8-bugbear==20.11.1 flake8-builtins==1.5.3 -flake8-comprehensions==3.3.0 +flake8-comprehensions==3.3.1 flake8-copyright==0.2.2 flake8-debugger==4.0.0 flake8-deprecated==1.3 @@ -13,7 +13,7 @@ flake8-future-import==0.4.6 flake8-mock==0.3 flake8-polyfill==1.0.2 flake8-string-format==0.3.0 -flake8-tidy-imports==4.2.0 +flake8-tidy-imports==4.2.1 flake8-tuple==0.4.1 mccabe==0.6.1 pep8-naming==0.11.1 diff --git a/misc/requirements/requirements-mypy.txt b/misc/requirements/requirements-mypy.txt index d640851c9..3eae67aa8 100644 --- a/misc/requirements/requirements-mypy.txt +++ b/misc/requirements/requirements-mypy.txt @@ -1,6 +1,7 @@ # This file is automatically generated by scripts/dev/recompile_requirements.py diff-cover==4.0.1 +importlib-resources==4.1.1 inflect==5.0.2 Jinja2==2.11.2 jinja2-pluralize==0.3.0 @@ -10,6 +11,6 @@ mypy==0.790 mypy-extensions==0.4.3 pluggy==0.13.1 Pygments==2.7.3 --e git+https://github.com/stlehmann/PyQt5-stubs.git@704207e90bee7b36ec9861dfa6b39f06a27c6718#egg=PyQt5_stubs +-e git+https://github.com/stlehmann/PyQt5-stubs.git@e9a2f2d9f5fd8d6a2665b0b0be2f1e96db3b58eb#egg=PyQt5_stubs typed-ast==1.4.1 typing-extensions==3.7.4.3 diff --git a/misc/requirements/requirements-mypy.txt-raw b/misc/requirements/requirements-mypy.txt-raw index 7c888ed1e..edc4b761a 100644 --- a/misc/requirements/requirements-mypy.txt-raw +++ b/misc/requirements/requirements-mypy.txt-raw @@ -1,4 +1,5 @@ mypy lxml # For HTML reports diff-cover +importlib_resources # So stubs are available even on newer Python versions -e git+https://github.com/stlehmann/PyQt5-stubs.git@master#egg=PyQt5-stubs diff --git a/misc/requirements/requirements-pyinstaller.txt b/misc/requirements/requirements-pyinstaller.txt index b1a3e98ee..5ddac7a6c 100644 --- a/misc/requirements/requirements-pyinstaller.txt +++ b/misc/requirements/requirements-pyinstaller.txt @@ -2,4 +2,4 @@ altgraph==0.17 pyinstaller==4.1 -pyinstaller-hooks-contrib==2020.10 +pyinstaller-hooks-contrib==2020.11 diff --git a/misc/requirements/requirements-pylint.txt b/misc/requirements/requirements-pylint.txt index 02165c497..034ec64d2 100644 --- a/misc/requirements/requirements-pylint.txt +++ b/misc/requirements/requirements-pylint.txt @@ -3,7 +3,7 @@ astroid==2.3.3 # rq.filter: < 2.4 certifi==2020.12.5 cffi==1.14.4 -chardet==3.0.4 +chardet==4.0.0 cryptography==3.3.1 github3.py==1.3.0 idna==2.10 @@ -15,7 +15,7 @@ pycparser==2.20 pylint==2.4.4 # rq.filter: < 2.5 python-dateutil==2.8.1 ./scripts/dev/pylint_checkers -requests==2.25.0 +requests==2.25.1 six==1.15.0 typed-ast==1.4.1 ; python_version<"3.8" uritemplate==3.0.1 diff --git a/misc/requirements/requirements-qutebrowser.txt-raw b/misc/requirements/requirements-qutebrowser.txt-raw index a01e56712..2d527aeef 100644 --- a/misc/requirements/requirements-qutebrowser.txt-raw +++ b/misc/requirements/requirements-qutebrowser.txt-raw @@ -4,6 +4,7 @@ pyPEG2 PyYAML colorama attrs +adblock # Optional, for improved adblocking importlib-resources #@ markers: importlib-resources python_version<"3.9" diff --git a/misc/requirements/requirements-sphinx.txt b/misc/requirements/requirements-sphinx.txt index 2cbf8d57d..54eb185bc 100644 --- a/misc/requirements/requirements-sphinx.txt +++ b/misc/requirements/requirements-sphinx.txt @@ -3,7 +3,7 @@ alabaster==0.7.12 Babel==2.9.0 certifi==2020.12.5 -chardet==3.0.4 +chardet==4.0.0 docutils==0.16 idna==2.10 imagesize==1.2.0 @@ -12,10 +12,10 @@ MarkupSafe==1.1.1 packaging==20.8 Pygments==2.7.3 pyparsing==2.4.7 -pytz==2020.4 -requests==2.25.0 +pytz==2020.5 +requests==2.25.1 snowballstemmer==2.0.0 -Sphinx==3.3.1 +Sphinx==3.4.1 sphinxcontrib-applehelp==1.0.2 sphinxcontrib-devhelp==1.0.2 sphinxcontrib-htmlhelp==1.0.3 diff --git a/misc/requirements/requirements-tests.txt b/misc/requirements/requirements-tests.txt index 28fd44126..b61ff1d6b 100644 --- a/misc/requirements/requirements-tests.txt +++ b/misc/requirements/requirements-tests.txt @@ -4,18 +4,18 @@ apipkg==1.5 attrs==20.3.0 beautifulsoup4==4.9.3 certifi==2020.12.5 -chardet==3.0.4 +chardet==4.0.0 cheroot==8.5.1 click==7.1.2 # colorama==0.4.4 -coverage==5.3 +coverage==5.3.1 EasyProcess==0.3 execnet==1.7.1 filelock==3.0.12 Flask==1.1.2 glob2==0.7 hunter==3.3.1 -hypothesis==5.43.3 +hypothesis==5.43.4 icdiff==1.9.1 idna==2.10 iniconfig==1.1.1 @@ -35,7 +35,7 @@ py==1.10.0 py-cpuinfo==7.0.0 Pygments==2.7.3 pyparsing==2.4.7 -pytest==6.2.0 +pytest==6.2.1 pytest-bdd==4.0.2 pytest-benchmark==3.2.3 pytest-clarity==0.3.0a0 @@ -43,14 +43,14 @@ pytest-cov==2.10.1 pytest-forked==1.3.0 pytest-icdiff==0.5 pytest-instafail==0.4.2 -pytest-mock==3.3.1 +pytest-mock==3.4.0 pytest-qt==3.3.0 pytest-repeat==0.9.1 pytest-rerunfailures==9.1.1 -pytest-xdist==2.1.0 +pytest-xdist==2.2.0 pytest-xvfb==2.0.0 PyVirtualDisplay==1.3.2 -requests==2.25.0 +requests==2.25.1 requests-file==1.5.1 six==1.15.0 sortedcontainers==2.3.0 diff --git a/misc/userscripts/README.md b/misc/userscripts/README.md index 669bfa664..938dd776d 100644 --- a/misc/userscripts/README.md +++ b/misc/userscripts/README.md @@ -35,6 +35,8 @@ The following userscripts are included in the current directory. - [qr](./qr): Show a QR code for the current webpage via [qrencode](https://fukuchi.org/works/qrencode/). - [kodi](./kodi): Play videos in Kodi. +- [add-nextcloud-bookmarks](./add-nextcloud-bookmarks): Create bookmarks in Nextcloud's Bookmarks app. +- [add-nextcloud-cookbook](./add-nextcloud-cookbook): Add recipes to Nextcloud's Cookbook app. [castnow]: https://github.com/xat/castnow [youtube-dl]: https://rg3.github.io/youtube-dl/ diff --git a/misc/userscripts/add-nextcloud-bookmarks b/misc/userscripts/add-nextcloud-bookmarks new file mode 100755 index 000000000..4e66620dc --- /dev/null +++ b/misc/userscripts/add-nextcloud-bookmarks @@ -0,0 +1,171 @@ +#!/usr/bin/env python + +""" +Behavior: + A qutebrowser userscript that creates bookmarks in Nextcloud's Bookmarks app. + +Requirements: + requests + +userscript setup: + Optionally create ~/.config/qutebrowser/add-nextcloud-bookmarks.ini like: + +[nextcloud] +HOST=https://nextcloud.example.com +USER=username +;PASSWORD=lamepassword +DESCRIPTION=None +;TAGS=just-one +TAGS=read-me-later,added-by-qutebrowser, Another-One + + If settings aren't in the configuration file, the user will be prompted during + bookmark creation. If DESCRIPTION and TAGS are set to None, they will be left + blank. If the user does not want to be prompted for a password, it is recommended + to set up an 'app password'. See the following for instructions: + https://docs.nextcloud.com/server/latest/user_manual/en/session_management.html#managing-devices # noqa: E501 + +qutebrowser setup: + add bookmark via hints + config.bind('X', 'hint links userscript add-nextcloud-bookmarks') + + add bookmark of current URL + config.bind('X', 'spawn --userscript add-nextcloud-bookmarks') + +troubleshooting: + Errors detected within this userscript will have an exit of 231. All other + exit codes will come from requests. +""" + +import configparser +from json import dumps +from os import environ, path +from sys import argv, exit + +from PyQt5.QtWidgets import QApplication, QInputDialog, QLineEdit +from requests import get, post +from requests.auth import HTTPBasicAuth + + +def get_text(name, info): + """Get input from the user.""" + _app = QApplication(argv) # noqa: F841 + if name == "password": + text, ok = QInputDialog.getText( + None, + "add-nextcloud-bookmarks userscript", + "Please enter {}".format(info), + QLineEdit.Password, + ) + else: + text, ok = QInputDialog.getText( + None, "add-nextcloud-bookmarks userscript", "Please enter {}".format(info) + ) + if not ok: + message("info", "Dialog box canceled.") + exit(0) + return text + + +def message(level, text): + """display message""" + with open(environ["QUTE_FIFO"], "w") as fifo: + fifo.write( + 'message-{} "add-nextcloud-bookmarks userscript: {}"\n'.format(level, text) + ) + fifo.flush() + + +if "QUTE_FIFO" not in environ: + print( + "This script is designed to run as a qutebrowser userscript, " + "not as a standalone script." + ) + exit(231) + +if "QUTE_CONFIG_DIR" not in environ: + if "XDG_CONFIG_HOME" in environ: + QUTE_CONFIG_DIR = environ["XDG_CONFIG_HOME"] + "/qutebrowser" + else: + QUTE_CONFIG_DIR = environ["HOME"] + "/.config/qutebrowser" +else: + QUTE_CONFIG_DIR = environ["QUTE_CONFIG_DIR"] + +config_file = QUTE_CONFIG_DIR + "/add-nextcloud-bookmarks.ini" +if path.isfile(config_file): + config = configparser.ConfigParser() + config.read(config_file) + settings = dict(config.items("nextcloud")) +else: + settings = {} + +settings_info = [ + ("host", "host information.", "required"), + ("user", "username.", "required"), + ("password", "password.", "required"), + ("description", "description or leave blank", "optional"), + ("tags", "tags (comma separated) or leave blank", "optional"), +] + +# check for settings that need user interaction and clear optional setting if need be +for setting in settings_info: + if setting[0] not in settings: + userInput = get_text(setting[0], setting[1]) + settings[setting[0]] = userInput + if setting[2] == "optional": + if settings[setting[0]] == "None": + settings[setting[0]] = "" + +tags = settings["tags"].split(",") + +QUTE_URL = environ["QUTE_URL"] +api_url = settings["host"] + "/index.php/apps/bookmarks/public/rest/v2/bookmark" + +headers = {"Content-Type": "application/json"} +auth = HTTPBasicAuth(settings["user"], settings["password"]) + +# check if there is already a bookmark for the URL +r = get( + "{}?url={}".format(api_url, QUTE_URL), + auth=auth, + headers=headers, + timeout=(3.05, 27), +) +if r.status_code != 200: + message( + "error", + "Could not connect to {} with status code {}".format( + settings["host"], r.status_code + ), + ) + exit(r.status_code) + +try: + r.json()["data"][0]["id"] +except IndexError: + pass +else: + message("info", "bookmark already exists for {}".format(QUTE_URL)) + exit(0) + +if environ["QUTE_MODE"] == "hints": + QUTE_TITLE = QUTE_URL +else: + QUTE_TITLE = environ["QUTE_TITLE"] + +# JSON format +# https://nextcloud-bookmarks.readthedocs.io/en/latest/bookmark.html#create-a-bookmark +dict = { + "url": QUTE_URL, + "title": QUTE_TITLE, + "description": settings["description"], + "tags": tags, +} +data = dumps(dict) + +r = post(api_url, data=data, headers=headers, auth=auth, timeout=(3.05, 27)) + +if r.status_code == 200: + message("info", "bookmark {} added".format(QUTE_URL)) +else: + message("error", "something went wrong {} bookmark not added".format(QUTE_URL)) + exit(r.status_code) diff --git a/misc/userscripts/add-nextcloud-cookbook b/misc/userscripts/add-nextcloud-cookbook new file mode 100755 index 000000000..a348417e0 --- /dev/null +++ b/misc/userscripts/add-nextcloud-cookbook @@ -0,0 +1,131 @@ +#!/usr/bin/env python + +""" +Behavior: + A qutebrowser userscript that adds recipes to Nextcloud's Cookbook app. + +Requirements: + requests + +userscript setup: + Optionally create ~/.config/qutebrowser/add-nextcloud-cookbook.ini like: + +[nextcloud] +HOST=https://nextcloud.example.com +USER=username +;PASSWORD=lamepassword + + If settings aren't in the configuration file, the user will be prompted. + If the user does not want to be prompted for a password, it is recommended + to set up an 'app password' with 'Allow filesystem access' enabled. + See the following for instructions: + https://docs.nextcloud.com/server/latest/user_manual/en/session_management.html#managing-devices # noqa: E501 + +qutebrowser setup: + add recipe via hints + config.bind('X', 'hint links userscript add-nextcloud-cookbook') + + add recipe of current URL + config.bind('X', 'spawn --userscript add-nextcloud-cookbook') + +troubleshooting: + Errors detected within this userscript will have an exit of 231. All other + exit codes will come from requests. +""" + +import configparser +from os import environ, path +from sys import argv, exit + +from PyQt5.QtWidgets import QApplication, QInputDialog, QLineEdit +from requests import post +from requests.auth import HTTPBasicAuth + + +def get_text(name, info): + """Get input from the user.""" + _app = QApplication(argv) # noqa: F841 + if name == "password": + text, ok = QInputDialog.getText( + None, + "add-nextcloud-cookbook userscript", + "Please enter {}".format(info), + QLineEdit.Password, + ) + else: + text, ok = QInputDialog.getText( + None, "add-nextcloud-cookbook userscript", "Please enter {}".format(info) + ) + if not ok: + message("info", "Dialog box canceled.") + exit(0) + return text + + +def message(level, text): + """display message""" + with open(environ["QUTE_FIFO"], "w") as fifo: + fifo.write( + "message-{} 'add-nextcloud-cookbook userscript: {}'\n".format(level, text) + ) + fifo.flush() + + +if "QUTE_FIFO" not in environ: + print( + "This script is designed to run as a qutebrowser userscript, " + "not as a standalone script." + ) + exit(231) + +if "QUTE_CONFIG_DIR" not in environ: + if "XDG_CONFIG_HOME" in environ: + QUTE_CONFIG_DIR = environ["XDG_CONFIG_HOME"] + "/qutebrowser" + else: + QUTE_CONFIG_DIR = environ["HOME"] + "/.config/qutebrowser" +else: + QUTE_CONFIG_DIR = environ["QUTE_CONFIG_DIR"] + +config_file = QUTE_CONFIG_DIR + "/add-nextcloud-cookbook.ini" +if path.isfile(config_file): + config = configparser.ConfigParser() + config.read(config_file) + settings = dict(config.items("nextcloud")) +else: + settings = {} + +settings_info = [ + ("host", "host information.", "required"), + ("user", "username.", "required"), + ("password", "password.", "required"), +] + +# check for settings that need user interaction +for setting in settings_info: + if setting[0] not in settings: + userInput = get_text(setting[0], setting[1]) + settings[setting[0]] = userInput + +api_url = settings["host"] + "/index.php/apps/cookbook/import" +headers = {"Content-Type": "application/x-www-form-urlencoded"} +auth = HTTPBasicAuth(settings["user"], settings["password"]) +data = "url=" + environ["QUTE_URL"] + +message("info", "starting to process {}".format(environ["QUTE_URL"])) + +r = post(api_url, data=data, headers=headers, auth=auth, timeout=(3.05, 27)) + +if r.status_code == 200: + message("info", "recipe from {} added.".format(environ["QUTE_URL"])) + exit(0) +elif r.status_code == 500: + message("warning", "Cookbook app reports {}".format(r.text)) + exit(0) +else: + message( + "error", + "Could not connect to {} with status code {}".format( + settings["host"], r.status_code + ), + ) + exit(r.status_code) diff --git a/misc/userscripts/qute-lastpass b/misc/userscripts/qute-lastpass index cd584ae3a..c92505600 100755 --- a/misc/userscripts/qute-lastpass +++ b/misc/userscripts/qute-lastpass @@ -102,12 +102,14 @@ def dmenu(items, invocation, encoding): def fake_key_raw(text): - sequence = '' - - for character in text: - # Escape all characters by default, space requires special handling - sequence += ('" "' if character == ' ' else '\\{}'.format(character)) - qute_command('fake-key {}'.format(sequence)) + # Escape all characters by default, space, '<' and '>' requires special handling + special_escapes = { + ' ': '" "', + '<': '<less>', + '>': '<greater>', + } + sequence = ''.join([special_escapes.get(c, '\\{}'.format(c)) for c in text]) + qute_command(f'fake-key {sequence}') def main(arguments): diff --git a/misc/userscripts/qute-pass b/misc/userscripts/qute-pass index b49e87dd8..388a7a737 100755 --- a/misc/userscripts/qute-pass +++ b/misc/userscripts/qute-pass @@ -152,6 +152,8 @@ def _run_pass(pass_arguments): def pass_(path): + if arguments.mode == "gopass": + return _run_pass(['show', '-o', path]) return _run_pass(['show', path]) diff --git a/pytest.ini b/pytest.ini index f936a02cb..3705a17ef 100644 --- a/pytest.ini +++ b/pytest.ini @@ -31,7 +31,7 @@ markers = qtwebengine_flaky: Tests which are flaky (and currently skipped) with QtWebEngine qtwebengine_mac_xfail: Tests which fail on macOS with QtWebEngine this: Used to mark tests during development - no_invalid_lines: Don't fail on unparseable lines in end2end tests + no_invalid_lines: Don't fail on unparsable lines in end2end tests fake_os: Fake utils.is_* to a fake operating system unicode_locale: Tests which need a unicode locale to work qtwebkit6021_xfail: Tests which would fail on WebKit version 602.1 diff --git a/qutebrowser/api/config.py b/qutebrowser/api/config.py index fb363d858..02d48ec3a 100644 --- a/qutebrowser/api/config.py +++ b/qutebrowser/api/config.py @@ -25,7 +25,7 @@ from PyQt5.QtCore import QUrl from qutebrowser.config import config -#: Simplified access to config values using attribute acccess. +#: Simplified access to config values using attribute access. #: For example, to access the ``content.javascript.enabled`` setting, #: you can do:: #: diff --git a/qutebrowser/app.py b/qutebrowser/app.py index 76d52470a..d2e9468aa 100644 --- a/qutebrowser/app.py +++ b/qutebrowser/app.py @@ -406,7 +406,7 @@ def open_desktopservices_url(url): # This is effectively a @config.change_filter -# Howerver, logging is initialized too early to use that annotation +# However, logging is initialized too early to use that annotation def _on_config_changed(name: str) -> None: if name.startswith('logging.'): log.init_from_config(config.val) diff --git a/qutebrowser/browser/downloads.py b/qutebrowser/browser/downloads.py index 96220897c..b179013fc 100644 --- a/qutebrowser/browser/downloads.py +++ b/qutebrowser/browser/downloads.py @@ -1291,7 +1291,7 @@ class TempDownloadManager: """Manager to handle temporary download files. - The downloads are downloaded to a temporary location and then openened with + The downloads are downloaded to a temporary location and then opened with the system standard application. The temporary files are deleted when qutebrowser is shutdown. diff --git a/qutebrowser/browser/greasemonkey.py b/qutebrowser/browser/greasemonkey.py index df8b2b0c2..9e25e49bd 100644 --- a/qutebrowser/browser/greasemonkey.py +++ b/qutebrowser/browser/greasemonkey.py @@ -299,7 +299,7 @@ class GreasemonkeyManager(QObject): """Add a GreasemonkeyScript to this manager. Args: - force: Fetch and overwrite any dependancies which are + force: Fetch and overwrite any dependencies which are already locally cached. """ if script.requires: @@ -345,7 +345,7 @@ class GreasemonkeyManager(QObject): def _add_script_with_requires(self, script, quiet=False): """Add a script with pending downloads to this GreasemonkeyManager. - Specifically a script that has dependancies specified via an + Specifically a script that has dependencies specified via an `@require` rule. Args: @@ -353,7 +353,7 @@ class GreasemonkeyManager(QObject): quiet: True to suppress the scripts_reloaded signal after adding `script`. Returns: True if the script was added, False if there are still - dependancies being downloaded. + dependencies being downloaded. """ # See if we are still waiting on any required scripts for this one for dl in self._in_progress_dls: diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index 89061cebf..04cc5a088 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -150,7 +150,7 @@ class WebHistory(sql.SqlTable): 'redirect': 'NOT NULL'}, parent=parent) self._progress = progress - # Store the last saved url to avoid duplicate immedate saves. + # Store the last saved url to avoid duplicate immediate saves. self._last_url = None self.completion = CompletionHistory(parent=self) diff --git a/qutebrowser/browser/shared.py b/qutebrowser/browser/shared.py index 9234e82d8..26cdace56 100644 --- a/qutebrowser/browser/shared.py +++ b/qutebrowser/browser/shared.py @@ -283,7 +283,7 @@ def get_user_stylesheet(searching=False): css += f.read() setting = config.val.scrolling.bar - if setting == 'overlay' and not utils.is_mac: + if setting == 'overlay' and utils.is_mac: setting = 'when-searching' if setting == 'never' or setting == 'when-searching' and not searching: diff --git a/qutebrowser/browser/webengine/darkmode.py b/qutebrowser/browser/webengine/darkmode.py index d067edea3..c9d69d52d 100644 --- a/qutebrowser/browser/webengine/darkmode.py +++ b/qutebrowser/browser/webengine/darkmode.py @@ -135,7 +135,7 @@ _DarkModeSettingsType = Iterable[ Tuple[ str, # qutebrowser option name str, # darkmode setting name - # Mapping from the config value to a string (or something convertable + # Mapping from the config value to a string (or something convertible # to a string) which gets passed to Chromium. Optional[Mapping[Any, Union[str, int]]], ], diff --git a/qutebrowser/browser/webengine/interceptor.py b/qutebrowser/browser/webengine/interceptor.py index b27509552..54bc5623b 100644 --- a/qutebrowser/browser/webengine/interceptor.py +++ b/qutebrowser/browser/webengine/interceptor.py @@ -45,16 +45,14 @@ class WebEngineRequest(interceptors.Request): def redirect(self, url: QUrl) -> None: if self._redirected: - raise interceptors.RedirectFailedException( - "Request already redirected.") + raise interceptors.RedirectException("Request already redirected.") if self._webengine_info is None: - raise interceptors.RedirectFailedException( - "Request improperly initialized.") + raise interceptors.RedirectException("Request improperly initialized.") # Redirecting a request that contains payload data is not allowed. # To be safe, abort on any request not in a whitelist. if (self._webengine_info.requestMethod() not in self._WHITELISTED_REQUEST_METHODS): - raise interceptors.RedirectFailedException( + raise interceptors.RedirectException( "Request method does not support redirection.") self._webengine_info.redirect(url) self._redirected = True diff --git a/qutebrowser/browser/webengine/webenginetab.py b/qutebrowser/browser/webengine/webenginetab.py index 98a6bf05d..8e5ea3a52 100644 --- a/qutebrowser/browser/webengine/webenginetab.py +++ b/qutebrowser/browser/webengine/webenginetab.py @@ -699,7 +699,7 @@ class WebEngineZoom(browsertab.AbstractZoom): class WebEngineElements(browsertab.AbstractElements): - """QtWebEngine implemementations related to elements on the page.""" + """QtWebEngine implementations related to elements on the page.""" _tab: 'WebEngineTab' @@ -772,7 +772,7 @@ class WebEngineElements(browsertab.AbstractElements): class WebEngineAudio(browsertab.AbstractAudio): - """QtWebEngine implemementations related to audio/muting. + """QtWebEngine implementations related to audio/muting. Attributes: _overridden: Whether the user toggled muting manually. diff --git a/qutebrowser/browser/webkit/webkittab.py b/qutebrowser/browser/webkit/webkittab.py index 69773fa57..c5f78cfe9 100644 --- a/qutebrowser/browser/webkit/webkittab.py +++ b/qutebrowser/browser/webkit/webkittab.py @@ -687,7 +687,7 @@ class WebKitHistory(browsertab.AbstractHistory): class WebKitElements(browsertab.AbstractElements): - """QtWebKit implemementations related to elements on the page.""" + """QtWebKit implementations related to elements on the page.""" _tab: 'WebKitTab' diff --git a/qutebrowser/completion/completer.py b/qutebrowser/completion/completer.py index b06611bc0..d66e3ee40 100644 --- a/qutebrowser/completion/completer.py +++ b/qutebrowser/completion/completer.py @@ -150,7 +150,7 @@ class Completer(QObject): parts.insert(i, '') prefix = [x.strip() for x in parts[:i]] center = parts[i].strip() - # strip trailing whitepsace included as a separate token + # strip trailing whitespace included as a separate token postfix = [x.strip() for x in parts[i+1:] if not x.isspace()] log.completion.debug( "partitioned: {} '{}' {}".format(prefix, center, postfix)) diff --git a/qutebrowser/completion/models/miscmodels.py b/qutebrowser/completion/models/miscmodels.py index 925f95bbb..d9d386365 100644 --- a/qutebrowser/completion/models/miscmodels.py +++ b/qutebrowser/completion/models/miscmodels.py @@ -293,6 +293,6 @@ def undo(*, info): enumerate(reversed(tabbed_browser.undo_stack), start=1) ] - cat = listcategory.ListCategory("Closed tabs", entries) + cat = listcategory.ListCategory("Closed tabs", entries, sort=False) model.add_category(cat) return model diff --git a/qutebrowser/components/adblock.py b/qutebrowser/components/adblock.py deleted file mode 100644 index f9b0a583b..000000000 --- a/qutebrowser/components/adblock.py +++ /dev/null @@ -1,344 +0,0 @@ -# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: - -# Copyright 2014-2020 Florian Bruhin (The Compiler) <mail@qutebrowser.org> -# -# This file is part of qutebrowser. -# -# qutebrowser is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# qutebrowser is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>. - -"""Functions related to ad blocking.""" - -import os.path -import functools -import posixpath -import zipfile -import logging -import pathlib -from typing import cast, IO, List, Set - -from PyQt5.QtCore import QUrl - -from qutebrowser.api import (cmdutils, hook, config, message, downloads, - interceptor, apitypes, qtutils) - - -logger = logging.getLogger('network') -_host_blocker = cast('HostBlocker', None) - - -def _guess_zip_filename(zf: zipfile.ZipFile) -> str: - """Guess which file to use inside a zip file.""" - files = zf.namelist() - if len(files) == 1: - return files[0] - else: - for e in files: - if posixpath.splitext(e)[0].lower() == 'hosts': - return e - raise FileNotFoundError("No hosts file found in zip") - - -def get_fileobj(byte_io: IO[bytes]) -> IO[bytes]: - """Get a usable file object to read the hosts file from.""" - byte_io.seek(0) # rewind downloaded file - if zipfile.is_zipfile(byte_io): - byte_io.seek(0) # rewind what zipfile.is_zipfile did - zf = zipfile.ZipFile(byte_io) - filename = _guess_zip_filename(zf) - byte_io = zf.open(filename, mode='r') - else: - byte_io.seek(0) # rewind what zipfile.is_zipfile did - return byte_io - - -def _is_whitelisted_url(url: QUrl) -> bool: - """Check if the given URL is on the adblock whitelist.""" - for pattern in config.val.content.host_blocking.whitelist: - if pattern.matches(url): - return True - return False - - -class _FakeDownload(downloads.TempDownload): - - """A download stub to use on_download_finished with local files.""" - - def __init__(self, # pylint: disable=super-init-not-called - fileobj: IO[bytes]) -> None: - self.fileobj = fileobj - self.successful = True - - -class HostBlocker: - - """Manage blocked hosts based from /etc/hosts-like files. - - Attributes: - _blocked_hosts: A set of blocked hosts. - _config_blocked_hosts: A set of blocked hosts from ~/.config. - _in_progress: The DownloadItems which are currently downloading. - _done_count: How many files have been read successfully. - _local_hosts_file: The path to the blocked-hosts file. - _config_hosts_file: The path to a blocked-hosts in ~/.config - _has_basedir: Whether a custom --basedir is set. - """ - - def __init__(self, *, data_dir: pathlib.Path, config_dir: pathlib.Path, - has_basedir: bool = False) -> None: - self._has_basedir = has_basedir - self._blocked_hosts: Set[str] = set() - self._config_blocked_hosts: Set[str] = set() - self._in_progress: List[downloads.TempDownload] = [] - self._done_count = 0 - - self._local_hosts_file = str(data_dir / 'blocked-hosts') - self.update_files() - - self._config_hosts_file = str(config_dir / 'blocked-hosts') - - def _is_blocked(self, request_url: QUrl, - first_party_url: QUrl = None) -> bool: - """Check whether the given request is blocked.""" - if first_party_url is not None and not first_party_url.isValid(): - first_party_url = None - - qtutils.ensure_valid(request_url) - - if not config.get('content.host_blocking.enabled', - url=first_party_url): - return False - - host = request_url.host() - return ((host in self._blocked_hosts or - host in self._config_blocked_hosts) and - not _is_whitelisted_url(request_url)) - - def filter_request(self, info: interceptor.Request) -> None: - """Block the given request if necessary.""" - if self._is_blocked(request_url=info.request_url, - first_party_url=info.first_party_url): - logger.debug("Request to {} blocked by host blocker." - .format(info.request_url.host())) - info.block() - - def _read_hosts_line(self, raw_line: bytes) -> Set[str]: - """Read hosts from the given line. - - Args: - line: The bytes object to read. - - Returns: - A set containing valid hosts found - in the line. - """ - if raw_line.startswith(b'#'): - # Ignoring comments early so we don't have to care about - # encoding errors in them - return set() - - line = raw_line.decode('utf-8') - - # Remove comments - hash_idx = line.find('#') - line = line if hash_idx == -1 else line[:hash_idx] - - parts = line.strip().split() - if len(parts) == 1: - # "one host per line" format - hosts = parts - else: - # /etc/hosts format - hosts = parts[1:] - - filtered_hosts = set() - for host in hosts: - if ('.' in host and - not host.endswith('.localdomain') and - host != '0.0.0.0'): - filtered_hosts.update([host]) - - return filtered_hosts - - def _read_hosts_file(self, filename: str, target: Set[str]) -> bool: - """Read hosts from the given filename. - - Args: - filename: The file to read. - target: The set to store the hosts in. - - Return: - True if a read was attempted, False otherwise - """ - if not os.path.exists(filename): - return False - - try: - with open(filename, 'rb') as f: - for line in f: - target |= self._read_hosts_line(line) - - except (OSError, UnicodeDecodeError): - logger.exception("Failed to read host blocklist!") - - return True - - def read_hosts(self) -> None: - """Read hosts from the existing blocked-hosts file.""" - self._blocked_hosts = set() - - self._read_hosts_file(self._config_hosts_file, - self._config_blocked_hosts) - - found = self._read_hosts_file(self._local_hosts_file, - self._blocked_hosts) - - if not found: - if (config.val.content.host_blocking.lists and - not self._has_basedir and - config.val.content.host_blocking.enabled): - message.info("Run :adblock-update to get adblock lists.") - - def adblock_update(self) -> None: - """Update the adblock block lists.""" - self._read_hosts_file(self._config_hosts_file, - self._config_blocked_hosts) - self._blocked_hosts = set() - self._done_count = 0 - for url in config.val.content.host_blocking.lists: - if url.scheme() == 'file': - filename = url.toLocalFile() - if os.path.isdir(filename): - for entry in os.scandir(filename): - if entry.is_file(): - self._import_local(entry.path) - else: - self._import_local(filename) - else: - download = downloads.download_temp(url) - self._in_progress.append(download) - download.finished.connect( - functools.partial(self._on_download_finished, download)) - - def _import_local(self, filename: str) -> None: - """Adds the contents of a file to the blocklist. - - Args: - filename: path to a local file to import. - """ - try: - fileobj = open(filename, 'rb') - except OSError as e: - message.error("adblock: Error while reading {}: {}".format( - filename, e.strerror)) - return - download = _FakeDownload(fileobj) - self._in_progress.append(download) - self._on_download_finished(download) - - def _merge_file(self, byte_io: IO[bytes]) -> None: - """Read and merge host files. - - Args: - byte_io: The BytesIO object of the completed download. - """ - error_count = 0 - line_count = 0 - try: - f = get_fileobj(byte_io) - except (OSError, zipfile.BadZipFile, zipfile.LargeZipFile, - LookupError) as e: - message.error("adblock: Error while reading {}: {} - {}".format( - byte_io.name, e.__class__.__name__, e)) - return - - for line in f: - line_count += 1 - try: - self._blocked_hosts |= self._read_hosts_line(line) - except UnicodeDecodeError: - logger.error("Failed to decode: {!r}".format(line)) - error_count += 1 - - logger.debug("{}: read {} lines".format(byte_io.name, line_count)) - if error_count > 0: - message.error("adblock: {} read errors for {}".format( - error_count, byte_io.name)) - - def _on_lists_downloaded(self) -> None: - """Install block lists after files have been downloaded.""" - with open(self._local_hosts_file, 'w', encoding='utf-8') as f: - for host in sorted(self._blocked_hosts): - f.write(host + '\n') - message.info("adblock: Read {} hosts from {} sources.".format( - len(self._blocked_hosts), self._done_count)) - - def update_files(self) -> None: - """Update files when the config changed.""" - if not config.val.content.host_blocking.lists: - try: - os.remove(self._local_hosts_file) - except FileNotFoundError: - pass - except OSError as e: - logger.exception("Failed to delete hosts file: {}".format(e)) - - def _on_download_finished(self, download: downloads.TempDownload) -> None: - """Check if all downloads are finished and if so, trigger reading. - - Arguments: - download: The finished download. - """ - self._in_progress.remove(download) - if download.successful: - self._done_count += 1 - assert not isinstance(download.fileobj, - downloads.UnsupportedAttribute) - assert download.fileobj is not None - try: - self._merge_file(download.fileobj) - finally: - download.fileobj.close() - if not self._in_progress: - try: - self._on_lists_downloaded() - except OSError: - logger.exception("Failed to write host block list!") - - -@cmdutils.register() -def adblock_update() -> None: - """Update the adblock block lists. - - This updates `~/.local/share/qutebrowser/blocked-hosts` with downloaded - host lists and re-reads `~/.config/qutebrowser/blocked-hosts`. - """ - # FIXME: As soon as we can register instances again, we should move this - # back to the class. - _host_blocker.adblock_update() - - -@hook.config_changed('content.host_blocking.lists') -def on_config_changed() -> None: - _host_blocker.update_files() - - -@hook.init() -def init(context: apitypes.InitContext) -> None: - """Initialize the host blocker.""" - global _host_blocker - _host_blocker = HostBlocker(data_dir=context.data_dir, - config_dir=context.config_dir, - has_basedir=context.args.basedir is not None) - _host_blocker.read_hosts() - interceptor.register(_host_blocker.filter_request) diff --git a/qutebrowser/components/adblockcommands.py b/qutebrowser/components/adblockcommands.py new file mode 100644 index 000000000..e507a2b5c --- /dev/null +++ b/qutebrowser/components/adblockcommands.py @@ -0,0 +1,31 @@ +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2020 Florian Bruhin (The Compiler) <mail@qutebrowser.org> +# +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>. + +"""Commands relating to ad blocking.""" + +from qutebrowser.api import cmdutils +from qutebrowser.components import braveadblock, hostblock + + +@cmdutils.register() +def adblock_update() -> None: + """Update block lists for both the host- and the Brave ad blocker.""" + if braveadblock.ad_blocker is not None: + braveadblock.ad_blocker.adblock_update() + hostblock.host_blocker.adblock_update() diff --git a/qutebrowser/components/braveadblock.py b/qutebrowser/components/braveadblock.py new file mode 100644 index 000000000..340ed0fac --- /dev/null +++ b/qutebrowser/components/braveadblock.py @@ -0,0 +1,294 @@ +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2020 Florian Bruhin (The Compiler) <mail@qutebrowser.org> +# +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>. + +"""Functions related to the Brave adblocker.""" + +import io +import logging +import pathlib +import functools +from typing import Optional, IO + +from PyQt5.QtCore import QUrl + +from qutebrowser.api import ( + hook, + config, + message, + interceptor, + apitypes, + qtutils, +) +from qutebrowser.api.interceptor import ResourceType +from qutebrowser.components.utils import blockutils +from qutebrowser.utils import version # FIXME: Move needed parts into api namespace? + +try: + import adblock +except ImportError: + adblock = None # type: ignore[assignment] + +logger = logging.getLogger("network") +ad_blocker: Optional["BraveAdBlocker"] = None + + +def _should_be_used() -> bool: + """Whether the Brave adblocker should be used or not. + + Here we assume the adblock dependency is satisfied. + """ + return config.val.content.blocking.method in ("auto", "both", "adblock") + + +def _possibly_show_missing_dependency_warning() -> None: + """Show missing dependency warning, if appropriate. + + If the adblocking method is configured such that the Brave adblocker + should be used, but the optional dependency is not satisfied, we show an + error message. + """ + adblock_info = version.MODULE_INFO["adblock"] + + method = config.val.content.blocking.method + if method not in ("both", "adblock"): + return + + if adblock_info.is_outdated(): + message.warning( + f"Installed version {adblock_info.get_version()} of the 'adblock' " + f"dependency is too old. Minimum supported is {adblock_info.min_version}." + ) + else: + assert not adblock_info.is_installed(), adblock_info + message.warning( + f"Ad blocking method is set to '{method}' but 'adblock' dependency is not " + "installed." + ) + + +_RESOURCE_TYPE_STRINGS = { + ResourceType.main_frame: "main_frame", + ResourceType.sub_frame: "sub_frame", + ResourceType.stylesheet: "stylesheet", + ResourceType.script: "script", + ResourceType.image: "image", + ResourceType.font_resource: "font", + ResourceType.sub_resource: "sub_frame", + ResourceType.object: "object", + ResourceType.media: "media", + ResourceType.worker: "other", + ResourceType.shared_worker: "other", + ResourceType.prefetch: "other", + ResourceType.favicon: "image", + ResourceType.xhr: "xhr", + ResourceType.ping: "ping", + ResourceType.service_worker: "other", + ResourceType.csp_report: "csp_report", + ResourceType.plugin_resource: "other", + ResourceType.preload_main_frame: "other", + ResourceType.preload_sub_frame: "other", + ResourceType.unknown: "other", + None: "", +} + + +def resource_type_to_string(resource_type: Optional[ResourceType]) -> str: + return _RESOURCE_TYPE_STRINGS.get(resource_type, "other") + + +class BraveAdBlocker: + + """Manage blocked hosts based on Brave's adblocker. + + Attributes: + enabled: Whether to block ads or not. + _has_basedir: Whether a custom --basedir is set. + _cache_path: The path of the adblock engine cache file + _engine: Brave ad-blocking engine. + """ + + def __init__(self, *, data_dir: pathlib.Path, has_basedir: bool = False) -> None: + self.enabled = _should_be_used() + self._has_basedir = has_basedir + self._cache_path = data_dir / "adblock-cache.dat" + self._engine = adblock.Engine(adblock.FilterSet()) + + def _is_blocked( + self, + request_url: QUrl, + first_party_url: Optional[QUrl] = None, + resource_type: Optional[interceptor.ResourceType] = None, + ) -> bool: + """Check whether the given request is blocked.""" + if not self.enabled: + # Do nothing if `content.blocking.method` is not set to enable the + # use of this adblocking module. + return False + + if first_party_url is None or not first_party_url.isValid(): + # FIXME: It seems that when `first_party_url` is None, every URL + # I try is blocked. This may have been a result of me incorrectly + # using the upstream library, or an upstream bug. For now we don't + # block any request with `first_party_url=None`. + return False + + qtutils.ensure_valid(request_url) + + if not config.get("content.blocking.enabled", url=first_party_url): + # Do nothing if adblocking is disabled for this site. + return False + + result = self._engine.check_network_urls( + request_url.toString(), + first_party_url.toString(), + resource_type_to_string(resource_type), + ) + + if not result.matched: + return False + elif result.exception is not None and not result.important: + # Exception is not `None` when the blocker matched on an exception + # rule. Effectively this means that there was a match, but the + # request should not be blocked. + # + # An `important` match means that exceptions should not apply and + # no further checking is necessary--the request should be blocked. + logger.debug( + "Excepting %s from being blocked by %s because of %s", + request_url.toDisplayString(), + result.filter, + result.exception, + ) + return False + elif blockutils.is_whitelisted_url(request_url): + logger.debug( + "Request to %s is whitelisted, thus not blocked", + request_url.toDisplayString(), + ) + return False + return True + + def filter_request(self, info: interceptor.Request) -> None: + """Block the given request if necessary.""" + if self._is_blocked(info.request_url, info.first_party_url, info.resource_type): + logger.debug( + "Request to %s blocked by ad blocker.", + info.request_url.toDisplayString(), + ) + info.block() + + def read_cache(self) -> None: + """Initialize the adblocking engine from cache file.""" + if self._cache_path.is_file(): + logger.debug("Loading cached adblock data: %s", self._cache_path) + self._engine.deserialize_from_file(str(self._cache_path)) + else: + if ( + config.val.content.blocking.adblock.lists + and not self._has_basedir + and config.val.content.blocking.enabled + and self.enabled + ): + message.info("Run :adblock-update to get adblock lists.") + + def adblock_update(self) -> blockutils.BlocklistDownloads: + """Update the adblock block lists.""" + logger.info("Downloading adblock filter lists...") + + filter_set = adblock.FilterSet() + dl = blockutils.BlocklistDownloads(config.val.content.blocking.adblock.lists) + dl.single_download_finished.connect( + functools.partial(self._on_download_finished, filter_set=filter_set) + ) + dl.all_downloads_finished.connect( + functools.partial(self._on_lists_downloaded, filter_set=filter_set) + ) + dl.initiate() + return dl + + def _on_lists_downloaded( + self, done_count: int, filter_set: "adblock.FilterSet" + ) -> None: + """Install block lists after files have been downloaded.""" + self._engine = adblock.Engine(filter_set) + self._engine.serialize_to_file(str(self._cache_path)) + message.info( + f"braveadblock: Filters successfully read from {done_count} sources.") + + def update_files(self) -> None: + """Update files when the config changed.""" + if not config.val.content.blocking.adblock.lists: + try: + self._cache_path.unlink() + except FileNotFoundError: + pass + except OSError as e: + logger.exception("Failed to remove adblock cache file: %s", e) + + def _on_download_finished( + self, fileobj: IO[bytes], filter_set: "adblock.FilterSet" + ) -> None: + """When a blocklist download finishes, add it to the given filter set. + + Arguments: + fileobj: The finished download's contents. + """ + fileobj.seek(0) + try: + with io.TextIOWrapper(fileobj, encoding="utf-8") as text_io: + filter_set.add_filter_list(text_io.read()) + except UnicodeDecodeError: + message.info("braveadblock: Block list is not valid utf-8") + + +@hook.config_changed("content.blocking.adblock.lists") +def on_lists_changed() -> None: + """Remove cached blocker from disk when blocklist changes.""" + if ad_blocker is not None: + ad_blocker.update_files() + + +@hook.config_changed("content.blocking.method") +def on_method_changed() -> None: + """When the adblocking method changes, update blocker accordingly.""" + if ad_blocker is not None: + # This implies the 'adblock' dependency is satisfied + ad_blocker.enabled = _should_be_used() + else: + _possibly_show_missing_dependency_warning() + + +@hook.init() +def init(context: apitypes.InitContext) -> None: + """Initialize the Brave ad blocker.""" + global ad_blocker + + adblock_info = version.MODULE_INFO["adblock"] + if not adblock_info.is_usable(): + # We want 'adblock' to be an optional dependency. If the module is + # not installed or is outdated, we simply keep the `ad_blocker` global at + # `None`. + _possibly_show_missing_dependency_warning() + return + + ad_blocker = BraveAdBlocker( + data_dir=context.data_dir, has_basedir=context.args.basedir is not None + ) + ad_blocker.read_cache() + interceptor.register(ad_blocker.filter_request) diff --git a/qutebrowser/components/hostblock.py b/qutebrowser/components/hostblock.py new file mode 100644 index 000000000..e1ef88667 --- /dev/null +++ b/qutebrowser/components/hostblock.py @@ -0,0 +1,307 @@ +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2014-2020 Florian Bruhin (The Compiler) <mail@qutebrowser.org> +# +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>. + +"""Functions related to host blocking.""" + +import os.path +import posixpath +import zipfile +import logging +import pathlib +from typing import cast, IO, Set + +from PyQt5.QtCore import QUrl + +from qutebrowser.api import ( + hook, + config, + message, + interceptor, + apitypes, + qtutils, +) +from qutebrowser.components.utils import blockutils +from qutebrowser.utils import version # FIXME: Move needed parts into api namespace? + + +logger = logging.getLogger("network") +host_blocker = cast("HostBlocker", None) + + +def _guess_zip_filename(zf: zipfile.ZipFile) -> str: + """Guess which file to use inside a zip file.""" + files = zf.namelist() + if len(files) == 1: + return files[0] + else: + for e in files: + if posixpath.splitext(e)[0].lower() == "hosts": + return e + raise FileNotFoundError("No hosts file found in zip") + + +def get_fileobj(byte_io: IO[bytes]) -> IO[bytes]: + """Get a usable file object to read the hosts file from.""" + byte_io.seek(0) # rewind downloaded file + if zipfile.is_zipfile(byte_io): + byte_io.seek(0) # rewind what zipfile.is_zipfile did + zf = zipfile.ZipFile(byte_io) + filename = _guess_zip_filename(zf) + byte_io = zf.open(filename, mode="r") + else: + byte_io.seek(0) # rewind what zipfile.is_zipfile did + return byte_io + + +def _should_be_used() -> bool: + """Whether the hostblocker should be used or not.""" + method = config.val.content.blocking.method + + adblock_info = version.MODULE_INFO["adblock"] + adblock_usable = adblock_info.is_usable() + + logger.debug(f"Configured adblock method {method}, adblock library usable: " + f"{adblock_usable}") + return method in ("both", "hosts") or (method == "auto" and not adblock_usable) + + +class HostBlocker: + + """Manage blocked hosts based from /etc/hosts-like files. + + Attributes: + enabled: Given the current blocking method, should the host blocker be enabled? + _blocked_hosts: A set of blocked hosts. + _config_blocked_hosts: A set of blocked hosts from ~/.config. + _local_hosts_file: The path to the blocked-hosts file. + _config_hosts_file: The path to a blocked-hosts in ~/.config + _has_basedir: Whether a custom --basedir is set. + """ + + def __init__( + self, + *, + data_dir: pathlib.Path, + config_dir: pathlib.Path, + has_basedir: bool = False + ) -> None: + self.enabled = _should_be_used() + self._has_basedir = has_basedir + self._blocked_hosts: Set[str] = set() + self._config_blocked_hosts: Set[str] = set() + + self._local_hosts_file = str(data_dir / "blocked-hosts") + self.update_files() + + self._config_hosts_file = str(config_dir / "blocked-hosts") + + def _is_blocked(self, request_url: QUrl, first_party_url: QUrl = None) -> bool: + """Check whether the given request is blocked.""" + if not self.enabled: + return False + + if first_party_url is not None and not first_party_url.isValid(): + first_party_url = None + + qtutils.ensure_valid(request_url) + + if not config.get("content.blocking.enabled", url=first_party_url): + return False + + host = request_url.host() + return ( + host in self._blocked_hosts or host in self._config_blocked_hosts + ) and not blockutils.is_whitelisted_url(request_url) + + def filter_request(self, info: interceptor.Request) -> None: + """Block the given request if necessary.""" + if self._is_blocked( + request_url=info.request_url, first_party_url=info.first_party_url + ): + logger.debug( + "Request to {} blocked by host blocker.".format(info.request_url.host()) + ) + info.block() + + def _read_hosts_line(self, raw_line: bytes) -> Set[str]: + """Read hosts from the given line. + + Args: + line: The bytes object to read. + + Returns: + A set containing valid hosts found + in the line. + """ + if raw_line.startswith(b"#"): + # Ignoring comments early so we don't have to care about + # encoding errors in them + return set() + + line = raw_line.decode("utf-8") + + # Remove comments + hash_idx = line.find("#") + line = line if hash_idx == -1 else line[:hash_idx] + + parts = line.strip().split() + if len(parts) == 1: + # "one host per line" format + hosts = parts + else: + # /etc/hosts format + hosts = parts[1:] + + filtered_hosts = set() + for host in hosts: + if "." in host and not host.endswith(".localdomain") and host != "0.0.0.0": + filtered_hosts.update([host]) + + return filtered_hosts + + def _read_hosts_file(self, filename: str, target: Set[str]) -> bool: + """Read hosts from the given filename. + + Args: + filename: The file to read. + target: The set to store the hosts in. + + Return: + True if a read was attempted, False otherwise + """ + if not os.path.exists(filename): + return False + + try: + with open(filename, "rb") as f: + for line in f: + target |= self._read_hosts_line(line) + + except (OSError, UnicodeDecodeError): + logger.exception("Failed to read host blocklist!") + + return True + + def read_hosts(self) -> None: + """Read hosts from the existing blocked-hosts file.""" + self._blocked_hosts = set() + + self._read_hosts_file(self._config_hosts_file, self._config_blocked_hosts) + + found = self._read_hosts_file(self._local_hosts_file, self._blocked_hosts) + + if not found: + if ( + config.val.content.blocking.hosts.lists + and not self._has_basedir + and config.val.content.blocking.enabled + and self.enabled + ): + message.info("Run :adblock-update to get adblock lists.") + + def adblock_update(self) -> blockutils.BlocklistDownloads: + """Update the adblock block lists.""" + self._read_hosts_file(self._config_hosts_file, self._config_blocked_hosts) + self._blocked_hosts = set() + + blocklists = config.val.content.blocking.hosts.lists + dl = blockutils.BlocklistDownloads(blocklists) + dl.single_download_finished.connect(self._merge_file) + dl.all_downloads_finished.connect(self._on_lists_downloaded) + dl.initiate() + return dl + + def _merge_file(self, byte_io: IO[bytes]) -> None: + """Read and merge host files. + + Args: + byte_io: The BytesIO object of the completed download. + """ + error_count = 0 + line_count = 0 + try: + f = get_fileobj(byte_io) + except (OSError, zipfile.BadZipFile, zipfile.LargeZipFile, LookupError) as e: + message.error( + "hostblock: Error while reading {}: {} - {}".format( + byte_io.name, e.__class__.__name__, e + ) + ) + return + + for line in f: + line_count += 1 + try: + self._blocked_hosts |= self._read_hosts_line(line) + except UnicodeDecodeError: + logger.error("Failed to decode: {!r}".format(line)) + error_count += 1 + + logger.debug("{}: read {} lines".format(byte_io.name, line_count)) + if error_count > 0: + message.error( + "hostblock: {} read errors for {}".format(error_count, byte_io.name) + ) + + def _on_lists_downloaded(self, done_count: int) -> None: + """Install block lists after files have been downloaded.""" + try: + with open(self._local_hosts_file, "w", encoding="utf-8") as f: + for host in sorted(self._blocked_hosts): + f.write(host + "\n") + message.info( + "hostblock: Read {} hosts from {} sources.".format( + len(self._blocked_hosts), done_count + ) + ) + except OSError: + logger.exception("Failed to write host block list!") + + def update_files(self) -> None: + """Update files when the config changed.""" + if not config.val.content.blocking.hosts.lists: + try: + os.remove(self._local_hosts_file) + except FileNotFoundError: + pass + except OSError as e: + logger.exception("Failed to delete hosts file: {}".format(e)) + + +@hook.config_changed("content.blocking.hosts.lists") +def on_lists_changed() -> None: + host_blocker.update_files() + + +@hook.config_changed("content.blocking.method") +def on_method_changed() -> None: + host_blocker.enabled = _should_be_used() + + +@hook.init() +def init(context: apitypes.InitContext) -> None: + """Initialize the host blocker.""" + global host_blocker + host_blocker = HostBlocker( + data_dir=context.data_dir, + config_dir=context.config_dir, + has_basedir=context.args.basedir is not None, + ) + host_blocker.read_hosts() + interceptor.register(host_blocker.filter_request) diff --git a/qutebrowser/components/utils/__init__.py b/qutebrowser/components/utils/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/qutebrowser/components/utils/__init__.py diff --git a/qutebrowser/components/utils/blockutils.py b/qutebrowser/components/utils/blockutils.py new file mode 100644 index 000000000..502038f48 --- /dev/null +++ b/qutebrowser/components/utils/blockutils.py @@ -0,0 +1,162 @@ +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2020 Florian Bruhin (The Compiler) <mail@qutebrowser.org> +# +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>. + + +"""Code that is shared between the host blocker and Brave ad blocker.""" + +import os +import functools +from typing import IO, List, Optional + +from PyQt5.QtCore import QUrl, QObject, pyqtSignal + +from qutebrowser.api import downloads, message, config + + +class FakeDownload(downloads.TempDownload): + + """A download stub to use on_download_finished with local files.""" + + def __init__(self, fileobj: IO[bytes]) -> None: + # pylint: disable=super-init-not-called + self.fileobj = fileobj + self.successful = True + + +class BlocklistDownloads(QObject): + """Download blocklists from the given URLs. + + Attributes: + single_download_finished: + A signal that is emitted when a single download has finished. The + listening slot is provided with the download object. + all_downloads_finished: + A signal that is emitted when all downloads have finished. The + first argument is the number of items downloaded. + _urls: The URLs to download. + _in_progress: The DownloadItems which are currently downloading. + _done_count: How many files have been read successfully. + _finished_registering_downloads: + Used to make sure that if all the downloads finish really quickly, + before all of the block-lists have been added to the download + queue, we don't emit `single_download_finished`. + _started: Has the `initiate` method been called? + _finished: Has `all_downloads_finished` been emitted? + """ + + single_download_finished = pyqtSignal(object) # arg: the file object + all_downloads_finished = pyqtSignal(int) # arg: download count + + def __init__(self, urls: List[QUrl], parent: Optional[QObject] = None) -> None: + super().__init__(parent) + self._urls = urls + + self._in_progress: List[downloads.TempDownload] = [] + self._done_count = 0 + self._finished_registering_downloads = False + self._started = False + self._finished = False + + def initiate(self) -> None: + """Initiate downloads of each url in `self._urls`.""" + if self._started: + raise ValueError("This download has already been initiated") + self._started = True + + if not self._urls: + self._finished = True + self.all_downloads_finished.emit(self._done_count) + return + + for url in self._urls: + self._download_blocklist_url(url) + self._finished_registering_downloads = True + + if not self._in_progress and not self._finished: + # The in-progress list is empty but we still haven't called the + # completion callback yet. This happens when all downloads finish + # before we've set `_finished_registering_dowloads` to False. + self._finished = True + self.all_downloads_finished.emit(self._done_count) + + def _download_blocklist_url(self, url: QUrl) -> None: + """Take a blocklist url and queue it for download. + + Args: + url: url to download + """ + if url.scheme() == "file": + # The URL describes a local file on disk if the url scheme is + # "file://". We handle those as a special case. + filename = url.toLocalFile() + if os.path.isdir(filename): + for entry in os.scandir(filename): + if entry.is_file(): + self._import_local(entry.path) + else: + self._import_local(filename) + else: + download = downloads.download_temp(url) + self._in_progress.append(download) + download.finished.connect( + functools.partial(self._on_download_finished, download) + ) + + def _import_local(self, filename: str) -> None: + """Pretend that a local file was downloaded from the internet. + + Args: + filename: path to a local file to import. + """ + try: + fileobj = open(filename, "rb") + except OSError as e: + message.error( + "blockutils: Error while reading {}: {}".format(filename, e.strerror) + ) + return + download = FakeDownload(fileobj) + self._in_progress.append(download) + self._on_download_finished(download) + + def _on_download_finished(self, download: downloads.TempDownload) -> None: + """Check if all downloads are finished and if so, trigger callback. + + Arguments: + download: The finished download. + """ + self._in_progress.remove(download) + if download.successful: + self._done_count += 1 + assert not isinstance(download.fileobj, downloads.UnsupportedAttribute) + assert download.fileobj is not None + try: + # Call the user-provided callback + self.single_download_finished.emit(download.fileobj) + finally: + download.fileobj.close() + if not self._in_progress and self._finished_registering_downloads: + self._finished = True + self.all_downloads_finished.emit(self._done_count) + + +def is_whitelisted_url(url: QUrl) -> bool: + """Check if the given URL is on the adblock whitelist.""" + whitelist = config.val.content.blocking.whitelist + return any(pattern.matches(url) for pattern in whitelist) diff --git a/qutebrowser/config/configdata.yml b/qutebrowser/config/configdata.yml index 645342767..6eb65dce7 100644 --- a/qutebrowser/config/configdata.yml +++ b/qutebrowser/config/configdata.yml @@ -592,12 +592,21 @@ content.headers.user_agent: to JavaScript requires a restart. content.host_blocking.enabled: + renamed: content.blocking.enabled + +content.host_blocking.lists: + renamed: content.blocking.hosts.lists + +content.host_blocking.whitelist: + renamed: content.blocking.whitelist + +content.blocking.enabled: default: true supports_pattern: true type: Bool - desc: Enable host blocking. + desc: Enable the ad/host blocker -content.host_blocking.lists: +content.blocking.hosts.lists: default: - "https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts" type: @@ -605,7 +614,9 @@ content.host_blocking.lists: valtype: Url none_ok: true desc: | - List of URLs of lists which contain hosts to block. + List of URLs to host blocklists for the host blocker. + + Only used when the simple host-blocker is used (see `content.blocking.method`). The file can be in one of the following formats: @@ -619,22 +630,53 @@ content.host_blocking.lists: The file `~/.config/qutebrowser/blocked-hosts` is always read if it exists. -content.host_blocking.whitelist: +content.blocking.method: + default: auto + type: + name: String + valid_values: + - auto: "Use Brave's ABP-style adblocker if available, host blocking + otherwise" + - adblock: Use Brave's ABP-style adblocker + - hosts: Use hosts blocking + - both: Use both hosts blocking and Brave's ABP-style adblocker + desc: | + Which method of blocking ads should be used. + + Support for Adblock Plus (ABP) syntax blocklists using Brave's Rust library requires + the `adblock` Python package to be installed, which is an optional dependency of + qutebrowser. It is required when either `adblock` or `both` are selected. + +content.blocking.adblock.lists: + default: + - "https://easylist.to/easylist/easylist.txt" + - "https://easylist.to/easylist/easyprivacy.txt" + type: + name: List + valtype: Url + none_ok: true + desc: | + List of URLs to ABP-style adblocking rulesets. + + Only used when Brave's ABP-style adblocker is used (see `content.blocking.method`). + +content.blocking.whitelist: default: [] type: name: List valtype: UrlPattern none_ok: true desc: >- - A list of patterns that should always be loaded, despite being ad-blocked. + A list of patterns that should always be loaded, despite being blocked by the + ad-/host-blocker. - Note this whitelists blocked hosts, not first-party URLs. As an example, if - `example.org` loads an ad from `ads.example.org`, the whitelisted host - should be `ads.example.org`. If you want to disable the adblocker on a - given page, use the `content.host_blocking.enabled` setting with a URL - pattern instead. + Local domains are always exempt from adblocking. - Local domains are always exempt from hostblocking. + Note this whitelists otherwise blocked requests, not first-party URLs. As + an example, if `example.org` loads an ad from `ads.example.org`, the + whitelist entry could be `https://ads.example.org/*`. If you want to + disable the adblocker on a given page, use the + `content.blocking.enabled` setting with a URL pattern instead. content.hyperlink_auditing: default: false @@ -1685,7 +1727,11 @@ tabs.last_close: - startpage: "Load the start page." - default-page: "Load the default page." - close: "Close the window." - desc: How to behave when the last tab is closed. + desc: >- + How to behave when the last tab is closed. + + If the `tabs.tabs_are_windows` setting is set, this is ignored and the behavior is + always identical to the `close` value. tabs.mousewheel_switching: default: true diff --git a/qutebrowser/extensions/interceptors.py b/qutebrowser/extensions/interceptors.py index fddeaabc9..9c50346d7 100644 --- a/qutebrowser/extensions/interceptors.py +++ b/qutebrowser/extensions/interceptors.py @@ -59,17 +59,9 @@ class ResourceType(enum.Enum): class RedirectException(Exception): - """Raised when there was an error with redirection.""" - - -class RedirectFailedException(RedirectException): """Raised when the request was invalid, or a request was already made.""" -class RedirectUnsupportedException(RedirectException): - """Raised when redirection is currently unsupported.""" - - @attr.s class Request: @@ -96,14 +88,13 @@ class Request: Only some types of requests can be successfully redirected. Improper use of this method can result in redirect loops. - This method will throw a RedirectFailedException if the request was not - possible. + This method will throw a RedirectException if the request was not possible. Args: url: The QUrl to try to redirect to. """ # Will be overridden if the backend supports redirection - raise RedirectUnsupportedException("Unsupported backend.") + raise NotImplementedError #: Type annotation for an interceptor function. diff --git a/qutebrowser/html/warning-webkit.html b/qutebrowser/html/warning-webkit.html index a46871089..975f98c1b 100644 --- a/qutebrowser/html/warning-webkit.html +++ b/qutebrowser/html/warning-webkit.html @@ -73,7 +73,7 @@ installed.</p> developers about QtWebEngine being "non-free" have repeatedly been disputed, and so far nobody came up with solid evidence about that being the case. Also, note that their qutebrowser package is usually very outdated (even qutebrowser -security fixes took months to arrive there). You might be better off chosing an +security fixes took months to arrive there). You might be better off choosing an <a href="https://qutebrowser.org/doc/install.html#tox"> alternative install method</a>.</p> diff --git a/qutebrowser/javascript/pac_utils.js b/qutebrowser/javascript/pac_utils.js index a7ac2d414..f93c85a87 100644 --- a/qutebrowser/javascript/pac_utils.js +++ b/qutebrowser/javascript/pac_utils.js @@ -145,7 +145,7 @@ function dateRange() { if (isGMT) { argc--; } - // function will work even without explict handling of this case + // function will work even without explicit handling of this case if (argc == 1) { var tmp = parseInt(arguments[0]); if (isNaN(tmp)) { diff --git a/qutebrowser/mainwindow/tabbedbrowser.py b/qutebrowser/mainwindow/tabbedbrowser.py index c67e5fa0e..78eb864a6 100644 --- a/qutebrowser/mainwindow/tabbedbrowser.py +++ b/qutebrowser/mainwindow/tabbedbrowser.py @@ -411,7 +411,11 @@ class TabbedBrowser(QWidget): add_undo: Whether the tab close can be undone. new_undo: Whether the undo entry should be a new item in the stack. """ - last_close = config.val.tabs.last_close + if config.val.tabs.tabs_are_windows: + last_close = 'close' + else: + last_close = config.val.tabs.last_close + count = self.widget.count() if last_close == 'ignore' and count == 1: diff --git a/qutebrowser/mainwindow/tabwidget.py b/qutebrowser/mainwindow/tabwidget.py index f853f8fd9..5263ecff9 100644 --- a/qutebrowser/mainwindow/tabwidget.py +++ b/qutebrowser/mainwindow/tabwidget.py @@ -212,7 +212,7 @@ class TabWidget(QTabWidget): Every single call to setTabText calls the size hinting functions for every single tab, which are slow. Since we know we are updating all the tab's titles, we can delay this processing by making the tab - non-visible. To avoid flickering, disable repaint updates whlie we + non-visible. To avoid flickering, disable repaint updates while we work. """ bar = self.tabBar() diff --git a/qutebrowser/misc/crashdialog.py b/qutebrowser/misc/crashdialog.py index 52cb8ad0c..41160198a 100644 --- a/qutebrowser/misc/crashdialog.py +++ b/qutebrowser/misc/crashdialog.py @@ -83,7 +83,7 @@ def parse_fatal_stacktrace(text): def _get_environment_vars(): """Gather environment variables for the crash info.""" masks = ('DESKTOP_SESSION', 'DE', 'QT_*', 'PYTHON*', 'LC_*', 'LANG', - 'XDG_*', 'QUTE_*', 'PATH') + 'XDG_*', 'QUTE_*', 'PATH', 'XMODIFIERS', 'XIM_*') info = [] for key, value in os.environ.items(): for m in masks: diff --git a/qutebrowser/utils/urlmatch.py b/qutebrowser/utils/urlmatch.py index e4e4984db..527d3403d 100644 --- a/qutebrowser/utils/urlmatch.py +++ b/qutebrowser/utils/urlmatch.py @@ -125,7 +125,7 @@ class UrlPattern: def _fixup_pattern(self, pattern: str) -> str: """Make sure the given pattern is parseable by urllib.parse.""" - if pattern.startswith('*:'): # Any scheme, but *:// is unparseable + if pattern.startswith('*:'): # Any scheme, but *:// is unparsable pattern = 'any:' + pattern[2:] schemes = tuple(s + ':' for s in self._SCHEMES_WITHOUT_HOST) diff --git a/qutebrowser/utils/utils.py b/qutebrowser/utils/utils.py index 331bf5f96..cf239805c 100644 --- a/qutebrowser/utils/utils.py +++ b/qutebrowser/utils/utils.py @@ -38,6 +38,15 @@ import ctypes import ctypes.util from typing import (Any, Callable, IO, Iterator, Optional, Sequence, Tuple, Type, Union, TYPE_CHECKING, cast) +try: + # Protocol was added in Python 3.8 + from typing import Protocol +except ImportError: # pragma: no cover + if not TYPE_CHECKING: + class Protocol: + + """Empty stub at runtime.""" + from PyQt5.QtCore import QUrl, QVersionNumber from PyQt5.QtGui import QClipboard, QDesktopServices @@ -72,16 +81,6 @@ is_windows = sys.platform.startswith('win') is_posix = os.name == 'posix' -try: - # Protocol was added in Python 3.8 - from typing import Protocol # pylint: disable=ungrouped-imports -except ImportError: # pragma: no cover - if not TYPE_CHECKING: - class Protocol: - - """Empty stub at runtime.""" - - class SupportsLessThan(Protocol): """Protocol for a "comparable" object.""" diff --git a/qutebrowser/utils/version.py b/qutebrowser/utils/version.py index 09aeb5a13..175d0d715 100644 --- a/qutebrowser/utils/version.py +++ b/qutebrowser/utils/version.py @@ -248,43 +248,139 @@ def _release_info() -> Sequence[Tuple[str, str]]: return data -def _module_versions() -> Sequence[str]: - """Get versions of optional modules. - - Return: - A list of lines with version info. +class ModuleInfo: + + """Class to query version information of qutebrowser dependencies. + + Attributes: + name: Name of the module as it is imported. + _version_attributes: + Sequence of attribute names belonging to the module which may hold + version information. + min_version: Minimum version of this module which qutebrowser can use. + _installed: Is the module installed? Determined at runtime. + _version: Version of the module. Determined at runtime. + _initialized: + Set to `True` if the `self._installed` and `self._version` + attributes have been set. """ - lines = [] - modules: Mapping[str, Sequence[str]] = collections.OrderedDict([ + + def __init__( + self, + name: str, + version_attributes: Sequence[str], + min_version: Optional[str] = None + ): + self.name = name + self._version_attributes = version_attributes + self.min_version = min_version + self._installed = False + self._version: Optional[str] = None + self._initialized = False + + def _reset_cache(self) -> None: + """Reset the version cache. + + It is necessary to call this method in unit tests that mock a module's + version number. + """ + self._installed = False + self._version = None + self._initialized = False + + def _initialize_info(self) -> None: + """Import module and set `self.installed` and `self.version`.""" + try: + module = importlib.import_module(self.name) + except (ImportError, ValueError): + self._installed = False + return + else: + self._installed = True + + for attribute_name in self._version_attributes: + if hasattr(module, attribute_name): + version = getattr(module, attribute_name) + assert isinstance(version, (str, float)) + self._version = str(version) + break + + self._initialized = True + + def get_version(self) -> Optional[str]: + """Finds the module version if it exists.""" + if not self._initialized: + self._initialize_info() + return self._version + + def is_installed(self) -> bool: + """Checks whether the module is installed.""" + if not self._initialized: + self._initialize_info() + return self._installed + + def is_outdated(self) -> Optional[bool]: + """Checks whether the module is outdated. + + Return: + A boolean when the version and minimum version are both defined. + Otherwise `None`. + """ + version = self.get_version() + if ( + not self.is_installed() + or version is None + or self.min_version is None + ): + return None + return version < self.min_version + + def is_usable(self) -> bool: + """Whether the module is both installed and not outdated.""" + return self.is_installed() and not self.is_outdated() + + def __str__(self) -> str: + if not self.is_installed(): + return f'{self.name}: no' + + version = self.get_version() + if version is None: + return f'{self.name}: yes' + + text = f'{self.name}: {version}' + if self.is_outdated(): + text += f" (< {self.min_version}, outdated)" + return text + + +MODULE_INFO: Mapping[str, ModuleInfo] = collections.OrderedDict([ + # FIXME: Mypy doesn't understand this. See https://github.com/python/mypy/issues/9706 + (name, ModuleInfo(name, *args)) # type: ignore[arg-type, misc] + for (name, *args) in + [ ('sip', ['SIP_VERSION_STR']), ('colorama', ['VERSION', '__version__']), ('pypeg2', ['__version__']), ('jinja2', ['__version__']), ('pygments', ['__version__']), ('yaml', ['__version__']), + ('adblock', ['__version__'], "0.3.2"), ('attr', ['__version__']), ('importlib_resources', []), ('PyQt5.QtWebEngineWidgets', []), ('PyQt5.QtWebEngine', ['PYQT_WEBENGINE_VERSION_STR']), ('PyQt5.QtWebKitWidgets', []), - ]) - for modname, attributes in modules.items(): - try: - module = importlib.import_module(modname) - except (ImportError, ValueError): - text = '{}: no'.format(modname) - else: - for name in attributes: - try: - text = '{}: {}'.format(modname, getattr(module, name)) - except AttributeError: - pass - else: - break - else: - text = '{}: yes'.format(modname) - lines.append(text) - return lines + ] +]) + + +def _module_versions() -> Sequence[str]: + """Get versions of optional modules. + + Return: + A list of lines with version info. + """ + return [str(mod_info) for mod_info in MODULE_INFO.values()] def _path_info() -> Mapping[str, str]: diff --git a/requirements.txt b/requirements.txt index 34412dadb..48c1991a3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,9 @@ # This file is automatically generated by scripts/dev/recompile_requirements.py +adblock==0.4.0 attrs==20.3.0 colorama==0.4.4 -importlib-resources==3.3.0 ; python_version<"3.9" +importlib-resources==4.1.1 ; python_version<"3.9" Jinja2==2.11.2 MarkupSafe==1.1.1 Pygments==2.7.3 diff --git a/scripts/dev/check_coverage.py b/scripts/dev/check_coverage.py index aa88c97a3..dae90d636 100644 --- a/scripts/dev/check_coverage.py +++ b/scripts/dev/check_coverage.py @@ -232,6 +232,7 @@ WHITELISTED_FILES = [ 'qutebrowser/keyinput/macros.py', 'qutebrowser/browser/webkit/webkitelem.py', 'qutebrowser/api/interceptor.py', + 'qutebrowser/extensions/interceptors.py', ] diff --git a/scripts/dev/misc_checks.py b/scripts/dev/misc_checks.py index ad446412c..4b5699086 100644 --- a/scripts/dev/misc_checks.py +++ b/scripts/dev/misc_checks.py @@ -161,7 +161,8 @@ def check_spelling(args: argparse.Namespace) -> Optional[bool]: 'exitted', 'mininum', 'resett?ed', 'recieved', 'regularily', 'underlaying', 'inexistant', 'elipsis', 'commiting', 'existant', 'resetted', 'similarily', 'informations', 'an url', 'treshold', - 'artefact', 'an unix', 'an utf', 'an unicode'} + 'artefact', 'an unix', 'an utf', 'an unicode', 'unparseable', + 'dependancies', 'convertable', 'chosing', 'authentification'} # Words which look better when splitted, but might need some fine tuning. words |= {'webelements', 'mouseevent', 'keysequence', 'normalmode', diff --git a/scripts/dev/recompile_requirements.py b/scripts/dev/recompile_requirements.py index 3b7c227f2..e43a3111d 100644 --- a/scripts/dev/recompile_requirements.py +++ b/scripts/dev/recompile_requirements.py @@ -173,8 +173,9 @@ CHANGELOG_URLS = { 'jinja2-pluralize': 'https://github.com/audreyfeldroy/jinja2_pluralize/blob/master/HISTORY.rst', 'mypy-extensions': 'https://github.com/python/mypy_extensions/commits/master', 'pyroma': 'https://github.com/regebro/pyroma/blob/master/HISTORY.txt', + 'adblock': 'https://github.com/ArniDagur/python-adblock/blob/master/CHANGELOG.md', 'pyPEG2': None, - 'importlib-resources': 'https://importlib-resources.readthedocs.io/en/latest/changelog%20%28links%29.html', + 'importlib-resources': 'https://importlib-resources.readthedocs.io/en/latest/history.html', } diff --git a/scripts/dev/run_vulture.py b/scripts/dev/run_vulture.py index 5e42febd4..05fcc9134 100755 --- a/scripts/dev/run_vulture.py +++ b/scripts/dev/run_vulture.py @@ -135,7 +135,10 @@ def whitelist_generator(): # noqa: C901 yield 'scripts.importer.import_moz_places.places.row_factory' # component hooks - yield 'qutebrowser.components.adblock.on_config_changed' + yield 'qutebrowser.components.hostblock.on_lists_changed' + yield 'qutebrowser.components.braveadblock.on_lists_changed' + yield 'qutebrowser.components.hostblock.on_method_changed' + yield 'qutebrowser.components.braveadblock.on_method_changed' # used in type comments yield 'pending_download_type' diff --git a/scripts/dev/src2asciidoc.py b/scripts/dev/src2asciidoc.py index aefbff7f8..60cb3d611 100755 --- a/scripts/dev/src2asciidoc.py +++ b/scripts/dev/src2asciidoc.py @@ -79,7 +79,7 @@ class UsageFormatter(argparse.HelpFormatter): def _metavar_formatter(self, action, default_metavar): """Override _metavar_formatter to add asciidoc markup to metavars. - Most code here is copied from Python 3.4's argparse.py. + Most code here is copied from Python 3.10's argparse.py. """ if action.metavar is not None: result = "'{}'".format(action.metavar) @@ -115,6 +115,19 @@ class UsageFormatter(argparse.HelpFormatter): action.option_strings = old_option_strings[action] return ret + def _format_args(self, action, default_metavar): + """Backport simplified star nargs usage. + + https://github.com/python/cpython/pull/17106 + """ + if sys.version_info >= (3, 9) or action.nargs != argparse.ZERO_OR_MORE: + return super()._format_args(action, default_metavar) + + get_metavar = self._metavar_formatter(action, default_metavar) + metavar = get_metavar(1) + assert len(metavar) == 1 + return f'[{metavar[0]} ...]' + def _open_file(name, mode='w'): """Open a file with a preset newline/encoding mode.""" diff --git a/scripts/hostblock_blame.py b/scripts/hostblock_blame.py index 2949b9e78..7b5d787a3 100644 --- a/scripts/hostblock_blame.py +++ b/scripts/hostblock_blame.py @@ -27,7 +27,7 @@ import os.path import urllib.request sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir)) -from qutebrowser.components import adblock +from qutebrowser.components import hostblock from qutebrowser.config import configdata @@ -39,11 +39,11 @@ def main(): configdata.init() - for url in configdata.DATA['content.host_blocking.lists'].default: + for url in configdata.DATA['content.blocking.hosts.lists'].default: print("checking {}...".format(url)) raw_file = urllib.request.urlopen(url) byte_io = io.BytesIO(raw_file.read()) - f = adblock.get_fileobj(byte_io) + f = hostblock.get_fileobj(byte_io) for line in f: line = line.decode('utf-8') if sys.argv[1] in line: diff --git a/scripts/importer.py b/scripts/importer.py index bae483b09..111082072 100755 --- a/scripts/importer.py +++ b/scripts/importer.py @@ -22,27 +22,19 @@ """Tool to import data from other browsers. -Currently importing bookmarks from Netscape Bookmark files and Mozilla -profiles is supported. +Currently importing bookmarks from Netscape HTML Bookmark files, Chrome +profiles, and Mozilla profiles is supported. """ import argparse +import textwrap import sqlite3 import os import urllib.parse import json import string -browser_default_input_format = { - 'chromium': 'chrome', - 'chrome': 'chrome', - 'ie': 'netscape', - 'firefox': 'mozilla', - 'seamonkey': 'mozilla', - 'palemoon': 'mozilla', -} - def main(): args = get_args() @@ -68,15 +60,9 @@ def main(): bookmark_types = ['bookmark', 'keyword'] if not output_format: output_format = 'quickmark' - if not input_format: - if args.browser: - input_format = browser_default_input_format[args.browser] - else: - #default to netscape - input_format = 'netscape' import_function = { - 'netscape': import_netscape_bookmarks, + 'html': import_html_bookmarks, 'mozilla': import_moz_places, 'chrome': import_chrome, } @@ -87,20 +73,25 @@ def main(): def get_args(): """Get the argparse parser.""" parser = argparse.ArgumentParser( - epilog="To import bookmarks from Chromium, Firefox or IE, " - "export them to HTML in your browsers bookmark manager. ") - parser.add_argument( - 'browser', - help="Which browser? {%(choices)s}", - choices=browser_default_input_format.keys(), - nargs='?', - metavar='browser') + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=textwrap.dedent(''' + To import bookmarks, you'll need the path to your profile or an + exported HTML file from your browser's bookmark manager. Redirect + the output from this script to the appropriate file in your + qutebrowser config directory (listed in the output of :version), + usually done with the '>' operator; for example, + ./importer.py -i mozilla your_profile_path > ~/.config/qutebrowser/quickmarks + + Common browsers with native input format support: + chrome: Chrome, Chromium, Edge + mozilla: Firefox, SeaMonkey, Pale Moon + ''')) parser.add_argument( '-i', '--input-format', - help='Which input format? (overrides browser default; "netscape" if ' - 'neither given)', - choices=set(browser_default_input_format.values()), + help="Which input format? Defaults to html", + choices=['html', 'mozilla', 'chrome'], + default='html', required=False) parser.add_argument( '-b', @@ -186,7 +177,7 @@ def opensearch_convert(url): return search_escape(url.format(**subst)).replace('%s', '{}') -def import_netscape_bookmarks(bookmarks_file, bookmark_types, output_format): +def import_html_bookmarks(bookmarks_file, bookmark_types, output_format): """Import bookmarks from a NETSCAPE-Bookmark-file v1. Generated by Chromium, Firefox, IE and possibly more browsers. Not all diff --git a/scripts/mkvenv.py b/scripts/mkvenv.py index 897088539..57e6fe3d1 100644 --- a/scripts/mkvenv.py +++ b/scripts/mkvenv.py @@ -57,6 +57,9 @@ def print_command(*cmd: Union[str, pathlib.Path], venv: bool) -> None: def parse_args(argv: List[str] = None) -> argparse.Namespace: """Parse commandline arguments.""" parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--update', + action='store_true', + help="Run 'git pull' before creating the environment.") parser.add_argument('--keep', action='store_true', help="Reuse an existing virtualenv.") @@ -395,10 +398,35 @@ def regenerate_docs(venv_dir: pathlib.Path, run_venv(venv_dir, 'python', str(script_path), *a2h_args) +def update_repo(): + """Update the git repository via git pull.""" + print_command('git pull', venv=False) + try: + subprocess.run(['git', 'pull'], check=True) + except subprocess.CalledProcessError as e: + raise Error("git pull failed, exiting") from e + + +def install_pyqt(venv_dir, args): + """Install PyQt in the virtualenv.""" + if args.pyqt_type == 'binary': + install_pyqt_binary(venv_dir, args.pyqt_version) + elif args.pyqt_type == 'source': + install_pyqt_source(venv_dir, args.pyqt_version) + elif args.pyqt_type == 'link': + install_pyqt_link(venv_dir) + elif args.pyqt_type == 'wheels': + wheels_dir = pathlib.Path(args.pyqt_wheels_dir) + install_pyqt_wheels(venv_dir, wheels_dir) + elif args.pyqt_type == 'skip': + pass + else: + raise AssertionError + + def run(args) -> None: """Install qutebrowser in a virtualenv..""" venv_dir = pathlib.Path(args.venv_dir) - wheels_dir = pathlib.Path(args.pyqt_wheels_dir) utils.change_cwd() if (args.pyqt_version != 'auto' and @@ -410,25 +438,17 @@ def run(args) -> None: raise Error('The --pyqt-wheels-dir option is only available when installing ' 'PyQt from wheels') + if args.update: + utils.print_title("Updating repository") + update_repo() + if not args.keep: utils.print_title("Creating virtual environment") delete_old_venv(venv_dir) create_venv(venv_dir, use_virtualenv=args.virtualenv) upgrade_seed_pkgs(venv_dir) - - if args.pyqt_type == 'binary': - install_pyqt_binary(venv_dir, args.pyqt_version) - elif args.pyqt_type == 'source': - install_pyqt_source(venv_dir, args.pyqt_version) - elif args.pyqt_type == 'link': - install_pyqt_link(venv_dir) - elif args.pyqt_type == 'wheels': - install_pyqt_wheels(venv_dir, wheels_dir) - elif args.pyqt_type == 'skip': - pass - else: - raise AssertionError + install_pyqt(venv_dir, args) apply_xcb_util_workaround(venv_dir, args.pyqt_type, args.pyqt_version) if args.pyqt_type != 'skip' and not args.skip_smoke_test: diff --git a/tests/end2end/data/adblock/simple b/tests/end2end/data/adblock/simple deleted file mode 100644 index 5778335db..000000000 --- a/tests/end2end/data/adblock/simple +++ /dev/null @@ -1 +0,0 @@ -example.org diff --git a/tests/end2end/data/adblock/external_logo.html b/tests/end2end/data/blocking/external_logo.html index 7fa7e9ebb..7fa7e9ebb 100644 --- a/tests/end2end/data/adblock/external_logo.html +++ b/tests/end2end/data/blocking/external_logo.html diff --git a/tests/end2end/data/blocking/qutebrowser-adblock b/tests/end2end/data/blocking/qutebrowser-adblock new file mode 100644 index 000000000..4b279b32c --- /dev/null +++ b/tests/end2end/data/blocking/qutebrowser-adblock @@ -0,0 +1 @@ +||qutebrowser.org^ diff --git a/tests/end2end/data/adblock/qutebrowser b/tests/end2end/data/blocking/qutebrowser-hosts index d104c0104..d104c0104 100644 --- a/tests/end2end/data/adblock/qutebrowser +++ b/tests/end2end/data/blocking/qutebrowser-hosts diff --git a/tests/end2end/data/brave-adblock/LICENSE b/tests/end2end/data/brave-adblock/LICENSE new file mode 100644 index 000000000..6f03f190d --- /dev/null +++ b/tests/end2end/data/brave-adblock/LICENSE @@ -0,0 +1,318 @@ +Mozilla Public License, version 2.0 + +Copyright (c) 2019, Andrius Aucinas + +1. Definitions + + 1.1. “Contributor” means each individual or legal entity that + creates, contributes to the creation of, or owns Covered Software. + + 1.2. “Contributor Version” means the combination of the + Contributions of others (if any) used by a Contributor and that + particular Contributor’s Contribution. + + 1.3. “Contribution” means Covered Software of a particular + Contributor. + + 1.4. “Covered Software” means Source Code Form to which the initial + Contributor has attached the notice in Exhibit A, the Executable + Form of such Source Code Form, and Modifications of such Source Code + Form, in each case including portions thereof. + + 1.5. “Incompatible With Secondary Licenses” means + + that the initial Contributor has attached the notice described in + Exhibit B to the Covered Software; or + + that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the terms + of a Secondary License. + + 1.6. “Executable Form” means any form of the work other than Source + Code Form. + + 1.7. “Larger Work” means a work that combines Covered Software with + other material, in a separate file or files, that is not Covered + Software. + + 1.8. “License” means this document. + + 1.9. “Licensable” means having the right to grant, to the maximum + extent possible, whether at the time of the initial grant or + subsequently, any and all of the rights conveyed by this License. + + 1.10. “Modifications” means any of the following: + + any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered Software; + or + + any new file in Source Code Form that contains any Covered Software. + + 1.11. “Patent Claims” of a Contributor means any patent claim(s), + including without limitation, method, process, and apparatus claims, + in any patent Licensable by such Contributor that would be + infringed, but for the grant of the License, by the making, using, + selling, offering for sale, having made, import, or transfer of + either its Contributions or its Contributor Version. + + 1.12. “Secondary License” means either the GNU General Public + License, Version 2.0, the GNU Lesser General Public License, Version + 2.1, the GNU Affero General Public License, Version 3.0, or any + later versions of those licenses. + + 1.13. “Source Code Form” means the form of the work preferred for + making modifications. + + 1.14. “You” (or “Your”) means an individual or a legal entity + exercising rights under this License. For legal entities, “You” + includes any entity that controls, is controlled by, or is under + common control with You. For purposes of this definition, “control” + means (a) the power, direct or indirect, to cause the direction or + management of such entity, whether by contract or otherwise, or (b) + ownership of more than fifty percent (50%) of the outstanding shares + or beneficial ownership of such entity. + + +2. License Grants and Conditions + + 2.1. Grants Each Contributor hereby grants You a world-wide, + royalty-free, non-exclusive license: + + under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + + under Patent Claims of such Contributor to make, use, sell, offer + for sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + + 2.2. Effective Date The licenses granted in Section 2.1 with respect + to any Contribution become effective for each Contribution on the + date the Contributor first distributes such Contribution. + + 2.3. Limitations on Grant Scope The licenses granted in this Section + 2 are the only rights granted under this License. No additional + rights or licenses will be implied from the distribution or + licensing of Covered Software under this License. Notwithstanding + Section 2.1(b) above, no patent license is granted by a Contributor: + + for any code that a Contributor has removed from Covered Software; + or + + for infringements caused by: (i) Your and any other third party’s + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + + under Patent Claims infringed by Covered Software in the absence of + its Contributions. + + This License does not grant any rights in the trademarks, service + marks, or logos of any Contributor (except as may be necessary to + comply with the notice requirements in Section 3.4). + + 2.4. Subsequent Licenses No Contributor makes additional grants as a + result of Your choice to distribute the Covered Software under a + subsequent version of this License (see Section 10.2) or under the + terms of a Secondary License (if permitted under the terms of + Section 3.3). + + 2.5. Representation Each Contributor represents that the Contributor + believes its Contributions are its original creation(s) or it has + sufficient rights to grant the rights to its Contributions conveyed + by this License. + + 2.6. Fair Use This License is not intended to limit any rights You + have under applicable copyright doctrines of fair use, fair dealing, + or other equivalents. + + 2.7. Conditions Sections 3.1, 3.2, 3.3, and 3.4 are conditions of + the licenses granted in Section 2.1. + + +3. Responsibilities + + 3.1. Distribution of Source Form All distribution of Covered + Software in Source Code Form, including any Modifications that You + create or to which You contribute, must be under the terms of this + License. You must inform recipients that the Source Code Form of the + Covered Software is governed by the terms of this License, and how + they can obtain a copy of this License. You may not attempt to alter + or restrict the recipients’ rights in the Source Code Form. + + 3.2. Distribution of Executable Form If You distribute Covered + Software in Executable Form then: + + such Covered Software must also be made available in Source Code + Form, as described in Section 3.1, and You must inform recipients of + the Executable Form how they can obtain a copy of such Source Code + Form by reasonable means in a timely manner, at a charge no more + than the cost of distribution to the recipient; and + + You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter + the recipients’ rights in the Source Code Form under this License. + + 3.3. Distribution of a Larger Work You may create and distribute a + Larger Work under terms of Your choice, provided that You also + comply with the requirements of this License for the Covered + Software. If the Larger Work is a combination of Covered Software + with a work governed by one or more Secondary Licenses, and the + Covered Software is not Incompatible With Secondary Licenses, this + License permits You to additionally distribute such Covered Software + under the terms of such Secondary License(s), so that the recipient + of the Larger Work may, at their option, further distribute the + Covered Software under the terms of either this License or such + Secondary License(s). + + 3.4. Notices You may not remove or alter the substance of any + license notices (including copyright notices, patent notices, + disclaimers of warranty, or limitations of liability) contained + within the Source Code Form of the Covered Software, except that You + may alter any license notices to the extent required to remedy known + factual inaccuracies. + + 3.5. Application of Additional Terms You may choose to offer, and to + charge a fee for, warranty, support, indemnity or liability + obligations to one or more recipients of Covered Software. However, + You may do so only on Your own behalf, and not on behalf of any + Contributor. You must make it absolutely clear that any such + warranty, support, indemnity, or liability obligation is offered by + You alone, and You hereby agree to indemnify every Contributor for + any liability incurred by such Contributor as a result of warranty, + support, indemnity or liability terms You offer. You may include + additional disclaimers of warranty and limitations of liability + specific to any jurisdiction. + + +4. Inability to Comply Due to Statute or Regulation + + If it is impossible for You to comply with any of the terms of this + License with respect to some or all of the Covered Software due to + statute, judicial order, or regulation then You must: (a) comply + with the terms of this License to the maximum extent possible; and + (b) describe the limitations and the code they affect. Such + description must be placed in a text file included with all + distributions of the Covered Software under this License. Except to + the extent prohibited by statute or regulation, such description + must be sufficiently detailed for a recipient of ordinary skill to + be able to understand it. + + +5. Termination + + 5.1. The rights granted under this License will terminate + automatically if You fail to comply with any of its terms. However, + if You become compliant, then the rights granted under this License + from a particular Contributor are reinstated (a) provisionally, + unless and until such Contributor explicitly and finally terminates + Your grants, and (b) on an ongoing basis, if such Contributor fails + to notify You of the non-compliance by some reasonable means prior + to 60 days after You have come back into compliance. Moreover, Your + grants from a particular Contributor are reinstated on an ongoing + basis if such Contributor notifies You of the non-compliance by some + reasonable means, this is the first time You have received notice of + non-compliance with this License from such Contributor, and You + become compliant prior to 30 days after Your receipt of the notice. + + 5.2. If You initiate litigation against any entity by asserting a + patent infringement claim (excluding declaratory judgment actions, + counter-claims, and cross-claims) alleging that a Contributor + Version directly or indirectly infringes any patent, then the rights + granted to You by any and all Contributors for the Covered Software + under Section 2.1 of this License shall terminate. + + 5.3. In the event of termination under Sections 5.1 or 5.2 above, + all end user license agreements (excluding distributors and + resellers) which have been validly granted by You or Your + distributors under this License prior to termination shall survive + termination. + +6. Disclaimer of Warranty Covered Software is provided under this +License on an “as is” basis, without warranty of any kind, either +expressed, implied, or statutory, including, without limitation, +warranties that the Covered Software is free of defects, merchantable, +fit for a particular purpose or non-infringing. The entire risk as to +the quality and performance of the Covered Software is with You. +Should any Covered Software prove defective in any respect, You (not +any Contributor) assume the cost of any necessary servicing, repair, +or correction. This disclaimer of warranty constitutes an essential +part of this License. No use of any Covered Software is authorized +under this License except under this disclaimer. + +7. Limitation of Liability Under no circumstances and under no legal +theory, whether tort (including negligence), contract, or otherwise, +shall any Contributor, or anyone who distributes Covered Software as +permitted above, be liable to You for any direct, indirect, special, +incidental, or consequential damages of any character including, +without limitation, damages for lost profits, loss of goodwill, work +stoppage, computer failure or malfunction, or any and all other +commercial damages or losses, even if such party shall have been +informed of the possibility of such damages. This limitation of +liability shall not apply to liability for death or personal injury +resulting from such party’s negligence to the extent applicable law +prohibits such limitation. Some jurisdictions do not allow the +exclusion or limitation of incidental or consequential damages, so +this exclusion and limitation may not apply to You. + +8. Litigation Any litigation relating to this License may be brought +only in the courts of a jurisdiction where the defendant maintains its +principal place of business and such litigation shall be governed by +laws of that jurisdiction, without reference to its conflict-of-law +provisions. Nothing in this Section shall prevent a party’s ability to +bring cross-claims or counter-claims. + +9. Miscellaneous This License represents the complete agreement +concerning the subject matter hereof. If any provision of this License +is held to be unenforceable, such provision shall be reformed only to +the extent necessary to make it enforceable. Any law or regulation +which provides that the language of a contract shall be construed +against the drafter shall not be used to construe this License against +a Contributor. + +10. Versions of the License + + 10.1. New Versions Mozilla Foundation is the license steward. Except + as provided in Section 10.3, no one other than the license steward + has the right to modify or publish new versions of this License. + Each version will be given a distinguishing version number. + + 10.2. Effect of New Versions You may distribute the Covered Software + under the terms of the version of the License under which You + originally received the Covered Software, or under the terms of any + subsequent version published by the license steward. + + 10.3. Modified Versions If you create software not governed by this + License, and you want to create a new license for such software, you + may create and use a modified version of this License if you rename + the license and remove any references to the name of the license + steward (except to note that such modified license differs from this + License). + + 10.4. Distributing Source Code Form that is Incompatible With + Secondary Licenses If You choose to distribute Source Code Form that + is Incompatible With Secondary Licenses under the terms of this + version of the License, the notice described in Exhibit B of this + License must be attached. + +Exhibit A - Source Code Form License Notice + + This Source Code Form is subject to the terms of the Mozilla Public + License, v. 2.0. If a copy of the MPL was not distributed with this + file, You can obtain one at http://mozilla.org/MPL/2.0/. + + If it is not possible or desirable to put the notice in a particular + file, then You may include the notice in a location (such as a + LICENSE file in a relevant directory) where a recipient would be + likely to look for such a notice. + + You may add additional accurate notices of copyright ownership. + +Exhibit B - “Incompatible With Secondary Licenses” Notice + + This Source Code Form is “Incompatible With Secondary Licenses”, as + defined by the Mozilla Public License, v. 2.0. diff --git a/tests/end2end/data/brave-adblock/README.md b/tests/end2end/data/brave-adblock/README.md new file mode 100644 index 000000000..0550e91a4 --- /dev/null +++ b/tests/end2end/data/brave-adblock/README.md @@ -0,0 +1,12 @@ +The `ublock-matches.tsv` file is [downloaded from][1] `adblock-rust`'s Github and preprocessed and compressed using `generate.py` to produce +`ublock-matches.tsv.gz`. + +## License + +The aforementioned file was released under terms of the Mozilla Public +License, version 2.0 (MPLv2) by Andrius Aucinas. A copy of the license may be +found in the [`LICENSE`][2] file of this directory, or on [Mozilla's website][3]. + +[1]: https://github.com/brave/adblock-rust/blob/master/data/ublock-matches.tsv +[2]: LICENSE +[3]: https://www.mozilla.org/en-US/MPL/2.0/ diff --git a/tests/end2end/data/brave-adblock/generate.py b/tests/end2end/data/brave-adblock/generate.py new file mode 100644 index 000000000..ae47c586b --- /dev/null +++ b/tests/end2end/data/brave-adblock/generate.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2020 Árni Dagur <arni@dagur.eu> +# +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>. + +import io +import gzip +import csv +import pathlib +import itertools +import urllib.request +import tempfile +from typing import Optional + +URL = "https://raw.githubusercontent.com/brave/adblock-rust/master/data/ublock-matches.tsv" +CACHE_PATH = pathlib.Path(tempfile.gettempdir(), "ublock-matches-cache.tsv") +ROWS_TO_USE = 30_000 + + +def type_rename(type_str: str) -> Optional[str]: + """Use the same resource type names as QtWebEngine.""" + if type_str == "other": + return "unknown" + if type_str == "xmlhttprequest": + return "xhr" + if type_str == "font": + return "font_resource" + if type_str in ["image", "stylesheet", "media", "script", "sub_frame"]: + return type_str + return None + + +def main(): + # Download file or use cached version + if CACHE_PATH.is_file(): + print(f"Using cached file {CACHE_PATH}") + data = io.StringIO(CACHE_PATH.read_text(encoding="utf-8")) + else: + request = urllib.request.Request(URL) + print(f"Downloading {URL} ...") + response = urllib.request.urlopen(request) + assert response.status == 200 + data_str = response.read().decode("utf-8") + print(f"Saving to cache file {CACHE_PATH} ...") + CACHE_PATH.write_text(data_str, encoding="utf-8") + data = io.StringIO(data_str) + + # We only want the first three columns and the first ROWS_TO_USE rows + print("Reading rows into memory...") + reader = csv.DictReader(data, delimiter="\t") + rows = list(itertools.islice(reader, ROWS_TO_USE)) + + print("Writing filtered file to memory...") + uncompressed_f = io.StringIO() + writer = csv.DictWriter( + uncompressed_f, ["url", "source_url", "type"], delimiter="\t" + ) + writer.writeheader() + for row in rows: + type_renamed = type_rename(row["type"]) + if type_renamed is None: + # Ignore request types we don't recognize + continue + writer.writerow( + { + "url": row["url"], + "source_url": row["sourceUrl"], + "type": type_renamed, + } + ) + uncompressed_f.seek(0) + + print("Compressing filtered file and saving to disk...") + # Compress the data before storing on the filesystem + with gzip.open("ublock-matches.tsv.gz", "wb", compresslevel=9) as gzip_f: + gzip_f.write(uncompressed_f.read().encode("utf-8")) + + +if __name__ == "__main__": + main() diff --git a/tests/end2end/data/brave-adblock/ublock-matches.tsv.gz b/tests/end2end/data/brave-adblock/ublock-matches.tsv.gz Binary files differnew file mode 100644 index 000000000..bced0da75 --- /dev/null +++ b/tests/end2end/data/brave-adblock/ublock-matches.tsv.gz diff --git a/tests/end2end/data/easylist.txt.gz b/tests/end2end/data/easylist.txt.gz Binary files differnew file mode 100644 index 000000000..b854af6f5 --- /dev/null +++ b/tests/end2end/data/easylist.txt.gz diff --git a/tests/end2end/data/easyprivacy.txt.gz b/tests/end2end/data/easyprivacy.txt.gz Binary files differnew file mode 100644 index 000000000..6ee5e2319 --- /dev/null +++ b/tests/end2end/data/easyprivacy.txt.gz diff --git a/tests/end2end/features/keyinput.feature b/tests/end2end/features/keyinput.feature index aa9009f7c..0e344947d 100644 --- a/tests/end2end/features/keyinput.feature +++ b/tests/end2end/features/keyinput.feature @@ -56,11 +56,17 @@ Feature: Keyboard input Scenario: :fake-key sending keychain to the website When I open data/keyinput/log.html - And I run :fake-key xy + And I run :fake-key x<greater>y<less>" " Then the javascript message "key press: 88" should be logged And the javascript message "key release: 88" should be logged + And the javascript message "key press: 190" should be logged + And the javascript message "key release: 190" should be logged And the javascript message "key press: 89" should be logged And the javascript message "key release: 89" should be logged + And the javascript message "key press: 188" should be logged + And the javascript message "key release: 188" should be logged + And the javascript message "key press: 32" should be logged + And the javascript message "key release: 32" should be logged Scenario: :fake-key sending keypress to qutebrowser When I run :fake-key -g x diff --git a/tests/end2end/features/misc.feature b/tests/end2end/features/misc.feature index 06dc0b805..570bd3321 100644 --- a/tests/end2end/features/misc.feature +++ b/tests/end2end/features/misc.feature @@ -525,12 +525,7 @@ Feature: Various utility commands. ## Other - Scenario: Simple adblock update - When I set up "simple" as block lists - And I run :adblock-update - Then the message "adblock: Read 1 hosts from 1 sources." should be shown - Scenario: Resource with invalid URL - When I open data/invalid_resource.html + When I open data/invalid_resource.html in a new tab Then "Ignoring invalid * URL: Invalid hostname (contains invalid characters); *" should be logged And no crash should happen diff --git a/tests/end2end/features/private.feature b/tests/end2end/features/private.feature index 2698555ab..11b2dc0ab 100644 --- a/tests/end2end/features/private.feature +++ b/tests/end2end/features/private.feature @@ -220,10 +220,11 @@ Feature: Using private browsing Scenario: Adblocking after reiniting private profile When I open about:blank in a private window And I run :close - And I set content.host_blocking.lists to ["http://localhost:(port)/data/adblock/qutebrowser"] + And I set content.blocking.hosts.lists to ["http://localhost:(port)/data/blocking/qutebrowser-hosts"] + And I set content.blocking.method to hosts And I run :adblock-update - And I wait for the message "adblock: Read 1 hosts from 1 sources." - And I open data/adblock/external_logo.html in a private window + And I wait for the message "hostblock: Read 1 hosts from 1 sources." + And I open data/blocking/external_logo.html in a private window Then "Request to qutebrowser.org blocked by host blocker." should be logged @pyqt!=5.15.0 # cookie filtering is broken on QtWebEngine 5.15.0 diff --git a/tests/end2end/features/prompts.feature b/tests/end2end/features/prompts.feature index a0b550054..8562b50c4 100644 --- a/tests/end2end/features/prompts.feature +++ b/tests/end2end/features/prompts.feature @@ -1,7 +1,7 @@ # vim: ft=cucumber fileencoding=utf-8 sts=4 sw=4 et: Feature: Prompts - Various prompts (javascript, SSL errors, authentification, etc.) + Various prompts (javascript, SSL errors, authentication, etc.) # Javascript @@ -319,7 +319,7 @@ Feature: Prompts # Page authentication - Scenario: Successful webpage authentification + Scenario: Successful webpage authentication When I open basic-auth/user1/password1 without waiting And I wait for a prompt And I press the keys "user1" @@ -372,7 +372,7 @@ Feature: Prompts } @qtwebengine_skip - Scenario: Cancellling webpage authentification with QtWebKit + Scenario: Cancellling webpage authentication with QtWebKit When I open basic-auth/user6/password6 without waiting And I wait for a prompt And I run :leave-mode diff --git a/tests/end2end/features/spawn.feature b/tests/end2end/features/spawn.feature index e0972da20..11b344439 100644 --- a/tests/end2end/features/spawn.feature +++ b/tests/end2end/features/spawn.feature @@ -14,7 +14,7 @@ Feature: :spawn When I run :spawn -u this_does_not_exist Then the error "Userscript 'this_does_not_exist' not found in userscript directories *" should be shown - Scenario: Starting a userscript with absoloute path which doesn't exist + Scenario: Starting a userscript with absolute path which doesn't exist When I run :spawn -u /this_does_not_exist Then the error "Userscript '/this_does_not_exist' not found" should be shown diff --git a/tests/end2end/features/tabs.feature b/tests/end2end/features/tabs.feature index 75051ad44..b17cbe1e6 100644 --- a/tests/end2end/features/tabs.feature +++ b/tests/end2end/features/tabs.feature @@ -6,6 +6,7 @@ Feature: Tab management Background: Given I clean up open tabs And I set tabs.tabs_are_windows to false + And I clear the log # :tab-close @@ -1415,6 +1416,21 @@ Feature: Tab management - history: - url: http://localhost:*/data/hello.txt + Scenario: Closing tab with tabs_are_windows + When I set tabs.tabs_are_windows to true + And I set tabs.last_close to ignore + And I open data/numbers/1.txt + And I open data/numbers/2.txt in a new tab + And I run :tab-close + And I wait for "removed: tabbed-browser" in the log + Then the session should look like: + windows: + - tabs: + - active: true + history: + - url: about:blank + - url: http://localhost:*/data/numbers/1.txt + # :tab-pin Scenario: :tab-pin command diff --git a/tests/end2end/features/test_misc_bdd.py b/tests/end2end/features/test_misc_bdd.py index 8dcf7edd1..7aeae2739 100644 --- a/tests/end2end/features/test_misc_bdd.py +++ b/tests/end2end/features/test_misc_bdd.py @@ -17,8 +17,6 @@ # You should have received a copy of the GNU General Public License # along with qutebrowser. If not, see <http://www.gnu.org/licenses/>. -import json - import pytest_bdd as bdd bdd.scenarios('misc.feature') @@ -28,10 +26,3 @@ def pdf_exists(quteproc, tmpdir, filename): path = tmpdir / filename data = path.read_binary() assert data.startswith(b'%PDF') - - -@bdd.when(bdd.parsers.parse('I set up "{lists}" as block lists')) -def set_up_blocking(quteproc, lists, server): - url = 'http://localhost:{}/data/adblock/'.format(server.port) - urls = [url + item.strip() for item in lists.split(',')] - quteproc.set_setting('content.host_blocking.lists', json.dumps(urls)) diff --git a/tests/end2end/fixtures/test_quteprocess.py b/tests/end2end/fixtures/test_quteprocess.py index 30c85d5ba..5693f11e9 100644 --- a/tests/end2end/fixtures/test_quteprocess.py +++ b/tests/end2end/fixtures/test_quteprocess.py @@ -351,7 +351,7 @@ def test_set(quteproc, value): @pytest.mark.parametrize('message, ignored', [ - # Unparseable + # Unparsable ('Hello World', False), # Without process/thread ID ('[0606/135039:ERROR:cert_verify_proc_nss.cc(925)] CERT_PKIXVerifyCert ' diff --git a/tests/end2end/test_adblock_e2e.py b/tests/end2end/test_adblock_e2e.py new file mode 100644 index 000000000..f8ecc596f --- /dev/null +++ b/tests/end2end/test_adblock_e2e.py @@ -0,0 +1,61 @@ +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2020 Florian Bruhin (The Compiler) <mail@qutebrowser.org> +# +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>. + +"""End to end tests for adblocking.""" + +import pytest + +try: + import adblock +except ImportError: + adblock = None + +needs_adblock_lib = pytest.mark.skipif( + adblock is None, reason="Needs 'adblock' library") + + +@pytest.mark.parametrize('method', [ + 'auto', + 'hosts', + pytest.param('adblock', marks=needs_adblock_lib), + pytest.param('both', marks=needs_adblock_lib), +]) +def test_adblock(method, quteproc, server): + for kind in ['hosts', 'adblock']: + quteproc.set_setting( + f'content.blocking.{kind}.lists', + f"['http://localhost:{server.port}/data/blocking/qutebrowser-{kind}']" + ) + + quteproc.set_setting('content.blocking.method', method) + quteproc.send_cmd(':adblock-update') + + quteproc.wait_for(message="hostblock: Read 1 hosts from 1 sources.") + if adblock is not None: + quteproc.wait_for( + message="braveadblock: Filters successfully read from 1 sources.") + + quteproc.open_path('data/blocking/external_logo.html') + + if method in ['hosts', 'both'] or (method == 'auto' and adblock is None): + message = "Request to qutebrowser.org blocked by host blocker." + else: + message = ("Request to https://qutebrowser.org/icons/qutebrowser.svg blocked " + "by ad blocker.") + quteproc.wait_for(message=message) diff --git a/tests/helpers/utils.py b/tests/helpers/utils.py index 2c275bf15..5f7297719 100644 --- a/tests/helpers/utils.py +++ b/tests/helpers/utils.py @@ -233,9 +233,25 @@ def ignore_bs4_warning(): yield +def _decompress_gzip_datafile(filename): + path = os.path.join(abs_datapath(), filename) + yield from io.TextIOWrapper(gzip.open(path), encoding="utf-8") + + def blocked_hosts(): - path = os.path.join(abs_datapath(), 'blocked-hosts.gz') - yield from io.TextIOWrapper(gzip.open(path), encoding='utf-8') + return _decompress_gzip_datafile("blocked-hosts.gz") + + +def adblock_dataset_tsv(): + return _decompress_gzip_datafile("brave-adblock/ublock-matches.tsv.gz") + + +def easylist_txt(): + return _decompress_gzip_datafile("easylist.txt.gz") + + +def easyprivacy_txt(): + return _decompress_gzip_datafile("easyprivacy.txt.gz") def seccomp_args(qt_flag): diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index 8b4653b58..082cf714a 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -1329,6 +1329,30 @@ def test_undo_completion(tabbed_browser_stubs, info): }) +def undo_completion_retains_sort_order(tabbed_browser_stubs, info): + """Test :undo completion sort order with > 10 entries.""" + created_dt = datetime(2020, 1, 1) + created_str = "2020-01-02 00:00" + + tabbed_browser_stubs[0].undo_stack = [ + tabbedbrowser._UndoEntry( + url=QUrl(f'https://example.org/{idx}'), + history=None, index=None, pinned=None, + created_at=created_dt, + ) + for idx in range(1, 11) + ] + + model = miscmodels.undo(info=info) + model.set_pattern('') + + expected = [ + (str(idx), f'https://example.org/{idx}', created_str) + for idx in range(1, 11) + ] + _check_completions(model, {"Closed tabs": expected}) + + @hypothesis.given(text=hypothesis.strategies.text()) def test_listcategory_hypothesis(text): """Make sure we can't produce invalid patterns.""" diff --git a/tests/unit/components/test_adblock.py b/tests/unit/components/test_adblock.py deleted file mode 100644 index 6ee236765..000000000 --- a/tests/unit/components/test_adblock.py +++ /dev/null @@ -1,474 +0,0 @@ -# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: -#!/usr/bin/env python3 - -# Copyright 2015 Corentin Julé <corentinjule@gmail.com> -# -# This file is part of qutebrowser. -# -# qutebrowser is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# qutebrowser is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>. - -import os -import os.path -import zipfile -import logging - -import pytest - -from PyQt5.QtCore import QUrl - -from qutebrowser.components import adblock -from qutebrowser.utils import urlmatch -from helpers import utils - - -pytestmark = pytest.mark.usefixtures('qapp') - -# TODO See ../utils/test_standarddirutils for OSError and caplog assertion - -WHITELISTED_HOSTS = ('qutebrowser.org', 'mediumhost.io', 'http://*.edu') - -BLOCKLIST_HOSTS = ('localhost', - 'mediumhost.io', - 'malware.badhost.org', - '4-verybadhost.com', - 'ads.worsthostever.net') - -CLEAN_HOSTS = ('goodhost.gov', 'verygoodhost.com') - -URLS_TO_CHECK = ('http://localhost', - 'http://mediumhost.io', - 'ftp://malware.badhost.org', - 'http://4-verybadhost.com', - 'http://ads.worsthostever.net', - 'http://goodhost.gov', - 'ftp://verygoodhost.com', - 'http://qutebrowser.org', - 'http://veryverygoodhost.edu') - - -@pytest.fixture -def host_blocker_factory(config_tmpdir, data_tmpdir, download_stub, - config_stub): - def factory(): - return adblock.HostBlocker(config_dir=config_tmpdir, - data_dir=data_tmpdir) - return factory - - -def create_zipfile(directory, files, zipname='test'): - """Return a path to a newly created zip file. - - Args: - directory: path object where to create the zip file. - files: list of filenames (relative to directory) to each file to add. - zipname: name to give to the zip file. - """ - zipfile_path = directory / zipname + '.zip' - with zipfile.ZipFile(str(zipfile_path), 'w') as new_zipfile: - for file_path in files: - new_zipfile.write(str(directory / file_path), - arcname=os.path.basename(str(file_path))) - # Removes path from file name - return str(zipname + '.zip') - - -def create_blocklist(directory, blocked_hosts=BLOCKLIST_HOSTS, - name='hosts', line_format='one_per_line'): - """Return a path to a blocklist file. - - Args: - directory: path object where to create the blocklist file - blocked_hosts: an iterable of string hosts to add to the blocklist - name: name to give to the blocklist file - line_format: 'etc_hosts' --> /etc/hosts format - 'one_per_line' --> one host per line format - 'not_correct' --> Not a correct hosts file format. - """ - blocklist_file = directory / name - with blocklist_file.open('w', encoding='UTF-8') as blocklist: - # ensure comments are ignored when processing blocklist - blocklist.write('# Blocked Hosts List #\n\n') - if line_format == 'etc_hosts': # /etc/hosts like format - for host in blocked_hosts: - blocklist.write('127.0.0.1 ' + host + '\n') - elif line_format == 'one_per_line': - for host in blocked_hosts: - blocklist.write(host + '\n') - elif line_format == 'not_correct': - for host in blocked_hosts: - blocklist.write(host + ' This is not a correct hosts file\n') - else: - raise ValueError('Incorrect line_format argument') - return name - - -def assert_urls(host_blocker, blocked=BLOCKLIST_HOSTS, - whitelisted=WHITELISTED_HOSTS, urls_to_check=URLS_TO_CHECK): - """Test if Urls to check are blocked or not by HostBlocker. - - Ensure URLs in 'blocked' and not in 'whitelisted' are blocked. - All other URLs must not be blocked. - - localhost is an example of a special case that shouldn't be blocked. - """ - whitelisted = list(whitelisted) + ['localhost'] - for str_url in urls_to_check: - url = QUrl(str_url) - host = url.host() - if host in blocked and host not in whitelisted: - assert host_blocker._is_blocked(url) - else: - assert not host_blocker._is_blocked(url) - - -def blocklist_to_url(filename): - """Get an example.com-URL with the given filename as path.""" - assert not os.path.isabs(filename), filename - url = QUrl('http://example.com/') - url.setPath('/' + filename) - assert url.isValid(), url.errorString() - return url - - -def generic_blocklists(directory): - """Return a generic list of files to be used in hosts-block-lists option. - - This list contains : - - a remote zip file with 1 hosts file and 2 useless files - - a remote zip file with only useless files - (Should raise a FileNotFoundError) - - a remote zip file with only one valid hosts file - - a local text file with valid hosts - - a remote text file without valid hosts format. - """ - # remote zip file with 1 hosts file and 2 useless files - file1 = create_blocklist(directory, blocked_hosts=CLEAN_HOSTS, - name='README', line_format='not_correct') - file2 = create_blocklist(directory, blocked_hosts=BLOCKLIST_HOSTS[:3], - name='hosts', line_format='etc_hosts') - file3 = create_blocklist(directory, blocked_hosts=CLEAN_HOSTS, - name='false_positive', line_format='one_per_line') - files_to_zip = [file1, file2, file3] - blocklist1 = blocklist_to_url( - create_zipfile(directory, files_to_zip, 'block1')) - - # remote zip file without file named hosts - # (Should raise a FileNotFoundError) - file1 = create_blocklist(directory, blocked_hosts=CLEAN_HOSTS, - name='md5sum', line_format='etc_hosts') - file2 = create_blocklist(directory, blocked_hosts=CLEAN_HOSTS, - name='README', line_format='not_correct') - file3 = create_blocklist(directory, blocked_hosts=CLEAN_HOSTS, - name='false_positive', line_format='one_per_line') - files_to_zip = [file1, file2, file3] - blocklist2 = blocklist_to_url( - create_zipfile(directory, files_to_zip, 'block2')) - - # remote zip file with only one valid hosts file inside - file1 = create_blocklist(directory, blocked_hosts=[BLOCKLIST_HOSTS[3]], - name='malwarelist', line_format='etc_hosts') - blocklist3 = blocklist_to_url(create_zipfile(directory, [file1], 'block3')) - - # local text file with valid hosts - blocklist4 = QUrl.fromLocalFile(str(directory / create_blocklist( - directory, blocked_hosts=[BLOCKLIST_HOSTS[4]], - name='mycustomblocklist', line_format='one_per_line'))) - assert blocklist4.isValid(), blocklist4.errorString() - - # remote text file without valid hosts format - blocklist5 = blocklist_to_url(create_blocklist( - directory, blocked_hosts=CLEAN_HOSTS, name='notcorrectlist', - line_format='not_correct')) - - return [blocklist1.toString(), blocklist2.toString(), - blocklist3.toString(), blocklist4.toString(), - blocklist5.toString()] - - -def test_disabled_blocking_update(config_stub, tmpdir, caplog, - host_blocker_factory): - """Ensure no URL is blocked when host blocking is disabled.""" - config_stub.val.content.host_blocking.lists = generic_blocklists(tmpdir) - config_stub.val.content.host_blocking.enabled = False - - host_blocker = host_blocker_factory() - host_blocker.adblock_update() - while host_blocker._in_progress: - current_download = host_blocker._in_progress[0] - with caplog.at_level(logging.ERROR): - current_download.successful = True - current_download.finished.emit() - host_blocker.read_hosts() - for str_url in URLS_TO_CHECK: - assert not host_blocker._is_blocked(QUrl(str_url)) - - -def test_disabled_blocking_per_url(config_stub, host_blocker_factory): - example_com = 'https://www.example.com/' - - config_stub.val.content.host_blocking.lists = [] - pattern = urlmatch.UrlPattern(example_com) - config_stub.set_obj('content.host_blocking.enabled', False, - pattern=pattern) - - url = QUrl('blocked.example.com') - - host_blocker = host_blocker_factory() - host_blocker._blocked_hosts.add(url.host()) - - assert host_blocker._is_blocked(url) - assert not host_blocker._is_blocked(url, first_party_url=QUrl(example_com)) - - -def test_no_blocklist_update(config_stub, download_stub, host_blocker_factory): - """Ensure no URL is blocked when no block list exists.""" - config_stub.val.content.host_blocking.lists = None - config_stub.val.content.host_blocking.enabled = True - - host_blocker = host_blocker_factory() - host_blocker.adblock_update() - host_blocker.read_hosts() - for dl in download_stub.downloads: - dl.successful = True - for str_url in URLS_TO_CHECK: - assert not host_blocker._is_blocked(QUrl(str_url)) - - -def test_successful_update(config_stub, tmpdir, caplog, host_blocker_factory): - """Ensure hosts from host_blocking.lists are blocked after an update.""" - config_stub.val.content.host_blocking.lists = generic_blocklists(tmpdir) - config_stub.val.content.host_blocking.enabled = True - config_stub.val.content.host_blocking.whitelist = None - - host_blocker = host_blocker_factory() - host_blocker.adblock_update() - # Simulate download is finished - while host_blocker._in_progress: - current_download = host_blocker._in_progress[0] - with caplog.at_level(logging.ERROR): - current_download.successful = True - current_download.finished.emit() - host_blocker.read_hosts() - assert_urls(host_blocker, whitelisted=[]) - - -def test_parsing_multiple_hosts_on_line(host_blocker_factory): - """Ensure multiple hosts on a line get parsed correctly.""" - host_blocker = host_blocker_factory() - bytes_host_line = ' '.join(BLOCKLIST_HOSTS).encode('utf-8') - parsed_hosts = host_blocker._read_hosts_line(bytes_host_line) - host_blocker._blocked_hosts |= parsed_hosts - assert_urls(host_blocker, whitelisted=[]) - - -@pytest.mark.parametrize('ip, host', [ - ('127.0.0.1', 'localhost'), - ('27.0.0.1', 'localhost.localdomain'), - ('27.0.0.1', 'local'), - ('55.255.255.255', 'broadcasthost'), - (':1', 'localhost'), - (':1', 'ip6-localhost'), - (':1', 'ip6-loopback'), - ('e80::1%lo0', 'localhost'), - ('f00::0', 'ip6-localnet'), - ('f00::0', 'ip6-mcastprefix'), - ('f02::1', 'ip6-allnodes'), - ('f02::2', 'ip6-allrouters'), - ('ff02::3', 'ip6-allhosts'), - ('.0.0.0', '0.0.0.0'), - ('127.0.1.1', 'myhostname'), - ('127.0.0.53', 'myhostname'), -]) -def test_whitelisted_lines(host_blocker_factory, ip, host): - """Make sure we don't block hosts we don't want to.""" - host_blocker = host_blocker_factory() - line = ('{} {}'.format(ip, host)).encode('ascii') - parsed_hosts = host_blocker._read_hosts_line(line) - assert host not in parsed_hosts - - -def test_failed_dl_update(config_stub, tmpdir, caplog, host_blocker_factory): - """One blocklist fails to download. - - Ensure hosts from this list are not blocked. - """ - dl_fail_blocklist = blocklist_to_url(create_blocklist( - tmpdir, blocked_hosts=CLEAN_HOSTS, name='download_will_fail', - line_format='one_per_line')) - hosts_to_block = (generic_blocklists(tmpdir) + - [dl_fail_blocklist.toString()]) - config_stub.val.content.host_blocking.lists = hosts_to_block - config_stub.val.content.host_blocking.enabled = True - config_stub.val.content.host_blocking.whitelist = None - - host_blocker = host_blocker_factory() - host_blocker.adblock_update() - while host_blocker._in_progress: - current_download = host_blocker._in_progress[0] - # if current download is the file we want to fail, make it fail - if current_download.name == dl_fail_blocklist.path(): - current_download.successful = False - else: - current_download.successful = True - with caplog.at_level(logging.ERROR): - current_download.finished.emit() - host_blocker.read_hosts() - assert_urls(host_blocker, whitelisted=[]) - - -@pytest.mark.parametrize('location', ['content', 'comment']) -def test_invalid_utf8(config_stub, tmpdir, caplog, host_blocker_factory, - location): - """Make sure invalid UTF-8 is handled correctly. - - See https://github.com/qutebrowser/qutebrowser/issues/2301 - """ - blocklist = tmpdir / 'blocklist' - if location == 'comment': - blocklist.write_binary(b'# nbsp: \xa0\n') - else: - assert location == 'content' - blocklist.write_binary(b'https://www.example.org/\xa0') - for url in BLOCKLIST_HOSTS: - blocklist.write(url + '\n', mode='a') - - url = blocklist_to_url('blocklist') - config_stub.val.content.host_blocking.lists = [url.toString()] - config_stub.val.content.host_blocking.enabled = True - config_stub.val.content.host_blocking.whitelist = None - - host_blocker = host_blocker_factory() - host_blocker.adblock_update() - current_download = host_blocker._in_progress[0] - - if location == 'content': - with caplog.at_level(logging.ERROR): - current_download.successful = True - current_download.finished.emit() - expected = (r"Failed to decode: " - r"b'https://www.example.org/\xa0localhost") - assert caplog.messages[-2].startswith(expected) - else: - current_download.successful = True - current_download.finished.emit() - - host_blocker.read_hosts() - assert_urls(host_blocker, whitelisted=[]) - - -def test_invalid_utf8_compiled(config_stub, config_tmpdir, data_tmpdir, - monkeypatch, caplog, host_blocker_factory): - """Make sure invalid UTF-8 in the compiled file is handled.""" - config_stub.val.content.host_blocking.lists = [] - - # Make sure the HostBlocker doesn't delete blocked-hosts in __init__ - monkeypatch.setattr(adblock.HostBlocker, 'update_files', - lambda _self: None) - - (config_tmpdir / 'blocked-hosts').write_binary( - b'https://www.example.org/\xa0') - (data_tmpdir / 'blocked-hosts').ensure() - - host_blocker = host_blocker_factory() - with caplog.at_level(logging.ERROR): - host_blocker.read_hosts() - assert caplog.messages[-1] == "Failed to read host blocklist!" - - -def test_blocking_with_whitelist(config_stub, data_tmpdir, host_blocker_factory): - """Ensure hosts in content.host_blocking.whitelist are never blocked.""" - # Simulate adblock_update has already been run - # by creating a file named blocked-hosts, - # Exclude localhost from it as localhost is never blocked via list - filtered_blocked_hosts = BLOCKLIST_HOSTS[1:] - blocklist = create_blocklist(data_tmpdir, - blocked_hosts=filtered_blocked_hosts, - name='blocked-hosts', - line_format='one_per_line') - config_stub.val.content.host_blocking.lists = [blocklist] - config_stub.val.content.host_blocking.enabled = True - config_stub.val.content.host_blocking.whitelist = list(WHITELISTED_HOSTS) - - host_blocker = host_blocker_factory() - host_blocker.read_hosts() - assert_urls(host_blocker) - - -def test_config_change_initial(config_stub, tmpdir, host_blocker_factory): - """Test emptying host_blocking.lists with existing blocked_hosts. - - - A blocklist is present in host_blocking.lists and blocked_hosts is - populated - - User quits qutebrowser, empties host_blocking.lists from his config - - User restarts qutebrowser, does adblock-update - """ - create_blocklist(tmpdir, blocked_hosts=BLOCKLIST_HOSTS, - name='blocked-hosts', line_format='one_per_line') - config_stub.val.content.host_blocking.lists = None - config_stub.val.content.host_blocking.enabled = True - config_stub.val.content.host_blocking.whitelist = None - - host_blocker = host_blocker_factory() - host_blocker.read_hosts() - for str_url in URLS_TO_CHECK: - assert not host_blocker._is_blocked(QUrl(str_url)) - - -def test_config_change(config_stub, tmpdir, host_blocker_factory): - """Ensure blocked-hosts resets if host-block-list is changed to None.""" - filtered_blocked_hosts = BLOCKLIST_HOSTS[1:] # Exclude localhost - blocklist = blocklist_to_url(create_blocklist( - tmpdir, blocked_hosts=filtered_blocked_hosts, name='blocked-hosts', - line_format='one_per_line')) - config_stub.val.content.host_blocking.lists = [blocklist.toString()] - config_stub.val.content.host_blocking.enabled = True - config_stub.val.content.host_blocking.whitelist = None - - host_blocker = host_blocker_factory() - host_blocker.read_hosts() - config_stub.val.content.host_blocking.lists = None - host_blocker.read_hosts() - for str_url in URLS_TO_CHECK: - assert not host_blocker._is_blocked(QUrl(str_url)) - - -def test_add_directory(config_stub, tmpdir, host_blocker_factory): - """Ensure adblocker can import all files in a directory.""" - blocklist_hosts2 = [] - for i in BLOCKLIST_HOSTS[1:]: - blocklist_hosts2.append('1' + i) - - create_blocklist(tmpdir, blocked_hosts=BLOCKLIST_HOSTS, - name='blocked-hosts', line_format='one_per_line') - create_blocklist(tmpdir, blocked_hosts=blocklist_hosts2, - name='blocked-hosts2', line_format='one_per_line') - - config_stub.val.content.host_blocking.lists = [tmpdir.strpath] - config_stub.val.content.host_blocking.enabled = True - host_blocker = host_blocker_factory() - host_blocker.adblock_update() - assert len(host_blocker._blocked_hosts) == len(blocklist_hosts2) * 2 - - -def test_adblock_benchmark(data_tmpdir, benchmark, host_blocker_factory): - blocked_hosts = data_tmpdir / 'blocked-hosts' - blocked_hosts.write_text('\n'.join(utils.blocked_hosts()), - encoding='utf-8') - - url = QUrl('https://www.example.org/') - blocker = host_blocker_factory() - blocker.read_hosts() - assert blocker._blocked_hosts - - benchmark(lambda: blocker._is_blocked(url)) diff --git a/tests/unit/components/test_blockutils.py b/tests/unit/components/test_blockutils.py new file mode 100644 index 000000000..480a6f9eb --- /dev/null +++ b/tests/unit/components/test_blockutils.py @@ -0,0 +1,83 @@ +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: +#!/usr/bin/env python3 + +# Copyright 2020 Árni Dagur <arni@dagur.eu> +# +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>. + +import os +import io +from typing import IO + +from PyQt5.QtCore import QUrl + +import pytest + +from qutebrowser.components.utils import blockutils + + +@pytest.fixture +def pretend_blocklists(tmpdir): + """Put fake blocklists into a tempdir. + + Put fake blocklists blocklists into a temporary directory, then return + both a list containing `file://` urls, and the residing dir. + """ + data = [ + (["cdn.malwarecorp.is", "evil-industries.com"], "malicious-hosts.txt"), + (["news.moms-against-icecream.net"], "blocklist.list"), + ] + # Add a bunch of automatically generated blocklist as well + for n in range(8): + data.append(([f"example{n}.com", f"example{n+1}.net"], f"blocklist{n}")) + + bl_dst_dir = tmpdir / "blocklists" + bl_dst_dir.mkdir() + urls = [] + for blocklist_lines, filename in data: + bl_dst_path = bl_dst_dir / filename + with open(bl_dst_path, "w", encoding="utf-8") as f: + f.write("\n".join(blocklist_lines)) + assert os.path.isfile(bl_dst_path) + urls.append(QUrl.fromLocalFile(str(bl_dst_path)).toString()) + return urls, bl_dst_dir + + +def test_blocklist_dl(qtbot, pretend_blocklists): + total_expected = 10 + num_single_dl_called = 0 + + def on_single_download(download: IO[bytes]) -> None: + nonlocal num_single_dl_called + num_single_dl_called += 1 + + num_lines = 0 + with io.TextIOWrapper(download, encoding="utf-8") as dl_io: + for line in dl_io: + assert line.split(".")[-1].strip() in ("com", "net", "is") + num_lines += 1 + assert num_lines >= 1 + + list_qurls = [QUrl(blocklist) for blocklist in pretend_blocklists[0]] + + dl = blockutils.BlocklistDownloads(list_qurls) + dl.single_download_finished.connect(on_single_download) + + with qtbot.waitSignal(dl.all_downloads_finished) as blocker: + dl.initiate() + assert blocker.args == [total_expected] + + assert num_single_dl_called == total_expected diff --git a/tests/unit/components/test_braveadblock.py b/tests/unit/components/test_braveadblock.py new file mode 100644 index 000000000..c653ce4e5 --- /dev/null +++ b/tests/unit/components/test_braveadblock.py @@ -0,0 +1,368 @@ +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2020 Florian Bruhin (The Compiler) <mail@qutebrowser.org> +# +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>. + +import pathlib +import logging +import csv +import os.path +from typing import Iterable, Tuple + +from PyQt5.QtCore import QUrl + +import pytest + +from qutebrowser.api.interceptor import ResourceType +from qutebrowser.components import braveadblock +from qutebrowser.components.utils import blockutils +from helpers import utils + +pytestmark = pytest.mark.usefixtures("qapp") + +OKAY_URLS = [ + ( + "https://qutebrowser.org/icons/qutebrowser.svg", + "https://qutebrowser.org", + ResourceType.image, + ), + ( + "https://qutebrowser.org/doc/img/main.png", + "https://qutebrowser.org", + ResourceType.image, + ), + ( + "https://qutebrowser.org/media/font.css", + "https://qutebrowser.org", + ResourceType.stylesheet, + ), + ( + "https://www.ruv.is/sites/default/files/styles/2000x1125/public/fr_20180719_091367_1.jpg?itok=0zTNSKKS×tamp=1561275315", + "https://www.ruv.is/frett/2020/04/23/today-is-the-first-day-of-summer", + ResourceType.image, + ), + ("https://easylist.to/easylist/easylist.txt", None, ResourceType.main_frame), + ("https://easylist.to/easylist/easyprivacy.txt", None, ResourceType.main_frame), +] + +NOT_OKAY_URLS = [ + ( + "https://pagead2.googlesyndication.com/pcs/activeview?xai=AKAOjsvBN5MuZsVQyE7HD18bD-JjK589TD3zkugwCoLE2C5nP26WFNCQb8WwxzZTelPEHwwnhaOCsGxYc8WeFgYZLReqLYl8r9BtAQ6r83OHa04&sig=Cg0ArKJSzKMgXuVbXAD1EAE&adk=1473563476&tt=-1&bs=1431%2C473&mtos=120250,120250,120250,120250,120250&tos=120250,0,0,0,0&p=60,352,150,1080&mcvt=120250&rs=0&ht=0&tfs=5491&tls=125682&mc=1<e=0&bas=0&bac=0&if=1&met=ie&avms=nio&exg=1&md=2&btr=0&lm=2&rst=1587887205533&dlt=226&rpt=1849&isd=0&msd=0&ext&xdi=0&ps=1431%2C7860&ss=1440%2C810&pt=-1&bin=4&deb=1-0-0-1192-5-1191-1191-0-0-0&tvt=125678&is=728%2C90&iframe_loc=https%3A%2F%2Ftpc.googlesyndication.com%2Fsafeframe%2F1-0-37%2Fhtml%2Fcontainer.html&r=u&id=osdtos&vs=4&uc=1192&upc=1&tgt=DIV&cl=1&cec=1&wf=0&cac=1&cd=0x0&itpl=19&v=20200422", + "https://google.com", + ResourceType.image, + ), + ( + "https://e.deployads.com/e/myanimelist.net", + "https://myanimelist.net", + ResourceType.xhr, + ), + ( + "https://c.amazon-adsystem.com/aax2/apstag.js", + "https://www.reddit.com", + ResourceType.script, + ), + ( + "https://c.aaxads.com/aax.js?pub=AAX763KC6&hst=www.reddit.com&ver=1.2", + "https://www.reddit.com", + ResourceType.script, + ), + ( + "https://pixel.mathtag.com/sync/img/?mt_exid=10009&mt_exuid=&mm_bnc&mm_bct&UUID=c7b65ea6-76cc-4700-b0c7-6dbcd10820ed", + "https://damndelicious.net/2019/04/03/easy-slow-cooker-chili/", + ResourceType.image, + ), +] + + +def run_function_on_dataset(given_function): + """Run the given function on a bunch of urls. + + In the data folder, we have a file called `adblock_dataset.tsv`, which + contains tuples of (url, source_url, type) in each line. We give these + to values to the given function, row by row. + """ + dataset = utils.adblock_dataset_tsv() + reader = csv.DictReader(dataset, delimiter="\t") + for row in reader: + url = QUrl(row["url"]) + source_url = QUrl(row["source_url"]) + resource_type = ResourceType[row["type"]] + given_function(url, source_url, resource_type) + + +def assert_none_blocked(ad_blocker): + assert_urls(ad_blocker, NOT_OKAY_URLS + OKAY_URLS, False) + + def assert_not_blocked(url, source_url, resource_type): + nonlocal ad_blocker + assert not ad_blocker._is_blocked(url, source_url, resource_type) + + run_function_on_dataset(assert_not_blocked) + + +@pytest.fixture +def blocklist_invalid_utf8(tmpdir): + dest_path = tmpdir / "invalid_utf8.txt" + dest_path.write_binary(b"invalidutf8\xa0") + return QUrl.fromLocalFile(str(dest_path)).toString() + + +@pytest.fixture +def easylist_easyprivacy_both(tmpdir): + """Put easyprivacy and easylist blocklists into a tempdir. + + Copy the easyprivacy and easylist blocklists into a temporary directory, + then return both a list containing `file://` urls, and the residing dir. + """ + bl_dst_dir = tmpdir / "blocklists" + bl_dst_dir.mkdir() + urls = [] + for blocklist, filename in [ + (utils.easylist_txt(), "easylist.txt"), + (utils.easyprivacy_txt(), "easyprivacy.txt"), + ]: + bl_dst_path = bl_dst_dir / filename + with open(bl_dst_path, "w", encoding="utf-8") as f: + f.write("\n".join(list(blocklist))) + assert os.path.isfile(bl_dst_path) + urls.append(QUrl.fromLocalFile(str(bl_dst_path)).toString()) + return urls, bl_dst_dir + + +@pytest.fixture +def empty_dir(tmpdir): + empty_dir_path = tmpdir / "empty_dir" + empty_dir_path.mkdir() + return empty_dir_path + + +@pytest.fixture +def easylist_easyprivacy(easylist_easyprivacy_both): + """The first return value of `easylist_easyprivacy_both`.""" + return easylist_easyprivacy_both[0] + + +@pytest.fixture +def ad_blocker(config_stub, data_tmpdir): + pytest.importorskip("adblock") + return braveadblock.BraveAdBlocker(data_dir=pathlib.Path(str(data_tmpdir))) + + +def assert_only_one_success_message(messages): + expected_msg = "braveadblock: Filters successfully read" + assert len([m for m in messages if m.startswith(expected_msg)]) == 1 + + +def assert_urls( + ad_blocker: braveadblock.BraveAdBlocker, + urls: Iterable[Tuple[str, str, ResourceType]], + should_be_blocked: bool, +) -> None: + for (str_url, source_str_url, request_type) in urls: + url = QUrl(str_url) + source_url = QUrl(source_str_url) + is_blocked = ad_blocker._is_blocked(url, source_url, request_type) + assert is_blocked == should_be_blocked + + +@pytest.mark.parametrize( + "blocking_enabled, method, should_be_blocked", + [ + (True, "auto", True), + (True, "adblock", True), + (True, "both", True), + (True, "hosts", False), + (False, "auto", False), + (False, "adblock", False), + (False, "both", False), + (False, "hosts", False), + ], +) +def test_blocking_enabled( + config_stub, + easylist_easyprivacy, + caplog, + ad_blocker, + blocking_enabled, + method, + should_be_blocked, +): + """Tests that the ads are blocked when the adblocker is enabled, and vice versa.""" + config_stub.val.content.blocking.adblock.lists = easylist_easyprivacy + config_stub.val.content.blocking.enabled = blocking_enabled + config_stub.val.content.blocking.method = method + # Simulate the method-changed hook being run, since it doesn't execute + # with pytest. + ad_blocker.enabled = braveadblock._should_be_used() + + downloads = ad_blocker.adblock_update() + while downloads._in_progress: + current_download = downloads._in_progress[0] + with caplog.at_level(logging.ERROR): + current_download.successful = True + current_download.finished.emit() + assert_urls(ad_blocker, NOT_OKAY_URLS, should_be_blocked) + assert_urls(ad_blocker, OKAY_URLS, False) + + +def test_adblock_cache(config_stub, easylist_easyprivacy, caplog, ad_blocker): + config_stub.val.content.blocking.adblock.lists = easylist_easyprivacy + config_stub.val.content.blocking.enabled = True + + for i in range(3): + print("At cache test iteration {}".format(i)) + # Trying to read the cache before calling the update command should return + # a log message. + with caplog.at_level(logging.INFO): + ad_blocker.read_cache() + caplog.messages[-1].startswith( + "Run :brave-adblock-update to get adblock lists." + ) + + if i == 0: + # We haven't initialized the ad blocker yet, so we shouldn't be blocking + # anything. + assert_none_blocked(ad_blocker) + + # Now we initialize the adblocker. + downloads = ad_blocker.adblock_update() + while downloads._in_progress: + current_download = downloads._in_progress[0] + with caplog.at_level(logging.ERROR): + current_download.successful = True + current_download.finished.emit() + + # After initializing the the adblocker, we should start seeing ads + # blocked. + assert_urls(ad_blocker, NOT_OKAY_URLS, True) + assert_urls(ad_blocker, OKAY_URLS, False) + + # After reading the cache, we should still be seeing ads blocked. + ad_blocker.read_cache() + assert_urls(ad_blocker, NOT_OKAY_URLS, True) + assert_urls(ad_blocker, OKAY_URLS, False) + + # Now we remove the cache file and try all over again... + ad_blocker._cache_path.unlink() + + +def test_invalid_utf8(ad_blocker, config_stub, blocklist_invalid_utf8, caplog): + """Test that the adblocker handles invalid utf-8 correctly.""" + config_stub.val.content.blocking.adblock.lists = [blocklist_invalid_utf8] + config_stub.val.content.blocking.enabled = True + + with caplog.at_level(logging.INFO): + ad_blocker.adblock_update() + expected = "braveadblock: Block list is not valid utf-8" + assert caplog.messages[-2].startswith(expected) + + +def test_config_changed(ad_blocker, config_stub, easylist_easyprivacy, caplog): + """Ensure blocked-hosts resets if host-block-list is changed to None.""" + config_stub.val.content.blocking.enabled = True + config_stub.val.content.blocking.whitelist = None + + for _ in range(2): + # We should be blocking like normal, since the block lists are set to + # easylist and easyprivacy. + config_stub.val.content.blocking.adblock.lists = easylist_easyprivacy + downloads = ad_blocker.adblock_update() + while downloads._in_progress: + current_download = downloads._in_progress[0] + with caplog.at_level(logging.ERROR): + current_download.successful = True + current_download.finished.emit() + assert_urls(ad_blocker, NOT_OKAY_URLS, True) + assert_urls(ad_blocker, OKAY_URLS, False) + + # After setting the ad blocking lists to None, the ads should still be + # blocked, since we haven't run `:brave-adblock-update`. + config_stub.val.content.blocking.adblock.lists = None + assert_urls(ad_blocker, NOT_OKAY_URLS, True) + assert_urls(ad_blocker, OKAY_URLS, False) + + # After updating the adblocker, nothing should be blocked, since we set + # the blocklist to None. + downloads = ad_blocker.adblock_update() + while downloads._in_progress: + current_download = downloads._in_progress[0] + with caplog.at_level(logging.ERROR): + current_download.successful = True + current_download.finished.emit() + assert_none_blocked(ad_blocker) + + +def test_whitelist_on_dataset(config_stub, easylist_easyprivacy): + config_stub.val.content.blocking.adblock.lists = easylist_easyprivacy + config_stub.val.content.blocking.enabled = True + config_stub.val.content.blocking.whitelist = None + + def assert_whitelisted(url, source_url, resource_type): + config_stub.val.content.blocking.whitelist = None + assert not blockutils.is_whitelisted_url(url) + config_stub.val.content.blocking.whitelist = [] + assert not blockutils.is_whitelisted_url(url) + whitelist_url = url.toString(QUrl.RemovePath) + "/*" + config_stub.val.content.blocking.whitelist = [whitelist_url] + assert blockutils.is_whitelisted_url(url) + + run_function_on_dataset(assert_whitelisted) + + +def test_update_easylist_easyprivacy_directory( + ad_blocker, config_stub, easylist_easyprivacy_both, caplog +): + # This directory should contain two text files, one for easylist, another + # for easyprivacy. + lists_directory = easylist_easyprivacy_both[1] + + config_stub.val.content.blocking.adblock.lists = [ + QUrl.fromLocalFile(str(lists_directory)).toString() + ] + config_stub.val.content.blocking.enabled = True + config_stub.val.content.blocking.whitelist = None + + with caplog.at_level(logging.INFO): + ad_blocker.adblock_update() + assert_only_one_success_message(caplog.messages) + assert ( + caplog.messages[-1] + == "braveadblock: Filters successfully read from 2 sources." + ) + assert_urls(ad_blocker, NOT_OKAY_URLS, True) + assert_urls(ad_blocker, OKAY_URLS, False) + + +def test_update_empty_directory_blocklist(ad_blocker, config_stub, empty_dir, caplog): + tmpdir_url = QUrl.fromLocalFile(str(empty_dir)).toString() + config_stub.val.content.blocking.adblock.lists = [tmpdir_url] + config_stub.val.content.blocking.enabled = True + config_stub.val.content.blocking.whitelist = None + + # The temporary directory we created should be empty + assert len(empty_dir.listdir()) == 0 + + with caplog.at_level(logging.INFO): + ad_blocker.adblock_update() + assert_only_one_success_message(caplog.messages) + assert ( + caplog.messages[-1] + == "braveadblock: Filters successfully read from 0 sources." + ) + + # There are no filters, so no ads should be blocked. + assert_none_blocked(ad_blocker) diff --git a/tests/unit/components/test_hostblock.py b/tests/unit/components/test_hostblock.py new file mode 100644 index 000000000..d1a65ade5 --- /dev/null +++ b/tests/unit/components/test_hostblock.py @@ -0,0 +1,567 @@ +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: +#!/usr/bin/env python3 + +# Copyright 2015 Corentin Julé <corentinjule@gmail.com> +# +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>. + +import os +import os.path +import zipfile +import logging + +import pytest + +from PyQt5.QtCore import QUrl + +from qutebrowser.components import hostblock +from qutebrowser.utils import urlmatch +from helpers import utils + + +pytestmark = pytest.mark.usefixtures("qapp") + +# TODO See ../utils/test_standarddirutils for OSError and caplog assertion + +WHITELISTED_HOSTS = ("qutebrowser.org", "mediumhost.io", "http://*.edu") + +BLOCKLIST_HOSTS = ( + "localhost", + "mediumhost.io", + "malware.badhost.org", + "4-verybadhost.com", + "ads.worsthostever.net", +) + +CLEAN_HOSTS = ("goodhost.gov", "verygoodhost.com") + +URLS_TO_CHECK = ( + "http://localhost", + "http://mediumhost.io", + "ftp://malware.badhost.org", + "http://4-verybadhost.com", + "http://ads.worsthostever.net", + "http://goodhost.gov", + "ftp://verygoodhost.com", + "http://qutebrowser.org", + "http://veryverygoodhost.edu", +) + + +@pytest.fixture +def host_blocker_factory(config_tmpdir, data_tmpdir, download_stub, config_stub): + def factory(): + return hostblock.HostBlocker(config_dir=config_tmpdir, data_dir=data_tmpdir) + + return factory + + +def create_zipfile(directory, files, zipname="test"): + """Return a path to a newly created zip file. + + Args: + directory: path object where to create the zip file. + files: list of filenames (relative to directory) to each file to add. + zipname: name to give to the zip file. + """ + zipfile_path = directory / zipname + ".zip" + with zipfile.ZipFile(str(zipfile_path), "w") as new_zipfile: + for file_path in files: + new_zipfile.write( + str(directory / file_path), arcname=os.path.basename(str(file_path)) + ) + # Removes path from file name + return str(zipname + ".zip") + + +def create_blocklist( + directory, blocked_hosts=BLOCKLIST_HOSTS, name="hosts", line_format="one_per_line" +): + """Return a path to a blocklist file. + + Args: + directory: path object where to create the blocklist file + blocked_hosts: an iterable of string hosts to add to the blocklist + name: name to give to the blocklist file + line_format: 'etc_hosts' --> /etc/hosts format + 'one_per_line' --> one host per line format + 'not_correct' --> Not a correct hosts file format. + """ + blocklist_file = directory / name + with blocklist_file.open("w", encoding="UTF-8") as blocklist: + # ensure comments are ignored when processing blocklist + blocklist.write("# Blocked Hosts List #\n\n") + if line_format == "etc_hosts": # /etc/hosts like format + for host in blocked_hosts: + blocklist.write("127.0.0.1 " + host + "\n") + elif line_format == "one_per_line": + for host in blocked_hosts: + blocklist.write(host + "\n") + elif line_format == "not_correct": + for host in blocked_hosts: + blocklist.write(host + " This is not a correct hosts file\n") + else: + raise ValueError("Incorrect line_format argument") + return name + + +def assert_urls( + host_blocker, + blocked=BLOCKLIST_HOSTS, + whitelisted=WHITELISTED_HOSTS, + urls_to_check=URLS_TO_CHECK, +): + """Test if Urls to check are blocked or not by HostBlocker. + + Ensure URLs in 'blocked' and not in 'whitelisted' are blocked. + All other URLs must not be blocked. + + localhost is an example of a special case that shouldn't be blocked. + """ + whitelisted = list(whitelisted) + ["localhost"] + for str_url in urls_to_check: + url = QUrl(str_url) + host = url.host() + if host in blocked and host not in whitelisted: + assert host_blocker._is_blocked(url) + else: + assert not host_blocker._is_blocked(url) + + +def blocklist_to_url(filename): + """Get an example.com-URL with the given filename as path.""" + assert not os.path.isabs(filename), filename + url = QUrl("http://example.com/") + url.setPath("/" + filename) + assert url.isValid(), url.errorString() + return url + + +def generic_blocklists(directory): + """Return a generic list of files to be used in hosts-block-lists option. + + This list contains : + - a remote zip file with 1 hosts file and 2 useless files + - a remote zip file with only useless files + (Should raise a FileNotFoundError) + - a remote zip file with only one valid hosts file + - a local text file with valid hosts + - a remote text file without valid hosts format. + """ + # remote zip file with 1 hosts file and 2 useless files + file1 = create_blocklist( + directory, blocked_hosts=CLEAN_HOSTS, name="README", line_format="not_correct" + ) + file2 = create_blocklist( + directory, + blocked_hosts=BLOCKLIST_HOSTS[:3], + name="hosts", + line_format="etc_hosts", + ) + file3 = create_blocklist( + directory, + blocked_hosts=CLEAN_HOSTS, + name="false_positive", + line_format="one_per_line", + ) + files_to_zip = [file1, file2, file3] + blocklist1 = blocklist_to_url(create_zipfile(directory, files_to_zip, "block1")) + + # remote zip file without file named hosts + # (Should raise a FileNotFoundError) + file1 = create_blocklist( + directory, blocked_hosts=CLEAN_HOSTS, name="md5sum", line_format="etc_hosts" + ) + file2 = create_blocklist( + directory, blocked_hosts=CLEAN_HOSTS, name="README", line_format="not_correct" + ) + file3 = create_blocklist( + directory, + blocked_hosts=CLEAN_HOSTS, + name="false_positive", + line_format="one_per_line", + ) + files_to_zip = [file1, file2, file3] + blocklist2 = blocklist_to_url(create_zipfile(directory, files_to_zip, "block2")) + + # remote zip file with only one valid hosts file inside + file1 = create_blocklist( + directory, + blocked_hosts=[BLOCKLIST_HOSTS[3]], + name="malwarelist", + line_format="etc_hosts", + ) + blocklist3 = blocklist_to_url(create_zipfile(directory, [file1], "block3")) + + # local text file with valid hosts + blocklist4 = QUrl.fromLocalFile( + str( + directory + / create_blocklist( + directory, + blocked_hosts=[BLOCKLIST_HOSTS[4]], + name="mycustomblocklist", + line_format="one_per_line", + ) + ) + ) + assert blocklist4.isValid(), blocklist4.errorString() + + # remote text file without valid hosts format + blocklist5 = blocklist_to_url( + create_blocklist( + directory, + blocked_hosts=CLEAN_HOSTS, + name="notcorrectlist", + line_format="not_correct", + ) + ) + + return [ + blocklist1.toString(), + blocklist2.toString(), + blocklist3.toString(), + blocklist4.toString(), + blocklist5.toString(), + ] + + +@pytest.mark.parametrize( + "blocking_enabled, method", + [ + # Assuming the adblock dependency is installed + (True, "auto"), + (True, "adblock"), + (False, "auto"), + (False, "adblock"), + (False, "both"), + (False, "hosts"), + ], +) +def test_disabled_blocking_update( + config_stub, tmpdir, caplog, host_blocker_factory, blocking_enabled, method +): + """Ensure no URL is blocked when host blocking should be disabled.""" + if blocking_enabled and method == 'auto': + pytest.importorskip('adblock') + + config_stub.val.content.blocking.hosts.lists = generic_blocklists(tmpdir) + config_stub.val.content.blocking.enabled = blocking_enabled + config_stub.val.content.blocking.method = method + + host_blocker = host_blocker_factory() + downloads = host_blocker.adblock_update() + while downloads._in_progress: + current_download = downloads._in_progress[0] + with caplog.at_level(logging.ERROR): + current_download.successful = True + current_download.finished.emit() + host_blocker.read_hosts() + for str_url in URLS_TO_CHECK: + assert not host_blocker._is_blocked(QUrl(str_url)) + + +def test_disabled_blocking_per_url(config_stub, host_blocker_factory): + example_com = "https://www.example.com/" + + config_stub.val.content.blocking.method = "hosts" + config_stub.val.content.blocking.hosts.lists = [] + pattern = urlmatch.UrlPattern(example_com) + config_stub.set_obj("content.blocking.enabled", False, pattern=pattern) + + url = QUrl("blocked.example.com") + + host_blocker = host_blocker_factory() + host_blocker._blocked_hosts.add(url.host()) + + assert host_blocker._is_blocked(url) + assert not host_blocker._is_blocked(url, first_party_url=QUrl(example_com)) + + +def test_no_blocklist_update(config_stub, download_stub, host_blocker_factory): + """Ensure no URL is blocked when no block list exists.""" + config_stub.val.content.blocking.hosts.lists = None + config_stub.val.content.blocking.method = "hosts" + config_stub.val.content.blocking.enabled = True + + host_blocker = host_blocker_factory() + host_blocker.adblock_update() + host_blocker.read_hosts() + for dl in download_stub.downloads: + dl.successful = True + for str_url in URLS_TO_CHECK: + assert not host_blocker._is_blocked(QUrl(str_url)) + + +def test_successful_update(config_stub, tmpdir, caplog, host_blocker_factory): + """Ensure hosts from host_blocking.lists are blocked after an update.""" + config_stub.val.content.blocking.hosts.lists = generic_blocklists(tmpdir) + config_stub.val.content.blocking.method = "hosts" + config_stub.val.content.blocking.enabled = True + config_stub.val.content.blocking.whitelist = None + + host_blocker = host_blocker_factory() + downloads = host_blocker.adblock_update() + # Simulate download is finished + while downloads._in_progress: + current_download = downloads._in_progress[0] + with caplog.at_level(logging.ERROR): + current_download.successful = True + current_download.finished.emit() + host_blocker.read_hosts() + assert_urls(host_blocker, whitelisted=[]) + + +def test_parsing_multiple_hosts_on_line(config_stub, host_blocker_factory): + """Ensure multiple hosts on a line get parsed correctly.""" + config_stub.val.content.blocking.method = "hosts" + config_stub.val.content.blocking.enabled = True + + host_blocker = host_blocker_factory() + bytes_host_line = " ".join(BLOCKLIST_HOSTS).encode("utf-8") + parsed_hosts = host_blocker._read_hosts_line(bytes_host_line) + host_blocker._blocked_hosts |= parsed_hosts + assert_urls(host_blocker, whitelisted=[]) + + +@pytest.mark.parametrize( + "ip, host", + [ + ("127.0.0.1", "localhost"), + ("27.0.0.1", "localhost.localdomain"), + ("27.0.0.1", "local"), + ("55.255.255.255", "broadcasthost"), + (":1", "localhost"), + (":1", "ip6-localhost"), + (":1", "ip6-loopback"), + ("e80::1%lo0", "localhost"), + ("f00::0", "ip6-localnet"), + ("f00::0", "ip6-mcastprefix"), + ("f02::1", "ip6-allnodes"), + ("f02::2", "ip6-allrouters"), + ("ff02::3", "ip6-allhosts"), + (".0.0.0", "0.0.0.0"), + ("127.0.1.1", "myhostname"), + ("127.0.0.53", "myhostname"), + ], +) +def test_whitelisted_lines(host_blocker_factory, ip, host): + """Make sure we don't block hosts we don't want to.""" + host_blocker = host_blocker_factory() + line = ("{} {}".format(ip, host)).encode("ascii") + parsed_hosts = host_blocker._read_hosts_line(line) + assert host not in parsed_hosts + + +def test_failed_dl_update(config_stub, tmpdir, caplog, host_blocker_factory): + """One blocklist fails to download. + + Ensure hosts from this list are not blocked. + """ + dl_fail_blocklist = blocklist_to_url( + create_blocklist( + tmpdir, + blocked_hosts=CLEAN_HOSTS, + name="download_will_fail", + line_format="one_per_line", + ) + ) + hosts_to_block = generic_blocklists(tmpdir) + [dl_fail_blocklist.toString()] + config_stub.val.content.blocking.hosts.lists = hosts_to_block + config_stub.val.content.blocking.enabled = True + config_stub.val.content.blocking.method = "hosts" + config_stub.val.content.blocking.whitelist = None + + host_blocker = host_blocker_factory() + downloads = host_blocker.adblock_update() + while downloads._in_progress: + current_download = downloads._in_progress[0] + # if current download is the file we want to fail, make it fail + if current_download.name == dl_fail_blocklist.path(): + current_download.successful = False + else: + current_download.successful = True + with caplog.at_level(logging.ERROR): + current_download.finished.emit() + host_blocker.read_hosts() + assert_urls(host_blocker, whitelisted=[]) + + +@pytest.mark.parametrize("location", ["content", "comment"]) +def test_invalid_utf8(config_stub, tmpdir, caplog, host_blocker_factory, location): + """Make sure invalid UTF-8 is handled correctly. + + See https://github.com/qutebrowser/qutebrowser/issues/2301 + """ + blocklist = tmpdir / "blocklist" + if location == "comment": + blocklist.write_binary(b"# nbsp: \xa0\n") + else: + assert location == "content" + blocklist.write_binary(b"https://www.example.org/\xa0") + for url in BLOCKLIST_HOSTS: + blocklist.write(url + "\n", mode="a") + + url = blocklist_to_url("blocklist") + config_stub.val.content.blocking.hosts.lists = [url.toString()] + config_stub.val.content.blocking.enabled = True + config_stub.val.content.blocking.method = "hosts" + config_stub.val.content.blocking.whitelist = None + + host_blocker = host_blocker_factory() + downloads = host_blocker.adblock_update() + current_download = downloads._in_progress[0] + + if location == "content": + with caplog.at_level(logging.ERROR): + current_download.successful = True + current_download.finished.emit() + expected = r"Failed to decode: " r"b'https://www.example.org/\xa0localhost" + assert caplog.messages[-2].startswith(expected) + else: + current_download.successful = True + current_download.finished.emit() + + host_blocker.read_hosts() + assert_urls(host_blocker, whitelisted=[]) + + +def test_invalid_utf8_compiled( + config_stub, config_tmpdir, data_tmpdir, monkeypatch, caplog, host_blocker_factory +): + """Make sure invalid UTF-8 in the compiled file is handled.""" + config_stub.val.content.blocking.hosts.lists = [] + + # Make sure the HostBlocker doesn't delete blocked-hosts in __init__ + monkeypatch.setattr(hostblock.HostBlocker, "update_files", lambda _self: None) + + (config_tmpdir / "blocked-hosts").write_binary(b"https://www.example.org/\xa0") + (data_tmpdir / "blocked-hosts").ensure() + + host_blocker = host_blocker_factory() + with caplog.at_level(logging.ERROR): + host_blocker.read_hosts() + assert caplog.messages[-1] == "Failed to read host blocklist!" + + +def test_blocking_with_whitelist(config_stub, data_tmpdir, host_blocker_factory): + """Ensure hosts in content.blocking.whitelist are never blocked.""" + # Simulate adblock_update has already been run + # by creating a file named blocked-hosts, + # Exclude localhost from it as localhost is never blocked via list + filtered_blocked_hosts = BLOCKLIST_HOSTS[1:] + blocklist = create_blocklist( + data_tmpdir, + blocked_hosts=filtered_blocked_hosts, + name="blocked-hosts", + line_format="one_per_line", + ) + config_stub.val.content.blocking.hosts.lists = [blocklist] + config_stub.val.content.blocking.enabled = True + config_stub.val.content.blocking.method = "hosts" + config_stub.val.content.blocking.whitelist = list(WHITELISTED_HOSTS) + + host_blocker = host_blocker_factory() + host_blocker.read_hosts() + assert_urls(host_blocker) + + +def test_config_change_initial(config_stub, tmpdir, host_blocker_factory): + """Test emptying host_blocking.lists with existing blocked_hosts. + + - A blocklist is present in host_blocking.lists and blocked_hosts is + populated + - User quits qutebrowser, empties host_blocking.lists from his config + - User restarts qutebrowser, does adblock-update + """ + create_blocklist( + tmpdir, + blocked_hosts=BLOCKLIST_HOSTS, + name="blocked-hosts", + line_format="one_per_line", + ) + config_stub.val.content.blocking.hosts.lists = None + config_stub.val.content.blocking.enabled = True + config_stub.val.content.blocking.method = "hosts" + config_stub.val.content.blocking.whitelist = None + + host_blocker = host_blocker_factory() + host_blocker.read_hosts() + for str_url in URLS_TO_CHECK: + assert not host_blocker._is_blocked(QUrl(str_url)) + + +def test_config_change(config_stub, tmpdir, host_blocker_factory): + """Ensure blocked-hosts resets if host-block-list is changed to None.""" + filtered_blocked_hosts = BLOCKLIST_HOSTS[1:] # Exclude localhost + blocklist = blocklist_to_url( + create_blocklist( + tmpdir, + blocked_hosts=filtered_blocked_hosts, + name="blocked-hosts", + line_format="one_per_line", + ) + ) + config_stub.val.content.blocking.hosts.lists = [blocklist.toString()] + config_stub.val.content.blocking.enabled = True + config_stub.val.content.blocking.method = "hosts" + config_stub.val.content.blocking.whitelist = None + + host_blocker = host_blocker_factory() + host_blocker.read_hosts() + config_stub.val.content.blocking.hosts.lists = None + host_blocker.read_hosts() + for str_url in URLS_TO_CHECK: + assert not host_blocker._is_blocked(QUrl(str_url)) + + +def test_add_directory(config_stub, tmpdir, host_blocker_factory): + """Ensure adblocker can import all files in a directory.""" + blocklist_hosts2 = [] + for i in BLOCKLIST_HOSTS[1:]: + blocklist_hosts2.append("1" + i) + + create_blocklist( + tmpdir, + blocked_hosts=BLOCKLIST_HOSTS, + name="blocked-hosts", + line_format="one_per_line", + ) + create_blocklist( + tmpdir, + blocked_hosts=blocklist_hosts2, + name="blocked-hosts2", + line_format="one_per_line", + ) + + config_stub.val.content.blocking.hosts.lists = [tmpdir.strpath] + config_stub.val.content.blocking.enabled = True + config_stub.val.content.blocking.method = "hosts" + host_blocker = host_blocker_factory() + host_blocker.adblock_update() + assert len(host_blocker._blocked_hosts) == len(blocklist_hosts2) * 2 + + +def test_adblock_benchmark(data_tmpdir, benchmark, host_blocker_factory): + blocked_hosts = data_tmpdir / "blocked-hosts" + blocked_hosts.write_text("\n".join(utils.blocked_hosts()), encoding="utf-8") + + url = QUrl("https://www.example.org/") + blocker = host_blocker_factory() + blocker.read_hosts() + assert blocker._blocked_hosts + + benchmark(lambda: blocker._is_blocked(url)) diff --git a/tests/unit/config/test_configcommands.py b/tests/unit/config/test_configcommands.py index 220aa40f7..dc62717e5 100644 --- a/tests/unit/config/test_configcommands.py +++ b/tests/unit/config/test_configcommands.py @@ -297,7 +297,7 @@ class TestAdd: @pytest.mark.parametrize('temp', [True, False]) @pytest.mark.parametrize('value', ['test1', 'test2']) def test_list_add(self, commands, config_stub, yaml_value, temp, value): - name = 'content.host_blocking.whitelist' + name = 'content.blocking.whitelist' commands.config_list_add(name, value, temp=temp) @@ -324,7 +324,7 @@ class TestAdd: with pytest.raises( cmdutils.CommandError, match="Invalid value '{}'".format(value)): - commands.config_list_add('content.host_blocking.whitelist', value) + commands.config_list_add('content.blocking.whitelist', value) @pytest.mark.parametrize('value', ['test1', 'test2']) @pytest.mark.parametrize('temp', [True, False]) diff --git a/tests/unit/misc/userscripts/test_qute_lastpass.py b/tests/unit/misc/userscripts/test_qute_lastpass.py index 20646edd0..229fcf09e 100644 --- a/tests/unit/misc/userscripts/test_qute_lastpass.py +++ b/tests/unit/misc/userscripts/test_qute_lastpass.py @@ -82,10 +82,11 @@ class TestQuteLastPassComponents: def test_fake_key_raw(self, qutecommand_mock): """Test if fake_key_raw properly escapes characters.""" - qute_lastpass.fake_key_raw('john.doe@example.com ') + qute_lastpass.fake_key_raw('john.<<doe>>@example.com ') qutecommand_mock.assert_called_once_with( - 'fake-key \\j\\o\\h\\n\\.\\d\\o\\e\\@\\e\\x\\a\\m\\p\\l\\e\\.\\c\\o\\m" "' + 'fake-key \\j\\o\\h\\n\\.<less><less>\\d\\o\\e<greater><greater>\\@' + '\\e\\x\\a\\m\\p\\l\\e\\.\\c\\o\\m" "' ) def test_dmenu(self, subprocess_mock): diff --git a/tests/unit/scripts/importer_sample/netscape/bookmarks b/tests/unit/scripts/importer_sample/html/bookmarks index 89d8ed8e9..89d8ed8e9 100644 --- a/tests/unit/scripts/importer_sample/netscape/bookmarks +++ b/tests/unit/scripts/importer_sample/html/bookmarks diff --git a/tests/unit/scripts/importer_sample/netscape/config_py b/tests/unit/scripts/importer_sample/html/config_py index 9232bc372..9232bc372 100644 --- a/tests/unit/scripts/importer_sample/netscape/config_py +++ b/tests/unit/scripts/importer_sample/html/config_py diff --git a/tests/unit/scripts/importer_sample/netscape/input b/tests/unit/scripts/importer_sample/html/input index 1e3cdec31..1e3cdec31 100644 --- a/tests/unit/scripts/importer_sample/netscape/input +++ b/tests/unit/scripts/importer_sample/html/input diff --git a/tests/unit/scripts/importer_sample/netscape/quickmarks b/tests/unit/scripts/importer_sample/html/quickmarks index a43bb338d..a43bb338d 100644 --- a/tests/unit/scripts/importer_sample/netscape/quickmarks +++ b/tests/unit/scripts/importer_sample/html/quickmarks diff --git a/tests/unit/scripts/test_importer.py b/tests/unit/scripts/test_importer.py index 4a70ae63e..fbf27a074 100644 --- a/tests/unit/scripts/test_importer.py +++ b/tests/unit/scripts/test_importer.py @@ -94,25 +94,25 @@ def test_chrome_searches(capsys): assert imported == search_expected('chrome') -def test_netscape_bookmarks(capsys): - importer.import_netscape_bookmarks( - sample_input('netscape'), ['bookmark', 'keyword'], 'bookmark') +def test_html_bookmarks(capsys): + importer.import_html_bookmarks( + sample_input('html'), ['bookmark', 'keyword'], 'bookmark') imported = capsys.readouterr()[0] - assert imported == bm_expected('netscape') + assert imported == bm_expected('html') -def test_netscape_quickmarks(capsys): - importer.import_netscape_bookmarks( - sample_input('netscape'), ['bookmark', 'keyword'], 'quickmark') +def test_html_quickmarks(capsys): + importer.import_html_bookmarks( + sample_input('html'), ['bookmark', 'keyword'], 'quickmark') imported = capsys.readouterr()[0] - assert imported == qm_expected('netscape') + assert imported == qm_expected('html') -def test_netscape_searches(capsys): - importer.import_netscape_bookmarks( - sample_input('netscape'), ['search'], 'search') +def test_html_searches(capsys): + importer.import_html_bookmarks( + sample_input('html'), ['search'], 'search') imported = capsys.readouterr()[0] - assert imported == search_expected('netscape') + assert imported == search_expected('html') def test_mozilla_bookmarks(capsys): diff --git a/tests/unit/utils/test_version.py b/tests/unit/utils/test_version.py index 922692fdd..e16bd2318 100644 --- a/tests/unit/utils/test_version.py +++ b/tests/unit/utils/test_version.py @@ -544,7 +544,7 @@ class ImportFake: Attributes: modules: A dict mapping module names to bools. If True, the import will - success. Otherwise, it'll fail with ImportError. + succeed. Otherwise, it'll fail with ImportError. version_attribute: The name to use in the fake modules for the version attribute. version: The version to use for the modules. @@ -553,24 +553,8 @@ class ImportFake: """ def __init__(self): - self.modules = collections.OrderedDict([ - ('sip', True), - ('colorama', True), - ('pypeg2', True), - ('jinja2', True), - ('pygments', True), - ('yaml', True), - ('attr', True), - ('importlib_resources', True), - ('PyQt5.QtWebEngineWidgets', True), - ('PyQt5.QtWebEngine', True), - ('PyQt5.QtWebKitWidgets', True), - ]) - self.no_version_attribute = ['sip', - 'importlib_resources', - 'PyQt5.QtWebEngineWidgets', - 'PyQt5.QtWebKitWidgets', - 'PyQt5.QtWebEngine'] + self.modules = collections.OrderedDict( + [(mod, True) for mod in version.MODULE_INFO]) self.version_attribute = '__version__' self.version = '1.2.3' self._real_import = builtins.__import__ @@ -623,13 +607,14 @@ def import_fake(monkeypatch): class TestModuleVersions: - """Tests for _module_versions().""" + """Tests for _module_versions() and ModuleInfo.""" def test_all_present(self, import_fake): """Test with all modules present in version 1.2.3.""" expected = [] for name in import_fake.modules: - if name in import_fake.no_version_attribute: + version.MODULE_INFO[name]._reset_cache() + if '__version__' not in version.MODULE_INFO[name]._version_attributes: expected.append('{}: yes'.format(name)) else: expected.append('{}: 1.2.3'.format(name)) @@ -637,6 +622,7 @@ class TestModuleVersions: @pytest.mark.parametrize('module, idx, expected', [ ('colorama', 1, 'colorama: no'), + ('adblock', 6, 'adblock: no'), ]) def test_missing_module(self, module, idx, expected, import_fake): """Test with a module missing. @@ -647,8 +633,45 @@ class TestModuleVersions: expected: The expected text. """ import_fake.modules[module] = False + # Needed after mocking the module + mod_info = version.MODULE_INFO[module] + mod_info._reset_cache() + assert version._module_versions()[idx] == expected + for method_name, expected_result in [ + ("is_installed", False), + ("is_usable", False), + ("get_version", None), + ("is_outdated", None) + ]: + method = getattr(mod_info, method_name) + # With hot cache + mod_info._initialize_info() + assert method() == expected_result + # With cold cache + mod_info._reset_cache() + assert method() == expected_result + + def test_outdated_adblock(self, import_fake): + """Test that warning is shown when adblock module is outdated.""" + mod_info = version.MODULE_INFO["adblock"] + fake_version = "0.1.0" + + # Needed after mocking version attribute + mod_info._reset_cache() + + assert mod_info.min_version is not None + assert fake_version < mod_info.min_version + import_fake.version = fake_version + + assert mod_info.is_installed() + assert mod_info.is_outdated() + assert not mod_info.is_usable() + + expected = f"adblock: {fake_version} (< {mod_info.min_version}, outdated)" + assert version._module_versions()[6] == expected + @pytest.mark.parametrize('attribute, expected_modules', [ ('VERSION', ['colorama']), ('SIP_VERSION_STR', ['sip']), @@ -665,12 +688,22 @@ class TestModuleVersions: expected: The expected return value. """ import_fake.version_attribute = attribute + + for mod_info in version.MODULE_INFO.values(): + # Invalidate the "version cache" since we just mocked some of the + # attributes. + mod_info._reset_cache() + expected = [] for name in import_fake.modules: + mod_info = version.MODULE_INFO[name] if name in expected_modules: + assert mod_info.get_version() == "1.2.3" expected.append('{}: 1.2.3'.format(name)) else: + assert mod_info.get_version() is None expected.append('{}: yes'.format(name)) + assert version._module_versions() == expected @pytest.mark.parametrize('name, has_version', [ @@ -680,6 +713,7 @@ class TestModuleVersions: ('jinja2', True), ('pygments', True), ('yaml', True), + ('adblock', True), ('attr', True), ]) def test_existing_attributes(self, name, has_version): @@ -692,7 +726,7 @@ class TestModuleVersions: name: The name of the module to check. has_version: Whether a __version__ attribute is expected. """ - module = importlib.import_module(name) + module = pytest.importorskip(name) assert hasattr(module, '__version__') == has_version def test_existing_sip_attribute(self): @@ -69,7 +69,7 @@ commands = {[testenv:vulture]commands} [testenv:pylint] -basepython = {env:PYTHON:python3} +basepython = {env:PYTHON:python3.8} ignore_errors = true passenv = deps = @@ -129,7 +129,7 @@ passenv = deps = -r{toxinidir}/misc/requirements/requirements-check-manifest.txt commands = - {envdir}/bin/check-manifest --ignore 'qutebrowser/git-commit-id,qutebrowser/html/doc,qutebrowser/html/doc/*,*/__pycache__' + {envdir}/bin/check-manifest --ignore 'qutebrowser/git-commit-id,qutebrowser/html/doc,qutebrowser/html/doc/*,qutebrowser/html/doc/img/cheatsheet-*.png,*/__pycache__' [testenv:docs] basepython = {env:PYTHON:python3} |