walk_module workaround for PyInstaller

Restored from dd2fc8e10bb9d4a1bd0158110173793a18736d6b

Now that we are putting our data files in the qutebrowser/ directory
again pkgutil/importlib is getting confused by that dir existing and
returning us a FileLoader for `qutebrowser.components`, I think that's
what's happening anyway. Should try reverting that and this commit and
see if extensions get loaded right again.

So bring back this workaround of using the toc on the PyInstaller loader
to get the list of component modules for now.

ref: #7803

2 files changed, 34 insertions, 2 deletions

diff --git a/qutebrowser/extensions/loader.py b/qutebrowser/extensions/loader.py
index b5b232c5a..7ccdabc88 100644
--- a/qutebrowser/extensions/loader.py
+++ b/qutebrowser/extensions/loader.py
@@ -6,11 +6,12 @@
 
 import pkgutil
 import types
+import sys
 import pathlib
 import importlib
 import argparse
 import dataclasses
-from typing import Callable, Iterator, List, Optional, Tuple
+from typing import Callable, Iterator, List, Optional, Set, Tuple
 
 from qutebrowser.qt.core import pyqtSlot
 
@@ -79,6 +80,14 @@ def load_components(*, skip_hooks: bool = False) -> None:
 
 def walk_components() -> Iterator[ExtensionInfo]:
     """Yield ExtensionInfo objects for all modules."""
+    if hasattr(sys, 'frozen'):
+        yield from _walk_pyinstaller()
+    else:
+        yield from _walk_normal()
+
+
+def _walk_normal() -> Iterator[ExtensionInfo]:
+    """Walk extensions when not using PyInstaller."""
     for _finder, name, ispkg in pkgutil.walk_packages(
             path=components.__path__,
             prefix=components.__name__ + '.',
@@ -93,6 +102,23 @@ def walk_components() -> Iterator[ExtensionInfo]:
         yield ExtensionInfo(name=name)
 
 
+def _walk_pyinstaller() -> Iterator[ExtensionInfo]:
+    """Walk extensions when using PyInstaller.
+
+    See https://github.com/pyinstaller/pyinstaller/issues/1905
+
+    Inspired by:
+    https://github.com/webcomics/dosage/blob/master/dosagelib/loader.py
+    """
+    toc: Set[str] = set()
+    for importer in pkgutil.iter_importers('qutebrowser'):
+        if hasattr(importer, 'toc'):
+            toc |= importer.toc
+    for name in toc:
+        if name.startswith(components.__name__ + '.'):
+            yield ExtensionInfo(name=name)
+
+
 def _get_init_context() -> InitContext:
     """Get an InitContext object."""
     return InitContext(data_dir=pathlib.Path(standarddir.data()),
diff --git a/tests/unit/extensions/test_loader.py b/tests/unit/extensions/test_loader.py
index a2a99f305..fd15130ba 100644
--- a/tests/unit/extensions/test_loader.py
+++ b/tests/unit/extensions/test_loader.py
@@ -20,10 +20,16 @@ def test_on_walk_error():
 
 
 def test_walk_normal():
-    names = [info.name for info in loader.walk_components()]
+    names = [info.name for info in loader._walk_normal()]
     assert 'qutebrowser.components.scrollcommands' in names
 
 
+def test_walk_pyinstaller():
+    # We can't test whether we get something back without being frozen by
+    # PyInstaller, but at least we can test that we don't crash.
+    list(loader._walk_pyinstaller())
+
+
 def test_load_component(monkeypatch):
     monkeypatch.setattr(objects, 'commands', {})