[enh] introduce categories_as_tabs

Previously all categories were displayed as search engine tabs.
This commit changes that so that only the categories listed under
categories_as_tabs in settings.yml are displayed.

This lets us introduce more categories without cluttering up the UI.
Categories not displayed as tabs  can still be searched with !bangs.

12 files changed, 82 insertions, 29 deletions

diff --git a/docs/admin/engines/configured_engines.rst b/docs/admin/engines/configured_engines.rst
index 82a1d2375..c7b6a1f52 100644
--- a/docs/admin/engines/configured_engines.rst
+++ b/docs/admin/engines/configured_engines.rst
@@ -16,11 +16,18 @@ Explanation of the :ref:`general engine configuration` shown in the table
 
    SearXNG supports {{engines | length}} search engines (of which {{enabled_engine_count}} are enabled by default).
 
-   {% for category, engines in categories.items() %}
+   {% for category, engines in categories_as_tabs.items() %}
 
    {{category}} search engines
    ---------------------------------------
 
+   {% for group, engines in engines | group_engines_in_tab %}
+
+   {% if loop.length > 1 %}
+   {{group}}
+   ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+   {% endif %}
+
    .. flat-table::
       :header-rows: 2
       :stub-columns: 1
@@ -39,7 +46,7 @@ Explanation of the :ref:`general engine configuration` shown in the table
         - Safe search
         - Time range
 
-      {% for mod in engines | sort_engines %}
+      {% for mod in engines %}
 
       * - `{{mod.name}} <{{mod.about and mod.about.website}}>`_
         - ``!{{mod.shortcut}}``
@@ -65,3 +72,4 @@ Explanation of the :ref:`general engine configuration` shown in the table
 
      {% endfor %}
      {% endfor %}
+     {% endfor %}
diff --git a/docs/admin/engines/settings.rst b/docs/admin/engines/settings.rst
index b04de7cb3..cf561a5ab 100644
--- a/docs/admin/engines/settings.rst
+++ b/docs/admin/engines/settings.rst
@@ -219,6 +219,26 @@ Communication with search engines.
 ``max_redirects`` :
   30 by default. Maximum redirect before it is an error.
 
+``categories_as_tabs:``
+-----------------------
+
+A list of the categories that are displayed as tabs in the user interface.
+Categories not listed here can still be searched with the :ref:`search-syntax`.
+
+.. code-block:: yaml
+
+  categories_as_tabs:
+    - general
+    - images
+    - videos
+    - news
+    - map
+    - music
+    - it
+    - science
+    - files
+    - social media
+
 .. _settings engine:
 
 Engine settings
diff --git a/docs/conf.py b/docs/conf.py
index 1b78ece60..da989e62e 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -50,14 +50,11 @@ jinja_contexts = {
         },
         'enabled_engine_count': sum(not x.disabled for x in searx.engines.engines.values()),
         'categories': searx.engines.categories,
+        'categories_as_tabs': {c: searx.engines.categories[c] for c in searx.settings['categories_as_tabs']},
     },
 }
 jinja_filters = {
-    'sort_engines':
-    lambda engines: sorted(
-        engines,
-        key=lambda engine: (engine.about.get('language', ''), engine.name)
-    )
+    'group_engines_in_tab': searx.engines.group_engines_in_tab,
 }
 
 # Let the Jinja template in configured_engines.rst access documented_modules
diff --git a/searx/engines/__init__.py b/searx/engines/__init__.py
index 1fc01600a..8d49bae8a 100644
--- a/searx/engines/__init__.py
+++ b/searx/engines/__init__.py
@@ -13,6 +13,7 @@ usage::
 
 import sys
 import copy
+import itertools
 
 from os.path import realpath, dirname
 from babel.localedata import locale_identifiers
@@ -260,3 +261,26 @@ def load_engines(engine_list):
         if engine:
             register_engine(engine)
     return engines
+
+
+DEFAULT_GROUP_NAME = 'others'
+
+
+def group_engines_in_tab(engines):  # pylint: disable=redefined-outer-name
+    def engine_sort_key(engine):
+        return (engine.about.get('language', ''), engine.name)
+
+    def group_sort_key(group):
+        return (group[0] == DEFAULT_GROUP_NAME, group[0].lower())
+
+    def get_group(eng):
+        non_tab_engines = [c for c in eng.categories if c not in settings['categories_as_tabs']]
+        return non_tab_engines[0] if len(non_tab_engines) > 0 else DEFAULT_GROUP_NAME
+
+    return [
+        (groupname, sorted(engines, key=engine_sort_key))
+        for groupname, engines in sorted(
+            ((name, list(engines)) for name, engines in itertools.groupby(sorted(engines, key=get_group), get_group)),
+            key=group_sort_key,
+        )
+    ]
diff --git a/searx/settings.yml b/searx/settings.yml
index 3227a5a55..18d047535 100644
--- a/searx/settings.yml
+++ b/searx/settings.yml
@@ -81,12 +81,6 @@ ui:
     simple_style: auto
     # Open result links in a new tab by default
     # results_on_new_tab: false
-    # categories_order :
-    #   - general
-    #   - files
-    #   - map
-    #   - it
-    #   - science
 
 # Lock arbitrary settings on the preferences page.  To find the ID of the user
 # setting you want to lock, check the ID of the form on the page "preferences".
@@ -233,6 +227,18 @@ checker:
         result_container:
           - has_infobox
 
+categories_as_tabs:
+  - general
+  - images
+  - videos
+  - news
+  - map
+  - music
+  - it
+  - science
+  - files
+  - social media
+
 engines:
   - name: apk mirror
     engine: apkmirror
diff --git a/searx/settings_defaults.py b/searx/settings_defaults.py
index 9c4711bfc..d8e3ec693 100644
--- a/searx/settings_defaults.py
+++ b/searx/settings_defaults.py
@@ -20,7 +20,7 @@ OUTPUT_FORMATS = ['html', 'csv', 'json', 'rss']
 LANGUAGE_CODES = ['all'] + list(l[0] for l in languages)
 OSCAR_STYLE = ('logicodev', 'logicodev-dark', 'pointhi')
 SIMPLE_STYLE = ('auto', 'light', 'dark')
-CATEGORY_ORDER = [
+CATEGORIES_AS_TABS = [
     'general',
     'images',
     'videos',
@@ -181,7 +181,6 @@ SCHEMA = {
         'results_on_new_tab': SettingsValue(bool, False),
         'advanced_search': SettingsValue(bool, False),
         'query_in_title': SettingsValue(bool, False),
-        'categories_order': SettingsValue(list, CATEGORY_ORDER),
     },
     'preferences': {
         'lock': SettingsValue(list, []),
@@ -212,6 +211,7 @@ SCHEMA = {
     'checker': {
         'off_when_debug': SettingsValue(bool, True),
     },
+    'categories_as_tabs': SettingsValue(list, CATEGORIES_AS_TABS),
     'engines': SettingsValue(list, []),
     'doi_resolvers': {},
 }
diff --git a/searx/templates/oscar/categories.html b/searx/templates/oscar/categories.html
index a5c5f61c7..39d1e1d33 100644
--- a/searx/templates/oscar/categories.html
+++ b/searx/templates/oscar/categories.html
@@ -1,11 +1,11 @@
 <div id="categories">
 {%- if rtl -%}
-    {% for category in categories | reverse -%}
+    {% for category in categories_as_tabs | reverse -%}
         <input class="hidden" type="checkbox" id="checkbox_{{ category|replace(' ', '_') }}" name="category_{{ category }}" {% if category in selected_categories %}checked="checked"{% endif %} />{{- '' -}}
         <label for="checkbox_{{ category|replace(' ', '_') }}">{{ _(category) }}</label>
     {%- endfor %}
 {%- else -%}
-    {% for category in categories -%}
+    {% for category in categories_as_tabs -%}
         <input class="hidden" type="checkbox" id="checkbox_{{ category|replace(' ', '_') }}" name="category_{{ category }}" {% if category in selected_categories %}checked="checked"{% endif %} />{{- '' -}}
         <label for="checkbox_{{ category|replace(' ', '_') }}">{{ _(category) }}</label>
     {%- endfor %}
diff --git a/searx/templates/oscar/preferences.html b/searx/templates/oscar/preferences.html
index b76ee44a1..da88cd755 100644
--- a/searx/templates/oscar/preferences.html
+++ b/searx/templates/oscar/preferences.html
@@ -298,7 +298,7 @@
             <div class="tab-pane active_if_nojs" id="tab_engine">
                 <!-- Nav tabs -->
                 <ul class="nav nav-tabs nav-justified hide_if_nojs" role="tablist">
-                    {% for categ in all_categories %}
+                    {% for categ in categories_as_tabs %}
                     <li{% if loop.first %} class="active"{% endif %}><a href="#tab_engine_{{ categ|replace(' ', '_') }}" role="tab" data-toggle="tab">{{ _(categ) }}</a></li>
                     {% endfor %}
                 </ul>
@@ -317,7 +317,7 @@
                         </p>
                     </div>
 
-                    {% for categ in all_categories %}
+                    {% for categ in categories_as_tabs %}
                     <noscript><label>{{ _(categ) }}</label>
                     </noscript>
                     <div class="tab-pane{% if loop.first %} active{% endif %} active_if_nojs" id="tab_engine_{{ categ|replace(' ', '_') }}">
diff --git a/searx/templates/simple/categories.html b/searx/templates/simple/categories.html
index ae8b2ec09..71a211904 100644
--- a/searx/templates/simple/categories.html
+++ b/searx/templates/simple/categories.html
@@ -14,7 +14,7 @@
 <div id="categories" class="search_categories">{{- '' -}}
     <div id="categories_container">
         {%- if display_tooltip %}<div class="help">{{ _('Click on the magnifier to perform search') }}</div>{% endif -%}
-        {%- for category in categories -%}
+        {%- for category in categories_as_tabs -%}
         <div class="category"><input type="checkbox" id="checkbox_{{ category|replace(' ', '_') }}" name="category_{{ category }}"{% if category in selected_categories %} checked="checked"{% endif %}/>
             <label for="checkbox_{{ category|replace(' ', '_') }}" class="tooltips">
                 {{- icon_big(category_icons[category]) if category in category_icons  else icon_big('globe-outline') -}}
diff --git a/searx/templates/simple/preferences.html b/searx/templates/simple/preferences.html
index 8f602cd1b..5f289c580 100644
--- a/searx/templates/simple/preferences.html
+++ b/searx/templates/simple/preferences.html
@@ -274,7 +274,7 @@
   {{ tab_header('maintab', 'engines', _('Engines')) }}
     <p>{{ _('Currently used search engines') }}</p>
     {{ tabs_open() }}
-    {% for categ in all_categories %}
+    {% for categ in categories_as_tabs %}
     {{ tab_header('enginetab', 'category' + categ, _(categ)) }}
     <div class="scrollx">
     <table class="striped">
diff --git a/searx/webapp.py b/searx/webapp.py
index a2aa84d9d..f884c35ef 100755
--- a/searx/webapp.py
+++ b/searx/webapp.py
@@ -390,12 +390,6 @@ def get_translations():
     }
 
 
-def _get_ordered_categories():
-    ordered_categories = list(settings['ui']['categories_order'])
-    ordered_categories.extend(x for x in sorted(categories.keys()) if x not in ordered_categories)
-    return ordered_categories
-
-
 def _get_enable_categories(all_categories):
     disabled_engines = request.preferences.engines.get_disabled()
     enabled_categories = set(
@@ -430,8 +424,8 @@ def render(template_name, override_theme=None, **kwargs):
     kwargs['query_in_title'] = request.preferences.get_value('query_in_title')
     kwargs['safesearch'] = str(request.preferences.get_value('safesearch'))
     kwargs['theme'] = get_current_theme_name(override=override_theme)
-    kwargs['all_categories'] = _get_ordered_categories()
-    kwargs['categories'] = _get_enable_categories(kwargs['all_categories'])
+    kwargs['categories_as_tabs'] = settings['categories_as_tabs']
+    kwargs['categories'] = _get_enable_categories(categories.keys())
 
     # i18n
     kwargs['language_codes'] = [l for l in languages if l[0] in settings['search']['languages']]
diff --git a/tests/robot/settings_robot.yml b/tests/robot/settings_robot.yml
index e0d01bb37..438ccf9a0 100644
--- a/tests/robot/settings_robot.yml
+++ b/tests/robot/settings_robot.yml
@@ -33,6 +33,10 @@ outgoing:
   request_timeout: 1.0  # seconds
   useragent_suffix: ""
 
+categories_as_tabs:
+  - general
+  - dummy
+
 engines:
   - name: general dummy
     engine: dummy