[enh] introduce categories_as_tabs

Previously all categories were displayed as search engine tabs.
This commit changes that so that only the categories listed under
categories_as_tabs in settings.yml are displayed.

This lets us introduce more categories without cluttering up the UI.
Categories not displayed as tabs  can still be searched with !bangs.

8 files changed, 46 insertions, 22 deletions

diff --git a/searx/engines/__init__.py b/searx/engines/__init__.py
index 1fc01600a..8d49bae8a 100644
--- a/searx/engines/__init__.py
+++ b/searx/engines/__init__.py
@@ -13,6 +13,7 @@ usage::
 
 import sys
 import copy
+import itertools
 
 from os.path import realpath, dirname
 from babel.localedata import locale_identifiers
@@ -260,3 +261,26 @@ def load_engines(engine_list):
         if engine:
             register_engine(engine)
     return engines
+
+
+DEFAULT_GROUP_NAME = 'others'
+
+
+def group_engines_in_tab(engines):  # pylint: disable=redefined-outer-name
+    def engine_sort_key(engine):
+        return (engine.about.get('language', ''), engine.name)
+
+    def group_sort_key(group):
+        return (group[0] == DEFAULT_GROUP_NAME, group[0].lower())
+
+    def get_group(eng):
+        non_tab_engines = [c for c in eng.categories if c not in settings['categories_as_tabs']]
+        return non_tab_engines[0] if len(non_tab_engines) > 0 else DEFAULT_GROUP_NAME
+
+    return [
+        (groupname, sorted(engines, key=engine_sort_key))
+        for groupname, engines in sorted(
+            ((name, list(engines)) for name, engines in itertools.groupby(sorted(engines, key=get_group), get_group)),
+            key=group_sort_key,
+        )
+    ]
diff --git a/searx/settings.yml b/searx/settings.yml
index 3227a5a55..18d047535 100644
--- a/searx/settings.yml
+++ b/searx/settings.yml
@@ -81,12 +81,6 @@ ui:
     simple_style: auto
     # Open result links in a new tab by default
     # results_on_new_tab: false
-    # categories_order :
-    #   - general
-    #   - files
-    #   - map
-    #   - it
-    #   - science
 
 # Lock arbitrary settings on the preferences page.  To find the ID of the user
 # setting you want to lock, check the ID of the form on the page "preferences".
@@ -233,6 +227,18 @@ checker:
         result_container:
           - has_infobox
 
+categories_as_tabs:
+  - general
+  - images
+  - videos
+  - news
+  - map
+  - music
+  - it
+  - science
+  - files
+  - social media
+
 engines:
   - name: apk mirror
     engine: apkmirror
diff --git a/searx/settings_defaults.py b/searx/settings_defaults.py
index 9c4711bfc..d8e3ec693 100644
--- a/searx/settings_defaults.py
+++ b/searx/settings_defaults.py
@@ -20,7 +20,7 @@ OUTPUT_FORMATS = ['html', 'csv', 'json', 'rss']
 LANGUAGE_CODES = ['all'] + list(l[0] for l in languages)
 OSCAR_STYLE = ('logicodev', 'logicodev-dark', 'pointhi')
 SIMPLE_STYLE = ('auto', 'light', 'dark')
-CATEGORY_ORDER = [
+CATEGORIES_AS_TABS = [
     'general',
     'images',
     'videos',
@@ -181,7 +181,6 @@ SCHEMA = {
         'results_on_new_tab': SettingsValue(bool, False),
         'advanced_search': SettingsValue(bool, False),
         'query_in_title': SettingsValue(bool, False),
-        'categories_order': SettingsValue(list, CATEGORY_ORDER),
     },
     'preferences': {
         'lock': SettingsValue(list, []),
@@ -212,6 +211,7 @@ SCHEMA = {
     'checker': {
         'off_when_debug': SettingsValue(bool, True),
     },
+    'categories_as_tabs': SettingsValue(list, CATEGORIES_AS_TABS),
     'engines': SettingsValue(list, []),
     'doi_resolvers': {},
 }
diff --git a/searx/templates/oscar/categories.html b/searx/templates/oscar/categories.html
index a5c5f61c7..39d1e1d33 100644
--- a/searx/templates/oscar/categories.html
+++ b/searx/templates/oscar/categories.html
@@ -1,11 +1,11 @@
 <div id="categories">
 {%- if rtl -%}
-    {% for category in categories | reverse -%}
+    {% for category in categories_as_tabs | reverse -%}
         <input class="hidden" type="checkbox" id="checkbox_{{ category|replace(' ', '_') }}" name="category_{{ category }}" {% if category in selected_categories %}checked="checked"{% endif %} />{{- '' -}}
         <label for="checkbox_{{ category|replace(' ', '_') }}">{{ _(category) }}</label>
     {%- endfor %}
 {%- else -%}
-    {% for category in categories -%}
+    {% for category in categories_as_tabs -%}
         <input class="hidden" type="checkbox" id="checkbox_{{ category|replace(' ', '_') }}" name="category_{{ category }}" {% if category in selected_categories %}checked="checked"{% endif %} />{{- '' -}}
         <label for="checkbox_{{ category|replace(' ', '_') }}">{{ _(category) }}</label>
     {%- endfor %}
diff --git a/searx/templates/oscar/preferences.html b/searx/templates/oscar/preferences.html
index b76ee44a1..da88cd755 100644
--- a/searx/templates/oscar/preferences.html
+++ b/searx/templates/oscar/preferences.html
@@ -298,7 +298,7 @@
             <div class="tab-pane active_if_nojs" id="tab_engine">
                 <!-- Nav tabs -->
                 <ul class="nav nav-tabs nav-justified hide_if_nojs" role="tablist">
-                    {% for categ in all_categories %}
+                    {% for categ in categories_as_tabs %}
                     <li{% if loop.first %} class="active"{% endif %}><a href="#tab_engine_{{ categ|replace(' ', '_') }}" role="tab" data-toggle="tab">{{ _(categ) }}</a></li>
                     {% endfor %}
                 </ul>
@@ -317,7 +317,7 @@
                         </p>
                     </div>
 
-                    {% for categ in all_categories %}
+                    {% for categ in categories_as_tabs %}
                     <noscript><label>{{ _(categ) }}</label>
                     </noscript>
                     <div class="tab-pane{% if loop.first %} active{% endif %} active_if_nojs" id="tab_engine_{{ categ|replace(' ', '_') }}">
diff --git a/searx/templates/simple/categories.html b/searx/templates/simple/categories.html
index ae8b2ec09..71a211904 100644
--- a/searx/templates/simple/categories.html
+++ b/searx/templates/simple/categories.html
@@ -14,7 +14,7 @@
 <div id="categories" class="search_categories">{{- '' -}}
     <div id="categories_container">
         {%- if display_tooltip %}<div class="help">{{ _('Click on the magnifier to perform search') }}</div>{% endif -%}
-        {%- for category in categories -%}
+        {%- for category in categories_as_tabs -%}
         <div class="category"><input type="checkbox" id="checkbox_{{ category|replace(' ', '_') }}" name="category_{{ category }}"{% if category in selected_categories %} checked="checked"{% endif %}/>
             <label for="checkbox_{{ category|replace(' ', '_') }}" class="tooltips">
                 {{- icon_big(category_icons[category]) if category in category_icons  else icon_big('globe-outline') -}}
diff --git a/searx/templates/simple/preferences.html b/searx/templates/simple/preferences.html
index 8f602cd1b..5f289c580 100644
--- a/searx/templates/simple/preferences.html
+++ b/searx/templates/simple/preferences.html
@@ -274,7 +274,7 @@
   {{ tab_header('maintab', 'engines', _('Engines')) }}
     <p>{{ _('Currently used search engines') }}</p>
     {{ tabs_open() }}
-    {% for categ in all_categories %}
+    {% for categ in categories_as_tabs %}
     {{ tab_header('enginetab', 'category' + categ, _(categ)) }}
     <div class="scrollx">
     <table class="striped">
diff --git a/searx/webapp.py b/searx/webapp.py
index a2aa84d9d..f884c35ef 100755
--- a/searx/webapp.py
+++ b/searx/webapp.py
@@ -390,12 +390,6 @@ def get_translations():
     }
 
 
-def _get_ordered_categories():
-    ordered_categories = list(settings['ui']['categories_order'])
-    ordered_categories.extend(x for x in sorted(categories.keys()) if x not in ordered_categories)
-    return ordered_categories
-
-
 def _get_enable_categories(all_categories):
     disabled_engines = request.preferences.engines.get_disabled()
     enabled_categories = set(
@@ -430,8 +424,8 @@ def render(template_name, override_theme=None, **kwargs):
     kwargs['query_in_title'] = request.preferences.get_value('query_in_title')
     kwargs['safesearch'] = str(request.preferences.get_value('safesearch'))
     kwargs['theme'] = get_current_theme_name(override=override_theme)
-    kwargs['all_categories'] = _get_ordered_categories()
-    kwargs['categories'] = _get_enable_categories(kwargs['all_categories'])
+    kwargs['categories_as_tabs'] = settings['categories_as_tabs']
+    kwargs['categories'] = _get_enable_categories(categories.keys())
 
     # i18n
     kwargs['language_codes'] = [l for l in languages if l[0] in settings['search']['languages']]