gaps: fix inner gaps for stacked/tabbed containers in splith/splitv (#5275)

Fixes https://github.com/i3/i3/issues/5261

2 files changed, 30 insertions, 8 deletions

diff --git a/src/con.c b/src/con.c
index 47f75167..5bf559a1 100644
--- a/src/con.c
+++ b/src/con.c
@@ -2579,3 +2579,18 @@ void con_merge_into(Con *old, Con *new) {
 
     tree_close_internal(old, DONT_KILL_WINDOW, false);
 }
+
+/*
+ * Returns true if the container is within any stacked/tabbed split container.
+ *
+ */
+bool con_inside_stacked_or_tabbed(Con *con) {
+    if (con->parent == NULL) {
+        return false;
+    }
+    if (con->parent->layout == L_STACKED ||
+        con->parent->layout == L_TABBED) {
+        return true;
+    }
+    return con_inside_stacked_or_tabbed(con->parent);
+}
diff --git a/src/gaps.c b/src/gaps.c
index 3babe2b3..545c8b65 100644
--- a/src/gaps.c
+++ b/src/gaps.c
@@ -47,20 +47,27 @@ gaps_t calculate_effective_gaps(Con *con) {
  * Decides whether the container should be inset.
  */
 bool gaps_should_inset_con(Con *con, int children) {
+    /* No parent? None of the conditionals below can be true. */
+    if (con->parent == NULL) {
+        return false;
+    }
+
+    const bool leaf_or_stacked_tabbed =
+        con_is_leaf(con) ||
+        (con->layout == L_STACKED || con->layout == L_TABBED);
+
     /* Inset direct children of the workspace that are leaf containers or
        stacked/tabbed containers. */
-    if (con->parent != NULL &&
-        con->parent->type == CT_WORKSPACE &&
-        (con_is_leaf(con) ||
-         (con->layout == L_STACKED || con->layout == L_TABBED))) {
+    if (leaf_or_stacked_tabbed &&
+        con->parent->type == CT_WORKSPACE) {
         return true;
     }
 
     /* Inset direct children of vertical or horizontal split containers at any
-       depth in the tree (only leaf containers, not split containers within
-       split containers, to avoid double insets). */
-    if (con_is_leaf(con) &&
-        con->parent != NULL &&
+       depth in the tree. Do not inset as soon as any parent is a stacked or
+       tabbed container, to avoid double insets. */
+    if (leaf_or_stacked_tabbed &&
+        !con_inside_stacked_or_tabbed(con) &&
         con->parent->type == CT_CON &&
         (con->parent->layout == L_SPLITH ||
          con->parent->layout == L_SPLITV)) {