Add support for patterns on %include #25140

Also adds generic tor_glob function to expand globs.

8 files changed, 428 insertions, 46 deletions

diff --git a/src/lib/fs/conffile.c b/src/lib/fs/conffile.c
index 9583093c12..f1f6d8ae5f 100644
--- a/src/lib/fs/conffile.c
+++ b/src/lib/fs/conffile.c
@@ -21,6 +21,8 @@
 #include "lib/malloc/malloc.h"
 #include "lib/string/printf.h"
 
+#include <stdbool.h>
+
 static smartlist_t *config_get_file_list(const char *path,
                                          smartlist_t *opened_files);
 static int config_get_included_config(const char *path, int recursion_level,
@@ -50,62 +52,109 @@ config_get_lines_include(const char *string, config_line_t **result,
                               opened_lst, 1, NULL, config_process_include);
 }
 
-/** Adds a list of configuration files present on <b>path</b> to
- * <b>file_list</b>. <b>path</b> can be a file or a directory. If it is a file,
- * only that file will be added to <b>file_list</b>. If it is a directory,
- * all paths for files on that directory root (no recursion) except for files
- * whose name starts with a dot will be added to <b>file_list</b>.
- * <b>opened_files</b> will have a list of files opened by this function
- * if provided. Return 0 on success, -1 on failure. Ignores empty files.
- */
+/** Returns a list of paths obtained when expading globs in <b>pattern</b>. If
+ * <b>pattern</b> has no globs, returns a list with <b>pattern</b> if it is an
+ * existing path or NULL otherwise. If <b>opened_files</b> is provided, adds
+ * paths opened by glob to it. Returns NULL on failure. */
 static smartlist_t *
-config_get_file_list(const char *path, smartlist_t *opened_files)
+expand_glob(const char *pattern, smartlist_t *opened_files)
 {
-  smartlist_t *file_list = smartlist_new();
+  smartlist_t *matches = tor_glob(pattern);
+  if (!matches) {
+    return NULL;
+  }
 
-  if (opened_files) {
-    smartlist_add_strdup(opened_files, path);
+  // if it is not a glob, return error when the path is missing
+  if (!has_glob(pattern) && smartlist_len(matches) == 0) {
+    smartlist_free(matches);
+    return NULL;
   }
 
-  file_status_t file_type = file_status(path);
-  if (file_type == FN_FILE) {
-    smartlist_add_strdup(file_list, path);
-    return file_list;
-  } else if (file_type == FN_DIR) {
-    smartlist_t *all_files = tor_listdir(path);
-    if (!all_files) {
-      smartlist_free(file_list);
+  if (opened_files) {
+    smartlist_t *glob_opened = get_glob_opened_files(pattern);
+    if (!glob_opened) {
+      SMARTLIST_FOREACH(matches, char *, f, tor_free(f));
+      smartlist_free(matches);
       return NULL;
     }
-    smartlist_sort_strings(all_files);
-    SMARTLIST_FOREACH_BEGIN(all_files, char *, f) {
-      if (f[0] == '.') {
-        tor_free(f);
-        continue;
-      }
+    smartlist_add_all(opened_files, glob_opened);
+    smartlist_free(glob_opened);
+  }
+  smartlist_sort_strings(matches);
+  return matches;
+}
+
+/** Returns a list of configuration files present on paths that match
+ * <b>pattern</b>. The pattern is expanded and then all the paths are
+ * processed. A path can be a file or a directory. If it is a file, that file
+ * will be added to the list to be returned. If it is a directory,
+ * all paths for files on that directory root (no recursion) except for files
+ * whose name starts with a dot will be added to the list to be returned.
+ * <b>opened_files</b> will have a list of files opened by this function
+ * if provided. Return NULL on failure. Ignores empty files.
+ */
+static smartlist_t *
+config_get_file_list(const char *pattern, smartlist_t *opened_files)
+{
+  smartlist_t *glob_matches = expand_glob(pattern, opened_files);
+  if (!glob_matches) {
+    return NULL;
+  }
 
-      char *fullname;
-      tor_asprintf(&fullname, "%s"PATH_SEPARATOR"%s", path, f);
-      tor_free(f);
+  bool error_found = false;
+  smartlist_t *file_list = smartlist_new();
+  SMARTLIST_FOREACH_BEGIN(glob_matches, char *, path) {
+    if (opened_files) {
+      smartlist_add_strdup(opened_files, path);
+    }
 
-      if (opened_files) {
-        smartlist_add_strdup(opened_files, fullname);
+    file_status_t file_type = file_status(path);
+    if (file_type == FN_FILE) {
+      smartlist_add_strdup(file_list, path);
+    } else if (file_type == FN_DIR) {
+      smartlist_t *all_files = tor_listdir(path);
+      if (!all_files) {
+        error_found = true;
+        break;
       }
-
-      if (file_status(fullname) != FN_FILE) {
-        tor_free(fullname);
+      smartlist_sort_strings(all_files);
+      SMARTLIST_FOREACH_BEGIN(all_files, char *, f) {
+        if (f[0] == '.') {
+          continue;
+        }
+
+        char *fullname;
+        tor_asprintf(&fullname, "%s"PATH_SEPARATOR"%s", path, f);
+
+        if (opened_files) {
+          smartlist_add_strdup(opened_files, fullname);
+        }
+
+        if (file_status(fullname) != FN_FILE) {
+          tor_free(fullname);
+          continue;
+        }
+        smartlist_add(file_list, fullname);
+      } SMARTLIST_FOREACH_END(f);
+      SMARTLIST_FOREACH(all_files, char *, f, tor_free(f));
+      smartlist_free(all_files);
+    } else if (file_type == FN_EMPTY) {
         continue;
-      }
-      smartlist_add(file_list, fullname);
-    } SMARTLIST_FOREACH_END(f);
-    smartlist_free(all_files);
-    return file_list;
-  } else if (file_type == FN_EMPTY) {
-      return file_list;
-  } else {
+    } else {
+      error_found = true;
+      break;
+    }
+  } SMARTLIST_FOREACH_END(path);
+  SMARTLIST_FOREACH(glob_matches, char *, f, tor_free(f));
+  smartlist_free(glob_matches);
+
+  if (error_found) {
+    SMARTLIST_FOREACH(file_list, char *, f, tor_free(f));
     smartlist_free(file_list);
-    return NULL;
+    file_list = NULL;
   }
+
+  return file_list;
 }
 
 /** Creates a list of config lines present on included <b>path</b>.
@@ -133,19 +182,19 @@ config_get_included_config(const char *path, int recursion_level, int extended,
   return 0;
 }
 
-/** Process an %include <b>path</b> in a config file. Set <b>list</b> to the
+/** Process an %include <b>pattern</b> in a config file. Set <b>list</b> to the
  * list of configuration settings obtained and <b>list_last</b> to the last
  * element of the same list. <b>opened_lst</b> will have a list of opened
  * files if provided. Return 0 on success, -1 on failure. */
 static int
-config_process_include(const char *path, int recursion_level, int extended,
+config_process_include(const char *pattern, int recursion_level, int extended,
                        config_line_t **list, config_line_t **list_last,
                        smartlist_t *opened_lst)
 {
   config_line_t *ret_list = NULL;
   config_line_t **next = &ret_list;
 
-  smartlist_t *config_files = config_get_file_list(path, opened_lst);
+  smartlist_t *config_files = config_get_file_list(pattern, opened_lst);
   if (!config_files) {
     return -1;
   }
diff --git a/src/lib/fs/files.c b/src/lib/fs/files.c
index a0b5a40aac..b4a432701f 100644
--- a/src/lib/fs/files.c
+++ b/src/lib/fs/files.c
@@ -247,6 +247,22 @@ file_status(const char *fname)
   }
 }
 
+/** Returns true if <b>file_type</b> represents an existing file (even if
+ * empty). Returns false otherwise. */
+bool
+is_file(file_status_t file_type)
+{
+  return file_type != FN_ERROR && file_type != FN_NOENT && file_type != FN_DIR;
+}
+
+/** Returns true if <b>file_type</b> represents an existing directory. Returns
+ * false otherwise. */
+bool
+is_dir(file_status_t file_type)
+{
+  return file_type == FN_DIR;
+}
+
 /** Create a file named <b>fname</b> with the contents <b>str</b>.  Overwrite
  * the previous <b>fname</b> if possible.  Return 0 on success, -1 on failure.
  *
diff --git a/src/lib/fs/files.h b/src/lib/fs/files.h
index a109cd6248..6eeba85e89 100644
--- a/src/lib/fs/files.h
+++ b/src/lib/fs/files.h
@@ -55,6 +55,8 @@ MOCK_DECL(int,tor_unlink,(const char *pathname));
 typedef enum { FN_ERROR, FN_NOENT, FN_FILE, FN_DIR, FN_EMPTY } file_status_t;
 
 file_status_t file_status(const char *filename);
+bool is_file(file_status_t file_type);
+bool is_dir(file_status_t file_type);
 
 int64_t tor_get_avail_disk_space(const char *path);
 
diff --git a/src/lib/fs/path.c b/src/lib/fs/path.c
index 0d57be4b06..4532bfb7d1 100644
--- a/src/lib/fs/path.c
+++ b/src/lib/fs/path.c
@@ -13,18 +13,44 @@
 #include "lib/malloc/malloc.h"
 #include "lib/log/log.h"
 #include "lib/log/util_bug.h"
+#include "lib/container/smartlist.h"
+#include "lib/sandbox/sandbox.h"
 #include "lib/string/printf.h"
 #include "lib/string/util_string.h"
 #include "lib/string/compat_ctype.h"
+#include "lib/string/compat_string.h"
+#include "lib/fs/files.h"
+#include "lib/fs/dir.h"
 #include "lib/fs/userdb.h"
 
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
 #ifdef HAVE_UNISTD_H
 #include <unistd.h>
 #endif
 
+#ifdef _WIN32
+#include <windows.h>
+#include <shlwapi.h>
+#else /* !(defined(_WIN32)) */
+#include <dirent.h>
+#include <glob.h>
+#endif /* defined(_WIN32) */
+
 #include <errno.h>
 #include <string.h>
 
+#ifdef _WIN32
+#define IS_GLOB_CHAR(s,i) (((s)[(i)]) == '*' || ((s)[(i)]) == '?')
+#else
+#define IS_GLOB_CHAR(s,i) ((((s)[(i)]) == '*' || ((s)[(i)]) == '?') &&\
+                           ((i) == 0 || (s)[(i)-1] != '\\')) /* check escape */
+#endif
+
 /** Removes enclosing quotes from <b>path</b> and unescapes quotes between the
  * enclosing quotes. Backslashes are not unescaped. Return the unquoted
  * <b>path</b> on success or 0 if <b>path</b> is not quoted correctly. */
@@ -294,3 +320,273 @@ make_path_absolute(const char *fname)
   return absfname;
 #endif /* defined(_WIN32) */
 }
+
+/** Expands globs in <b>pattern</b> for the path fragment between
+ * <b>prev_sep</b> and <b>next_sep</b>. Returns NULL on failure. */
+typedef struct smartlist_t * unglob_fn(const char *pattern, int prev_sep,
+                                       int next_sep);
+
+/** Adds <b>path</b> to <b>result</b> if it exists and is a file type we can
+ * handle. Returns false if <b>path</b> is a file type we cannot handle,
+ * returns true otherwise. Used on tor_glob for WIN32. */
+static bool
+add_non_glob_path(const char *path, struct smartlist_t *result)
+{
+  file_status_t file_type = file_status(path);
+  if (file_type == FN_ERROR) {
+    return false;
+  } else if (file_type != FN_NOENT) {
+    char *to_add = tor_strdup(path);
+    clean_fname_for_stat(to_add);
+    smartlist_add(result, to_add);
+  }
+  /* If WIN32 tor_glob is called with a non-existing path, we want it to
+   * return an empty list instead of error to match the regular version */
+  return true;
+}
+
+/** Auxiliary function used by get_glob_opened_files and WIN32 tor_glob.
+ * Returns a list of paths obtained from <b>pattern</b> using <b>unglob</b> to
+ * expand each path fragment. If <b>final</b> is true, the paths are the result
+ * of the glob expansion of <b>pattern</b> (implements tor_glob). Otherwise,
+ * the paths are the paths opened by glob while expanding <b>pattern</b>
+ * (implements get_glb_opened_files). Returns NULL on failure. */
+static struct smartlist_t *
+get_glob_paths(const char *pattern, unglob_fn unglob, bool final)
+{
+  smartlist_t *result = smartlist_new();
+  int i, prev_sep = -1, next_sep = -1;
+  bool is_glob = false, error_found = false, is_sep = false, is_last = false;
+
+  // find first path fragment with globs
+  for (i = 0; pattern[i]; i++) {
+    is_glob = is_glob || IS_GLOB_CHAR(pattern, i);
+    is_last = !pattern[i+1];
+    is_sep = pattern[i] == *PATH_SEPARATOR || pattern[i] == '/';
+    if (is_sep || is_last) {
+      prev_sep = next_sep;
+      next_sep = i; // next_sep+1 is start of next fragment or end of string
+      if (is_glob) {
+        break;
+      }
+    }
+  }
+
+  if (!is_glob) { // pattern fully expanded or no glob in pattern
+    if (final && !add_non_glob_path(pattern, result)) {
+      error_found = true;
+      goto end;
+    }
+    return result;
+  }
+
+  if (!final) {
+    // add path before the glob to result
+    int len = prev_sep < 1 ? prev_sep + 1 : prev_sep; // handle /*
+    char *path_until_glob = tor_strndup(pattern, len);
+    smartlist_add(result, path_until_glob);
+  }
+
+  smartlist_t *unglobbed_paths = unglob(pattern, prev_sep, next_sep);
+  if (!unglobbed_paths) {
+    error_found = true;
+  } else {
+    // for each path for current fragment, add the rest of the pattern
+    // and call recursively to get all expanded paths
+    SMARTLIST_FOREACH_BEGIN(unglobbed_paths, char *, current_path) {
+      char *next_path;
+      tor_asprintf(&next_path, "%s"PATH_SEPARATOR"%s", current_path,
+                   &pattern[next_sep+1]);
+      smartlist_t *opened_next = get_glob_paths(next_path, unglob, final);
+      tor_free(next_path);
+      if (!opened_next) {
+        error_found = true;
+        break;
+      }
+      smartlist_add_all(result, opened_next);
+      smartlist_free(opened_next);
+    } SMARTLIST_FOREACH_END(current_path);
+    SMARTLIST_FOREACH(unglobbed_paths, char *, p, tor_free(p));
+    smartlist_free(unglobbed_paths);
+  }
+
+end:
+  if (error_found) {
+    SMARTLIST_FOREACH(result, char *, p, tor_free(p));
+    smartlist_free(result);
+    result = NULL;
+  }
+  return result;
+}
+
+#ifdef _WIN32
+/** Expands globs in <b>pattern</b> for the path fragment between
+ * <b>prev_sep</b> and <b>next_sep</b> using the WIN32 API. Returns NULL on
+ * failure. Used by the WIN32 implementation of tor_glob. */
+static struct smartlist_t *
+unglob_win32(const char *pattern, int prev_sep, int next_sep)
+{
+  smartlist_t *result = smartlist_new();
+  int len = prev_sep < 1 ? prev_sep + 1 : prev_sep; // handle /*
+  char *path_until_glob = tor_strndup(pattern, len);
+
+  if (!is_file(file_status(path_until_glob))) {
+    smartlist_t *filenames = tor_listdir(path_until_glob);
+    if (!filenames) {
+      smartlist_free(result);
+      result = NULL;
+    } else {
+      SMARTLIST_FOREACH_BEGIN(filenames, char *, filename) {
+        TCHAR tpattern[MAX_PATH] = {0};
+        TCHAR tfile[MAX_PATH] = {0};
+        char *full_path;
+        tor_asprintf(&full_path, "%s"PATH_SEPARATOR"%s",
+                     path_until_glob, filename);
+        char *path_curr_glob = tor_strndup(pattern, next_sep + 1);
+        // *\ must return only dirs, remove \ from the pattern so it matches
+        if (is_dir(file_status(full_path))) {
+          clean_fname_for_stat(path_curr_glob);
+        }
+#ifdef UNICODE
+        mbstowcs(tpattern, path_curr_glob, MAX_PATH);
+        mbstowcs(tfile, full_path, MAX_PATH);
+#else
+        strlcpy(tpattern, path_curr_glob, MAX_PATH);
+        strlcpy(tfile, full_path, MAX_PATH);
+#endif
+        if (PathMatchSpec(tfile, tpattern)) {
+          smartlist_add(result, full_path);
+        } else {
+          tor_free(full_path);
+        }
+        tor_free(path_curr_glob);
+      } SMARTLIST_FOREACH_END(filename);
+      SMARTLIST_FOREACH(filenames, char *, p, tor_free(p));
+      smartlist_free(filenames);
+    }
+  }
+  tor_free(path_until_glob);
+  return result;
+}
+#else /* !defined(_WIN32) */
+/** Same as opendir but calls sandbox_intern_string before */
+static DIR *
+prot_opendir(const char *name)
+{
+  return opendir(sandbox_intern_string(name));
+}
+
+/** Same as stat but calls sandbox_intern_string before */
+static int
+prot_stat(const char *pathname, struct stat *buf)
+{
+  return stat(sandbox_intern_string(pathname), buf);
+}
+
+/** Same as lstat but calls sandbox_intern_string before */
+static int
+prot_lstat(const char *pathname, struct stat *buf)
+{
+  return lstat(sandbox_intern_string(pathname), buf);
+}
+#endif /* defined(_WIN32) */
+
+/** Return a new list containing the paths that match the pattern
+ * <b>pattern</b>. Return NULL on error.
+ */
+struct smartlist_t *
+tor_glob(const char *pattern)
+{
+  smartlist_t *result;
+#ifdef _WIN32
+  // PathMatchSpec does not support forward slashes, change them to backslashes
+  char *pattern_normalized = tor_strdup(pattern);
+  tor_strreplacechar(pattern_normalized, '/', *PATH_SEPARATOR);
+  result = get_glob_paths(pattern_normalized, unglob_win32, true);
+  tor_free(pattern_normalized);
+#else /* !(defined(_WIN32)) */
+  glob_t matches;
+  int flags = GLOB_ERR | GLOB_NOSORT;
+#ifdef GLOB_ALTDIRFUNC
+  /* use functions that call sandbox_intern_string */
+  flags |= GLOB_ALTDIRFUNC;
+  typedef void *(*gl_opendir)(const char * name);
+  typedef struct dirent *(*gl_readdir)(void *);
+  typedef void (*gl_closedir)(void *);
+  matches.gl_opendir = (gl_opendir) &prot_opendir;
+  matches.gl_readdir = (gl_readdir) &readdir;
+  matches.gl_closedir = (gl_closedir) &closedir;
+  matches.gl_stat = &prot_stat;
+  matches.gl_lstat = &prot_lstat;
+#endif /* defined(GLOB_ALTDIRFUNC) */
+  int ret = glob(pattern, flags, NULL, &matches);
+  if (ret == GLOB_NOMATCH) {
+    return smartlist_new();
+  } else if (ret != 0) {
+    return NULL;
+  }
+
+  result = smartlist_new();
+  size_t i;
+  for (i = 0; i < matches.gl_pathc; i++) {
+    char *match = tor_strdup(matches.gl_pathv[i]);
+    size_t len = strlen(match);
+    if (len > 0 && match[len-1] == *PATH_SEPARATOR) {
+      match[len-1] = '\0';
+    }
+    smartlist_add(result, match);
+  }
+  globfree(&matches);
+#endif /* defined(_WIN32) */
+  return result;
+}
+
+/** Returns true if <b>s</b> contains characters that can be globbed.
+ * Returns false otherwise. */
+bool
+has_glob(const char *s)
+{
+  int i;
+  for (i = 0; s[i]; i++) {
+    if (IS_GLOB_CHAR(s, i)) {
+      return true;
+    }
+  }
+  return false;
+}
+
+/** Expands globs in <b>pattern</b> for the path fragment between
+ * <b>prev_sep</b> and <b>next_sep</b> using tor_glob. Returns NULL on
+ * failure. Used by get_glob_opened_files. */
+static struct smartlist_t *
+unglob_opened_files(const char *pattern, int prev_sep, int next_sep)
+{
+  (void)prev_sep;
+  smartlist_t *result = smartlist_new();
+  // if the following fragments have no globs, we're done
+  if (has_glob(&pattern[next_sep+1])) {
+    // if there is a glob after next_sep, we know it is a separator and not the
+    // last char and glob_path will have the path without the separator
+    char *glob_path = tor_strndup(pattern, next_sep);
+    smartlist_t *child_paths = tor_glob(glob_path);
+    tor_free(glob_path);
+    if (!child_paths) {
+      smartlist_free(result);
+      result = NULL;
+    } else {
+      smartlist_add_all(result, child_paths);
+      smartlist_free(child_paths);
+    }
+  }
+  return result;
+}
+
+/** Returns a list of files that are opened by the tor_glob function when
+ * called with <b>pattern</b>. Returns NULL on error. The purpose of this
+ * function is to create a list of files to be added to the sandbox white list
+ * before the sandbox is enabled. */
+struct smartlist_t *
+get_glob_opened_files(const char *pattern)
+{
+  return get_glob_paths(pattern, unglob_opened_files, false);
+}
diff --git a/src/lib/fs/path.h b/src/lib/fs/path.h
index f0e253c556..425bd12516 100644
--- a/src/lib/fs/path.h
+++ b/src/lib/fs/path.h
@@ -12,6 +12,10 @@
 #ifndef TOR_PATH_H
 #define TOR_PATH_H
 
+#include <stdbool.h>
+#ifdef _WIN32
+#include <windows.h>
+#endif
 #include "lib/cc/compat_compiler.h"
 
 #ifdef _WIN32
@@ -26,5 +30,8 @@ int path_is_relative(const char *filename);
 void clean_fname_for_stat(char *name);
 int get_parent_directory(char *fname);
 char *make_path_absolute(const char *fname);
+struct smartlist_t *tor_glob(const char *pattern);
+bool has_glob(const char *s);
+struct smartlist_t *get_glob_opened_files(const char *pattern);
 
 #endif /* !defined(TOR_PATH_H) */
diff --git a/src/lib/sandbox/sandbox.c b/src/lib/sandbox/sandbox.c
index 820e4fd1a5..81240bcfdb 100644
--- a/src/lib/sandbox/sandbox.c
+++ b/src/lib/sandbox/sandbox.c
@@ -200,6 +200,8 @@ static int filter_nopar_gen[] = {
 #ifdef __NR__llseek
     SCMP_SYS(_llseek),
 #endif
+    // glob uses this..
+    SCMP_SYS(lstat),
     SCMP_SYS(mkdir),
     SCMP_SYS(mlockall),
 #ifdef __NR_mmap
diff --git a/src/lib/string/util_string.c b/src/lib/string/util_string.c
index c8f12d780e..ba5f9f2203 100644
--- a/src/lib/string/util_string.c
+++ b/src/lib/string/util_string.c
@@ -143,6 +143,15 @@ tor_strupper(char *s)
   }
 }
 
+/** Replaces <b>old</b> with <b>replacement</b> in <b>s</b> */
+void
+tor_strreplacechar(char *s, char find, char replacement)
+{
+  for (s = strchr(s, find); s; s = strchr(s + 1, find)) {
+    *s = replacement;
+  }
+}
+
 /** Return 1 if every character in <b>s</b> is printable, else return 0.
  */
 int
diff --git a/src/lib/string/util_string.h b/src/lib/string/util_string.h
index e89233df88..15d35415fe 100644
--- a/src/lib/string/util_string.h
+++ b/src/lib/string/util_string.h
@@ -31,6 +31,7 @@ int tor_digest256_is_zero(const char *digest);
 #define HEX_CHARACTERS "0123456789ABCDEFabcdef"
 void tor_strlower(char *s);
 void tor_strupper(char *s);
+void tor_strreplacechar(char *s, char find, char replacement);
 int tor_strisprint(const char *s);
 int tor_strisnonupper(const char *s);
 int tor_strisspace(const char *s);