aboutsummaryrefslogtreecommitdiff
path: root/src/lib/fs
diff options
context:
space:
mode:
Diffstat (limited to 'src/lib/fs')
-rw-r--r--src/lib/fs/.may_include16
-rw-r--r--src/lib/fs/conffile.c174
-rw-r--r--src/lib/fs/conffile.h23
-rw-r--r--src/lib/fs/dir.c367
-rw-r--r--src/lib/fs/dir.h33
-rw-r--r--src/lib/fs/files.c721
-rw-r--r--src/lib/fs/files.h145
-rw-r--r--src/lib/fs/freespace.c63
-rw-r--r--src/lib/fs/include.am37
-rw-r--r--src/lib/fs/lockfile.c145
-rw-r--r--src/lib/fs/lockfile.h20
-rw-r--r--src/lib/fs/mmap.c240
-rw-r--r--src/lib/fs/mmap.h41
-rw-r--r--src/lib/fs/path.c295
-rw-r--r--src/lib/fs/path.h30
-rw-r--r--src/lib/fs/storagedir.c606
-rw-r--r--src/lib/fs/storagedir.h64
-rw-r--r--src/lib/fs/userdb.c138
-rw-r--r--src/lib/fs/userdb.h26
-rw-r--r--src/lib/fs/winlib.c30
-rw-r--r--src/lib/fs/winlib.h22
21 files changed, 3236 insertions, 0 deletions
diff --git a/src/lib/fs/.may_include b/src/lib/fs/.may_include
new file mode 100644
index 0000000000..b1e49fc891
--- /dev/null
+++ b/src/lib/fs/.may_include
@@ -0,0 +1,16 @@
+orconfig.h
+
+ext/getdelim.c
+
+lib/cc/*.h
+lib/container/*.h
+lib/encoding/*.h
+lib/err/*.h
+lib/fdio/*.h
+lib/fs/*.h
+lib/log/*.h
+lib/malloc/*.h
+lib/memarea/*.h
+lib/sandbox/*.h
+lib/string/*.h
+lib/testsupport/testsupport.h
diff --git a/src/lib/fs/conffile.c b/src/lib/fs/conffile.c
new file mode 100644
index 0000000000..7bb2f23931
--- /dev/null
+++ b/src/lib/fs/conffile.c
@@ -0,0 +1,174 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file conffile.h
+ *
+ * \brief Read configuration files from disk, with full `%include` support.
+ **/
+
+#include "lib/fs/conffile.h"
+
+#include "lib/container/smartlist.h"
+#include "lib/encoding/confline.h"
+#include "lib/fs/dir.h"
+#include "lib/fs/files.h"
+#include "lib/fs/path.h"
+#include "lib/log/log.h"
+#include "lib/malloc/malloc.h"
+#include "lib/string/printf.h"
+
+static smartlist_t *config_get_file_list(const char *path,
+ smartlist_t *opened_files);
+static int config_get_included_config(const char *path, int recursion_level,
+ int extended, config_line_t **config,
+ config_line_t **config_last,
+ smartlist_t *opened_lst);
+static int config_process_include(const char *path, int recursion_level,
+ int extended, config_line_t **list,
+ config_line_t **list_last,
+ smartlist_t *opened_lst);
+
+/** Helper: parse the config string and strdup into key/value
+ * strings. Set *result to the list, or NULL if parsing the string
+ * failed. Set *has_include to 1 if <b>result</b> has values from
+ * %included files. <b>opened_lst</b> will have a list of opened files if
+ * provided. Return 0 on success, -1 on failure. Warn and ignore any
+ * misformatted lines.
+ *
+ * If <b>extended</b> is set, then treat keys beginning with / and with + as
+ * indicating "clear" and "append" respectively. */
+int
+config_get_lines_include(const char *string, config_line_t **result,
+ int extended, int *has_include,
+ smartlist_t *opened_lst)
+{
+ return config_get_lines_aux(string, result, extended, 1, has_include,
+ opened_lst, 1, NULL, config_process_include);
+}
+
+/** Adds a list of configuration files present on <b>path</b> to
+ * <b>file_list</b>. <b>path</b> can be a file or a directory. If it is a file,
+ * only that file will be added to <b>file_list</b>. If it is a directory,
+ * all paths for files on that directory root (no recursion) except for files
+ * whose name starts with a dot will be added to <b>file_list</b>.
+ * <b>opened_files</b> will have a list of files opened by this function
+ * if provided. Return 0 on success, -1 on failure. Ignores empty files.
+ */
+static smartlist_t *
+config_get_file_list(const char *path, smartlist_t *opened_files)
+{
+ smartlist_t *file_list = smartlist_new();
+
+ if (opened_files) {
+ smartlist_add_strdup(opened_files, path);
+ }
+
+ file_status_t file_type = file_status(path);
+ if (file_type == FN_FILE) {
+ smartlist_add_strdup(file_list, path);
+ return file_list;
+ } else if (file_type == FN_DIR) {
+ smartlist_t *all_files = tor_listdir(path);
+ if (!all_files) {
+ smartlist_free(file_list);
+ return NULL;
+ }
+ smartlist_sort_strings(all_files);
+ SMARTLIST_FOREACH_BEGIN(all_files, char *, f) {
+ if (f[0] == '.') {
+ tor_free(f);
+ continue;
+ }
+
+ char *fullname;
+ tor_asprintf(&fullname, "%s"PATH_SEPARATOR"%s", path, f);
+ tor_free(f);
+
+ if (opened_files) {
+ smartlist_add_strdup(opened_files, fullname);
+ }
+
+ if (file_status(fullname) != FN_FILE) {
+ tor_free(fullname);
+ continue;
+ }
+ smartlist_add(file_list, fullname);
+ } SMARTLIST_FOREACH_END(f);
+ smartlist_free(all_files);
+ return file_list;
+ } else if (file_type == FN_EMPTY) {
+ return file_list;
+ } else {
+ smartlist_free(file_list);
+ return NULL;
+ }
+}
+
+/** Creates a list of config lines present on included <b>path</b>.
+ * Set <b>config</b> to the list and <b>config_last</b> to the last element of
+ * <b>config</b>. <b>opened_lst</b> will have a list of opened files if
+ * provided. Return 0 on success, -1 on failure. */
+static int
+config_get_included_config(const char *path, int recursion_level, int extended,
+ config_line_t **config, config_line_t **config_last,
+ smartlist_t *opened_lst)
+{
+ char *included_conf = read_file_to_str(path, 0, NULL);
+ if (!included_conf) {
+ return -1;
+ }
+
+ if (config_get_lines_aux(included_conf, config, extended, 1, NULL,
+ opened_lst, recursion_level+1, config_last,
+ config_process_include) < 0) {
+ tor_free(included_conf);
+ return -1;
+ }
+
+ tor_free(included_conf);
+ return 0;
+}
+
+/** Process an %include <b>path</b> in a config file. Set <b>list</b> to the
+ * list of configuration settings obtained and <b>list_last</b> to the last
+ * element of the same list. <b>opened_lst</b> will have a list of opened
+ * files if provided. Return 0 on success, -1 on failure. */
+static int
+config_process_include(const char *path, int recursion_level, int extended,
+ config_line_t **list, config_line_t **list_last,
+ smartlist_t *opened_lst)
+{
+ config_line_t *ret_list = NULL;
+ config_line_t **next = &ret_list;
+
+ smartlist_t *config_files = config_get_file_list(path, opened_lst);
+ if (!config_files) {
+ return -1;
+ }
+
+ int rv = -1;
+ SMARTLIST_FOREACH_BEGIN(config_files, const char *, config_file) {
+ config_line_t *included_config = NULL;
+ if (config_get_included_config(config_file, recursion_level, extended,
+ &included_config, list_last,
+ opened_lst) < 0) {
+ goto done;
+ }
+
+ *next = included_config;
+ if (*list_last)
+ next = &(*list_last)->next;
+
+ } SMARTLIST_FOREACH_END(config_file);
+ *list = ret_list;
+ rv = 0;
+
+ done:
+ SMARTLIST_FOREACH(config_files, char *, f, tor_free(f));
+ smartlist_free(config_files);
+ return rv;
+}
diff --git a/src/lib/fs/conffile.h b/src/lib/fs/conffile.h
new file mode 100644
index 0000000000..7af9119dbb
--- /dev/null
+++ b/src/lib/fs/conffile.h
@@ -0,0 +1,23 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_CONFFILE_H
+#define TOR_CONFFILE_H
+
+/**
+ * \file conffile.h
+ *
+ * \brief Header for conffile.c
+ **/
+
+struct smartlist_t;
+struct config_line_t;
+
+int config_get_lines_include(const char *string, struct config_line_t **result,
+ int extended, int *has_include,
+ struct smartlist_t *opened_lst);
+
+#endif /* !defined(TOR_CONFLINE_H) */
diff --git a/src/lib/fs/dir.c b/src/lib/fs/dir.c
new file mode 100644
index 0000000000..3c31e00d99
--- /dev/null
+++ b/src/lib/fs/dir.c
@@ -0,0 +1,367 @@
+/* Copyright (c) 2003, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file dir.c
+ *
+ * \brief Read directories, and create directories with restrictive
+ * permissions.
+ **/
+
+#include "lib/fs/dir.h"
+#include "lib/fs/path.h"
+#include "lib/fs/userdb.h"
+
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/log/win32err.h"
+#include "lib/container/smartlist.h"
+#include "lib/sandbox/sandbox.h"
+#include "lib/malloc/malloc.h"
+#include "lib/string/printf.h"
+#include "lib/string/compat_string.h"
+
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#ifdef HAVE_FCNTL_H
+#include <fcntl.h>
+#endif
+
+#ifdef _WIN32
+#include <io.h>
+#include <direct.h>
+#include <windows.h>
+#else /* !(defined(_WIN32)) */
+#include <dirent.h>
+#include <pwd.h>
+#include <grp.h>
+#endif /* defined(_WIN32) */
+
+#include <errno.h>
+#include <string.h>
+
+/** Check whether <b>dirname</b> exists and is private. If yes return 0.
+ * If <b>dirname</b> does not exist:
+ * - if <b>check</b>&CPD_CREATE, try to create it and return 0 on success.
+ * - if <b>check</b>&CPD_CHECK, and we think we can create it, return 0.
+ * - if <b>check</b>&CPD_CHECK is false, and the directory exists, return 0.
+ * - otherwise, return -1.
+ * If CPD_GROUP_OK is set, then it's okay if the directory
+ * is group-readable, but in all cases we create the directory mode 0700.
+ * If CPD_GROUP_READ is set, existing directory behaves as CPD_GROUP_OK and
+ * if the directory is created it will use mode 0750 with group read
+ * permission. Group read privileges also assume execute permission
+ * as norm for directories. If CPD_CHECK_MODE_ONLY is set, then we don't
+ * alter the directory permissions if they are too permissive:
+ * we just return -1.
+ * When effective_user is not NULL, check permissions against the given user
+ * and its primary group.
+ */
+MOCK_IMPL(int,
+check_private_dir,(const char *dirname, cpd_check_t check,
+ const char *effective_user))
+{
+ int r;
+ struct stat st;
+
+ tor_assert(dirname);
+
+#ifndef _WIN32
+ int fd;
+ const struct passwd *pw = NULL;
+ uid_t running_uid;
+ gid_t running_gid;
+
+ /*
+ * Goal is to harden the implementation by removing any
+ * potential for race between stat() and chmod().
+ * chmod() accepts filename as argument. If an attacker can move
+ * the file between stat() and chmod(), a potential race exists.
+ *
+ * Several suggestions taken from:
+ * https://developer.apple.com/library/mac/documentation/
+ * Security/Conceptual/SecureCodingGuide/Articles/RaceConditions.html
+ */
+
+ /* Open directory.
+ * O_NOFOLLOW to ensure that it does not follow symbolic links */
+ fd = open(sandbox_intern_string(dirname), O_NOFOLLOW);
+
+ /* Was there an error? Maybe the directory does not exist? */
+ if (fd == -1) {
+
+ if (errno != ENOENT) {
+ /* Other directory error */
+ log_warn(LD_FS, "Directory %s cannot be read: %s", dirname,
+ strerror(errno));
+ return -1;
+ }
+
+ /* Received ENOENT: Directory does not exist */
+
+ /* Should we create the directory? */
+ if (check & CPD_CREATE) {
+ log_info(LD_GENERAL, "Creating directory %s", dirname);
+ if (check & CPD_GROUP_READ) {
+ r = mkdir(dirname, 0750);
+ } else {
+ r = mkdir(dirname, 0700);
+ }
+
+ /* check for mkdir() error */
+ if (r) {
+ log_warn(LD_FS, "Error creating directory %s: %s", dirname,
+ strerror(errno));
+ return -1;
+ }
+
+ /* we just created the directory. try to open it again.
+ * permissions on the directory will be checked again below.*/
+ fd = open(sandbox_intern_string(dirname), O_NOFOLLOW);
+
+ if (fd == -1) {
+ log_warn(LD_FS, "Could not reopen recently created directory %s: %s",
+ dirname,
+ strerror(errno));
+ return -1;
+ } else {
+ close(fd);
+ }
+
+ } else if (!(check & CPD_CHECK)) {
+ log_warn(LD_FS, "Directory %s does not exist.", dirname);
+ return -1;
+ }
+
+ /* XXXX In the case where check==CPD_CHECK, we should look at the
+ * parent directory a little harder. */
+ return 0;
+ }
+
+ tor_assert(fd >= 0);
+
+ //f = tor_strdup(dirname);
+ //clean_name_for_stat(f);
+ log_debug(LD_FS, "stat()ing %s", dirname);
+ //r = stat(sandbox_intern_string(f), &st);
+ r = fstat(fd, &st);
+ if (r == -1) {
+ log_warn(LD_FS, "fstat() on directory %s failed.", dirname);
+ close(fd);
+ return -1;
+ }
+ //tor_free(f);
+
+ /* check that dirname is a directory */
+ if (!(st.st_mode & S_IFDIR)) {
+ log_warn(LD_FS, "%s is not a directory", dirname);
+ close(fd);
+ return -1;
+ }
+
+ if (effective_user) {
+ /* Look up the user and group information.
+ * If we have a problem, bail out. */
+ pw = tor_getpwnam(effective_user);
+ if (pw == NULL) {
+ log_warn(LD_CONFIG, "Error setting configured user: %s not found",
+ effective_user);
+ close(fd);
+ return -1;
+ }
+ running_uid = pw->pw_uid;
+ running_gid = pw->pw_gid;
+ } else {
+ running_uid = getuid();
+ running_gid = getgid();
+ }
+ if (st.st_uid != running_uid) {
+ char *process_ownername = NULL, *file_ownername = NULL;
+
+ {
+ const struct passwd *pw_running = tor_getpwuid(running_uid);
+ process_ownername = pw_running ? tor_strdup(pw_running->pw_name) :
+ tor_strdup("<unknown>");
+ }
+
+ {
+ const struct passwd *pw_stat = tor_getpwuid(st.st_uid);
+ file_ownername = pw_stat ? tor_strdup(pw_stat->pw_name) :
+ tor_strdup("<unknown>");
+ }
+
+ log_warn(LD_FS, "%s is not owned by this user (%s, %d) but by "
+ "%s (%d). Perhaps you are running Tor as the wrong user?",
+ dirname, process_ownername, (int)running_uid,
+ file_ownername, (int)st.st_uid);
+
+ tor_free(process_ownername);
+ tor_free(file_ownername);
+ close(fd);
+ return -1;
+ }
+ if ( (check & (CPD_GROUP_OK|CPD_GROUP_READ))
+ && (st.st_gid != running_gid) && (st.st_gid != 0)) {
+ struct group *gr;
+ char *process_groupname = NULL;
+ gr = getgrgid(running_gid);
+ process_groupname = gr ? tor_strdup(gr->gr_name) : tor_strdup("<unknown>");
+ gr = getgrgid(st.st_gid);
+
+ log_warn(LD_FS, "%s is not owned by this group (%s, %d) but by group "
+ "%s (%d). Are you running Tor as the wrong user?",
+ dirname, process_groupname, (int)running_gid,
+ gr ? gr->gr_name : "<unknown>", (int)st.st_gid);
+
+ tor_free(process_groupname);
+ close(fd);
+ return -1;
+ }
+ unsigned unwanted_bits = 0;
+ if (check & (CPD_GROUP_OK|CPD_GROUP_READ)) {
+ unwanted_bits = 0027;
+ } else {
+ unwanted_bits = 0077;
+ }
+ unsigned check_bits_filter = ~0;
+ if (check & CPD_RELAX_DIRMODE_CHECK) {
+ check_bits_filter = 0022;
+ }
+ if ((st.st_mode & unwanted_bits & check_bits_filter) != 0) {
+ unsigned new_mode;
+ if (check & CPD_CHECK_MODE_ONLY) {
+ log_warn(LD_FS, "Permissions on directory %s are too permissive.",
+ dirname);
+ close(fd);
+ return -1;
+ }
+ log_warn(LD_FS, "Fixing permissions on directory %s", dirname);
+ new_mode = st.st_mode;
+ new_mode |= 0700; /* Owner should have rwx */
+ if (check & CPD_GROUP_READ) {
+ new_mode |= 0050; /* Group should have rx */
+ }
+ new_mode &= ~unwanted_bits; /* Clear the bits that we didn't want set...*/
+ if (fchmod(fd, new_mode)) {
+ log_warn(LD_FS, "Could not chmod directory %s: %s", dirname,
+ strerror(errno));
+ close(fd);
+ return -1;
+ } else {
+ close(fd);
+ return 0;
+ }
+ }
+ close(fd);
+#else /* !(!defined(_WIN32)) */
+ /* Win32 case: we can't open() a directory. */
+ (void)effective_user;
+
+ char *f = tor_strdup(dirname);
+ clean_fname_for_stat(f);
+ log_debug(LD_FS, "stat()ing %s", f);
+ r = stat(sandbox_intern_string(f), &st);
+ tor_free(f);
+ if (r) {
+ if (errno != ENOENT) {
+ log_warn(LD_FS, "Directory %s cannot be read: %s", dirname,
+ strerror(errno));
+ return -1;
+ }
+ if (check & CPD_CREATE) {
+ log_info(LD_GENERAL, "Creating directory %s", dirname);
+ r = mkdir(dirname);
+ if (r) {
+ log_warn(LD_FS, "Error creating directory %s: %s", dirname,
+ strerror(errno));
+ return -1;
+ }
+ } else if (!(check & CPD_CHECK)) {
+ log_warn(LD_FS, "Directory %s does not exist.", dirname);
+ return -1;
+ }
+ return 0;
+ }
+ if (!(st.st_mode & S_IFDIR)) {
+ log_warn(LD_FS, "%s is not a directory", dirname);
+ return -1;
+ }
+
+#endif /* !defined(_WIN32) */
+ return 0;
+}
+
+/** Return a new list containing the filenames in the directory <b>dirname</b>.
+ * Return NULL on error or if <b>dirname</b> is not a directory.
+ */
+MOCK_IMPL(smartlist_t *,
+tor_listdir, (const char *dirname))
+{
+ smartlist_t *result;
+#ifdef _WIN32
+ char *pattern=NULL;
+ TCHAR tpattern[MAX_PATH] = {0};
+ char name[MAX_PATH*2+1] = {0};
+ HANDLE handle;
+ WIN32_FIND_DATA findData;
+ tor_asprintf(&pattern, "%s\\*", dirname);
+#ifdef UNICODE
+ mbstowcs(tpattern,pattern,MAX_PATH);
+#else
+ strlcpy(tpattern, pattern, MAX_PATH);
+#endif
+ if (INVALID_HANDLE_VALUE == (handle = FindFirstFile(tpattern, &findData))) {
+ tor_free(pattern);
+ return NULL;
+ }
+ result = smartlist_new();
+ while (1) {
+#ifdef UNICODE
+ wcstombs(name,findData.cFileName,MAX_PATH);
+ name[sizeof(name)-1] = '\0';
+#else
+ strlcpy(name,findData.cFileName,sizeof(name));
+#endif /* defined(UNICODE) */
+ if (strcmp(name, ".") &&
+ strcmp(name, "..")) {
+ smartlist_add_strdup(result, name);
+ }
+ if (!FindNextFile(handle, &findData)) {
+ DWORD err;
+ if ((err = GetLastError()) != ERROR_NO_MORE_FILES) {
+ char *errstr = format_win32_error(err);
+ log_warn(LD_FS, "Error reading directory '%s': %s", dirname, errstr);
+ tor_free(errstr);
+ }
+ break;
+ }
+ }
+ FindClose(handle);
+ tor_free(pattern);
+#else /* !(defined(_WIN32)) */
+ const char *prot_dname = sandbox_intern_string(dirname);
+ DIR *d;
+ struct dirent *de;
+ if (!(d = opendir(prot_dname)))
+ return NULL;
+
+ result = smartlist_new();
+ while ((de = readdir(d))) {
+ if (!strcmp(de->d_name, ".") ||
+ !strcmp(de->d_name, ".."))
+ continue;
+ smartlist_add_strdup(result, de->d_name);
+ }
+ closedir(d);
+#endif /* defined(_WIN32) */
+ return result;
+}
diff --git a/src/lib/fs/dir.h b/src/lib/fs/dir.h
new file mode 100644
index 0000000000..826bc2dfc5
--- /dev/null
+++ b/src/lib/fs/dir.h
@@ -0,0 +1,33 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_DIR_H
+#define TOR_DIR_H
+
+/**
+ * \file dir.h
+ *
+ * \brief Header for dir.c
+ **/
+
+#include "lib/cc/compat_compiler.h"
+#include "lib/testsupport/testsupport.h"
+
+/** Possible behaviors for check_private_dir() on encountering a nonexistent
+ * directory; see that function's documentation for details. */
+typedef unsigned int cpd_check_t;
+#define CPD_NONE 0
+#define CPD_CREATE (1u << 0)
+#define CPD_CHECK (1u << 1)
+#define CPD_GROUP_OK (1u << 2)
+#define CPD_GROUP_READ (1u << 3)
+#define CPD_CHECK_MODE_ONLY (1u << 4)
+#define CPD_RELAX_DIRMODE_CHECK (1u << 5)
+MOCK_DECL(int, check_private_dir, (const char *dirname, cpd_check_t check,
+ const char *effective_user));
+
+MOCK_DECL(struct smartlist_t *, tor_listdir, (const char *dirname));
+
+#endif
diff --git a/src/lib/fs/files.c b/src/lib/fs/files.c
new file mode 100644
index 0000000000..b98a51a287
--- /dev/null
+++ b/src/lib/fs/files.c
@@ -0,0 +1,721 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file files.h
+ *
+ * \brief Wrappers for reading and writing data to files on disk.
+ **/
+
+#ifdef _WIN32
+#include <windows.h>
+#endif
+
+#include "lib/fs/files.h"
+#include "lib/fs/path.h"
+#include "lib/container/smartlist.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/log/escape.h"
+#include "lib/err/torerr.h"
+#include "lib/malloc/malloc.h"
+#include "lib/sandbox/sandbox.h"
+#include "lib/string/printf.h"
+#include "lib/string/util_string.h"
+#include "lib/fdio/fdio.h"
+
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+#ifdef HAVE_UTIME_H
+#include <utime.h>
+#endif
+#ifdef HAVE_SYS_TIME_H
+#include <sys/time.h>
+#endif
+#ifdef HAVE_FCNTL_H
+#include <fcntl.h>
+#endif
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#include <errno.h>
+#include <stdio.h>
+#include <string.h>
+
+/** As open(path, flags, mode), but return an fd with the close-on-exec mode
+ * set. */
+int
+tor_open_cloexec(const char *path, int flags, unsigned mode)
+{
+ int fd;
+ const char *p = sandbox_intern_string(path);
+#ifdef O_CLOEXEC
+ fd = open(p, flags|O_CLOEXEC, mode);
+ if (fd >= 0)
+ return fd;
+ /* If we got an error, see if it is EINVAL. EINVAL might indicate that,
+ * even though we were built on a system with O_CLOEXEC support, we
+ * are running on one without. */
+ if (errno != EINVAL)
+ return -1;
+#endif /* defined(O_CLOEXEC) */
+
+ log_debug(LD_FS, "Opening %s with flags %x", p, flags);
+ fd = open(p, flags, mode);
+#ifdef FD_CLOEXEC
+ if (fd >= 0) {
+ if (fcntl(fd, F_SETFD, FD_CLOEXEC) == -1) {
+ log_warn(LD_FS,"Couldn't set FD_CLOEXEC: %s", strerror(errno));
+ close(fd);
+ return -1;
+ }
+ }
+#endif /* defined(FD_CLOEXEC) */
+ return fd;
+}
+
+/** As fopen(path,mode), but ensures that the O_CLOEXEC bit is set on the
+ * underlying file handle. */
+FILE *
+tor_fopen_cloexec(const char *path, const char *mode)
+{
+ FILE *result = fopen(path, mode);
+#ifdef FD_CLOEXEC
+ if (result != NULL) {
+ if (fcntl(fileno(result), F_SETFD, FD_CLOEXEC) == -1) {
+ log_warn(LD_FS,"Couldn't set FD_CLOEXEC: %s", strerror(errno));
+ fclose(result);
+ return NULL;
+ }
+ }
+#endif /* defined(FD_CLOEXEC) */
+ return result;
+}
+
+/** As rename(), but work correctly with the sandbox. */
+int
+tor_rename(const char *path_old, const char *path_new)
+{
+ log_debug(LD_FS, "Renaming %s to %s", path_old, path_new);
+ return rename(sandbox_intern_string(path_old),
+ sandbox_intern_string(path_new));
+}
+
+/**
+ * Rename the file <b>from</b> to the file <b>to</b>. On Unix, this is
+ * the same as rename(2). On windows, this removes <b>to</b> first if
+ * it already exists.
+ * Returns 0 on success. Returns -1 and sets errno on failure.
+ */
+int
+replace_file(const char *from, const char *to)
+{
+#ifndef _WIN32
+ return tor_rename(from, to);
+#else
+ switch (file_status(to))
+ {
+ case FN_NOENT:
+ break;
+ case FN_FILE:
+ case FN_EMPTY:
+ if (unlink(to)) return -1;
+ break;
+ case FN_ERROR:
+ return -1;
+ case FN_DIR:
+ errno = EISDIR;
+ return -1;
+ }
+ return tor_rename(from,to);
+#endif /* !defined(_WIN32) */
+}
+
+/** Change <b>fname</b>'s modification time to now. */
+int
+touch_file(const char *fname)
+{
+ if (utime(fname, NULL)!=0)
+ return -1;
+ return 0;
+}
+
+/** Wrapper for unlink() to make it mockable for the test suite; returns 0
+ * if unlinking the file succeeded, -1 and sets errno if unlinking fails.
+ */
+
+MOCK_IMPL(int,
+tor_unlink,(const char *pathname))
+{
+ return unlink(pathname);
+}
+
+/** Write <b>count</b> bytes from <b>buf</b> to <b>fd</b>. Return the number
+ * of bytes written, or -1 on error. Only use if fd is a blocking fd. */
+ssize_t
+write_all_to_fd(int fd, const char *buf, size_t count)
+{
+ size_t written = 0;
+ ssize_t result;
+ raw_assert(count < SSIZE_MAX);
+
+ while (written != count) {
+ result = write(fd, buf+written, count-written);
+ if (result<0)
+ return -1;
+ written += result;
+ }
+ return (ssize_t)count;
+}
+
+/** Read from <b>fd</b> to <b>buf</b>, until we get <b>count</b> bytes or
+ * reach the end of the file. Return the number of bytes read, or -1 on
+ * error. Only use if fd is a blocking fd. */
+ssize_t
+read_all_from_fd(int fd, char *buf, size_t count)
+{
+ size_t numread = 0;
+ ssize_t result;
+
+ if (count > SIZE_T_CEILING || count > SSIZE_MAX) {
+ errno = EINVAL;
+ return -1;
+ }
+
+ while (numread < count) {
+ result = read(fd, buf+numread, count-numread);
+ if (result<0)
+ return -1;
+ else if (result == 0)
+ break;
+ numread += result;
+ }
+ return (ssize_t)numread;
+}
+
+/** Return:
+ * FN_ERROR if filename can't be read, is NULL, or is zero-length,
+ * FN_NOENT if it doesn't exist,
+ * FN_FILE if it is a non-empty regular file, or a FIFO on unix-like systems,
+ * FN_EMPTY for zero-byte regular files,
+ * FN_DIR if it's a directory, and
+ * FN_ERROR for any other file type.
+ * On FN_ERROR and FN_NOENT, sets errno. (errno is not set when FN_ERROR
+ * is returned due to an unhandled file type.) */
+file_status_t
+file_status(const char *fname)
+{
+ struct stat st;
+ char *f;
+ int r;
+ if (!fname || strlen(fname) == 0) {
+ return FN_ERROR;
+ }
+ f = tor_strdup(fname);
+ clean_fname_for_stat(f);
+ log_debug(LD_FS, "stat()ing %s", f);
+ r = stat(sandbox_intern_string(f), &st);
+ tor_free(f);
+ if (r) {
+ if (errno == ENOENT) {
+ return FN_NOENT;
+ }
+ return FN_ERROR;
+ }
+ if (st.st_mode & S_IFDIR) {
+ return FN_DIR;
+ } else if (st.st_mode & S_IFREG) {
+ if (st.st_size > 0) {
+ return FN_FILE;
+ } else if (st.st_size == 0) {
+ return FN_EMPTY;
+ } else {
+ return FN_ERROR;
+ }
+#ifndef _WIN32
+ } else if (st.st_mode & S_IFIFO) {
+ return FN_FILE;
+#endif
+ } else {
+ return FN_ERROR;
+ }
+}
+
+/** Create a file named <b>fname</b> with the contents <b>str</b>. Overwrite
+ * the previous <b>fname</b> if possible. Return 0 on success, -1 on failure.
+ *
+ * This function replaces the old file atomically, if possible. This
+ * function, and all other functions in util.c that create files, create them
+ * with mode 0600.
+ */
+MOCK_IMPL(int,
+write_str_to_file,(const char *fname, const char *str, int bin))
+{
+#ifdef _WIN32
+ if (!bin && strchr(str, '\r')) {
+ log_warn(LD_BUG,
+ "We're writing a text string that already contains a CR to %s",
+ escaped(fname));
+ }
+#endif /* defined(_WIN32) */
+ return write_bytes_to_file(fname, str, strlen(str), bin);
+}
+
+/** Represents a file that we're writing to, with support for atomic commit:
+ * we can write into a temporary file, and either remove the file on
+ * failure, or replace the original file on success. */
+struct open_file_t {
+ char *tempname; /**< Name of the temporary file. */
+ char *filename; /**< Name of the original file. */
+ unsigned rename_on_close:1; /**< Are we using the temporary file or not? */
+ unsigned binary:1; /**< Did we open in binary mode? */
+ int fd; /**< fd for the open file. */
+ FILE *stdio_file; /**< stdio wrapper for <b>fd</b>. */
+};
+
+/** Try to start writing to the file in <b>fname</b>, passing the flags
+ * <b>open_flags</b> to the open() syscall, creating the file (if needed) with
+ * access value <b>mode</b>. If the O_APPEND flag is set, we append to the
+ * original file. Otherwise, we open a new temporary file in the same
+ * directory, and either replace the original or remove the temporary file
+ * when we're done.
+ *
+ * Return the fd for the newly opened file, and store working data in
+ * *<b>data_out</b>. The caller should not close the fd manually:
+ * instead, call finish_writing_to_file() or abort_writing_to_file().
+ * Returns -1 on failure.
+ *
+ * NOTE: When not appending, the flags O_CREAT and O_TRUNC are treated
+ * as true and the flag O_EXCL is treated as false.
+ *
+ * NOTE: Ordinarily, O_APPEND means "seek to the end of the file before each
+ * write()". We don't do that.
+ */
+int
+start_writing_to_file(const char *fname, int open_flags, int mode,
+ open_file_t **data_out)
+{
+ open_file_t *new_file = tor_malloc_zero(sizeof(open_file_t));
+ const char *open_name;
+ int append = 0;
+
+ tor_assert(fname);
+ tor_assert(data_out);
+#if (O_BINARY != 0 && O_TEXT != 0)
+ tor_assert((open_flags & (O_BINARY|O_TEXT)) != 0);
+#endif
+ new_file->fd = -1;
+ new_file->filename = tor_strdup(fname);
+ if (open_flags & O_APPEND) {
+ open_name = fname;
+ new_file->rename_on_close = 0;
+ append = 1;
+ open_flags &= ~O_APPEND;
+ } else {
+ tor_asprintf(&new_file->tempname, "%s.tmp", fname);
+ open_name = new_file->tempname;
+ /* We always replace an existing temporary file if there is one. */
+ open_flags |= O_CREAT|O_TRUNC;
+ open_flags &= ~O_EXCL;
+ new_file->rename_on_close = 1;
+ }
+#if O_BINARY != 0
+ if (open_flags & O_BINARY)
+ new_file->binary = 1;
+#endif
+
+ new_file->fd = tor_open_cloexec(open_name, open_flags, mode);
+ if (new_file->fd < 0) {
+ log_warn(LD_FS, "Couldn't open \"%s\" (%s) for writing: %s",
+ open_name, fname, strerror(errno));
+ goto err;
+ }
+ if (append) {
+ if (tor_fd_seekend(new_file->fd) < 0) {
+ log_warn(LD_FS, "Couldn't seek to end of file \"%s\": %s", open_name,
+ strerror(errno));
+ goto err;
+ }
+ }
+
+ *data_out = new_file;
+
+ return new_file->fd;
+
+ err:
+ if (new_file->fd >= 0)
+ close(new_file->fd);
+ *data_out = NULL;
+ tor_free(new_file->filename);
+ tor_free(new_file->tempname);
+ tor_free(new_file);
+ return -1;
+}
+
+/** Given <b>file_data</b> from start_writing_to_file(), return a stdio FILE*
+ * that can be used to write to the same file. The caller should not mix
+ * stdio calls with non-stdio calls. */
+FILE *
+fdopen_file(open_file_t *file_data)
+{
+ tor_assert(file_data);
+ if (file_data->stdio_file)
+ return file_data->stdio_file;
+ tor_assert(file_data->fd >= 0);
+ if (!(file_data->stdio_file = fdopen(file_data->fd,
+ file_data->binary?"ab":"a"))) {
+ log_warn(LD_FS, "Couldn't fdopen \"%s\" [%d]: %s", file_data->filename,
+ file_data->fd, strerror(errno));
+ }
+ return file_data->stdio_file;
+}
+
+/** Combines start_writing_to_file with fdopen_file(): arguments are as
+ * for start_writing_to_file, but */
+FILE *
+start_writing_to_stdio_file(const char *fname, int open_flags, int mode,
+ open_file_t **data_out)
+{
+ FILE *res;
+ if (start_writing_to_file(fname, open_flags, mode, data_out)<0)
+ return NULL;
+ if (!(res = fdopen_file(*data_out))) {
+ abort_writing_to_file(*data_out);
+ *data_out = NULL;
+ }
+ return res;
+}
+
+/** Helper function: close and free the underlying file and memory in
+ * <b>file_data</b>. If we were writing into a temporary file, then delete
+ * that file (if abort_write is true) or replaces the target file with
+ * the temporary file (if abort_write is false). */
+static int
+finish_writing_to_file_impl(open_file_t *file_data, int abort_write)
+{
+ int r = 0;
+
+ tor_assert(file_data && file_data->filename);
+ if (file_data->stdio_file) {
+ if (fclose(file_data->stdio_file)) {
+ log_warn(LD_FS, "Error closing \"%s\": %s", file_data->filename,
+ strerror(errno));
+ abort_write = r = -1;
+ }
+ } else if (file_data->fd >= 0 && close(file_data->fd) < 0) {
+ log_warn(LD_FS, "Error flushing \"%s\": %s", file_data->filename,
+ strerror(errno));
+ abort_write = r = -1;
+ }
+
+ if (file_data->rename_on_close) {
+ tor_assert(file_data->tempname && file_data->filename);
+ if (!abort_write) {
+ tor_assert(strcmp(file_data->filename, file_data->tempname));
+ if (replace_file(file_data->tempname, file_data->filename)) {
+ log_warn(LD_FS, "Error replacing \"%s\": %s", file_data->filename,
+ strerror(errno));
+ abort_write = r = -1;
+ }
+ }
+ if (abort_write) {
+ int res = unlink(file_data->tempname);
+ if (res != 0) {
+ /* We couldn't unlink and we'll leave a mess behind */
+ log_warn(LD_FS, "Failed to unlink %s: %s",
+ file_data->tempname, strerror(errno));
+ r = -1;
+ }
+ }
+ }
+
+ tor_free(file_data->filename);
+ tor_free(file_data->tempname);
+ tor_free(file_data);
+
+ return r;
+}
+
+/** Finish writing to <b>file_data</b>: close the file handle, free memory as
+ * needed, and if using a temporary file, replace the original file with
+ * the temporary file. */
+int
+finish_writing_to_file(open_file_t *file_data)
+{
+ return finish_writing_to_file_impl(file_data, 0);
+}
+
+/** Finish writing to <b>file_data</b>: close the file handle, free memory as
+ * needed, and if using a temporary file, delete it. */
+int
+abort_writing_to_file(open_file_t *file_data)
+{
+ return finish_writing_to_file_impl(file_data, 1);
+}
+
+/** Helper: given a set of flags as passed to open(2), open the file
+ * <b>fname</b> and write all the sized_chunk_t structs in <b>chunks</b> to
+ * the file. Do so as atomically as possible e.g. by opening temp files and
+ * renaming. */
+static int
+write_chunks_to_file_impl(const char *fname, const smartlist_t *chunks,
+ int open_flags)
+{
+ open_file_t *file = NULL;
+ int fd;
+ ssize_t result;
+ fd = start_writing_to_file(fname, open_flags, 0600, &file);
+ if (fd<0)
+ return -1;
+ SMARTLIST_FOREACH(chunks, sized_chunk_t *, chunk,
+ {
+ result = write_all_to_fd(fd, chunk->bytes, chunk->len);
+ if (result < 0) {
+ log_warn(LD_FS, "Error writing to \"%s\": %s", fname,
+ strerror(errno));
+ goto err;
+ }
+ tor_assert((size_t)result == chunk->len);
+ });
+
+ return finish_writing_to_file(file);
+ err:
+ abort_writing_to_file(file);
+ return -1;
+}
+
+/** Given a smartlist of sized_chunk_t, write them to a file
+ * <b>fname</b>, overwriting or creating the file as necessary.
+ * If <b>no_tempfile</b> is 0 then the file will be written
+ * atomically. */
+int
+write_chunks_to_file(const char *fname, const smartlist_t *chunks, int bin,
+ int no_tempfile)
+{
+ int flags = OPEN_FLAGS_REPLACE|(bin?O_BINARY:O_TEXT);
+
+ if (no_tempfile) {
+ /* O_APPEND stops write_chunks_to_file from using tempfiles */
+ flags |= O_APPEND;
+ }
+ return write_chunks_to_file_impl(fname, chunks, flags);
+}
+
+/** Write <b>len</b> bytes, starting at <b>str</b>, to <b>fname</b>
+ using the open() flags passed in <b>flags</b>. */
+static int
+write_bytes_to_file_impl(const char *fname, const char *str, size_t len,
+ int flags)
+{
+ int r;
+ sized_chunk_t c = { str, len };
+ smartlist_t *chunks = smartlist_new();
+ smartlist_add(chunks, &c);
+ r = write_chunks_to_file_impl(fname, chunks, flags);
+ smartlist_free(chunks);
+ return r;
+}
+
+/** As write_str_to_file, but does not assume a NUL-terminated
+ * string. Instead, we write <b>len</b> bytes, starting at <b>str</b>. */
+MOCK_IMPL(int,
+write_bytes_to_file,(const char *fname, const char *str, size_t len,
+ int bin))
+{
+ return write_bytes_to_file_impl(fname, str, len,
+ OPEN_FLAGS_REPLACE|(bin?O_BINARY:O_TEXT));
+}
+
+/** As write_bytes_to_file, but if the file already exists, append the bytes
+ * to the end of the file instead of overwriting it. */
+int
+append_bytes_to_file(const char *fname, const char *str, size_t len,
+ int bin)
+{
+ return write_bytes_to_file_impl(fname, str, len,
+ OPEN_FLAGS_APPEND|(bin?O_BINARY:O_TEXT));
+}
+
+/** Like write_str_to_file(), but also return -1 if there was a file
+ already residing in <b>fname</b>. */
+int
+write_bytes_to_new_file(const char *fname, const char *str, size_t len,
+ int bin)
+{
+ return write_bytes_to_file_impl(fname, str, len,
+ OPEN_FLAGS_DONT_REPLACE|
+ (bin?O_BINARY:O_TEXT));
+}
+
+/**
+ * Read the contents of the open file <b>fd</b> presuming it is a FIFO
+ * (or similar) file descriptor for which the size of the file isn't
+ * known ahead of time. Return NULL on failure, and a NUL-terminated
+ * string on success. On success, set <b>sz_out</b> to the number of
+ * bytes read.
+ */
+char *
+read_file_to_str_until_eof(int fd, size_t max_bytes_to_read, size_t *sz_out)
+{
+ ssize_t r;
+ size_t pos = 0;
+ char *string = NULL;
+ size_t string_max = 0;
+
+ if (max_bytes_to_read+1 >= SIZE_T_CEILING) {
+ errno = EINVAL;
+ return NULL;
+ }
+
+ do {
+ /* XXXX This "add 1K" approach is a little goofy; if we care about
+ * performance here, we should be doubling. But in practice we shouldn't
+ * be using this function on big files anyway. */
+ string_max = pos + 1024;
+ if (string_max > max_bytes_to_read)
+ string_max = max_bytes_to_read + 1;
+ string = tor_realloc(string, string_max);
+ r = read(fd, string + pos, string_max - pos - 1);
+ if (r < 0) {
+ int save_errno = errno;
+ tor_free(string);
+ errno = save_errno;
+ return NULL;
+ }
+
+ pos += r;
+ } while (r > 0 && pos < max_bytes_to_read);
+
+ tor_assert(pos < string_max);
+ *sz_out = pos;
+ string[pos] = '\0';
+ return string;
+}
+
+/** Read the contents of <b>filename</b> into a newly allocated
+ * string; return the string on success or NULL on failure.
+ *
+ * If <b>stat_out</b> is provided, store the result of stat()ing the
+ * file into <b>stat_out</b>.
+ *
+ * If <b>flags</b> &amp; RFTS_BIN, open the file in binary mode.
+ * If <b>flags</b> &amp; RFTS_IGNORE_MISSING, don't warn if the file
+ * doesn't exist.
+ */
+/*
+ * This function <em>may</em> return an erroneous result if the file
+ * is modified while it is running, but must not crash or overflow.
+ * Right now, the error case occurs when the file length grows between
+ * the call to stat and the call to read_all: the resulting string will
+ * be truncated.
+ */
+MOCK_IMPL(char *,
+read_file_to_str, (const char *filename, int flags, struct stat *stat_out))
+{
+ int fd; /* router file */
+ struct stat statbuf;
+ char *string;
+ ssize_t r;
+ int bin = flags & RFTS_BIN;
+
+ tor_assert(filename);
+
+ fd = tor_open_cloexec(filename,O_RDONLY|(bin?O_BINARY:O_TEXT),0);
+ if (fd<0) {
+ int severity = LOG_WARN;
+ int save_errno = errno;
+ if (errno == ENOENT && (flags & RFTS_IGNORE_MISSING))
+ severity = LOG_INFO;
+ log_fn(severity, LD_FS,"Could not open \"%s\": %s",filename,
+ strerror(errno));
+ errno = save_errno;
+ return NULL;
+ }
+
+ if (fstat(fd, &statbuf)<0) {
+ int save_errno = errno;
+ close(fd);
+ log_warn(LD_FS,"Could not fstat \"%s\".",filename);
+ errno = save_errno;
+ return NULL;
+ }
+
+#ifndef _WIN32
+/** When we detect that we're reading from a FIFO, don't read more than
+ * this many bytes. It's insane overkill for most uses. */
+#define FIFO_READ_MAX (1024*1024)
+ if (S_ISFIFO(statbuf.st_mode)) {
+ size_t sz = 0;
+ string = read_file_to_str_until_eof(fd, FIFO_READ_MAX, &sz);
+ int save_errno = errno;
+ if (string && stat_out) {
+ statbuf.st_size = sz;
+ memcpy(stat_out, &statbuf, sizeof(struct stat));
+ }
+ close(fd);
+ if (!string)
+ errno = save_errno;
+ return string;
+ }
+#endif /* !defined(_WIN32) */
+
+ if ((uint64_t)(statbuf.st_size)+1 >= SIZE_T_CEILING) {
+ close(fd);
+ errno = EINVAL;
+ return NULL;
+ }
+
+ string = tor_malloc((size_t)(statbuf.st_size+1));
+
+ r = read_all_from_fd(fd,string,(size_t)statbuf.st_size);
+ if (r<0) {
+ int save_errno = errno;
+ log_warn(LD_FS,"Error reading from file \"%s\": %s", filename,
+ strerror(errno));
+ tor_free(string);
+ close(fd);
+ errno = save_errno;
+ return NULL;
+ }
+ string[r] = '\0'; /* NUL-terminate the result. */
+
+#if defined(_WIN32) || defined(__CYGWIN__)
+ if (!bin && strchr(string, '\r')) {
+ log_debug(LD_FS, "We didn't convert CRLF to LF as well as we hoped "
+ "when reading %s. Coping.",
+ filename);
+ tor_strstrip(string, "\r");
+ r = strlen(string);
+ }
+ if (!bin) {
+ statbuf.st_size = (size_t) r;
+ } else
+#endif /* defined(_WIN32) || defined(__CYGWIN__) */
+ if (r != statbuf.st_size) {
+ /* Unless we're using text mode on win32, we'd better have an exact
+ * match for size. */
+ int save_errno = errno;
+ log_warn(LD_FS,"Could read only %d of %ld bytes of file \"%s\".",
+ (int)r, (long)statbuf.st_size,filename);
+ tor_free(string);
+ close(fd);
+ errno = save_errno;
+ return NULL;
+ }
+ close(fd);
+ if (stat_out) {
+ memcpy(stat_out, &statbuf, sizeof(struct stat));
+ }
+
+ return string;
+}
+
+#if !defined(HAVE_GETDELIM) || defined(TOR_UNIT_TESTS)
+#include "ext/getdelim.c"
+#endif
diff --git a/src/lib/fs/files.h b/src/lib/fs/files.h
new file mode 100644
index 0000000000..52c94c914f
--- /dev/null
+++ b/src/lib/fs/files.h
@@ -0,0 +1,145 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file files.h
+ *
+ * \brief Header for files.c
+ **/
+
+#ifndef TOR_FS_H
+#define TOR_FS_H
+
+#include "lib/cc/compat_compiler.h"
+#include "lib/cc/torint.h"
+#include "lib/testsupport/testsupport.h"
+
+#include <stddef.h>
+#include <stdio.h>
+
+#ifdef _WIN32
+/* We need these for struct stat to work */
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+#endif
+
+#ifndef O_BINARY
+#define O_BINARY 0
+#endif
+#ifndef O_TEXT
+#define O_TEXT 0
+#endif
+#ifndef O_NOFOLLOW
+#define O_NOFOLLOW 0
+#endif
+
+struct stat;
+
+int tor_open_cloexec(const char *path, int flags, unsigned mode);
+FILE *tor_fopen_cloexec(const char *path, const char *mode);
+int tor_rename(const char *path_old, const char *path_new);
+
+int replace_file(const char *from, const char *to);
+int touch_file(const char *fname);
+
+MOCK_DECL(int,tor_unlink,(const char *pathname));
+
+/** Return values from file_status(); see that function's documentation
+ * for details. */
+typedef enum { FN_ERROR, FN_NOENT, FN_FILE, FN_DIR, FN_EMPTY } file_status_t;
+
+file_status_t file_status(const char *filename);
+
+int64_t tor_get_avail_disk_space(const char *path);
+
+ssize_t write_all_to_fd(int fd, const char *buf, size_t count);
+ssize_t read_all_from_fd(int fd, char *buf, size_t count);
+
+#define OPEN_FLAGS_REPLACE (O_WRONLY|O_CREAT|O_TRUNC)
+#define OPEN_FLAGS_APPEND (O_WRONLY|O_CREAT|O_APPEND)
+#define OPEN_FLAGS_DONT_REPLACE (O_CREAT|O_EXCL|O_APPEND|O_WRONLY)
+typedef struct open_file_t open_file_t;
+int start_writing_to_file(const char *fname, int open_flags, int mode,
+ open_file_t **data_out);
+FILE *start_writing_to_stdio_file(const char *fname, int open_flags, int mode,
+ open_file_t **data_out);
+FILE *fdopen_file(open_file_t *file_data);
+int finish_writing_to_file(open_file_t *file_data);
+int abort_writing_to_file(open_file_t *file_data);
+MOCK_DECL(int, write_str_to_file,(const char *fname, const char *str,
+ int bin));
+MOCK_DECL(int, write_bytes_to_file,(const char *fname, const char *str,
+ size_t len,int bin));
+
+/** An ad-hoc type to hold a string of characters and a count; used by
+ * write_chunks_to_file. */
+typedef struct sized_chunk_t {
+ const char *bytes;
+ size_t len;
+} sized_chunk_t;
+struct smartlist_t;
+int write_chunks_to_file(const char *fname, const struct smartlist_t *chunks,
+ int bin, int no_tempfile);
+int append_bytes_to_file(const char *fname, const char *str, size_t len,
+ int bin);
+int write_bytes_to_new_file(const char *fname, const char *str, size_t len,
+ int bin);
+
+/** Flag for read_file_to_str: open the file in binary mode. */
+#define RFTS_BIN 1
+/** Flag for read_file_to_str: it's okay if the file doesn't exist. */
+#define RFTS_IGNORE_MISSING 2
+
+MOCK_DECL_ATTR(char *, read_file_to_str,(const char *filename, int flags,
+ struct stat *stat_out),
+ ATTR_MALLOC);
+char *read_file_to_str_until_eof(int fd, size_t max_bytes_to_read,
+ size_t *sz_out)
+ ATTR_MALLOC;
+
+#if !defined(HAVE_GETDELIM) || defined(TOR_UNIT_TESTS)
+/** Internal back-end function to implement getdelim(): only exists when
+ * Tor is built for unit tests, or when Tor is built on an operating system
+ * without its own getdelim(). */
+ssize_t compat_getdelim_(char **lineptr, size_t *n, int delim, FILE *stream);
+#endif
+
+#ifdef HAVE_GETDELIM
+/**
+ * Cross-platform wrapper for getdelim(): behaves as the POSIX-standard
+ * getdelim() function.
+ *
+ * See `getdelim(3)` for more information.
+ *
+ * Note that this function will use the libc memory allocator -- so any memory
+ * passed to this function must come from raw_malloc(), and must be freed by
+ * raw_free() -- don't use tor_malloc() and tor_free() with this.
+ */
+#define tor_getdelim(lineptr, n, delim, stream) \
+ getdelim((lineptr), (n), (delim), (stream))
+#else
+#define tor_getdelim(lineptr, n, delim, stream) \
+ compat_getdelim_((lineptr), (n), (delim), (stream))
+#endif
+
+#ifdef HAVE_GETLINE
+/**
+ * Cross-platform wrapper for getline(): behaves as the POSIX-standard
+ * getline() function.
+ *
+ * See tor_getdelim() for usage notes.
+ */
+#define tor_getline(lineptr, n, stream) \
+ getline((lineptr), (n), (stream))
+#else
+#define tor_getline(lineptr, n, stream) \
+ tor_getdelim((lineptr), (n), '\n', (stream))
+#endif
+
+#endif
diff --git a/src/lib/fs/freespace.c b/src/lib/fs/freespace.c
new file mode 100644
index 0000000000..ee0f93073d
--- /dev/null
+++ b/src/lib/fs/freespace.c
@@ -0,0 +1,63 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file freespace.c
+ *
+ * \brief Find the available disk space on the current volume.
+ **/
+
+#include "lib/fs/files.h"
+#include "lib/cc/torint.h"
+
+#ifdef HAVE_SYS_STATVFS_H
+#include <sys/statvfs.h>
+#endif
+#ifdef _WIN32
+#include <windows.h>
+#endif
+
+#include <errno.h>
+#include <string.h>
+
+/** Return the amount of free disk space we have permission to use, in
+ * bytes. Return -1 if the amount of free space can't be determined. */
+int64_t
+tor_get_avail_disk_space(const char *path)
+{
+#ifdef HAVE_STATVFS
+ struct statvfs st;
+ int r;
+ memset(&st, 0, sizeof(st));
+
+ r = statvfs(path, &st);
+ if (r < 0)
+ return -1;
+
+ int64_t result = st.f_bavail;
+ if (st.f_frsize) {
+ result *= st.f_frsize;
+ } else if (st.f_bsize) {
+ result *= st.f_bsize;
+ } else {
+ return -1;
+ }
+
+ return result;
+#elif defined(_WIN32)
+ ULARGE_INTEGER freeBytesAvail;
+ BOOL ok;
+
+ ok = GetDiskFreeSpaceEx(path, &freeBytesAvail, NULL, NULL);
+ if (!ok) {
+ return -1;
+ }
+ return (int64_t)freeBytesAvail.QuadPart;
+#else
+ (void)path;
+ errno = ENOSYS;
+ return -1;
+#endif /* defined(HAVE_STATVFS) || ... */
+}
diff --git a/src/lib/fs/include.am b/src/lib/fs/include.am
new file mode 100644
index 0000000000..f33e4d6430
--- /dev/null
+++ b/src/lib/fs/include.am
@@ -0,0 +1,37 @@
+
+noinst_LIBRARIES += src/lib/libtor-fs.a
+
+if UNITTESTS_ENABLED
+noinst_LIBRARIES += src/lib/libtor-fs-testing.a
+endif
+
+src_lib_libtor_fs_a_SOURCES = \
+ src/lib/fs/conffile.c \
+ src/lib/fs/dir.c \
+ src/lib/fs/files.c \
+ src/lib/fs/freespace.c \
+ src/lib/fs/lockfile.c \
+ src/lib/fs/mmap.c \
+ src/lib/fs/path.c \
+ src/lib/fs/storagedir.c \
+ src/lib/fs/userdb.c
+
+if WIN32
+src_lib_libtor_fs_a_SOURCES += src/lib/fs/winlib.c
+endif
+
+src_lib_libtor_fs_testing_a_SOURCES = \
+ $(src_lib_libtor_fs_a_SOURCES)
+src_lib_libtor_fs_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
+src_lib_libtor_fs_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
+
+noinst_HEADERS += \
+ src/lib/fs/conffile.h \
+ src/lib/fs/dir.h \
+ src/lib/fs/files.h \
+ src/lib/fs/lockfile.h \
+ src/lib/fs/mmap.h \
+ src/lib/fs/path.h \
+ src/lib/fs/storagedir.h \
+ src/lib/fs/userdb.h \
+ src/lib/fs/winlib.h
diff --git a/src/lib/fs/lockfile.c b/src/lib/fs/lockfile.c
new file mode 100644
index 0000000000..933ff1e02f
--- /dev/null
+++ b/src/lib/fs/lockfile.c
@@ -0,0 +1,145 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file lockfile.c
+ *
+ * \brief Implements lock files to prevent two Tor processes from using the
+ * same data directory at the same time.
+ **/
+
+#include "orconfig.h"
+#include "lib/fs/files.h"
+#include "lib/fs/lockfile.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/malloc/malloc.h"
+
+#ifdef HAVE_SYS_FILE_H
+#include <sys/file.h>
+#endif
+#ifdef HAVE_FCNTL_H
+#include <fcntl.h>
+#endif
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#ifdef _WIN32
+#include <windows.h>
+#include <sys/locking.h>
+#endif
+
+#include <errno.h>
+#include <string.h>
+
+/** Represents a lockfile on which we hold the lock. */
+struct tor_lockfile_t {
+ /** Name of the file */
+ char *filename;
+ /** File descriptor used to hold the file open */
+ int fd;
+};
+
+/** Try to get a lock on the lockfile <b>filename</b>, creating it as
+ * necessary. If someone else has the lock and <b>blocking</b> is true,
+ * wait until the lock is available. Otherwise return immediately whether
+ * we succeeded or not.
+ *
+ * Set *<b>locked_out</b> to true if somebody else had the lock, and to false
+ * otherwise.
+ *
+ * Return a <b>tor_lockfile_t</b> on success, NULL on failure.
+ *
+ * (Implementation note: because we need to fall back to fcntl on some
+ * platforms, these locks are per-process, not per-thread. If you want
+ * to do in-process locking, use tor_mutex_t like a normal person.
+ * On Windows, when <b>blocking</b> is true, the maximum time that
+ * is actually waited is 10 seconds, after which NULL is returned
+ * and <b>locked_out</b> is set to 1.)
+ */
+tor_lockfile_t *
+tor_lockfile_lock(const char *filename, int blocking, int *locked_out)
+{
+ tor_lockfile_t *result;
+ int fd;
+ *locked_out = 0;
+
+ log_info(LD_FS, "Locking \"%s\"", filename);
+ fd = tor_open_cloexec(filename, O_RDWR|O_CREAT|O_TRUNC, 0600);
+ if (fd < 0) {
+ log_warn(LD_FS,"Couldn't open \"%s\" for locking: %s", filename,
+ strerror(errno));
+ return NULL;
+ }
+
+#ifdef _WIN32
+ _lseek(fd, 0, SEEK_SET);
+ if (_locking(fd, blocking ? _LK_LOCK : _LK_NBLCK, 1) < 0) {
+ if (errno != EACCES && errno != EDEADLOCK)
+ log_warn(LD_FS,"Couldn't lock \"%s\": %s", filename, strerror(errno));
+ else
+ *locked_out = 1;
+ close(fd);
+ return NULL;
+ }
+#elif defined(HAVE_FLOCK)
+ if (flock(fd, LOCK_EX|(blocking ? 0 : LOCK_NB)) < 0) {
+ if (errno != EWOULDBLOCK)
+ log_warn(LD_FS,"Couldn't lock \"%s\": %s", filename, strerror(errno));
+ else
+ *locked_out = 1;
+ close(fd);
+ return NULL;
+ }
+#else
+ {
+ struct flock lock;
+ memset(&lock, 0, sizeof(lock));
+ lock.l_type = F_WRLCK;
+ lock.l_whence = SEEK_SET;
+ if (fcntl(fd, blocking ? F_SETLKW : F_SETLK, &lock) < 0) {
+ if (errno != EACCES && errno != EAGAIN)
+ log_warn(LD_FS, "Couldn't lock \"%s\": %s", filename, strerror(errno));
+ else
+ *locked_out = 1;
+ close(fd);
+ return NULL;
+ }
+ }
+#endif /* defined(_WIN32) || ... */
+
+ result = tor_malloc(sizeof(tor_lockfile_t));
+ result->filename = tor_strdup(filename);
+ result->fd = fd;
+ return result;
+}
+
+/** Release the lock held as <b>lockfile</b>. */
+void
+tor_lockfile_unlock(tor_lockfile_t *lockfile)
+{
+ tor_assert(lockfile);
+
+ log_info(LD_FS, "Unlocking \"%s\"", lockfile->filename);
+#ifdef _WIN32
+ _lseek(lockfile->fd, 0, SEEK_SET);
+ if (_locking(lockfile->fd, _LK_UNLCK, 1) < 0) {
+ log_warn(LD_FS,"Error unlocking \"%s\": %s", lockfile->filename,
+ strerror(errno));
+ }
+#elif defined(HAVE_FLOCK)
+ if (flock(lockfile->fd, LOCK_UN) < 0) {
+ log_warn(LD_FS, "Error unlocking \"%s\": %s", lockfile->filename,
+ strerror(errno));
+ }
+#else
+ /* Closing the lockfile is sufficient. */
+#endif /* defined(_WIN32) || ... */
+
+ close(lockfile->fd);
+ lockfile->fd = -1;
+ tor_free(lockfile->filename);
+ tor_free(lockfile);
+}
diff --git a/src/lib/fs/lockfile.h b/src/lib/fs/lockfile.h
new file mode 100644
index 0000000000..8aeee4cc7f
--- /dev/null
+++ b/src/lib/fs/lockfile.h
@@ -0,0 +1,20 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file lockfile.h
+ *
+ * \brief Header for lockfile.c
+ **/
+
+#ifndef TOR_LOCKFILE_H
+#define TOR_LOCKFILE_H
+
+typedef struct tor_lockfile_t tor_lockfile_t;
+tor_lockfile_t *tor_lockfile_lock(const char *filename, int blocking,
+ int *locked_out);
+void tor_lockfile_unlock(tor_lockfile_t *lockfile);
+
+#endif
diff --git a/src/lib/fs/mmap.c b/src/lib/fs/mmap.c
new file mode 100644
index 0000000000..daaee1f9b1
--- /dev/null
+++ b/src/lib/fs/mmap.c
@@ -0,0 +1,240 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file mmap.c
+ *
+ * \brief Cross-platform support for mapping files into our address space.
+ **/
+
+#include "lib/fs/mmap.h"
+#include "lib/fs/files.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/log/win32err.h"
+#include "lib/string/compat_string.h"
+#include "lib/malloc/malloc.h"
+
+#ifdef HAVE_MMAP
+#include <sys/mman.h>
+#endif
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#ifdef HAVE_FCNTL_H
+#include <fcntl.h>
+#endif
+
+#ifdef _WIN32
+#include <windows.h>
+#endif
+
+#include <errno.h>
+#include <string.h>
+
+#if defined(HAVE_MMAP) || defined(RUNNING_DOXYGEN)
+/** Try to create a memory mapping for <b>filename</b> and return it. On
+ * failure, return NULL. Sets errno properly, using ERANGE to mean
+ * "empty file". Must only be called on trusted Tor-owned files, as changing
+ * the underlying file's size causes unspecified behavior. */
+tor_mmap_t *
+tor_mmap_file(const char *filename)
+{
+ int fd; /* router file */
+ char *string;
+ int result;
+ tor_mmap_t *res;
+ size_t size, filesize;
+ struct stat st;
+
+ tor_assert(filename);
+
+ fd = tor_open_cloexec(filename, O_RDONLY, 0);
+ if (fd<0) {
+ int save_errno = errno;
+ int severity = (errno == ENOENT) ? LOG_INFO : LOG_WARN;
+ log_fn(severity, LD_FS,"Could not open \"%s\" for mmap(): %s",filename,
+ strerror(errno));
+ errno = save_errno;
+ return NULL;
+ }
+
+ /* Get the size of the file */
+ result = fstat(fd, &st);
+ if (result != 0) {
+ int save_errno = errno;
+ log_warn(LD_FS,
+ "Couldn't fstat opened descriptor for \"%s\" during mmap: %s",
+ filename, strerror(errno));
+ close(fd);
+ errno = save_errno;
+ return NULL;
+ }
+ size = filesize = (size_t)(st.st_size);
+
+ if (st.st_size > SSIZE_T_CEILING || (off_t)size < st.st_size) {
+ log_warn(LD_FS, "File \"%s\" is too large. Ignoring.",filename);
+ errno = EFBIG;
+ close(fd);
+ return NULL;
+ }
+ if (!size) {
+ /* Zero-length file. If we call mmap on it, it will succeed but
+ * return NULL, and bad things will happen. So just fail. */
+ log_info(LD_FS,"File \"%s\" is empty. Ignoring.",filename);
+ errno = ERANGE;
+ close(fd);
+ return NULL;
+ }
+
+ string = mmap(0, size, PROT_READ, MAP_PRIVATE, fd, 0);
+ close(fd);
+ if (string == MAP_FAILED) {
+ int save_errno = errno;
+ log_warn(LD_FS,"Could not mmap file \"%s\": %s", filename,
+ strerror(errno));
+ errno = save_errno;
+ return NULL;
+ }
+
+ res = tor_malloc_zero(sizeof(tor_mmap_t));
+ res->data = string;
+ res->size = filesize;
+ res->mapping_size = size;
+
+ return res;
+}
+/** Release storage held for a memory mapping; returns 0 on success,
+ * or -1 on failure (and logs a warning). */
+int
+tor_munmap_file(tor_mmap_t *handle)
+{
+ int res;
+
+ if (handle == NULL)
+ return 0;
+
+ res = munmap((char*)handle->data, handle->mapping_size);
+ if (res == 0) {
+ /* munmap() succeeded */
+ tor_free(handle);
+ } else {
+ log_warn(LD_FS, "Failed to munmap() in tor_munmap_file(): %s",
+ strerror(errno));
+ res = -1;
+ }
+
+ return res;
+}
+#elif defined(_WIN32)
+tor_mmap_t *
+tor_mmap_file(const char *filename)
+{
+ TCHAR tfilename[MAX_PATH]= {0};
+ tor_mmap_t *res = tor_malloc_zero(sizeof(tor_mmap_t));
+ int empty = 0;
+ HANDLE file_handle = INVALID_HANDLE_VALUE;
+ DWORD size_low, size_high;
+ uint64_t real_size;
+ res->mmap_handle = NULL;
+#ifdef UNICODE
+ mbstowcs(tfilename,filename,MAX_PATH);
+#else
+ strlcpy(tfilename,filename,MAX_PATH);
+#endif
+ file_handle = CreateFile(tfilename,
+ GENERIC_READ, FILE_SHARE_READ,
+ NULL,
+ OPEN_EXISTING,
+ FILE_ATTRIBUTE_NORMAL,
+ 0);
+
+ if (file_handle == INVALID_HANDLE_VALUE)
+ goto win_err;
+
+ size_low = GetFileSize(file_handle, &size_high);
+
+ if (size_low == INVALID_FILE_SIZE && GetLastError() != NO_ERROR) {
+ log_warn(LD_FS,"Error getting size of \"%s\".",filename);
+ goto win_err;
+ }
+ if (size_low == 0 && size_high == 0) {
+ log_info(LD_FS,"File \"%s\" is empty. Ignoring.",filename);
+ empty = 1;
+ goto err;
+ }
+ real_size = (((uint64_t)size_high)<<32) | size_low;
+ if (real_size > SIZE_MAX) {
+ log_warn(LD_FS,"File \"%s\" is too big to map; not trying.",filename);
+ goto err;
+ }
+ res->size = real_size;
+
+ res->mmap_handle = CreateFileMapping(file_handle,
+ NULL,
+ PAGE_READONLY,
+ size_high,
+ size_low,
+ NULL);
+ if (res->mmap_handle == NULL)
+ goto win_err;
+ res->data = (char*) MapViewOfFile(res->mmap_handle,
+ FILE_MAP_READ,
+ 0, 0, 0);
+ if (!res->data)
+ goto win_err;
+
+ CloseHandle(file_handle);
+ return res;
+ win_err: {
+ DWORD e = GetLastError();
+ int severity = (e == ERROR_FILE_NOT_FOUND || e == ERROR_PATH_NOT_FOUND) ?
+ LOG_INFO : LOG_WARN;
+ char *msg = format_win32_error(e);
+ log_fn(severity, LD_FS, "Couldn't mmap file \"%s\": %s", filename, msg);
+ tor_free(msg);
+ if (e == ERROR_FILE_NOT_FOUND || e == ERROR_PATH_NOT_FOUND)
+ errno = ENOENT;
+ else
+ errno = EINVAL;
+ }
+ err:
+ if (empty)
+ errno = ERANGE;
+ if (file_handle != INVALID_HANDLE_VALUE)
+ CloseHandle(file_handle);
+ tor_munmap_file(res);
+ return NULL;
+}
+
+/* Unmap the file, and return 0 for success or -1 for failure */
+int
+tor_munmap_file(tor_mmap_t *handle)
+{
+ if (handle == NULL)
+ return 0;
+
+ if (handle->data) {
+ /* This is an ugly cast, but without it, "data" in struct tor_mmap_t would
+ have to be redefined as non-const. */
+ BOOL ok = UnmapViewOfFile( (LPVOID) handle->data);
+ if (!ok) {
+ log_warn(LD_FS, "Failed to UnmapViewOfFile() in tor_munmap_file(): %d",
+ (int)GetLastError());
+ }
+ }
+
+ if (handle->mmap_handle != NULL)
+ CloseHandle(handle->mmap_handle);
+ tor_free(handle);
+
+ return 0;
+}
+#else
+#error "cannot implement tor_mmap_file"
+#endif /* defined(HAVE_MMAP) || ... || ... */
diff --git a/src/lib/fs/mmap.h b/src/lib/fs/mmap.h
new file mode 100644
index 0000000000..18fb18a13c
--- /dev/null
+++ b/src/lib/fs/mmap.h
@@ -0,0 +1,41 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file mmap.h
+ *
+ * \brief Header for mmap.c
+ **/
+
+#ifndef TOR_MMAP_H
+#define TOR_MMAP_H
+
+#include "lib/cc/compat_compiler.h"
+#include <stddef.h>
+
+#ifdef _WIN32
+#include <windef.h>
+#endif
+
+/** Represents an mmaped file. Allocated via tor_mmap_file; freed with
+ * tor_munmap_file. */
+typedef struct tor_mmap_t {
+ const char *data; /**< Mapping of the file's contents. */
+ size_t size; /**< Size of the file. */
+
+ /* None of the fields below should be accessed from outside compat.c */
+#ifdef HAVE_MMAP
+ size_t mapping_size; /**< Size of the actual mapping. (This is this file
+ * size, rounded up to the nearest page.) */
+#elif defined _WIN32
+ HANDLE mmap_handle;
+#endif /* defined(HAVE_MMAP) || ... */
+
+} tor_mmap_t;
+
+tor_mmap_t *tor_mmap_file(const char *filename);
+int tor_munmap_file(tor_mmap_t *handle);
+
+#endif
diff --git a/src/lib/fs/path.c b/src/lib/fs/path.c
new file mode 100644
index 0000000000..b3ef61979d
--- /dev/null
+++ b/src/lib/fs/path.c
@@ -0,0 +1,295 @@
+/* Copyright (c) 2003, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file path.c
+ *
+ * \brief Manipulate strings that contain filesystem paths.
+ **/
+
+#include "lib/fs/path.h"
+#include "lib/malloc/malloc.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/string/printf.h"
+#include "lib/string/util_string.h"
+#include "lib/string/compat_ctype.h"
+#include "lib/fs/userdb.h"
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+#include <errno.h>
+#include <string.h>
+
+/** Removes enclosing quotes from <b>path</b> and unescapes quotes between the
+ * enclosing quotes. Backslashes are not unescaped. Return the unquoted
+ * <b>path</b> on success or 0 if <b>path</b> is not quoted correctly. */
+char *
+get_unquoted_path(const char *path)
+{
+ size_t len = strlen(path);
+
+ if (len == 0) {
+ return tor_strdup("");
+ }
+
+ int has_start_quote = (path[0] == '\"');
+ int has_end_quote = (len > 0 && path[len-1] == '\"');
+ if (has_start_quote != has_end_quote || (len == 1 && has_start_quote)) {
+ return NULL;
+ }
+
+ char *unquoted_path = tor_malloc(len - has_start_quote - has_end_quote + 1);
+ char *s = unquoted_path;
+ size_t i;
+ for (i = has_start_quote; i < len - has_end_quote; i++) {
+ if (path[i] == '\"' && (i > 0 && path[i-1] == '\\')) {
+ *(s-1) = path[i];
+ } else if (path[i] != '\"') {
+ *s++ = path[i];
+ } else { /* unescaped quote */
+ tor_free(unquoted_path);
+ return NULL;
+ }
+ }
+ *s = '\0';
+ return unquoted_path;
+}
+
+/** Expand any homedir prefix on <b>filename</b>; return a newly allocated
+ * string. */
+char *
+expand_filename(const char *filename)
+{
+ tor_assert(filename);
+#ifdef _WIN32
+ /* Might consider using GetFullPathName() as described here:
+ * http://etutorials.org/Programming/secure+programming/
+ * Chapter+3.+Input+Validation/3.7+Validating+Filenames+and+Paths/
+ */
+ return tor_strdup(filename);
+#else /* !(defined(_WIN32)) */
+ if (*filename == '~') {
+ char *home, *result=NULL;
+ const char *rest;
+
+ if (filename[1] == '/' || filename[1] == '\0') {
+ home = getenv("HOME");
+ if (!home) {
+ log_warn(LD_CONFIG, "Couldn't find $HOME environment variable while "
+ "expanding \"%s\"; defaulting to \"\".", filename);
+ home = tor_strdup("");
+ } else {
+ home = tor_strdup(home);
+ }
+ rest = strlen(filename)>=2?(filename+2):"";
+ } else {
+#ifdef HAVE_PWD_H
+ char *username, *slash;
+ slash = strchr(filename, '/');
+ if (slash)
+ username = tor_strndup(filename+1,slash-filename-1);
+ else
+ username = tor_strdup(filename+1);
+ if (!(home = get_user_homedir(username))) {
+ log_warn(LD_CONFIG,"Couldn't get homedir for \"%s\"",username);
+ tor_free(username);
+ return NULL;
+ }
+ tor_free(username);
+ rest = slash ? (slash+1) : "";
+#else /* !(defined(HAVE_PWD_H)) */
+ log_warn(LD_CONFIG, "Couldn't expand homedir on system without pwd.h");
+ return tor_strdup(filename);
+#endif /* defined(HAVE_PWD_H) */
+ }
+ tor_assert(home);
+ /* Remove trailing slash. */
+ if (strlen(home)>1 && !strcmpend(home,PATH_SEPARATOR)) {
+ home[strlen(home)-1] = '\0';
+ }
+ tor_asprintf(&result,"%s"PATH_SEPARATOR"%s",home,rest);
+ tor_free(home);
+ return result;
+ } else {
+ return tor_strdup(filename);
+ }
+#endif /* defined(_WIN32) */
+}
+
+/** Return true iff <b>filename</b> is a relative path. */
+int
+path_is_relative(const char *filename)
+{
+ if (filename && filename[0] == '/')
+ return 0;
+#ifdef _WIN32
+ else if (filename && filename[0] == '\\')
+ return 0;
+ else if (filename && strlen(filename)>3 && TOR_ISALPHA(filename[0]) &&
+ filename[1] == ':' && filename[2] == '\\')
+ return 0;
+#endif /* defined(_WIN32) */
+ else
+ return 1;
+}
+
+/** Clean up <b>name</b> so that we can use it in a call to "stat". On Unix,
+ * we do nothing. On Windows, we remove a trailing slash, unless the path is
+ * the root of a disk. */
+void
+clean_fname_for_stat(char *name)
+{
+#ifdef _WIN32
+ size_t len = strlen(name);
+ if (!len)
+ return;
+ if (name[len-1]=='\\' || name[len-1]=='/') {
+ if (len == 1 || (len==3 && name[1]==':'))
+ return;
+ name[len-1]='\0';
+ }
+#else /* !(defined(_WIN32)) */
+ (void)name;
+#endif /* defined(_WIN32) */
+}
+
+/** Modify <b>fname</b> to contain the name of its parent directory. Doesn't
+ * actually examine the filesystem; does a purely syntactic modification.
+ *
+ * The parent of the root director is considered to be iteself.
+ *
+ * Path separators are the forward slash (/) everywhere and additionally
+ * the backslash (\) on Win32.
+ *
+ * Cuts off any number of trailing path separators but otherwise ignores
+ * them for purposes of finding the parent directory.
+ *
+ * Returns 0 if a parent directory was successfully found, -1 otherwise (fname
+ * did not have any path separators or only had them at the end).
+ * */
+int
+get_parent_directory(char *fname)
+{
+ char *cp;
+ int at_end = 1;
+ tor_assert(fname);
+#ifdef _WIN32
+ /* If we start with, say, c:, then don't consider that the start of the path
+ */
+ if (fname[0] && fname[1] == ':') {
+ fname += 2;
+ }
+#endif /* defined(_WIN32) */
+ /* Now we want to remove all path-separators at the end of the string,
+ * and to remove the end of the string starting with the path separator
+ * before the last non-path-separator. In perl, this would be
+ * s#[/]*$##; s#/[^/]*$##;
+ * on a unixy platform.
+ */
+ cp = fname + strlen(fname);
+ at_end = 1;
+ while (--cp >= fname) {
+ int is_sep = (*cp == '/'
+#ifdef _WIN32
+ || *cp == '\\'
+#endif
+ );
+ if (is_sep) {
+ if (cp == fname) {
+ /* This is the first separator in the file name; don't remove it! */
+ cp[1] = '\0';
+ return 0;
+ }
+ *cp = '\0';
+ if (! at_end)
+ return 0;
+ } else {
+ at_end = 0;
+ }
+ }
+ return -1;
+}
+
+#ifndef _WIN32
+/** Return a newly allocated string containing the output of getcwd(). Return
+ * NULL on failure. (We can't just use getcwd() into a PATH_MAX buffer, since
+ * Hurd hasn't got a PATH_MAX.)
+ */
+static char *
+alloc_getcwd(void)
+{
+#ifdef HAVE_GET_CURRENT_DIR_NAME
+ /* Glibc makes this nice and simple for us. */
+ char *cwd = get_current_dir_name();
+ char *result = NULL;
+ if (cwd) {
+ /* We make a copy here, in case tor_malloc() is not malloc(). */
+ result = tor_strdup(cwd);
+ raw_free(cwd); // alias for free to avoid tripping check-spaces.
+ }
+ return result;
+#else /* !(defined(HAVE_GET_CURRENT_DIR_NAME)) */
+ size_t size = 1024;
+ char *buf = NULL;
+ char *ptr = NULL;
+
+ while (ptr == NULL) {
+ buf = tor_realloc(buf, size);
+ ptr = getcwd(buf, size);
+
+ if (ptr == NULL && errno != ERANGE) {
+ tor_free(buf);
+ return NULL;
+ }
+
+ size *= 2;
+ }
+ return buf;
+#endif /* defined(HAVE_GET_CURRENT_DIR_NAME) */
+}
+#endif /* !defined(_WIN32) */
+
+/** Expand possibly relative path <b>fname</b> to an absolute path.
+ * Return a newly allocated string, possibly equal to <b>fname</b>. */
+char *
+make_path_absolute(char *fname)
+{
+#ifdef _WIN32
+ char *absfname_malloced = _fullpath(NULL, fname, 1);
+
+ /* We don't want to assume that tor_free can free a string allocated
+ * with malloc. On failure, return fname (it's better than nothing). */
+ char *absfname = tor_strdup(absfname_malloced ? absfname_malloced : fname);
+ if (absfname_malloced) raw_free(absfname_malloced);
+
+ return absfname;
+#else /* !(defined(_WIN32)) */
+ char *absfname = NULL, *path = NULL;
+
+ tor_assert(fname);
+
+ if (fname[0] == '/') {
+ absfname = tor_strdup(fname);
+ } else {
+ path = alloc_getcwd();
+ if (path) {
+ tor_asprintf(&absfname, "%s/%s", path, fname);
+ tor_free(path);
+ } else {
+ /* LCOV_EXCL_START Can't make getcwd fail. */
+ /* If getcwd failed, the best we can do here is keep using the
+ * relative path. (Perhaps / isn't readable by this UID/GID.) */
+ log_warn(LD_GENERAL, "Unable to find current working directory: %s",
+ strerror(errno));
+ absfname = tor_strdup(fname);
+ /* LCOV_EXCL_STOP */
+ }
+ }
+ return absfname;
+#endif /* defined(_WIN32) */
+}
diff --git a/src/lib/fs/path.h b/src/lib/fs/path.h
new file mode 100644
index 0000000000..4675ac84e8
--- /dev/null
+++ b/src/lib/fs/path.h
@@ -0,0 +1,30 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file path.h
+ *
+ * \brief Header for path.c
+ **/
+
+#ifndef TOR_PATH_H
+#define TOR_PATH_H
+
+#include "lib/cc/compat_compiler.h"
+
+#ifdef _WIN32
+#define PATH_SEPARATOR "\\"
+#else
+#define PATH_SEPARATOR "/"
+#endif
+
+char *get_unquoted_path(const char *path);
+char *expand_filename(const char *filename);
+int path_is_relative(const char *filename);
+void clean_fname_for_stat(char *name);
+int get_parent_directory(char *fname);
+char *make_path_absolute(char *fname);
+
+#endif
diff --git a/src/lib/fs/storagedir.c b/src/lib/fs/storagedir.c
new file mode 100644
index 0000000000..2caddf1ad9
--- /dev/null
+++ b/src/lib/fs/storagedir.c
@@ -0,0 +1,606 @@
+/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file storagedir.c
+ *
+ * \brief An abstraction for a directory full of similar files.
+ *
+ * Storagedirs are used by our consensus cache code, and may someday also get
+ * used for unparseable objects. A large part of the need for this type is to
+ * work around the limitations in our sandbox code, where all filenames need
+ * to be registered in advance.
+ **/
+
+#include "lib/fs/storagedir.h"
+
+#include "lib/container/smartlist.h"
+#include "lib/encoding/confline.h"
+#include "lib/fs/dir.h"
+#include "lib/fs/files.h"
+#include "lib/fs/mmap.h"
+#include "lib/log/escape.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/malloc/malloc.h"
+#include "lib/memarea/memarea.h"
+#include "lib/sandbox/sandbox.h"
+#include "lib/string/printf.h"
+#include "lib/string/util_string.h"
+
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#include <stdlib.h>
+#include <errno.h>
+#include <string.h>
+
+#define FNAME_MIN_NUM 1000
+
+/** A storage_dir_t represents a directory full of similar cached
+ * files. Filenames are decimal integers. Files can be cleaned as needed
+ * to limit total disk usage. */
+struct storage_dir_t {
+ /** Directory holding the files for this storagedir. */
+ char *directory;
+ /** Either NULL, or a directory listing of the directory (as a smartlist
+ * of strings */
+ smartlist_t *contents;
+ /** The largest number of non-temporary files we'll place in the
+ * directory. */
+ int max_files;
+ /** If true, then 'usage' has been computed. */
+ int usage_known;
+ /** The total number of bytes used in this directory */
+ uint64_t usage;
+};
+
+/** Create or open a new storage directory at <b>dirname</b>, with
+ * capacity for up to <b>max_files</b> files.
+ */
+storage_dir_t *
+storage_dir_new(const char *dirname, int max_files)
+{
+ if (check_private_dir(dirname, CPD_CREATE, NULL) < 0)
+ return NULL;
+
+ storage_dir_t *d = tor_malloc_zero(sizeof(storage_dir_t));
+ d->directory = tor_strdup(dirname);
+ d->max_files = max_files;
+ return d;
+}
+
+/**
+ * Drop all in-RAM storage for <b>d</b>. Does not delete any files.
+ */
+void
+storage_dir_free_(storage_dir_t *d)
+{
+ if (d == NULL)
+ return;
+ tor_free(d->directory);
+ if (d->contents) {
+ SMARTLIST_FOREACH(d->contents, char *, cp, tor_free(cp));
+ smartlist_free(d->contents);
+ }
+ tor_free(d);
+}
+
+/**
+ * Tell the sandbox (if any) configured by <b>cfg</b> to allow the
+ * operations that <b>d</b> will need.
+ *
+ * The presence of this function is why we need an upper limit on the
+ * number of files in a storage_dir_t: we need to approve file operations
+ * one by one.
+ */
+int
+storage_dir_register_with_sandbox(storage_dir_t *d, sandbox_cfg_t **cfg)
+{
+ int problems = 0;
+ int idx;
+ for (idx = FNAME_MIN_NUM; idx < FNAME_MIN_NUM + d->max_files; ++idx) {
+ char *path = NULL, *tmppath = NULL;
+ tor_asprintf(&path, "%s/%d", d->directory, idx);
+ tor_asprintf(&tmppath, "%s/%d.tmp", d->directory, idx);
+
+ problems += sandbox_cfg_allow_open_filename(cfg, tor_strdup(path));
+ problems += sandbox_cfg_allow_open_filename(cfg, tor_strdup(tmppath));
+ problems += sandbox_cfg_allow_stat_filename(cfg, tor_strdup(path));
+ problems += sandbox_cfg_allow_stat_filename(cfg, tor_strdup(tmppath));
+ problems += sandbox_cfg_allow_rename(cfg,
+ tor_strdup(tmppath), tor_strdup(path));
+
+ tor_free(path);
+ tor_free(tmppath);
+ }
+
+ return problems ? -1 : 0;
+}
+
+/**
+ * Remove all files in <b>d</b> whose names end with ".tmp".
+ *
+ * Requires that the contents field of <b>d</b> is set.
+ */
+static void
+storage_dir_clean_tmpfiles(storage_dir_t *d)
+{
+ if (!d->contents)
+ return;
+ SMARTLIST_FOREACH_BEGIN(d->contents, char *, fname) {
+ if (strcmpend(fname, ".tmp"))
+ continue;
+ char *path = NULL;
+ tor_asprintf(&path, "%s/%s", d->directory, fname);
+ if (unlink(sandbox_intern_string(path))) {
+ log_warn(LD_FS, "Unable to unlink %s while cleaning "
+ "temporary files: %s", escaped(path), strerror(errno));
+ tor_free(path);
+ continue;
+ }
+ tor_free(path);
+ SMARTLIST_DEL_CURRENT(d->contents, fname);
+ tor_free(fname);
+ } SMARTLIST_FOREACH_END(fname);
+
+ d->usage_known = 0;
+}
+
+/**
+ * Re-scan the directory <b>d</b> to learn its contents.
+ */
+static int
+storage_dir_rescan(storage_dir_t *d)
+{
+ if (d->contents) {
+ SMARTLIST_FOREACH(d->contents, char *, cp, tor_free(cp));
+ smartlist_free(d->contents);
+ }
+ d->usage = 0;
+ d->usage_known = 0;
+ if (NULL == (d->contents = tor_listdir(d->directory))) {
+ return -1;
+ }
+ storage_dir_clean_tmpfiles(d);
+ return 0;
+}
+
+/**
+ * Return a smartlist containing the filenames within <b>d</b>.
+ */
+const smartlist_t *
+storage_dir_list(storage_dir_t *d)
+{
+ if (! d->contents)
+ storage_dir_rescan(d);
+ return d->contents;
+}
+
+/**
+ * Return the total number of bytes used for storage in <b>d</b>.
+ */
+uint64_t
+storage_dir_get_usage(storage_dir_t *d)
+{
+ if (d->usage_known)
+ return d->usage;
+
+ uint64_t total = 0;
+ SMARTLIST_FOREACH_BEGIN(storage_dir_list(d), const char *, cp) {
+ char *path = NULL;
+ struct stat st;
+ tor_asprintf(&path, "%s/%s", d->directory, cp);
+ if (stat(sandbox_intern_string(path), &st) == 0) {
+ total += st.st_size;
+ }
+ tor_free(path);
+ } SMARTLIST_FOREACH_END(cp);
+
+ d->usage = total;
+ d->usage_known = 1;
+ return d->usage;
+}
+
+/** Mmap a specified file within <b>d</b>.
+ *
+ * On failure, return NULL and set errno as for tor_mmap_file(). */
+tor_mmap_t *
+storage_dir_map(storage_dir_t *d, const char *fname)
+{
+ char *path = NULL;
+ tor_asprintf(&path, "%s/%s", d->directory, fname);
+ tor_mmap_t *result = tor_mmap_file(path);
+ int errval = errno;
+ tor_free(path);
+ if (result == NULL)
+ errno = errval;
+ return result;
+}
+
+/** Read a file within <b>d</b> into a newly allocated buffer. Set
+ * *<b>sz_out</b> to its size. */
+uint8_t *
+storage_dir_read(storage_dir_t *d, const char *fname, int bin, size_t *sz_out)
+{
+ const int flags = bin ? RFTS_BIN : 0;
+
+ char *path = NULL;
+ tor_asprintf(&path, "%s/%s", d->directory, fname);
+ struct stat st;
+ char *contents = read_file_to_str(path, flags, &st);
+ if (contents && sz_out) {
+ // it fits in RAM, so we know its size is less than SIZE_MAX
+#if UINT64_MAX > SIZE_MAX
+ tor_assert((uint64_t)st.st_size <= SIZE_MAX);
+#endif
+ *sz_out = (size_t) st.st_size;
+ }
+
+ tor_free(path);
+ return (uint8_t *) contents;
+}
+
+/** Helper: Find an unused filename within the directory */
+static char *
+find_unused_fname(storage_dir_t *d)
+{
+ if (!d->contents) {
+ if (storage_dir_rescan(d) < 0)
+ return NULL;
+ }
+
+ char buf[16];
+ int i;
+ /* Yuck; this is quadratic. Fortunately, that shouldn't matter much,
+ * since disk writes are more expensive by a lot. */
+ for (i = FNAME_MIN_NUM; i < FNAME_MIN_NUM + d->max_files; ++i) {
+ tor_snprintf(buf, sizeof(buf), "%d", i);
+ if (!smartlist_contains_string(d->contents, buf)) {
+ return tor_strdup(buf);
+ }
+ }
+ return NULL;
+}
+
+/** Helper: As storage_dir_save_bytes_to_file, but store a smartlist of
+ * sized_chunk_t rather than a single byte array. */
+static int
+storage_dir_save_chunks_to_file(storage_dir_t *d,
+ const smartlist_t *chunks,
+ int binary,
+ char **fname_out)
+{
+ uint64_t total_length = 0;
+ char *fname = find_unused_fname(d);
+ if (!fname)
+ return -1;
+
+ SMARTLIST_FOREACH(chunks, const sized_chunk_t *, ch,
+ total_length += ch->len);
+
+ char *path = NULL;
+ tor_asprintf(&path, "%s/%s", d->directory, fname);
+
+ int r = write_chunks_to_file(path, chunks, binary, 0);
+ if (r == 0) {
+ if (d->usage_known)
+ d->usage += total_length;
+ if (fname_out) {
+ *fname_out = tor_strdup(fname);
+ }
+ if (d->contents)
+ smartlist_add(d->contents, tor_strdup(fname));
+ }
+ tor_free(fname);
+ tor_free(path);
+ return r;
+}
+
+/** Try to write the <b>length</b> bytes at <b>data</b> into a new file
+ * in <b>d</b>. On success, return 0 and set *<b>fname_out</b> to a
+ * newly allocated string containing the filename. On failure, return
+ * -1. */
+int
+storage_dir_save_bytes_to_file(storage_dir_t *d,
+ const uint8_t *data,
+ size_t length,
+ int binary,
+ char **fname_out)
+{
+ smartlist_t *chunks = smartlist_new();
+ sized_chunk_t chunk = { (const char *)data, length };
+ smartlist_add(chunks, &chunk);
+ int r = storage_dir_save_chunks_to_file(d, chunks, binary, fname_out);
+ smartlist_free(chunks);
+ return r;
+}
+
+/**
+ * As storage_dir_save_bytes_to_file, but saves a NUL-terminated string
+ * <b>str</b>.
+ */
+int
+storage_dir_save_string_to_file(storage_dir_t *d,
+ const char *str,
+ int binary,
+ char **fname_out)
+{
+ return storage_dir_save_bytes_to_file(d,
+ (const uint8_t*)str, strlen(str), binary, fname_out);
+}
+
+/**
+ * As storage_dir_save_bytes_to_file, but associates the data with the
+ * key-value pairs in <b>labels</b>. Files stored in this format can be
+ * recovered with storage_dir_map_labeled() or storage_dir_read_labeled().
+ */
+int
+storage_dir_save_labeled_to_file(storage_dir_t *d,
+ const config_line_t *labels,
+ const uint8_t *data,
+ size_t length,
+ char **fname_out)
+{
+ /*
+ * The storage format is to prefix the data with the key-value pairs in
+ * <b>labels</b>, and a single NUL separator. But code outside this module
+ * MUST NOT rely on that format.
+ */
+
+ smartlist_t *chunks = smartlist_new();
+ memarea_t *area = memarea_new();
+ const config_line_t *line;
+ for (line = labels; line; line = line->next) {
+ sized_chunk_t *sz = memarea_alloc(area, sizeof(sized_chunk_t));
+ sz->len = strlen(line->key) + 1 + strlen(line->value) + 1;
+ const size_t allocated = sz->len + 1;
+ char *bytes = memarea_alloc(area, allocated);
+ tor_snprintf(bytes, allocated, "%s %s\n", line->key, line->value);
+ sz->bytes = bytes;
+ smartlist_add(chunks, sz);
+ }
+
+ sized_chunk_t *nul = memarea_alloc(area, sizeof(sized_chunk_t));
+ nul->len = 1;
+ nul->bytes = "\0";
+ smartlist_add(chunks, nul);
+
+ sized_chunk_t *datachunk = memarea_alloc(area, sizeof(sized_chunk_t));
+ datachunk->bytes = (const char *)data;
+ datachunk->len = length;
+ smartlist_add(chunks, datachunk);
+
+ int r = storage_dir_save_chunks_to_file(d, chunks, 1, fname_out);
+ smartlist_free(chunks);
+ memarea_drop_all(area);
+ return r;
+}
+
+/**
+ * Map a file that was created with storage_dir_save_labeled_to_file(). On
+ * failure, return NULL. On success, write a set of newly allocated labels
+ * into *<b>labels_out</b>, a pointer to the data into *<b>data_out</b>, and
+ * the data's size into *<b>sz_out</b>. On success, also return a tor_mmap_t
+ * object whose contents should not be used -- it needs to be kept around,
+ * though, for as long as <b>data_out</b> is going to be valid.
+ *
+ * On failure, set errno as for tor_mmap_file() if the file was missing or
+ * empty, and set errno to EINVAL if the file was not in the labeled
+ * format expected.
+ */
+tor_mmap_t *
+storage_dir_map_labeled(storage_dir_t *dir,
+ const char *fname,
+ config_line_t **labels_out,
+ const uint8_t **data_out,
+ size_t *sz_out)
+{
+ tor_mmap_t *m = storage_dir_map(dir, fname);
+ int errval;
+ if (! m) {
+ errval = errno;
+ goto err;
+ }
+ const char *nulp = memchr(m->data, '\0', m->size);
+ if (! nulp) {
+ errval = EINVAL;
+ goto err;
+ }
+ if (labels_out && config_get_lines(m->data, labels_out, 0) < 0) {
+ errval = EINVAL;
+ goto err;
+ }
+ size_t offset = nulp - m->data + 1;
+ tor_assert(offset <= m->size);
+ *data_out = (const uint8_t *)(m->data + offset);
+ *sz_out = m->size - offset;
+
+ return m;
+ err:
+ tor_munmap_file(m);
+ errno = errval;
+ return NULL;
+}
+
+/** As storage_dir_map_labeled, but return a new byte array containing the
+ * data. */
+uint8_t *
+storage_dir_read_labeled(storage_dir_t *dir,
+ const char *fname,
+ config_line_t **labels_out,
+ size_t *sz_out)
+{
+ const uint8_t *data = NULL;
+ tor_mmap_t *m = storage_dir_map_labeled(dir, fname, labels_out,
+ &data, sz_out);
+ if (m == NULL)
+ return NULL;
+ uint8_t *result = tor_memdup(data, *sz_out);
+ tor_munmap_file(m);
+ return result;
+}
+
+/* Reduce the cached usage amount in <b>d</b> by <b>removed_file_size</b>.
+ * This function is a no-op if <b>d->usage_known</b> is 0. */
+static void
+storage_dir_reduce_usage(storage_dir_t *d, uint64_t removed_file_size)
+{
+ if (d->usage_known) {
+ if (! BUG(d->usage < removed_file_size)) {
+ /* This bug can also be triggered if an external process resized a file
+ * between the call to storage_dir_get_usage() that last checked
+ * actual usage (rather than relaying on cached usage), and the call to
+ * this function. */
+ d->usage -= removed_file_size;
+ } else {
+ /* If we underflowed the cached directory size, re-check the sizes of all
+ * the files in the directory. This makes storage_dir_shrink() quadratic,
+ * but only if a process is continually changing file sizes in the
+ * storage directory (in which case, we have bigger issues).
+ *
+ * We can't just reset usage_known, because storage_dir_shrink() relies
+ * on knowing the usage. */
+ storage_dir_rescan(d);
+ (void)storage_dir_get_usage(d);
+ }
+ }
+}
+
+/**
+ * Remove the file called <b>fname</b> from <b>d</b>.
+ */
+void
+storage_dir_remove_file(storage_dir_t *d,
+ const char *fname)
+{
+ char *path = NULL;
+ tor_asprintf(&path, "%s/%s", d->directory, fname);
+ const char *ipath = sandbox_intern_string(path);
+
+ uint64_t size = 0;
+ if (d->usage_known) {
+ struct stat st;
+ if (stat(ipath, &st) == 0) {
+ size = st.st_size;
+ }
+ }
+ if (unlink(ipath) == 0) {
+ storage_dir_reduce_usage(d, size);
+ } else {
+ log_warn(LD_FS, "Unable to unlink %s while removing file: %s",
+ escaped(path), strerror(errno));
+ tor_free(path);
+ return;
+ }
+ if (d->contents) {
+ smartlist_string_remove(d->contents, fname);
+ }
+
+ tor_free(path);
+}
+
+/** Helper type: used to sort the members of storage directory by mtime. */
+typedef struct shrinking_dir_entry_t {
+ time_t mtime;
+ uint64_t size;
+ char *path;
+} shrinking_dir_entry_t;
+
+/** Helper: use with qsort to sort shrinking_dir_entry_t structs. */
+static int
+shrinking_dir_entry_compare(const void *a_, const void *b_)
+{
+ const shrinking_dir_entry_t *a = a_;
+ const shrinking_dir_entry_t *b = b_;
+
+ if (a->mtime < b->mtime)
+ return -1;
+ else if (a->mtime > b->mtime)
+ return 1;
+ else
+ return 0;
+}
+
+/**
+ * Try to free space by removing the oldest files in <b>d</b>. Delete
+ * until no more than <b>target_size</b> bytes are left, and at least
+ * <b>min_to_remove</b> files have been removed... or until there is
+ * nothing left to remove.
+ *
+ * Return 0 on success; -1 on failure.
+ */
+int
+storage_dir_shrink(storage_dir_t *d,
+ uint64_t target_size,
+ int min_to_remove)
+{
+ if (d->usage_known && d->usage <= target_size && !min_to_remove) {
+ /* Already small enough. */
+ return 0;
+ }
+
+ if (storage_dir_rescan(d) < 0)
+ return -1;
+
+ const uint64_t orig_usage = storage_dir_get_usage(d);
+ if (orig_usage <= target_size && !min_to_remove) {
+ /* Okay, small enough after rescan! */
+ return 0;
+ }
+
+ const int n = smartlist_len(d->contents);
+ shrinking_dir_entry_t *ents = tor_calloc(n, sizeof(shrinking_dir_entry_t));
+ SMARTLIST_FOREACH_BEGIN(d->contents, const char *, fname) {
+ shrinking_dir_entry_t *ent = &ents[fname_sl_idx];
+ struct stat st;
+ tor_asprintf(&ent->path, "%s/%s", d->directory, fname);
+ if (stat(sandbox_intern_string(ent->path), &st) == 0) {
+ ent->mtime = st.st_mtime;
+ ent->size = st.st_size;
+ }
+ } SMARTLIST_FOREACH_END(fname);
+
+ qsort(ents, n, sizeof(shrinking_dir_entry_t), shrinking_dir_entry_compare);
+
+ int idx = 0;
+ while ((d->usage > target_size || min_to_remove > 0) && idx < n) {
+ if (unlink(sandbox_intern_string(ents[idx].path)) == 0) {
+ storage_dir_reduce_usage(d, ents[idx].size);
+ --min_to_remove;
+ }
+ ++idx;
+ }
+
+ for (idx = 0; idx < n; ++idx) {
+ tor_free(ents[idx].path);
+ }
+ tor_free(ents);
+
+ storage_dir_rescan(d);
+
+ return 0;
+}
+
+/** Remove all files in <b>d</b>. */
+int
+storage_dir_remove_all(storage_dir_t *d)
+{
+ return storage_dir_shrink(d, 0, d->max_files);
+}
+
+/**
+ * Return the largest number of non-temporary files we're willing to
+ * store in <b>d</b>.
+ */
+int
+storage_dir_get_max_files(storage_dir_t *d)
+{
+ return d->max_files;
+}
diff --git a/src/lib/fs/storagedir.h b/src/lib/fs/storagedir.h
new file mode 100644
index 0000000000..7e6633a0bb
--- /dev/null
+++ b/src/lib/fs/storagedir.h
@@ -0,0 +1,64 @@
+/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file storagedir.h
+ *
+ * \brief Header for storagedir.c
+ **/
+
+#ifndef TOR_STORAGEDIR_H
+#define TOR_STORAGEDIR_H
+
+#include "lib/cc/torint.h"
+#include <stddef.h>
+
+typedef struct storage_dir_t storage_dir_t;
+struct config_line_t;
+struct sandbox_cfg_elem;
+struct tor_mmap_t;
+struct smartlist_t;
+
+storage_dir_t * storage_dir_new(const char *dirname, int n_files);
+void storage_dir_free_(storage_dir_t *d);
+#define storage_dir_free(d) \
+ FREE_AND_NULL(storage_dir_t, storage_dir_free_, (d))
+
+int storage_dir_register_with_sandbox(storage_dir_t *d,
+ struct sandbox_cfg_elem **cfg);
+const struct smartlist_t *storage_dir_list(storage_dir_t *d);
+uint64_t storage_dir_get_usage(storage_dir_t *d);
+struct tor_mmap_t *storage_dir_map(storage_dir_t *d, const char *fname);
+uint8_t *storage_dir_read(storage_dir_t *d, const char *fname, int bin,
+ size_t *sz_out);
+int storage_dir_save_bytes_to_file(storage_dir_t *d,
+ const uint8_t *data,
+ size_t length,
+ int binary,
+ char **fname_out);
+int storage_dir_save_string_to_file(storage_dir_t *d,
+ const char *data,
+ int binary,
+ char **fname_out);
+int storage_dir_save_labeled_to_file(storage_dir_t *d,
+ const struct config_line_t *labels,
+ const uint8_t *data,
+ size_t length,
+ char **fname_out);
+struct tor_mmap_t *storage_dir_map_labeled(storage_dir_t *dir,
+ const char *fname,
+ struct config_line_t **labels_out,
+ const uint8_t **data_out,
+ size_t *size_out);
+uint8_t *storage_dir_read_labeled(storage_dir_t *d, const char *fname,
+ struct config_line_t **labels_out,
+ size_t *sz_out);
+void storage_dir_remove_file(storage_dir_t *d,
+ const char *fname);
+int storage_dir_shrink(storage_dir_t *d,
+ uint64_t target_size,
+ int min_to_remove);
+int storage_dir_remove_all(storage_dir_t *d);
+int storage_dir_get_max_files(storage_dir_t *d);
+
+#endif /* !defined(TOR_STORAGEDIR_H) */
diff --git a/src/lib/fs/userdb.c b/src/lib/fs/userdb.c
new file mode 100644
index 0000000000..95205c670e
--- /dev/null
+++ b/src/lib/fs/userdb.c
@@ -0,0 +1,138 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file userdb.c
+ *
+ * \brief Access the POSIX user database.
+ **/
+
+#include "lib/fs/userdb.h"
+
+#ifndef _WIN32
+#include "lib/malloc/malloc.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+
+#include <pwd.h>
+#include <stddef.h>
+#include <string.h>
+
+/** Cached struct from the last getpwname() call we did successfully. */
+static struct passwd *passwd_cached = NULL;
+
+/** Helper: copy a struct passwd object.
+ *
+ * We only copy the fields pw_uid, pw_gid, pw_name, pw_dir. Tor doesn't use
+ * any others, and I don't want to run into incompatibilities.
+ */
+static struct passwd *
+tor_passwd_dup(const struct passwd *pw)
+{
+ struct passwd *new_pw = tor_malloc_zero(sizeof(struct passwd));
+ if (pw->pw_name)
+ new_pw->pw_name = tor_strdup(pw->pw_name);
+ if (pw->pw_dir)
+ new_pw->pw_dir = tor_strdup(pw->pw_dir);
+ new_pw->pw_uid = pw->pw_uid;
+ new_pw->pw_gid = pw->pw_gid;
+
+ return new_pw;
+}
+
+#define tor_passwd_free(pw) \
+ FREE_AND_NULL(struct passwd, tor_passwd_free_, (pw))
+
+/** Helper: free one of our cached 'struct passwd' values. */
+static void
+tor_passwd_free_(struct passwd *pw)
+{
+ if (!pw)
+ return;
+
+ tor_free(pw->pw_name);
+ tor_free(pw->pw_dir);
+ tor_free(pw);
+}
+
+/** Wrapper around getpwnam() that caches result. Used so that we don't need
+ * to give the sandbox access to /etc/passwd.
+ *
+ * The following fields alone will definitely be copied in the output: pw_uid,
+ * pw_gid, pw_name, pw_dir. Other fields are not present in cached values.
+ *
+ * When called with a NULL argument, this function clears storage associated
+ * with static variables it uses.
+ **/
+const struct passwd *
+tor_getpwnam(const char *username)
+{
+ struct passwd *pw;
+
+ if (username == NULL) {
+ tor_passwd_free(passwd_cached);
+ passwd_cached = NULL;
+ return NULL;
+ }
+
+ if ((pw = getpwnam(username))) {
+ tor_passwd_free(passwd_cached);
+ passwd_cached = tor_passwd_dup(pw);
+ log_info(LD_GENERAL, "Caching new entry %s for %s",
+ passwd_cached->pw_name, username);
+ return pw;
+ }
+
+ /* Lookup failed */
+ if (! passwd_cached || ! passwd_cached->pw_name)
+ return NULL;
+
+ if (! strcmp(username, passwd_cached->pw_name))
+ return passwd_cached; // LCOV_EXCL_LINE - would need to make getpwnam flaky
+
+ return NULL;
+}
+
+/** Wrapper around getpwnam() that can use cached result from
+ * tor_getpwnam(). Used so that we don't need to give the sandbox access to
+ * /etc/passwd.
+ *
+ * The following fields alone will definitely be copied in the output: pw_uid,
+ * pw_gid, pw_name, pw_dir. Other fields are not present in cached values.
+ */
+const struct passwd *
+tor_getpwuid(uid_t uid)
+{
+ struct passwd *pw;
+
+ if ((pw = getpwuid(uid))) {
+ return pw;
+ }
+
+ /* Lookup failed */
+ if (! passwd_cached)
+ return NULL;
+
+ if (uid == passwd_cached->pw_uid)
+ return passwd_cached; // LCOV_EXCL_LINE - would need to make getpwnam flaky
+
+ return NULL;
+}
+
+/** Allocate and return a string containing the home directory for the
+ * user <b>username</b>. Only works on posix-like systems. */
+char *
+get_user_homedir(const char *username)
+{
+ const struct passwd *pw;
+ tor_assert(username);
+
+ if (!(pw = tor_getpwnam(username))) {
+ log_err(LD_CONFIG,"User \"%s\" not found.", username);
+ return NULL;
+ }
+ return tor_strdup(pw->pw_dir);
+}
+#endif /* !defined(_WIN32) */
diff --git a/src/lib/fs/userdb.h b/src/lib/fs/userdb.h
new file mode 100644
index 0000000000..5c39794873
--- /dev/null
+++ b/src/lib/fs/userdb.h
@@ -0,0 +1,26 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file userdb.h
+ *
+ * \brief Header for userdb.c
+ **/
+
+#ifndef TOR_USERDB_H
+#define TOR_USERDB_H
+
+#include "orconfig.h"
+
+#ifndef _WIN32
+#include <sys/types.h>
+
+struct passwd;
+const struct passwd *tor_getpwnam(const char *username);
+const struct passwd *tor_getpwuid(uid_t uid);
+char *get_user_homedir(const char *username);
+#endif
+
+#endif
diff --git a/src/lib/fs/winlib.c b/src/lib/fs/winlib.c
new file mode 100644
index 0000000000..b7302bd4ca
--- /dev/null
+++ b/src/lib/fs/winlib.c
@@ -0,0 +1,30 @@
+/* Copyright (c) 2003, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file winlib.c
+ *
+ * \brief Find and load windows system libraries.
+ *
+ * We use this function to dynamically load code at runtime that might not be
+ * available on all versions of Windows that we support.
+ **/
+
+#ifdef _WIN32
+#include "lib/fs/winlib.h"
+
+HANDLE
+load_windows_system_library(const TCHAR *library_name)
+{
+ TCHAR path[MAX_PATH];
+ unsigned n;
+ n = GetSystemDirectory(path, MAX_PATH);
+ if (n == 0 || n + _tcslen(library_name) + 2 >= MAX_PATH)
+ return 0;
+ _tcscat(path, TEXT("\\"));
+ _tcscat(path, library_name);
+ return LoadLibrary(path);
+}
+#endif /* defined(_WIN32) */
diff --git a/src/lib/fs/winlib.h b/src/lib/fs/winlib.h
new file mode 100644
index 0000000000..64a22439e5
--- /dev/null
+++ b/src/lib/fs/winlib.h
@@ -0,0 +1,22 @@
+/* Copyright (c) 2003, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file winlib.h
+ *
+ * \brief Header for winlib.c
+ **/
+
+#ifndef TOR_WINLIB_H
+#define TOR_WINLIB_H
+
+#ifdef _WIN32
+#include <windows.h>
+#include <tchar.h>
+
+HANDLE load_windows_system_library(const TCHAR *library_name);
+#endif
+
+#endif
7apu2bq3rhoylwmucxizk54afw5jyda7pho4j5zdcqpwkufke7zdzwad.onion