Extract threading code into a new library.

Note that the workqueue code does *not* go here: it is logically at
a higher level, since it needs to use libevent and the networking
stack.

6 files changed, 791 insertions, 0 deletions

diff --git a/src/lib/thread/.may_include b/src/lib/thread/.may_include
new file mode 100644
index 0000000000..93ad0cd734
--- /dev/null
+++ b/src/lib/thread/.may_include
@@ -0,0 +1,6 @@
+orconfig.h
+lib/cc/*.h
+lib/lock/*.h
+lib/log/*.h
+lib/testsupport/*.h
+lib/thread/*.h
diff --git a/src/lib/thread/compat_pthreads.c b/src/lib/thread/compat_pthreads.c
new file mode 100644
index 0000000000..246c6254bb
--- /dev/null
+++ b/src/lib/thread/compat_pthreads.c
@@ -0,0 +1,269 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file compat_pthreads.c
+ *
+ * \brief Implementation for the pthreads-based multithreading backend
+ * functions.
+ */
+
+#include "orconfig.h"
+#include "lib/thread/threads.h"
+#include "lib/log/torlog.h"
+#include "lib/log/util_bug.h"
+
+#include <sys/time.h>
+#include <pthread.h>
+#include <signal.h>
+#include <time.h>
+#include <errno.h>
+#include <string.h>
+
+/** Wraps a void (*)(void*) function and its argument so we can
+ * invoke them in a way pthreads would expect.
+ */
+typedef struct tor_pthread_data_t {
+  void (*func)(void *);
+  void *data;
+} tor_pthread_data_t;
+/** Given a tor_pthread_data_t <b>_data</b>, call _data-&gt;func(d-&gt;data)
+ * and free _data.  Used to make sure we can call functions the way pthread
+ * expects. */
+static void *
+tor_pthread_helper_fn(void *_data)
+{
+  tor_pthread_data_t *data = _data;
+  void (*func)(void*);
+  void *arg;
+  /* mask signals to worker threads to avoid SIGPIPE, etc */
+  sigset_t sigs;
+  /* We're in a subthread; don't handle any signals here. */
+  sigfillset(&sigs);
+  pthread_sigmask(SIG_SETMASK, &sigs, NULL);
+
+  func = data->func;
+  arg = data->data;
+  tor_free(_data);
+  func(arg);
+  return NULL;
+}
+/**
+ * A pthread attribute to make threads start detached.
+ */
+static pthread_attr_t attr_detached;
+/** True iff we've called tor_threads_init() */
+static int threads_initialized = 0;
+
+/** Minimalist interface to run a void function in the background.  On
+ * Unix calls pthread_create, on win32 calls beginthread.  Returns -1 on
+ * failure.
+ * func should not return, but rather should call spawn_exit.
+ *
+ * NOTE: if <b>data</b> is used, it should not be allocated on the stack,
+ * since in a multithreaded environment, there is no way to be sure that
+ * the caller's stack will still be around when the called function is
+ * running.
+ */
+int
+spawn_func(void (*func)(void *), void *data)
+{
+  pthread_t thread;
+  tor_pthread_data_t *d;
+  if (PREDICT_UNLIKELY(!threads_initialized)) {
+    tor_threads_init();
+  }
+  d = tor_malloc(sizeof(tor_pthread_data_t));
+  d->data = data;
+  d->func = func;
+  if (pthread_create(&thread, &attr_detached, tor_pthread_helper_fn, d)) {
+    tor_free(d);
+    return -1;
+  }
+
+  return 0;
+}
+
+/** End the current thread/process.
+ */
+void
+spawn_exit(void)
+{
+  pthread_exit(NULL);
+}
+
+/** Return an integer representing this thread. */
+unsigned long
+tor_get_thread_id(void)
+{
+  union {
+    pthread_t thr;
+    unsigned long id;
+  } r;
+  r.thr = pthread_self();
+  return r.id;
+}
+
+/* Conditions. */
+
+/** Initialize an already-allocated condition variable. */
+int
+tor_cond_init(tor_cond_t *cond)
+{
+  pthread_condattr_t condattr;
+
+  memset(cond, 0, sizeof(tor_cond_t));
+  /* Default condition attribute. Might be used if clock monotonic is
+   * available else this won't affect anything. */
+  if (pthread_condattr_init(&condattr)) {
+    return -1;
+  }
+
+#if defined(HAVE_CLOCK_GETTIME)
+#if defined(HAVE_PTHREAD_CONDATTR_SETCLOCK) && \
+  defined(CLOCK_MONOTONIC)
+  /* Use monotonic time so when we timedwait() on it, any clock adjustment
+   * won't affect the timeout value. */
+  if (pthread_condattr_setclock(&condattr, CLOCK_MONOTONIC)) {
+    return -1;
+  }
+#define USE_COND_CLOCK CLOCK_MONOTONIC
+#else /* !(defined(HAVE_PTHREAD_CONDATTR_SETCLOCK) && ...) */
+  /* On OSX Sierra, there is no pthread_condattr_setclock, so we are stuck
+   * with the realtime clock.
+   */
+#define USE_COND_CLOCK CLOCK_REALTIME
+#endif /* defined(HAVE_PTHREAD_CONDATTR_SETCLOCK) && ... */
+#endif /* defined(HAVE_CLOCK_GETTIME) */
+  if (pthread_cond_init(&cond->cond, &condattr)) {
+    return -1;
+  }
+  return 0;
+}
+
+/** Release all resources held by <b>cond</b>, but do not free <b>cond</b>
+ * itself. */
+void
+tor_cond_uninit(tor_cond_t *cond)
+{
+  if (pthread_cond_destroy(&cond->cond)) {
+    // LCOV_EXCL_START
+    log_warn(LD_GENERAL,"Error freeing condition: %s", strerror(errno));
+    return;
+    // LCOV_EXCL_STOP
+  }
+}
+/** Wait until one of the tor_cond_signal functions is called on <b>cond</b>.
+ * (If <b>tv</b> is set, and that amount of time passes with no signal to
+ * <b>cond</b>, return anyway.  All waiters on the condition must wait holding
+ * the same <b>mutex</b>.  All signallers should hold that mutex.  The mutex
+ * needs to have been allocated with tor_mutex_init_for_cond().
+ *
+ * Returns 0 on success, -1 on failure, 1 on timeout. */
+int
+tor_cond_wait(tor_cond_t *cond, tor_mutex_t *mutex, const struct timeval *tv)
+{
+  int r;
+  if (tv == NULL) {
+    while (1) {
+      r = pthread_cond_wait(&cond->cond, &mutex->mutex);
+      if (r == EINTR) {
+        /* EINTR should be impossible according to POSIX, but POSIX, like the
+         * Pirate's Code, is apparently treated "more like what you'd call
+         * guidelines than actual rules." */
+        continue; // LCOV_EXCL_LINE
+      }
+      return r ? -1 : 0;
+    }
+  } else {
+    struct timeval tvnow, tvsum;
+    struct timespec ts;
+    while (1) {
+#if defined(HAVE_CLOCK_GETTIME) && defined(USE_COND_CLOCK)
+      if (clock_gettime(USE_COND_CLOCK, &ts) < 0) {
+        return -1;
+      }
+      tvnow.tv_sec = ts.tv_sec;
+      tvnow.tv_usec = (int)(ts.tv_nsec / 1000);
+      timeradd(tv, &tvnow, &tvsum);
+#else /* !(defined(HAVE_CLOCK_GETTIME) && defined(USE_COND_CLOCK)) */
+      if (gettimeofday(&tvnow, NULL) < 0)
+        return -1;
+      timeradd(tv, &tvnow, &tvsum);
+#endif /* defined(HAVE_CLOCK_GETTIME) && defined(USE_COND_CLOCK) */
+
+      ts.tv_sec = tvsum.tv_sec;
+      ts.tv_nsec = tvsum.tv_usec * 1000;
+
+      r = pthread_cond_timedwait(&cond->cond, &mutex->mutex, &ts);
+      if (r == 0)
+        return 0;
+      else if (r == ETIMEDOUT)
+        return 1;
+      else if (r == EINTR)
+        continue;
+      else
+        return -1;
+    }
+  }
+}
+/** Wake up one of the waiters on <b>cond</b>. */
+void
+tor_cond_signal_one(tor_cond_t *cond)
+{
+  pthread_cond_signal(&cond->cond);
+}
+/** Wake up all of the waiters on <b>cond</b>. */
+void
+tor_cond_signal_all(tor_cond_t *cond)
+{
+  pthread_cond_broadcast(&cond->cond);
+}
+
+int
+tor_threadlocal_init(tor_threadlocal_t *threadlocal)
+{
+  int err = pthread_key_create(&threadlocal->key, NULL);
+  return err ? -1 : 0;
+}
+
+void
+tor_threadlocal_destroy(tor_threadlocal_t *threadlocal)
+{
+  pthread_key_delete(threadlocal->key);
+  memset(threadlocal, 0, sizeof(tor_threadlocal_t));
+}
+
+void *
+tor_threadlocal_get(tor_threadlocal_t *threadlocal)
+{
+  return pthread_getspecific(threadlocal->key);
+}
+
+void
+tor_threadlocal_set(tor_threadlocal_t *threadlocal, void *value)
+{
+  int err = pthread_setspecific(threadlocal->key, value);
+  tor_assert(err == 0);
+}
+
+/** Set up common structures for use by threading. */
+void
+tor_threads_init(void)
+{
+  if (!threads_initialized) {
+    tor_locking_init();
+    const int ret1 = pthread_attr_init(&attr_detached);
+    tor_assert(ret1 == 0);
+#ifndef PTHREAD_CREATE_DETACHED
+#define PTHREAD_CREATE_DETACHED 1
+#endif
+    const int ret2 =
+      pthread_attr_setdetachstate(&attr_detached, PTHREAD_CREATE_DETACHED);
+    tor_assert(ret2 == 0);
+    threads_initialized = 1;
+    set_main_thread();
+  }
+}
diff --git a/src/lib/thread/compat_threads.c b/src/lib/thread/compat_threads.c
new file mode 100644
index 0000000000..1d685b2c3e
--- /dev/null
+++ b/src/lib/thread/compat_threads.c
@@ -0,0 +1,109 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file compat_threads.c
+ *
+ * \brief Cross-platform threading and inter-thread communication logic.
+ *  (Platform-specific parts are written in the other compat_*threads
+ *  modules.)
+ */
+
+#include "orconfig.h"
+#include <stdlib.h>
+#include "lib/thread/threads.h"
+
+#include "lib/log/torlog.h"
+#include "lib/log/util_bug.h"
+
+/** Allocate and return a new condition variable. */
+tor_cond_t *
+tor_cond_new(void)
+{
+  tor_cond_t *cond = tor_malloc(sizeof(tor_cond_t));
+  if (BUG(tor_cond_init(cond)<0))
+    tor_free(cond); // LCOV_EXCL_LINE
+  return cond;
+}
+
+/** Free all storage held in <b>c</b>. */
+void
+tor_cond_free_(tor_cond_t *c)
+{
+  if (!c)
+    return;
+  tor_cond_uninit(c);
+  tor_free(c);
+}
+
+/** Identity of the "main" thread */
+static unsigned long main_thread_id = -1;
+
+/** Start considering the current thread to be the 'main thread'.  This has
+ * no effect on anything besides in_main_thread(). */
+void
+set_main_thread(void)
+{
+  main_thread_id = tor_get_thread_id();
+}
+/** Return true iff called from the main thread. */
+int
+in_main_thread(void)
+{
+  return main_thread_id == tor_get_thread_id();
+}
+
+#ifndef HAVE_STDATOMIC_H
+/** Initialize a new atomic counter with the value 0 */
+void
+atomic_counter_init(atomic_counter_t *counter)
+{
+  memset(counter, 0, sizeof(*counter));
+  tor_mutex_init_nonrecursive(&counter->mutex);
+}
+/** Clean up all resources held by an atomic counter. */
+void
+atomic_counter_destroy(atomic_counter_t *counter)
+{
+  tor_mutex_uninit(&counter->mutex);
+  memset(counter, 0, sizeof(*counter));
+}
+/** Add a value to an atomic counter. */
+void
+atomic_counter_add(atomic_counter_t *counter, size_t add)
+{
+  tor_mutex_acquire(&counter->mutex);
+  counter->val += add;
+  tor_mutex_release(&counter->mutex);
+}
+/** Subtract a value from an atomic counter. */
+void
+atomic_counter_sub(atomic_counter_t *counter, size_t sub)
+{
+  // this relies on unsigned overflow, but that's fine.
+  atomic_counter_add(counter, -sub);
+}
+/** Return the current value of an atomic counter */
+size_t
+atomic_counter_get(atomic_counter_t *counter)
+{
+  size_t val;
+  tor_mutex_acquire(&counter->mutex);
+  val = counter->val;
+  tor_mutex_release(&counter->mutex);
+  return val;
+}
+/** Replace the value of an atomic counter; return the old one. */
+size_t
+atomic_counter_exchange(atomic_counter_t *counter, size_t newval)
+{
+  size_t oldval;
+  tor_mutex_acquire(&counter->mutex);
+  oldval = counter->val;
+  counter->val = newval;
+  tor_mutex_release(&counter->mutex);
+  return oldval;
+}
+#endif /* !defined(HAVE_STDATOMIC_H) */
diff --git a/src/lib/thread/compat_winthreads.c b/src/lib/thread/compat_winthreads.c
new file mode 100644
index 0000000000..7f9877d21e
--- /dev/null
+++ b/src/lib/thread/compat_winthreads.c
@@ -0,0 +1,223 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file compat_winthreads.c
+ *
+ * \brief Implementation for the windows-based multithreading backend
+ * functions.
+ */
+
+#ifdef _WIN32
+
+#include <windows.h>
+#include <process.h>
+#include "lib/thread/threads.h"
+#include "lib/log/torlog.h"
+#include "lib/log/util_bug.h"
+#include "lib/log/win32err.h"
+
+/* This value is more or less total cargo-cult */
+#define SPIN_COUNT 2000
+
+/** Minimalist interface to run a void function in the background.  On
+ * Unix calls fork, on win32 calls beginthread.  Returns -1 on failure.
+ * func should not return, but rather should call spawn_exit.
+ *
+ * NOTE: if <b>data</b> is used, it should not be allocated on the stack,
+ * since in a multithreaded environment, there is no way to be sure that
+ * the caller's stack will still be around when the called function is
+ * running.
+ */
+int
+spawn_func(void (*func)(void *), void *data)
+{
+  int rv;
+  rv = (int)_beginthread(func, 0, data);
+  if (rv == (int)-1)
+    return -1;
+  return 0;
+}
+
+/** End the current thread/process.
+ */
+void
+spawn_exit(void)
+{
+  _endthread();
+  // LCOV_EXCL_START
+  //we should never get here. my compiler thinks that _endthread returns, this
+  //is an attempt to fool it.
+  tor_assert(0);
+  _exit(0); // exit ok: unreachable.
+  // LCOV_EXCL_STOP
+}
+
+unsigned long
+tor_get_thread_id(void)
+{
+  return (unsigned long)GetCurrentThreadId();
+}
+
+int
+tor_cond_init(tor_cond_t *cond)
+{
+  memset(cond, 0, sizeof(tor_cond_t));
+  if (InitializeCriticalSectionAndSpinCount(&cond->lock, SPIN_COUNT)==0) {
+    return -1;
+  }
+  if ((cond->event = CreateEvent(NULL,TRUE,FALSE,NULL)) == NULL) {
+    DeleteCriticalSection(&cond->lock);
+    return -1;
+  }
+  cond->n_waiting = cond->n_to_wake = cond->generation = 0;
+  return 0;
+}
+void
+tor_cond_uninit(tor_cond_t *cond)
+{
+  DeleteCriticalSection(&cond->lock);
+  CloseHandle(cond->event);
+}
+
+static void
+tor_cond_signal_impl(tor_cond_t *cond, int broadcast)
+{
+  EnterCriticalSection(&cond->lock);
+  if (broadcast)
+    cond->n_to_wake = cond->n_waiting;
+  else
+    ++cond->n_to_wake;
+  cond->generation++;
+  SetEvent(cond->event);
+  LeaveCriticalSection(&cond->lock);
+}
+void
+tor_cond_signal_one(tor_cond_t *cond)
+{
+  tor_cond_signal_impl(cond, 0);
+}
+void
+tor_cond_signal_all(tor_cond_t *cond)
+{
+  tor_cond_signal_impl(cond, 1);
+}
+
+int
+tor_threadlocal_init(tor_threadlocal_t *threadlocal)
+{
+  threadlocal->index = TlsAlloc();
+  return (threadlocal->index == TLS_OUT_OF_INDEXES) ? -1 : 0;
+}
+
+void
+tor_threadlocal_destroy(tor_threadlocal_t *threadlocal)
+{
+  TlsFree(threadlocal->index);
+  memset(threadlocal, 0, sizeof(tor_threadlocal_t));
+}
+
+void *
+tor_threadlocal_get(tor_threadlocal_t *threadlocal)
+{
+  void *value = TlsGetValue(threadlocal->index);
+  if (value == NULL) {
+    DWORD err = GetLastError();
+    if (err != ERROR_SUCCESS) {
+      char *msg = format_win32_error(err);
+      log_err(LD_GENERAL, "Error retrieving thread-local value: %s", msg);
+      tor_free(msg);
+      tor_assert(err == ERROR_SUCCESS);
+    }
+  }
+  return value;
+}
+
+void
+tor_threadlocal_set(tor_threadlocal_t *threadlocal, void *value)
+{
+  BOOL ok = TlsSetValue(threadlocal->index, value);
+  if (!ok) {
+    DWORD err = GetLastError();
+    char *msg = format_win32_error(err);
+    log_err(LD_GENERAL, "Error adjusting thread-local value: %s", msg);
+    tor_free(msg);
+    tor_assert(ok);
+  }
+}
+
+int
+tor_cond_wait(tor_cond_t *cond, tor_mutex_t *lock_, const struct timeval *tv)
+{
+  CRITICAL_SECTION *lock = &lock_->mutex;
+  int generation_at_start;
+  int waiting = 1;
+  int result = -1;
+  DWORD ms = INFINITE, ms_orig = INFINITE, startTime, endTime;
+  if (tv)
+    ms_orig = ms = tv->tv_sec*1000 + (tv->tv_usec+999)/1000;
+
+  EnterCriticalSection(&cond->lock);
+  ++cond->n_waiting;
+  generation_at_start = cond->generation;
+  LeaveCriticalSection(&cond->lock);
+
+  LeaveCriticalSection(lock);
+
+  startTime = GetTickCount();
+  do {
+    DWORD res;
+    res = WaitForSingleObject(cond->event, ms);
+    EnterCriticalSection(&cond->lock);
+    if (cond->n_to_wake &&
+        cond->generation != generation_at_start) {
+      --cond->n_to_wake;
+      --cond->n_waiting;
+      result = 0;
+      waiting = 0;
+      goto out;
+    } else if (res != WAIT_OBJECT_0) {
+      result = (res==WAIT_TIMEOUT) ? 1 : -1;
+      --cond->n_waiting;
+      waiting = 0;
+      goto out;
+    } else if (ms != INFINITE) {
+      endTime = GetTickCount();
+      if (startTime + ms_orig <= endTime) {
+        result = 1; /* Timeout */
+        --cond->n_waiting;
+        waiting = 0;
+        goto out;
+      } else {
+        ms = startTime + ms_orig - endTime;
+      }
+    }
+    /* If we make it here, we are still waiting. */
+    if (cond->n_to_wake == 0) {
+      /* There is nobody else who should wake up; reset
+       * the event. */
+      ResetEvent(cond->event);
+    }
+  out:
+    LeaveCriticalSection(&cond->lock);
+  } while (waiting);
+
+  EnterCriticalSection(lock);
+
+  EnterCriticalSection(&cond->lock);
+  if (!cond->n_waiting)
+    ResetEvent(cond->event);
+  LeaveCriticalSection(&cond->lock);
+
+  return result;
+}
+
+void
+tor_threads_init(void)
+{
+  set_main_thread();
+}
+
+#endif /* defined(_WIN32) */
diff --git a/src/lib/thread/include.am b/src/lib/thread/include.am
new file mode 100644
index 0000000000..0eac7edeeb
--- /dev/null
+++ b/src/lib/thread/include.am
@@ -0,0 +1,25 @@
+
+noinst_LIBRARIES += src/lib/libtor-thread.a
+
+if UNITTESTS_ENABLED
+noinst_LIBRARIES += src/lib/libtor-thread-testing.a
+endif
+
+if THREADS_PTHREADS
+threads_impl_source=src/lib/thread/compat_pthreads.c
+endif
+if THREADS_WIN32
+threads_impl_source=src/lib/thread/compat_winthreads.c
+endif
+
+src_lib_libtor_thread_a_SOURCES =			\
+	src/lib/thread/compat_threads.c			\
+	$(threads_impl_source)
+
+src_lib_libtor_thread_testing_a_SOURCES = \
+	$(src_lib_libtor_thread_a_SOURCES)
+src_lib_libtor_thread_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
+src_lib_libtor_thread_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
+
+noinst_HEADERS +=					\
+	src/lib/thread/threads.h
diff --git a/src/lib/thread/threads.h b/src/lib/thread/threads.h
new file mode 100644
index 0000000000..fcccc643d5
--- /dev/null
+++ b/src/lib/thread/threads.h
@@ -0,0 +1,159 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_COMPAT_THREADS_H
+#define TOR_COMPAT_THREADS_H
+
+#include "orconfig.h"
+#include "lib/cc/torint.h"
+#include "lib/testsupport/testsupport.h"
+#include "lib/lock/compat_mutex.h"
+
+#ifdef HAVE_STDATOMIC_H
+#include <stdatomic.h>
+#endif
+
+struct timeval;
+
+int spawn_func(void (*func)(void *), void *data);
+void spawn_exit(void) ATTR_NORETURN;
+
+unsigned long tor_get_thread_id(void);
+void tor_threads_init(void);
+
+/** Conditions need nonrecursive mutexes with pthreads. */
+#define tor_mutex_init_for_cond(m) tor_mutex_init_nonrecursive(m)
+
+void set_main_thread(void);
+int in_main_thread(void);
+
+typedef struct tor_cond_t {
+#ifdef USE_PTHREADS
+  pthread_cond_t cond;
+#elif defined(USE_WIN32_THREADS)
+  HANDLE event;
+
+  CRITICAL_SECTION lock;
+  int n_waiting;
+  int n_to_wake;
+  int generation;
+#else
+#error no known condition implementation.
+#endif /* defined(USE_PTHREADS) || ... */
+} tor_cond_t;
+
+tor_cond_t *tor_cond_new(void);
+void tor_cond_free_(tor_cond_t *cond);
+#define tor_cond_free(c) FREE_AND_NULL(tor_cond_t, tor_cond_free_, (c))
+int tor_cond_init(tor_cond_t *cond);
+void tor_cond_uninit(tor_cond_t *cond);
+int tor_cond_wait(tor_cond_t *cond, tor_mutex_t *mutex,
+                  const struct timeval *tv);
+void tor_cond_signal_one(tor_cond_t *cond);
+void tor_cond_signal_all(tor_cond_t *cond);
+
+typedef struct tor_threadlocal_s {
+#ifdef _WIN32
+  DWORD index;
+#else
+  pthread_key_t key;
+#endif
+} tor_threadlocal_t;
+
+/** Initialize a thread-local variable.
+ *
+ * After you call this function on a tor_threadlocal_t, you can call
+ * tor_threadlocal_set to change the current value of this variable for the
+ * current thread, and tor_threadlocal_get to retrieve the current value for
+ * the current thread.  Each thread has its own value.
+ **/
+int tor_threadlocal_init(tor_threadlocal_t *threadlocal);
+/**
+ * Release all resource associated with a thread-local variable.
+ */
+void tor_threadlocal_destroy(tor_threadlocal_t *threadlocal);
+/**
+ * Return the current value of a thread-local variable for this thread.
+ *
+ * It's undefined behavior to use this function if the threadlocal hasn't
+ * been initialized, or has been destroyed.
+ */
+void *tor_threadlocal_get(tor_threadlocal_t *threadlocal);
+/**
+ * Change the current value of a thread-local variable for this thread to
+ * <b>value</b>.
+ *
+ * It's undefined behavior to use this function if the threadlocal hasn't
+ * been initialized, or has been destroyed.
+ */
+void tor_threadlocal_set(tor_threadlocal_t *threadlocal, void *value);
+
+/**
+ * Atomic counter type; holds a size_t value.
+ */
+#ifdef HAVE_STDATOMIC_H
+typedef struct atomic_counter_t {
+  atomic_size_t val;
+} atomic_counter_t;
+#define ATOMIC_LINKAGE static
+#else /* !(defined(HAVE_STDATOMIC_H)) */
+typedef struct atomic_counter_t {
+  tor_mutex_t mutex;
+  size_t val;
+} atomic_counter_t;
+#define ATOMIC_LINKAGE
+#endif /* defined(HAVE_STDATOMIC_H) */
+
+ATOMIC_LINKAGE void atomic_counter_init(atomic_counter_t *counter);
+ATOMIC_LINKAGE void atomic_counter_destroy(atomic_counter_t *counter);
+ATOMIC_LINKAGE void atomic_counter_add(atomic_counter_t *counter, size_t add);
+ATOMIC_LINKAGE void atomic_counter_sub(atomic_counter_t *counter, size_t sub);
+ATOMIC_LINKAGE size_t atomic_counter_get(atomic_counter_t *counter);
+ATOMIC_LINKAGE size_t atomic_counter_exchange(atomic_counter_t *counter,
+                                              size_t newval);
+#undef ATOMIC_LINKAGE
+
+#ifdef HAVE_STDATOMIC_H
+/** Initialize a new atomic counter with the value 0 */
+static inline void
+atomic_counter_init(atomic_counter_t *counter)
+{
+  atomic_init(&counter->val, 0);
+}
+/** Clean up all resources held by an atomic counter. */
+static inline void
+atomic_counter_destroy(atomic_counter_t *counter)
+{
+  (void)counter;
+}
+/** Add a value to an atomic counter. */
+static inline void
+atomic_counter_add(atomic_counter_t *counter, size_t add)
+{
+  (void) atomic_fetch_add(&counter->val, add);
+}
+/** Subtract a value from an atomic counter. */
+static inline void
+atomic_counter_sub(atomic_counter_t *counter, size_t sub)
+{
+  (void) atomic_fetch_sub(&counter->val, sub);
+}
+/** Return the current value of an atomic counter */
+static inline size_t
+atomic_counter_get(atomic_counter_t *counter)
+{
+  return atomic_load(&counter->val);
+}
+/** Replace the value of an atomic counter; return the old one. */
+static inline size_t
+atomic_counter_exchange(atomic_counter_t *counter, size_t newval)
+{
+  return atomic_exchange(&counter->val, newval);
+}
+
+#else /* !(defined(HAVE_STDATOMIC_H)) */
+#endif /* defined(HAVE_STDATOMIC_H) */
+
+#endif /* !defined(TOR_COMPAT_THREADS_H) */