Merge pull request #1227 from trapexit/gidcache

Add ability to invalidate gid cache on demand

libfuse/lib/fuse.c

@@ -753,7 +753,6 @@ find_node(struct fuse *f,
       if(f->conf.remember)
         inc_nlookup(node);
 
-      printf("hash_name = %s\n",name);
       if(hash_name(f,node,parent,name) == -1)
         {
           free_node(f,node);

src/fuse_ioctl.cpp

@@ -24,6 +24,7 @@
 #include "fs_ioctl.hpp"
 #include "fs_open.hpp"
 #include "fs_path.hpp"
+#include "gidcache.hpp"
 #include "str.hpp"
 #include "ugid.hpp"
 
@@ -46,6 +47,8 @@ typedef char IOCTL_BUF[4096];
 #define IOCTL_GC                   _IO(IOCTL_APP_TYPE,1)
 #define IOCTL_GC1                  _IO(IOCTL_APP_TYPE,2)
 #define IOCTL_INVALIDATE_ALL_NODES _IO(IOCTL_APP_TYPE,3)
+#define IOCTL_INVALIDATE_GID_CACHE _IO(IOCTL_APP_TYPE,4)
+
 
 // From linux/btrfs.h
 #define BTRFS_IOCTL_MAGIC 0x94
@@ -346,6 +349,9 @@ namespace l
       case IOCTL_INVALIDATE_ALL_NODES:
         fuse_invalidate_all_nodes();
         return 0;
+      case IOCTL_INVALIDATE_GID_CACHE:
+        GIDCache::invalidate_all_caches();
+        break;
       }
 
     return -ENOTTY;

src/gidcache.cpp

@@ -26,35 +26,59 @@
 # include <sys/param.h>
 #endif
 
+#include <cassert>
 #include <cstdlib>
 #include <algorithm>
+#include <unordered_map>
+#include <mutex>
+
 
 #include "gidcache.hpp"
 
+std::mutex                          g_REGISTERED_CACHES_MUTEX;
+std::unordered_map<pid_t,GIDCache*> g_REGISTERED_CACHES;
+
+
 inline
 bool
-gid_t_rec::operator<(const struct gid_t_rec &b) const
+GIDRecord::operator<(const struct GIDRecord &b) const
 {
   return uid < b.uid;
 }
 
+GIDCache::GIDCache()
+  : invalidate(false),
+    size(0),
+    recs()
+{
+  std::lock_guard<std::mutex> guard(g_REGISTERED_CACHES_MUTEX);
+
+  pid_t tid;
+  bool  inserted;
+
+  tid = ::gettid();
+  inserted = g_REGISTERED_CACHES.emplace(tid,this).second;
+
+  assert(inserted == true);
+}
+
 inline
-gid_t_rec *
-gid_t_cache::begin(void)
+GIDRecord *
+GIDCache::begin(void)
 {
-  return recs;
+  return &recs[0];
 }
 
 inline
-gid_t_rec *
-gid_t_cache::end(void)
+GIDRecord *
+GIDCache::end(void)
 {
-  return recs + size;
+  return &recs[size];
 }
 
 inline
-gid_t_rec *
-gid_t_cache::allocrec(void)
+GIDRecord *
+GIDCache::allocrec(void)
 {
   if(size == MAXRECS)
     return &recs[rand() % MAXRECS];
@@ -63,14 +87,14 @@ gid_t_cache::allocrec(void)
 }
 
 inline
-gid_t_rec *
-gid_t_cache::lower_bound(gid_t_rec   *begin,
-                         gid_t_rec   *end,
+GIDRecord *
+GIDCache::lower_bound(GIDRecord   *begin,
+                      GIDRecord   *end,
                       const uid_t  uid)
 {
   int step;
   int count;
-  gid_t_rec *iter;
+  GIDRecord *iter;
 
   count = std::distance(begin,end);
   while(count > 0)
@@ -106,15 +130,15 @@ _getgrouplist(const char  *user,
 #endif
 }
 
-gid_t_rec *
-gid_t_cache::cache(const uid_t uid,
+GIDRecord *
+GIDCache::cache(const uid_t uid,
                 const gid_t gid)
 {
   int rv;
   char buf[4096];
   struct passwd pwd;
   struct passwd *pwdrv;
-  gid_t_rec *rec;
+  GIDRecord *rec;
 
   rec = allocrec();
 
@@ -139,7 +163,7 @@ gid_t_cache::cache(const uid_t uid,
 static
 inline
 int
-setgroups(const gid_t_rec *rec)
+setgroups(const GIDRecord *rec)
 {
 #if defined __linux__ and UGID_USE_RWLOCK == 0
 # if defined SYS_setgroups32
@@ -153,11 +177,17 @@ setgroups(const gid_t_rec *rec)
 }
 
 int
-gid_t_cache::initgroups(const uid_t uid,
+GIDCache::initgroups(const uid_t uid,
                      const gid_t gid)
 {
   int rv;
-  gid_t_rec *rec;
+  GIDRecord *rec;
+
+  if(invalidate)
+    {
+      size       = 0;
+      invalidate = false;
+    }
 
   rec = lower_bound(begin(),end(),uid);
   if(rec == end() || rec->uid != uid)
@@ -173,3 +203,12 @@ gid_t_cache::initgroups(const uid_t uid,
 
   return rv;
 }
+
+void
+GIDCache::invalidate_all_caches()
+{
+  std::lock_guard<std::mutex> guard(g_REGISTERED_CACHES_MUTEX);
+
+  for(auto &p : g_REGISTERED_CACHES)
+    p.second->invalidate = true;
+}

src/gidcache.hpp

@@ -19,37 +19,55 @@
 #include <sys/types.h>
 #include <unistd.h>
 
+#include <array>
+
 #define MAXGIDS 32
 #define MAXRECS 256
 
-struct gid_t_rec
+// GIDCache is a global, per thread cache of uid to gid + supplemental
+// groups mapping for use when threads change credentials. This is
+// needed due to the high cost of querying such information. The cache
+// instance should always be thread local and live the lifetime of the
+// app. The constructor will register the instance so they can each be
+// told to invalidate the cache on demand. A second instance on the
+// same thread will cause an assert to be triggered.
+
+
+struct GIDRecord
 {
   uid_t uid;
   int   size;
   gid_t gids[MAXGIDS];
 
   bool
-  operator<(const struct gid_t_rec &b) const;
+  operator<(const struct GIDRecord &b) const;
 };
 
-struct gid_t_cache
+struct GIDCache
 {
 public:
+  GIDCache();
+
+public:
+  bool   invalidate;
   size_t size;
-  gid_t_rec  recs[MAXRECS];
+  std::array<GIDRecord,MAXRECS> recs;
 
 private:
-  gid_t_rec * begin(void);
-  gid_t_rec * end(void);
-  gid_t_rec * allocrec(void);
-  gid_t_rec * lower_bound(gid_t_rec   *begin,
-                          gid_t_rec   *end,
+  GIDRecord *begin(void);
+  GIDRecord *end(void);
+  GIDRecord *allocrec(void);
+  GIDRecord *lower_bound(GIDRecord   *begin,
+                         GIDRecord   *end,
                          const uid_t  uid);
-  gid_t_rec * cache(const uid_t uid,
+  GIDRecord *cache(const uid_t uid,
                    const gid_t gid);
 
 public:
   int
   initgroups(const uid_t uid,
              const gid_t gid);
+
+public:
+  static void invalidate_all_caches();
 };

src/mergerfs.cpp

@@ -25,6 +25,7 @@
 #include "procfs_get_name.hpp"
 #include "resources.hpp"
 #include "strvec.hpp"
+#include "gidcache.hpp"
 
 #include "fuse_access.hpp"
 #include "fuse_bmap.hpp"
@@ -218,6 +219,7 @@ namespace l
   {
     syslog_info("Received SIGUSR2 - triggering thorough gc");
     fuse_gc();
+    GIDCache::invalidate_all_caches();
   }
 
   static

src/ugid.cpp

@@ -28,7 +28,7 @@ namespace ugid
   initgroups(const uid_t uid_,
              const gid_t gid_)
   {
-    static thread_local gid_t_cache cache = {0};
+    static thread_local GIDCache cache;
 
     cache.initgroups(uid_,gid_);
   }

src/ugid_linux.hpp

@@ -53,9 +53,9 @@
 
 namespace ugid
 {
-  extern __thread uid_t currentuid;
-  extern __thread gid_t currentgid;
-  extern __thread bool  initialized;
+  extern thread_local uid_t currentuid;
+  extern thread_local gid_t currentgid;
+  extern thread_local bool  initialized;
 
   struct Set
   {

src/ugid_linux.icpp

@@ -23,9 +23,9 @@
 
 namespace ugid
 {
-  __thread uid_t currentuid  = 0;
-  __thread gid_t currentgid  = 0;
-  __thread bool  initialized = false;
+  thread_local uid_t currentuid  = 0;
+  thread_local gid_t currentgid  = 0;
+  thread_local bool  initialized = false;
 
   void
   init()