Merge pull request #584 from trapexit/timeouts

make attr, entry, negative_entry cache timeouts runtime configurable

README.md

@@ -1,6 +1,6 @@
 % mergerfs(1) mergerfs user manual
 % Antonio SJ Musumeci <trapexit@spawn.link>
-% 2019-02-17
+% 2019-02-22
 
 # NAME
 
@@ -85,6 +85,9 @@ mergerfs does **not** support the copy-on-write (CoW) behavior found in **aufs**
 * **category.&lt;category&gt;=&lt;policy&gt;**: Sets policy of all FUSE functions in the provided category. Example: **category.create=mfs**
 * **cache.open=&lt;int&gt;**: 'open' policy cache timeout in seconds. (default: 0)
 * **cache.statfs=&lt;int&gt;**: 'statfs' cache timeout in seconds. (default: 0)
+* **cache.attr=&lt;int&gt;**: file attribute cache timeout in seconds. (default: 1)
+* **cache.entry=&lt;int&gt;**: file name lookup cache timeout in seconds. (default: 1)
+* **cache.negative_entry=&lt;int&gt;**: negative file name lookup cache timeout in seconds. (default: 0)
 
 **NOTE:** Options are evaluated in the order listed so if the options are **func.rmdir=rand,category.action=ff** the **action** category setting will override the **rmdir** setting.
 

libfuse/include/fuse.h

@@ -701,7 +701,17 @@ int fuse_loop(struct fuse *f);
  */
 void fuse_exit(struct fuse *f);
 
-int fuse_config_num_threads(const struct fuse *f);
+void fuse_config_set_entry_timeout(struct fuse  *fuse_,
+                                   const double  entry_timeout_);
+void fuse_config_set_negative_entry_timeout(struct fuse  *fuse_,
+                                            const double  entry_timeout_);
+void fuse_config_set_attr_timeout(struct fuse  *fuse_,
+                                  const double  attr_timeout_);
+
+int    fuse_config_num_threads(const struct fuse *fuse_);
+double fuse_config_get_entry_timeout(const struct fuse *fuse_);
+double fuse_config_get_negative_entry_timeout(const struct fuse *fuse_);
+double fuse_config_get_attr_timeout(const struct fuse *fuse_);
 
 /**
  * FUSE event loop with multiple threads

libfuse/lib/fuse.c

@@ -4723,7 +4723,47 @@ struct fuse *fuse_new_compat25(int fd, struct fuse_args *args,
 
 FUSE_SYMVER(".symver fuse_new_compat25,fuse_new@FUSE_2.5");
 
-int fuse_config_num_threads(const struct fuse *f)
+int
+fuse_config_num_threads(const struct fuse *fuse_)
 {
-  return f->conf.threads;
+  return fuse_->conf.threads;
+}
+
+void
+fuse_config_set_entry_timeout(struct fuse  *fuse_,
+                              const double  entry_timeout_)
+{
+  fuse_->conf.entry_timeout = entry_timeout_;
+}
+
+double
+fuse_config_get_entry_timeout(const struct fuse *fuse_)
+{
+  return fuse_->conf.entry_timeout;
+}
+
+void
+fuse_config_set_negative_entry_timeout(struct fuse *fuse_,
+                                       const double entry_timeout_)
+{
+  fuse_->conf.negative_timeout = entry_timeout_;
+}
+
+double
+fuse_config_get_negative_entry_timeout(const struct fuse *fuse_)
+{
+  return fuse_->conf.negative_timeout;
+}
+
+void
+fuse_config_set_attr_timeout(struct fuse  *fuse_,
+                             const double  attr_timeout_)
+{
+  fuse_->conf.attr_timeout = attr_timeout_;
+}
+
+double
+fuse_config_get_attr_timeout(const struct fuse *fuse_)
+{
+  return fuse_->conf.attr_timeout;
 }

man/mergerfs.1

@@ -1,7 +1,7 @@
 .\"t
 .\" Automatically generated by Pandoc 1.19.2.4
 .\"
-.TH "mergerfs" "1" "2019\-02\-17" "mergerfs user manual" ""
+.TH "mergerfs" "1" "2019\-02\-22" "mergerfs user manual" ""
 .hy
 .SH NAME
 .PP
@@ -213,6 +213,16 @@ seconds.
 .IP \[bu] 2
 \f[B]cache.statfs=<int>\f[]: \[aq]statfs\[aq] cache timeout in seconds.
 (default: 0)
+.IP \[bu] 2
+\f[B]cache.attr=<int>\f[]: file attribute cache timeout in seconds.
+(default: 1)
+.IP \[bu] 2
+\f[B]cache.entry=<int>\f[]: file name lookup cache timeout in seconds.
+(default: 1)
+.IP \[bu] 2
+\f[B]cache.negative_entry=<int>\f[]: negative file name lookup cache
+timeout in seconds.
+(default: 0)
 .PP
 \f[B]NOTE:\f[] Options are evaluated in the order listed so if the
 options are \f[B]func.rmdir=rand,category.action=ff\f[] the

src/fuse_getxattr.cpp

@@ -130,6 +130,18 @@ namespace l
     attrvalue_ = os.str();
   }
 
+  static
+  void
+  getxattr_controlfile_double(const double  d_,
+                              string       &attrvalue_)
+  {
+    std::ostringstream os;
+
+    os << d_;
+
+    attrvalue_ = os.str();
+  }
+
   static
   void
   getxattr_controlfile_time_t(const time_t  time,
@@ -236,7 +248,7 @@ namespace l
 
   static
   void
-  getxattr_pid(string &attrvalue)
+  getxattr_controlfile_pid(string &attrvalue)
   {
     int pid;
     char buf[32];
@@ -247,6 +259,39 @@ namespace l
     attrvalue = buf;
   }
 
+  static
+  void
+  getxattr_controlfile_cache_attr(string &attrvalue)
+  {
+    double d;
+
+    d = fuse_config_get_attr_timeout(fuse_get_context()->fuse);
+
+    l::getxattr_controlfile_double(d,attrvalue);
+  }
+
+  static
+  void
+  getxattr_controlfile_cache_entry(string &attrvalue)
+  {
+    double d;
+
+    d = fuse_config_get_entry_timeout(fuse_get_context()->fuse);
+
+    l::getxattr_controlfile_double(d,attrvalue);
+  }
+
+  static
+  void
+  getxattr_controlfile_cache_negative_entry(string &attrvalue)
+  {
+    double d;
+
+    d = fuse_config_get_negative_entry_timeout(fuse_get_context()->fuse);
+
+    l::getxattr_controlfile_double(d,attrvalue);
+  }
+
   static
   int
   getxattr_controlfile(const Config &config,
@@ -298,7 +343,7 @@ namespace l
         else if(attr[2] == "version")
           l::getxattr_controlfile_version(attrvalue);
         else if(attr[2] == "pid")
-          l::getxattr_pid(attrvalue);
+          l::getxattr_controlfile_pid(attrvalue);
         else if(attr[2] == "direct_io")
           l::getxattr_controlfile_bool(config.direct_io,attrvalue);
         break;
@@ -312,6 +357,12 @@ namespace l
           l::getxattr_controlfile_uint64_t(config.open_cache.timeout,attrvalue);
         else if((attr[2] == "cache") && (attr[3] == "statfs"))
           l::getxattr_controlfile_uint64_t(fs::statvfs_cache_timeout(),attrvalue);
+        else if((attr[2] == "cache") && (attr[3] == "attr"))
+          l::getxattr_controlfile_cache_attr(attrvalue);
+        else if((attr[2] == "cache") && (attr[3] == "entry"))
+          l::getxattr_controlfile_cache_entry(attrvalue);
+        else if((attr[2] == "cache") && (attr[3] == "negative_entry"))
+          l::getxattr_controlfile_cache_negative_entry(attrvalue);
         break;
       }
 

src/fuse_listxattr.cpp

@@ -45,6 +45,9 @@ namespace l
     const vector<string> strs =
       buildvector<string>
       ("user.mergerfs.branches")
+      ("user.mergerfs.cache.attr")
+      ("user.mergerfs.cache.entry")
+      ("user.mergerfs.cache.negative_entry")
       ("user.mergerfs.cache.open")
       ("user.mergerfs.cache.statfs")
       ("user.mergerfs.direct_io")

src/fuse_setxattr.cpp

@@ -119,6 +119,24 @@ namespace l
     return 0;
   }
 
+  static
+  int
+  setxattr_double(const string &attrval_,
+                  const int     flags_,
+                  double       *d_)
+  {
+    int rv;
+
+    if((flags_ & XATTR_CREATE) == XATTR_CREATE)
+      return -EEXIST;
+
+    rv = num::to_double(attrval_,d_);
+    if(rv == -1)
+      return -EINVAL;
+
+    return 0;
+  }
+
   static
   int
   setxattr_time_t(const string &attrval,
@@ -276,6 +294,51 @@ namespace l
     return rv;
   }
 
+  static
+  int
+  setxattr_controlfile_cache_attr(const string &attrval_,
+                                  const int     flags_)
+  {
+    int rv;
+    double d;
+
+    rv = l::setxattr_double(attrval_,flags_,&d);
+    if(rv >= 0)
+      fuse_config_set_attr_timeout(fuse_get_context()->fuse,d);
+
+    return rv;
+  }
+
+  static
+  int
+  setxattr_controlfile_cache_entry(const string &attrval_,
+                                   const int     flags_)
+  {
+    int rv;
+    double d;
+
+    rv = l::setxattr_double(attrval_,flags_,&d);
+    if(rv >= 0)
+      fuse_config_set_entry_timeout(fuse_get_context()->fuse,d);
+
+    return rv;
+  }
+
+  static
+  int
+  setxattr_controlfile_cache_negative_entry(const string &attrval_,
+                                            const int     flags_)
+  {
+    int rv;
+    double d;
+
+    rv = l::setxattr_double(attrval_,flags_,&d);
+    if(rv >= 0)
+      fuse_config_set_negative_entry_timeout(fuse_get_context()->fuse,d);
+
+    return rv;
+  }
+
   static
   int
   setxattr_controlfile(Config       &config,
@@ -367,6 +430,12 @@ namespace l
                                       config.open_cache.timeout);
         else if((attr[2] == "cache") && (attr[3] == "statfs"))
           return l::setxattr_statfs_timeout(attrval,flags);
+        else if((attr[2] == "cache") && (attr[3] == "attr"))
+          return l::setxattr_controlfile_cache_attr(attrval,flags);
+        else if((attr[2] == "cache") && (attr[3] == "entry"))
+          return l::setxattr_controlfile_cache_entry(attrval,flags);
+        else if((attr[2] == "cache") && (attr[3] == "negative_entry"))
+          return l::setxattr_controlfile_cache_negative_entry(attrval,flags);
         break;
 
       default:

src/mergerfs.cpp

@@ -158,7 +158,7 @@ namespace l
     args.argv      = argv_;
     args.allocated = 0;
 
-    options::parse(args,config);
+    options::parse(&args,&config);
 
     l::setup_resources();
     l::get_fuse_operations(ops,config.nullrw);

src/num.cpp

@@ -47,6 +47,11 @@ namespace num
         tmp *= (1024 * 1024 * 1024);
         break;
 
+      case 't':
+      case 'T':
+        tmp *= (1024ULL * 1024 * 1024 * 1024);
+        break;
+
       case '\0':
         break;
 
@@ -59,6 +64,22 @@ namespace num
     return 0;
   }
 
+  int
+  to_double(const std::string &str_,
+            double            *d_)
+  {
+    double tmp;
+    char *endptr;
+
+    tmp = strtod(str_.c_str(),&endptr);
+    if(*endptr != '\0')
+      return -1;
+
+    *d_ = tmp;
+
+    return 0;
+  }
+
   int
   to_time_t(const std::string &str,
             time_t            &value)

src/num.hpp

@@ -22,6 +22,7 @@
 
 namespace num
 {
-  int to_uint64_t(const std::string &str, uint64_t &value);
-  int to_time_t(const std::string &str, time_t &value);
+  int to_uint64_t(const std::string &str_, uint64_t &value_);
+  int to_double(const std::string &str_, double *value_);
+  int to_time_t(const std::string &str_, time_t &value_);
 }

src/option_parser.cpp

@@ -48,17 +48,17 @@ enum
 
 static
 void
-set_option(fuse_args         &args,
+set_option(fuse_args         *args,
            const std::string &option_)
 {
 
-  fuse_opt_add_arg(&args,"-o");
-  fuse_opt_add_arg(&args,option_.c_str());
+  fuse_opt_add_arg(args,"-o");
+  fuse_opt_add_arg(args,option_.c_str());
 }
 
 static
 void
-set_kv_option(fuse_args         &args,
+set_kv_option(fuse_args         *args,
               const std::string &key,
               const std::string &value)
 {
@@ -71,7 +71,7 @@ set_kv_option(fuse_args         &args,
 
 static
 void
-set_fsname(fuse_args      &args,
+set_fsname(fuse_args      *args,
            const Branches &branches_)
 {
   vector<string> branches;
@@ -90,14 +90,14 @@ set_fsname(fuse_args      &args,
 
 static
 void
-set_subtype(fuse_args &args)
+set_subtype(fuse_args *args)
 {
   set_kv_option(args,"subtype","mergerfs");
 }
 
 static
 void
-set_default_options(fuse_args &args)
+set_default_options(fuse_args *args)
 {
   set_option(args,"atomic_o_trunc");
   set_option(args,"big_writes");
@@ -216,12 +216,19 @@ static
 int
 parse_and_process_cache(Config       &config_,
                         const string &func_,
-                        const string &value_)
+                        const string &value_,
+                        fuse_args    *outargs)
 {
   if(func_ == "open")
     return parse_and_process(value_,config_.open_cache.timeout);
   else if(func_ == "statfs")
     return parse_and_process_statfs_cache(value_);
+  else if(func_ == "entry")
+    return (set_kv_option(outargs,"entry_timeout",value_),0);
+  else if(func_ == "negative_entry")
+    return (set_kv_option(outargs,"negative_timeout",value_),0);
+  else if(func_ == "attr")
+    return (set_kv_option(outargs,"attr_timeout",value_),0);
 
   return 1;
 }
@@ -229,8 +236,7 @@ parse_and_process_cache(Config       &config_,
 static
 int
 parse_and_process_arg(Config            &config,
-                      const std::string &arg,
-                      fuse_args         *outargs)
+                      const std::string &arg)
 {
   if(arg == "defaults")
     return 0;
@@ -244,7 +250,8 @@ static
 int
 parse_and_process_kv_arg(Config            &config,
                          const std::string &key,
-                         const std::string &value)
+                         const std::string &value,
+                         fuse_args         *outargs)
 {
   int rv;
   std::vector<std::string> keypart;
@@ -258,7 +265,7 @@ parse_and_process_kv_arg(Config            &config,
       else if(keypart[0] == "category")
         rv = config.set_category_policy(keypart[1],value);
       else if(keypart[0] == "cache")
-        rv = parse_and_process_cache(config,keypart[1],value);
+        rv = parse_and_process_cache(config,keypart[1],value,outargs);
     }
   else
     {
@@ -307,11 +314,11 @@ process_opt(Config            &config,
   switch(argvalue.size())
     {
     case 1:
-      rv = parse_and_process_arg(config,argvalue[0],outargs);
+      rv = parse_and_process_arg(config,argvalue[0]);
       break;
 
     case 2:
-      rv = parse_and_process_kv_arg(config,argvalue[0],argvalue[1]);
+      rv = parse_and_process_kv_arg(config,argvalue[0],argvalue[1],outargs);
       break;
 
     default:
@@ -362,6 +369,13 @@ usage(void)
     "                           default = 0 (disabled)\n"
     "    -o cache.statfs=<int>  'statfs' cache timeout in seconds. Used by\n"
     "                           policies. default = 0 (disabled)\n"
+    "    -o cache.attr=<int>    file attribute cache timeout in seconds.\n"
+    "                           default = 1\n"
+    "    -o cache.entry=<int>   file name lookup cache timeout in seconds.\n"
+    "                           default = 1\n"
+    "    -o cache.negative_entry=<int>\n"
+    "                           negative file name lookup cache timeout in\n"
+    "                           seconds. default = 0\n"
     "    -o direct_io           Bypass page caching, may increase write\n"
     "                           speeds at the cost of reads. Please read docs\n"
     "                           for more details as there are tradeoffs.\n"
@@ -456,8 +470,8 @@ option_processor(void       *data,
 namespace options
 {
   void
-  parse(fuse_args &args,
-        Config    &config)
+  parse(fuse_args *args,
+        Config    *config)
   {
     const struct fuse_opt opts[] =
       {
@@ -469,13 +483,13 @@ namespace options
         {NULL,-1U,0}
       };
 
-    fuse_opt_parse(&args,
-                   &config,
+    fuse_opt_parse(args,
+                   config,
                    opts,
                    ::option_processor);
 
     set_default_options(args);
-    set_fsname(args,config.branches);
+    set_fsname(args,config->branches);
     set_subtype(args);
   }
 }

src/option_parser.hpp

@@ -23,6 +23,6 @@
 namespace options
 {
   void
-  parse(fuse_args &args,
-        Config    &config);
+  parse(fuse_args *args,
+        Config    *config);
 }