From 5f737cb7bfa9f8fbfe5c323b1d78e6cebc0a49e2 Mon Sep 17 00:00:00 2001 From: Antonio SJ Musumeci Date: Sun, 10 Oct 2021 18:37:14 -0400 Subject: [PATCH] Add option to log node memory usage metrics --- libfuse/include/fuse.h | 2 ++ libfuse/lib/fmp.h | 8 +++++ libfuse/lib/fuse.c | 71 ++++++++++++++++++++++++++++++++++++++++++ libfuse/lib/lfmp.h | 14 +++++++++ src/fuse_ioctl.cpp | 48 +++++++++++++++++++++++----- 5 files changed, 136 insertions(+), 7 deletions(-) diff --git a/libfuse/include/fuse.h b/libfuse/include/fuse.h index 74691926..2f837e82 100644 --- a/libfuse/include/fuse.h +++ b/libfuse/include/fuse.h @@ -694,6 +694,8 @@ int fuse_main_real(int argc, char *argv[], const struct fuse_operations *op, siz int fuse_start_maintenance_thread(struct fuse *fuse); void fuse_stop_maintenance_thread(struct fuse *fuse); +void fuse_log_metrics(int enabled); + /** * Iterate over cache removing stale entries * use in conjunction with "-oremember" diff --git a/libfuse/lib/fmp.h b/libfuse/lib/fmp.h index dfa8057e..e1131e37 100644 --- a/libfuse/lib/fmp.h +++ b/libfuse/lib/fmp.h @@ -354,3 +354,11 @@ fmp_slab_usage_ratio(fmp_t *fmp_) return rv; } + +static +inline +uint64_t +fmp_total_allocated_memory(fmp_t *fmp_) +{ + return (fmp_->slab_size * kv_size(fmp_->slabs)); +} diff --git a/libfuse/lib/fuse.c b/libfuse/lib/fuse.c index cedd1d61..9f87cc24 100644 --- a/libfuse/lib/fuse.c +++ b/libfuse/lib/fuse.c @@ -53,6 +53,8 @@ #define NODE_TABLE_MIN_SIZE 8192 +static int g_LOG_METRICS = 0; + struct fuse_config { unsigned int uid; @@ -4006,6 +4008,64 @@ node_table_init(struct node_table *t) return 0; } +static +void +metrics_log_nodes_info(struct fuse *f_, + FILE *file_) +{ + pthread_mutex_lock(&f_->lock); + + fprintf(file_, + "time: %zu\n" + "sizeof(node): %zu\n" + "node id_table size: %zu\n" + "node id_table usage: %zu\n" + "node id_table total allocated memory: %zu\n" + "node name_table size: %zu\n" + "node name_table usage: %zu\n" + "node name_table total allocated memory: %zu\n" + "node memory pool slab count: %zu\n" + "node memory pool usage ratio: %f\n" + "node memory pool avail objs: %zu\n" + "node memory pool total allocated memory: %zu\n" + "\n" + , + time(NULL), + sizeof(struct node), + f_->id_table.size, + f_->id_table.use, + (f_->id_table.size * sizeof(struct node*)), + f_->name_table.size, + f_->name_table.use, + (f_->name_table.size * sizeof(struct node*)), + lfmp_slab_count(&f_->node_fmp), + lfmp_slab_usage_ratio(&f_->node_fmp), + lfmp_avail_objs(&f_->node_fmp), + lfmp_total_allocated_memory(&f_->node_fmp) + ); + + pthread_mutex_unlock(&f_->lock); +} + +static +void +metrics_log_nodes_info_to_tmp_dir(struct fuse *f_) +{ + FILE *file; + char filepath[256]; + + sprintf(filepath,"/tmp/mergerfs.%d.info",getpid()); + + file = fopen(filepath,"w"); + if(file == NULL) + return; + + metrics_log_nodes_info(f_,file); + + fclose(file); +} + + static void fuse_malloc_trim(void) @@ -4038,6 +4098,9 @@ fuse_maintenance_loop(void *fuse_) if(loops % 15) fuse_malloc_trim(); + if(g_LOG_METRICS) + metrics_log_nodes_info_to_tmp_dir(f); + loops++; sleep(sleep_time); } @@ -4097,6 +4160,8 @@ fuse_new_common(struct fuse_chan *ch, if(fuse_opt_parse(args,&f->conf,fuse_lib_opts,fuse_lib_opt_proc) == -1) goto out_free_fs; + g_LOG_METRICS = f->conf.debug; + f->se = fuse_lowlevel_new_common(args,&llop,sizeof(llop),f); if(f->se == NULL) goto out_free_fs; @@ -4215,3 +4280,9 @@ fuse_config_num_threads(const struct fuse *fuse_) { return fuse_->conf.threads; } + +void +fuse_log_metrics(int log_) +{ + g_LOG_METRICS = log_; +} diff --git a/libfuse/lib/lfmp.h b/libfuse/lib/lfmp.h index c2267387..49097cb7 100644 --- a/libfuse/lib/lfmp.h +++ b/libfuse/lib/lfmp.h @@ -211,3 +211,17 @@ lfmp_slab_usage_ratio(lfmp_t *lfmp_) return rv; } + +static +inline +uint64_t +lfmp_total_allocated_memory(lfmp_t *lfmp_) +{ + uint64_t rv; + + pthread_mutex_lock(&lfmp_->lock); + rv = fmp_total_allocated_memory(&lfmp_->fmp); + pthread_mutex_unlock(&lfmp_->lock); + + return rv; +} diff --git a/src/fuse_ioctl.cpp b/src/fuse_ioctl.cpp index 8e74559d..e07b24b9 100644 --- a/src/fuse_ioctl.cpp +++ b/src/fuse_ioctl.cpp @@ -38,10 +38,19 @@ using std::string; using std::vector; +#ifndef _IOC_TYPE +#define _IOC_TYPE(X) (((X) >> 8) & 0xFF) +#endif + typedef char IOCTL_BUF[4096]; #define IOCTL_APP_TYPE 0xDF -//#define IOCTL_FILE_INFO 0xD000DF00 -#define IOCTL_FILE_INFO _IOWR(IOCTL_APP_TYPE,0,IOCTL_BUF) +#define IOCTL_FILE_INFO _IOWR(IOCTL_APP_TYPE,0,IOCTL_BUF) +#define IOCTL_METRICS_ENABLE _IOWR(IOCTL_APP_TYPE,1,IOCTL_BUF) +#define IOCTL_METRICS_DISABLE _IOWR(IOCTL_APP_TYPE,2,IOCTL_BUF) + +static_assert(IOCTL_FILE_INFO == 0xD000DF00,""); +static_assert(IOCTL_METRICS_ENABLE == 0xD000DF01,""); +static_assert(IOCTL_METRICS_DISABLE == 0xD000DF02,""); #ifndef FS_IOC_GETFLAGS # define FS_IOC_GETFLAGS _IOR('f',1,long) @@ -305,6 +314,34 @@ namespace l return -ENOATTR; } + + static + bool + is_mergerfs_ioctl_cmd(const unsigned long cmd_) + { + return (_IOC_TYPE(cmd_) == IOCTL_APP_TYPE); + } + + static + int + ioctl_custom(const fuse_file_info_t *ffi_, + unsigned long cmd_, + void *data_) + { + switch(cmd_) + { + case IOCTL_FILE_INFO: + return l::file_info(ffi_,data_); + case IOCTL_METRICS_ENABLE: + fuse_log_metrics(1); + return 0; + case IOCTL_METRICS_DISABLE: + fuse_log_metrics(0); + return 0; + } + + return -ENOTTY; + } } namespace FUSE @@ -317,11 +354,8 @@ namespace FUSE void *data_, uint32_t *out_bufsz_) { - switch(cmd_) - { - case IOCTL_FILE_INFO: - return l::file_info(ffi_,data_); - } + if(l::is_mergerfs_ioctl_cmd(cmd_)) + return l::ioctl_custom(ffi_,cmd_,data_); if(flags_ & FUSE_IOCTL_DIR) return l::ioctl_dir(ffi_,cmd_,data_,out_bufsz_);