diff --git a/weed/command/command.go b/weed/command/command.go index c451936e5..d04d6eb15 100644 --- a/weed/command/command.go +++ b/weed/command/command.go @@ -57,7 +57,7 @@ func (c *Command) Name() string { } func (c *Command) Usage() { - fmt.Fprintf(os.Stderr, "Example: weed %s\n", c.UsageLine) + fmt.Fprintf(os.Stderr, "Example filer: weed %s\n", c.UsageLine) fmt.Fprintf(os.Stderr, "Default Usage:\n") c.Flag.PrintDefaults() fmt.Fprintf(os.Stderr, "Description:\n") diff --git a/weed/command/filer.go b/weed/command/filer.go index 0bd508e0b..c9b2e7ada 100644 --- a/weed/command/filer.go +++ b/weed/command/filer.go @@ -24,13 +24,17 @@ type FilerOptions struct { dir *string redirectOnRead *bool disableDirListing *bool - maxMB *int + maxMB *int secretKey *string cassandra_server *string cassandra_keyspace *string redis_server *string redis_password *string redis_database *int + mysql_dsn *string + mysql_table *string + mysql_fnameCol *string + mysql_fidCol *string } func init() { @@ -50,7 +54,10 @@ func init() { f.redis_password = cmdFiler.Flag.String("redis.password", "", "password in clear text") f.redis_database = cmdFiler.Flag.Int("redis.database", 0, "the database on the redis server") f.secretKey = cmdFiler.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)") - + f.mysql_dsn = cmdFiler.Flag.String("mysql.dsn", "", "database source name") + f.mysql_table = cmdFiler.Flag.String("mysql.table", "weed", "mysql table") + f.mysql_fnameCol = cmdFiler.Flag.String("mysql.fnameCol", "fname", "file name column") + f.mysql_fidCol = cmdFiler.Flag.String("mysql.fidCol", "fid", "file id column") } var cmdFiler = &Command{ @@ -88,6 +95,7 @@ func runFiler(cmd *Command, args []string) bool { *f.secretKey, *f.cassandra_server, *f.cassandra_keyspace, *f.redis_server, *f.redis_password, *f.redis_database, + *f.mysql_dsn, *f.mysql_table, *f.mysql_fnameCol, *f.mysql_fidCol, ) if nfs_err != nil { glog.Fatalf("Filer startup error: %v", nfs_err) diff --git a/weed/command/server.go b/weed/command/server.go index 7a6677a65..11ce2fa31 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -174,6 +174,7 @@ func runServer(cmd *Command, args []string) bool { *filerOptions.secretKey, *filerOptions.cassandra_server, *filerOptions.cassandra_keyspace, *filerOptions.redis_server, *filerOptions.redis_password, *filerOptions.redis_database, + *filerOptions.mysql_dsn, *filerOptions.mysql_table, *filerOptions.mysql_fnameCol, *filerOptions.mysql_fidCol, ) if nfs_err != nil { glog.Fatalf("Filer startup error: %v", nfs_err) diff --git a/weed/filer/mysql_store/mysql_store.go b/weed/filer/mysql_store/mysql_store.go new file mode 100644 index 000000000..db9925900 --- /dev/null +++ b/weed/filer/mysql_store/mysql_store.go @@ -0,0 +1,72 @@ +package mysql_store + +import ( + "database/sql" + "fmt" + _ "github.com/go-sql-driver/mysql" +) + +var maxOpen int = 100 + +type MysqlStore struct { + Client *sql.DB + table string + fnameCol string + fidCol string + getStmt *sql.Stmt + putStmt *sql.Stmt + delStmt *sql.Stmt +} + +func NewMysqlStore(dataSourceName string, table, fnameCol, fidCol string) (m *MysqlStore, err error) { + db, err := sql.Open("mysql", dataSourceName) + if err != nil { + return nil, err + } + db.SetMaxOpenConns(maxOpen) + db.SetMaxIdleConns(maxOpen / 2) + getStmt, err := db.Prepare(fmt.Sprintf("select %s from %s where %s=?", fidCol, table, fnameCol)) + if err != nil { + return nil, err + } + putStmt, err := db.Prepare(fmt.Sprintf("insert into %s(%s,%s) values(?,?)", table, fnameCol, fidCol)) + if err != nil { + return nil, err + } + delStmt, err := db.Prepare(fmt.Sprintf("delete from %s where %s=?", table, fnameCol)) + if err != nil { + return nil, err + } + + return &MysqlStore{db, table, fnameCol, fidCol, getStmt, putStmt, delStmt}, nil +} + +func (s *MysqlStore) Get(fullFileName string) (fid string, err error) { + err = s.getStmt.QueryRow(fullFileName).Scan(&fid) + return fid, err +} + +func (s *MysqlStore) Put(fullFileName string, fid string) (err error) { + _, err = s.putStmt.Exec(fullFileName, fid) + return err +} + +func (s *MysqlStore) Delete(fullFileName string) (err error) { + _, err = s.delStmt.Exec(fullFileName) + return err +} + +func (s *MysqlStore) Close() { + if s.getStmt != nil { + s.getStmt.Close() + } + if s.putStmt != nil { + s.putStmt.Close() + } + if s.delStmt != nil { + s.delStmt.Close() + } + if s.Client != nil { + s.Client.Close() + } +} diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go index 3c7c1fd9e..fc5242a96 100644 --- a/weed/server/filer_server.go +++ b/weed/server/filer_server.go @@ -11,6 +11,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/filer/cassandra_store" "github.com/chrislusf/seaweedfs/weed/filer/embedded_filer" "github.com/chrislusf/seaweedfs/weed/filer/flat_namespace" + "github.com/chrislusf/seaweedfs/weed/filer/mysql_store" "github.com/chrislusf/seaweedfs/weed/filer/redis_store" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/security" @@ -28,7 +29,7 @@ type FilerServer struct { disableDirListing bool secret security.Secret filer filer.Filer - maxMB int + maxMB int masterNodes *storage.MasterNodes } @@ -38,6 +39,7 @@ func NewFilerServer(r *http.ServeMux, ip string, port int, master string, dir st secret string, cassandra_server string, cassandra_keyspace string, redis_server string, redis_password string, redis_database int, + mysql_dsn string, mysql_table, mysql_fnameCol, mysql_fidCol string, ) (fs *FilerServer, err error) { fs = &FilerServer{ master: master, @@ -45,7 +47,7 @@ func NewFilerServer(r *http.ServeMux, ip string, port int, master string, dir st defaultReplication: replication, redirectOnRead: redirectOnRead, disableDirListing: disableDirListing, - maxMB: maxMB, + maxMB: maxMB, port: ip + ":" + strconv.Itoa(port), } @@ -58,6 +60,12 @@ func NewFilerServer(r *http.ServeMux, ip string, port int, master string, dir st } else if redis_server != "" { redis_store := redis_store.NewRedisStore(redis_server, redis_password, redis_database) fs.filer = flat_namespace.NewFlatNamespaceFiler(master, redis_store) + } else if mysql_dsn != "" { + mysql_store, err := mysql_store.NewMysqlStore(mysql_dsn, mysql_table, mysql_fnameCol, mysql_fidCol) + if err != nil { + glog.Fatalf("Can not connect to mysql server %s err %v", mysql_dsn, err) + } + fs.filer = flat_namespace.NewFlatNamespaceFiler(master, mysql_store) } else { if fs.filer, err = embedded_filer.NewFilerEmbedded(master, dir); err != nil { glog.Fatalf("Can not start filer in dir %s : %v", dir, err)