Merge f7b1b83cc6 into 9992fe648d

weed/command/command.go

@@ -57,7 +57,7 @@ func (c *Command) Name() string {
 }
 
 func (c *Command) Usage() {
-	fmt.Fprintf(os.Stderr, "Example: weed %s\n", c.UsageLine)
+	fmt.Fprintf(os.Stderr, "Example filer: weed %s\n", c.UsageLine)
 	fmt.Fprintf(os.Stderr, "Default Usage:\n")
 	c.Flag.PrintDefaults()
 	fmt.Fprintf(os.Stderr, "Description:\n")

weed/command/filer.go

@@ -24,13 +24,17 @@ type FilerOptions struct {
 	dir                     *string
 	redirectOnRead          *bool
 	disableDirListing       *bool
-	maxMB			*int
+	maxMB                   *int
 	secretKey               *string
 	cassandra_server        *string
 	cassandra_keyspace      *string
 	redis_server            *string
 	redis_password          *string
 	redis_database          *int
+	mysql_dsn               *string
+	mysql_table             *string
+	mysql_fnameCol          *string
+	mysql_fidCol            *string
 }
 
 func init() {
@@ -50,7 +54,10 @@ func init() {
 	f.redis_password = cmdFiler.Flag.String("redis.password", "", "password in clear text")
 	f.redis_database = cmdFiler.Flag.Int("redis.database", 0, "the database on the redis server")
 	f.secretKey = cmdFiler.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)")
-
+	f.mysql_dsn = cmdFiler.Flag.String("mysql.dsn", "", "database source name")
+	f.mysql_table = cmdFiler.Flag.String("mysql.table", "weed", "mysql table")
+	f.mysql_fnameCol = cmdFiler.Flag.String("mysql.fnameCol", "fname", "file name column")
+	f.mysql_fidCol = cmdFiler.Flag.String("mysql.fidCol", "fid", "file id column")
 }
 
 var cmdFiler = &Command{
@@ -88,6 +95,7 @@ func runFiler(cmd *Command, args []string) bool {
 		*f.secretKey,
 		*f.cassandra_server, *f.cassandra_keyspace,
 		*f.redis_server, *f.redis_password, *f.redis_database,
+		*f.mysql_dsn, *f.mysql_table, *f.mysql_fnameCol, *f.mysql_fidCol,
 	)
 	if nfs_err != nil {
 		glog.Fatalf("Filer startup error: %v", nfs_err)

weed/command/server.go

@@ -174,6 +174,7 @@ func runServer(cmd *Command, args []string) bool {
 				*filerOptions.secretKey,
 				*filerOptions.cassandra_server, *filerOptions.cassandra_keyspace,
 				*filerOptions.redis_server, *filerOptions.redis_password, *filerOptions.redis_database,
+				*filerOptions.mysql_dsn, *filerOptions.mysql_table, *filerOptions.mysql_fnameCol, *filerOptions.mysql_fidCol,
 			)
 			if nfs_err != nil {
 				glog.Fatalf("Filer startup error: %v", nfs_err)

weed/filer/mysql_store/mysql_store.go

@@ -0,0 +1,72 @@
+package mysql_store
+
+import (
+	"database/sql"
+	"fmt"
+	_ "github.com/go-sql-driver/mysql"
+)
+
+var maxOpen int = 100
+
+type MysqlStore struct {
+	Client   *sql.DB
+	table    string
+	fnameCol string
+	fidCol   string
+	getStmt  *sql.Stmt
+	putStmt  *sql.Stmt
+	delStmt  *sql.Stmt
+}
+
+func NewMysqlStore(dataSourceName string, table, fnameCol, fidCol string) (m *MysqlStore, err error) {
+	db, err := sql.Open("mysql", dataSourceName)
+	if err != nil {
+		return nil, err
+	}
+	db.SetMaxOpenConns(maxOpen)
+	db.SetMaxIdleConns(maxOpen / 2)
+	getStmt, err := db.Prepare(fmt.Sprintf("select %s from %s where %s=?", fidCol, table, fnameCol))
+	if err != nil {
+		return nil, err
+	}
+	putStmt, err := db.Prepare(fmt.Sprintf("insert into %s(%s,%s) values(?,?)", table, fnameCol, fidCol))
+	if err != nil {
+		return nil, err
+	}
+	delStmt, err := db.Prepare(fmt.Sprintf("delete from %s where %s=?", table, fnameCol))
+	if err != nil {
+		return nil, err
+	}
+
+	return &MysqlStore{db, table, fnameCol, fidCol, getStmt, putStmt, delStmt}, nil
+}
+
+func (s *MysqlStore) Get(fullFileName string) (fid string, err error) {
+	err = s.getStmt.QueryRow(fullFileName).Scan(&fid)
+	return fid, err
+}
+
+func (s *MysqlStore) Put(fullFileName string, fid string) (err error) {
+	_, err = s.putStmt.Exec(fullFileName, fid)
+	return err
+}
+
+func (s *MysqlStore) Delete(fullFileName string) (err error) {
+	_, err = s.delStmt.Exec(fullFileName)
+	return err
+}
+
+func (s *MysqlStore) Close() {
+	if s.getStmt != nil {
+		s.getStmt.Close()
+	}
+	if s.putStmt != nil {
+		s.putStmt.Close()
+	}
+	if s.delStmt != nil {
+		s.delStmt.Close()
+	}
+	if s.Client != nil {
+		s.Client.Close()
+	}
+}

weed/server/filer_server.go

@@ -11,6 +11,7 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/filer/cassandra_store"
 	"github.com/chrislusf/seaweedfs/weed/filer/embedded_filer"
 	"github.com/chrislusf/seaweedfs/weed/filer/flat_namespace"
+	"github.com/chrislusf/seaweedfs/weed/filer/mysql_store"
 	"github.com/chrislusf/seaweedfs/weed/filer/redis_store"
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/security"
@@ -28,7 +29,7 @@ type FilerServer struct {
 	disableDirListing  bool
 	secret             security.Secret
 	filer              filer.Filer
-	maxMB		   int
+	maxMB              int
 	masterNodes        *storage.MasterNodes
 }
 
@@ -38,6 +39,7 @@ func NewFilerServer(r *http.ServeMux, ip string, port int, master string, dir st
 	secret string,
 	cassandra_server string, cassandra_keyspace string,
 	redis_server string, redis_password string, redis_database int,
+	mysql_dsn string, mysql_table, mysql_fnameCol, mysql_fidCol string,
 ) (fs *FilerServer, err error) {
 	fs = &FilerServer{
 		master:             master,
@@ -45,7 +47,7 @@ func NewFilerServer(r *http.ServeMux, ip string, port int, master string, dir st
 		defaultReplication: replication,
 		redirectOnRead:     redirectOnRead,
 		disableDirListing:  disableDirListing,
-		maxMB:		    maxMB,
+		maxMB:              maxMB,
 		port:               ip + ":" + strconv.Itoa(port),
 	}
 
@@ -58,6 +60,12 @@ func NewFilerServer(r *http.ServeMux, ip string, port int, master string, dir st
 	} else if redis_server != "" {
 		redis_store := redis_store.NewRedisStore(redis_server, redis_password, redis_database)
 		fs.filer = flat_namespace.NewFlatNamespaceFiler(master, redis_store)
+	} else if mysql_dsn != "" {
+		mysql_store, err := mysql_store.NewMysqlStore(mysql_dsn, mysql_table, mysql_fnameCol, mysql_fidCol)
+		if err != nil {
+			glog.Fatalf("Can not connect to mysql server %s err %v", mysql_dsn, err)
+		}
+		fs.filer = flat_namespace.NewFlatNamespaceFiler(master, mysql_store)
 	} else {
 		if fs.filer, err = embedded_filer.NewFilerEmbedded(master, dir); err != nil {
 			glog.Fatalf("Can not start filer in dir %s : %v", dir, err)