filer store: add mysql2

weed/command/scaffold.go

@@ -121,6 +121,27 @@ connection_max_open = 100
 connection_max_lifetime_seconds = 0
 interpolateParams = false
 
+[mysql2]  # or memsql, tidb
+enabled = false
+createTable = """
+  CREATE TABLE IF NOT EXISTS %s (
+    dirhash BIGINT, 
+    name VARCHAR(1000), 
+    directory TEXT, 
+    meta LONGBLOB, 
+    PRIMARY KEY (dirhash, name)
+  ) DEFAULT CHARSET=utf8;
+"""
+hostname = "localhost"
+port = 3306
+username = "root"
+password = ""
+database = ""              # create or use an existing database
+connection_max_idle = 2
+connection_max_open = 100
+connection_max_lifetime_seconds = 0
+interpolateParams = false
+
 [postgres] # or cockroachdb, YugabyteDB
 # CREATE TABLE IF NOT EXISTS filemeta (
 #   dirhash     BIGINT,

weed/filer/abstract_sql/abstract_sql_store.go

@@ -91,6 +91,8 @@ func (store *AbstractSqlStore) getTxOrDB(ctx context.Context, fullpath util.Full
 	if t < 0 && !isForChildren {
 		return
 	}
+	bucket = bucketAndObjectKey
+	shortPath = "/"
 	if t > 0 {
 		bucket = bucketAndObjectKey[:t]
 		shortPath = util.FullPath(bucketAndObjectKey[t:])
@@ -241,11 +243,13 @@ func (store *AbstractSqlStore) DeleteFolderChildren(ctx context.Context, fullpat
 	}
 
 	if isValidBucket(bucket) && shortPath == "/" {
-		if err = store.deleteTable(ctx, bucket); err != nil {
+		if err = store.deleteTable(ctx, bucket); err == nil {
 			store.dbsLock.Lock()
 			delete(store.dbs, bucket)
 			store.dbsLock.Unlock()
 			return nil
+		} else {
+			return err
 		}
 	}
 

weed/filer/mysql/mysql_sql_gen.go

@@ -0,0 +1,52 @@
+package mysql
+
+import (
+	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/filer/abstract_sql"
+	_ "github.com/go-sql-driver/mysql"
+)
+
+type SqlGenMysql struct {
+	CreateTableSqlTemplate string
+	DropTableSqlTemplate   string
+}
+
+var (
+	_ = abstract_sql.SqlGenerator(&SqlGenMysql{})
+)
+
+func (gen *SqlGenMysql) GetSqlInsert(bucket string) string {
+	return fmt.Sprintf("INSERT INTO %s (dirhash,name,directory,meta) VALUES(?,?,?,?)", bucket)
+}
+
+func (gen *SqlGenMysql) GetSqlUpdate(bucket string) string {
+	return fmt.Sprintf("UPDATE %s SET meta=? WHERE dirhash=? AND name=? AND directory=?", bucket)
+}
+
+func (gen *SqlGenMysql) GetSqlFind(bucket string) string {
+	return fmt.Sprintf("SELECT meta FROM %s WHERE dirhash=? AND name=? AND directory=?", bucket)
+}
+
+func (gen *SqlGenMysql) GetSqlDelete(bucket string) string {
+	return fmt.Sprintf("DELETE FROM %s WHERE dirhash=? AND name=? AND directory=?", bucket)
+}
+
+func (gen *SqlGenMysql) GetSqlDeleteFolderChildren(bucket string) string {
+	return fmt.Sprintf("DELETE FROM %s WHERE dirhash=? AND directory=?", bucket)
+}
+
+func (gen *SqlGenMysql) GetSqlListExclusive(bucket string) string {
+	return fmt.Sprintf("SELECT NAME, meta FROM %s WHERE dirhash=? AND name>? AND directory=? AND name like ? ORDER BY NAME ASC LIMIT ?", bucket)
+}
+
+func (gen *SqlGenMysql) GetSqlListInclusive(bucket string) string {
+	return fmt.Sprintf("SELECT NAME, meta FROM %s WHERE dirhash=? AND name>=? AND directory=? AND name like ? ORDER BY NAME ASC LIMIT ?", bucket)
+}
+
+func (gen *SqlGenMysql) GetSqlCreateTable(bucket string) string {
+	return fmt.Sprintf(gen.CreateTableSqlTemplate, bucket)
+}
+
+func (gen *SqlGenMysql) GetSqlDropTable(bucket string) string {
+	return fmt.Sprintf(gen.DropTableSqlTemplate, bucket)
+}

weed/filer/mysql/mysql_store.go

@@ -3,9 +3,9 @@ package mysql
 import (
 	"database/sql"
 	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/filer"
 	"time"
 
-	"github.com/chrislusf/seaweedfs/weed/filer"
 	"github.com/chrislusf/seaweedfs/weed/filer/abstract_sql"
 	"github.com/chrislusf/seaweedfs/weed/util"
 	_ "github.com/go-sql-driver/mysql"
@@ -15,51 +15,6 @@ const (
 	CONNECTION_URL_PATTERN = "%s:%s@tcp(%s:%d)/%s?charset=utf8"
 )
 
-type SqlGenMysql struct {
-	createTableSqlTemplate string
-	dropTableSqlTemplate string
-}
-
-var (
-	_ = abstract_sql.SqlGenerator(&SqlGenMysql{})
-)
-
-func (gen *SqlGenMysql) GetSqlInsert(bucket string) string {
-	return "INSERT INTO filemeta (dirhash,name,directory,meta) VALUES(?,?,?,?)"
-}
-
-func (gen *SqlGenMysql) GetSqlUpdate(bucket string) string {
-	return "UPDATE filemeta SET meta=? WHERE dirhash=? AND name=? AND directory=?"
-}
-
-func (gen *SqlGenMysql) GetSqlFind(bucket string) string {
-	return "SELECT meta FROM filemeta WHERE dirhash=? AND name=? AND directory=?"
-}
-
-func (gen *SqlGenMysql) GetSqlDelete(bucket string) string {
-	return "DELETE FROM filemeta WHERE dirhash=? AND name=? AND directory=?"
-}
-
-func (gen *SqlGenMysql) GetSqlDeleteFolderChildren(bucket string) string {
-	return "DELETE FROM filemeta WHERE dirhash=? AND directory=?"
-}
-
-func (gen *SqlGenMysql) GetSqlListExclusive(bucket string) string {
-	return "SELECT NAME, meta FROM filemeta WHERE dirhash=? AND name>? AND directory=? AND name like ? ORDER BY NAME ASC LIMIT ?"
-}
-
-func (gen *SqlGenMysql) GetSqlListInclusive(bucket string) string {
-	return "SELECT NAME, meta FROM filemeta WHERE dirhash=? AND name>=? AND directory=? AND name like ? ORDER BY NAME ASC LIMIT ?"
-}
-
-func (gen *SqlGenMysql) GetSqlCreateTable(bucket string) string {
-	return fmt.Sprintf(gen.createTableSqlTemplate, bucket)
-}
-
-func (gen *SqlGenMysql) GetSqlDropTable(bucket string) string {
-	return fmt.Sprintf(gen.dropTableSqlTemplate, bucket)
-}
-
 func init() {
 	filer.Stores = append(filer.Stores, &MysqlStore{})
 }
@@ -91,8 +46,8 @@ func (store *MysqlStore) initialize(user, password, hostname string, port int, d
 
 	store.SupportBucketTable = false
 	store.SqlGenerator = &SqlGenMysql{
-		createTableSqlTemplate: "",
+		CreateTableSqlTemplate: "",
-		dropTableSqlTemplate: "drop table %s",
+		DropTableSqlTemplate:   "drop table %s",
 	}
 
 	sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, user, password, hostname, port, database)

weed/filer/mysql2/mysql_store.go

@@ -0,0 +1,77 @@
+package mysql2
+
+import (
+	"database/sql"
+	"fmt"
+	"time"
+
+	"github.com/chrislusf/seaweedfs/weed/filer"
+	"github.com/chrislusf/seaweedfs/weed/filer/abstract_sql"
+	"github.com/chrislusf/seaweedfs/weed/filer/mysql"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	_ "github.com/go-sql-driver/mysql"
+)
+
+const (
+	CONNECTION_URL_PATTERN = "%s:%s@tcp(%s:%d)/%s?charset=utf8"
+)
+
+func init() {
+	filer.Stores = append(filer.Stores, &MysqlStore2{})
+}
+
+type MysqlStore2 struct {
+	abstract_sql.AbstractSqlStore
+}
+
+func (store *MysqlStore2) GetName() string {
+	return "mysql2"
+}
+
+func (store *MysqlStore2) Initialize(configuration util.Configuration, prefix string) (err error) {
+	return store.initialize(
+		configuration.GetString(prefix+"createTable"),
+		configuration.GetString(prefix+"username"),
+		configuration.GetString(prefix+"password"),
+		configuration.GetString(prefix+"hostname"),
+		configuration.GetInt(prefix+"port"),
+		configuration.GetString(prefix+"database"),
+		configuration.GetInt(prefix+"connection_max_idle"),
+		configuration.GetInt(prefix+"connection_max_open"),
+		configuration.GetInt(prefix+"connection_max_lifetime_seconds"),
+		configuration.GetBool(prefix+"interpolateParams"),
+	)
+}
+
+func (store *MysqlStore2) initialize(createTable, user, password, hostname string, port int, database string, maxIdle, maxOpen,
+	maxLifetimeSeconds int, interpolateParams bool) (err error) {
+
+	store.SupportBucketTable = true
+	store.SqlGenerator = &mysql.SqlGenMysql{
+		CreateTableSqlTemplate: createTable,
+		DropTableSqlTemplate:   "drop table %s",
+	}
+
+	sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, user, password, hostname, port, database)
+	if interpolateParams {
+		sqlUrl += "&interpolateParams=true"
+	}
+
+	var dbErr error
+	store.DB, dbErr = sql.Open("mysql", sqlUrl)
+	if dbErr != nil {
+		store.DB.Close()
+		store.DB = nil
+		return fmt.Errorf("can not connect to %s error:%v", sqlUrl, err)
+	}
+
+	store.DB.SetMaxIdleConns(maxIdle)
+	store.DB.SetMaxOpenConns(maxOpen)
+	store.DB.SetConnMaxLifetime(time.Duration(maxLifetimeSeconds) * time.Second)
+
+	if err = store.DB.Ping(); err != nil {
+		return fmt.Errorf("connect to %s error:%v", sqlUrl, err)
+	}
+
+	return nil
+}

weed/server/filer_server.go

@@ -29,6 +29,7 @@ import (
 	_ "github.com/chrislusf/seaweedfs/weed/filer/leveldb3"
 	_ "github.com/chrislusf/seaweedfs/weed/filer/mongodb"
 	_ "github.com/chrislusf/seaweedfs/weed/filer/mysql"
+	_ "github.com/chrislusf/seaweedfs/weed/filer/mysql2"
 	_ "github.com/chrislusf/seaweedfs/weed/filer/postgres"
 	_ "github.com/chrislusf/seaweedfs/weed/filer/redis"
 	_ "github.com/chrislusf/seaweedfs/weed/filer/redis2"

weed/util/config.go

@@ -29,11 +29,11 @@ func LoadConfiguration(configFileName string, required bool) (loaded bool) {
 	glog.V(1).Infof("Reading %s.toml from %s", configFileName, viper.ConfigFileUsed())
 
 	if err := viper.MergeInConfig(); err != nil { // Handle errors reading the config file
-		logLevel := glog.Level(0)
 		if strings.Contains(err.Error(), "Not Found") {
-			logLevel = 1
+			glog.V(1).Infof("Reading %s: %v", viper.ConfigFileUsed(), err)
+		} else {
+			glog.Fatalf("Reading %s: %v", viper.ConfigFileUsed(), err)
 		}
-		glog.V(logLevel).Infof("Reading %s: %v", viper.ConfigFileUsed(), err)
 		if required {
 			glog.Fatalf("Failed to load %s.toml file from current directory, or $HOME/.seaweedfs/, or /etc/seaweedfs/"+
 				"\n\nPlease use this command to generate the default %s.toml file\n"+