gitlab-com-seaweedfs/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-01-19 02:48:24 +00:00
Code Issues Projects Releases Wiki Activity

filer store: add mysql2

Browse source
This commit is contained in:
Chris Lu 2021-01-19 17:21:50 -08:00
parent 93b3adba98
commit 52a8f1470e
7 changed files with 162 additions and 52 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
weed/command/scaffold.go
View file

@ -121,6 +121,27 @@ connection_max_open = 100
connection_max_lifetime_seconds = 0
interpolateParams = false
[mysql2] # or memsql, tidb
enabled = false
createTable = """
CREATE TABLE IF NOT EXISTS %s (
dirhash BIGINT,
name VARCHAR(1000),
directory TEXT,
meta LONGBLOB,
PRIMARY KEY (dirhash, name)
) DEFAULT CHARSET=utf8;
"""
hostname = "localhost"
port = 3306
username = "root"
password = ""
database = "" # create or use an existing database
connection_max_idle = 2
connection_max_open = 100
connection_max_lifetime_seconds = 0
interpolateParams = false
[postgres] # or cockroachdb, YugabyteDB
# CREATE TABLE IF NOT EXISTS filemeta (
# dirhash BIGINT,

6
weed/filer/abstract_sql/abstract_sql_store.go
View file

@ -91,6 +91,8 @@ func (store *AbstractSqlStore) getTxOrDB(ctx context.Context, fullpath util.Full
if t < 0 && !isForChildren {
return
}
bucket = bucketAndObjectKey
shortPath = "/"
if t > 0 {
bucket = bucketAndObjectKey[:t]
shortPath = util.FullPath(bucketAndObjectKey[t:])
@ -241,11 +243,13 @@ func (store *AbstractSqlStore) DeleteFolderChildren(ctx context.Context, fullpat
}
if isValidBucket(bucket) && shortPath == "/" {
if err = store.deleteTable(ctx, bucket); err != nil {
if err = store.deleteTable(ctx, bucket); err == nil {
store.dbsLock.Lock()
delete(store.dbs, bucket)
store.dbsLock.Unlock()
return nil
} else {
return err
}
}

52
weed/filer/mysql/mysql_sql_gen.go Normal file
View file

@ -0,0 +1,52 @@
package mysql
import (
"fmt"
"github.com/chrislusf/seaweedfs/weed/filer/abstract_sql"
_ "github.com/go-sql-driver/mysql"
)
type SqlGenMysql struct {
CreateTableSqlTemplate string
DropTableSqlTemplate string
}
var (
_ = abstract_sql.SqlGenerator(&SqlGenMysql{})
)
func (gen *SqlGenMysql) GetSqlInsert(bucket string) string {
return fmt.Sprintf("INSERT INTO %s (dirhash,name,directory,meta) VALUES(?,?,?,?)", bucket)
}
func (gen *SqlGenMysql) GetSqlUpdate(bucket string) string {
return fmt.Sprintf("UPDATE %s SET meta=? WHERE dirhash=? AND name=? AND directory=?", bucket)
}
func (gen *SqlGenMysql) GetSqlFind(bucket string) string {
return fmt.Sprintf("SELECT meta FROM %s WHERE dirhash=? AND name=? AND directory=?", bucket)
}
func (gen *SqlGenMysql) GetSqlDelete(bucket string) string {
return fmt.Sprintf("DELETE FROM %s WHERE dirhash=? AND name=? AND directory=?", bucket)
}
func (gen *SqlGenMysql) GetSqlDeleteFolderChildren(bucket string) string {
return fmt.Sprintf("DELETE FROM %s WHERE dirhash=? AND directory=?", bucket)
}
func (gen *SqlGenMysql) GetSqlListExclusive(bucket string) string {
return fmt.Sprintf("SELECT NAME, meta FROM %s WHERE dirhash=? AND name>? AND directory=? AND name like ? ORDER BY NAME ASC LIMIT ?", bucket)
}
func (gen *SqlGenMysql) GetSqlListInclusive(bucket string) string {
return fmt.Sprintf("SELECT NAME, meta FROM %s WHERE dirhash=? AND name>=? AND directory=? AND name like ? ORDER BY NAME ASC LIMIT ?", bucket)
}
func (gen *SqlGenMysql) GetSqlCreateTable(bucket string) string {
return fmt.Sprintf(gen.CreateTableSqlTemplate, bucket)
}
func (gen *SqlGenMysql) GetSqlDropTable(bucket string) string {
return fmt.Sprintf(gen.DropTableSqlTemplate, bucket)
}

51
weed/filer/mysql/mysql_store.go
View file

@ -3,9 +3,9 @@ package mysql
import (
"database/sql"
"fmt"
"github.com/chrislusf/seaweedfs/weed/filer"
"time"
"github.com/chrislusf/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/filer/abstract_sql"
"github.com/chrislusf/seaweedfs/weed/util"
_ "github.com/go-sql-driver/mysql"
@ -15,51 +15,6 @@ const (
CONNECTION_URL_PATTERN = "%s:%s@tcp(%s:%d)/%s?charset=utf8"
)
type SqlGenMysql struct {
createTableSqlTemplate string
dropTableSqlTemplate string
}
var (
_ = abstract_sql.SqlGenerator(&SqlGenMysql{})
)
func (gen *SqlGenMysql) GetSqlInsert(bucket string) string {
return "INSERT INTO filemeta (dirhash,name,directory,meta) VALUES(?,?,?,?)"
}
func (gen *SqlGenMysql) GetSqlUpdate(bucket string) string {
return "UPDATE filemeta SET meta=? WHERE dirhash=? AND name=? AND directory=?"
}
func (gen *SqlGenMysql) GetSqlFind(bucket string) string {
return "SELECT meta FROM filemeta WHERE dirhash=? AND name=? AND directory=?"
}
func (gen *SqlGenMysql) GetSqlDelete(bucket string) string {
return "DELETE FROM filemeta WHERE dirhash=? AND name=? AND directory=?"
}
func (gen *SqlGenMysql) GetSqlDeleteFolderChildren(bucket string) string {
return "DELETE FROM filemeta WHERE dirhash=? AND directory=?"
}
func (gen *SqlGenMysql) GetSqlListExclusive(bucket string) string {
return "SELECT NAME, meta FROM filemeta WHERE dirhash=? AND name>? AND directory=? AND name like ? ORDER BY NAME ASC LIMIT ?"
}
func (gen *SqlGenMysql) GetSqlListInclusive(bucket string) string {
return "SELECT NAME, meta FROM filemeta WHERE dirhash=? AND name>=? AND directory=? AND name like ? ORDER BY NAME ASC LIMIT ?"
}
func (gen *SqlGenMysql) GetSqlCreateTable(bucket string) string {
return fmt.Sprintf(gen.createTableSqlTemplate, bucket)
}
func (gen *SqlGenMysql) GetSqlDropTable(bucket string) string {
return fmt.Sprintf(gen.dropTableSqlTemplate, bucket)
}
func init() {
filer.Stores = append(filer.Stores, &MysqlStore{})
}
@ -91,8 +46,8 @@ func (store *MysqlStore) initialize(user, password, hostname string, port int, d
store.SupportBucketTable = false
store.SqlGenerator = &SqlGenMysql{
createTableSqlTemplate: "",
dropTableSqlTemplate: "drop table %s",
CreateTableSqlTemplate: "",
DropTableSqlTemplate: "drop table %s",
}
sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, user, password, hostname, port, database)

77
weed/filer/mysql2/mysql_store.go Normal file
View file

@ -0,0 +1,77 @@
package mysql2
import (
"database/sql"
"fmt"
"time"
"github.com/chrislusf/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/filer/abstract_sql"
"github.com/chrislusf/seaweedfs/weed/filer/mysql"
"github.com/chrislusf/seaweedfs/weed/util"
_ "github.com/go-sql-driver/mysql"
)
const (
CONNECTION_URL_PATTERN = "%s:%s@tcp(%s:%d)/%s?charset=utf8"
)
func init() {
filer.Stores = append(filer.Stores, &MysqlStore2{})
}
type MysqlStore2 struct {
abstract_sql.AbstractSqlStore
}
func (store *MysqlStore2) GetName() string {
return "mysql2"
}
func (store *MysqlStore2) Initialize(configuration util.Configuration, prefix string) (err error) {
return store.initialize(
configuration.GetString(prefix+"createTable"),
configuration.GetString(prefix+"username"),
configuration.GetString(prefix+"password"),
configuration.GetString(prefix+"hostname"),
configuration.GetInt(prefix+"port"),
configuration.GetString(prefix+"database"),
configuration.GetInt(prefix+"connection_max_idle"),
configuration.GetInt(prefix+"connection_max_open"),
configuration.GetInt(prefix+"connection_max_lifetime_seconds"),
configuration.GetBool(prefix+"interpolateParams"),
)
}
func (store *MysqlStore2) initialize(createTable, user, password, hostname string, port int, database string, maxIdle, maxOpen,
maxLifetimeSeconds int, interpolateParams bool) (err error) {
store.SupportBucketTable = true
store.SqlGenerator = &mysql.SqlGenMysql{
CreateTableSqlTemplate: createTable,
DropTableSqlTemplate: "drop table %s",
}
sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, user, password, hostname, port, database)
if interpolateParams {
sqlUrl += "&interpolateParams=true"
}
var dbErr error
store.DB, dbErr = sql.Open("mysql", sqlUrl)
if dbErr != nil {
store.DB.Close()
store.DB = nil
return fmt.Errorf("can not connect to %s error:%v", sqlUrl, err)
}
store.DB.SetMaxIdleConns(maxIdle)
store.DB.SetMaxOpenConns(maxOpen)
store.DB.SetConnMaxLifetime(time.Duration(maxLifetimeSeconds) * time.Second)
if err = store.DB.Ping(); err != nil {
return fmt.Errorf("connect to %s error:%v", sqlUrl, err)
}
return nil
}

1
weed/server/filer_server.go
View file

@ -29,6 +29,7 @@ import (
_ "github.com/chrislusf/seaweedfs/weed/filer/leveldb3"
_ "github.com/chrislusf/seaweedfs/weed/filer/mongodb"
_ "github.com/chrislusf/seaweedfs/weed/filer/mysql"
_ "github.com/chrislusf/seaweedfs/weed/filer/mysql2"
_ "github.com/chrislusf/seaweedfs/weed/filer/postgres"
_ "github.com/chrislusf/seaweedfs/weed/filer/redis"
_ "github.com/chrislusf/seaweedfs/weed/filer/redis2"

6
weed/util/config.go
View file

@ -29,11 +29,11 @@ func LoadConfiguration(configFileName string, required bool) (loaded bool) {
glog.V(1).Infof("Reading %s.toml from %s", configFileName, viper.ConfigFileUsed())
if err := viper.MergeInConfig(); err != nil { // Handle errors reading the config file
logLevel := glog.Level(0)
if strings.Contains(err.Error(), "Not Found") {
logLevel = 1
glog.V(1).Infof("Reading %s: %v", viper.ConfigFileUsed(), err)
} else {
glog.Fatalf("Reading %s: %v", viper.ConfigFileUsed(), err)
}
glog.V(logLevel).Infof("Reading %s: %v", viper.ConfigFileUsed(), err)
if required {
glog.Fatalf("Failed to load %s.toml file from current directory, or $HOME/.seaweedfs/, or /etc/seaweedfs/"+
"\n\nPlease use this command to generate the default %s.toml file\n"+