From 2da84ed33115a4f2908479c86442b1936ed1b616 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 26 May 2018 13:35:56 -0700 Subject: [PATCH] working fine now but index length can be improved --- weed/filer2/abstract_sql/abstract_sql_store.go | 17 +++++++++++------ weed/filer2/configuration.go | 18 +++++++++++------- weed/filer2/mysql/mysql_store.go | 9 +++++++-- 3 files changed, 29 insertions(+), 15 deletions(-) diff --git a/weed/filer2/abstract_sql/abstract_sql_store.go b/weed/filer2/abstract_sql/abstract_sql_store.go index 252a3c549..dd037b6af 100644 --- a/weed/filer2/abstract_sql/abstract_sql_store.go +++ b/weed/filer2/abstract_sql/abstract_sql_store.go @@ -20,7 +20,7 @@ func (store *AbstractSqlStore) InsertEntry(entry *filer2.Entry) (err error) { return fmt.Errorf("mysql encode %s: %s", entry.FullPath, err) } - res, err := store.DB.Exec("INSERT INTO seaweedfs (directory,name,meta) VALUES(?,?,?)", dir, name, meta) + res, err := store.DB.Exec("INSERT INTO filemeta (directory,name,meta) VALUES(?,?,?)", dir, name, meta) if err != nil { return fmt.Errorf("mysql insert %s: %s", entry.FullPath, err) } @@ -40,7 +40,7 @@ func (store *AbstractSqlStore) UpdateEntry(entry *filer2.Entry) (err error) { return fmt.Errorf("mysql encode %s: %s", entry.FullPath, err) } - res, err := store.DB.Exec("UPDATE seaweedfs SET meta=? WHERE directory=? and name=?", dir, name, meta) + res, err := store.DB.Exec("UPDATE filemeta SET meta=? WHERE directory=? and name=?", dir, name, meta) if err != nil { return fmt.Errorf("mysql update %s: %s", entry.FullPath, err) } @@ -55,7 +55,7 @@ func (store *AbstractSqlStore) UpdateEntry(entry *filer2.Entry) (err error) { func (store *AbstractSqlStore) FindEntry(fullpath filer2.FullPath) (*filer2.Entry, error) { dir, name := fullpath.DirAndName() - row := store.DB.QueryRow("SELECT meta FROM seaweedfs WHERE directory=? and name=?", dir, name) + row := store.DB.QueryRow("SELECT meta FROM filemeta WHERE directory=? and name=?", dir, name) var data []byte if err := row.Scan(&data); err != nil { return nil, fmt.Errorf("mysql read entry %s: %v", fullpath, err) @@ -77,7 +77,7 @@ func (store *AbstractSqlStore) DeleteEntry(fullpath filer2.FullPath) (*filer2.En dir, name := fullpath.DirAndName() - res, err := store.DB.Exec("DELETE FROM seaweedfs WHERE directory=? and name=?", dir, name) + res, err := store.DB.Exec("DELETE FROM filemeta WHERE directory=? and name=?", dir, name) if err != nil { return nil, fmt.Errorf("mysql delete %s: %s", fullpath, err) } @@ -92,7 +92,12 @@ func (store *AbstractSqlStore) DeleteEntry(fullpath filer2.FullPath) (*filer2.En func (store *AbstractSqlStore) ListDirectoryEntries(fullpath filer2.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) { - rows, err := store.DB.Query("SELECT NAME, meta FROM seaweedfs WHERE directory=? and name>?", fullpath, startFileName) + sqlText := "SELECT NAME, meta FROM filemeta WHERE directory=? and name>? LIMIT ?" + if inclusive { + sqlText = "SELECT NAME, meta FROM filemeta WHERE directory=? and name>=? LIMIT ?" + } + + rows, err := store.DB.Query(sqlText, string(fullpath), startFileName, limit) if err != nil { return nil, fmt.Errorf("mysql list %s : %v", fullpath, err) } @@ -107,7 +112,7 @@ func (store *AbstractSqlStore) ListDirectoryEntries(fullpath filer2.FullPath, st } entry := &filer2.Entry{ - FullPath: fullpath, + FullPath: filer2.NewFullPath(string(fullpath), name), } if err = entry.DecodeAttributesAndChunks(data); err != nil { glog.V(0).Infof("mysql scan decode %s : %v", entry.FullPath, err) diff --git a/weed/filer2/configuration.go b/weed/filer2/configuration.go index caca0a49c..ddd06a568 100644 --- a/weed/filer2/configuration.go +++ b/weed/filer2/configuration.go @@ -11,28 +11,32 @@ const ( FILER_TOML_EXAMPLE = ` # A sample TOML config file for SeaweedFS filer store -# local in memory, mostly for testing purpose [memory] +# local in memory, mostly for testing purpose enabled = false [leveldb] +# local on disk, mostly for simple single-machine setup, fairly scalable enabled = false dir = "." # directory to store level db files [mysql] -# also need to manually create seaweedfs table. -# CREATE TABLE IF NOT EXISTS seaweedfs ( -# directory VARCHAR(4096) NOT NULL DEFAULT "" COMMENT 'full path to parent directory', -# name VARCHAR(1024) NOT NULL DEFAULT "" COMMENT 'directory or file name', +# multiple filers on shared storage, fairly scalable +# +# need to choose or create a database. +# need to manually create a table "filemeta". +# CREATE TABLE IF NOT EXISTS filemeta ( +# directory VARCHAR(512) NOT NULL DEFAULT "" COMMENT 'full path to parent directory', +# name VARCHAR(512) NOT NULL DEFAULT "" COMMENT 'directory or file name', # meta BLOB, -# PRIMARY KEY (directory, name), +# PRIMARY KEY (directory, name) # ) DEFAULT CHARSET=utf8; enabled = true server = "localhost" port = 3306 username = "root" password = "" -database = "" +database = "" # create or use an existing database, create the seaweedfs table. connection_max_idle = 2 connection_max_open = 100 diff --git a/weed/filer2/mysql/mysql_store.go b/weed/filer2/mysql/mysql_store.go index 4accc9a9f..16fd4311f 100644 --- a/weed/filer2/mysql/mysql_store.go +++ b/weed/filer2/mysql/mysql_store.go @@ -31,14 +31,14 @@ func (store *MysqlStore) Initialize(viper *viper.Viper) (err error) { viper.GetString("username"), viper.GetString("password"), viper.GetString("hostname"), - viper.GetString("port"), + viper.GetInt("port"), viper.GetString("database"), viper.GetInt("connection_max_idle"), viper.GetInt("connection_max_open"), ) } -func (store *MysqlStore) initialize(user, password, hostname, port, database string, maxIdle, maxOpen int) (err error) { +func (store *MysqlStore) initialize(user, password, hostname string, port int, database string, maxIdle, maxOpen int) (err error) { sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, user, password, hostname, port, database) var dbErr error store.DB, dbErr = sql.Open("mysql", sqlUrl) @@ -50,5 +50,10 @@ func (store *MysqlStore) initialize(user, password, hostname, port, database str store.DB.SetMaxIdleConns(maxIdle) store.DB.SetMaxOpenConns(maxOpen) + + if err = store.DB.Ping(); err != nil { + return fmt.Errorf("connect to %s error:%v", sqlUrl, err) + } + return nil }