seaweedfs/weed/filer/arangodb/helpers.go
gfx 48db56ddad
arangodb s3 bucket name compatibility (#3588)
* Update arangodb_store.go

* update readme, properly escape queries, add name patching

* use underscore

* use underscore

* better comment

* fix readme

Co-authored-by: a <a@a.a>
2022-09-09 09:43:42 -07:00

155 lines
4.1 KiB
Go

package arangodb
import (
"context"
"crypto/md5"
"encoding/binary"
"encoding/hex"
"io"
"strings"
"github.com/arangodb/go-driver"
"github.com/seaweedfs/seaweedfs/weed/util"
)
// convert a string into arango-key safe hex bytes hash
func hashString(dir string) string {
h := md5.New()
io.WriteString(h, dir)
b := h.Sum(nil)
return hex.EncodeToString(b)
}
// convert slice of bytes into slice of uint64
// the first uint64 indicates the length in bytes
func bytesToArray(bs []byte) []uint64 {
out := make([]uint64, 0, 2+len(bs)/8)
out = append(out, uint64(len(bs)))
for len(bs)%8 != 0 {
bs = append(bs, 0)
}
for i := 0; i < len(bs); i = i + 8 {
out = append(out, binary.BigEndian.Uint64(bs[i:]))
}
return out
}
// convert from slice of uint64 back to bytes
// if input length is 0 or 1, will return nil
func arrayToBytes(xs []uint64) []byte {
if len(xs) < 2 {
return nil
}
first := xs[0]
out := make([]byte, len(xs)*8) // i think this can actually be len(xs)*8-8, but i dont think an extra 8 bytes hurts...
for i := 1; i < len(xs); i = i + 1 {
binary.BigEndian.PutUint64(out[((i-1)*8):], xs[i])
}
return out[:first]
}
// gets the collection the bucket points to from filepath
func (store *ArangodbStore) extractBucketCollection(ctx context.Context, fullpath util.FullPath) (c driver.Collection, err error) {
bucket, _ := extractBucket(fullpath)
if bucket == "" {
bucket = DEFAULT_COLLECTION
}
c, err = store.ensureBucket(ctx, bucket)
if err != nil {
return nil, err
}
return c, err
}
// called by extractBucketCollection
func extractBucket(fullpath util.FullPath) (string, string) {
if !strings.HasPrefix(string(fullpath), BUCKET_PREFIX+"/") {
return "", string(fullpath)
}
if strings.Count(string(fullpath), "/") < 3 {
return "", string(fullpath)
}
bucketAndObjectKey := string(fullpath)[len(BUCKET_PREFIX+"/"):]
t := strings.Index(bucketAndObjectKey, "/")
bucket := bucketAndObjectKey
shortPath := "/"
if t > 0 {
bucket = bucketAndObjectKey[:t]
shortPath = string(util.FullPath(bucketAndObjectKey[t:]))
}
return bucket, shortPath
}
// get bucket collection from cache. if not exist, creates the buckets collection and grab it
func (store *ArangodbStore) ensureBucket(ctx context.Context, bucket string) (bc driver.Collection, err error) {
var ok bool
store.mu.RLock()
bc, ok = store.buckets[bucket]
store.mu.RUnlock()
if ok && bc != nil {
return bc, nil
}
store.mu.Lock()
defer store.mu.Unlock()
store.buckets[bucket], err = store.ensureCollection(ctx, bucket)
if err != nil {
return nil, err
}
return store.buckets[bucket], nil
}
// transform to an arango compliant name
func bucketToCollectionName(s string) string {
if len(s) == 0 {
return ""
}
// replace all "." with _
s = strings.ReplaceAll(s, ".", "_")
// if starts with number or '.' then add a special prefix
if (s[0] >= '0' && s[0] <= '9') || (s[0] == '.' || s[0] == '_' || s[0] == '-') {
s = "xN--" + s
}
return s
}
// creates collection if not exist, ensures indices if not exist
func (store *ArangodbStore) ensureCollection(ctx context.Context, bucket_name string) (c driver.Collection, err error) {
// convert the bucket to collection name
name := bucketToCollectionName(bucket_name)
ok, err := store.database.CollectionExists(ctx, name)
if err != nil {
return
}
if ok {
c, err = store.database.Collection(ctx, name)
} else {
c, err = store.database.CreateCollection(ctx, name, &driver.CreateCollectionOptions{})
}
if err != nil {
return
}
// ensure indices
if _, _, err = c.EnsurePersistentIndex(ctx, []string{"directory", "name"},
&driver.EnsurePersistentIndexOptions{
Name: "directory_name_multi", Unique: true,
}); err != nil {
return
}
if _, _, err = c.EnsurePersistentIndex(ctx, []string{"directory"},
&driver.EnsurePersistentIndexOptions{Name: "IDX_directory"}); err != nil {
return
}
if _, _, err = c.EnsureTTLIndex(ctx, "ttl", 1,
&driver.EnsureTTLIndexOptions{Name: "IDX_TTL"}); err != nil {
return
}
if _, _, err = c.EnsurePersistentIndex(ctx, []string{"name"}, &driver.EnsurePersistentIndexOptions{
Name: "IDX_name",
}); err != nil {
return
}
return c, nil
}