support 128 bit NeedleId

weed/storage/needle.go

@@ -166,19 +166,19 @@ func (n *Needle) ParsePath(fid string) (err error) {
 
 func ParseNeedleIdCookie(key_hash_string string) (NeedleId, Cookie, error) {
 	if len(key_hash_string) <= CookieSize*2 {
-		return 0, 0, fmt.Errorf("KeyHash is too short.")
+		return NeedleIdEmpty, 0, fmt.Errorf("KeyHash is too short.")
 	}
 	if len(key_hash_string) > (NeedleIdSize+CookieSize)*2 {
-		return 0, 0, fmt.Errorf("KeyHash is too long.")
+		return NeedleIdEmpty, 0, fmt.Errorf("KeyHash is too long.")
 	}
 	split := len(key_hash_string) - CookieSize*2
 	needleId, err := ParseNeedleId(key_hash_string[:split])
 	if err != nil {
-		return 0, 0, fmt.Errorf("Parse needleId error: %v", err)
+		return NeedleIdEmpty, 0, fmt.Errorf("Parse needleId error: %v", err)
 	}
 	cookie, err := ParseCookie(key_hash_string[split:])
 	if err != nil {
-		return 0, 0, fmt.Errorf("Parse cookie error: %v", err)
+		return NeedleIdEmpty, 0, fmt.Errorf("Parse cookie error: %v", err)
 	}
 	return needleId, cookie, nil
 }

weed/storage/types/needle_id_128bit.go

@@ -0,0 +1,44 @@
+// +build 128BitNeedleId
+
+package types
+
+import (
+	"encoding/hex"
+)
+
+const (
+	NeedleIdSize = 16
+	NeedleIdEmpty = ""
+)
+
+// this is a 128 bit needle id implementation.
+// Usually a FileId has 32bit volume id, 64bit needle id, 32 bit cookie.
+// But when your system is using UUID, which is 128 bit, a custom 128-bit needle id can be easier to manage.
+// Caveat: In this mode, the fildId from master /dir/assign can not be directly used.
+// Only the volume id and cookie from the fileId are usuable.
+type NeedleId string
+
+func NeedleIdToBytes(bytes []byte, needleId NeedleId) {
+	hex.Decode(bytes, []byte(needleId))
+}
+
+// NeedleIdToUint64 used to send max needle id to master
+func NeedleIdToUint64(needleId NeedleId) uint64 {
+	return 0
+}
+
+func Uint64ToNeedleId(needleId uint64) NeedleId {
+	return NeedleId("")
+}
+
+func BytesToNeedleId(bytes []byte) (needleId NeedleId) {
+	return NeedleId(hex.EncodeToString(bytes))
+}
+
+func (k NeedleId) String() string {
+	return string(k)
+}
+
+func ParseNeedleId(idString string) (NeedleId, error) {
+	return NeedleId(idString), nil
+}

weed/storage/types/needle_id_type.go

@@ -1,3 +1,5 @@
+// +build !128BitNeedleId
+
 package types
 
 import (
@@ -10,6 +12,7 @@ type NeedleId uint64
 
 const (
 	NeedleIdSize = 8
+	NeedleIdEmpty = 0
 )
 
 func NeedleIdToBytes(bytes []byte, needleId NeedleId) {

weed/storage/volume_sync.go

@@ -91,7 +91,7 @@ func (v *Volume) trySynchronizing(volumeServer string, masterMap *needle.Compact
 	}
 	var delta []needle.NeedleValue
 	if err := masterMap.Visit(func(needleValue needle.NeedleValue) error {
-		if needleValue.Key == 0 {
+		if needleValue.Key == NeedleIdEmpty {
 			return nil
 		}
 		if _, ok := slaveMap.Get(needleValue.Key); ok {
@@ -103,7 +103,7 @@ func (v *Volume) trySynchronizing(volumeServer string, masterMap *needle.Compact
 		return fmt.Errorf("Add master entry: %v", err)
 	}
 	if err := slaveMap.m.Visit(func(needleValue needle.NeedleValue) error {
-		if needleValue.Key == 0 {
+		if needleValue.Key == NeedleIdEmpty {
 			return nil
 		}
 		if _, ok := masterMap.Get(needleValue.Key); ok {