diff --git a/go/operation/sync_volume.go b/go/operation/sync_volume.go index 54944a64e..626b2a7ba 100644 --- a/go/operation/sync_volume.go +++ b/go/operation/sync_volume.go @@ -1,6 +1,7 @@ package operation import ( + "encoding/binary" "encoding/json" "fmt" "net/url" @@ -42,9 +43,9 @@ func GetVolumeIdxEntries(server string, vid string, eachEntryFn func(key uint64, values.Add("volume", vid) line := make([]byte, 16) err := util.GetBufferStream("http://"+server+"/admin/sync/index", values, line, func(bytes []byte) { - key := util.BytesToUint64(bytes[:8]) - offset := util.BytesToUint32(bytes[8:12]) - size := util.BytesToUint32(bytes[12:16]) + key := binary.BigEndian.Uint64(bytes[:8]) + offset := binary.BigEndian.Uint32(bytes[8:12]) + size := binary.BigEndian.Uint32(bytes[12:16]) eachEntryFn(key, offset, size) }) if err != nil { diff --git a/go/storage/compact_map_perf_test.go b/go/storage/compact_map_perf_test.go index 1b429f263..7fd4c2dfb 100644 --- a/go/storage/compact_map_perf_test.go +++ b/go/storage/compact_map_perf_test.go @@ -1,12 +1,12 @@ package storage import ( + "encoding/binary" "log" "os" "testing" "github.com/chrislusf/seaweedfs/go/glog" - "github.com/chrislusf/seaweedfs/go/util" ) func TestMemoryUsage(t *testing.T) { @@ -29,9 +29,9 @@ func LoadNewNeedleMap(file *os.File) CompactMap { } for count > 0 && e == nil { for i := 0; i < count; i += 16 { - key := util.BytesToUint64(bytes[i : i+8]) - offset := util.BytesToUint32(bytes[i+8 : i+12]) - size := util.BytesToUint32(bytes[i+12 : i+16]) + key := binary.BigEndian.Uint64(bytes[i : i+8]) + offset := binary.BigEndian.Uint32(bytes[i+8 : i+12]) + size := binary.BigEndian.Uint32(bytes[i+12 : i+16]) if offset > 0 { m.Set(Key(key), offset, size) } else { diff --git a/go/storage/crc.go b/go/storage/crc.go index 21e384854..3150b786c 100644 --- a/go/storage/crc.go +++ b/go/storage/crc.go @@ -1,10 +1,10 @@ package storage import ( + "encoding/binary" "fmt" - "github.com/klauspost/crc32" - "github.com/chrislusf/seaweedfs/go/util" + "github.com/klauspost/crc32" ) var table = crc32.MakeTable(crc32.Castagnoli) @@ -25,6 +25,6 @@ func (c CRC) Value() uint32 { func (n *Needle) Etag() string { bits := make([]byte, 4) - util.Uint32toBytes(bits, uint32(n.Checksum)) + binary.BigEndian.PutUint32(bits, uint32(n.Checksum)) return fmt.Sprintf("\"%x\"", bits) } diff --git a/go/storage/file_id.go b/go/storage/file_id.go index 64b61ba89..3d2944403 100644 --- a/go/storage/file_id.go +++ b/go/storage/file_id.go @@ -1,12 +1,12 @@ package storage import ( + "encoding/binary" "encoding/hex" "errors" "strings" "github.com/chrislusf/seaweedfs/go/glog" - "github.com/chrislusf/seaweedfs/go/util" ) type FileId struct { @@ -34,8 +34,8 @@ func ParseFileId(fid string) (*FileId, error) { } func (n *FileId) String() string { bytes := make([]byte, 12) - util.Uint64toBytes(bytes[0:8], n.Key) - util.Uint32toBytes(bytes[8:12], n.Hashcode) + binary.BigEndian.PutUint64(bytes[0:8], n.Key) + binary.BigEndian.PutUint32(bytes[8:12], n.Hashcode) nonzero_index := 0 for ; bytes[nonzero_index] == 0; nonzero_index++ { } diff --git a/go/storage/needle.go b/go/storage/needle.go index e49368820..66cf5b22a 100644 --- a/go/storage/needle.go +++ b/go/storage/needle.go @@ -1,7 +1,6 @@ package storage import ( - "encoding/hex" "errors" "fmt" "io/ioutil" @@ -15,7 +14,6 @@ import ( "github.com/chrislusf/seaweedfs/go/glog" "github.com/chrislusf/seaweedfs/go/images" "github.com/chrislusf/seaweedfs/go/operation" - "github.com/chrislusf/seaweedfs/go/util" ) const ( @@ -213,16 +211,25 @@ func (n *Needle) ParsePath(fid string) (err error) { } func ParseKeyHash(key_hash_string string) (uint64, uint32, error) { - if len(key_hash_string)%2 == 1 { - key_hash_string = "0" + key_hash_string - } - key_hash_bytes, khe := hex.DecodeString(key_hash_string) - key_hash_len := len(key_hash_bytes) - if khe != nil || key_hash_len <= 4 { - glog.V(0).Infoln("Invalid key_hash", key_hash_string, "length:", key_hash_len, "error", khe) + key, hash, ok := parseKeyHash(key_hash_string) + if !ok { return 0, 0, errors.New("Invalid key and hash:" + key_hash_string) } - key := util.BytesToUint64(key_hash_bytes[0 : key_hash_len-4]) - hash := util.BytesToUint32(key_hash_bytes[key_hash_len-4 : key_hash_len]) return key, hash, nil } + +func parseKeyHash(keyhash string) (uint64, uint32, bool) { + if len(keyhash) <= 8 || len(keyhash) > 24 { + return 0, 0, false + } + split := len(keyhash) - 8 + key, err := strconv.ParseUint(keyhash[:split], 16, 64) + if err != nil { + return 0, 0, false + } + hash, err := strconv.ParseUint(keyhash[split:], 16, 32) + if err != nil { + return 0, 0, false + } + return key, uint32(hash), true +} diff --git a/go/storage/needle_map.go b/go/storage/needle_map.go index 814789616..41f53b90b 100644 --- a/go/storage/needle_map.go +++ b/go/storage/needle_map.go @@ -1,12 +1,11 @@ package storage import ( + "encoding/binary" "fmt" "io/ioutil" "os" "sync" - - "github.com/chrislusf/seaweedfs/go/util" ) type NeedleMapType int @@ -53,16 +52,16 @@ func (nm baseNeedleMapper) IndexFileName() string { } func idxFileEntry(bytes []byte) (key uint64, offset uint32, size uint32) { - key = util.BytesToUint64(bytes[:8]) - offset = util.BytesToUint32(bytes[8:12]) - size = util.BytesToUint32(bytes[12:16]) + key = binary.BigEndian.Uint64(bytes[:8]) + offset = binary.BigEndian.Uint32(bytes[8:12]) + size = binary.BigEndian.Uint32(bytes[12:16]) return } func (nm baseNeedleMapper) appendToIndexFile(key uint64, offset uint32, size uint32) error { bytes := make([]byte, 16) - util.Uint64toBytes(bytes[0:8], key) - util.Uint32toBytes(bytes[8:12], offset) - util.Uint32toBytes(bytes[12:16], size) + binary.BigEndian.PutUint64(bytes[0:8], key) + binary.BigEndian.PutUint32(bytes[8:12], offset) + binary.BigEndian.PutUint32(bytes[12:16], size) nm.indexFileAccessLock.Lock() defer nm.indexFileAccessLock.Unlock() diff --git a/go/storage/needle_map_boltdb.go b/go/storage/needle_map_boltdb.go index e95c016bb..a17e6ccf6 100644 --- a/go/storage/needle_map_boltdb.go +++ b/go/storage/needle_map_boltdb.go @@ -1,13 +1,12 @@ package storage import ( + "encoding/binary" "fmt" "os" "github.com/boltdb/bolt" - "github.com/chrislusf/seaweedfs/go/glog" - "github.com/chrislusf/seaweedfs/go/util" ) type BoltDbNeedleMap struct { @@ -75,7 +74,7 @@ func generateBoltDbFile(dbFileName string, indexFile *os.File) error { func (m *BoltDbNeedleMap) Get(key uint64) (element *NeedleValue, ok bool) { bytes := make([]byte, 8) var data []byte - util.Uint64toBytes(bytes, key) + binary.BigEndian.PutUint64(bytes, key) err := m.db.View(func(tx *bolt.Tx) error { bucket := tx.Bucket(boltdbBucket) if bucket == nil { @@ -89,8 +88,8 @@ func (m *BoltDbNeedleMap) Get(key uint64) (element *NeedleValue, ok bool) { if err != nil || len(data) != 8 { return nil, false } - offset := util.BytesToUint32(data[0:4]) - size := util.BytesToUint32(data[4:8]) + offset := binary.BigEndian.Uint32(data[0:4]) + size := binary.BigEndian.Uint32(data[4:8]) return &NeedleValue{Key: Key(key), Offset: offset, Size: size}, true } @@ -110,9 +109,9 @@ func (m *BoltDbNeedleMap) Put(key uint64, offset uint32, size uint32) error { func boltDbWrite(db *bolt.DB, key uint64, offset uint32, size uint32) error { bytes := make([]byte, 16) - util.Uint64toBytes(bytes[0:8], key) - util.Uint32toBytes(bytes[8:12], offset) - util.Uint32toBytes(bytes[12:16], size) + binary.BigEndian.PutUint64(bytes[0:8], key) + binary.BigEndian.PutUint32(bytes[8:12], offset) + binary.BigEndian.PutUint32(bytes[12:16], size) return db.Update(func(tx *bolt.Tx) error { bucket, err := tx.CreateBucketIfNotExists(boltdbBucket) if err != nil { @@ -128,7 +127,7 @@ func boltDbWrite(db *bolt.DB, } func boltDbDelete(db *bolt.DB, key uint64) error { bytes := make([]byte, 8) - util.Uint64toBytes(bytes, key) + binary.BigEndian.PutUint64(bytes, key) return db.Update(func(tx *bolt.Tx) error { bucket, err := tx.CreateBucketIfNotExists(boltdbBucket) if err != nil { diff --git a/go/storage/needle_map_leveldb.go b/go/storage/needle_map_leveldb.go index 47f63e3ae..f9e580f6a 100644 --- a/go/storage/needle_map_leveldb.go +++ b/go/storage/needle_map_leveldb.go @@ -1,12 +1,12 @@ package storage import ( + "encoding/binary" "fmt" "os" "path/filepath" "github.com/chrislusf/seaweedfs/go/glog" - "github.com/chrislusf/seaweedfs/go/util" "github.com/syndtr/goleveldb/leveldb" ) @@ -72,13 +72,13 @@ func generateLevelDbFile(dbFileName string, indexFile *os.File) error { func (m *LevelDbNeedleMap) Get(key uint64) (element *NeedleValue, ok bool) { bytes := make([]byte, 8) - util.Uint64toBytes(bytes, key) + binary.BigEndian.PutUint64(bytes, key) data, err := m.db.Get(bytes, nil) if err != nil || len(data) != 8 { return nil, false } - offset := util.BytesToUint32(data[0:4]) - size := util.BytesToUint32(data[4:8]) + offset := binary.BigEndian.Uint32(data[0:4]) + size := binary.BigEndian.Uint32(data[4:8]) return &NeedleValue{Key: Key(key), Offset: offset, Size: size}, true } @@ -98,9 +98,9 @@ func (m *LevelDbNeedleMap) Put(key uint64, offset uint32, size uint32) error { func levelDbWrite(db *leveldb.DB, key uint64, offset uint32, size uint32) error { bytes := make([]byte, 16) - util.Uint64toBytes(bytes[0:8], key) - util.Uint32toBytes(bytes[8:12], offset) - util.Uint32toBytes(bytes[12:16], size) + binary.BigEndian.PutUint64(bytes[0:8], key) + binary.BigEndian.PutUint32(bytes[8:12], offset) + binary.BigEndian.PutUint32(bytes[12:16], size) if err := db.Put(bytes[0:8], bytes[8:16], nil); err != nil { return fmt.Errorf("failed to write leveldb: %v", err) } @@ -108,7 +108,7 @@ func levelDbWrite(db *leveldb.DB, } func levelDbDelete(db *leveldb.DB, key uint64) error { bytes := make([]byte, 8) - util.Uint64toBytes(bytes, key) + binary.BigEndian.PutUint64(bytes, key) return db.Delete(bytes, nil) } diff --git a/go/storage/needle_read_write.go b/go/storage/needle_read_write.go index ccfe1d498..67adaa65f 100644 --- a/go/storage/needle_read_write.go +++ b/go/storage/needle_read_write.go @@ -1,13 +1,13 @@ package storage import ( + "encoding/binary" "errors" "fmt" "io" "os" "github.com/chrislusf/seaweedfs/go/glog" - "github.com/chrislusf/seaweedfs/go/util" ) const ( @@ -43,11 +43,11 @@ func (n *Needle) Append(w io.Writer, version Version) (size uint32, err error) { switch version { case Version1: header := make([]byte, NeedleHeaderSize) - util.Uint32toBytes(header[0:4], n.Cookie) - util.Uint64toBytes(header[4:12], n.Id) + binary.BigEndian.PutUint32(header[0:4], n.Cookie) + binary.BigEndian.PutUint64(header[4:12], n.Id) n.Size = uint32(len(n.Data)) size = n.Size - util.Uint32toBytes(header[12:16], n.Size) + binary.BigEndian.PutUint32(header[12:16], n.Size) if _, err = w.Write(header); err != nil { return } @@ -55,13 +55,13 @@ func (n *Needle) Append(w io.Writer, version Version) (size uint32, err error) { return } padding := NeedlePaddingSize - ((NeedleHeaderSize + n.Size + NeedleChecksumSize) % NeedlePaddingSize) - util.Uint32toBytes(header[0:NeedleChecksumSize], n.Checksum.Value()) + binary.BigEndian.PutUint32(header[0:NeedleChecksumSize], n.Checksum.Value()) _, err = w.Write(header[0 : NeedleChecksumSize+padding]) return case Version2: header := make([]byte, NeedleHeaderSize) - util.Uint32toBytes(header[0:4], n.Cookie) - util.Uint64toBytes(header[4:12], n.Id) + binary.BigEndian.PutUint32(header[0:4], n.Cookie) + binary.BigEndian.PutUint64(header[4:12], n.Id) n.DataSize, n.NameSize, n.MimeSize = uint32(len(n.Data)), uint8(len(n.Name)), uint8(len(n.Mime)) if n.DataSize > 0 { n.Size = 4 + n.DataSize + 1 @@ -81,24 +81,24 @@ func (n *Needle) Append(w io.Writer, version Version) (size uint32, err error) { n.Size = 0 } size = n.DataSize - util.Uint32toBytes(header[12:16], n.Size) + binary.BigEndian.PutUint32(header[12:16], n.Size) if _, err = w.Write(header); err != nil { return } if n.DataSize > 0 { - util.Uint32toBytes(header[0:4], n.DataSize) + binary.BigEndian.PutUint32(header[0:4], n.DataSize) if _, err = w.Write(header[0:4]); err != nil { return } if _, err = w.Write(n.Data); err != nil { return } - util.Uint8toBytes(header[0:1], n.Flags) + header[0] = n.Flags if _, err = w.Write(header[0:1]); err != nil { return } if n.HasName() { - util.Uint8toBytes(header[0:1], n.NameSize) + header[0] = n.NameSize if _, err = w.Write(header[0:1]); err != nil { return } @@ -107,7 +107,7 @@ func (n *Needle) Append(w io.Writer, version Version) (size uint32, err error) { } } if n.HasMime() { - util.Uint8toBytes(header[0:1], n.MimeSize) + header[0] = n.MimeSize if _, err = w.Write(header[0:1]); err != nil { return } @@ -116,7 +116,7 @@ func (n *Needle) Append(w io.Writer, version Version) (size uint32, err error) { } } if n.HasLastModifiedDate() { - util.Uint64toBytes(header[0:8], n.LastModified) + binary.BigEndian.PutUint64(header[0:8], n.LastModified) if _, err = w.Write(header[8-LastModifiedBytesLength : 8]); err != nil { return } @@ -129,7 +129,7 @@ func (n *Needle) Append(w io.Writer, version Version) (size uint32, err error) { } } padding := NeedlePaddingSize - ((NeedleHeaderSize + n.Size + NeedleChecksumSize) % NeedlePaddingSize) - util.Uint32toBytes(header[0:NeedleChecksumSize], n.Checksum.Value()) + binary.BigEndian.PutUint32(header[0:NeedleChecksumSize], n.Checksum.Value()) _, err = w.Write(header[0 : NeedleChecksumSize+padding]) return n.DataSize, err } @@ -158,7 +158,7 @@ func (n *Needle) ReadData(r *os.File, offset int64, size uint32, version Version case Version2: n.readNeedleDataVersion2(bytes[NeedleHeaderSize : NeedleHeaderSize+int(n.Size)]) } - checksum := util.BytesToUint32(bytes[NeedleHeaderSize+size : NeedleHeaderSize+size+NeedleChecksumSize]) + checksum := binary.BigEndian.Uint32(bytes[NeedleHeaderSize+size : NeedleHeaderSize+size+NeedleChecksumSize]) newChecksum := NewCRC(n.Data) if checksum != newChecksum.Value() { return errors.New("CRC error! Data On Disk Corrupted") @@ -167,14 +167,14 @@ func (n *Needle) ReadData(r *os.File, offset int64, size uint32, version Version return nil } func (n *Needle) ParseNeedleHeader(bytes []byte) { - n.Cookie = util.BytesToUint32(bytes[0:4]) - n.Id = util.BytesToUint64(bytes[4:12]) - n.Size = util.BytesToUint32(bytes[12:NeedleHeaderSize]) + n.Cookie = binary.BigEndian.Uint32(bytes[0:4]) + n.Id = binary.BigEndian.Uint64(bytes[4:12]) + n.Size = binary.BigEndian.Uint32(bytes[12:NeedleHeaderSize]) } func (n *Needle) readNeedleDataVersion2(bytes []byte) { index, lenBytes := 0, len(bytes) if index < lenBytes { - n.DataSize = util.BytesToUint32(bytes[index : index+4]) + n.DataSize = binary.BigEndian.Uint32(bytes[index : index+4]) index = index + 4 if int(n.DataSize)+index > lenBytes { // this if clause is due to bug #87 and #93, fixed in v0.69 @@ -199,7 +199,7 @@ func (n *Needle) readNeedleDataVersion2(bytes []byte) { index = index + int(n.MimeSize) } if index < lenBytes && n.HasLastModifiedDate() { - n.LastModified = util.BytesToUint64(bytes[index : index+LastModifiedBytesLength]) + n.LastModified = uint64(bytes[index])<<32 | uint64(binary.BigEndian.Uint32(bytes[index+1:index+LastModifiedBytesLength])) index = index + LastModifiedBytesLength } if index < lenBytes && n.HasTtl() { diff --git a/go/storage/needle_test.go b/go/storage/needle_test.go new file mode 100644 index 000000000..c05afda2f --- /dev/null +++ b/go/storage/needle_test.go @@ -0,0 +1,45 @@ +package storage + +import "testing" + +func TestParseKeyHash(t *testing.T) { + testcases := []struct { + KeyHash string + ID uint64 + Cookie uint32 + Err bool + }{ + // normal + {"4ed4c8116e41", 0x4ed4, 0xc8116e41, false}, + // cookie with leading zeros + {"4ed401116e41", 0x4ed4, 0x01116e41, false}, + // odd length + {"ed400116e41", 0xed4, 0x00116e41, false}, + // uint + {"fed4c8114ed4c811f0116e41", 0xfed4c8114ed4c811, 0xf0116e41, false}, + // err: too short + {"4ed4c811", 0, 0, true}, + // err: too long + {"4ed4c8114ed4c8114ed4c8111", 0, 0, true}, + // err: invalid character + {"helloworld", 0, 0, true}, + } + + for _, tc := range testcases { + if id, cookie, err := ParseKeyHash(tc.KeyHash); err != nil && !tc.Err { + t.Fatalf("Parse %s error: %v", tc.KeyHash, err) + } else if err == nil && tc.Err { + t.Fatalf("Parse %s expected error got nil", tc.KeyHash) + } else if id != tc.ID || cookie != tc.Cookie { + t.Fatalf("Parse %s wrong result. Expected: (%d, %d) got: (%d, %d)", tc.KeyHash, tc.ID, tc.Cookie, id, cookie) + } + } +} + +func BenchmarkParseKeyHash(b *testing.B) { + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + ParseKeyHash("4ed44ed44ed44ed4c8116e41") + } +} diff --git a/go/storage/volume_super_block.go b/go/storage/volume_super_block.go index e37360075..76116090d 100644 --- a/go/storage/volume_super_block.go +++ b/go/storage/volume_super_block.go @@ -1,11 +1,11 @@ package storage import ( + "encoding/binary" "fmt" "os" "github.com/chrislusf/seaweedfs/go/glog" - "github.com/chrislusf/seaweedfs/go/util" ) const ( @@ -35,7 +35,7 @@ func (s *SuperBlock) Bytes() []byte { header[0] = byte(s.version) header[1] = s.ReplicaPlacement.Byte() s.Ttl.ToBytes(header[2:4]) - util.Uint16toBytes(header[4:6], s.CompactRevision) + binary.BigEndian.PutUint16(header[4:6], s.CompactRevision) return header } @@ -76,6 +76,6 @@ func ParseSuperBlock(header []byte) (superBlock SuperBlock, err error) { err = fmt.Errorf("cannot read replica type: %s", err.Error()) } superBlock.Ttl = LoadTTLFromBytes(header[2:4]) - superBlock.CompactRevision = util.BytesToUint16(header[4:6]) + superBlock.CompactRevision = binary.BigEndian.Uint16(header[4:6]) return } diff --git a/go/util/bytes.go b/go/util/bytes.go deleted file mode 100644 index dfa4ae665..000000000 --- a/go/util/bytes.go +++ /dev/null @@ -1,45 +0,0 @@ -package util - -// big endian - -func BytesToUint64(b []byte) (v uint64) { - length := uint(len(b)) - for i := uint(0); i < length-1; i++ { - v += uint64(b[i]) - v <<= 8 - } - v += uint64(b[length-1]) - return -} -func BytesToUint32(b []byte) (v uint32) { - length := uint(len(b)) - for i := uint(0); i < length-1; i++ { - v += uint32(b[i]) - v <<= 8 - } - v += uint32(b[length-1]) - return -} -func BytesToUint16(b []byte) (v uint16) { - v += uint16(b[0]) - v <<= 8 - v += uint16(b[1]) - return -} -func Uint64toBytes(b []byte, v uint64) { - for i := uint(0); i < 8; i++ { - b[7-i] = byte(v >> (i * 8)) - } -} -func Uint32toBytes(b []byte, v uint32) { - for i := uint(0); i < 4; i++ { - b[3-i] = byte(v >> (i * 8)) - } -} -func Uint16toBytes(b []byte, v uint16) { - b[0] = byte(v >> 8) - b[1] = byte(v) -} -func Uint8toBytes(b []byte, v uint8) { - b[0] = byte(v) -}