From b39c384d6dfa63133990a1e190525ef3133f1530 Mon Sep 17 00:00:00 2001 From: chrislusf Date: Sun, 10 Apr 2016 00:24:22 -0700 Subject: [PATCH] Revert "Merge pull request #284 from thinxer/binary" This reverts commit 3523ad523929870aa8d4a7741ee8e152cfd40489, reversing changes made to 5d100994b1b9c6679113c8dd73f97aff85397f02. --- go/operation/sync_volume.go | 7 ++--- go/storage/compact_map_perf_test.go | 8 ++--- go/storage/crc.go | 6 ++-- go/storage/file_id.go | 6 ++-- go/storage/needle.go | 29 +++++++------------ go/storage/needle_map.go | 15 +++++----- go/storage/needle_map_boltdb.go | 17 ++++++----- go/storage/needle_map_leveldb.go | 16 +++++----- go/storage/needle_read_write.go | 40 ++++++++++++------------- go/storage/needle_test.go | 45 ----------------------------- go/storage/volume_super_block.go | 6 ++-- go/util/bytes.go | 45 +++++++++++++++++++++++++++++ 12 files changed, 117 insertions(+), 123 deletions(-) delete mode 100644 go/storage/needle_test.go create mode 100644 go/util/bytes.go diff --git a/go/operation/sync_volume.go b/go/operation/sync_volume.go index 626b2a7ba..54944a64e 100644 --- a/go/operation/sync_volume.go +++ b/go/operation/sync_volume.go @@ -1,7 +1,6 @@ package operation import ( - "encoding/binary" "encoding/json" "fmt" "net/url" @@ -43,9 +42,9 @@ func GetVolumeIdxEntries(server string, vid string, eachEntryFn func(key uint64, values.Add("volume", vid) line := make([]byte, 16) err := util.GetBufferStream("http://"+server+"/admin/sync/index", values, line, func(bytes []byte) { - key := binary.BigEndian.Uint64(bytes[:8]) - offset := binary.BigEndian.Uint32(bytes[8:12]) - size := binary.BigEndian.Uint32(bytes[12:16]) + key := util.BytesToUint64(bytes[:8]) + offset := util.BytesToUint32(bytes[8:12]) + size := util.BytesToUint32(bytes[12:16]) eachEntryFn(key, offset, size) }) if err != nil { diff --git a/go/storage/compact_map_perf_test.go b/go/storage/compact_map_perf_test.go index 7fd4c2dfb..1b429f263 100644 --- a/go/storage/compact_map_perf_test.go +++ b/go/storage/compact_map_perf_test.go @@ -1,12 +1,12 @@ package storage import ( - "encoding/binary" "log" "os" "testing" "github.com/chrislusf/seaweedfs/go/glog" + "github.com/chrislusf/seaweedfs/go/util" ) func TestMemoryUsage(t *testing.T) { @@ -29,9 +29,9 @@ func LoadNewNeedleMap(file *os.File) CompactMap { } for count > 0 && e == nil { for i := 0; i < count; i += 16 { - key := binary.BigEndian.Uint64(bytes[i : i+8]) - offset := binary.BigEndian.Uint32(bytes[i+8 : i+12]) - size := binary.BigEndian.Uint32(bytes[i+12 : i+16]) + key := util.BytesToUint64(bytes[i : i+8]) + offset := util.BytesToUint32(bytes[i+8 : i+12]) + size := util.BytesToUint32(bytes[i+12 : i+16]) if offset > 0 { m.Set(Key(key), offset, size) } else { diff --git a/go/storage/crc.go b/go/storage/crc.go index 3150b786c..21e384854 100644 --- a/go/storage/crc.go +++ b/go/storage/crc.go @@ -1,10 +1,10 @@ package storage import ( - "encoding/binary" "fmt" - "github.com/klauspost/crc32" + + "github.com/chrislusf/seaweedfs/go/util" ) var table = crc32.MakeTable(crc32.Castagnoli) @@ -25,6 +25,6 @@ func (c CRC) Value() uint32 { func (n *Needle) Etag() string { bits := make([]byte, 4) - binary.BigEndian.PutUint32(bits, uint32(n.Checksum)) + util.Uint32toBytes(bits, uint32(n.Checksum)) return fmt.Sprintf("\"%x\"", bits) } diff --git a/go/storage/file_id.go b/go/storage/file_id.go index 3d2944403..64b61ba89 100644 --- a/go/storage/file_id.go +++ b/go/storage/file_id.go @@ -1,12 +1,12 @@ package storage import ( - "encoding/binary" "encoding/hex" "errors" "strings" "github.com/chrislusf/seaweedfs/go/glog" + "github.com/chrislusf/seaweedfs/go/util" ) type FileId struct { @@ -34,8 +34,8 @@ func ParseFileId(fid string) (*FileId, error) { } func (n *FileId) String() string { bytes := make([]byte, 12) - binary.BigEndian.PutUint64(bytes[0:8], n.Key) - binary.BigEndian.PutUint32(bytes[8:12], n.Hashcode) + util.Uint64toBytes(bytes[0:8], n.Key) + util.Uint32toBytes(bytes[8:12], n.Hashcode) nonzero_index := 0 for ; bytes[nonzero_index] == 0; nonzero_index++ { } diff --git a/go/storage/needle.go b/go/storage/needle.go index 66cf5b22a..e49368820 100644 --- a/go/storage/needle.go +++ b/go/storage/needle.go @@ -1,6 +1,7 @@ package storage import ( + "encoding/hex" "errors" "fmt" "io/ioutil" @@ -14,6 +15,7 @@ import ( "github.com/chrislusf/seaweedfs/go/glog" "github.com/chrislusf/seaweedfs/go/images" "github.com/chrislusf/seaweedfs/go/operation" + "github.com/chrislusf/seaweedfs/go/util" ) const ( @@ -211,25 +213,16 @@ func (n *Needle) ParsePath(fid string) (err error) { } func ParseKeyHash(key_hash_string string) (uint64, uint32, error) { - key, hash, ok := parseKeyHash(key_hash_string) - if !ok { + if len(key_hash_string)%2 == 1 { + key_hash_string = "0" + key_hash_string + } + key_hash_bytes, khe := hex.DecodeString(key_hash_string) + key_hash_len := len(key_hash_bytes) + if khe != nil || key_hash_len <= 4 { + glog.V(0).Infoln("Invalid key_hash", key_hash_string, "length:", key_hash_len, "error", khe) return 0, 0, errors.New("Invalid key and hash:" + key_hash_string) } + key := util.BytesToUint64(key_hash_bytes[0 : key_hash_len-4]) + hash := util.BytesToUint32(key_hash_bytes[key_hash_len-4 : key_hash_len]) return key, hash, nil } - -func parseKeyHash(keyhash string) (uint64, uint32, bool) { - if len(keyhash) <= 8 || len(keyhash) > 24 { - return 0, 0, false - } - split := len(keyhash) - 8 - key, err := strconv.ParseUint(keyhash[:split], 16, 64) - if err != nil { - return 0, 0, false - } - hash, err := strconv.ParseUint(keyhash[split:], 16, 32) - if err != nil { - return 0, 0, false - } - return key, uint32(hash), true -} diff --git a/go/storage/needle_map.go b/go/storage/needle_map.go index 13fe4ffb1..638a9b4af 100644 --- a/go/storage/needle_map.go +++ b/go/storage/needle_map.go @@ -1,11 +1,12 @@ package storage import ( - "encoding/binary" "fmt" "io/ioutil" "os" "sync" + + "github.com/chrislusf/seaweedfs/go/util" ) type NeedleMapType int @@ -52,16 +53,16 @@ func (nm *baseNeedleMapper) IndexFileName() string { } func idxFileEntry(bytes []byte) (key uint64, offset uint32, size uint32) { - key = binary.BigEndian.Uint64(bytes[:8]) - offset = binary.BigEndian.Uint32(bytes[8:12]) - size = binary.BigEndian.Uint32(bytes[12:16]) + key = util.BytesToUint64(bytes[:8]) + offset = util.BytesToUint32(bytes[8:12]) + size = util.BytesToUint32(bytes[12:16]) return } func (nm *baseNeedleMapper) appendToIndexFile(key uint64, offset uint32, size uint32) error { bytes := make([]byte, 16) - binary.BigEndian.PutUint64(bytes[0:8], key) - binary.BigEndian.PutUint32(bytes[8:12], offset) - binary.BigEndian.PutUint32(bytes[12:16], size) + util.Uint64toBytes(bytes[0:8], key) + util.Uint32toBytes(bytes[8:12], offset) + util.Uint32toBytes(bytes[12:16], size) nm.indexFileAccessLock.Lock() defer nm.indexFileAccessLock.Unlock() diff --git a/go/storage/needle_map_boltdb.go b/go/storage/needle_map_boltdb.go index a17e6ccf6..e95c016bb 100644 --- a/go/storage/needle_map_boltdb.go +++ b/go/storage/needle_map_boltdb.go @@ -1,12 +1,13 @@ package storage import ( - "encoding/binary" "fmt" "os" "github.com/boltdb/bolt" + "github.com/chrislusf/seaweedfs/go/glog" + "github.com/chrislusf/seaweedfs/go/util" ) type BoltDbNeedleMap struct { @@ -74,7 +75,7 @@ func generateBoltDbFile(dbFileName string, indexFile *os.File) error { func (m *BoltDbNeedleMap) Get(key uint64) (element *NeedleValue, ok bool) { bytes := make([]byte, 8) var data []byte - binary.BigEndian.PutUint64(bytes, key) + util.Uint64toBytes(bytes, key) err := m.db.View(func(tx *bolt.Tx) error { bucket := tx.Bucket(boltdbBucket) if bucket == nil { @@ -88,8 +89,8 @@ func (m *BoltDbNeedleMap) Get(key uint64) (element *NeedleValue, ok bool) { if err != nil || len(data) != 8 { return nil, false } - offset := binary.BigEndian.Uint32(data[0:4]) - size := binary.BigEndian.Uint32(data[4:8]) + offset := util.BytesToUint32(data[0:4]) + size := util.BytesToUint32(data[4:8]) return &NeedleValue{Key: Key(key), Offset: offset, Size: size}, true } @@ -109,9 +110,9 @@ func (m *BoltDbNeedleMap) Put(key uint64, offset uint32, size uint32) error { func boltDbWrite(db *bolt.DB, key uint64, offset uint32, size uint32) error { bytes := make([]byte, 16) - binary.BigEndian.PutUint64(bytes[0:8], key) - binary.BigEndian.PutUint32(bytes[8:12], offset) - binary.BigEndian.PutUint32(bytes[12:16], size) + util.Uint64toBytes(bytes[0:8], key) + util.Uint32toBytes(bytes[8:12], offset) + util.Uint32toBytes(bytes[12:16], size) return db.Update(func(tx *bolt.Tx) error { bucket, err := tx.CreateBucketIfNotExists(boltdbBucket) if err != nil { @@ -127,7 +128,7 @@ func boltDbWrite(db *bolt.DB, } func boltDbDelete(db *bolt.DB, key uint64) error { bytes := make([]byte, 8) - binary.BigEndian.PutUint64(bytes, key) + util.Uint64toBytes(bytes, key) return db.Update(func(tx *bolt.Tx) error { bucket, err := tx.CreateBucketIfNotExists(boltdbBucket) if err != nil { diff --git a/go/storage/needle_map_leveldb.go b/go/storage/needle_map_leveldb.go index f9e580f6a..47f63e3ae 100644 --- a/go/storage/needle_map_leveldb.go +++ b/go/storage/needle_map_leveldb.go @@ -1,12 +1,12 @@ package storage import ( - "encoding/binary" "fmt" "os" "path/filepath" "github.com/chrislusf/seaweedfs/go/glog" + "github.com/chrislusf/seaweedfs/go/util" "github.com/syndtr/goleveldb/leveldb" ) @@ -72,13 +72,13 @@ func generateLevelDbFile(dbFileName string, indexFile *os.File) error { func (m *LevelDbNeedleMap) Get(key uint64) (element *NeedleValue, ok bool) { bytes := make([]byte, 8) - binary.BigEndian.PutUint64(bytes, key) + util.Uint64toBytes(bytes, key) data, err := m.db.Get(bytes, nil) if err != nil || len(data) != 8 { return nil, false } - offset := binary.BigEndian.Uint32(data[0:4]) - size := binary.BigEndian.Uint32(data[4:8]) + offset := util.BytesToUint32(data[0:4]) + size := util.BytesToUint32(data[4:8]) return &NeedleValue{Key: Key(key), Offset: offset, Size: size}, true } @@ -98,9 +98,9 @@ func (m *LevelDbNeedleMap) Put(key uint64, offset uint32, size uint32) error { func levelDbWrite(db *leveldb.DB, key uint64, offset uint32, size uint32) error { bytes := make([]byte, 16) - binary.BigEndian.PutUint64(bytes[0:8], key) - binary.BigEndian.PutUint32(bytes[8:12], offset) - binary.BigEndian.PutUint32(bytes[12:16], size) + util.Uint64toBytes(bytes[0:8], key) + util.Uint32toBytes(bytes[8:12], offset) + util.Uint32toBytes(bytes[12:16], size) if err := db.Put(bytes[0:8], bytes[8:16], nil); err != nil { return fmt.Errorf("failed to write leveldb: %v", err) } @@ -108,7 +108,7 @@ func levelDbWrite(db *leveldb.DB, } func levelDbDelete(db *leveldb.DB, key uint64) error { bytes := make([]byte, 8) - binary.BigEndian.PutUint64(bytes, key) + util.Uint64toBytes(bytes, key) return db.Delete(bytes, nil) } diff --git a/go/storage/needle_read_write.go b/go/storage/needle_read_write.go index 67adaa65f..ccfe1d498 100644 --- a/go/storage/needle_read_write.go +++ b/go/storage/needle_read_write.go @@ -1,13 +1,13 @@ package storage import ( - "encoding/binary" "errors" "fmt" "io" "os" "github.com/chrislusf/seaweedfs/go/glog" + "github.com/chrislusf/seaweedfs/go/util" ) const ( @@ -43,11 +43,11 @@ func (n *Needle) Append(w io.Writer, version Version) (size uint32, err error) { switch version { case Version1: header := make([]byte, NeedleHeaderSize) - binary.BigEndian.PutUint32(header[0:4], n.Cookie) - binary.BigEndian.PutUint64(header[4:12], n.Id) + util.Uint32toBytes(header[0:4], n.Cookie) + util.Uint64toBytes(header[4:12], n.Id) n.Size = uint32(len(n.Data)) size = n.Size - binary.BigEndian.PutUint32(header[12:16], n.Size) + util.Uint32toBytes(header[12:16], n.Size) if _, err = w.Write(header); err != nil { return } @@ -55,13 +55,13 @@ func (n *Needle) Append(w io.Writer, version Version) (size uint32, err error) { return } padding := NeedlePaddingSize - ((NeedleHeaderSize + n.Size + NeedleChecksumSize) % NeedlePaddingSize) - binary.BigEndian.PutUint32(header[0:NeedleChecksumSize], n.Checksum.Value()) + util.Uint32toBytes(header[0:NeedleChecksumSize], n.Checksum.Value()) _, err = w.Write(header[0 : NeedleChecksumSize+padding]) return case Version2: header := make([]byte, NeedleHeaderSize) - binary.BigEndian.PutUint32(header[0:4], n.Cookie) - binary.BigEndian.PutUint64(header[4:12], n.Id) + util.Uint32toBytes(header[0:4], n.Cookie) + util.Uint64toBytes(header[4:12], n.Id) n.DataSize, n.NameSize, n.MimeSize = uint32(len(n.Data)), uint8(len(n.Name)), uint8(len(n.Mime)) if n.DataSize > 0 { n.Size = 4 + n.DataSize + 1 @@ -81,24 +81,24 @@ func (n *Needle) Append(w io.Writer, version Version) (size uint32, err error) { n.Size = 0 } size = n.DataSize - binary.BigEndian.PutUint32(header[12:16], n.Size) + util.Uint32toBytes(header[12:16], n.Size) if _, err = w.Write(header); err != nil { return } if n.DataSize > 0 { - binary.BigEndian.PutUint32(header[0:4], n.DataSize) + util.Uint32toBytes(header[0:4], n.DataSize) if _, err = w.Write(header[0:4]); err != nil { return } if _, err = w.Write(n.Data); err != nil { return } - header[0] = n.Flags + util.Uint8toBytes(header[0:1], n.Flags) if _, err = w.Write(header[0:1]); err != nil { return } if n.HasName() { - header[0] = n.NameSize + util.Uint8toBytes(header[0:1], n.NameSize) if _, err = w.Write(header[0:1]); err != nil { return } @@ -107,7 +107,7 @@ func (n *Needle) Append(w io.Writer, version Version) (size uint32, err error) { } } if n.HasMime() { - header[0] = n.MimeSize + util.Uint8toBytes(header[0:1], n.MimeSize) if _, err = w.Write(header[0:1]); err != nil { return } @@ -116,7 +116,7 @@ func (n *Needle) Append(w io.Writer, version Version) (size uint32, err error) { } } if n.HasLastModifiedDate() { - binary.BigEndian.PutUint64(header[0:8], n.LastModified) + util.Uint64toBytes(header[0:8], n.LastModified) if _, err = w.Write(header[8-LastModifiedBytesLength : 8]); err != nil { return } @@ -129,7 +129,7 @@ func (n *Needle) Append(w io.Writer, version Version) (size uint32, err error) { } } padding := NeedlePaddingSize - ((NeedleHeaderSize + n.Size + NeedleChecksumSize) % NeedlePaddingSize) - binary.BigEndian.PutUint32(header[0:NeedleChecksumSize], n.Checksum.Value()) + util.Uint32toBytes(header[0:NeedleChecksumSize], n.Checksum.Value()) _, err = w.Write(header[0 : NeedleChecksumSize+padding]) return n.DataSize, err } @@ -158,7 +158,7 @@ func (n *Needle) ReadData(r *os.File, offset int64, size uint32, version Version case Version2: n.readNeedleDataVersion2(bytes[NeedleHeaderSize : NeedleHeaderSize+int(n.Size)]) } - checksum := binary.BigEndian.Uint32(bytes[NeedleHeaderSize+size : NeedleHeaderSize+size+NeedleChecksumSize]) + checksum := util.BytesToUint32(bytes[NeedleHeaderSize+size : NeedleHeaderSize+size+NeedleChecksumSize]) newChecksum := NewCRC(n.Data) if checksum != newChecksum.Value() { return errors.New("CRC error! Data On Disk Corrupted") @@ -167,14 +167,14 @@ func (n *Needle) ReadData(r *os.File, offset int64, size uint32, version Version return nil } func (n *Needle) ParseNeedleHeader(bytes []byte) { - n.Cookie = binary.BigEndian.Uint32(bytes[0:4]) - n.Id = binary.BigEndian.Uint64(bytes[4:12]) - n.Size = binary.BigEndian.Uint32(bytes[12:NeedleHeaderSize]) + n.Cookie = util.BytesToUint32(bytes[0:4]) + n.Id = util.BytesToUint64(bytes[4:12]) + n.Size = util.BytesToUint32(bytes[12:NeedleHeaderSize]) } func (n *Needle) readNeedleDataVersion2(bytes []byte) { index, lenBytes := 0, len(bytes) if index < lenBytes { - n.DataSize = binary.BigEndian.Uint32(bytes[index : index+4]) + n.DataSize = util.BytesToUint32(bytes[index : index+4]) index = index + 4 if int(n.DataSize)+index > lenBytes { // this if clause is due to bug #87 and #93, fixed in v0.69 @@ -199,7 +199,7 @@ func (n *Needle) readNeedleDataVersion2(bytes []byte) { index = index + int(n.MimeSize) } if index < lenBytes && n.HasLastModifiedDate() { - n.LastModified = uint64(bytes[index])<<32 | uint64(binary.BigEndian.Uint32(bytes[index+1:index+LastModifiedBytesLength])) + n.LastModified = util.BytesToUint64(bytes[index : index+LastModifiedBytesLength]) index = index + LastModifiedBytesLength } if index < lenBytes && n.HasTtl() { diff --git a/go/storage/needle_test.go b/go/storage/needle_test.go deleted file mode 100644 index c05afda2f..000000000 --- a/go/storage/needle_test.go +++ /dev/null @@ -1,45 +0,0 @@ -package storage - -import "testing" - -func TestParseKeyHash(t *testing.T) { - testcases := []struct { - KeyHash string - ID uint64 - Cookie uint32 - Err bool - }{ - // normal - {"4ed4c8116e41", 0x4ed4, 0xc8116e41, false}, - // cookie with leading zeros - {"4ed401116e41", 0x4ed4, 0x01116e41, false}, - // odd length - {"ed400116e41", 0xed4, 0x00116e41, false}, - // uint - {"fed4c8114ed4c811f0116e41", 0xfed4c8114ed4c811, 0xf0116e41, false}, - // err: too short - {"4ed4c811", 0, 0, true}, - // err: too long - {"4ed4c8114ed4c8114ed4c8111", 0, 0, true}, - // err: invalid character - {"helloworld", 0, 0, true}, - } - - for _, tc := range testcases { - if id, cookie, err := ParseKeyHash(tc.KeyHash); err != nil && !tc.Err { - t.Fatalf("Parse %s error: %v", tc.KeyHash, err) - } else if err == nil && tc.Err { - t.Fatalf("Parse %s expected error got nil", tc.KeyHash) - } else if id != tc.ID || cookie != tc.Cookie { - t.Fatalf("Parse %s wrong result. Expected: (%d, %d) got: (%d, %d)", tc.KeyHash, tc.ID, tc.Cookie, id, cookie) - } - } -} - -func BenchmarkParseKeyHash(b *testing.B) { - b.ReportAllocs() - - for i := 0; i < b.N; i++ { - ParseKeyHash("4ed44ed44ed44ed4c8116e41") - } -} diff --git a/go/storage/volume_super_block.go b/go/storage/volume_super_block.go index 76116090d..e37360075 100644 --- a/go/storage/volume_super_block.go +++ b/go/storage/volume_super_block.go @@ -1,11 +1,11 @@ package storage import ( - "encoding/binary" "fmt" "os" "github.com/chrislusf/seaweedfs/go/glog" + "github.com/chrislusf/seaweedfs/go/util" ) const ( @@ -35,7 +35,7 @@ func (s *SuperBlock) Bytes() []byte { header[0] = byte(s.version) header[1] = s.ReplicaPlacement.Byte() s.Ttl.ToBytes(header[2:4]) - binary.BigEndian.PutUint16(header[4:6], s.CompactRevision) + util.Uint16toBytes(header[4:6], s.CompactRevision) return header } @@ -76,6 +76,6 @@ func ParseSuperBlock(header []byte) (superBlock SuperBlock, err error) { err = fmt.Errorf("cannot read replica type: %s", err.Error()) } superBlock.Ttl = LoadTTLFromBytes(header[2:4]) - superBlock.CompactRevision = binary.BigEndian.Uint16(header[4:6]) + superBlock.CompactRevision = util.BytesToUint16(header[4:6]) return } diff --git a/go/util/bytes.go b/go/util/bytes.go new file mode 100644 index 000000000..dfa4ae665 --- /dev/null +++ b/go/util/bytes.go @@ -0,0 +1,45 @@ +package util + +// big endian + +func BytesToUint64(b []byte) (v uint64) { + length := uint(len(b)) + for i := uint(0); i < length-1; i++ { + v += uint64(b[i]) + v <<= 8 + } + v += uint64(b[length-1]) + return +} +func BytesToUint32(b []byte) (v uint32) { + length := uint(len(b)) + for i := uint(0); i < length-1; i++ { + v += uint32(b[i]) + v <<= 8 + } + v += uint32(b[length-1]) + return +} +func BytesToUint16(b []byte) (v uint16) { + v += uint16(b[0]) + v <<= 8 + v += uint16(b[1]) + return +} +func Uint64toBytes(b []byte, v uint64) { + for i := uint(0); i < 8; i++ { + b[7-i] = byte(v >> (i * 8)) + } +} +func Uint32toBytes(b []byte, v uint32) { + for i := uint(0); i < 4; i++ { + b[3-i] = byte(v >> (i * 8)) + } +} +func Uint16toBytes(b []byte, v uint16) { + b[0] = byte(v >> 8) + b[1] = byte(v) +} +func Uint8toBytes(b []byte, v uint8) { + b[0] = byte(v) +}