From 784c5bb73a71bad615e500e4254c11e736b99a41 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 8 Apr 2019 22:01:29 -0700 Subject: [PATCH] add build option support 5-byte offset --- weed/storage/types/needle_types.go | 53 ------------------- weed/storage/types/offset_4bytes.go | 63 +++++++++++++++++++++++ weed/storage/types/offset_5bytes.go | 80 +++++++++++++++++++++++++++++ 3 files changed, 143 insertions(+), 53 deletions(-) create mode 100644 weed/storage/types/offset_4bytes.go create mode 100644 weed/storage/types/offset_5bytes.go diff --git a/weed/storage/types/needle_types.go b/weed/storage/types/needle_types.go index 86eae03ac..b591dd7c3 100644 --- a/weed/storage/types/needle_types.go +++ b/weed/storage/types/needle_types.go @@ -12,10 +12,6 @@ type Offset struct { OffsetLower } -type OffsetHigher struct { - // b4 byte -} - type OffsetLower struct { b3 byte b2 byte @@ -26,12 +22,10 @@ type OffsetLower struct { type Cookie uint32 const ( - OffsetSize = 4 // + 1 SizeSize = 4 // uint32 size NeedleEntrySize = CookieSize + NeedleIdSize + SizeSize TimestampSize = 8 // int64 size NeedlePaddingSize = 8 - MaxPossibleVolumeSize = 4 * 1024 * 1024 * 1024 * 8 TombstoneFileSize = math.MaxUint32 CookieSize = 4 ) @@ -54,50 +48,3 @@ func ParseCookie(cookieString string) (Cookie, error) { } return Cookie(cookie), nil } - -func OffsetToBytes(bytes []byte, offset Offset) { - bytes[3] = offset.b0 - bytes[2] = offset.b1 - bytes[1] = offset.b2 - bytes[0] = offset.b3 -} - -// only for testing, will be removed later. -func Uint32ToOffset(offset uint32) Offset { - return Offset{ - OffsetLower: OffsetLower{ - b0: byte(offset), - b1: byte(offset >> 8), - b2: byte(offset >> 16), - b3: byte(offset >> 24), - }, - } -} - -func BytesToOffset(bytes []byte) Offset { - return Offset{ - OffsetLower: OffsetLower{ - b0: bytes[3], - b1: bytes[2], - b2: bytes[1], - b3: bytes[0], - }, - } -} - -func (offset Offset) IsZero() bool { - return offset.b0 == 0 && offset.b1 == 0 && offset.b2 == 0 && offset.b3 == 0 -} - -func ToOffset(offset int64) Offset { - smaller := uint32(offset / int64(NeedlePaddingSize)) - return Uint32ToOffset(smaller) -} - -func (offset Offset) ToAcutalOffset() (actualOffset int64) { - return (int64(offset.b0) + int64(offset.b1)<<8 + int64(offset.b2)<<16 + int64(offset.b3)<<24) * int64(NeedlePaddingSize) -} - -func (offset Offset) String() string { - return fmt.Sprintf("%d", int64(offset.b0)+int64(offset.b1)<<8+int64(offset.b2)<<16+int64(offset.b3)<<24) -} diff --git a/weed/storage/types/offset_4bytes.go b/weed/storage/types/offset_4bytes.go new file mode 100644 index 000000000..9acd069d3 --- /dev/null +++ b/weed/storage/types/offset_4bytes.go @@ -0,0 +1,63 @@ +// +build !5BytesOffset + +package types + +import ( + "fmt" +) + +type OffsetHigher struct { + // b4 byte +} + +const ( + OffsetSize = 4 + MaxPossibleVolumeSize = 4 * 1024 * 1024 * 1024 * 8 // 32GB +) + +func OffsetToBytes(bytes []byte, offset Offset) { + bytes[3] = offset.b0 + bytes[2] = offset.b1 + bytes[1] = offset.b2 + bytes[0] = offset.b3 +} + +// only for testing, will be removed later. +func Uint32ToOffset(offset uint32) Offset { + return Offset{ + OffsetLower: OffsetLower{ + b0: byte(offset), + b1: byte(offset >> 8), + b2: byte(offset >> 16), + b3: byte(offset >> 24), + }, + } +} + +func BytesToOffset(bytes []byte) Offset { + return Offset{ + OffsetLower: OffsetLower{ + b0: bytes[3], + b1: bytes[2], + b2: bytes[1], + b3: bytes[0], + }, + } +} + +func (offset Offset) IsZero() bool { + return offset.b0 == 0 && offset.b1 == 0 && offset.b2 == 0 && offset.b3 == 0 +} + +func ToOffset(offset int64) Offset { + smaller := uint32(offset / int64(NeedlePaddingSize)) + return Uint32ToOffset(smaller) +} + +func (offset Offset) ToAcutalOffset() (actualOffset int64) { + return (int64(offset.b0) + int64(offset.b1)<<8 + int64(offset.b2)<<16 + int64(offset.b3)<<24) * int64(NeedlePaddingSize) +} + +func (offset Offset) String() string { + return fmt.Sprintf("%d", int64(offset.b0)+int64(offset.b1)<<8+int64(offset.b2)<<16+int64(offset.b3)<<24) +} diff --git a/weed/storage/types/offset_5bytes.go b/weed/storage/types/offset_5bytes.go new file mode 100644 index 000000000..f57e4f6d4 --- /dev/null +++ b/weed/storage/types/offset_5bytes.go @@ -0,0 +1,80 @@ +// +build 5BytesOffset + +package types + +import ( + "fmt" +) + +type OffsetHigher struct { + b4 byte +} + +const ( + OffsetSize = 4 + 1 + MaxPossibleVolumeSize = 4 * 1024 * 1024 * 1024 * 8 * 256 /* 256 is from the extra byte */ // 8TB +) + +func OffsetToBytes(bytes []byte, offset Offset) { + bytes[4] = offset.b4 + bytes[3] = offset.b0 + bytes[2] = offset.b1 + bytes[1] = offset.b2 + bytes[0] = offset.b3 +} + +// only for testing, will be removed later. +func Uint32ToOffset(offset uint32) Offset { + return Offset{ + OffsetHigher: OffsetHigher{ + b4: byte(offset >> 32), + }, + OffsetLower: OffsetLower{ + b0: byte(offset), + b1: byte(offset >> 8), + b2: byte(offset >> 16), + b3: byte(offset >> 24), + }, + } +} + +func BytesToOffset(bytes []byte) Offset { + return Offset{ + OffsetHigher: OffsetHigher{ + b4: bytes[4], + }, + OffsetLower: OffsetLower{ + b0: bytes[3], + b1: bytes[2], + b2: bytes[1], + b3: bytes[0], + }, + } +} + +func (offset Offset) IsZero() bool { + return offset.b0 == 0 && offset.b1 == 0 && offset.b2 == 0 && offset.b3 == 0 && offset.b4 == 0 +} + +func ToOffset(offset int64) Offset { + smaller := offset / int64(NeedlePaddingSize) + return Offset{ + OffsetHigher: OffsetHigher{ + b4: byte(smaller >> 32), + }, + OffsetLower: OffsetLower{ + b0: byte(smaller), + b1: byte(smaller >> 8), + b2: byte(smaller >> 16), + b3: byte(smaller >> 24), + }, + } +} + +func (offset Offset) ToAcutalOffset() (actualOffset int64) { + return (int64(offset.b0) + int64(offset.b1)<<8 + int64(offset.b2)<<16 + int64(offset.b3)<<24 + int64(offset.b4)<<32) * int64(NeedlePaddingSize) +} + +func (offset Offset) String() string { + return fmt.Sprintf("%d", int64(offset.b0)+int64(offset.b1)<<8+int64(offset.b2)<<16+int64(offset.b3)<<24+int64(offset.b4)<<32) +}