From de5ca9b25809e846026784b6721cfc76471c87bf Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 20 Jun 2020 12:50:40 -0700 Subject: [PATCH] remove fixJpgOrientation --- .../templates/volume-statefulset.yaml | 3 - k8s/seaweedfs/values.yaml | 3 - weed/command/server.go | 1 - weed/command/volume.go | 4 +- weed/images/orientation.go | 182 ------------------ weed/images/orientation_test.go | 20 -- weed/images/preprocess.go | 29 --- weed/pb/shared_values.go | 2 +- weed/server/volume_server.go | 3 - weed/server/volume_server_handlers_write.go | 2 +- weed/storage/needle/needle.go | 10 +- weed/wdclient/masterclient.go | 18 +- 12 files changed, 13 insertions(+), 264 deletions(-) delete mode 100644 weed/images/orientation.go delete mode 100644 weed/images/orientation_test.go delete mode 100644 weed/images/preprocess.go diff --git a/k8s/seaweedfs/templates/volume-statefulset.yaml b/k8s/seaweedfs/templates/volume-statefulset.yaml index 9c6ddcd9f..6180bc7df 100644 --- a/k8s/seaweedfs/templates/volume-statefulset.yaml +++ b/k8s/seaweedfs/templates/volume-statefulset.yaml @@ -88,9 +88,6 @@ spec: {{- if .Values.volume.whiteList }} -whiteList={{ .Values.volume.whiteList }} \ {{- end }} - {{- if .Values.volume.imagesFixOrientation }} - -images.fix.orientation \ - {{- end }} -ip=${POD_NAME}.${SEAWEEDFS_FULLNAME}-volume \ -compactionMBps={{ .Values.volume.compactionMBps }} \ -mserver={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }} diff --git a/k8s/seaweedfs/values.yaml b/k8s/seaweedfs/values.yaml index 4c8ae6b88..9bdbb0fd8 100644 --- a/k8s/seaweedfs/values.yaml +++ b/k8s/seaweedfs/values.yaml @@ -123,9 +123,6 @@ volume: # Comma separated Ip addresses having write permission. No limit if empty. whiteList: null - # Adjust jpg orientation when uploading. - imagesFixOrientation: false - extraVolumes: "" extraVolumeMounts: "" diff --git a/weed/command/server.go b/weed/command/server.go index a2b401dbf..fadd08a66 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -92,7 +92,6 @@ func init() { serverOptions.v.port = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port") serverOptions.v.publicPort = cmdServer.Flag.Int("volume.port.public", 0, "volume server public port") serverOptions.v.indexType = cmdServer.Flag.String("volume.index", "memory", "Choose [memory|leveldb|leveldbMedium|leveldbLarge] mode for memory~performance balance.") - serverOptions.v.fixJpgOrientation = cmdServer.Flag.Bool("volume.images.fix.orientation", false, "Adjust jpg orientation when uploading.") serverOptions.v.readRedirect = cmdServer.Flag.Bool("volume.read.redirect", true, "Redirect moved or non-local volumes.") serverOptions.v.compactionMBPerSecond = cmdServer.Flag.Int("volume.compactionMBps", 0, "limit compaction speed in mega bytes per second") serverOptions.v.fileSizeLimitMB = cmdServer.Flag.Int("volume.fileSizeLimitMB", 256, "limit file size to avoid out of memory") diff --git a/weed/command/volume.go b/weed/command/volume.go index ba8c1a550..a20ac8c81 100644 --- a/weed/command/volume.go +++ b/weed/command/volume.go @@ -47,7 +47,6 @@ type VolumeServerOptions struct { rack *string whiteList []string indexType *string - fixJpgOrientation *bool readRedirect *bool cpuProfile *string memProfile *string @@ -71,7 +70,6 @@ func init() { v.dataCenter = cmdVolume.Flag.String("dataCenter", "", "current volume server's data center name") v.rack = cmdVolume.Flag.String("rack", "", "current volume server's rack name") v.indexType = cmdVolume.Flag.String("index", "memory", "Choose [memory|leveldb|leveldbMedium|leveldbLarge] mode for memory~performance balance.") - v.fixJpgOrientation = cmdVolume.Flag.Bool("images.fix.orientation", false, "Adjust jpg orientation when uploading.") v.readRedirect = cmdVolume.Flag.Bool("read.redirect", true, "Redirect moved or non-local volumes.") v.cpuProfile = cmdVolume.Flag.String("cpuprofile", "", "cpu profile output file") v.memProfile = cmdVolume.Flag.String("memprofile", "", "memory profile output file") @@ -192,7 +190,7 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v volumeNeedleMapKind, strings.Split(masters, ","), 5, *v.dataCenter, *v.rack, v.whiteList, - *v.fixJpgOrientation, *v.readRedirect, + *v.readRedirect, *v.compactionMBPerSecond, *v.fileSizeLimitMB, ) diff --git a/weed/images/orientation.go b/weed/images/orientation.go deleted file mode 100644 index a592a7d8b..000000000 --- a/weed/images/orientation.go +++ /dev/null @@ -1,182 +0,0 @@ -package images - -import ( - "bytes" - "image" - "image/draw" - "image/jpeg" - "log" - - "github.com/seaweedfs/goexif/exif" -) - -//many code is copied from http://camlistore.org/pkg/images/images.go -func FixJpgOrientation(data []byte) (oriented []byte) { - ex, err := exif.Decode(bytes.NewReader(data)) - if err != nil { - return data - } - tag, err := ex.Get(exif.Orientation) - if err != nil { - return data - } - angle := 0 - flipMode := FlipDirection(0) - orient, err := tag.Int(0) - if err != nil { - return data - } - switch orient { - case topLeftSide: - // do nothing - return data - case topRightSide: - flipMode = 2 - case bottomRightSide: - angle = 180 - case bottomLeftSide: - angle = 180 - flipMode = 2 - case leftSideTop: - angle = -90 - flipMode = 2 - case rightSideTop: - angle = -90 - case rightSideBottom: - angle = 90 - flipMode = 2 - case leftSideBottom: - angle = 90 - } - - if srcImage, _, err := image.Decode(bytes.NewReader(data)); err == nil { - dstImage := flip(rotate(srcImage, angle), flipMode) - var buf bytes.Buffer - jpeg.Encode(&buf, dstImage, nil) - return buf.Bytes() - } - - return data -} - -// Exif Orientation Tag values -// http://sylvana.net/jpegcrop/exif_orientation.html -const ( - topLeftSide = 1 - topRightSide = 2 - bottomRightSide = 3 - bottomLeftSide = 4 - leftSideTop = 5 - rightSideTop = 6 - rightSideBottom = 7 - leftSideBottom = 8 -) - -// The FlipDirection type is used by the Flip option in DecodeOpts -// to indicate in which direction to flip an image. -type FlipDirection int - -// FlipVertical and FlipHorizontal are two possible FlipDirections -// values to indicate in which direction an image will be flipped. -const ( - FlipVertical FlipDirection = 1 << iota - FlipHorizontal -) - -type DecodeOpts struct { - // Rotate specifies how to rotate the image. - // If nil, the image is rotated automatically based on EXIF metadata. - // If an int, Rotate is the number of degrees to rotate - // counter clockwise and must be one of 0, 90, -90, 180, or - // -180. - Rotate interface{} - - // Flip specifies how to flip the image. - // If nil, the image is flipped automatically based on EXIF metadata. - // Otherwise, Flip is a FlipDirection bitfield indicating how to flip. - Flip interface{} -} - -func rotate(im image.Image, angle int) image.Image { - var rotated *image.NRGBA - // trigonometric (i.e counter clock-wise) - switch angle { - case 90: - newH, newW := im.Bounds().Dx(), im.Bounds().Dy() - rotated = image.NewNRGBA(image.Rect(0, 0, newW, newH)) - for y := 0; y < newH; y++ { - for x := 0; x < newW; x++ { - rotated.Set(x, y, im.At(newH-1-y, x)) - } - } - case -90: - newH, newW := im.Bounds().Dx(), im.Bounds().Dy() - rotated = image.NewNRGBA(image.Rect(0, 0, newW, newH)) - for y := 0; y < newH; y++ { - for x := 0; x < newW; x++ { - rotated.Set(x, y, im.At(y, newW-1-x)) - } - } - case 180, -180: - newW, newH := im.Bounds().Dx(), im.Bounds().Dy() - rotated = image.NewNRGBA(image.Rect(0, 0, newW, newH)) - for y := 0; y < newH; y++ { - for x := 0; x < newW; x++ { - rotated.Set(x, y, im.At(newW-1-x, newH-1-y)) - } - } - default: - return im - } - return rotated -} - -// flip returns a flipped version of the image im, according to -// the direction(s) in dir. -// It may flip the imput im in place and return it, or it may allocate a -// new NRGBA (if im is an *image.YCbCr). -func flip(im image.Image, dir FlipDirection) image.Image { - if dir == 0 { - return im - } - ycbcr := false - var nrgba image.Image - dx, dy := im.Bounds().Dx(), im.Bounds().Dy() - di, ok := im.(draw.Image) - if !ok { - if _, ok := im.(*image.YCbCr); !ok { - log.Printf("failed to flip image: input does not satisfy draw.Image") - return im - } - // because YCbCr does not implement Set, we replace it with a new NRGBA - ycbcr = true - nrgba = image.NewNRGBA(image.Rect(0, 0, dx, dy)) - di, ok = nrgba.(draw.Image) - if !ok { - log.Print("failed to flip image: could not cast an NRGBA to a draw.Image") - return im - } - } - if dir&FlipHorizontal != 0 { - for y := 0; y < dy; y++ { - for x := 0; x < dx/2; x++ { - old := im.At(x, y) - di.Set(x, y, im.At(dx-1-x, y)) - di.Set(dx-1-x, y, old) - } - } - } - if dir&FlipVertical != 0 { - for y := 0; y < dy/2; y++ { - for x := 0; x < dx; x++ { - old := im.At(x, y) - di.Set(x, y, im.At(x, dy-1-y)) - di.Set(x, dy-1-y, old) - } - } - } - if ycbcr { - return nrgba - } - return im -} diff --git a/weed/images/orientation_test.go b/weed/images/orientation_test.go deleted file mode 100644 index 32fa38f76..000000000 --- a/weed/images/orientation_test.go +++ /dev/null @@ -1,20 +0,0 @@ -package images - -import ( - "io/ioutil" - "os" - "testing" -) - -func TestXYZ(t *testing.T) { - fname := "sample1.jpg" - - dat, _ := ioutil.ReadFile(fname) - - fixed_data := FixJpgOrientation(dat) - - ioutil.WriteFile("fixed1.jpg", fixed_data, 0644) - - os.Remove("fixed1.jpg") - -} diff --git a/weed/images/preprocess.go b/weed/images/preprocess.go deleted file mode 100644 index f6f3b554d..000000000 --- a/weed/images/preprocess.go +++ /dev/null @@ -1,29 +0,0 @@ -package images - -import ( - "bytes" - "io" - "path/filepath" - "strings" -) - -/* -* Preprocess image files on client side. -* 1. possibly adjust the orientation -* 2. resize the image to a width or height limit -* 3. remove the exif data -* Call this function on any file uploaded to SeaweedFS -* - */ -func MaybePreprocessImage(filename string, data []byte, width, height int) (resized io.ReadSeeker, w int, h int) { - ext := filepath.Ext(filename) - ext = strings.ToLower(ext) - switch ext { - case ".png", ".gif": - return Resized(ext, bytes.NewReader(data), width, height, "") - case ".jpg", ".jpeg": - data = FixJpgOrientation(data) - return Resized(ext, bytes.NewReader(data), width, height, "") - } - return bytes.NewReader(data), 0, 0 -} diff --git a/weed/pb/shared_values.go b/weed/pb/shared_values.go index acc3bb56d..1af19e51a 100644 --- a/weed/pb/shared_values.go +++ b/weed/pb/shared_values.go @@ -1,5 +1,5 @@ package pb const ( - AdminShellClient = "shell" + AdminShellClient = "adminShell" ) diff --git a/weed/server/volume_server.go b/weed/server/volume_server.go index 62fbc19a7..367e39d8a 100644 --- a/weed/server/volume_server.go +++ b/weed/server/volume_server.go @@ -25,7 +25,6 @@ type VolumeServer struct { grpcDialOption grpc.DialOption needleMapKind storage.NeedleMapType - FixJpgOrientation bool ReadRedirect bool compactionBytePerSecond int64 MetricsAddress string @@ -40,7 +39,6 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string, masterNodes []string, pulseSeconds int, dataCenter string, rack string, whiteList []string, - fixJpgOrientation bool, readRedirect bool, compactionMBPerSecond int, fileSizeLimitMB int, @@ -61,7 +59,6 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string, dataCenter: dataCenter, rack: rack, needleMapKind: needleMapKind, - FixJpgOrientation: fixJpgOrientation, ReadRedirect: readRedirect, grpcDialOption: security.LoadClientTLS(util.GetViper(), "grpc.volume"), compactionBytePerSecond: int64(compactionMBPerSecond) * 1024 * 1024, diff --git a/weed/server/volume_server_handlers_write.go b/weed/server/volume_server_handlers_write.go index 74dad28de..5ece46ed0 100644 --- a/weed/server/volume_server_handlers_write.go +++ b/weed/server/volume_server_handlers_write.go @@ -42,7 +42,7 @@ func (vs *VolumeServer) PostHandler(w http.ResponseWriter, r *http.Request) { return } - reqNeedle, originalSize, ne := needle.CreateNeedleFromRequest(r, vs.FixJpgOrientation, vs.fileSizeLimitBytes) + reqNeedle, originalSize, ne := needle.CreateNeedleFromRequest(r, vs.fileSizeLimitBytes) if ne != nil { writeJsonError(w, r, http.StatusBadRequest, ne) return diff --git a/weed/storage/needle/needle.go b/weed/storage/needle/needle.go index 7d02758d6..4c62aa00b 100644 --- a/weed/storage/needle/needle.go +++ b/weed/storage/needle/needle.go @@ -8,7 +8,6 @@ import ( "strings" "time" - "github.com/chrislusf/seaweedfs/weed/images" . "github.com/chrislusf/seaweedfs/weed/storage/types" ) @@ -48,7 +47,7 @@ func (n *Needle) String() (str string) { return } -func CreateNeedleFromRequest(r *http.Request, fixJpgOrientation bool, sizeLimit int64) (n *Needle, originalSize int, e error) { +func CreateNeedleFromRequest(r *http.Request, sizeLimit int64) (n *Needle, originalSize int, e error) { n = new(Needle) pu, e := ParseUpload(r, sizeLimit) if e != nil { @@ -95,13 +94,6 @@ func CreateNeedleFromRequest(r *http.Request, fixJpgOrientation bool, sizeLimit n.SetIsChunkManifest() } - if fixJpgOrientation { - loweredName := strings.ToLower(pu.FileName) - if pu.MimeType == "image/jpeg" || strings.HasSuffix(loweredName, ".jpg") || strings.HasSuffix(loweredName, ".jpeg") { - n.Data = images.FixJpgOrientation(n.Data) - } - } - n.Checksum = NewCRC(n.Data) commaSep := strings.LastIndex(r.URL.Path, ",") diff --git a/weed/wdclient/masterclient.go b/weed/wdclient/masterclient.go index 4f8e0d5ef..4c066d535 100644 --- a/weed/wdclient/masterclient.go +++ b/weed/wdclient/masterclient.go @@ -45,7 +45,7 @@ func (mc *MasterClient) WaitUntilConnected() { } func (mc *MasterClient) KeepConnectedToMaster() { - glog.V(1).Infof("%s bootstraps with masters %v", mc.clientType, mc.masters) + glog.V(1).Infof("%s masterClient bootstraps with masters %v", mc.clientType, mc.masters) for { mc.tryAllMasters() time.Sleep(time.Second) @@ -67,27 +67,27 @@ func (mc *MasterClient) tryAllMasters() { } func (mc *MasterClient) tryConnectToMaster(master string) (nextHintedLeader string) { - glog.V(1).Infof("%s Connecting to master %v", mc.clientType, master) + glog.V(1).Infof("%s masterClient Connecting to master %v", mc.clientType, master) gprcErr := pb.WithMasterClient(master, mc.grpcDialOption, func(client master_pb.SeaweedClient) error { stream, err := client.KeepConnected(context.Background()) if err != nil { - glog.V(0).Infof("%s failed to keep connected to %s: %v", mc.clientType, master, err) + glog.V(0).Infof("%s masterClient failed to keep connected to %s: %v", mc.clientType, master, err) return err } if err = stream.Send(&master_pb.KeepConnectedRequest{Name: mc.clientType, GrpcPort: mc.grpcPort}); err != nil { - glog.V(0).Infof("%s failed to send to %s: %v", mc.clientType, master, err) + glog.V(0).Infof("%s masterClient failed to send to %s: %v", mc.clientType, master, err) return err } - glog.V(1).Infof("%s Connected to %v", mc.clientType, master) + glog.V(1).Infof("%s masterClient Connected to %v", mc.clientType, master) mc.currentMaster = master for { volumeLocation, err := stream.Recv() if err != nil { - glog.V(0).Infof("%s failed to receive from %s: %v", mc.clientType, master, err) + glog.V(0).Infof("%s masterClient failed to receive from %s: %v", mc.clientType, master, err) return err } @@ -104,18 +104,18 @@ func (mc *MasterClient) tryConnectToMaster(master string) (nextHintedLeader stri PublicUrl: volumeLocation.PublicUrl, } for _, newVid := range volumeLocation.NewVids { - glog.V(1).Infof("%s: %s adds volume %d", mc.clientType, loc.Url, newVid) + glog.V(1).Infof("%s: %s masterClient adds volume %d", mc.clientType, loc.Url, newVid) mc.addLocation(newVid, loc) } for _, deletedVid := range volumeLocation.DeletedVids { - glog.V(1).Infof("%s: %s removes volume %d", mc.clientType, loc.Url, deletedVid) + glog.V(1).Infof("%s: %s masterClient removes volume %d", mc.clientType, loc.Url, deletedVid) mc.deleteLocation(deletedVid, loc) } } }) if gprcErr != nil { - glog.V(0).Infof("%s failed to connect with master %v: %v", mc.clientType, master, gprcErr) + glog.V(0).Infof("%s masterClient failed to connect with master %v: %v", mc.clientType, master, gprcErr) } return }