diff --git a/weed/operation/assign_file_id.go b/weed/operation/assign_file_id.go index 924b29c0e..00e1caad5 100644 --- a/weed/operation/assign_file_id.go +++ b/weed/operation/assign_file_id.go @@ -3,6 +3,7 @@ package operation import ( "context" "fmt" + "time" "github.com/chrislusf/seaweedfs/weed/pb/master_pb" ) @@ -40,6 +41,9 @@ func Assign(server string, primaryRequest *VolumeAssignRequest, alternativeReque } lastError = withMasterServerClient(server, func(masterClient master_pb.SeaweedClient) error { + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(5*time.Second)) + defer cancel() + req := &master_pb.AssignRequest{ Count: primaryRequest.Count, Replication: primaryRequest.Replication, @@ -49,7 +53,7 @@ func Assign(server string, primaryRequest *VolumeAssignRequest, alternativeReque Rack: primaryRequest.Rack, DataNode: primaryRequest.DataNode, } - resp, grpcErr := masterClient.Assign(context.Background(), req) + resp, grpcErr := masterClient.Assign(ctx, req) if grpcErr != nil { return grpcErr } diff --git a/weed/operation/delete_content.go b/weed/operation/delete_content.go index e15859d15..3e468e1a3 100644 --- a/weed/operation/delete_content.go +++ b/weed/operation/delete_content.go @@ -7,6 +7,7 @@ import ( "net/http" "strings" "sync" + "time" "github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb" ) @@ -108,12 +109,14 @@ func DeleteFilesWithLookupVolumeId(fileIds []string, lookupFunc func(vid []strin func DeleteFilesAtOneVolumeServer(volumeServer string, fileIds []string) (ret []*volume_server_pb.DeleteResult, err error) { err = WithVolumeServerClient(volumeServer, func(volumeServerClient volume_server_pb.VolumeServerClient) error { + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(5*time.Second)) + defer cancel() req := &volume_server_pb.BatchDeleteRequest{ FileIds: fileIds, } - resp, err := volumeServerClient.BatchDelete(context.Background(), req) + resp, err := volumeServerClient.BatchDelete(ctx, req) // fmt.Printf("deleted %v %v: %v\n", fileIds, err, resp) diff --git a/weed/operation/lookup.go b/weed/operation/lookup.go index c495af1af..562a11580 100644 --- a/weed/operation/lookup.go +++ b/weed/operation/lookup.go @@ -99,10 +99,13 @@ func LookupVolumeIds(server string, vids []string) (map[string]LookupResult, err //only query unknown_vids err := withMasterServerClient(server, func(masterClient master_pb.SeaweedClient) error { + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(5*time.Second)) + defer cancel() + req := &master_pb.LookupVolumeRequest{ VolumeIds: unknown_vids, } - resp, grpcErr := masterClient.LookupVolume(context.Background(), req) + resp, grpcErr := masterClient.LookupVolume(ctx, req) if grpcErr != nil { return grpcErr } diff --git a/weed/operation/stats.go b/weed/operation/stats.go index 277b0e835..364727272 100644 --- a/weed/operation/stats.go +++ b/weed/operation/stats.go @@ -2,6 +2,7 @@ package operation import ( "context" + "time" "github.com/chrislusf/seaweedfs/weed/pb/master_pb" ) @@ -9,7 +10,10 @@ import ( func Statistics(server string, req *master_pb.StatisticsRequest) (resp *master_pb.StatisticsResponse, err error) { err = withMasterServerClient(server, func(masterClient master_pb.SeaweedClient) error { - grpcResponse, grpcErr := masterClient.Statistics(context.Background(), req) + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(5*time.Second)) + defer cancel() + + grpcResponse, grpcErr := masterClient.Statistics(ctx, req) if grpcErr != nil { return grpcErr } diff --git a/weed/operation/sync_volume.go b/weed/operation/sync_volume.go index ac2e2bf79..e40c7de41 100644 --- a/weed/operation/sync_volume.go +++ b/weed/operation/sync_volume.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "io" + "time" "github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb" . "github.com/chrislusf/seaweedfs/weed/storage/types" @@ -13,7 +14,10 @@ import ( func GetVolumeSyncStatus(server string, vid uint32) (resp *volume_server_pb.VolumeSyncStatusResponse, err error) { WithVolumeServerClient(server, func(client volume_server_pb.VolumeServerClient) error { - resp, err = client.VolumeSyncStatus(context.Background(), &volume_server_pb.VolumeSyncStatusRequest{ + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(5*time.Second)) + defer cancel() + + resp, err = client.VolumeSyncStatus(ctx, &volume_server_pb.VolumeSyncStatusRequest{ VolumdId: vid, }) return nil diff --git a/weed/server/master_server_handlers_admin.go b/weed/server/master_server_handlers_admin.go index 490693d97..3a2662908 100644 --- a/weed/server/master_server_handlers_admin.go +++ b/weed/server/master_server_handlers_admin.go @@ -7,6 +7,7 @@ import ( "math/rand" "net/http" "strconv" + "time" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/operation" @@ -24,7 +25,10 @@ func (ms *MasterServer) collectionDeleteHandler(w http.ResponseWriter, r *http.R } for _, server := range collection.ListVolumeServers() { err := operation.WithVolumeServerClient(server.Url(), func(client volume_server_pb.VolumeServerClient) error { - _, deleteErr := client.DeleteCollection(context.Background(), &volume_server_pb.DeleteCollectionRequest{ + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(5*time.Second)) + defer cancel() + + _, deleteErr := client.DeleteCollection(ctx, &volume_server_pb.DeleteCollectionRequest{ Collection: collection.Name, }) return deleteErr diff --git a/weed/topology/allocate_volume.go b/weed/topology/allocate_volume.go index 61fc9d479..55796ab43 100644 --- a/weed/topology/allocate_volume.go +++ b/weed/topology/allocate_volume.go @@ -2,6 +2,7 @@ package topology import ( "context" + "time" "github.com/chrislusf/seaweedfs/weed/operation" "github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb" @@ -15,7 +16,10 @@ type AllocateVolumeResult struct { func AllocateVolume(dn *DataNode, vid storage.VolumeId, option *VolumeGrowOption) error { return operation.WithVolumeServerClient(dn.Url(), func(client volume_server_pb.VolumeServerClient) error { - _, deleteErr := client.AssignVolume(context.Background(), &volume_server_pb.AssignVolumeRequest{ + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(5*time.Second)) + defer cancel() + + _, deleteErr := client.AssignVolume(ctx, &volume_server_pb.AssignVolumeRequest{ VolumdId: uint32(vid), Collection: option.Collection, Replication: option.ReplicaPlacement.String(), diff --git a/weed/topology/topology_vacuum.go b/weed/topology/topology_vacuum.go index e386dabdb..95d9376d5 100644 --- a/weed/topology/topology_vacuum.go +++ b/weed/topology/topology_vacuum.go @@ -15,7 +15,10 @@ func batchVacuumVolumeCheck(vl *VolumeLayout, vid storage.VolumeId, locationlist for index, dn := range locationlist.list { go func(index int, url string, vid storage.VolumeId) { err := operation.WithVolumeServerClient(url, func(volumeServerClient volume_server_pb.VolumeServerClient) error { - resp, err := volumeServerClient.VacuumVolumeCheck(context.Background(), &volume_server_pb.VacuumVolumeCheckRequest{ + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(5*time.Second)) + defer cancel() + + resp, err := volumeServerClient.VacuumVolumeCheck(ctx, &volume_server_pb.VacuumVolumeCheckRequest{ VolumdId: uint32(vid), }) if err != nil { @@ -48,6 +51,7 @@ func batchVacuumVolumeCompact(vl *VolumeLayout, vid storage.VolumeId, locationli ch := make(chan bool, locationlist.Length()) for index, dn := range locationlist.list { go func(index int, url string, vid storage.VolumeId) { + // TODO: set timeout according to actual compact duration glog.V(0).Infoln(index, "Start vacuuming", vid, "on", url) err := operation.WithVolumeServerClient(url, func(volumeServerClient volume_server_pb.VolumeServerClient) error { _, err := volumeServerClient.VacuumVolumeCompact(context.Background(), &volume_server_pb.VacuumVolumeCompactRequest{ @@ -81,7 +85,10 @@ func batchVacuumVolumeCommit(vl *VolumeLayout, vid storage.VolumeId, locationlis for _, dn := range locationlist.list { glog.V(0).Infoln("Start Commiting vacuum", vid, "on", dn.Url()) err := operation.WithVolumeServerClient(dn.Url(), func(volumeServerClient volume_server_pb.VolumeServerClient) error { - _, err := volumeServerClient.VacuumVolumeCommit(context.Background(), &volume_server_pb.VacuumVolumeCommitRequest{ + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(5*time.Second)) + defer cancel() + + _, err := volumeServerClient.VacuumVolumeCommit(ctx, &volume_server_pb.VacuumVolumeCommitRequest{ VolumdId: uint32(vid), }) return err @@ -102,7 +109,10 @@ func batchVacuumVolumeCleanup(vl *VolumeLayout, vid storage.VolumeId, locationli for _, dn := range locationlist.list { glog.V(0).Infoln("Start cleaning up", vid, "on", dn.Url()) err := operation.WithVolumeServerClient(dn.Url(), func(volumeServerClient volume_server_pb.VolumeServerClient) error { - _, err := volumeServerClient.VacuumVolumeCleanup(context.Background(), &volume_server_pb.VacuumVolumeCleanupRequest{ + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(5*time.Second)) + defer cancel() + + _, err := volumeServerClient.VacuumVolumeCleanup(ctx, &volume_server_pb.VacuumVolumeCleanupRequest{ VolumdId: uint32(vid), }) return err