seaweedfs/weed/pb/volume_server.proto
Konstantin Lebedev 2f72103c83
avoid load volume file with BytesOffset mismatch (#3841)
* avoid load volume file with BytesOffset mismatch

https://github.com/seaweedfs/seaweedfs/issues/2966

* set BytesOffset if has not VolumeInfoFile

* typos fail => failed

* exit if bytesOffset mismatch
2022-10-14 00:18:09 -07:00

610 lines
15 KiB
Protocol Buffer

syntax = "proto3";
package volume_server_pb;
option go_package = "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb";
import "remote.proto";
//////////////////////////////////////////////////
service VolumeServer {
//Experts only: takes multiple fid parameters. This function does not propagate deletes to replicas.
rpc BatchDelete (BatchDeleteRequest) returns (BatchDeleteResponse) {
}
rpc VacuumVolumeCheck (VacuumVolumeCheckRequest) returns (VacuumVolumeCheckResponse) {
}
rpc VacuumVolumeCompact (VacuumVolumeCompactRequest) returns (stream VacuumVolumeCompactResponse) {
}
rpc VacuumVolumeCommit (VacuumVolumeCommitRequest) returns (VacuumVolumeCommitResponse) {
}
rpc VacuumVolumeCleanup (VacuumVolumeCleanupRequest) returns (VacuumVolumeCleanupResponse) {
}
rpc DeleteCollection (DeleteCollectionRequest) returns (DeleteCollectionResponse) {
}
rpc AllocateVolume (AllocateVolumeRequest) returns (AllocateVolumeResponse) {
}
rpc VolumeSyncStatus (VolumeSyncStatusRequest) returns (VolumeSyncStatusResponse) {
}
rpc VolumeIncrementalCopy (VolumeIncrementalCopyRequest) returns (stream VolumeIncrementalCopyResponse) {
}
rpc VolumeMount (VolumeMountRequest) returns (VolumeMountResponse) {
}
rpc VolumeUnmount (VolumeUnmountRequest) returns (VolumeUnmountResponse) {
}
rpc VolumeDelete (VolumeDeleteRequest) returns (VolumeDeleteResponse) {
}
rpc VolumeMarkReadonly (VolumeMarkReadonlyRequest) returns (VolumeMarkReadonlyResponse) {
}
rpc VolumeMarkWritable (VolumeMarkWritableRequest) returns (VolumeMarkWritableResponse) {
}
rpc VolumeConfigure (VolumeConfigureRequest) returns (VolumeConfigureResponse) {
}
rpc VolumeStatus (VolumeStatusRequest) returns (VolumeStatusResponse) {
}
// copy the .idx .dat files, and mount this volume
rpc VolumeCopy (VolumeCopyRequest) returns (stream VolumeCopyResponse) {
}
rpc ReadVolumeFileStatus (ReadVolumeFileStatusRequest) returns (ReadVolumeFileStatusResponse) {
}
rpc CopyFile (CopyFileRequest) returns (stream CopyFileResponse) {
}
rpc ReadNeedleBlob (ReadNeedleBlobRequest) returns (ReadNeedleBlobResponse) {
}
rpc ReadNeedleMeta (ReadNeedleMetaRequest) returns (ReadNeedleMetaResponse) {
}
rpc WriteNeedleBlob (WriteNeedleBlobRequest) returns (WriteNeedleBlobResponse) {
}
rpc ReadAllNeedles (ReadAllNeedlesRequest) returns (stream ReadAllNeedlesResponse) {
}
rpc VolumeTailSender (VolumeTailSenderRequest) returns (stream VolumeTailSenderResponse) {
}
rpc VolumeTailReceiver (VolumeTailReceiverRequest) returns (VolumeTailReceiverResponse) {
}
// erasure coding
rpc VolumeEcShardsGenerate (VolumeEcShardsGenerateRequest) returns (VolumeEcShardsGenerateResponse) {
}
rpc VolumeEcShardsRebuild (VolumeEcShardsRebuildRequest) returns (VolumeEcShardsRebuildResponse) {
}
rpc VolumeEcShardsCopy (VolumeEcShardsCopyRequest) returns (VolumeEcShardsCopyResponse) {
}
rpc VolumeEcShardsDelete (VolumeEcShardsDeleteRequest) returns (VolumeEcShardsDeleteResponse) {
}
rpc VolumeEcShardsMount (VolumeEcShardsMountRequest) returns (VolumeEcShardsMountResponse) {
}
rpc VolumeEcShardsUnmount (VolumeEcShardsUnmountRequest) returns (VolumeEcShardsUnmountResponse) {
}
rpc VolumeEcShardRead (VolumeEcShardReadRequest) returns (stream VolumeEcShardReadResponse) {
}
rpc VolumeEcBlobDelete (VolumeEcBlobDeleteRequest) returns (VolumeEcBlobDeleteResponse) {
}
rpc VolumeEcShardsToVolume (VolumeEcShardsToVolumeRequest) returns (VolumeEcShardsToVolumeResponse) {
}
// tiered storage
rpc VolumeTierMoveDatToRemote (VolumeTierMoveDatToRemoteRequest) returns (stream VolumeTierMoveDatToRemoteResponse) {
}
rpc VolumeTierMoveDatFromRemote (VolumeTierMoveDatFromRemoteRequest) returns (stream VolumeTierMoveDatFromRemoteResponse) {
}
rpc VolumeServerStatus (VolumeServerStatusRequest) returns (VolumeServerStatusResponse) {
}
rpc VolumeServerLeave (VolumeServerLeaveRequest) returns (VolumeServerLeaveResponse) {
}
// remote storage
rpc FetchAndWriteNeedle (FetchAndWriteNeedleRequest) returns (FetchAndWriteNeedleResponse) {
}
// <experimental> query
rpc Query (QueryRequest) returns (stream QueriedStripe) {
}
rpc VolumeNeedleStatus (VolumeNeedleStatusRequest) returns (VolumeNeedleStatusResponse) {
}
rpc Ping (PingRequest) returns (PingResponse) {
}
}
//////////////////////////////////////////////////
message BatchDeleteRequest {
repeated string file_ids = 1;
bool skip_cookie_check = 2;
}
message BatchDeleteResponse {
repeated DeleteResult results = 1;
}
message DeleteResult {
string file_id = 1;
int32 status = 2;
string error = 3;
uint32 size = 4;
uint32 version = 5;
}
message Empty {
}
message VacuumVolumeCheckRequest {
uint32 volume_id = 1;
}
message VacuumVolumeCheckResponse {
double garbage_ratio = 1;
}
message VacuumVolumeCompactRequest {
uint32 volume_id = 1;
int64 preallocate = 2;
}
message VacuumVolumeCompactResponse {
int64 processed_bytes = 1;
float load_avg_1m = 2;
}
message VacuumVolumeCommitRequest {
uint32 volume_id = 1;
}
message VacuumVolumeCommitResponse {
bool is_read_only = 1;
}
message VacuumVolumeCleanupRequest {
uint32 volume_id = 1;
}
message VacuumVolumeCleanupResponse {
}
message DeleteCollectionRequest {
string collection = 1;
}
message DeleteCollectionResponse {
}
message AllocateVolumeRequest {
uint32 volume_id = 1;
string collection = 2;
int64 preallocate = 3;
string replication = 4;
string ttl = 5;
uint32 memory_map_max_size_mb = 6;
string disk_type = 7;
}
message AllocateVolumeResponse {
}
message VolumeSyncStatusRequest {
uint32 volume_id = 1;
}
message VolumeSyncStatusResponse {
uint32 volume_id = 1;
string collection = 2;
string replication = 4;
string ttl = 5;
uint64 tail_offset = 6;
uint32 compact_revision = 7;
uint64 idx_file_size = 8;
}
message VolumeIncrementalCopyRequest {
uint32 volume_id = 1;
uint64 since_ns = 2;
}
message VolumeIncrementalCopyResponse {
bytes file_content = 1;
}
message VolumeMountRequest {
uint32 volume_id = 1;
}
message VolumeMountResponse {
}
message VolumeUnmountRequest {
uint32 volume_id = 1;
}
message VolumeUnmountResponse {
}
message VolumeDeleteRequest {
uint32 volume_id = 1;
}
message VolumeDeleteResponse {
}
message VolumeMarkReadonlyRequest {
uint32 volume_id = 1;
}
message VolumeMarkReadonlyResponse {
}
message VolumeMarkWritableRequest {
uint32 volume_id = 1;
}
message VolumeMarkWritableResponse {
}
message VolumeConfigureRequest {
uint32 volume_id = 1;
string replication = 2;
}
message VolumeConfigureResponse {
string error = 1;
}
message VolumeStatusRequest {
uint32 volume_id = 1;
}
message VolumeStatusResponse {
bool is_read_only = 1;
}
message VolumeCopyRequest {
uint32 volume_id = 1;
string collection = 2;
string replication = 3;
string ttl = 4;
string source_data_node = 5;
string disk_type = 6;
int64 io_byte_per_second = 7;
}
message VolumeCopyResponse {
uint64 last_append_at_ns = 1;
int64 processed_bytes = 2;
}
message CopyFileRequest {
uint32 volume_id = 1;
string ext = 2;
uint32 compaction_revision = 3;
uint64 stop_offset = 4;
string collection = 5;
bool is_ec_volume = 6;
bool ignore_source_file_not_found = 7;
}
message CopyFileResponse {
bytes file_content = 1;
int64 modified_ts_ns = 2;
}
message ReadNeedleBlobRequest {
uint32 volume_id = 1;
int64 offset = 3; // actual offset
int32 size = 4;
}
message ReadNeedleBlobResponse {
bytes needle_blob = 1;
}
message ReadNeedleMetaRequest {
uint32 volume_id = 1;
uint64 needle_id = 2;
int64 offset = 3; // actual offset
int32 size = 4;
}
message ReadNeedleMetaResponse {
uint32 cookie = 1;
uint64 last_modified = 2;
uint32 crc = 3;
string ttl = 4;
}
message WriteNeedleBlobRequest {
uint32 volume_id = 1;
uint64 needle_id = 2;
int32 size = 3;
bytes needle_blob = 4;
}
message WriteNeedleBlobResponse {
}
message ReadAllNeedlesRequest {
repeated uint32 volume_ids = 1;
}
message ReadAllNeedlesResponse {
uint32 volume_id = 1;
uint64 needle_id = 2;
uint32 cookie = 3;
bytes needle_blob = 5;
}
message VolumeTailSenderRequest {
uint32 volume_id = 1;
uint64 since_ns = 2;
uint32 idle_timeout_seconds = 3;
}
message VolumeTailSenderResponse {
bytes needle_header = 1;
bytes needle_body = 2;
bool is_last_chunk = 3;
}
message VolumeTailReceiverRequest {
uint32 volume_id = 1;
uint64 since_ns = 2;
uint32 idle_timeout_seconds = 3;
string source_volume_server = 4;
}
message VolumeTailReceiverResponse {
}
message VolumeEcShardsGenerateRequest {
uint32 volume_id = 1;
string collection = 2;
}
message VolumeEcShardsGenerateResponse {
}
message VolumeEcShardsRebuildRequest {
uint32 volume_id = 1;
string collection = 2;
}
message VolumeEcShardsRebuildResponse {
repeated uint32 rebuilt_shard_ids = 1;
}
message VolumeEcShardsCopyRequest {
uint32 volume_id = 1;
string collection = 2;
repeated uint32 shard_ids = 3;
bool copy_ecx_file = 4;
string source_data_node = 5;
bool copy_ecj_file = 6;
bool copy_vif_file = 7;
}
message VolumeEcShardsCopyResponse {
}
message VolumeEcShardsDeleteRequest {
uint32 volume_id = 1;
string collection = 2;
repeated uint32 shard_ids = 3;
}
message VolumeEcShardsDeleteResponse {
}
message VolumeEcShardsMountRequest {
uint32 volume_id = 1;
string collection = 2;
repeated uint32 shard_ids = 3;
}
message VolumeEcShardsMountResponse {
}
message VolumeEcShardsUnmountRequest {
uint32 volume_id = 1;
repeated uint32 shard_ids = 3;
}
message VolumeEcShardsUnmountResponse {
}
message VolumeEcShardReadRequest {
uint32 volume_id = 1;
uint32 shard_id = 2;
int64 offset = 3;
int64 size = 4;
uint64 file_key = 5;
}
message VolumeEcShardReadResponse {
bytes data = 1;
bool is_deleted = 2;
}
message VolumeEcBlobDeleteRequest {
uint32 volume_id = 1;
string collection = 2;
uint64 file_key = 3;
uint32 version = 4;
}
message VolumeEcBlobDeleteResponse {
}
message VolumeEcShardsToVolumeRequest {
uint32 volume_id = 1;
string collection = 2;
}
message VolumeEcShardsToVolumeResponse {
}
message ReadVolumeFileStatusRequest {
uint32 volume_id = 1;
}
message ReadVolumeFileStatusResponse {
uint32 volume_id = 1;
uint64 idx_file_timestamp_seconds = 2;
uint64 idx_file_size = 3;
uint64 dat_file_timestamp_seconds = 4;
uint64 dat_file_size = 5;
uint64 file_count = 6;
uint32 compaction_revision = 7;
string collection = 8;
string disk_type = 9;
}
message DiskStatus {
string dir = 1;
uint64 all = 2;
uint64 used = 3;
uint64 free = 4;
float percent_free = 5;
float percent_used = 6;
string disk_type = 7;
}
message MemStatus {
int32 goroutines = 1;
uint64 all = 2;
uint64 used = 3;
uint64 free = 4;
uint64 self = 5;
uint64 heap = 6;
uint64 stack = 7;
}
// tired storage on volume servers
message RemoteFile {
string backend_type = 1;
string backend_id = 2;
string key = 3;
uint64 offset = 4;
uint64 file_size = 5;
uint64 modified_time = 6;
string extension = 7;
}
message VolumeInfo {
repeated RemoteFile files = 1;
uint32 version = 2;
string replication = 3;
uint32 BytesOffset = 4;
}
// tiered storage
message VolumeTierMoveDatToRemoteRequest {
uint32 volume_id = 1;
string collection = 2;
string destination_backend_name = 3;
bool keep_local_dat_file = 4;
}
message VolumeTierMoveDatToRemoteResponse {
int64 processed = 1;
float processedPercentage = 2;
}
message VolumeTierMoveDatFromRemoteRequest {
uint32 volume_id = 1;
string collection = 2;
bool keep_remote_dat_file = 3;
}
message VolumeTierMoveDatFromRemoteResponse {
int64 processed = 1;
float processedPercentage = 2;
}
message VolumeServerStatusRequest {
}
message VolumeServerStatusResponse {
repeated DiskStatus disk_statuses = 1;
MemStatus memory_status = 2;
string version = 3;
string data_center = 4;
string rack = 5;
}
message VolumeServerLeaveRequest {
}
message VolumeServerLeaveResponse {
}
// remote storage
message FetchAndWriteNeedleRequest {
uint32 volume_id = 1;
uint64 needle_id = 2;
uint32 cookie = 3;
int64 offset = 4;
int64 size = 5;
message Replica {
string url = 1;
string public_url = 2;
int32 grpc_port = 3;
}
repeated Replica replicas = 6;
string auth = 7;
// remote conf
remote_pb.RemoteConf remote_conf = 15;
remote_pb.RemoteStorageLocation remote_location = 16;
}
message FetchAndWriteNeedleResponse {
}
// select on volume servers
message QueryRequest {
repeated string selections = 1;
repeated string from_file_ids = 2;
message Filter {
string field = 1;
string operand = 2;
string value = 3;
}
Filter filter = 3;
message InputSerialization {
// NONE | GZIP | BZIP2
string compression_type = 1;
message CSVInput {
string file_header_info = 1; // Valid values: NONE | USE | IGNORE
string record_delimiter = 2; // Default: \n
string field_delimiter = 3; // Default: ,
string quote_character = 4; // Default: "
string quote_escape_character = 5; // Default: "
string comments = 6; // Default: #
// If true, records might contain record delimiters within quote characters
bool allow_quoted_record_delimiter = 7; // default False.
}
message JSONInput {
string type = 1; // Valid values: DOCUMENT | LINES
}
message ParquetInput {
}
CSVInput csv_input = 2;
JSONInput json_input = 3;
ParquetInput parquet_input = 4;
}
InputSerialization input_serialization = 4;
message OutputSerialization {
message CSVOutput {
string quote_fields = 1; // Valid values: ALWAYS | ASNEEDED
string record_delimiter = 2; // Default: \n
string field_delimiter = 3; // Default: ,
string quote_character = 4; // Default: "
string quote_escape_character = 5; // Default: "
}
message JSONOutput {
string record_delimiter = 1;
}
CSVOutput csv_output = 2;
JSONOutput json_output = 3;
}
OutputSerialization output_serialization = 5;
}
message QueriedStripe {
bytes records = 1;
}
message VolumeNeedleStatusRequest {
uint32 volume_id = 1;
uint64 needle_id = 2;
}
message VolumeNeedleStatusResponse {
uint64 needle_id = 1;
uint32 cookie = 2;
uint32 size = 3;
uint64 last_modified = 4;
uint32 crc = 5;
string ttl = 6;
}
message PingRequest {
string target = 1; // default to ping itself
string target_type = 2;
}
message PingResponse {
int64 start_time_ns = 1;
int64 remote_time_ns = 2;
int64 stop_time_ns = 3;
}