seaweedfs/weed/pb/volume_server.proto

618 lines
15 KiB
Protocol Buffer
Raw Normal View History

2018-10-11 08:16:33 +00:00
syntax = "proto3";
package volume_server_pb;
option go_package = "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb";
2018-10-11 08:16:33 +00:00
2021-08-26 22:18:34 +00:00
import "remote.proto";
2018-10-11 08:16:33 +00:00
//////////////////////////////////////////////////
service VolumeServer {
//Experts only: takes multiple fid parameters. This function does not propagate deletes to replicas.
rpc BatchDelete (BatchDeleteRequest) returns (BatchDeleteResponse) {
}
2018-10-15 06:12:43 +00:00
rpc VacuumVolumeCheck (VacuumVolumeCheckRequest) returns (VacuumVolumeCheckResponse) {
}
rpc VacuumVolumeCompact (VacuumVolumeCompactRequest) returns (stream VacuumVolumeCompactResponse) {
2018-10-15 06:12:43 +00:00
}
rpc VacuumVolumeCommit (VacuumVolumeCommitRequest) returns (VacuumVolumeCommitResponse) {
}
rpc VacuumVolumeCleanup (VacuumVolumeCleanupRequest) returns (VacuumVolumeCleanupResponse) {
}
rpc DeleteCollection (DeleteCollectionRequest) returns (DeleteCollectionResponse) {
}
2019-04-11 04:41:17 +00:00
rpc AllocateVolume (AllocateVolumeRequest) returns (AllocateVolumeResponse) {
}
2018-10-16 04:44:41 +00:00
rpc VolumeSyncStatus (VolumeSyncStatusRequest) returns (VolumeSyncStatusResponse) {
}
2019-04-18 05:04:49 +00:00
rpc VolumeIncrementalCopy (VolumeIncrementalCopyRequest) returns (stream VolumeIncrementalCopyResponse) {
2019-03-25 16:16:12 +00:00
}
rpc VolumeMount (VolumeMountRequest) returns (VolumeMountResponse) {
}
rpc VolumeUnmount (VolumeUnmountRequest) returns (VolumeUnmountResponse) {
}
rpc VolumeDelete (VolumeDeleteRequest) returns (VolumeDeleteResponse) {
}
rpc VolumeMarkReadonly (VolumeMarkReadonlyRequest) returns (VolumeMarkReadonlyResponse) {
}
rpc VolumeMarkWritable (VolumeMarkWritableRequest) returns (VolumeMarkWritableResponse) {
}
rpc VolumeConfigure (VolumeConfigureRequest) returns (VolumeConfigureResponse) {
}
rpc VolumeStatus (VolumeStatusRequest) returns (VolumeStatusResponse) {
}
2019-04-18 05:04:49 +00:00
// copy the .idx .dat files, and mount this volume
rpc VolumeCopy (VolumeCopyRequest) returns (stream VolumeCopyResponse) {
}
rpc ReadVolumeFileStatus (ReadVolumeFileStatusRequest) returns (ReadVolumeFileStatusResponse) {
}
rpc CopyFile (CopyFileRequest) returns (stream CopyFileResponse) {
}
rpc ReadNeedleBlob (ReadNeedleBlobRequest) returns (ReadNeedleBlobResponse) {
}
rpc ReadNeedleMeta (ReadNeedleMetaRequest) returns (ReadNeedleMetaResponse) {
}
rpc WriteNeedleBlob (WriteNeedleBlobRequest) returns (WriteNeedleBlobResponse) {
}
rpc ReadAllNeedles (ReadAllNeedlesRequest) returns (stream ReadAllNeedlesResponse) {
}
2019-04-20 18:35:20 +00:00
rpc VolumeTailSender (VolumeTailSenderRequest) returns (stream VolumeTailSenderResponse) {
}
rpc VolumeTailReceiver (VolumeTailReceiverRequest) returns (VolumeTailReceiverResponse) {
}
// erasure coding
rpc VolumeEcShardsGenerate (VolumeEcShardsGenerateRequest) returns (VolumeEcShardsGenerateResponse) {
}
2019-06-03 09:26:31 +00:00
rpc VolumeEcShardsRebuild (VolumeEcShardsRebuildRequest) returns (VolumeEcShardsRebuildResponse) {
}
rpc VolumeEcShardsCopy (VolumeEcShardsCopyRequest) returns (VolumeEcShardsCopyResponse) {
}
rpc VolumeEcShardsDelete (VolumeEcShardsDeleteRequest) returns (VolumeEcShardsDeleteResponse) {
}
2019-05-26 06:23:19 +00:00
rpc VolumeEcShardsMount (VolumeEcShardsMountRequest) returns (VolumeEcShardsMountResponse) {
}
rpc VolumeEcShardsUnmount (VolumeEcShardsUnmountRequest) returns (VolumeEcShardsUnmountResponse) {
}
2019-05-27 18:59:03 +00:00
rpc VolumeEcShardRead (VolumeEcShardReadRequest) returns (stream VolumeEcShardReadResponse) {
}
2019-06-20 07:17:11 +00:00
rpc VolumeEcBlobDelete (VolumeEcBlobDeleteRequest) returns (VolumeEcBlobDeleteResponse) {
}
2019-12-23 20:48:20 +00:00
rpc VolumeEcShardsToVolume (VolumeEcShardsToVolumeRequest) returns (VolumeEcShardsToVolumeResponse) {
}
2019-11-27 11:09:42 +00:00
// tiered storage
rpc VolumeTierMoveDatToRemote (VolumeTierMoveDatToRemoteRequest) returns (stream VolumeTierMoveDatToRemoteResponse) {
}
rpc VolumeTierMoveDatFromRemote (VolumeTierMoveDatFromRemoteRequest) returns (stream VolumeTierMoveDatFromRemoteResponse) {
2019-11-27 11:09:42 +00:00
}
2020-02-22 05:45:03 +00:00
rpc VolumeServerStatus (VolumeServerStatusRequest) returns (VolumeServerStatusResponse) {
}
2020-09-14 04:25:51 +00:00
rpc VolumeServerLeave (VolumeServerLeaveRequest) returns (VolumeServerLeaveResponse) {
}
2020-02-22 05:45:03 +00:00
2021-08-07 21:18:53 +00:00
// remote storage
rpc FetchAndWriteNeedle (FetchAndWriteNeedleRequest) returns (FetchAndWriteNeedleResponse) {
}
2020-02-22 05:45:03 +00:00
// <experimental> query
rpc Query (QueryRequest) returns (stream QueriedStripe) {
}
rpc VolumeNeedleStatus (VolumeNeedleStatusRequest) returns (VolumeNeedleStatusResponse) {
}
2022-04-01 23:37:06 +00:00
rpc Ping (PingRequest) returns (PingResponse) {
}
2018-10-11 08:16:33 +00:00
}
//////////////////////////////////////////////////
message BatchDeleteRequest {
repeated string file_ids = 1;
2020-03-30 08:19:33 +00:00
bool skip_cookie_check = 2;
2018-10-11 08:16:33 +00:00
}
message BatchDeleteResponse {
repeated DeleteResult results = 1;
}
message DeleteResult {
string file_id = 1;
int32 status = 2;
string error = 3;
uint32 size = 4;
2019-06-20 07:17:11 +00:00
uint32 version = 5;
2018-10-11 08:16:33 +00:00
}
message Empty {
}
2018-10-15 06:12:43 +00:00
message VacuumVolumeCheckRequest {
uint32 volume_id = 1;
2018-10-15 06:12:43 +00:00
}
message VacuumVolumeCheckResponse {
double garbage_ratio = 1;
}
message VacuumVolumeCompactRequest {
uint32 volume_id = 1;
2018-10-15 06:12:43 +00:00
int64 preallocate = 2;
}
message VacuumVolumeCompactResponse {
int64 processed_bytes = 1;
2022-08-01 16:32:21 +00:00
float load_avg_1m = 2;
2018-10-15 06:12:43 +00:00
}
message VacuumVolumeCommitRequest {
uint32 volume_id = 1;
2018-10-15 06:12:43 +00:00
}
message VacuumVolumeCommitResponse {
bool is_read_only = 1;
2018-10-15 06:12:43 +00:00
}
message VacuumVolumeCleanupRequest {
uint32 volume_id = 1;
2018-10-15 06:12:43 +00:00
}
message VacuumVolumeCleanupResponse {
}
message DeleteCollectionRequest {
string collection = 1;
}
message DeleteCollectionResponse {
}
2019-04-11 04:41:17 +00:00
message AllocateVolumeRequest {
uint32 volume_id = 1;
string collection = 2;
int64 preallocate = 3;
string replication = 4;
string ttl = 5;
2019-10-22 05:57:01 +00:00
uint32 memory_map_max_size_mb = 6;
2020-12-14 07:08:21 +00:00
string disk_type = 7;
}
2019-04-11 04:41:17 +00:00
message AllocateVolumeResponse {
}
message VolumeSyncStatusRequest {
uint32 volume_id = 1;
}
message VolumeSyncStatusResponse {
uint32 volume_id = 1;
string collection = 2;
string replication = 4;
string ttl = 5;
uint64 tail_offset = 6;
uint32 compact_revision = 7;
uint64 idx_file_size = 8;
}
2019-04-18 05:04:49 +00:00
message VolumeIncrementalCopyRequest {
2019-03-25 16:16:12 +00:00
uint32 volume_id = 1;
uint64 since_ns = 2;
2019-03-25 16:16:12 +00:00
}
2019-04-18 05:04:49 +00:00
message VolumeIncrementalCopyResponse {
2019-03-25 16:16:12 +00:00
bytes file_content = 1;
}
message VolumeMountRequest {
uint32 volume_id = 1;
}
message VolumeMountResponse {
}
message VolumeUnmountRequest {
uint32 volume_id = 1;
}
message VolumeUnmountResponse {
}
message VolumeDeleteRequest {
uint32 volume_id = 1;
}
message VolumeDeleteResponse {
}
message VolumeMarkReadonlyRequest {
uint32 volume_id = 1;
}
message VolumeMarkReadonlyResponse {
}
message VolumeMarkWritableRequest {
uint32 volume_id = 1;
}
message VolumeMarkWritableResponse {
}
message VolumeConfigureRequest {
uint32 volume_id = 1;
string replication = 2;
}
message VolumeConfigureResponse {
string error = 1;
}
message VolumeStatusRequest {
uint32 volume_id = 1;
}
message VolumeStatusResponse {
bool is_read_only = 1;
}
2019-04-18 05:04:49 +00:00
message VolumeCopyRequest {
uint32 volume_id = 1;
string collection = 2;
string replication = 3;
string ttl = 4;
string source_data_node = 5;
string disk_type = 6;
int64 io_byte_per_second = 7;
}
2019-04-18 05:04:49 +00:00
message VolumeCopyResponse {
2019-04-19 07:39:34 +00:00
uint64 last_append_at_ns = 1;
int64 processed_bytes = 2;
}
message CopyFileRequest {
uint32 volume_id = 1;
2019-05-20 04:37:49 +00:00
string ext = 2;
uint32 compaction_revision = 3;
uint64 stop_offset = 4;
2019-06-03 09:26:31 +00:00
string collection = 5;
bool is_ec_volume = 6;
2019-12-23 20:48:20 +00:00
bool ignore_source_file_not_found = 7;
}
message CopyFileResponse {
bytes file_content = 1;
int64 modified_ts_ns = 2;
}
message ReadNeedleBlobRequest {
uint32 volume_id = 1;
int64 offset = 3; // actual offset
int32 size = 4;
}
message ReadNeedleBlobResponse {
bytes needle_blob = 1;
}
message ReadNeedleMetaRequest {
uint32 volume_id = 1;
uint64 needle_id = 2;
int64 offset = 3; // actual offset
int32 size = 4;
}
message ReadNeedleMetaResponse {
uint32 cookie = 1;
uint64 last_modified = 2;
uint32 crc = 3;
string ttl = 4;
uint64 append_at_ns = 5;
}
message WriteNeedleBlobRequest {
uint32 volume_id = 1;
uint64 needle_id = 2;
int32 size = 3;
bytes needle_blob = 4;
}
message WriteNeedleBlobResponse {
}
message ReadAllNeedlesRequest {
repeated uint32 volume_ids = 1;
}
message ReadAllNeedlesResponse {
uint32 volume_id = 1;
uint64 needle_id = 2;
uint32 cookie = 3;
bytes needle_blob = 5;
bool needle_blob_compressed = 6;
uint64 last_modified = 7;
uint32 crc = 8;
bytes name = 9;
bytes mime = 10;
}
2019-04-20 18:35:20 +00:00
message VolumeTailSenderRequest {
uint32 volume_id = 1;
uint64 since_ns = 2;
2019-04-20 19:05:28 +00:00
uint32 idle_timeout_seconds = 3;
}
2019-04-20 18:35:20 +00:00
message VolumeTailSenderResponse {
bytes needle_header = 1;
bytes needle_body = 2;
2019-04-19 02:22:13 +00:00
bool is_last_chunk = 3;
}
2019-04-20 18:35:20 +00:00
message VolumeTailReceiverRequest {
uint32 volume_id = 1;
uint64 since_ns = 2;
2019-04-20 19:05:28 +00:00
uint32 idle_timeout_seconds = 3;
2019-04-20 18:35:20 +00:00
string source_volume_server = 4;
}
message VolumeTailReceiverResponse {
}
message VolumeEcShardsGenerateRequest {
uint32 volume_id = 1;
string collection = 2;
}
message VolumeEcShardsGenerateResponse {
}
2019-06-03 09:26:31 +00:00
message VolumeEcShardsRebuildRequest {
uint32 volume_id = 1;
string collection = 2;
}
message VolumeEcShardsRebuildResponse {
repeated uint32 rebuilt_shard_ids = 1;
}
message VolumeEcShardsCopyRequest {
uint32 volume_id = 1;
string collection = 2;
2019-05-27 18:59:03 +00:00
repeated uint32 shard_ids = 3;
2019-06-03 09:26:31 +00:00
bool copy_ecx_file = 4;
string source_data_node = 5;
2019-12-23 20:48:20 +00:00
bool copy_ecj_file = 6;
2019-12-28 20:44:59 +00:00
bool copy_vif_file = 7;
}
message VolumeEcShardsCopyResponse {
}
message VolumeEcShardsDeleteRequest {
uint32 volume_id = 1;
2019-06-03 09:26:31 +00:00
string collection = 2;
2019-05-27 18:59:03 +00:00
repeated uint32 shard_ids = 3;
}
message VolumeEcShardsDeleteResponse {
}
2019-05-26 06:23:19 +00:00
message VolumeEcShardsMountRequest {
uint32 volume_id = 1;
string collection = 2;
2019-05-27 18:59:03 +00:00
repeated uint32 shard_ids = 3;
2019-05-26 06:23:19 +00:00
}
message VolumeEcShardsMountResponse {
}
message VolumeEcShardsUnmountRequest {
uint32 volume_id = 1;
2019-05-27 18:59:03 +00:00
repeated uint32 shard_ids = 3;
2019-05-26 06:23:19 +00:00
}
message VolumeEcShardsUnmountResponse {
}
2019-05-27 18:59:03 +00:00
message VolumeEcShardReadRequest {
uint32 volume_id = 1;
uint32 shard_id = 2;
int64 offset = 3;
int64 size = 4;
2019-06-21 08:14:10 +00:00
uint64 file_key = 5;
2019-05-27 18:59:03 +00:00
}
message VolumeEcShardReadResponse {
bytes data = 1;
2019-06-21 08:14:10 +00:00
bool is_deleted = 2;
2019-05-27 18:59:03 +00:00
}
2019-06-20 07:17:11 +00:00
message VolumeEcBlobDeleteRequest {
uint32 volume_id = 1;
string collection = 2;
uint64 file_key = 3;
uint32 version = 4;
}
message VolumeEcBlobDeleteResponse {
}
2019-12-23 20:48:20 +00:00
message VolumeEcShardsToVolumeRequest {
uint32 volume_id = 1;
string collection = 2;
}
message VolumeEcShardsToVolumeResponse {
}
message ReadVolumeFileStatusRequest {
uint32 volume_id = 1;
}
message ReadVolumeFileStatusResponse {
uint32 volume_id = 1;
2019-04-19 07:39:34 +00:00
uint64 idx_file_timestamp_seconds = 2;
uint64 idx_file_size = 3;
2019-04-19 07:39:34 +00:00
uint64 dat_file_timestamp_seconds = 4;
uint64 dat_file_size = 5;
uint64 file_count = 6;
uint32 compaction_revision = 7;
2019-04-20 18:35:20 +00:00
string collection = 8;
2020-12-14 07:08:21 +00:00
string disk_type = 9;
VolumeInfo volume_info = 10;
}
message DiskStatus {
string dir = 1;
uint64 all = 2;
uint64 used = 3;
uint64 free = 4;
float percent_free = 5;
float percent_used = 6;
2020-12-14 08:51:57 +00:00
string disk_type = 7;
}
message MemStatus {
int32 goroutines = 1;
uint64 all = 2;
uint64 used = 3;
uint64 free = 4;
uint64 self = 5;
uint64 heap = 6;
uint64 stack = 7;
}
2019-11-27 11:09:42 +00:00
// tired storage on volume servers
message RemoteFile {
2019-11-27 11:09:42 +00:00
string backend_type = 1;
string backend_id = 2;
string key = 3;
uint64 offset = 4;
uint64 file_size = 5;
uint64 modified_time = 6;
2019-12-28 19:16:10 +00:00
string extension = 7;
}
2019-12-28 19:21:49 +00:00
message VolumeInfo {
repeated RemoteFile files = 1;
uint32 version = 2;
string replication = 3;
uint32 BytesOffset = 4;
2019-11-27 11:09:42 +00:00
}
2021-08-07 21:18:53 +00:00
// tiered storage
message VolumeTierMoveDatToRemoteRequest {
2019-11-27 11:09:42 +00:00
uint32 volume_id = 1;
string collection = 2;
string destination_backend_name = 3;
bool keep_local_dat_file = 4;
2019-11-27 11:09:42 +00:00
}
message VolumeTierMoveDatToRemoteResponse {
int64 processed = 1;
float processedPercentage = 2;
}
message VolumeTierMoveDatFromRemoteRequest {
uint32 volume_id = 1;
string collection = 2;
bool keep_remote_dat_file = 3;
}
message VolumeTierMoveDatFromRemoteResponse {
int64 processed = 1;
float processedPercentage = 2;
2019-11-27 11:09:42 +00:00
}
2020-02-22 05:45:03 +00:00
message VolumeServerStatusRequest {
}
message VolumeServerStatusResponse {
repeated DiskStatus disk_statuses = 1;
MemStatus memory_status = 2;
2022-06-12 18:56:23 +00:00
string version = 3;
string data_center = 4;
string rack = 5;
2020-02-22 05:45:03 +00:00
}
2020-09-14 04:25:51 +00:00
message VolumeServerLeaveRequest {
}
message VolumeServerLeaveResponse {
}
2021-08-07 21:18:53 +00:00
// remote storage
message FetchAndWriteNeedleRequest {
uint32 volume_id = 1;
uint64 needle_id = 2;
2021-08-09 21:35:18 +00:00
uint32 cookie = 3;
int64 offset = 4;
int64 size = 5;
2021-09-07 01:30:44 +00:00
message Replica {
string url = 1;
string public_url = 2;
int32 grpc_port = 3;
2021-09-07 01:30:44 +00:00
}
repeated Replica replicas = 6;
string auth = 7;
2021-08-07 21:18:53 +00:00
// remote conf
2021-08-26 22:18:34 +00:00
remote_pb.RemoteConf remote_conf = 15;
remote_pb.RemoteStorageLocation remote_location = 16;
2021-08-07 21:18:53 +00:00
}
message FetchAndWriteNeedleResponse {
string e_tag = 1;
2021-08-07 21:18:53 +00:00
}
2019-11-27 11:09:42 +00:00
// select on volume servers
message QueryRequest {
repeated string selections = 1;
repeated string from_file_ids = 2;
message Filter {
string field = 1;
string operand = 2;
string value = 3;
}
Filter filter = 3;
message InputSerialization {
// NONE | GZIP | BZIP2
string compression_type = 1;
message CSVInput {
string file_header_info = 1; // Valid values: NONE | USE | IGNORE
string record_delimiter = 2; // Default: \n
string field_delimiter = 3; // Default: ,
string quote_character = 4; // Default: "
string quote_escape_character = 5; // Default: "
string comments = 6; // Default: #
// If true, records might contain record delimiters within quote characters
bool allow_quoted_record_delimiter = 7; // default False.
}
message JSONInput {
string type = 1; // Valid values: DOCUMENT | LINES
}
message ParquetInput {
}
CSVInput csv_input = 2;
JSONInput json_input = 3;
ParquetInput parquet_input = 4;
}
InputSerialization input_serialization = 4;
message OutputSerialization {
message CSVOutput {
string quote_fields = 1; // Valid values: ALWAYS | ASNEEDED
string record_delimiter = 2; // Default: \n
string field_delimiter = 3; // Default: ,
string quote_character = 4; // Default: "
string quote_escape_character = 5; // Default: "
}
message JSONOutput {
string record_delimiter = 1;
}
CSVOutput csv_output = 2;
JSONOutput json_output = 3;
}
OutputSerialization output_serialization = 5;
}
message QueriedStripe {
bytes records = 1;
}
message VolumeNeedleStatusRequest {
uint32 volume_id = 1;
uint64 needle_id = 2;
}
message VolumeNeedleStatusResponse {
uint64 needle_id = 1;
uint32 cookie = 2;
uint32 size = 3;
uint64 last_modified = 4;
uint32 crc = 5;
string ttl = 6;
}
2022-04-01 23:37:06 +00:00
message PingRequest {
string target = 1; // default to ping itself
string target_type = 2;
}
message PingResponse {
2022-04-16 19:45:49 +00:00
int64 start_time_ns = 1;
int64 remote_time_ns = 2;
int64 stop_time_ns = 3;
2022-04-01 23:37:06 +00:00
}