seaweedfs/weed/shell/command_volume_fsck.go

611 lines
18 KiB
Go
Raw Normal View History

package shell
import (
"bufio"
"context"
2020-03-25 05:38:33 +00:00
"flag"
"fmt"
"io"
"io/ioutil"
"math"
"net/http"
"net/url"
"os"
"path/filepath"
2020-03-25 09:21:15 +00:00
"sync"
2020-09-01 07:21:19 +00:00
"github.com/chrislusf/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/operation"
"github.com/chrislusf/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
"github.com/chrislusf/seaweedfs/weed/storage/needle"
"github.com/chrislusf/seaweedfs/weed/storage/needle_map"
"github.com/chrislusf/seaweedfs/weed/storage/types"
"github.com/chrislusf/seaweedfs/weed/util"
)
func init() {
Commands = append(Commands, &commandVolumeFsck{})
}
type commandVolumeFsck struct {
env *CommandEnv
}
func (c *commandVolumeFsck) Name() string {
return "volume.fsck"
}
func (c *commandVolumeFsck) Help() string {
return `check all volumes to find entries not used by the filer
Important assumption!!!
the system is all used by one filer.
This command works this way:
1. collect all file ids from all volumes, as set A
2. collect all file ids from the filer, as set B
3. find out the set A subtract B
If -findMissingChunksInFiler is enabled, this works
in a reverse way:
1. collect all file ids from all volumes, as set A
2. collect all file ids from the filer, as set B
3. find out the set B subtract A
`
}
func (c *commandVolumeFsck) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
2020-03-25 05:38:33 +00:00
fsckCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
verbose := fsckCommand.Bool("v", false, "verbose mode")
2021-06-28 06:32:57 +00:00
findMissingChunksInFiler := fsckCommand.Bool("findMissingChunksInFiler", false, "see \"help volume.fsck\"")
findMissingChunksInFilerPath := fsckCommand.String("findMissingChunksInFilerPath", "/", "used together with findMissingChunksInFiler")
applyPurging := fsckCommand.Bool("reallyDeleteFromVolume", false, "<expert only!> after detection, delete missing data from volumes / delete missing file entries from filer")
2022-02-12 23:53:35 +00:00
purgeAbsent := fsckCommand.Bool("reallyDeleteFilerEntries", false, "<expert only!> delete missing file entries from filer if the corresponding volume is missing for any reason, please ensure all still existing/expected volumes are connected! used together with findMissingChunksInFiler")
2020-03-25 05:38:33 +00:00
if err = fsckCommand.Parse(args); err != nil {
return nil
}
2021-12-10 21:24:38 +00:00
if err = commandEnv.confirmIsLocked(args); err != nil {
return
}
c.env = commandEnv
// create a temp folder
tempFolder, err := os.MkdirTemp("", "sw_fsck")
if err != nil {
return fmt.Errorf("failed to create temp folder: %v", err)
}
2020-03-25 05:38:33 +00:00
if *verbose {
fmt.Fprintf(writer, "working directory: %s\n", tempFolder)
}
defer os.RemoveAll(tempFolder)
2020-03-25 07:56:47 +00:00
// collect all volume id locations
2021-02-22 08:28:42 +00:00
volumeIdToVInfo, err := c.collectVolumeIds(commandEnv, *verbose, writer)
2020-03-25 07:56:47 +00:00
if err != nil {
return fmt.Errorf("failed to collect all volume locations: %v", err)
}
// collect each volume file ids
for volumeId, vinfo := range volumeIdToVInfo {
2020-03-25 07:56:47 +00:00
err = c.collectOneVolumeFileIds(tempFolder, volumeId, vinfo, *verbose, writer)
if err != nil {
return fmt.Errorf("failed to collect file ids from volume %d on %s: %v", volumeId, vinfo.server, err)
}
}
if *findMissingChunksInFiler {
// collect all filer file ids and paths
if err = c.collectFilerFileIdAndPaths(volumeIdToVInfo, tempFolder, writer, *findMissingChunksInFilerPath, *verbose, *purgeAbsent); err != nil {
return fmt.Errorf("collectFilerFileIdAndPaths: %v", err)
}
// for each volume, check filer file ids
if err = c.findFilerChunksMissingInVolumeServers(volumeIdToVInfo, tempFolder, writer, *verbose, *applyPurging); err != nil {
2021-07-29 05:25:04 +00:00
return fmt.Errorf("findFilerChunksMissingInVolumeServers: %v", err)
}
} else {
// collect all filer file ids
if err = c.collectFilerFileIds(volumeIdToVInfo, tempFolder, writer, *verbose); err != nil {
return fmt.Errorf("failed to collect file ids from filer: %v", err)
}
// volume file ids subtract filer file ids
if err = c.findExtraChunksInVolumeServers(volumeIdToVInfo, tempFolder, writer, *verbose, *applyPurging); err != nil {
return fmt.Errorf("findExtraChunksInVolumeServers: %v", err)
}
}
return nil
}
func (c *commandVolumeFsck) collectFilerFileIdAndPaths(volumeIdToServer map[uint32]VInfo, tempFolder string, writer io.Writer, filerPath string, verbose bool, purgeAbsent bool) error {
if verbose {
fmt.Fprintf(writer, "checking each file from filer ...\n")
}
files := make(map[uint32]*os.File)
for vid := range volumeIdToServer {
dst, openErr := os.OpenFile(getFilerFileIdFile(tempFolder, vid), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
if openErr != nil {
return fmt.Errorf("failed to create file %s: %v", getFilerFileIdFile(tempFolder, vid), openErr)
}
files[vid] = dst
}
defer func() {
for _, f := range files {
f.Close()
}
}()
type Item struct {
vid uint32
fileKey uint64
cookie uint32
path util.FullPath
}
2022-01-21 20:08:58 +00:00
return doTraverseBfsAndSaving(c.env, nil, filerPath, false, func(entry *filer_pb.FullEntry, outputChan chan interface{}) (err error) {
2021-06-28 06:32:57 +00:00
if verbose && entry.Entry.IsDirectory {
fmt.Fprintf(writer, "checking directory %s\n", util.NewFullPath(entry.Dir, entry.Entry.Name))
}
dChunks, mChunks, resolveErr := filer.ResolveChunkManifest(filer.LookupFn(c.env), entry.Entry.Chunks, 0, math.MaxInt64)
if resolveErr != nil {
return nil
}
dChunks = append(dChunks, mChunks...)
for _, chunk := range dChunks {
outputChan <- &Item{
vid: chunk.Fid.VolumeId,
fileKey: chunk.Fid.FileKey,
cookie: chunk.Fid.Cookie,
path: util.NewFullPath(entry.Dir, entry.Entry.Name),
}
}
return nil
2022-01-21 20:08:58 +00:00
}, func(outputChan chan interface{}) {
buffer := make([]byte, 16)
for item := range outputChan {
i := item.(*Item)
if f, ok := files[i.vid]; ok {
util.Uint64toBytes(buffer, i.fileKey)
util.Uint32toBytes(buffer[8:], i.cookie)
util.Uint32toBytes(buffer[12:], uint32(len(i.path)))
f.Write(buffer)
f.Write([]byte(i.path))
// fmt.Fprintf(writer, "%d,%x%08x %d %s\n", i.vid, i.fileKey, i.cookie, len(i.path), i.path)
} else {
fmt.Fprintf(writer, "%d,%x%08x %s volume not found\n", i.vid, i.fileKey, i.cookie, i.path)
if purgeAbsent {
fmt.Printf("deleting path %s after volume not found", i.path)
c.httpDelete(i.path, verbose)
}
2022-01-21 20:08:58 +00:00
}
}
})
}
func (c *commandVolumeFsck) findFilerChunksMissingInVolumeServers(volumeIdToVInfo map[uint32]VInfo, tempFolder string, writer io.Writer, verbose bool, applyPurging bool) error {
2021-06-25 00:22:53 +00:00
for volumeId, vinfo := range volumeIdToVInfo {
checkErr := c.oneVolumeFileIdsCheckOneVolume(tempFolder, volumeId, writer, verbose, applyPurging)
if checkErr != nil {
return fmt.Errorf("failed to collect file ids from volume %d on %s: %v", volumeId, vinfo.server, checkErr)
}
}
return nil
2021-06-25 00:22:53 +00:00
}
func (c *commandVolumeFsck) findExtraChunksInVolumeServers(volumeIdToVInfo map[uint32]VInfo, tempFolder string, writer io.Writer, verbose bool, applyPurging bool) error {
2020-03-25 05:38:33 +00:00
var totalInUseCount, totalOrphanChunkCount, totalOrphanDataSize uint64
for volumeId, vinfo := range volumeIdToVInfo {
inUseCount, orphanFileIds, orphanDataSize, checkErr := c.oneVolumeFileIdsSubtractFilerFileIds(tempFolder, volumeId, writer, verbose)
if checkErr != nil {
2020-03-25 01:40:52 +00:00
return fmt.Errorf("failed to collect file ids from volume %d on %s: %v", volumeId, vinfo.server, checkErr)
}
2020-03-25 05:38:33 +00:00
totalInUseCount += inUseCount
2020-03-25 09:21:15 +00:00
totalOrphanChunkCount += uint64(len(orphanFileIds))
totalOrphanDataSize += orphanDataSize
2020-03-25 09:21:15 +00:00
if verbose {
for _, fid := range orphanFileIds {
fmt.Fprintf(writer, "%s\n", fid)
}
}
if applyPurging && len(orphanFileIds) > 0 {
if verbose {
fmt.Fprintf(writer, "purging process for volume %d", volumeId)
}
2020-03-25 09:41:22 +00:00
if vinfo.isEcVolume {
fmt.Fprintf(writer, "skip purging for Erasure Coded volume %d.\n", volumeId)
2021-12-13 06:55:27 +00:00
continue
2020-03-25 09:41:22 +00:00
}
needleVID := needle.VolumeId(volumeId)
if vinfo.isReadOnly {
err := markVolumeWritable(c.env.option.GrpcDialOption, needleVID, vinfo.server, true)
if err != nil {
return fmt.Errorf("mark volume %d read/write: %v", volumeId, err)
}
fmt.Fprintf(writer, "temporarily marked %d on server %v writable for forced purge\n", volumeId, vinfo.server)
defer markVolumeWritable(c.env.option.GrpcDialOption, needleVID, vinfo.server, false)
2020-03-25 09:21:15 +00:00
}
fmt.Fprintf(writer, "marked %d on server %v writable for forced purge\n", volumeId, vinfo.server)
if verbose {
fmt.Fprintf(writer, "purging files from volume %d\n", volumeId)
}
if err := c.purgeFileIdsForOneVolume(volumeId, orphanFileIds, writer); err != nil {
return fmt.Errorf("purging volume %d: %v", volumeId, err)
2020-03-25 09:21:15 +00:00
}
}
}
if !applyPurging {
2020-03-25 09:21:15 +00:00
pct := float64(totalOrphanChunkCount*100) / (float64(totalOrphanChunkCount + totalInUseCount))
fmt.Fprintf(writer, "\nTotal\t\tentries:%d\torphan:%d\t%.2f%%\t%dB\n",
totalOrphanChunkCount+totalInUseCount, totalOrphanChunkCount, pct, totalOrphanDataSize)
2020-03-25 05:38:33 +00:00
2020-03-25 09:21:15 +00:00
fmt.Fprintf(writer, "This could be normal if multiple filers or no filers are used.\n")
2020-03-25 05:38:33 +00:00
}
if totalOrphanChunkCount == 0 {
fmt.Fprintf(writer, "no orphan data\n")
//return nil
}
return nil
}
2020-03-25 07:56:47 +00:00
func (c *commandVolumeFsck) collectOneVolumeFileIds(tempFolder string, volumeId uint32, vinfo VInfo, verbose bool, writer io.Writer) error {
if verbose {
fmt.Fprintf(writer, "collecting volume %d file ids from %s ...\n", volumeId, vinfo.server)
}
return operation.WithVolumeServerClient(false, vinfo.server, c.env.option.GrpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
ext := ".idx"
if vinfo.isEcVolume {
ext = ".ecx"
}
copyFileClient, err := volumeServerClient.CopyFile(context.Background(), &volume_server_pb.CopyFileRequest{
VolumeId: volumeId,
Ext: ext,
CompactionRevision: math.MaxUint32,
StopOffset: math.MaxInt64,
Collection: vinfo.collection,
IsEcVolume: vinfo.isEcVolume,
IgnoreSourceFileNotFound: false,
})
if err != nil {
return fmt.Errorf("failed to start copying volume %d%s: %v", volumeId, ext, err)
}
err = writeToFile(copyFileClient, getVolumeFileIdFile(tempFolder, volumeId))
if err != nil {
return fmt.Errorf("failed to copy %d%s from %s: %v", volumeId, ext, vinfo.server, err)
}
return nil
})
}
func (c *commandVolumeFsck) collectFilerFileIds(volumeIdToServer map[uint32]VInfo, tempFolder string, writer io.Writer, verbose bool) error {
2020-03-25 07:56:47 +00:00
if verbose {
fmt.Fprintf(writer, "collecting file ids from filer ...\n")
}
files := make(map[uint32]*os.File)
for vid := range volumeIdToServer {
dst, openErr := os.OpenFile(getFilerFileIdFile(tempFolder, vid), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
if openErr != nil {
return fmt.Errorf("failed to create file %s: %v", getFilerFileIdFile(tempFolder, vid), openErr)
}
files[vid] = dst
}
defer func() {
for _, f := range files {
f.Close()
}
}()
type Item struct {
vid uint32
fileKey uint64
}
2022-01-21 20:08:58 +00:00
return doTraverseBfsAndSaving(c.env, nil, "/", false, func(entry *filer_pb.FullEntry, outputChan chan interface{}) (err error) {
dChunks, mChunks, resolveErr := filer.ResolveChunkManifest(filer.LookupFn(c.env), entry.Entry.Chunks, 0, math.MaxInt64)
2020-07-21 05:02:05 +00:00
if resolveErr != nil {
2021-08-13 09:57:14 +00:00
if verbose {
fmt.Fprintf(writer, "resolving manifest chunks in %s: %v\n", util.NewFullPath(entry.Dir, entry.Entry.Name), resolveErr)
}
2020-07-21 05:02:05 +00:00
return nil
}
dChunks = append(dChunks, mChunks...)
for _, chunk := range dChunks {
outputChan <- &Item{
vid: chunk.Fid.VolumeId,
fileKey: chunk.Fid.FileKey,
}
}
return nil
2022-01-21 20:08:58 +00:00
}, func(outputChan chan interface{}) {
buffer := make([]byte, 8)
for item := range outputChan {
i := item.(*Item)
util.Uint64toBytes(buffer, i.fileKey)
files[i.vid].Write(buffer)
}
})
}
func (c *commandVolumeFsck) oneVolumeFileIdsCheckOneVolume(tempFolder string, volumeId uint32, writer io.Writer, verbose bool, applyPurging bool) (err error) {
2021-06-28 06:32:57 +00:00
if verbose {
fmt.Fprintf(writer, "find missing file chunks in volume %d ...\n", volumeId)
2021-06-28 06:32:57 +00:00
}
db := needle_map.NewMemDb()
defer db.Close()
if err = db.LoadFromIdx(getVolumeFileIdFile(tempFolder, volumeId)); err != nil {
return
}
file := getFilerFileIdFile(tempFolder, volumeId)
fp, err := os.Open(file)
if err != nil {
return
}
defer fp.Close()
type Item struct {
fileKey uint64
cookie uint32
path util.FullPath
}
br := bufio.NewReader(fp)
buffer := make([]byte, 16)
item := &Item{}
var readSize int
for {
2021-06-28 06:32:57 +00:00
readSize, err = io.ReadFull(br, buffer)
if err != nil || readSize != 16 {
break
}
item.fileKey = util.BytesToUint64(buffer[:8])
item.cookie = util.BytesToUint32(buffer[8:12])
pathSize := util.BytesToUint32(buffer[12:16])
pathBytes := make([]byte, int(pathSize))
2021-06-28 06:32:57 +00:00
n, err := io.ReadFull(br, pathBytes)
if err != nil {
fmt.Fprintf(writer, "%d,%x%08x in unexpected error: %v\n", volumeId, item.fileKey, item.cookie, err)
}
if n != int(pathSize) {
fmt.Fprintf(writer, "%d,%x%08x %d unexpected file name size %d\n", volumeId, item.fileKey, item.cookie, pathSize, n)
}
item.path = util.FullPath(string(pathBytes))
needleId := types.NeedleId(item.fileKey)
if _, found := db.Get(needleId); !found {
fmt.Fprintf(writer, "%s\n", item.path)
if applyPurging {
// defining the URL this way automatically escapes complex path names
c.httpDelete(item.path, verbose)
}
}
}
return nil
}
func (c *commandVolumeFsck) httpDelete(path util.FullPath, verbose bool) {
req, err := http.NewRequest(http.MethodDelete, "", nil)
req.URL = &url.URL{
Scheme: "http",
Host: c.env.option.FilerAddress.ToHttpAddress(),
Path: string(path),
}
if verbose {
fmt.Printf("full HTTP delete request to be sent: %v\n", req)
}
if err != nil {
fmt.Errorf("HTTP delete request error: %v\n", err)
}
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
fmt.Errorf("DELETE fetch error: %v\n", err)
}
defer resp.Body.Close()
_, err = ioutil.ReadAll(resp.Body)
if err != nil {
fmt.Errorf("DELETE response error: %v\n", err)
}
if verbose {
fmt.Println("delete response Status : ", resp.Status)
fmt.Println("delete response Headers : ", resp.Header)
}
}
2020-03-25 09:21:15 +00:00
func (c *commandVolumeFsck) oneVolumeFileIdsSubtractFilerFileIds(tempFolder string, volumeId uint32, writer io.Writer, verbose bool) (inUseCount uint64, orphanFileIds []string, orphanDataSize uint64, err error) {
db := needle_map.NewMemDb()
defer db.Close()
if err = db.LoadFromIdx(getVolumeFileIdFile(tempFolder, volumeId)); err != nil {
return
}
filerFileIdsData, err := os.ReadFile(getFilerFileIdFile(tempFolder, volumeId))
if err != nil {
return
}
dataLen := len(filerFileIdsData)
if dataLen%8 != 0 {
2020-03-25 09:21:15 +00:00
return 0, nil, 0, fmt.Errorf("filer data is corrupted")
}
for i := 0; i < len(filerFileIdsData); i += 8 {
fileKey := util.BytesToUint64(filerFileIdsData[i : i+8])
db.Delete(types.NeedleId(fileKey))
2020-03-25 05:38:33 +00:00
inUseCount++
}
2020-03-25 09:21:15 +00:00
var orphanFileCount uint64
db.AscendingVisit(func(n needle_map.NeedleValue) error {
2020-03-24 09:34:28 +00:00
// fmt.Printf("%d,%x\n", volumeId, n.Key)
orphanFileIds = append(orphanFileIds, fmt.Sprintf("%d,%s00000000", volumeId, n.Key.String()))
2020-03-25 09:21:15 +00:00
orphanFileCount++
orphanDataSize += uint64(n.Size)
return nil
})
2020-03-25 09:21:15 +00:00
if orphanFileCount > 0 {
pct := float64(orphanFileCount*100) / (float64(orphanFileCount + inUseCount))
2020-03-25 05:38:33 +00:00
fmt.Fprintf(writer, "volume:%d\tentries:%d\torphan:%d\t%.2f%%\t%dB\n",
2020-03-25 09:21:15 +00:00
volumeId, orphanFileCount+inUseCount, orphanFileCount, pct, orphanDataSize)
2020-03-24 09:34:28 +00:00
}
return
}
type VInfo struct {
server pb.ServerAddress
collection string
isEcVolume bool
isReadOnly bool
}
2021-02-22 08:28:42 +00:00
func (c *commandVolumeFsck) collectVolumeIds(commandEnv *CommandEnv, verbose bool, writer io.Writer) (volumeIdToServer map[uint32]VInfo, err error) {
2020-03-25 07:56:47 +00:00
if verbose {
fmt.Fprintf(writer, "collecting volume id and locations from master ...\n")
}
volumeIdToServer = make(map[uint32]VInfo)
2021-02-22 08:28:42 +00:00
// collect topology information
topologyInfo, _, err := collectTopologyInfo(commandEnv, 0)
if err != nil {
return
}
2021-02-22 08:28:42 +00:00
eachDataNode(topologyInfo, func(dc string, rack RackId, t *master_pb.DataNodeInfo) {
2021-02-18 04:57:08 +00:00
for _, diskInfo := range t.DiskInfos {
2021-02-16 10:47:02 +00:00
for _, vi := range diskInfo.VolumeInfos {
volumeIdToServer[vi.Id] = VInfo{
server: pb.NewServerAddressFromDataNode(t),
2021-02-16 10:47:02 +00:00
collection: vi.Collection,
isEcVolume: false,
isReadOnly: vi.ReadOnly,
2021-02-16 10:47:02 +00:00
}
}
2021-02-16 10:47:02 +00:00
for _, ecShardInfo := range diskInfo.EcShardInfos {
volumeIdToServer[ecShardInfo.Id] = VInfo{
server: pb.NewServerAddressFromDataNode(t),
2021-02-16 10:47:02 +00:00
collection: ecShardInfo.Collection,
isEcVolume: true,
isReadOnly: true,
2021-02-16 10:47:02 +00:00
}
}
}
})
2020-03-25 07:56:47 +00:00
if verbose {
fmt.Fprintf(writer, "collected %d volumes and locations.\n", len(volumeIdToServer))
}
return
}
2020-03-25 09:21:15 +00:00
func (c *commandVolumeFsck) purgeFileIdsForOneVolume(volumeId uint32, fileIds []string, writer io.Writer) (err error) {
fmt.Fprintf(writer, "purging orphan data for volume %d...\n", volumeId)
locations, found := c.env.MasterClient.GetLocations(volumeId)
if !found {
return fmt.Errorf("failed to find volume %d locations", volumeId)
}
resultChan := make(chan []*volume_server_pb.DeleteResult, len(locations))
var wg sync.WaitGroup
for _, location := range locations {
wg.Add(1)
go func(server pb.ServerAddress, fidList []string) {
2020-03-25 09:21:15 +00:00
defer wg.Done()
if deleteResults, deleteErr := operation.DeleteFilesAtOneVolumeServer(server, c.env.option.GrpcDialOption, fidList, false); deleteErr != nil {
err = deleteErr
} else if deleteResults != nil {
resultChan <- deleteResults
}
}(location.ServerAddress(), fileIds)
2020-03-25 09:21:15 +00:00
}
wg.Wait()
close(resultChan)
for results := range resultChan {
for _, result := range results {
if result.Error != "" {
fmt.Fprintf(writer, "purge error: %s\n", result.Error)
}
}
}
return
}
func getVolumeFileIdFile(tempFolder string, vid uint32) string {
return filepath.Join(tempFolder, fmt.Sprintf("%d.idx", vid))
}
func getFilerFileIdFile(tempFolder string, vid uint32) string {
return filepath.Join(tempFolder, fmt.Sprintf("%d.fid", vid))
}
func writeToFile(client volume_server_pb.VolumeServer_CopyFileClient, fileName string) error {
flags := os.O_WRONLY | os.O_CREATE | os.O_TRUNC
dst, err := os.OpenFile(fileName, flags, 0644)
if err != nil {
return nil
}
defer dst.Close()
for {
resp, receiveErr := client.Recv()
if receiveErr == io.EOF {
break
}
if receiveErr != nil {
return fmt.Errorf("receiving %s: %v", fileName, receiveErr)
}
dst.Write(resp.FileContent)
}
return nil
}