From e5a0787653e78178e3d1357127c9d4f954bbeea6 Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Fri, 22 May 2020 10:54:42 -0700
Subject: [PATCH] filer: leveldb store add auto recovery, report correct error

fix https://github.com/chrislusf/seaweedfs/issues/1326
---
 unmaintained/compact_leveldb/compact_leveldb.go |  4 ++++
 weed/filer2/leveldb/leveldb_store.go            |  4 ++++
 weed/filer2/leveldb2/leveldb2_store.go          |  6 +++++-
 weed/storage/needle_map_leveldb.go              | 10 ++++++++--
 4 files changed, 21 insertions(+), 3 deletions(-)

diff --git a/unmaintained/compact_leveldb/compact_leveldb.go b/unmaintained/compact_leveldb/compact_leveldb.go
index 317356c3f..9be5697de 100644
--- a/unmaintained/compact_leveldb/compact_leveldb.go
+++ b/unmaintained/compact_leveldb/compact_leveldb.go
@@ -5,6 +5,7 @@ import (
 	"log"
 
 	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/errors"
 	"github.com/syndtr/goleveldb/leveldb/opt"
 	"github.com/syndtr/goleveldb/leveldb/util"
 )
@@ -25,6 +26,9 @@ func main() {
 	}
 
 	db, err := leveldb.OpenFile(*dir, opts)
+	if errors.IsCorrupted(err) {
+		db, err = leveldb.RecoverFile(*dir, opts)
+	}
 	if err != nil {
 		log.Fatal(err)
 	}
diff --git a/weed/filer2/leveldb/leveldb_store.go b/weed/filer2/leveldb/leveldb_store.go
index f8e56d93c..675b6e19d 100644
--- a/weed/filer2/leveldb/leveldb_store.go
+++ b/weed/filer2/leveldb/leveldb_store.go
@@ -6,6 +6,7 @@ import (
 	"fmt"
 
 	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/errors"
 	"github.com/syndtr/goleveldb/leveldb/opt"
 	leveldb_util "github.com/syndtr/goleveldb/leveldb/util"
 
@@ -49,6 +50,9 @@ func (store *LevelDBStore) initialize(dir string) (err error) {
 	}
 
 	if store.db, err = leveldb.OpenFile(dir, opts); err != nil {
+		if errors.IsCorrupted(err) {
+			store.db, err = leveldb.RecoverFile(dir, opts)
+		}
 		glog.Infof("filer store open dir %s: %v", dir, err)
 		return
 	}
diff --git a/weed/filer2/leveldb2/leveldb2_store.go b/weed/filer2/leveldb2/leveldb2_store.go
index 61fd2e9e6..c907e8746 100644
--- a/weed/filer2/leveldb2/leveldb2_store.go
+++ b/weed/filer2/leveldb2/leveldb2_store.go
@@ -9,6 +9,7 @@ import (
 	"os"
 
 	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/errors"
 	"github.com/syndtr/goleveldb/leveldb/opt"
 	leveldb_util "github.com/syndtr/goleveldb/leveldb/util"
 
@@ -52,9 +53,12 @@ func (store *LevelDB2Store) initialize(dir string, dbCount int) (err error) {
 		dbFolder := fmt.Sprintf("%s/%02d", dir, d)
 		os.MkdirAll(dbFolder, 0755)
 		db, dbErr := leveldb.OpenFile(dbFolder, opts)
+		if errors.IsCorrupted(dbErr) {
+			db, dbErr = leveldb.RecoverFile(dbFolder, opts)
+		}
 		if dbErr != nil {
 			glog.Errorf("filer store open dir %s: %v", dbFolder, dbErr)
-			return
+			return dbErr
 		}
 		store.dbs = append(store.dbs, db)
 	}
diff --git a/weed/storage/needle_map_leveldb.go b/weed/storage/needle_map_leveldb.go
index 3bb258559..ee223425e 100644
--- a/weed/storage/needle_map_leveldb.go
+++ b/weed/storage/needle_map_leveldb.go
@@ -5,14 +5,17 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/chrislusf/seaweedfs/weed/storage/idx"
+	"github.com/syndtr/goleveldb/leveldb/errors"
 	"github.com/syndtr/goleveldb/leveldb/opt"
 
+	"github.com/chrislusf/seaweedfs/weed/storage/idx"
+
+	"github.com/syndtr/goleveldb/leveldb"
+
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/storage/needle_map"
 	. "github.com/chrislusf/seaweedfs/weed/storage/types"
 	"github.com/chrislusf/seaweedfs/weed/util"
-	"github.com/syndtr/goleveldb/leveldb"
 )
 
 type LevelDbNeedleMap struct {
@@ -32,6 +35,9 @@ func NewLevelDbNeedleMap(dbFileName string, indexFile *os.File, opts *opt.Option
 	glog.V(1).Infof("Opening %s...", dbFileName)
 
 	if m.db, err = leveldb.OpenFile(dbFileName, opts); err != nil {
+		if errors.IsCorrupted(err) {
+			m.db, err = leveldb.RecoverFile(dbFileName, opts)
+		}
 		return
 	}
 	glog.V(1).Infof("Loading %s...", indexFile.Name())