Add command to restore index of a cloud storage

8 files changed, 122 insertions, 1 deletions

diff --git a/main.go b/main.go
index 21473c5..bc4ef30 100644
--- a/main.go
+++ b/main.go
@@ -19,6 +19,7 @@ var subcmds = map[string]subcmd{
 	"list-snapshots":   ListSnapshots,
 	"restore-snapshot": RestoreSnapshot,
 	"fsck":             Fsck,
+	"storagecmd":       StorageCmd,
 }
 
 func subcmdUsage(name string, usage string, flags *flag.FlagSet) func() {
diff --git a/objects/object.go b/objects/object.go
index f9907f2..08525a9 100644
--- a/objects/object.go
+++ b/objects/object.go
@@ -23,6 +23,15 @@ var AllObjectTypes = []ObjectType{
 	OTSnapshot,
 }
 
+func (ot ObjectType) IsKnown() bool {
+	for _, t := range AllObjectTypes {
+		if t == ot {
+			return true
+		}
+	}
+	return false
+}
+
 // RawObject describes a serialized object plus it's type header. This is the content that will be saved.
 // It is serialized as the type (ObjectType), a space byte, the size of the payload in bytes (encoded as decimal ASCII number), a newline ('\n') character and the payload.
 // The encoding of the payload depends on the type.
diff --git a/storage/cloud/cloudstorage.go b/storage/cloud/cloudstorage.go
index f93bb5c..f72fb7b 100644
--- a/storage/cloud/cloudstorage.go
+++ b/storage/cloud/cloudstorage.go
@@ -5,11 +5,14 @@ package cloud
 import (
 	"bytes"
 	"code.laria.me/petrific/config"
+	"code.laria.me/petrific/logging"
 	"code.laria.me/petrific/objects"
 	"code.laria.me/petrific/storage"
 	"errors"
 	"fmt"
 	"math/rand"
+	"strings"
+	"time"
 )
 
 type CloudStorage interface {
@@ -92,7 +95,7 @@ func (cbos CloudBasedObjectStorage) Set(id objects.ObjectId, typ objects.ObjectT
 		return err
 	}
 
-	// could be used to repopulate the index (not implemented yet)
+	// can be used to repopulate the index
 	if err := cbos.CS.Put(cbos.Prefix+"typeof/"+id.String(), []byte(typ)); err != nil {
 		return err
 	}
@@ -106,6 +109,67 @@ func (cbos CloudBasedObjectStorage) List(typ objects.ObjectType) ([]objects.Obje
 	return cbos.index.List(typ), nil
 }
 
+func (cbos CloudBasedObjectStorage) retryGet(key string, tries int, retryDelay time.Duration, log *logging.Log) (p []byte, err error) {
+	for i := 0; i < tries; i++ {
+		p, err = cbos.CS.Get(key)
+		if err == nil {
+			return
+		}
+		log.Info().Printf("Failed getting %s (err=%s), retrying...", key, err)
+		time.Sleep(retryDelay)
+	}
+	return
+}
+
+func (cbos CloudBasedObjectStorage) restoreIndex(log *logging.Log) error {
+	prefix := cbos.Prefix + "typeof/"
+
+	typeof_objs, err := cbos.CS.List(prefix)
+	if err != nil {
+		return err
+	}
+
+	for _, key := range typeof_objs {
+		log.Debug().Printf("processing %s", key)
+
+		id, err := objects.ParseObjectId(key[len(prefix):])
+		if err != nil {
+			log.Error().Printf("Skip %s, can't parse id: %s", key, err)
+			continue
+		}
+
+		// At least OVHs Swift object storage apparently doesn't like being sent
+		// many small requests in a short amount of time. This retries getting
+		// an object, if the cloud storage returned an error.
+		p, err := cbos.retryGet(key, 3, 10*time.Second, log)
+		if err != nil {
+			return err
+		}
+
+		ot := objects.ObjectType(strings.TrimSpace(string(p)))
+		if !ot.IsKnown() {
+			log.Error().Printf("Skip %s, unknown object type %s", key, ot)
+			continue
+		}
+
+		cbos.index.Set(id, ot)
+	}
+
+	return nil
+}
+
+func (cbos CloudBasedObjectStorage) Subcmds() map[string]storage.StorageSubcmd {
+	return map[string]storage.StorageSubcmd{
+		"restore-index": func(args []string, log *logging.Log, conf config.Config) int {
+			if err := cbos.restoreIndex(log); err != nil {
+				log.Error().Print(err)
+				return 1
+			}
+			return 0
+		},
+	}
+}
+
 func (cbos CloudBasedObjectStorage) Close() (outerr error) {
 	defer func() {
 		err := cbos.CS.Close()
diff --git a/storage/filter/filter.go b/storage/filter/filter.go
index 1628c1e..2e60430 100644
--- a/storage/filter/filter.go
+++ b/storage/filter/filter.go
@@ -87,6 +87,10 @@ func (filt FilterStorage) List(typ objects.ObjectType) ([]objects.ObjectId, erro
 	return filt.Base.List(typ)
 }
 
+func (filt FilterStorage) Subcmds() map[string]storage.StorageSubcmd {
+	return filt.Base.Subcmds()
+}
+
 func (filt FilterStorage) Close() error {
 	return filt.Base.Close()
 }
diff --git a/storage/local/local.go b/storage/local/local.go
index ae39eb5..8b48b2e 100644
--- a/storage/local/local.go
+++ b/storage/local/local.go
@@ -128,6 +128,10 @@ func (l LocalStorage) List(typ objects.ObjectType) ([]objects.ObjectId, error) {
 	return l.index.List(typ), nil
 }
 
+func (LocalStorage) Subcmds() map[string]storage.StorageSubcmd {
+	return make(map[string]storage.StorageSubcmd)
+}
+
 func (l LocalStorage) Close() error {
 	f, err := os.Create(joinPath(l.Path, "index"))
 	if err != nil {
diff --git a/storage/memory/memory.go b/storage/memory/memory.go
index dc72f70..a43414d 100644
--- a/storage/memory/memory.go
+++ b/storage/memory/memory.go
@@ -55,6 +55,10 @@ func (ms MemoryStorage) List(typ objects.ObjectType) ([]objects.ObjectId, error)
 	return ms.bytype[typ], nil
 }
 
+func (MemoryStorage) Subcmds() map[string]storage.StorageSubcmd {
+	return make(map[string]storage.StorageSubcmd)
+}
+
 func (MemoryStorage) Close() error {
 	return nil
 }
diff --git a/storage/storage.go b/storage/storage.go
index 176e75d..eb71447 100644
--- a/storage/storage.go
+++ b/storage/storage.go
@@ -3,6 +3,7 @@ package storage
 import (
 	"bytes"
 	"code.laria.me/petrific/config"
+	"code.laria.me/petrific/logging"
 	"code.laria.me/petrific/objects"
 	"errors"
 	"fmt"
@@ -14,12 +15,16 @@ var (
 	ObjectNotFound = errors.New("Object not found")
 )
 
+type StorageSubcmd func(args []string, log *logging.Log, conf config.Config) int
+
 type Storage interface {
 	Get(id objects.ObjectId) ([]byte, error)
 	Has(id objects.ObjectId) (bool, error)
 	Set(id objects.ObjectId, typ objects.ObjectType, raw []byte) error
 	List(typ objects.ObjectType) ([]objects.ObjectId, error)
 
+	Subcmds() map[string]StorageSubcmd
+
 	Close() error
 }
 
diff --git a/storagecmd.go b/storagecmd.go
new file mode 100644
index 0000000..d2b603a
--- /dev/null
+++ b/storagecmd.go
@@ -0,0 +1,30 @@
+package main
+
+import (
+	"fmt"
+	"os"
+)
+
+func StorageCmd(env *Env, args []string) int {
+	cmds := env.Store.Subcmds()
+
+	if len(args) == 0 {
+		if len(cmds) == 0 {
+			fmt.Fprintln(os.Stderr, "No storage subcommands available")
+			return 2
+		}
+
+		fmt.Fprintln(os.Stderr, "Availabe storage subcommands:")
+		for name := range cmds {
+			fmt.Fprintln(os.Stderr, "  "+name)
+		}
+		return 2
+	}
+
+	cmd, ok := cmds[args[0]]
+	if !ok {
+		fmt.Fprintf(os.Stderr, "Unknown storage subcommand %s\n", args[0])
+	}
+
+	return cmd(args[1:], env.Log, env.Conf)
+}