aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--main.go1
-rw-r--r--objects/object.go9
-rw-r--r--storage/cloud/cloudstorage.go66
-rw-r--r--storage/filter/filter.go4
-rw-r--r--storage/local/local.go4
-rw-r--r--storage/memory/memory.go4
-rw-r--r--storage/storage.go5
-rw-r--r--storagecmd.go30
8 files changed, 122 insertions, 1 deletions
diff --git a/main.go b/main.go
index 21473c5..bc4ef30 100644
--- a/main.go
+++ b/main.go
@@ -19,6 +19,7 @@ var subcmds = map[string]subcmd{
"list-snapshots": ListSnapshots,
"restore-snapshot": RestoreSnapshot,
"fsck": Fsck,
+ "storagecmd": StorageCmd,
}
func subcmdUsage(name string, usage string, flags *flag.FlagSet) func() {
diff --git a/objects/object.go b/objects/object.go
index f9907f2..08525a9 100644
--- a/objects/object.go
+++ b/objects/object.go
@@ -23,6 +23,15 @@ var AllObjectTypes = []ObjectType{
OTSnapshot,
}
+func (ot ObjectType) IsKnown() bool {
+ for _, t := range AllObjectTypes {
+ if t == ot {
+ return true
+ }
+ }
+ return false
+}
+
// RawObject describes a serialized object plus it's type header. This is the content that will be saved.
// It is serialized as the type (ObjectType), a space byte, the size of the payload in bytes (encoded as decimal ASCII number), a newline ('\n') character and the payload.
// The encoding of the payload depends on the type.
diff --git a/storage/cloud/cloudstorage.go b/storage/cloud/cloudstorage.go
index f93bb5c..f72fb7b 100644
--- a/storage/cloud/cloudstorage.go
+++ b/storage/cloud/cloudstorage.go
@@ -5,11 +5,14 @@ package cloud
import (
"bytes"
"code.laria.me/petrific/config"
+ "code.laria.me/petrific/logging"
"code.laria.me/petrific/objects"
"code.laria.me/petrific/storage"
"errors"
"fmt"
"math/rand"
+ "strings"
+ "time"
)
type CloudStorage interface {
@@ -92,7 +95,7 @@ func (cbos CloudBasedObjectStorage) Set(id objects.ObjectId, typ objects.ObjectT
return err
}
- // could be used to repopulate the index (not implemented yet)
+ // can be used to repopulate the index
if err := cbos.CS.Put(cbos.Prefix+"typeof/"+id.String(), []byte(typ)); err != nil {
return err
}
@@ -106,6 +109,67 @@ func (cbos CloudBasedObjectStorage) List(typ objects.ObjectType) ([]objects.Obje
return cbos.index.List(typ), nil
}
+func (cbos CloudBasedObjectStorage) retryGet(key string, tries int, retryDelay time.Duration, log *logging.Log) (p []byte, err error) {
+ for i := 0; i < tries; i++ {
+ p, err = cbos.CS.Get(key)
+ if err == nil {
+ return
+ }
+ log.Info().Printf("Failed getting %s (err=%s), retrying...", key, err)
+ time.Sleep(retryDelay)
+ }
+ return
+}
+
+func (cbos CloudBasedObjectStorage) restoreIndex(log *logging.Log) error {
+ prefix := cbos.Prefix + "typeof/"
+
+ typeof_objs, err := cbos.CS.List(prefix)
+ if err != nil {
+ return err
+ }
+
+ for _, key := range typeof_objs {
+ log.Debug().Printf("processing %s", key)
+
+ id, err := objects.ParseObjectId(key[len(prefix):])
+ if err != nil {
+ log.Error().Printf("Skip %s, can't parse id: %s", key, err)
+ continue
+ }
+
+ // At least OVHs Swift object storage apparently doesn't like being sent
+ // many small requests in a short amount of time. This retries getting
+ // an object, if the cloud storage returned an error.
+ p, err := cbos.retryGet(key, 3, 10*time.Second, log)
+ if err != nil {
+ return err
+ }
+
+ ot := objects.ObjectType(strings.TrimSpace(string(p)))
+ if !ot.IsKnown() {
+ log.Error().Printf("Skip %s, unknown object type %s", key, ot)
+ continue
+ }
+
+ cbos.index.Set(id, ot)
+ }
+
+ return nil
+}
+
+func (cbos CloudBasedObjectStorage) Subcmds() map[string]storage.StorageSubcmd {
+ return map[string]storage.StorageSubcmd{
+ "restore-index": func(args []string, log *logging.Log, conf config.Config) int {
+ if err := cbos.restoreIndex(log); err != nil {
+ log.Error().Print(err)
+ return 1
+ }
+ return 0
+ },
+ }
+}
+
func (cbos CloudBasedObjectStorage) Close() (outerr error) {
defer func() {
err := cbos.CS.Close()
diff --git a/storage/filter/filter.go b/storage/filter/filter.go
index 1628c1e..2e60430 100644
--- a/storage/filter/filter.go
+++ b/storage/filter/filter.go
@@ -87,6 +87,10 @@ func (filt FilterStorage) List(typ objects.ObjectType) ([]objects.ObjectId, erro
return filt.Base.List(typ)
}
+func (filt FilterStorage) Subcmds() map[string]storage.StorageSubcmd {
+ return filt.Base.Subcmds()
+}
+
func (filt FilterStorage) Close() error {
return filt.Base.Close()
}
diff --git a/storage/local/local.go b/storage/local/local.go
index ae39eb5..8b48b2e 100644
--- a/storage/local/local.go
+++ b/storage/local/local.go
@@ -128,6 +128,10 @@ func (l LocalStorage) List(typ objects.ObjectType) ([]objects.ObjectId, error) {
return l.index.List(typ), nil
}
+func (LocalStorage) Subcmds() map[string]storage.StorageSubcmd {
+ return make(map[string]storage.StorageSubcmd)
+}
+
func (l LocalStorage) Close() error {
f, err := os.Create(joinPath(l.Path, "index"))
if err != nil {
diff --git a/storage/memory/memory.go b/storage/memory/memory.go
index dc72f70..a43414d 100644
--- a/storage/memory/memory.go
+++ b/storage/memory/memory.go
@@ -55,6 +55,10 @@ func (ms MemoryStorage) List(typ objects.ObjectType) ([]objects.ObjectId, error)
return ms.bytype[typ], nil
}
+func (MemoryStorage) Subcmds() map[string]storage.StorageSubcmd {
+ return make(map[string]storage.StorageSubcmd)
+}
+
func (MemoryStorage) Close() error {
return nil
}
diff --git a/storage/storage.go b/storage/storage.go
index 176e75d..eb71447 100644
--- a/storage/storage.go
+++ b/storage/storage.go
@@ -3,6 +3,7 @@ package storage
import (
"bytes"
"code.laria.me/petrific/config"
+ "code.laria.me/petrific/logging"
"code.laria.me/petrific/objects"
"errors"
"fmt"
@@ -14,12 +15,16 @@ var (
ObjectNotFound = errors.New("Object not found")
)
+type StorageSubcmd func(args []string, log *logging.Log, conf config.Config) int
+
type Storage interface {
Get(id objects.ObjectId) ([]byte, error)
Has(id objects.ObjectId) (bool, error)
Set(id objects.ObjectId, typ objects.ObjectType, raw []byte) error
List(typ objects.ObjectType) ([]objects.ObjectId, error)
+ Subcmds() map[string]StorageSubcmd
+
Close() error
}
diff --git a/storagecmd.go b/storagecmd.go
new file mode 100644
index 0000000..d2b603a
--- /dev/null
+++ b/storagecmd.go
@@ -0,0 +1,30 @@
+package main
+
+import (
+ "fmt"
+ "os"
+)
+
+func StorageCmd(env *Env, args []string) int {
+ cmds := env.Store.Subcmds()
+
+ if len(args) == 0 {
+ if len(cmds) == 0 {
+ fmt.Fprintln(os.Stderr, "No storage subcommands available")
+ return 2
+ }
+
+ fmt.Fprintln(os.Stderr, "Availabe storage subcommands:")
+ for name := range cmds {
+ fmt.Fprintln(os.Stderr, " "+name)
+ }
+ return 2
+ }
+
+ cmd, ok := cmds[args[0]]
+ if !ok {
+ fmt.Fprintf(os.Stderr, "Unknown storage subcommand %s\n", args[0])
+ }
+
+ return cmd(args[1:], env.Log, env.Conf)
+}