diff options
| -rw-r--r-- | main.go | 1 | ||||
| -rw-r--r-- | objects/object.go | 9 | ||||
| -rw-r--r-- | storage/cloud/cloudstorage.go | 66 | ||||
| -rw-r--r-- | storage/filter/filter.go | 4 | ||||
| -rw-r--r-- | storage/local/local.go | 4 | ||||
| -rw-r--r-- | storage/memory/memory.go | 4 | ||||
| -rw-r--r-- | storage/storage.go | 5 | ||||
| -rw-r--r-- | storagecmd.go | 30 |
8 files changed, 122 insertions, 1 deletions
@@ -19,6 +19,7 @@ var subcmds = map[string]subcmd{ "list-snapshots": ListSnapshots, "restore-snapshot": RestoreSnapshot, "fsck": Fsck, + "storagecmd": StorageCmd, } func subcmdUsage(name string, usage string, flags *flag.FlagSet) func() { diff --git a/objects/object.go b/objects/object.go index f9907f2..08525a9 100644 --- a/objects/object.go +++ b/objects/object.go @@ -23,6 +23,15 @@ var AllObjectTypes = []ObjectType{ OTSnapshot, } +func (ot ObjectType) IsKnown() bool { + for _, t := range AllObjectTypes { + if t == ot { + return true + } + } + return false +} + // RawObject describes a serialized object plus it's type header. This is the content that will be saved. // It is serialized as the type (ObjectType), a space byte, the size of the payload in bytes (encoded as decimal ASCII number), a newline ('\n') character and the payload. // The encoding of the payload depends on the type. diff --git a/storage/cloud/cloudstorage.go b/storage/cloud/cloudstorage.go index f93bb5c..f72fb7b 100644 --- a/storage/cloud/cloudstorage.go +++ b/storage/cloud/cloudstorage.go @@ -5,11 +5,14 @@ package cloud import ( "bytes" "code.laria.me/petrific/config" + "code.laria.me/petrific/logging" "code.laria.me/petrific/objects" "code.laria.me/petrific/storage" "errors" "fmt" "math/rand" + "strings" + "time" ) type CloudStorage interface { @@ -92,7 +95,7 @@ func (cbos CloudBasedObjectStorage) Set(id objects.ObjectId, typ objects.ObjectT return err } - // could be used to repopulate the index (not implemented yet) + // can be used to repopulate the index if err := cbos.CS.Put(cbos.Prefix+"typeof/"+id.String(), []byte(typ)); err != nil { return err } @@ -106,6 +109,67 @@ func (cbos CloudBasedObjectStorage) List(typ objects.ObjectType) ([]objects.Obje return cbos.index.List(typ), nil } +func (cbos CloudBasedObjectStorage) retryGet(key string, tries int, retryDelay time.Duration, log *logging.Log) (p []byte, err error) { + for i := 0; i < tries; i++ { + p, err = cbos.CS.Get(key) + if err == nil { + return + } + log.Info().Printf("Failed getting %s (err=%s), retrying...", key, err) + time.Sleep(retryDelay) + } + return +} + +func (cbos CloudBasedObjectStorage) restoreIndex(log *logging.Log) error { + prefix := cbos.Prefix + "typeof/" + + typeof_objs, err := cbos.CS.List(prefix) + if err != nil { + return err + } + + for _, key := range typeof_objs { + log.Debug().Printf("processing %s", key) + + id, err := objects.ParseObjectId(key[len(prefix):]) + if err != nil { + log.Error().Printf("Skip %s, can't parse id: %s", key, err) + continue + } + + // At least OVHs Swift object storage apparently doesn't like being sent + // many small requests in a short amount of time. This retries getting + // an object, if the cloud storage returned an error. + p, err := cbos.retryGet(key, 3, 10*time.Second, log) + if err != nil { + return err + } + + ot := objects.ObjectType(strings.TrimSpace(string(p))) + if !ot.IsKnown() { + log.Error().Printf("Skip %s, unknown object type %s", key, ot) + continue + } + + cbos.index.Set(id, ot) + } + + return nil +} + +func (cbos CloudBasedObjectStorage) Subcmds() map[string]storage.StorageSubcmd { + return map[string]storage.StorageSubcmd{ + "restore-index": func(args []string, log *logging.Log, conf config.Config) int { + if err := cbos.restoreIndex(log); err != nil { + log.Error().Print(err) + return 1 + } + return 0 + }, + } +} + func (cbos CloudBasedObjectStorage) Close() (outerr error) { defer func() { err := cbos.CS.Close() diff --git a/storage/filter/filter.go b/storage/filter/filter.go index 1628c1e..2e60430 100644 --- a/storage/filter/filter.go +++ b/storage/filter/filter.go @@ -87,6 +87,10 @@ func (filt FilterStorage) List(typ objects.ObjectType) ([]objects.ObjectId, erro return filt.Base.List(typ) } +func (filt FilterStorage) Subcmds() map[string]storage.StorageSubcmd { + return filt.Base.Subcmds() +} + func (filt FilterStorage) Close() error { return filt.Base.Close() } diff --git a/storage/local/local.go b/storage/local/local.go index ae39eb5..8b48b2e 100644 --- a/storage/local/local.go +++ b/storage/local/local.go @@ -128,6 +128,10 @@ func (l LocalStorage) List(typ objects.ObjectType) ([]objects.ObjectId, error) { return l.index.List(typ), nil } +func (LocalStorage) Subcmds() map[string]storage.StorageSubcmd { + return make(map[string]storage.StorageSubcmd) +} + func (l LocalStorage) Close() error { f, err := os.Create(joinPath(l.Path, "index")) if err != nil { diff --git a/storage/memory/memory.go b/storage/memory/memory.go index dc72f70..a43414d 100644 --- a/storage/memory/memory.go +++ b/storage/memory/memory.go @@ -55,6 +55,10 @@ func (ms MemoryStorage) List(typ objects.ObjectType) ([]objects.ObjectId, error) return ms.bytype[typ], nil } +func (MemoryStorage) Subcmds() map[string]storage.StorageSubcmd { + return make(map[string]storage.StorageSubcmd) +} + func (MemoryStorage) Close() error { return nil } diff --git a/storage/storage.go b/storage/storage.go index 176e75d..eb71447 100644 --- a/storage/storage.go +++ b/storage/storage.go @@ -3,6 +3,7 @@ package storage import ( "bytes" "code.laria.me/petrific/config" + "code.laria.me/petrific/logging" "code.laria.me/petrific/objects" "errors" "fmt" @@ -14,12 +15,16 @@ var ( ObjectNotFound = errors.New("Object not found") ) +type StorageSubcmd func(args []string, log *logging.Log, conf config.Config) int + type Storage interface { Get(id objects.ObjectId) ([]byte, error) Has(id objects.ObjectId) (bool, error) Set(id objects.ObjectId, typ objects.ObjectType, raw []byte) error List(typ objects.ObjectType) ([]objects.ObjectId, error) + Subcmds() map[string]StorageSubcmd + Close() error } diff --git a/storagecmd.go b/storagecmd.go new file mode 100644 index 0000000..d2b603a --- /dev/null +++ b/storagecmd.go @@ -0,0 +1,30 @@ +package main + +import ( + "fmt" + "os" +) + +func StorageCmd(env *Env, args []string) int { + cmds := env.Store.Subcmds() + + if len(args) == 0 { + if len(cmds) == 0 { + fmt.Fprintln(os.Stderr, "No storage subcommands available") + return 2 + } + + fmt.Fprintln(os.Stderr, "Availabe storage subcommands:") + for name := range cmds { + fmt.Fprintln(os.Stderr, " "+name) + } + return 2 + } + + cmd, ok := cmds[args[0]] + if !ok { + fmt.Fprintf(os.Stderr, "Unknown storage subcommand %s\n", args[0]) + } + + return cmd(args[1:], env.Log, env.Conf) +} |