aboutsummaryrefslogtreecommitdiff
path: root/storage/cloud
diff options
context:
space:
mode:
authorLaria Carolin Chabowski <laria@laria.me>2018-01-15 07:39:49 +0100
committerLaria Carolin Chabowski <laria@laria.me>2018-01-15 07:39:49 +0100
commit62eef384330f652edd17732240ae0d82685c0914 (patch)
tree1ebf60a5baada7e5bd32c44ed0bafbe264958ca9 /storage/cloud
parentfc37a9d8e4e240a8fbf297919490c6df7e676d63 (diff)
downloadpetrific-62eef384330f652edd17732240ae0d82685c0914.tar.gz
petrific-62eef384330f652edd17732240ae0d82685c0914.tar.bz2
petrific-62eef384330f652edd17732240ae0d82685c0914.zip
Add command to restore index of a cloud storage
Diffstat (limited to 'storage/cloud')
-rw-r--r--storage/cloud/cloudstorage.go66
1 files changed, 65 insertions, 1 deletions
diff --git a/storage/cloud/cloudstorage.go b/storage/cloud/cloudstorage.go
index f93bb5c..f72fb7b 100644
--- a/storage/cloud/cloudstorage.go
+++ b/storage/cloud/cloudstorage.go
@@ -5,11 +5,14 @@ package cloud
import (
"bytes"
"code.laria.me/petrific/config"
+ "code.laria.me/petrific/logging"
"code.laria.me/petrific/objects"
"code.laria.me/petrific/storage"
"errors"
"fmt"
"math/rand"
+ "strings"
+ "time"
)
type CloudStorage interface {
@@ -92,7 +95,7 @@ func (cbos CloudBasedObjectStorage) Set(id objects.ObjectId, typ objects.ObjectT
return err
}
- // could be used to repopulate the index (not implemented yet)
+ // can be used to repopulate the index
if err := cbos.CS.Put(cbos.Prefix+"typeof/"+id.String(), []byte(typ)); err != nil {
return err
}
@@ -106,6 +109,67 @@ func (cbos CloudBasedObjectStorage) List(typ objects.ObjectType) ([]objects.Obje
return cbos.index.List(typ), nil
}
+func (cbos CloudBasedObjectStorage) retryGet(key string, tries int, retryDelay time.Duration, log *logging.Log) (p []byte, err error) {
+ for i := 0; i < tries; i++ {
+ p, err = cbos.CS.Get(key)
+ if err == nil {
+ return
+ }
+ log.Info().Printf("Failed getting %s (err=%s), retrying...", key, err)
+ time.Sleep(retryDelay)
+ }
+ return
+}
+
+func (cbos CloudBasedObjectStorage) restoreIndex(log *logging.Log) error {
+ prefix := cbos.Prefix + "typeof/"
+
+ typeof_objs, err := cbos.CS.List(prefix)
+ if err != nil {
+ return err
+ }
+
+ for _, key := range typeof_objs {
+ log.Debug().Printf("processing %s", key)
+
+ id, err := objects.ParseObjectId(key[len(prefix):])
+ if err != nil {
+ log.Error().Printf("Skip %s, can't parse id: %s", key, err)
+ continue
+ }
+
+ // At least OVHs Swift object storage apparently doesn't like being sent
+ // many small requests in a short amount of time. This retries getting
+ // an object, if the cloud storage returned an error.
+ p, err := cbos.retryGet(key, 3, 10*time.Second, log)
+ if err != nil {
+ return err
+ }
+
+ ot := objects.ObjectType(strings.TrimSpace(string(p)))
+ if !ot.IsKnown() {
+ log.Error().Printf("Skip %s, unknown object type %s", key, ot)
+ continue
+ }
+
+ cbos.index.Set(id, ot)
+ }
+
+ return nil
+}
+
+func (cbos CloudBasedObjectStorage) Subcmds() map[string]storage.StorageSubcmd {
+ return map[string]storage.StorageSubcmd{
+ "restore-index": func(args []string, log *logging.Log, conf config.Config) int {
+ if err := cbos.restoreIndex(log); err != nil {
+ log.Error().Print(err)
+ return 1
+ }
+ return 0
+ },
+ }
+}
+
func (cbos CloudBasedObjectStorage) Close() (outerr error) {
defer func() {
err := cbos.CS.Close()