From 62eef384330f652edd17732240ae0d82685c0914 Mon Sep 17 00:00:00 2001
From: Laria Carolin Chabowski <laria@laria.me>
Date: Mon, 15 Jan 2018 07:39:49 +0100
Subject: Add command to restore index of a cloud storage

---
 storage/cloud/cloudstorage.go | 66 ++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 65 insertions(+), 1 deletion(-)

(limited to 'storage/cloud/cloudstorage.go')

diff --git a/storage/cloud/cloudstorage.go b/storage/cloud/cloudstorage.go
index f93bb5c..f72fb7b 100644
--- a/storage/cloud/cloudstorage.go
+++ b/storage/cloud/cloudstorage.go
@@ -5,11 +5,14 @@ package cloud
 import (
 	"bytes"
 	"code.laria.me/petrific/config"
+	"code.laria.me/petrific/logging"
 	"code.laria.me/petrific/objects"
 	"code.laria.me/petrific/storage"
 	"errors"
 	"fmt"
 	"math/rand"
+	"strings"
+	"time"
 )
 
 type CloudStorage interface {
@@ -92,7 +95,7 @@ func (cbos CloudBasedObjectStorage) Set(id objects.ObjectId, typ objects.ObjectT
 		return err
 	}
 
-	// could be used to repopulate the index (not implemented yet)
+	// can be used to repopulate the index
 	if err := cbos.CS.Put(cbos.Prefix+"typeof/"+id.String(), []byte(typ)); err != nil {
 		return err
 	}
@@ -106,6 +109,67 @@ func (cbos CloudBasedObjectStorage) List(typ objects.ObjectType) ([]objects.Obje
 	return cbos.index.List(typ), nil
 }
 
+func (cbos CloudBasedObjectStorage) retryGet(key string, tries int, retryDelay time.Duration, log *logging.Log) (p []byte, err error) {
+	for i := 0; i < tries; i++ {
+		p, err = cbos.CS.Get(key)
+		if err == nil {
+			return
+		}
+		log.Info().Printf("Failed getting %s (err=%s), retrying...", key, err)
+		time.Sleep(retryDelay)
+	}
+	return
+}
+
+func (cbos CloudBasedObjectStorage) restoreIndex(log *logging.Log) error {
+	prefix := cbos.Prefix + "typeof/"
+
+	typeof_objs, err := cbos.CS.List(prefix)
+	if err != nil {
+		return err
+	}
+
+	for _, key := range typeof_objs {
+		log.Debug().Printf("processing %s", key)
+
+		id, err := objects.ParseObjectId(key[len(prefix):])
+		if err != nil {
+			log.Error().Printf("Skip %s, can't parse id: %s", key, err)
+			continue
+		}
+
+		// At least OVHs Swift object storage apparently doesn't like being sent
+		// many small requests in a short amount of time. This retries getting
+		// an object, if the cloud storage returned an error.
+		p, err := cbos.retryGet(key, 3, 10*time.Second, log)
+		if err != nil {
+			return err
+		}
+
+		ot := objects.ObjectType(strings.TrimSpace(string(p)))
+		if !ot.IsKnown() {
+			log.Error().Printf("Skip %s, unknown object type %s", key, ot)
+			continue
+		}
+
+		cbos.index.Set(id, ot)
+	}
+
+	return nil
+}
+
+func (cbos CloudBasedObjectStorage) Subcmds() map[string]storage.StorageSubcmd {
+	return map[string]storage.StorageSubcmd{
+		"restore-index": func(args []string, log *logging.Log, conf config.Config) int {
+			if err := cbos.restoreIndex(log); err != nil {
+				log.Error().Print(err)
+				return 1
+			}
+			return 0
+		},
+	}
+}
+
 func (cbos CloudBasedObjectStorage) Close() (outerr error) {
 	defer func() {
 		err := cbos.CS.Close()
-- 
cgit v1.2.3-54-g00ecf