From 62eef384330f652edd17732240ae0d82685c0914 Mon Sep 17 00:00:00 2001
From: Laria Carolin Chabowski <laria@laria.me>
Date: Mon, 15 Jan 2018 07:39:49 +0100
Subject: Add command to restore index of a cloud storage

---
 storage/cloud/cloudstorage.go | 66 ++++++++++++++++++++++++++++++++++++++++++-
 storage/filter/filter.go      |  4 +++
 storage/local/local.go        |  4 +++
 storage/memory/memory.go      |  4 +++
 storage/storage.go            |  5 ++++
 5 files changed, 82 insertions(+), 1 deletion(-)

(limited to 'storage')

diff --git a/storage/cloud/cloudstorage.go b/storage/cloud/cloudstorage.go
index f93bb5c..f72fb7b 100644
--- a/storage/cloud/cloudstorage.go
+++ b/storage/cloud/cloudstorage.go
@@ -5,11 +5,14 @@ package cloud
 import (
 	"bytes"
 	"code.laria.me/petrific/config"
+	"code.laria.me/petrific/logging"
 	"code.laria.me/petrific/objects"
 	"code.laria.me/petrific/storage"
 	"errors"
 	"fmt"
 	"math/rand"
+	"strings"
+	"time"
 )
 
 type CloudStorage interface {
@@ -92,7 +95,7 @@ func (cbos CloudBasedObjectStorage) Set(id objects.ObjectId, typ objects.ObjectT
 		return err
 	}
 
-	// could be used to repopulate the index (not implemented yet)
+	// can be used to repopulate the index
 	if err := cbos.CS.Put(cbos.Prefix+"typeof/"+id.String(), []byte(typ)); err != nil {
 		return err
 	}
@@ -106,6 +109,67 @@ func (cbos CloudBasedObjectStorage) List(typ objects.ObjectType) ([]objects.Obje
 	return cbos.index.List(typ), nil
 }
 
+func (cbos CloudBasedObjectStorage) retryGet(key string, tries int, retryDelay time.Duration, log *logging.Log) (p []byte, err error) {
+	for i := 0; i < tries; i++ {
+		p, err = cbos.CS.Get(key)
+		if err == nil {
+			return
+		}
+		log.Info().Printf("Failed getting %s (err=%s), retrying...", key, err)
+		time.Sleep(retryDelay)
+	}
+	return
+}
+
+func (cbos CloudBasedObjectStorage) restoreIndex(log *logging.Log) error {
+	prefix := cbos.Prefix + "typeof/"
+
+	typeof_objs, err := cbos.CS.List(prefix)
+	if err != nil {
+		return err
+	}
+
+	for _, key := range typeof_objs {
+		log.Debug().Printf("processing %s", key)
+
+		id, err := objects.ParseObjectId(key[len(prefix):])
+		if err != nil {
+			log.Error().Printf("Skip %s, can't parse id: %s", key, err)
+			continue
+		}
+
+		// At least OVHs Swift object storage apparently doesn't like being sent
+		// many small requests in a short amount of time. This retries getting
+		// an object, if the cloud storage returned an error.
+		p, err := cbos.retryGet(key, 3, 10*time.Second, log)
+		if err != nil {
+			return err
+		}
+
+		ot := objects.ObjectType(strings.TrimSpace(string(p)))
+		if !ot.IsKnown() {
+			log.Error().Printf("Skip %s, unknown object type %s", key, ot)
+			continue
+		}
+
+		cbos.index.Set(id, ot)
+	}
+
+	return nil
+}
+
+func (cbos CloudBasedObjectStorage) Subcmds() map[string]storage.StorageSubcmd {
+	return map[string]storage.StorageSubcmd{
+		"restore-index": func(args []string, log *logging.Log, conf config.Config) int {
+			if err := cbos.restoreIndex(log); err != nil {
+				log.Error().Print(err)
+				return 1
+			}
+			return 0
+		},
+	}
+}
+
 func (cbos CloudBasedObjectStorage) Close() (outerr error) {
 	defer func() {
 		err := cbos.CS.Close()
diff --git a/storage/filter/filter.go b/storage/filter/filter.go
index 1628c1e..2e60430 100644
--- a/storage/filter/filter.go
+++ b/storage/filter/filter.go
@@ -87,6 +87,10 @@ func (filt FilterStorage) List(typ objects.ObjectType) ([]objects.ObjectId, erro
 	return filt.Base.List(typ)
 }
 
+func (filt FilterStorage) Subcmds() map[string]storage.StorageSubcmd {
+	return filt.Base.Subcmds()
+}
+
 func (filt FilterStorage) Close() error {
 	return filt.Base.Close()
 }
diff --git a/storage/local/local.go b/storage/local/local.go
index ae39eb5..8b48b2e 100644
--- a/storage/local/local.go
+++ b/storage/local/local.go
@@ -128,6 +128,10 @@ func (l LocalStorage) List(typ objects.ObjectType) ([]objects.ObjectId, error) {
 	return l.index.List(typ), nil
 }
 
+func (LocalStorage) Subcmds() map[string]storage.StorageSubcmd {
+	return make(map[string]storage.StorageSubcmd)
+}
+
 func (l LocalStorage) Close() error {
 	f, err := os.Create(joinPath(l.Path, "index"))
 	if err != nil {
diff --git a/storage/memory/memory.go b/storage/memory/memory.go
index dc72f70..a43414d 100644
--- a/storage/memory/memory.go
+++ b/storage/memory/memory.go
@@ -55,6 +55,10 @@ func (ms MemoryStorage) List(typ objects.ObjectType) ([]objects.ObjectId, error)
 	return ms.bytype[typ], nil
 }
 
+func (MemoryStorage) Subcmds() map[string]storage.StorageSubcmd {
+	return make(map[string]storage.StorageSubcmd)
+}
+
 func (MemoryStorage) Close() error {
 	return nil
 }
diff --git a/storage/storage.go b/storage/storage.go
index 176e75d..eb71447 100644
--- a/storage/storage.go
+++ b/storage/storage.go
@@ -3,6 +3,7 @@ package storage
 import (
 	"bytes"
 	"code.laria.me/petrific/config"
+	"code.laria.me/petrific/logging"
 	"code.laria.me/petrific/objects"
 	"errors"
 	"fmt"
@@ -14,12 +15,16 @@ var (
 	ObjectNotFound = errors.New("Object not found")
 )
 
+type StorageSubcmd func(args []string, log *logging.Log, conf config.Config) int
+
 type Storage interface {
 	Get(id objects.ObjectId) ([]byte, error)
 	Has(id objects.ObjectId) (bool, error)
 	Set(id objects.ObjectId, typ objects.ObjectType, raw []byte) error
 	List(typ objects.ObjectType) ([]objects.ObjectId, error)
 
+	Subcmds() map[string]StorageSubcmd
+
 	Close() error
 }
 
-- 
cgit v1.2.3-54-g00ecf