Add filter storage method

Also remove de/encryption in cloud storage, can be provided with a
filter storage

7 files changed, 213 insertions, 65 deletions

diff --git a/gpg/gpg.go b/gpg/gpg.go
index 1ab71ec..cb62f32 100644
--- a/gpg/gpg.go
+++ b/gpg/gpg.go
@@ -36,24 +36,3 @@ func (Verifyer) Verify(b []byte) error {
 	cmd.Stdin = bytes.NewReader(b)
 	return cmd.Run()
 }
-
-type Encrypter struct {
-	Key string
-}
-
-func (e Encrypter) Encrypt(b []byte) ([]byte, error) {
-	cmd := exec.Command("gpg", "--encrypt", "--recipient", e.Key)
-	return filter(cmd, b)
-}
-
-type Decrypter struct{}
-
-func (Decrypter) Decrypt(b []byte) ([]byte, error) {
-	cmd := exec.Command("gpg", "--decrypt")
-	return filter(cmd, b)
-}
-
-type Crypter struct {
-	Encrypter
-	Decrypter
-}
diff --git a/storage/cloud/cloudstorage.go b/storage/cloud/cloudstorage.go
index 261a034..f93bb5c 100644
--- a/storage/cloud/cloudstorage.go
+++ b/storage/cloud/cloudstorage.go
@@ -5,7 +5,6 @@ package cloud
 import (
 	"bytes"
 	"code.laria.me/petrific/config"
-	"code.laria.me/petrific/gpg"
 	"code.laria.me/petrific/objects"
 	"code.laria.me/petrific/storage"
 	"errors"
@@ -27,22 +26,9 @@ var (
 	NotFoundErr = errors.New("Object not found") // Cloud object could not be found
 )
 
-// Crypter provides de-/encrypting facilities for CloudBasedObjectStorage
-type Crypter interface {
-	Encrypt([]byte) ([]byte, error)
-	Decrypt([]byte) ([]byte, error)
-}
-
-// NopCrypter implements Crypter by not de/-encrypting at all
-type NopCrypter struct{}
-
-func (NopCrypter) Encrypt(in []byte) ([]byte, error) { return in, nil }
-func (NopCrypter) Decrypt(in []byte) ([]byte, error) { return in, nil }
-
 type CloudBasedObjectStorage struct {
-	CS      CloudStorage
-	Prefix  string
-	Crypter Crypter
+	CS     CloudStorage
+	Prefix string
 
 	index storage.Index
 }
@@ -94,24 +80,14 @@ func (cbos *CloudBasedObjectStorage) Init() error {
 }
 
 func (cbos CloudBasedObjectStorage) Get(id objects.ObjectId) ([]byte, error) {
-	b, err := cbos.CS.Get(cbos.objidToKey(id))
-	if err != nil {
-		return []byte{}, err
-	}
-
-	return cbos.Crypter.Decrypt(b)
+	return cbos.CS.Get(cbos.objidToKey(id))
 }
 
 func (cbos CloudBasedObjectStorage) Has(id objects.ObjectId) (bool, error) {
 	return cbos.CS.Has(cbos.objidToKey(id))
 }
 
-func (cbos CloudBasedObjectStorage) Set(id objects.ObjectId, typ objects.ObjectType, raw []byte) error {
-	b, err := cbos.Crypter.Encrypt(raw)
-	if err != nil {
-		return err
-	}
-
+func (cbos CloudBasedObjectStorage) Set(id objects.ObjectId, typ objects.ObjectType, b []byte) error {
 	if err := cbos.CS.Put(cbos.objidToKey(id), b); err != nil {
 		return err
 	}
@@ -159,8 +135,7 @@ func cloudStorageCreator(cloudCreator cloudObjectStorageCreator) storage.CreateS
 		var cbos CloudBasedObjectStorage
 
 		var storageconf struct {
-			Prefix     string `toml:"prefix,omitempty"`
-			EncryptFor string `toml:"encrypt_for,omitempty"`
+			Prefix string `toml:"prefix,omitempty"`
 		}
 
 		if err := conf.GetStorageConfData(name, &storageconf); err != nil {
@@ -169,13 +144,6 @@ func cloudStorageCreator(cloudCreator cloudObjectStorageCreator) storage.CreateS
 
 		cbos.Prefix = storageconf.Prefix
 
-		if storageconf.EncryptFor != "" {
-			cbos.Crypter = gpg.Crypter{
-				gpg.Encrypter{Key: storageconf.EncryptFor},
-				gpg.Decrypter{},
-			}
-		}
-
 		var err error
 		if cbos.CS, err = cloudCreator(conf, name); err != nil {
 			return nil, err
diff --git a/storage/filter/filter.go b/storage/filter/filter.go
new file mode 100644
index 0000000..52ecbfd
--- /dev/null
+++ b/storage/filter/filter.go
@@ -0,0 +1,77 @@
+package filter
+
+import (
+	"bytes"
+	"code.laria.me/petrific/objects"
+	"code.laria.me/petrific/storage"
+	"errors"
+	"os/exec"
+)
+
+type Filter interface {
+	Transform([]byte) ([]byte, error)
+}
+
+type PipeFilter []string
+
+var emptyCommand = errors.New("Need at least one argument for pipeFilter")
+
+func (pf PipeFilter) Transform(b []byte) ([]byte, error) {
+	if len(pf) == 0 {
+		return []byte{}, emptyCommand
+	}
+
+	cmd := exec.Command(pf[0], pf[1:]...)
+
+	buf := new(bytes.Buffer)
+	cmd.Stdout = buf
+	cmd.Stdin = bytes.NewReader(b)
+
+	if err := cmd.Run(); err != nil {
+		return []byte{}, err
+	}
+
+	return buf.Bytes(), nil
+}
+
+type FilterStorage struct {
+	Base           storage.Storage
+	Decode, Encode Filter
+}
+
+func (filt FilterStorage) Get(id objects.ObjectId) ([]byte, error) {
+	data, err := filt.Base.Get(id)
+	if err != nil {
+		return data, err
+	}
+
+	if filt.Decode == nil {
+		return data, nil
+	}
+
+	return filt.Decode.Transform(data)
+}
+
+func (filt FilterStorage) Has(id objects.ObjectId) (bool, error) {
+	return filt.Base.Has(id)
+}
+
+func (filt FilterStorage) Set(id objects.ObjectId, typ objects.ObjectType, raw []byte) error {
+	if filt.Encode != nil {
+		var err error
+		raw, err = filt.Encode.Transform(raw)
+		if err != nil {
+			return err
+		}
+	}
+
+	return filt.Base.Set(id, typ, raw)
+}
+
+func (filt FilterStorage) List(typ objects.ObjectType) ([]objects.ObjectId, error) {
+	return filt.Base.List(typ)
+}
+
+func (filt FilterStorage) Close() error {
+	return filt.Base.Close()
+}
diff --git a/storage/filter/filter_test.go b/storage/filter/filter_test.go
new file mode 100644
index 0000000..78d4e7f
--- /dev/null
+++ b/storage/filter/filter_test.go
@@ -0,0 +1,76 @@
+package filter
+
+import (
+	"bytes"
+	"code.laria.me/petrific/backup"
+	"code.laria.me/petrific/storage/memory"
+	"testing"
+)
+
+type XorFilter byte
+
+func (xf XorFilter) Transform(b []byte) ([]byte, error) {
+	for i := range b {
+		b[i] ^= byte(xf)
+	}
+	return b, nil
+}
+
+func genTestData(size int) []byte {
+	data := make([]byte, size)
+	for i := range data {
+		data[i] = byte(i & 0xff)
+	}
+	return data
+}
+
+func TestXorReverse(t *testing.T) {
+	orig := genTestData(1 << 20)
+
+	filter := XorFilter(42)
+
+	filtered := make([]byte, len(orig))
+	copy(filtered, orig)
+
+	filtered, _ = filter.Transform(filtered)
+	filtered, _ = filter.Transform(filtered)
+
+	if !bytes.Equal(filtered, orig) {
+		t.Errorf("orig != filtered:\n{%x}\n{%x}", orig, filtered)
+	}
+}
+
+func testFilter(t *testing.T, encode, decode Filter) {
+	base := memory.NewMemoryStorage()
+	storage := FilterStorage{
+		Base:   base,
+		Encode: encode,
+		Decode: decode,
+	}
+
+	size := backup.BlobChunkSize*2 + 10
+	data := genTestData(size)
+
+	id, err := backup.WriteFile(storage, bytes.NewReader(data))
+	if err != nil {
+		t.Fatalf("Unexpeced error from WriteFile: %s", err)
+	}
+
+	buf := new(bytes.Buffer)
+	if err := backup.RestoreFile(storage, id, buf); err != nil {
+		t.Fatalf("Unexpeced error from RestoreFile: %s", err)
+	}
+
+	if !bytes.Equal(data, buf.Bytes()) {
+		t.Errorf("data != buf.Bytes()")
+	}
+
+}
+
+func TestFilterStorage(t *testing.T) {
+	testFilter(t, XorFilter(42), XorFilter(42))
+}
+
+func TestPipeFilter(t *testing.T) {
+	testFilter(t, PipeFilter([]string{"cat"}), PipeFilter([]string{"cat"}))
+}
diff --git a/storage/memory/memory.go b/storage/memory/memory.go
index 456049b..e73c51a 100644
--- a/storage/memory/memory.go
+++ b/storage/memory/memory.go
@@ -22,12 +22,18 @@ func MemoryStorageFromConfig(conf config.Config, name string) (storage.Storage,
 	return NewMemoryStorage(), nil
 }
 
+func copyBytes(b []byte) []byte {
+	c := make([]byte, len(b))
+	copy(c, b)
+	return c
+}
+
 func (ms MemoryStorage) Get(id objects.ObjectId) ([]byte, error) {
 	b, ok := ms.objects[id.String()]
 	if !ok {
 		return nil, storage.ObjectNotFound
 	}
-	return b, nil
+	return copyBytes(b), nil
 }
 
 func (ms MemoryStorage) Has(id objects.ObjectId) (bool, error) {
@@ -36,7 +42,7 @@ func (ms MemoryStorage) Has(id objects.ObjectId) (bool, error) {
 }
 
 func (ms MemoryStorage) Set(id objects.ObjectId, typ objects.ObjectType, raw []byte) error {
-	ms.objects[id.String()] = raw
+	ms.objects[id.String()] = copyBytes(raw)
 	ms.bytype[typ] = append(ms.bytype[typ], id)
 
 	return nil
diff --git a/storage/registry/filter_conf.go b/storage/registry/filter_conf.go
new file mode 100644
index 0000000..593f238
--- /dev/null
+++ b/storage/registry/filter_conf.go
@@ -0,0 +1,39 @@
+package registry
+
+import (
+	"code.laria.me/petrific/config"
+	"code.laria.me/petrific/storage"
+	"code.laria.me/petrific/storage/filter"
+)
+
+// Unlike the other storage engines, we can not define FilterStorages
+// *FromConfig function in the package itself, because we need to reference the
+// registry package and circular imports are not allowed
+
+func filterStorageFromConfig(conf config.Config, name string) (storage.Storage, error) {
+	var storage_conf struct {
+		Base   string
+		Encode []string
+		Decode []string
+	}
+
+	if err := conf.GetStorageConfData(name, &storage_conf); err != nil {
+		return nil, err
+	}
+
+	base, err := LoadStorage(conf, storage_conf.Base)
+	if err != nil {
+		return nil, err
+	}
+
+	st := filter.FilterStorage{Base: base}
+
+	if len(storage_conf.Encode) > 0 {
+		st.Encode = filter.PipeFilter(storage_conf.Encode)
+	}
+	if len(storage_conf.Decode) > 0 {
+		st.Decode = filter.PipeFilter(storage_conf.Decode)
+	}
+
+	return st, nil
+}
diff --git a/storage/registry/registry.go b/storage/registry/registry.go
index f792946..4b48ef4 100644
--- a/storage/registry/registry.go
+++ b/storage/registry/registry.go
@@ -11,10 +11,13 @@ import (
 )
 
 // List af all available storage types
-var StorageTypes = map[string]storage.CreateStorageFromConfig{
-	"local":           local.LocalStorageFromConfig,
-	"memory":          memory.MemoryStorageFromConfig,
-	"openstack-swift": cloud.SwiftStorageCreator(),
+func getStorageTypes() map[string]storage.CreateStorageFromConfig {
+	return map[string]storage.CreateStorageFromConfig{
+		"local":           local.LocalStorageFromConfig,
+		"memory":          memory.MemoryStorageFromConfig,
+		"filter":          filterStorageFromConfig,
+		"openstack-swift": cloud.SwiftStorageCreator(),
+	}
 }
 
 var notFoundErr = errors.New("Storage not found")
@@ -41,7 +44,7 @@ func loadStorage(conf config.Config, storageName string) (storage.Storage, error
 		return nil, err
 	}
 
-	st, ok := StorageTypes[method]
+	st, ok := getStorageTypes()[method]
 	if !ok {
 		return nil, unknownMethodErr(method)
 	}