aboutsummaryrefslogtreecommitdiff
path: root/storage
diff options
context:
space:
mode:
Diffstat (limited to 'storage')
-rw-r--r--storage/cloud/cloudstorage.go42
-rw-r--r--storage/filter/filter.go77
-rw-r--r--storage/filter/filter_test.go76
-rw-r--r--storage/memory/memory.go10
-rw-r--r--storage/registry/filter_conf.go39
-rw-r--r--storage/registry/registry.go13
6 files changed, 213 insertions, 44 deletions
diff --git a/storage/cloud/cloudstorage.go b/storage/cloud/cloudstorage.go
index 261a034..f93bb5c 100644
--- a/storage/cloud/cloudstorage.go
+++ b/storage/cloud/cloudstorage.go
@@ -5,7 +5,6 @@ package cloud
import (
"bytes"
"code.laria.me/petrific/config"
- "code.laria.me/petrific/gpg"
"code.laria.me/petrific/objects"
"code.laria.me/petrific/storage"
"errors"
@@ -27,22 +26,9 @@ var (
NotFoundErr = errors.New("Object not found") // Cloud object could not be found
)
-// Crypter provides de-/encrypting facilities for CloudBasedObjectStorage
-type Crypter interface {
- Encrypt([]byte) ([]byte, error)
- Decrypt([]byte) ([]byte, error)
-}
-
-// NopCrypter implements Crypter by not de/-encrypting at all
-type NopCrypter struct{}
-
-func (NopCrypter) Encrypt(in []byte) ([]byte, error) { return in, nil }
-func (NopCrypter) Decrypt(in []byte) ([]byte, error) { return in, nil }
-
type CloudBasedObjectStorage struct {
- CS CloudStorage
- Prefix string
- Crypter Crypter
+ CS CloudStorage
+ Prefix string
index storage.Index
}
@@ -94,24 +80,14 @@ func (cbos *CloudBasedObjectStorage) Init() error {
}
func (cbos CloudBasedObjectStorage) Get(id objects.ObjectId) ([]byte, error) {
- b, err := cbos.CS.Get(cbos.objidToKey(id))
- if err != nil {
- return []byte{}, err
- }
-
- return cbos.Crypter.Decrypt(b)
+ return cbos.CS.Get(cbos.objidToKey(id))
}
func (cbos CloudBasedObjectStorage) Has(id objects.ObjectId) (bool, error) {
return cbos.CS.Has(cbos.objidToKey(id))
}
-func (cbos CloudBasedObjectStorage) Set(id objects.ObjectId, typ objects.ObjectType, raw []byte) error {
- b, err := cbos.Crypter.Encrypt(raw)
- if err != nil {
- return err
- }
-
+func (cbos CloudBasedObjectStorage) Set(id objects.ObjectId, typ objects.ObjectType, b []byte) error {
if err := cbos.CS.Put(cbos.objidToKey(id), b); err != nil {
return err
}
@@ -159,8 +135,7 @@ func cloudStorageCreator(cloudCreator cloudObjectStorageCreator) storage.CreateS
var cbos CloudBasedObjectStorage
var storageconf struct {
- Prefix string `toml:"prefix,omitempty"`
- EncryptFor string `toml:"encrypt_for,omitempty"`
+ Prefix string `toml:"prefix,omitempty"`
}
if err := conf.GetStorageConfData(name, &storageconf); err != nil {
@@ -169,13 +144,6 @@ func cloudStorageCreator(cloudCreator cloudObjectStorageCreator) storage.CreateS
cbos.Prefix = storageconf.Prefix
- if storageconf.EncryptFor != "" {
- cbos.Crypter = gpg.Crypter{
- gpg.Encrypter{Key: storageconf.EncryptFor},
- gpg.Decrypter{},
- }
- }
-
var err error
if cbos.CS, err = cloudCreator(conf, name); err != nil {
return nil, err
diff --git a/storage/filter/filter.go b/storage/filter/filter.go
new file mode 100644
index 0000000..52ecbfd
--- /dev/null
+++ b/storage/filter/filter.go
@@ -0,0 +1,77 @@
+package filter
+
+import (
+ "bytes"
+ "code.laria.me/petrific/objects"
+ "code.laria.me/petrific/storage"
+ "errors"
+ "os/exec"
+)
+
+type Filter interface {
+ Transform([]byte) ([]byte, error)
+}
+
+type PipeFilter []string
+
+var emptyCommand = errors.New("Need at least one argument for pipeFilter")
+
+func (pf PipeFilter) Transform(b []byte) ([]byte, error) {
+ if len(pf) == 0 {
+ return []byte{}, emptyCommand
+ }
+
+ cmd := exec.Command(pf[0], pf[1:]...)
+
+ buf := new(bytes.Buffer)
+ cmd.Stdout = buf
+ cmd.Stdin = bytes.NewReader(b)
+
+ if err := cmd.Run(); err != nil {
+ return []byte{}, err
+ }
+
+ return buf.Bytes(), nil
+}
+
+type FilterStorage struct {
+ Base storage.Storage
+ Decode, Encode Filter
+}
+
+func (filt FilterStorage) Get(id objects.ObjectId) ([]byte, error) {
+ data, err := filt.Base.Get(id)
+ if err != nil {
+ return data, err
+ }
+
+ if filt.Decode == nil {
+ return data, nil
+ }
+
+ return filt.Decode.Transform(data)
+}
+
+func (filt FilterStorage) Has(id objects.ObjectId) (bool, error) {
+ return filt.Base.Has(id)
+}
+
+func (filt FilterStorage) Set(id objects.ObjectId, typ objects.ObjectType, raw []byte) error {
+ if filt.Encode != nil {
+ var err error
+ raw, err = filt.Encode.Transform(raw)
+ if err != nil {
+ return err
+ }
+ }
+
+ return filt.Base.Set(id, typ, raw)
+}
+
+func (filt FilterStorage) List(typ objects.ObjectType) ([]objects.ObjectId, error) {
+ return filt.Base.List(typ)
+}
+
+func (filt FilterStorage) Close() error {
+ return filt.Base.Close()
+}
diff --git a/storage/filter/filter_test.go b/storage/filter/filter_test.go
new file mode 100644
index 0000000..78d4e7f
--- /dev/null
+++ b/storage/filter/filter_test.go
@@ -0,0 +1,76 @@
+package filter
+
+import (
+ "bytes"
+ "code.laria.me/petrific/backup"
+ "code.laria.me/petrific/storage/memory"
+ "testing"
+)
+
+type XorFilter byte
+
+func (xf XorFilter) Transform(b []byte) ([]byte, error) {
+ for i := range b {
+ b[i] ^= byte(xf)
+ }
+ return b, nil
+}
+
+func genTestData(size int) []byte {
+ data := make([]byte, size)
+ for i := range data {
+ data[i] = byte(i & 0xff)
+ }
+ return data
+}
+
+func TestXorReverse(t *testing.T) {
+ orig := genTestData(1 << 20)
+
+ filter := XorFilter(42)
+
+ filtered := make([]byte, len(orig))
+ copy(filtered, orig)
+
+ filtered, _ = filter.Transform(filtered)
+ filtered, _ = filter.Transform(filtered)
+
+ if !bytes.Equal(filtered, orig) {
+ t.Errorf("orig != filtered:\n{%x}\n{%x}", orig, filtered)
+ }
+}
+
+func testFilter(t *testing.T, encode, decode Filter) {
+ base := memory.NewMemoryStorage()
+ storage := FilterStorage{
+ Base: base,
+ Encode: encode,
+ Decode: decode,
+ }
+
+ size := backup.BlobChunkSize*2 + 10
+ data := genTestData(size)
+
+ id, err := backup.WriteFile(storage, bytes.NewReader(data))
+ if err != nil {
+ t.Fatalf("Unexpeced error from WriteFile: %s", err)
+ }
+
+ buf := new(bytes.Buffer)
+ if err := backup.RestoreFile(storage, id, buf); err != nil {
+ t.Fatalf("Unexpeced error from RestoreFile: %s", err)
+ }
+
+ if !bytes.Equal(data, buf.Bytes()) {
+ t.Errorf("data != buf.Bytes()")
+ }
+
+}
+
+func TestFilterStorage(t *testing.T) {
+ testFilter(t, XorFilter(42), XorFilter(42))
+}
+
+func TestPipeFilter(t *testing.T) {
+ testFilter(t, PipeFilter([]string{"cat"}), PipeFilter([]string{"cat"}))
+}
diff --git a/storage/memory/memory.go b/storage/memory/memory.go
index 456049b..e73c51a 100644
--- a/storage/memory/memory.go
+++ b/storage/memory/memory.go
@@ -22,12 +22,18 @@ func MemoryStorageFromConfig(conf config.Config, name string) (storage.Storage,
return NewMemoryStorage(), nil
}
+func copyBytes(b []byte) []byte {
+ c := make([]byte, len(b))
+ copy(c, b)
+ return c
+}
+
func (ms MemoryStorage) Get(id objects.ObjectId) ([]byte, error) {
b, ok := ms.objects[id.String()]
if !ok {
return nil, storage.ObjectNotFound
}
- return b, nil
+ return copyBytes(b), nil
}
func (ms MemoryStorage) Has(id objects.ObjectId) (bool, error) {
@@ -36,7 +42,7 @@ func (ms MemoryStorage) Has(id objects.ObjectId) (bool, error) {
}
func (ms MemoryStorage) Set(id objects.ObjectId, typ objects.ObjectType, raw []byte) error {
- ms.objects[id.String()] = raw
+ ms.objects[id.String()] = copyBytes(raw)
ms.bytype[typ] = append(ms.bytype[typ], id)
return nil
diff --git a/storage/registry/filter_conf.go b/storage/registry/filter_conf.go
new file mode 100644
index 0000000..593f238
--- /dev/null
+++ b/storage/registry/filter_conf.go
@@ -0,0 +1,39 @@
+package registry
+
+import (
+ "code.laria.me/petrific/config"
+ "code.laria.me/petrific/storage"
+ "code.laria.me/petrific/storage/filter"
+)
+
+// Unlike the other storage engines, we can not define FilterStorages
+// *FromConfig function in the package itself, because we need to reference the
+// registry package and circular imports are not allowed
+
+func filterStorageFromConfig(conf config.Config, name string) (storage.Storage, error) {
+ var storage_conf struct {
+ Base string
+ Encode []string
+ Decode []string
+ }
+
+ if err := conf.GetStorageConfData(name, &storage_conf); err != nil {
+ return nil, err
+ }
+
+ base, err := LoadStorage(conf, storage_conf.Base)
+ if err != nil {
+ return nil, err
+ }
+
+ st := filter.FilterStorage{Base: base}
+
+ if len(storage_conf.Encode) > 0 {
+ st.Encode = filter.PipeFilter(storage_conf.Encode)
+ }
+ if len(storage_conf.Decode) > 0 {
+ st.Decode = filter.PipeFilter(storage_conf.Decode)
+ }
+
+ return st, nil
+}
diff --git a/storage/registry/registry.go b/storage/registry/registry.go
index f792946..4b48ef4 100644
--- a/storage/registry/registry.go
+++ b/storage/registry/registry.go
@@ -11,10 +11,13 @@ import (
)
// List af all available storage types
-var StorageTypes = map[string]storage.CreateStorageFromConfig{
- "local": local.LocalStorageFromConfig,
- "memory": memory.MemoryStorageFromConfig,
- "openstack-swift": cloud.SwiftStorageCreator(),
+func getStorageTypes() map[string]storage.CreateStorageFromConfig {
+ return map[string]storage.CreateStorageFromConfig{
+ "local": local.LocalStorageFromConfig,
+ "memory": memory.MemoryStorageFromConfig,
+ "filter": filterStorageFromConfig,
+ "openstack-swift": cloud.SwiftStorageCreator(),
+ }
}
var notFoundErr = errors.New("Storage not found")
@@ -41,7 +44,7 @@ func loadStorage(conf config.Config, storageName string) (storage.Storage, error
return nil, err
}
- st, ok := StorageTypes[method]
+ st, ok := getStorageTypes()[method]
if !ok {
return nil, unknownMethodErr(method)
}
generated by cgit v1.2.3-70-g09d2 (git 2.47.0) at 2024-11-21 19:43:09 +0000