From db0c023fd0d756912c3f575c6ac65e99fda573cc Mon Sep 17 00:00:00 2001 From: Laria Carolin Chabowski Date: Mon, 2 Oct 2017 14:26:26 +0200 Subject: Add filter storage method Also remove de/encryption in cloud storage, can be provided with a filter storage --- gpg/gpg.go | 21 ----------- storage/cloud/cloudstorage.go | 42 +++------------------- storage/filter/filter.go | 77 +++++++++++++++++++++++++++++++++++++++++ storage/filter/filter_test.go | 76 ++++++++++++++++++++++++++++++++++++++++ storage/memory/memory.go | 10 ++++-- storage/registry/filter_conf.go | 39 +++++++++++++++++++++ storage/registry/registry.go | 13 ++++--- 7 files changed, 213 insertions(+), 65 deletions(-) create mode 100644 storage/filter/filter.go create mode 100644 storage/filter/filter_test.go create mode 100644 storage/registry/filter_conf.go diff --git a/gpg/gpg.go b/gpg/gpg.go index 1ab71ec..cb62f32 100644 --- a/gpg/gpg.go +++ b/gpg/gpg.go @@ -36,24 +36,3 @@ func (Verifyer) Verify(b []byte) error { cmd.Stdin = bytes.NewReader(b) return cmd.Run() } - -type Encrypter struct { - Key string -} - -func (e Encrypter) Encrypt(b []byte) ([]byte, error) { - cmd := exec.Command("gpg", "--encrypt", "--recipient", e.Key) - return filter(cmd, b) -} - -type Decrypter struct{} - -func (Decrypter) Decrypt(b []byte) ([]byte, error) { - cmd := exec.Command("gpg", "--decrypt") - return filter(cmd, b) -} - -type Crypter struct { - Encrypter - Decrypter -} diff --git a/storage/cloud/cloudstorage.go b/storage/cloud/cloudstorage.go index 261a034..f93bb5c 100644 --- a/storage/cloud/cloudstorage.go +++ b/storage/cloud/cloudstorage.go @@ -5,7 +5,6 @@ package cloud import ( "bytes" "code.laria.me/petrific/config" - "code.laria.me/petrific/gpg" "code.laria.me/petrific/objects" "code.laria.me/petrific/storage" "errors" @@ -27,22 +26,9 @@ var ( NotFoundErr = errors.New("Object not found") // Cloud object could not be found ) -// Crypter provides de-/encrypting facilities for CloudBasedObjectStorage -type Crypter interface { - Encrypt([]byte) ([]byte, error) - Decrypt([]byte) ([]byte, error) -} - -// NopCrypter implements Crypter by not de/-encrypting at all -type NopCrypter struct{} - -func (NopCrypter) Encrypt(in []byte) ([]byte, error) { return in, nil } -func (NopCrypter) Decrypt(in []byte) ([]byte, error) { return in, nil } - type CloudBasedObjectStorage struct { - CS CloudStorage - Prefix string - Crypter Crypter + CS CloudStorage + Prefix string index storage.Index } @@ -94,24 +80,14 @@ func (cbos *CloudBasedObjectStorage) Init() error { } func (cbos CloudBasedObjectStorage) Get(id objects.ObjectId) ([]byte, error) { - b, err := cbos.CS.Get(cbos.objidToKey(id)) - if err != nil { - return []byte{}, err - } - - return cbos.Crypter.Decrypt(b) + return cbos.CS.Get(cbos.objidToKey(id)) } func (cbos CloudBasedObjectStorage) Has(id objects.ObjectId) (bool, error) { return cbos.CS.Has(cbos.objidToKey(id)) } -func (cbos CloudBasedObjectStorage) Set(id objects.ObjectId, typ objects.ObjectType, raw []byte) error { - b, err := cbos.Crypter.Encrypt(raw) - if err != nil { - return err - } - +func (cbos CloudBasedObjectStorage) Set(id objects.ObjectId, typ objects.ObjectType, b []byte) error { if err := cbos.CS.Put(cbos.objidToKey(id), b); err != nil { return err } @@ -159,8 +135,7 @@ func cloudStorageCreator(cloudCreator cloudObjectStorageCreator) storage.CreateS var cbos CloudBasedObjectStorage var storageconf struct { - Prefix string `toml:"prefix,omitempty"` - EncryptFor string `toml:"encrypt_for,omitempty"` + Prefix string `toml:"prefix,omitempty"` } if err := conf.GetStorageConfData(name, &storageconf); err != nil { @@ -169,13 +144,6 @@ func cloudStorageCreator(cloudCreator cloudObjectStorageCreator) storage.CreateS cbos.Prefix = storageconf.Prefix - if storageconf.EncryptFor != "" { - cbos.Crypter = gpg.Crypter{ - gpg.Encrypter{Key: storageconf.EncryptFor}, - gpg.Decrypter{}, - } - } - var err error if cbos.CS, err = cloudCreator(conf, name); err != nil { return nil, err diff --git a/storage/filter/filter.go b/storage/filter/filter.go new file mode 100644 index 0000000..52ecbfd --- /dev/null +++ b/storage/filter/filter.go @@ -0,0 +1,77 @@ +package filter + +import ( + "bytes" + "code.laria.me/petrific/objects" + "code.laria.me/petrific/storage" + "errors" + "os/exec" +) + +type Filter interface { + Transform([]byte) ([]byte, error) +} + +type PipeFilter []string + +var emptyCommand = errors.New("Need at least one argument for pipeFilter") + +func (pf PipeFilter) Transform(b []byte) ([]byte, error) { + if len(pf) == 0 { + return []byte{}, emptyCommand + } + + cmd := exec.Command(pf[0], pf[1:]...) + + buf := new(bytes.Buffer) + cmd.Stdout = buf + cmd.Stdin = bytes.NewReader(b) + + if err := cmd.Run(); err != nil { + return []byte{}, err + } + + return buf.Bytes(), nil +} + +type FilterStorage struct { + Base storage.Storage + Decode, Encode Filter +} + +func (filt FilterStorage) Get(id objects.ObjectId) ([]byte, error) { + data, err := filt.Base.Get(id) + if err != nil { + return data, err + } + + if filt.Decode == nil { + return data, nil + } + + return filt.Decode.Transform(data) +} + +func (filt FilterStorage) Has(id objects.ObjectId) (bool, error) { + return filt.Base.Has(id) +} + +func (filt FilterStorage) Set(id objects.ObjectId, typ objects.ObjectType, raw []byte) error { + if filt.Encode != nil { + var err error + raw, err = filt.Encode.Transform(raw) + if err != nil { + return err + } + } + + return filt.Base.Set(id, typ, raw) +} + +func (filt FilterStorage) List(typ objects.ObjectType) ([]objects.ObjectId, error) { + return filt.Base.List(typ) +} + +func (filt FilterStorage) Close() error { + return filt.Base.Close() +} diff --git a/storage/filter/filter_test.go b/storage/filter/filter_test.go new file mode 100644 index 0000000..78d4e7f --- /dev/null +++ b/storage/filter/filter_test.go @@ -0,0 +1,76 @@ +package filter + +import ( + "bytes" + "code.laria.me/petrific/backup" + "code.laria.me/petrific/storage/memory" + "testing" +) + +type XorFilter byte + +func (xf XorFilter) Transform(b []byte) ([]byte, error) { + for i := range b { + b[i] ^= byte(xf) + } + return b, nil +} + +func genTestData(size int) []byte { + data := make([]byte, size) + for i := range data { + data[i] = byte(i & 0xff) + } + return data +} + +func TestXorReverse(t *testing.T) { + orig := genTestData(1 << 20) + + filter := XorFilter(42) + + filtered := make([]byte, len(orig)) + copy(filtered, orig) + + filtered, _ = filter.Transform(filtered) + filtered, _ = filter.Transform(filtered) + + if !bytes.Equal(filtered, orig) { + t.Errorf("orig != filtered:\n{%x}\n{%x}", orig, filtered) + } +} + +func testFilter(t *testing.T, encode, decode Filter) { + base := memory.NewMemoryStorage() + storage := FilterStorage{ + Base: base, + Encode: encode, + Decode: decode, + } + + size := backup.BlobChunkSize*2 + 10 + data := genTestData(size) + + id, err := backup.WriteFile(storage, bytes.NewReader(data)) + if err != nil { + t.Fatalf("Unexpeced error from WriteFile: %s", err) + } + + buf := new(bytes.Buffer) + if err := backup.RestoreFile(storage, id, buf); err != nil { + t.Fatalf("Unexpeced error from RestoreFile: %s", err) + } + + if !bytes.Equal(data, buf.Bytes()) { + t.Errorf("data != buf.Bytes()") + } + +} + +func TestFilterStorage(t *testing.T) { + testFilter(t, XorFilter(42), XorFilter(42)) +} + +func TestPipeFilter(t *testing.T) { + testFilter(t, PipeFilter([]string{"cat"}), PipeFilter([]string{"cat"})) +} diff --git a/storage/memory/memory.go b/storage/memory/memory.go index 456049b..e73c51a 100644 --- a/storage/memory/memory.go +++ b/storage/memory/memory.go @@ -22,12 +22,18 @@ func MemoryStorageFromConfig(conf config.Config, name string) (storage.Storage, return NewMemoryStorage(), nil } +func copyBytes(b []byte) []byte { + c := make([]byte, len(b)) + copy(c, b) + return c +} + func (ms MemoryStorage) Get(id objects.ObjectId) ([]byte, error) { b, ok := ms.objects[id.String()] if !ok { return nil, storage.ObjectNotFound } - return b, nil + return copyBytes(b), nil } func (ms MemoryStorage) Has(id objects.ObjectId) (bool, error) { @@ -36,7 +42,7 @@ func (ms MemoryStorage) Has(id objects.ObjectId) (bool, error) { } func (ms MemoryStorage) Set(id objects.ObjectId, typ objects.ObjectType, raw []byte) error { - ms.objects[id.String()] = raw + ms.objects[id.String()] = copyBytes(raw) ms.bytype[typ] = append(ms.bytype[typ], id) return nil diff --git a/storage/registry/filter_conf.go b/storage/registry/filter_conf.go new file mode 100644 index 0000000..593f238 --- /dev/null +++ b/storage/registry/filter_conf.go @@ -0,0 +1,39 @@ +package registry + +import ( + "code.laria.me/petrific/config" + "code.laria.me/petrific/storage" + "code.laria.me/petrific/storage/filter" +) + +// Unlike the other storage engines, we can not define FilterStorages +// *FromConfig function in the package itself, because we need to reference the +// registry package and circular imports are not allowed + +func filterStorageFromConfig(conf config.Config, name string) (storage.Storage, error) { + var storage_conf struct { + Base string + Encode []string + Decode []string + } + + if err := conf.GetStorageConfData(name, &storage_conf); err != nil { + return nil, err + } + + base, err := LoadStorage(conf, storage_conf.Base) + if err != nil { + return nil, err + } + + st := filter.FilterStorage{Base: base} + + if len(storage_conf.Encode) > 0 { + st.Encode = filter.PipeFilter(storage_conf.Encode) + } + if len(storage_conf.Decode) > 0 { + st.Decode = filter.PipeFilter(storage_conf.Decode) + } + + return st, nil +} diff --git a/storage/registry/registry.go b/storage/registry/registry.go index f792946..4b48ef4 100644 --- a/storage/registry/registry.go +++ b/storage/registry/registry.go @@ -11,10 +11,13 @@ import ( ) // List af all available storage types -var StorageTypes = map[string]storage.CreateStorageFromConfig{ - "local": local.LocalStorageFromConfig, - "memory": memory.MemoryStorageFromConfig, - "openstack-swift": cloud.SwiftStorageCreator(), +func getStorageTypes() map[string]storage.CreateStorageFromConfig { + return map[string]storage.CreateStorageFromConfig{ + "local": local.LocalStorageFromConfig, + "memory": memory.MemoryStorageFromConfig, + "filter": filterStorageFromConfig, + "openstack-swift": cloud.SwiftStorageCreator(), + } } var notFoundErr = errors.New("Storage not found") @@ -41,7 +44,7 @@ func loadStorage(conf config.Config, storageName string) (storage.Storage, error return nil, err } - st, ok := StorageTypes[method] + st, ok := getStorageTypes()[method] if !ok { return nil, unknownMethodErr(method) } -- cgit v1.2.3-54-g00ecf