From c04bbf9c82f216891ba44b8b5b621b8020963086 Mon Sep 17 00:00:00 2001 From: Laria Carolin Chabowski Date: Fri, 29 Sep 2017 07:57:09 +0200 Subject: Restructure package storage --- backup/backup_test.go | 5 +- backup/restore_test.go | 6 +- main.go | 22 +------- storage/local/local.go | 131 +++++++++++++++++++++++++++++++++++++++++++ storage/local_storage.go | 130 ------------------------------------------ storage/memory/memory.go | 51 +++++++++++++++++ storage/memory_storage.go | 50 ----------------- storage/registry/registry.go | 58 ++++++++++++++++++- 8 files changed, 246 insertions(+), 207 deletions(-) create mode 100644 storage/local/local.go delete mode 100644 storage/local_storage.go create mode 100644 storage/memory/memory.go delete mode 100644 storage/memory_storage.go diff --git a/backup/backup_test.go b/backup/backup_test.go index da14117..a7f7ff9 100644 --- a/backup/backup_test.go +++ b/backup/backup_test.go @@ -6,6 +6,7 @@ import ( "code.laria.me/petrific/fs" "code.laria.me/petrific/objects" "code.laria.me/petrific/storage" + "code.laria.me/petrific/storage/memory" "testing" ) @@ -27,7 +28,7 @@ func wantObject( } func TestWriteLargeFile(t *testing.T) { - s := storage.NewMemoryStorage() + s := memory.NewMemoryStorage() id, err := WriteFile(s, bytes.NewReader(content_largefile)) if err != nil { @@ -61,7 +62,7 @@ func mkfile(t *testing.T, d fs.Dir, name string, exec bool, content []byte) { } func TestWriteDir(t *testing.T) { - s := storage.NewMemoryStorage() + s := memory.NewMemoryStorage() root := fs.NewMemoryFSRoot("root") diff --git a/backup/restore_test.go b/backup/restore_test.go index 9e6e525..db66e11 100644 --- a/backup/restore_test.go +++ b/backup/restore_test.go @@ -4,7 +4,7 @@ import ( "bytes" "code.laria.me/petrific/fs" "code.laria.me/petrific/objects" - "code.laria.me/petrific/storage" + "code.laria.me/petrific/storage/memory" "io" "testing" ) @@ -71,7 +71,7 @@ func wantDir(n int, fx func(*testing.T, fs.Dir)) func(*testing.T, fs.File) { } func TestRestoreDir(t *testing.T) { - s := storage.NewMemoryStorage() + s := memory.NewMemoryStorage() s.Set(objid_emptyfile, objects.OTFile, obj_emptyfile) s.Set(objid_fooblob, objects.OTBlob, obj_fooblob) @@ -109,7 +109,7 @@ func TestRestoreDir(t *testing.T) { } func TestRestoreLargeFile(t *testing.T) { - s := storage.NewMemoryStorage() + s := memory.NewMemoryStorage() s.Set(objid_largefile_blob0, objects.OTBlob, obj_largefile_blob0) s.Set(objid_largefile_blob1, objects.OTBlob, obj_largefile_blob1) s.Set(objid_largefile, objects.OTFile, obj_largefile) diff --git a/main.go b/main.go index 6b2b55e..11202f3 100644 --- a/main.go +++ b/main.go @@ -97,27 +97,9 @@ func loadConfig() bool { storageName = conf.DefaultStorage } - storageOptions, ok := conf.Storage[storageName] - if !ok { - fmt.Fprintf(os.Stderr, "Storage %s not found\n", storageName) - return false - } - - var method string - if err := storageOptions.Get("method", &method); err != nil { - fmt.Fprintf(os.Stderr, "Failed setting up storage %s: %s\n", storageName, err) - return false - } - - st, ok := registry.StorageTypes[method] - if !ok { - fmt.Fprintf(os.Stderr, "Failed setting up storage %s: Method %s unknown", storageName, method) - return false - } - - s, err := st(conf, storageName) + s, err := registry.LoadStorage(conf, storageName) if err != nil { - fmt.Fprintf(os.Stderr, "Failed setting up storage %s: %s\n", storageName, err) + fmt.Fprintln(os.Stderr, err) return false } diff --git a/storage/local/local.go b/storage/local/local.go new file mode 100644 index 0000000..4869bb2 --- /dev/null +++ b/storage/local/local.go @@ -0,0 +1,131 @@ +package local + +import ( + "bytes" + "code.laria.me/petrific/config" + "code.laria.me/petrific/objects" + "code.laria.me/petrific/storage" + "encoding/hex" + "fmt" + "io" + "os" + "strings" +) + +func joinPath(parts ...string) string { + return strings.Join(parts, string(os.PathSeparator)) +} + +func objectDir(id objects.ObjectId) string { + return joinPath(string(id.Algo), hex.EncodeToString(id.Sum[0:1])) +} + +type LocalStorage struct { + Path string + index storage.Index +} + +func LocalStorageFromConfig(conf config.Config, name string) (storage.Storage, error) { + var path string + if err := conf.Storage[name].Get("path", &path); err != nil { + return nil, err + } + + return OpenLocalStorage(config.ExpandTilde(path)) +} + +func OpenLocalStorage(path string) (l LocalStorage, err error) { + l.Path = path + l.index = storage.NewIndex() + + if fi, err := os.Stat(path); os.IsNotExist(err) { + if err := os.MkdirAll(path, 0755); err != nil { + return l, err + } + } else if err != nil { + return l, err + } else if !fi.Mode().IsDir() { + return l, fmt.Errorf("%s: Not a directory", path) + } + + f, err := os.Open(joinPath(path, "index")) + if err != nil { + if os.IsNotExist(err) { + err = nil + } + return l, err + } + + if err == nil { + defer f.Close() + err = l.index.Load(f) + } + + return +} + +func objectPath(id objects.ObjectId) string { + return joinPath(objectDir(id), hex.EncodeToString(id.Sum[1:])) +} + +func (l LocalStorage) Get(id objects.ObjectId) ([]byte, error) { + f, err := os.Open(joinPath(l.Path, objectPath(id))) + if os.IsNotExist(err) { + return []byte{}, storage.ObjectNotFound + } else if err != nil { + return []byte{}, err + } + defer f.Close() + + buf := new(bytes.Buffer) + _, err = io.Copy(buf, f) + return buf.Bytes(), err +} + +func (l LocalStorage) Has(id objects.ObjectId) (bool, error) { + _, err := os.Stat(joinPath(l.Path, objectPath(id))) + if err == nil { + return true, nil + } else if os.IsNotExist(err) { + return false, nil + } else { + return false, err + } +} + +func (l LocalStorage) Set(id objects.ObjectId, typ objects.ObjectType, raw []byte) error { + // First, check if the directory exists + dir := joinPath(l.Path, objectDir(id)) + _, err := os.Stat(dir) + if os.IsNotExist(err) { + if err := os.MkdirAll(dir, 0755); err != nil { + return err + } + } else if err != nil { + return err + } + + f, err := os.Create(joinPath(l.Path, objectPath(id))) + if err != nil { + return err + } + defer f.Close() + + _, err = f.Write(raw) + l.index.Set(id, typ) + return err +} + +func (l LocalStorage) List(typ objects.ObjectType) ([]objects.ObjectId, error) { + return l.index.List(typ), nil +} + +func (l LocalStorage) Close() error { + f, err := os.Create(joinPath(l.Path, "index")) + if err != nil { + return err + } + defer f.Close() + + return l.index.Save(f) +} diff --git a/storage/local_storage.go b/storage/local_storage.go deleted file mode 100644 index c080564..0000000 --- a/storage/local_storage.go +++ /dev/null @@ -1,130 +0,0 @@ -package storage - -import ( - "bytes" - "code.laria.me/petrific/config" - "code.laria.me/petrific/objects" - "encoding/hex" - "fmt" - "io" - "os" - "strings" -) - -func joinPath(parts ...string) string { - return strings.Join(parts, string(os.PathSeparator)) -} - -func objectDir(id objects.ObjectId) string { - return joinPath(string(id.Algo), hex.EncodeToString(id.Sum[0:1])) -} - -type LocalStorage struct { - Path string - index Index -} - -func LocalStorageFromConfig(conf config.Config, name string) (Storage, error) { - var path string - if err := conf.Storage[name].Get("path", &path); err != nil { - return nil, err - } - - return OpenLocalStorage(config.ExpandTilde(path)) -} - -func OpenLocalStorage(path string) (l LocalStorage, err error) { - l.Path = path - l.index = NewIndex() - - if fi, err := os.Stat(path); os.IsNotExist(err) { - if err := os.MkdirAll(path, 0755); err != nil { - return l, err - } - } else if err != nil { - return l, err - } else if !fi.Mode().IsDir() { - return l, fmt.Errorf("%s: Not a directory", path) - } - - f, err := os.Open(joinPath(path, "index")) - if err != nil { - if os.IsNotExist(err) { - err = nil - } - return l, err - } - - if err == nil { - defer f.Close() - err = l.index.Load(f) - } - - return -} - -func objectPath(id objects.ObjectId) string { - return joinPath(objectDir(id), hex.EncodeToString(id.Sum[1:])) -} - -func (l LocalStorage) Get(id objects.ObjectId) ([]byte, error) { - f, err := os.Open(joinPath(l.Path, objectPath(id))) - if os.IsNotExist(err) { - return []byte{}, ObjectNotFound - } else if err != nil { - return []byte{}, err - } - defer f.Close() - - buf := new(bytes.Buffer) - _, err = io.Copy(buf, f) - return buf.Bytes(), err -} - -func (l LocalStorage) Has(id objects.ObjectId) (bool, error) { - _, err := os.Stat(joinPath(l.Path, objectPath(id))) - if err == nil { - return true, nil - } else if os.IsNotExist(err) { - return false, nil - } else { - return false, err - } -} - -func (l LocalStorage) Set(id objects.ObjectId, typ objects.ObjectType, raw []byte) error { - // First, check if the directory exists - dir := joinPath(l.Path, objectDir(id)) - _, err := os.Stat(dir) - if os.IsNotExist(err) { - if err := os.MkdirAll(dir, 0755); err != nil { - return err - } - } else if err != nil { - return err - } - - f, err := os.Create(joinPath(l.Path, objectPath(id))) - if err != nil { - return err - } - defer f.Close() - - _, err = f.Write(raw) - l.index.Set(id, typ) - return err -} - -func (l LocalStorage) List(typ objects.ObjectType) ([]objects.ObjectId, error) { - return l.index.List(typ), nil -} - -func (l LocalStorage) Close() error { - f, err := os.Create(joinPath(l.Path, "index")) - if err != nil { - return err - } - defer f.Close() - - return l.index.Save(f) -} diff --git a/storage/memory/memory.go b/storage/memory/memory.go new file mode 100644 index 0000000..456049b --- /dev/null +++ b/storage/memory/memory.go @@ -0,0 +1,51 @@ +package memory + +import ( + "code.laria.me/petrific/config" + "code.laria.me/petrific/objects" + "code.laria.me/petrific/storage" +) + +type MemoryStorage struct { + objects map[string][]byte + bytype map[objects.ObjectType][]objects.ObjectId +} + +func NewMemoryStorage() storage.Storage { + return MemoryStorage{ + objects: make(map[string][]byte), + bytype: make(map[objects.ObjectType][]objects.ObjectId), + } +} + +func MemoryStorageFromConfig(conf config.Config, name string) (storage.Storage, error) { + return NewMemoryStorage(), nil +} + +func (ms MemoryStorage) Get(id objects.ObjectId) ([]byte, error) { + b, ok := ms.objects[id.String()] + if !ok { + return nil, storage.ObjectNotFound + } + return b, nil +} + +func (ms MemoryStorage) Has(id objects.ObjectId) (bool, error) { + _, ok := ms.objects[id.String()] + return ok, nil +} + +func (ms MemoryStorage) Set(id objects.ObjectId, typ objects.ObjectType, raw []byte) error { + ms.objects[id.String()] = raw + ms.bytype[typ] = append(ms.bytype[typ], id) + + return nil +} + +func (ms MemoryStorage) List(typ objects.ObjectType) ([]objects.ObjectId, error) { + return ms.bytype[typ], nil +} + +func (MemoryStorage) Close() error { + return nil +} diff --git a/storage/memory_storage.go b/storage/memory_storage.go deleted file mode 100644 index b9ed54a..0000000 --- a/storage/memory_storage.go +++ /dev/null @@ -1,50 +0,0 @@ -package storage - -import ( - "code.laria.me/petrific/config" - "code.laria.me/petrific/objects" -) - -type MemoryStorage struct { - objects map[string][]byte - bytype map[objects.ObjectType][]objects.ObjectId -} - -func NewMemoryStorage() Storage { - return MemoryStorage{ - objects: make(map[string][]byte), - bytype: make(map[objects.ObjectType][]objects.ObjectId), - } -} - -func MemoryStorageFromConfig(conf config.Config, name string) (Storage, error) { - return NewMemoryStorage(), nil -} - -func (ms MemoryStorage) Get(id objects.ObjectId) ([]byte, error) { - b, ok := ms.objects[id.String()] - if !ok { - return nil, ObjectNotFound - } - return b, nil -} - -func (ms MemoryStorage) Has(id objects.ObjectId) (bool, error) { - _, ok := ms.objects[id.String()] - return ok, nil -} - -func (ms MemoryStorage) Set(id objects.ObjectId, typ objects.ObjectType, raw []byte) error { - ms.objects[id.String()] = raw - ms.bytype[typ] = append(ms.bytype[typ], id) - - return nil -} - -func (ms MemoryStorage) List(typ objects.ObjectType) ([]objects.ObjectId, error) { - return ms.bytype[typ], nil -} - -func (MemoryStorage) Close() error { - return nil -} diff --git a/storage/registry/registry.go b/storage/registry/registry.go index dec7c32..ad66c6a 100644 --- a/storage/registry/registry.go +++ b/storage/registry/registry.go @@ -1,13 +1,67 @@ package registry import ( + "code.laria.me/petrific/config" "code.laria.me/petrific/storage" "code.laria.me/petrific/storage/cloud" + "code.laria.me/petrific/storage/local" + "code.laria.me/petrific/storage/memory" + "errors" + "fmt" ) // List af all available storage types var StorageTypes = map[string]storage.CreateStorageFromConfig{ - "local": storage.LocalStorageFromConfig, - "memory": storage.MemoryStorageFromConfig, + "local": local.LocalStorageFromConfig, + "memory": memory.MemoryStorageFromConfig, "openstack-swift": cloud.SwiftStorageCreator(), } + +var notFoundErr = errors.New("Storage not found") + +type unknownMethodErr string + +func (method unknownMethodErr) Error() string { + return fmt.Sprintf("Method %s unknown", string(method)) +} + +type storageConfErr struct { + name string + err error +} + +func (e storageConfErr) Error() string { + return fmt.Sprintf("Failed setting up storage %s: %s", e.name, e.err.Error()) +} + +func loadStorage(conf config.Config, storageName string) (storage.Storage, error) { + storageOptions, ok := conf.Storage[storageName] + if !ok { + return nil, notFoundErr + } + + var method string + if err := storageOptions.Get("method", &method); err != nil { + return nil, err + } + + st, ok := StorageTypes[method] + if !ok { + return nil, unknownMethodErr(method) + } + + s, err := st(conf, storageName) + if err != nil { + return nil, err + } + + return s, nil +} + +func LoadStorage(conf config.Config, storageName string) (storage.Storage, error) { + s, err := loadStorage(conf, storageName) + if err != nil { + return nil, storageConfErr{storageName, err} + } + return s, nil +} -- cgit v1.2.3-54-g00ecf