From f98b4e4b8165497b64d28d3e54babd1a43ad46d6 Mon Sep 17 00:00:00 2001 From: Laria Carolin Chabowski Date: Sun, 25 Jun 2017 21:49:04 +0200 Subject: File and Blob objects --- objects/object.go | 6 +++ objects/object_blob.go | 17 ++++++++ objects/object_file.go | 103 ++++++++++++++++++++++++++++++++++++++++++++ objects/object_file_test.go | 98 +++++++++++++++++++++++++++++++++++++++++ objects/properties.go | 93 +++++++++++++++++++++++++++++++++++++++ objects/properties_test.go | 72 +++++++++++++++++++++++++++++++ 6 files changed, 389 insertions(+) create mode 100644 objects/object_blob.go create mode 100644 objects/object_file.go create mode 100644 objects/object_file_test.go create mode 100644 objects/properties.go create mode 100644 objects/properties_test.go diff --git a/objects/object.go b/objects/object.go index fbe1f98..ced78bd 100644 --- a/objects/object.go +++ b/objects/object.go @@ -112,3 +112,9 @@ func Unserialize(r io.Reader) (RawObject, error) { return o, nil } + +type Object interface { + Type() ObjectType + Payload() []byte + FromPayload([]byte) error +} diff --git a/objects/object_blob.go b/objects/object_blob.go new file mode 100644 index 0000000..c35cb2a --- /dev/null +++ b/objects/object_blob.go @@ -0,0 +1,17 @@ +package objects + +type Blob []byte + +func (b Blob) Type() ObjectType { + return OTBlob +} + +func (b Blob) Payload() []byte { + return []byte(b) +} + +func (b *Blob) FromPayload(bytes []byte) error { + // TODO: perhaps it is better to copy the bytes? + *b = bytes + return nil +} diff --git a/objects/object_file.go b/objects/object_file.go new file mode 100644 index 0000000..7551193 --- /dev/null +++ b/objects/object_file.go @@ -0,0 +1,103 @@ +package objects + +import ( + "bufio" + "bytes" + "errors" + "strconv" +) + +type FileFragment struct { + Blob ObjectId + Size uint64 +} + +func (ff FileFragment) toProperties() properties { + return properties{"blob": ff.Blob.String(), "size": strconv.FormatUint(ff.Size, 10)} +} + +func (ff *FileFragment) fromProperties(p properties) error { + blob, ok := p["blob"] + if !ok { + return errors.New("Field `blob` is missing") + } + + var err error + ff.Blob, err = ParseObjectId(blob) + if err != nil { + return err + } + + size, ok := p["size"] + if !ok { + return errors.New("Field `size` is missing") + } + + ff.Size, err = strconv.ParseUint(size, 10, 64) + return err +} + +func (a FileFragment) Equals(b FileFragment) bool { + return a.Blob.Equals(b.Blob) && a.Size == b.Size +} + +type File []FileFragment + +func (f File) Type() ObjectType { + return OTFile +} + +func (f File) Payload() []byte { + out := []byte{} + + for _, ff := range f { + b, err := ff.toProperties().MarshalText() + if err != nil { + panic(err) + } + + out = append(out, b...) + out = append(out, '\n') + } + + return out +} + +func (f *File) FromPayload(payload []byte) error { + sc := bufio.NewScanner(bytes.NewReader(payload)) + + for sc.Scan() { + line := sc.Bytes() + if len(line) == 0 { + continue + } + + props := make(properties) + if err := props.UnmarshalText(line); err != nil { + return nil + } + + ff := FileFragment{} + if err := ff.fromProperties(props); err != nil { + return err + } + + *f = append(*f, ff) + } + + return sc.Err() +} + +func (a File) Equals(b File) bool { + if len(a) != len(b) { + return false + } + + for i := range a { + if !a[i].Equals(b[i]) { + return false + } + } + + return true +} diff --git a/objects/object_file_test.go b/objects/object_file_test.go new file mode 100644 index 0000000..f1cb584 --- /dev/null +++ b/objects/object_file_test.go @@ -0,0 +1,98 @@ +package objects + +import ( + "bytes" + "testing" +) + +var ( + testFileObj = File{ + FileFragment{Blob: genId(0x11), Size: 10}, + FileFragment{Blob: genId(0x22), Size: 20}, + FileFragment{Blob: genId(0x33), Size: 30}, + FileFragment{Blob: genId(0x44), Size: 40}, + FileFragment{Blob: genId(0x55), Size: 50}, + } + + testFileSerialization = []byte("" + + "blob=sha3-256:1111111111111111111111111111111111111111111111111111111111111111&size=10\n" + + "blob=sha3-256:2222222222222222222222222222222222222222222222222222222222222222&size=20\n" + + "blob=sha3-256:3333333333333333333333333333333333333333333333333333333333333333&size=30\n" + + "blob=sha3-256:4444444444444444444444444444444444444444444444444444444444444444&size=40\n" + + "blob=sha3-256:5555555555555555555555555555555555555555555555555555555555555555&size=50\n") +) + +func genId(b byte) (oid ObjectId) { + oid.Algo = OIdAlgoSHA3_256 + oid.Sum = make([]byte, OIdAlgoSHA3_256.sumLength()) + for i := 0; i < OIdAlgoSHA3_256.sumLength(); i++ { + oid.Sum[i] = b + } + + return +} + +func TestSerializeFile(t *testing.T) { + have := testFileObj.Payload() + + if !bytes.Equal(have, testFileSerialization) { + t.Errorf("Unexpected serialization result: %s", have) + } +} + +func TestSerializeEmptyFile(t *testing.T) { + f := File{} + + have := f.Payload() + want := []byte{} + + if !bytes.Equal(have, want) { + t.Errorf("Unexpected serialization result: %s", have) + } +} + +func TestUnserializeFile(t *testing.T) { + have := File{} + err := have.FromPayload(testFileSerialization) + + if err != nil { + t.Fatalf("Unexpected error: %s", err) + } + + if !have.Equals(testFileObj) { + t.Errorf("Unexpeced unserialization result: %v", have) + } +} + +func TestUnserializeEmptyFile(t *testing.T) { + have := File{} + err := have.FromPayload([]byte{}) + + if err != nil { + t.Fatalf("Unexpected error: %s", err) + } + + if len(have) != 0 { + t.Errorf("Unexpeced unserialization result: %v", have) + } +} + +func TestUnserializeFailure(t *testing.T) { + subtests := []struct{ name, payload string }{ + {"missing blob", "size=100\n"}, + {"empty blob", "blob=&size=100"}, + {"invalid blob", "blob=foobar&size=100"}, // Variations of invalid IDs are tested elsewhere + {"missing size", "blob=sha3-256:0000000000000000000000000000000000000000000000000000000000000000\n"}, + {"empty size", "blob=sha3-256:0000000000000000000000000000000000000000000000000000000000000000&size=\n"}, + {"invalid size", "blob=sha3-256:0000000000000000000000000000000000000000000000000000000000000000&size=foobar\n"}, + {"no props", "foobar\n"}, + } + + for _, subtest := range subtests { + have := File{} + err := have.FromPayload([]byte(subtest.payload)) + if err == nil { + t.Errorf("Unexpected unserialization success: %v", have) + } + } +} diff --git a/objects/properties.go b/objects/properties.go new file mode 100644 index 0000000..6b4acc1 --- /dev/null +++ b/objects/properties.go @@ -0,0 +1,93 @@ +package objects + +import ( + "encoding/hex" + "fmt" + "net/url" + "sort" +) + +// properties are mappings from strings to strings that are encoded as a restricted version of URL query strings +// (only the characters [a-zA-Z0-9.:_-] are allowed, values are ordered by their key) +type properties map[string]string + +// escapePropertyString escapes all bytes not in [a-zA-Z0-9.:_-] as %XX, where XX represents the hexadecimal value of the byte. +// Compatible with URL query strings +func escapePropertyString(s string) []byte { + out := []byte{} + esc := []byte("%XX") + + for _, b := range []byte(s) { + if (b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || (b >= '0' && b <= '9') || b == '.' || b == ':' || b == '_' || b == '-' { + out = append(out, b) + } else { + hex.Encode(esc[1:], []byte{b}) + out = append(out, esc...) + } + } + + return out +} + +func (p properties) MarshalText() ([]byte, error) { // Guaranteed to not fail, error is only here to satisfy encoding.TextMarshaler + keys := make([]string, len(p)) + i := 0 + for k := range p { + keys[i] = k + i++ + } + + sort.Strings(keys) + + out := []byte{} + + first := true + for _, k := range keys { + if first { + first = false + } else { + out = append(out, '&') + } + + out = append(out, escapePropertyString(k)...) + out = append(out, '=') + out = append(out, escapePropertyString(p[k])...) + } + + return out, nil +} + +func (p properties) UnmarshalText(text []byte) error { + vals, err := url.ParseQuery(string(text)) + if err != nil { + return err + } + + for k, v := range vals { + if len(v) != 1 { + return fmt.Errorf("Got %d values for key %s, expected 1", len(v), k) + } + + p[k] = v[0] + } + + return nil +} + +func (a properties) Equals(b properties) bool { + for k, va := range a { + vb, ok := b[k] + if !ok || vb != va { + return false + } + } + + for k := range b { + _, ok := a[k] + if !ok { + return false + } + } + + return true +} diff --git a/objects/properties_test.go b/objects/properties_test.go new file mode 100644 index 0000000..96df5c8 --- /dev/null +++ b/objects/properties_test.go @@ -0,0 +1,72 @@ +package objects + +import ( + "testing" +) + +func TestPropEscape(t *testing.T) { + tests := []struct{ name, in, want string }{ + {"empty", "", ""}, + {"no escape", "foo:bar_BAZ-123", "foo:bar_BAZ-123"}, + {"reserved chars", "foo=bar%baz%%=", "foo%3dbar%25baz%25%25%3d"}, + } + + for _, subtest := range tests { + have := string(escapePropertyString(subtest.in)) + if have != subtest.want { + t.Errorf("%s: want: '%s', have: '%s'", subtest.name, subtest.want, have) + } + } +} + +func TestPropertyMarshalling(t *testing.T) { + tests := []struct { + name string + in properties + want string + }{ + {"empty", properties{}, ""}, + {"single", properties{"foo": "bar"}, "foo=bar"}, + {"simple", properties{"foo": "bar", "bar": "baz"}, "bar=baz&foo=bar"}, + {"escapes", properties{"foo&bar": "%=baz", "?": "!"}, "%3f=%21&foo%26bar=%25%3dbaz"}, + } + + for _, subtest := range tests { + have, err := subtest.in.MarshalText() + if err != nil { + t.Errorf("%s: Got an error: %s", err) + continue + } + + if string(have) != subtest.want { + t.Errorf("%s: want: '%s', have: '%s'", subtest.name, subtest.want, have) + } + } +} + +func TestPropertyUnmarshalling(t *testing.T) { + tests := []struct { + name string + in string + want properties + }{ + {"empty", "", properties{}}, + {"single", "foo=bar", properties{"foo": "bar"}}, + {"simple", "bar=baz&foo=bar", properties{"foo": "bar", "bar": "baz"}}, + {"escapes", "%3f=%21&foo%26bar=%25%3dbaz", properties{"foo&bar": "%=baz", "?": "!"}}, + } + + for _, subtest := range tests { + have := make(properties) + err := have.UnmarshalText([]byte(subtest.in)) + + if err != nil { + t.Errorf("%s: Got an error: %s", err) + continue + } + + if !have.Equals(subtest.want) { + t.Errorf("%s: want: '%v', have: '%v'", subtest.name, subtest.want, have) + } + } +} -- cgit v1.2.3-54-g00ecf