diff options
Diffstat (limited to 'objects')
| -rw-r--r-- | objects/object.go | 3 | ||||
| -rw-r--r-- | objects/object_blob.go | 1 | ||||
| -rw-r--r-- | objects/object_file.go | 25 | ||||
| -rw-r--r-- | objects/object_snapshot.go | 5 | ||||
| -rw-r--r-- | objects/object_tree.go | 38 | ||||
| -rw-r--r-- | objects/properties.go | 13 | ||||
| -rw-r--r-- | objects/properties_test.go | 22 |
7 files changed, 76 insertions, 31 deletions
diff --git a/objects/object.go b/objects/object.go index 1bc9dbf..f9907f2 100644 --- a/objects/object.go +++ b/objects/object.go @@ -23,6 +23,9 @@ var AllObjectTypes = []ObjectType{ OTSnapshot, } +// RawObject describes a serialized object plus it's type header. This is the content that will be saved. +// It is serialized as the type (ObjectType), a space byte, the size of the payload in bytes (encoded as decimal ASCII number), a newline ('\n') character and the payload. +// The encoding of the payload depends on the type. type RawObject struct { Type ObjectType Payload []byte diff --git a/objects/object_blob.go b/objects/object_blob.go index c35cb2a..a6b3f7b 100644 --- a/objects/object_blob.go +++ b/objects/object_blob.go @@ -1,5 +1,6 @@ package objects +// Blob is an object containing raw bytes. It's serialized payload is just that content. type Blob []byte func (b Blob) Type() ObjectType { diff --git a/objects/object_file.go b/objects/object_file.go index 7551193..6143ec2 100644 --- a/objects/object_file.go +++ b/objects/object_file.go @@ -7,16 +7,20 @@ import ( "strconv" ) +// FileFragment describes a fragment of a file. It consists of the ID of a blob and the size of the blob. +// The size is not necessary for reconstruction (the blob object already has a size) +// but it can speed up random access to the whole file by skipping previous fragments. +// It is serialized as `Properties` (see there) with the keys `blob` (ID of blob object) and `size` (decimal size of the blob) type FileFragment struct { Blob ObjectId Size uint64 } -func (ff FileFragment) toProperties() properties { - return properties{"blob": ff.Blob.String(), "size": strconv.FormatUint(ff.Size, 10)} +func (ff FileFragment) toProperties() Properties { + return Properties{"blob": ff.Blob.String(), "size": strconv.FormatUint(ff.Size, 10)} } -func (ff *FileFragment) fromProperties(p properties) error { +func (ff *FileFragment) fromProperties(p Properties) error { blob, ok := p["blob"] if !ok { return errors.New("Field `blob` is missing") @@ -41,6 +45,19 @@ func (a FileFragment) Equals(b FileFragment) bool { return a.Blob.Equals(b.Blob) && a.Size == b.Size } +// File describes a file object which ties together multiple Blob objects to a file. +// It is an ordered list of `FileFragment`s. The referenced blobs concatencated in the given order are the content of the file. +// +// Example: +// +// blob=sha3-256:1111111111111111111111111111111111111111111111111111111111111111&size=123 +// blob=sha3-256:1111111111111111111111111111111111111111111111111111111111111111&size=123 +// blob=sha3-256:2222222222222222222222222222222222222222222222222222222222222222&size=10 +// +// The file described by this object is 123+123+10 = 256 bytes long and consists of the blobs 111... (two times) and 222... +// +// Since the blob IDs and sizes don't change unless the file itself changes and the format will always serialize the same, +// serializing the same file twice results in exactly the same object with exactly the same ID. It will therefor only be stored once. type File []FileFragment func (f File) Type() ObjectType { @@ -72,7 +89,7 @@ func (f *File) FromPayload(payload []byte) error { continue } - props := make(properties) + props := make(Properties) if err := props.UnmarshalText(line); err != nil { return nil } diff --git a/objects/object_snapshot.go b/objects/object_snapshot.go index e86484d..49ca463 100644 --- a/objects/object_snapshot.go +++ b/objects/object_snapshot.go @@ -17,6 +17,11 @@ const ( snapshot_end_line = snapshot_end_marker + "\n" ) +// Snapshot objects describe the state of a directory structure at a given time. It references a tree object by it's ID, +// records the snapshot time and associates it ton an archive (which is just a short freeform text grouping multiple snapshots together). +// A snapshot can optionally contain a comment and can be signed with a gpg key. +// If the snapshot is signed and you trust the signature, you can automatically trust the whole associated file tree, +// since all references are really cryptographic hashes, guaranteeing data integrity. type Snapshot struct { Tree ObjectId Date time.Time diff --git a/objects/object_tree.go b/objects/object_tree.go index 609aa7e..2708aaa 100644 --- a/objects/object_tree.go +++ b/objects/object_tree.go @@ -29,7 +29,7 @@ type TreeEntry interface { User() string Group() string equalContent(TreeEntry) bool - toProperties() properties + toProperties() Properties } func compareTreeEntries(a, b TreeEntry) bool { @@ -62,8 +62,8 @@ func (teb TreeEntryBase) Group() string { return teb.group } -func (teb TreeEntryBase) toProperties() properties { - props := properties{"acl": teb.acl.String()} +func (teb TreeEntryBase) toProperties() Properties { + props := Properties{"acl": teb.acl.String()} if teb.user != "" { props["user"] = teb.user } @@ -93,7 +93,7 @@ func (tef TreeEntryFile) Type() TreeEntryType { return TETFile } -func (tef TreeEntryFile) toProperties() properties { +func (tef TreeEntryFile) toProperties() Properties { props := tef.TreeEntryBase.toProperties() props["ref"] = tef.Ref.String() return props @@ -120,7 +120,7 @@ func (ted TreeEntryDir) Type() TreeEntryType { return TETDir } -func (ted TreeEntryDir) toProperties() properties { +func (ted TreeEntryDir) toProperties() Properties { props := ted.TreeEntryBase.toProperties() props["ref"] = ted.Ref.String() return props @@ -147,7 +147,7 @@ func (tes TreeEntrySymlink) Type() TreeEntryType { return TETSymlink } -func (tes TreeEntrySymlink) toProperties() properties { +func (tes TreeEntrySymlink) toProperties() Properties { props := tes.TreeEntryBase.toProperties() props["target"] = tes.Target return props @@ -158,6 +158,24 @@ func (a TreeEntrySymlink) equalContent(_b TreeEntry) bool { return ok && a.TreeEntryBase.equalContent(b.TreeEntryBase) && a.Target == b.Target } +// Tree objects represent a filesystem tree / directory. +// It contains references to files (See `File`), symlinks and other trees plus their metadata. +// It is serialized as a sorted list of `Property` serializations (seperated by newline '\n'). +// All entries have the property keys "name" and "type" (and optionally "user", "group" and "acl" representing a posix ACL. +// Currently only the execution bit is actually considerer. Choosing posix ACLs gives us the +// possibility to extend the privilege system later). +// Further keys depend on the value of type: +// +// type=file, type=dir => +// +// ref: Holding the ID referencing a file / subtree +// +// type=symlink +// +// target: Holding the (relative) symlink path +// +// The Property format allows easy extension in the future while remaining compatible +// to older versions (they then simply ignore the additional properties). type Tree map[string]TreeEntry func (t Tree) Type() ObjectType { @@ -188,7 +206,7 @@ func (t Tree) Payload() (out []byte) { return } -func getObjectIdFromProps(p properties, key string) (ObjectId, error) { +func getObjectIdFromProps(p Properties, key string) (ObjectId, error) { raw, ok := p[key] if !ok { return ObjectId{}, fmt.Errorf("Missing key: %s", key) @@ -198,7 +216,7 @@ func getObjectIdFromProps(p properties, key string) (ObjectId, error) { return oid, err } -func defaultFileTreeEntryBase(_acl *acl.ACL, props properties) (base TreeEntryBase) { +func defaultFileTreeEntryBase(_acl *acl.ACL, props Properties) (base TreeEntryBase) { base.user = props["user"] base.group = props["group"] if _acl == nil { @@ -209,7 +227,7 @@ func defaultFileTreeEntryBase(_acl *acl.ACL, props properties) (base TreeEntryBa return } -func defaultDirTreeEntryBase(_acl *acl.ACL, props properties) (base TreeEntryBase) { +func defaultDirTreeEntryBase(_acl *acl.ACL, props Properties) (base TreeEntryBase) { base.user = props["user"] base.group = props["group"] if _acl == nil { @@ -229,7 +247,7 @@ func (t Tree) FromPayload(payload []byte) error { continue } - props := make(properties) + props := make(Properties) if err := props.UnmarshalText(line); err != nil { return err } diff --git a/objects/properties.go b/objects/properties.go index 8c92932..2e8c8ac 100644 --- a/objects/properties.go +++ b/objects/properties.go @@ -7,9 +7,10 @@ import ( "sort" ) -// properties are mappings from strings to strings that are encoded as a restricted version of URL query strings -// (only the characters [a-zA-Z0-9.:_-] are allowed, values are ordered by their key) -type properties map[string]string +// Properties are mappings from strings to strings that are encoded as a restricted version of URL query strings +// (only the characters [a-zA-Z0-9.:_-] are allowed, values are ordered by their key. +// This ordering and this character restrictions guarantee reproducible serialization) +type Properties map[string]string // escapePropertyString escapes all bytes not in [a-zA-Z0-9.,:_-] as %XX, where XX represents the hexadecimal value of the byte. // Compatible with URL query strings @@ -29,7 +30,7 @@ func escapePropertyString(s string) []byte { return out } -func (p properties) MarshalText() ([]byte, error) { // Guaranteed to not fail, error is only here to satisfy encoding.TextMarshaler +func (p Properties) MarshalText() ([]byte, error) { // Guaranteed to not fail, error is only here to satisfy encoding.TextMarshaler keys := make([]string, len(p)) i := 0 for k := range p { @@ -57,7 +58,7 @@ func (p properties) MarshalText() ([]byte, error) { // Guaranteed to not fail, e return out, nil } -func (p properties) UnmarshalText(text []byte) error { +func (p Properties) UnmarshalText(text []byte) error { vals, err := url.ParseQuery(string(text)) if err != nil { return err @@ -74,7 +75,7 @@ func (p properties) UnmarshalText(text []byte) error { return nil } -func (a properties) Equals(b properties) bool { +func (a Properties) Equals(b Properties) bool { for k, va := range a { vb, ok := b[k] if !ok || vb != va { diff --git a/objects/properties_test.go b/objects/properties_test.go index 96df5c8..7243ce3 100644 --- a/objects/properties_test.go +++ b/objects/properties_test.go @@ -22,13 +22,13 @@ func TestPropEscape(t *testing.T) { func TestPropertyMarshalling(t *testing.T) { tests := []struct { name string - in properties + in Properties want string }{ - {"empty", properties{}, ""}, - {"single", properties{"foo": "bar"}, "foo=bar"}, - {"simple", properties{"foo": "bar", "bar": "baz"}, "bar=baz&foo=bar"}, - {"escapes", properties{"foo&bar": "%=baz", "?": "!"}, "%3f=%21&foo%26bar=%25%3dbaz"}, + {"empty", Properties{}, ""}, + {"single", Properties{"foo": "bar"}, "foo=bar"}, + {"simple", Properties{"foo": "bar", "bar": "baz"}, "bar=baz&foo=bar"}, + {"escapes", Properties{"foo&bar": "%=baz", "?": "!"}, "%3f=%21&foo%26bar=%25%3dbaz"}, } for _, subtest := range tests { @@ -48,16 +48,16 @@ func TestPropertyUnmarshalling(t *testing.T) { tests := []struct { name string in string - want properties + want Properties }{ - {"empty", "", properties{}}, - {"single", "foo=bar", properties{"foo": "bar"}}, - {"simple", "bar=baz&foo=bar", properties{"foo": "bar", "bar": "baz"}}, - {"escapes", "%3f=%21&foo%26bar=%25%3dbaz", properties{"foo&bar": "%=baz", "?": "!"}}, + {"empty", "", Properties{}}, + {"single", "foo=bar", Properties{"foo": "bar"}}, + {"simple", "bar=baz&foo=bar", Properties{"foo": "bar", "bar": "baz"}}, + {"escapes", "%3f=%21&foo%26bar=%25%3dbaz", Properties{"foo&bar": "%=baz", "?": "!"}}, } for _, subtest := range tests { - have := make(properties) + have := make(Properties) err := have.UnmarshalText([]byte(subtest.in)) if err != nil { |