From 1014cf54546776c411483768cf90734addd4e1d5 Mon Sep 17 00:00:00 2001 From: Ayke van Laethem Date: Sat, 30 Apr 2016 15:10:18 +0200 Subject: Fix warning in Go 1.6: -Wimplicit-function-declaration --- librsync/librsync.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/librsync/librsync.go b/librsync/librsync.go index a3a4e87..234b1f3 100644 --- a/librsync/librsync.go +++ b/librsync/librsync.go @@ -11,6 +11,8 @@ static inline rs_buffers_t* new_rs_buffers() { return (rs_buffers_t*) malloc(sizeof(rs_buffers_t)); } +rs_result patchCallbackGo(void *_patcher, rs_long_t pos, size_t *len, void *_buf); + rs_result patchCallback(void* _patcher, rs_long_t pos, size_t* len, void** _buf) { return patchCallbackGo(_patcher, pos, len, _buf); } -- cgit v1.2.3-70-g09d2 From 4ce8be3db5b6afe8e984367c2aad1f21bed9d47d Mon Sep 17 00:00:00 2001 From: Ayke van Laethem Date: Sat, 30 Apr 2016 16:21:03 +0200 Subject: Do not use Go pointers containing Go pointers in a CGo call https://github.com/golang/proposal/blob/master/design/12416-cgo-pointers.md --- librsync/librsync.go | 14 ++++++++++-- librsync/librsync_callback.go | 4 ++-- librsync/pointers.go | 51 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 65 insertions(+), 4 deletions(-) create mode 100644 librsync/pointers.go diff --git a/librsync/librsync.go b/librsync/librsync.go index 234b1f3..641e480 100644 --- a/librsync/librsync.go +++ b/librsync/librsync.go @@ -275,7 +275,7 @@ func NewDeltaGen(sig Signature, newfile io.Reader) (job *Job, err error) { // Patcher is a job with additional hidden data for patching. // -// IMPORTANT: You still need to Close() this! +// This patcher must be closed after use to free memory. type Patcher struct { *Job basis io.ReaderAt @@ -299,11 +299,21 @@ func NewPatcher(delta io.Reader, basis io.ReaderAt) (job *Patcher, err error) { Job: _job, basis: basis} - job.job = C.rs_patch_begin((*C.rs_copy_cb)(patchCallback), unsafe.Pointer(job)) + id := uintptr(unsafe.Pointer(_job.rsbufs)) // this is a unique, unchanging number (C doesn't change pointers under the hood) + storePatcher(job, id) + job.job = C.rs_patch_begin((*C.rs_copy_cb)(patchCallback), unsafe.Pointer(id)) if job.job == nil { + dropPatcher(id) job.Close() return nil, errors.New("rs_patch_begin failed") } return } + +// Close unreferences memory that the garbage collector would not otherwise be +// able to free. +func (patch *Patcher) Close() error { + dropPatcher(uintptr(unsafe.Pointer(patch.Job.rsbufs))) + return patch.Job.Close() +} diff --git a/librsync/librsync_callback.go b/librsync/librsync_callback.go index c8b9e4f..ace9779 100644 --- a/librsync/librsync_callback.go +++ b/librsync/librsync_callback.go @@ -12,8 +12,8 @@ import ( ) //export patchCallbackGo -func patchCallbackGo(_patcher unsafe.Pointer, pos C.rs_long_t, len *C.size_t, _buf *unsafe.Pointer) C.rs_result { - patcher := (*Patcher)(_patcher) +func patchCallbackGo(_patcher uintptr, pos C.rs_long_t, len *C.size_t, _buf *unsafe.Pointer) C.rs_result { + patcher := getPatcher(_patcher) patcher.buf = make([]byte, int(*len)) n, err := patcher.basis.ReadAt(patcher.buf, int64(pos)) diff --git a/librsync/pointers.go b/librsync/pointers.go new file mode 100644 index 0000000..d14f41f --- /dev/null +++ b/librsync/pointers.go @@ -0,0 +1,51 @@ +package librsync + +import ( + "sync" +) + +// pointerMap holds Go *Patcher objects to pass to C. +// Don't touch this data structure, instead use the storePatcher, getPatcher, +// and dropPatcher functions. +var patcherStore = struct { + lock sync.Mutex + store map[uintptr]*Patcher +}{ + store: make(map[uintptr]*Patcher), +} + +// storePatcher stores the value and returns a reference to it, for use in a CGo +// call. Use the same reference for dropPatcher. C callbacks can use getPatcher +// to get the original value. +func storePatcher(patcher *Patcher, id uintptr) { + patcherStore.lock.Lock() + defer patcherStore.lock.Unlock() + + if _, ok := patcherStore.store[id]; ok { + // Just to be on the safe side. + panic("pointer already stored") + } + patcherStore.store[id] = patcher +} + +// getPatcher returns the value for the reference id. It returns nil if there +// is no such reference. +func getPatcher(id uintptr) *Patcher { + patcherStore.lock.Lock() + defer patcherStore.lock.Unlock() + + return patcherStore.store[id] +} + +// dropPatcher unreferences the value so the garbage collector can free it's +// memory. +func dropPatcher(id uintptr) { + patcherStore.lock.Lock() + defer patcherStore.lock.Unlock() + + if _, ok := patcherStore.store[id]; !ok { + panic("pointer not stored") + } + + delete(patcherStore.store, id) +} -- cgit v1.2.3-70-g09d2 From 181d4286ad8dbca39bb4d43475b31ab9b3c626ee Mon Sep 17 00:00:00 2001 From: Ayke van Laethem Date: Sat, 30 Apr 2016 22:21:45 +0200 Subject: Support the new librsync (version 1.0.0) Compatibility with older versions is kept (tested 0.9.7). --- librsync/librsync.go | 58 +++++++++++++++++++++++++++++++---- librsync/librsync_test.go | 9 +++++- librsync/testdata/random_data.sig.go | 30 +++++++++++++----- librsync/testdata/random_data.sig2 | Bin 0 -> 156 bytes 4 files changed, 83 insertions(+), 14 deletions(-) create mode 100644 librsync/testdata/random_data.sig2 diff --git a/librsync/librsync.go b/librsync/librsync.go index 641e480..1ffc7ec 100644 --- a/librsync/librsync.go +++ b/librsync/librsync.go @@ -6,6 +6,7 @@ package librsync #include #include #include +#include static inline rs_buffers_t* new_rs_buffers() { return (rs_buffers_t*) malloc(sizeof(rs_buffers_t)); @@ -17,6 +18,27 @@ rs_result patchCallback(void* _patcher, rs_long_t pos, size_t* len, void** _buf) return patchCallbackGo(_patcher, pos, len, _buf); } +#ifndef RS_DEFAULT_STRONG_LEN +// librsync >= 1.0.0, 0 is the full size (32 bytes) +#define DEFAULT_STRONG_LEN 0 +#else +// librsync < 1.0.0, using md4 (8 bytes) +#define DEFAULT_STRONG_LEN RS_DEFAULT_STRONG_LEN +#endif + +static inline rs_job_t* sig_begin(size_t new_block_len, size_t strong_sum_len, bool compat) { +#ifndef RS_DEFAULT_STRONG_LEN + // librsync >= 1.0.0, supporting the newer hash function (blake2b) + if (compat) { + return rs_sig_begin(new_block_len, strong_sum_len, RS_MD4_SIG_MAGIC); + } + return rs_sig_begin(new_block_len, strong_sum_len, RS_BLAKE2_SIG_MAGIC); +#else + // not supporting the newer hash function, fall back to the md4 hash + return rs_sig_begin(new_block_len, strong_sum_len); +#endif +} + */ import "C" @@ -32,6 +54,11 @@ const ( outbufSize = 16 * 1024 ) +const ( + DefaultBlockLen = C.RS_DEFAULT_BLOCK_LEN + DefaultStrongLen = C.DEFAULT_STRONG_LEN +) + var ( ErrInputEnded = errors.New("Input ended (possibly unexpected)") ErrBadMagic = errors.New("Bad magic number. Probably not an librsync file.") @@ -74,24 +101,43 @@ func newJob(input io.Reader) (job *Job, err error) { return } -// NewDefaultSignatureGen is like NewSignatureGen, but uses default values for blocklen and stronglen. +// Config sets parameters for NewSignatureGen. May be the zero value for default +// values. +type Config struct { + BlockLen uint // length of a block, e.g. 2048 + StrongLen uint // length of a strong hash, e.g. 32 or 0 + CompatMD4 bool // enable for compatibility with librsync < 1.0.0 +} + +func (c *Config) setup() { + if c.BlockLen == 0 { + c.BlockLen = DefaultBlockLen + } + if c.StrongLen == 0 { + c.StrongLen = DefaultStrongLen + } +} + +// NewDefaultSignatureGen is like NewSignatureGen, but uses the default +// configuration. func NewDefaultSignatureGen(basis io.Reader) (job *Job, err error) { - job, err = NewSignatureGen(C.RS_DEFAULT_BLOCK_LEN, C.RS_DEFAULT_STRONG_LEN, basis) + job, err = NewSignatureGen(Config{}, basis) return } // NewSignatureGen creates a signature generation job. // -// blocklen is the length of a block. -// stronglen is the length of the stong hash. +// config is a Config object for more options. // basis is an io.Reader that provides data of the basis file. -func NewSignatureGen(blocklen, stronglen uint, basis io.Reader) (job *Job, err error) { +func NewSignatureGen(config Config, basis io.Reader) (job *Job, err error) { job, err = newJob(basis) if err != nil { return } - job.job = C.rs_sig_begin(C.size_t(blocklen), C.size_t(stronglen)) + config.setup() + + job.job = C.sig_begin(C.size_t(config.BlockLen), C.size_t(config.StrongLen), C.bool(config.CompatMD4)) if job.job == nil { job.Close() return nil, errors.New("rs_sig_begin failed") diff --git a/librsync/librsync_test.go b/librsync/librsync_test.go index 7c6a278..c291fac 100644 --- a/librsync/librsync_test.go +++ b/librsync/librsync_test.go @@ -22,7 +22,14 @@ func TestSignatureDeltaPatch(t *testing.T) { t.Fatalf("Creating the signature failed: %s", err) } - if !bytes.Equal(sigbuf.Bytes(), testdata.RandomDataSig()) { + matches := false + // Check both possible signatures. + for _, sigcheck := range testdata.RandomDataSig() { + if bytes.Equal(sigbuf.Bytes(), sigcheck) { + matches = true + } + } + if !matches { if path, err := dump(sigbuf); err == nil { t.Fatalf("Signatures do not match. Generated signature dumped to %s", path) } else { diff --git a/librsync/testdata/random_data.sig.go b/librsync/testdata/random_data.sig.go index 8822edf..e5e57a3 100644 --- a/librsync/testdata/random_data.sig.go +++ b/librsync/testdata/random_data.sig.go @@ -1,12 +1,28 @@ package testdata -func RandomDataSig() []byte { +func RandomDataSig() [][]byte { // Was generated with `rdiff signature random_data random_data.sig` - return []byte{ - 0x72, 0x73, 0x01, 0x36, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x08, - 0x60, 0xcb, 0xf7, 0x64, 0x2e, 0xbe, 0x39, 0x5e, 0x1a, 0xf2, 0x77, 0x07, - 0xf5, 0x8f, 0xf3, 0xdc, 0x1d, 0xcc, 0xe0, 0x3b, 0x3b, 0xc9, 0x7e, 0x0d, - 0x2e, 0xcf, 0xed, 0xc5, 0x09, 0x1f, 0xbb, 0x75, 0x27, 0xbb, 0x46, 0xa9, - 0x86, 0x26, 0x04, 0xa1, 0x9d, 0x4d, 0xa3, 0x7a, 0xfa, 0xdd, 0x5d, 0xc5, + return [][]byte{ + { + 0x72, 0x73, 0x01, 0x36, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x08, + 0x60, 0xcb, 0xf7, 0x64, 0x2e, 0xbe, 0x39, 0x5e, 0x1a, 0xf2, 0x77, 0x07, + 0xf5, 0x8f, 0xf3, 0xdc, 0x1d, 0xcc, 0xe0, 0x3b, 0x3b, 0xc9, 0x7e, 0x0d, + 0x2e, 0xcf, 0xed, 0xc5, 0x09, 0x1f, 0xbb, 0x75, 0x27, 0xbb, 0x46, 0xa9, + 0x86, 0x26, 0x04, 0xa1, 0x9d, 0x4d, 0xa3, 0x7a, 0xfa, 0xdd, 0x5d, 0xc5, + }, { + 0x72, 0x73, 0x01, 0x37, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x20, + 0x60, 0xcb, 0xf7, 0x64, 0x3a, 0x2b, 0x95, 0x58, 0xad, 0xd8, 0xa9, 0x71, + 0x70, 0x7d, 0x38, 0x64, 0xd5, 0xff, 0xa2, 0xc1, 0xae, 0xdf, 0x62, 0x76, + 0x7a, 0x84, 0xf8, 0xa5, 0xc7, 0x68, 0xd9, 0xbd, 0x04, 0xea, 0x09, 0x2b, + 0xf5, 0x8f, 0xf3, 0xdc, 0x13, 0x67, 0x5e, 0x50, 0xd9, 0xd3, 0x38, 0x47, + 0x1c, 0xb9, 0x27, 0x60, 0x26, 0x8d, 0x23, 0x39, 0x47, 0x0e, 0x57, 0x2c, + 0xfd, 0x2a, 0x27, 0xf1, 0xab, 0xee, 0x88, 0xe6, 0x74, 0x60, 0x78, 0xf8, + 0x2e, 0xcf, 0xed, 0xc5, 0x3b, 0xef, 0xab, 0x41, 0x13, 0x93, 0x0e, 0x26, + 0x38, 0x79, 0xc6, 0xed, 0x4d, 0xce, 0x65, 0xcd, 0x02, 0x5f, 0x2f, 0x19, + 0x42, 0x4a, 0x2d, 0xac, 0x8f, 0xb1, 0x40, 0xa2, 0x3c, 0xfc, 0x90, 0x66, + 0x86, 0x26, 0x04, 0xa1, 0x08, 0x25, 0xe4, 0x2c, 0xf7, 0x77, 0xa8, 0x7e, + 0x8f, 0xbf, 0xe9, 0xdf, 0x7a, 0x2d, 0x89, 0xc3, 0xe8, 0x15, 0x65, 0x46, + 0xa2, 0xe0, 0x33, 0x38, 0xc9, 0x73, 0xf0, 0xc3, 0x20, 0x79, 0x5a, 0x11, + }, } } diff --git a/librsync/testdata/random_data.sig2 b/librsync/testdata/random_data.sig2 new file mode 100644 index 0000000..684eacc Binary files /dev/null and b/librsync/testdata/random_data.sig2 differ -- cgit v1.2.3-70-g09d2