commit b210f39f074eef565d2f2d23e95547ebb42d2ab2 Author: icedream Date: Tue Feb 21 08:26:46 2017 +0100 Initial commit. diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..2d3d068 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "bsdiff"] + path = bsdiff + url = https://github.com/mendsley/bsdiff.git diff --git a/bsdiff b/bsdiff new file mode 160000 index 0000000..7d70d8f --- /dev/null +++ b/bsdiff @@ -0,0 +1 @@ +Subproject commit 7d70d8f4ff48345bc76e314c9d98da91f78873fa diff --git a/cmd/bsdiff/main.go b/cmd/bsdiff/main.go new file mode 100644 index 0000000..77cf81a --- /dev/null +++ b/cmd/bsdiff/main.go @@ -0,0 +1,31 @@ +package main + +import ( + "log" + "os" + + "github.com/icedream/go-bsdiff" +) + +func must(err error) { + if err == nil { + return + } + + log.Fatal(err) +} + +func main() { + oldFile, err := os.Open("test.txt") + must(err) + + newFile, err := os.Open("test_new.txt") + must(err) + + patchFile, err := os.Create("test.patch") + must(err) + + must(bsdiff.Diff(oldFile, newFile, patchFile)) + + log.Println("Done.") +} diff --git a/diff.go b/diff.go new file mode 100644 index 0000000..5e1de99 --- /dev/null +++ b/diff.go @@ -0,0 +1,33 @@ +package bsdiff + +import ( + "io" + "io/ioutil" + + "github.com/dsnet/compress/bzip2" + "github.com/icedream/go-bsdiff/internal/native" +) + +func Diff(oldReader, newReader io.Reader, patchWriter io.Writer) (err error) { + oldBytes, err := ioutil.ReadAll(oldReader) + if err != nil { + return + } + newBytes, err := ioutil.ReadAll(newReader) + if err != nil { + return + } + + if err = writeHeader(patchWriter, uint64(len(newBytes))); err != nil { + return + } + + // Compression + bz2Writer, err := bzip2.NewWriter(patchWriter, nil) + if err != nil { + return + } + defer bz2Writer.Close() + + return native.Diff(oldBytes, newBytes, bz2Writer) +} diff --git a/internal/native/cgo.c b/internal/native/cgo.c new file mode 100644 index 0000000..576101c --- /dev/null +++ b/internal/native/cgo.c @@ -0,0 +1,56 @@ +#include "cgo.h" + +#include "bsdiff.h" +#include "bspatch.h" + +extern int cgo_write_buffer(int bufferIndex, void* buf, int size); + +int cgo_write(struct bsdiff_stream* stream, + const void* buf, int size) { + struct buffer_table_index* bufferEntry; + + bufferEntry = (struct buffer_table_index*)stream->opaque; + + return cgo_write_buffer(bufferEntry->index, (void*)buf, size); +} + +extern int cgo_read_buffer(int bufferIndex, void* buf, int size); + +int cgo_read(const struct bspatch_stream* stream, + void* buf, int size) { + struct buffer_table_index* bufferEntry; + + bufferEntry = (struct buffer_table_index*)stream->opaque; + + return cgo_read_buffer(bufferEntry->index, buf, size) ; +} + +int bsdiff_cgo(uint8_t* oldptr, int64_t oldsize, + uint8_t* newptr, int64_t newsize, + int bufferIndex) +{ + struct bsdiff_stream stream; + stream.malloc = malloc; + stream.free = free; + stream.write = cgo_write; + + struct buffer_table_index bufferEntry; + bufferEntry.index = bufferIndex; + stream.opaque = &bufferEntry; + + return bsdiff(oldptr, oldsize, newptr, newsize, &stream); +} + +int bspatch_cgo(uint8_t* oldptr, int64_t oldsize, + uint8_t* newptr, int64_t newsize, + int bufferIndex) +{ + struct bspatch_stream stream; + stream.read = cgo_read; + + struct buffer_table_index bufferEntry; + bufferEntry.index = bufferIndex; + stream.opaque = &bufferEntry; + + return bspatch(oldptr, oldsize, newptr, newsize, &stream); +} \ No newline at end of file diff --git a/internal/native/cgo.h b/internal/native/cgo.h new file mode 100644 index 0000000..cf692e8 --- /dev/null +++ b/internal/native/cgo.h @@ -0,0 +1,15 @@ +#include +#include "stdint.h" + +struct buffer_table_index +{ + int index; +}; + +int bsdiff_cgo(uint8_t* oldptr, int64_t oldsize, + uint8_t* newptr, int64_t newsize, + int bufferIndex); + +int bspatch_cgo(uint8_t* oldptr, int64_t oldsize, + uint8_t* newptr, int64_t newsize, + int bufferIndex); diff --git a/internal/native/cgo_read.go b/internal/native/cgo_read.go new file mode 100644 index 0000000..4e4967a --- /dev/null +++ b/internal/native/cgo_read.go @@ -0,0 +1,43 @@ +package native + +/* +#include "bspatch.h" +*/ +import "C" +import ( + "io" + "log" + "unsafe" +) + +//export cgo_read_buffer +func cgo_read_buffer(bufferIndex C.int, + bufPtr unsafe.Pointer, length C.int) C.int { + goLength := int(length) + + if goLength == 0 { + return 0 + } + + sourceBuffer := readers.Get(int(bufferIndex)) + targetBuffer := cPtrToSlice(bufPtr, goLength) + + errCode := 0 + offset := 0 + for offset < goLength { + n, err := sourceBuffer.Read(targetBuffer) + + if err == io.EOF { + break + } else if err != nil { + log.Println("cgo_read_buffer failed:", err) + errCode = 1 + break + } + + offset += n + targetBuffer = targetBuffer[n:] + } + + return C.int(errCode) +} diff --git a/internal/native/cgo_write.go b/internal/native/cgo_write.go new file mode 100644 index 0000000..1a4e6bd --- /dev/null +++ b/internal/native/cgo_write.go @@ -0,0 +1,18 @@ +package native + +/* +#include "bsdiff.h" +*/ +import "C" +import "unsafe" + +//export cgo_write_buffer +func cgo_write_buffer(bufferIndex C.int, + dataPtr unsafe.Pointer, size C.int) C.int { + buffer := writers.Get(int(bufferIndex)) + errCode := 0 + if _, err := buffer.Write(cPtrToSlice(dataPtr, int(size))); err != nil { + errCode = 1 + } + return C.int(errCode) +} diff --git a/internal/native/diff.go b/internal/native/diff.go new file mode 100644 index 0000000..97b78e5 --- /dev/null +++ b/internal/native/diff.go @@ -0,0 +1,28 @@ +package native + +/* +#cgo CFLAGS: -I../../bsdiff + +#include "bsdiff.h" +#include "cgo.h" +*/ +import "C" +import ( + "errors" + "io" +) + +func Diff(oldbytes, newbytes []byte, patch io.Writer) (err error) { + oldptr, oldsize := bytesToUint8PtrAndSize(oldbytes) + newptr, newsize := bytesToUint8PtrAndSize(newbytes) + + bufferIndex := writers.Add(patch) + + errCode := int(C.bsdiff_cgo(oldptr, oldsize, newptr, newsize, C.int(bufferIndex))) + if errCode != 0 { + err = errors.New("bsdiff failed") + return + } + + return +} diff --git a/internal/native/ext_bsdiff.c b/internal/native/ext_bsdiff.c new file mode 100644 index 0000000..a9558c1 --- /dev/null +++ b/internal/native/ext_bsdiff.c @@ -0,0 +1,2 @@ +#include "bsdiff.c" +#include "bspatch.c" \ No newline at end of file diff --git a/internal/native/native.go b/internal/native/native.go new file mode 100644 index 0000000..8a9348f --- /dev/null +++ b/internal/native/native.go @@ -0,0 +1,31 @@ +package native + +/* +#include +*/ +import "C" +import ( + "reflect" + "unsafe" +) + +var ( + writers = writerTable{} + readers = readerTable{} +) + +func bytesToUint8PtrAndSize(bytes []byte) (ptr *C.uint8_t, size C.int64_t) { + ptr = (*C.uint8_t)(unsafe.Pointer(&bytes[0])) + size = C.int64_t(int64(len(bytes))) + return +} + +func cPtrToSlice(ptr unsafe.Pointer, size int) []byte { + var slice []byte + sliceHeader := (*reflect.SliceHeader)(unsafe.Pointer(&slice)) + sliceHeader.Cap = size + sliceHeader.Len = size + sliceHeader.Data = uintptr(ptr) + + return slice +} diff --git a/internal/native/patch.go b/internal/native/patch.go new file mode 100644 index 0000000..f797cf5 --- /dev/null +++ b/internal/native/patch.go @@ -0,0 +1,29 @@ +package native + +/* +#cgo CFLAGS: -I../../bsdiff + +#include "bspatch.h" +#include "cgo.h" +*/ +import "C" +import ( + "errors" + "io" + "strconv" +) + +func Patch(oldbytes, newbytes []byte, patch io.Reader) (err error) { + oldptr, oldsize := bytesToUint8PtrAndSize(oldbytes) + newptr, newsize := bytesToUint8PtrAndSize(newbytes) + + bufferIndex := readers.Add(patch) + + errCode := int(C.bspatch_cgo(oldptr, oldsize, newptr, newsize, C.int(bufferIndex))) + if errCode != 0 { + err = errors.New("bspatch failed with code " + strconv.Itoa(errCode)) + return + } + + return +} diff --git a/internal/native/table_reader.go b/internal/native/table_reader.go new file mode 100644 index 0000000..620e683 --- /dev/null +++ b/internal/native/table_reader.go @@ -0,0 +1,37 @@ +package native + +import ( + "io" + "sync" +) + +type readerTable struct { + nextIndex int + table map[int]io.Reader + mutex sync.Mutex +} + +func (bt *readerTable) Add(reader io.Reader) (index int) { + bt.mutex.Lock() + defer bt.mutex.Unlock() + + if bt.table == nil { + bt.table = map[int]io.Reader{} + } + + index = bt.nextIndex + bt.table[index] = reader + + // TODO - Handle int overflow + + bt.nextIndex++ + + return +} + +func (bt *readerTable) Get(index int) io.Reader { + bt.mutex.Lock() + defer bt.mutex.Unlock() + + return bt.table[index] +} diff --git a/internal/native/table_writer.go b/internal/native/table_writer.go new file mode 100644 index 0000000..42b57c2 --- /dev/null +++ b/internal/native/table_writer.go @@ -0,0 +1,37 @@ +package native + +import ( + "io" + "sync" +) + +type writerTable struct { + nextIndex int + table map[int]io.Writer + mutex sync.Mutex +} + +func (bt *writerTable) Add(writer io.Writer) (index int) { + bt.mutex.Lock() + defer bt.mutex.Unlock() + + if bt.table == nil { + bt.table = map[int]io.Writer{} + } + + index = bt.nextIndex + bt.table[index] = writer + + // TODO - Handle int overflow + + bt.nextIndex++ + + return +} + +func (bt *writerTable) Get(index int) io.Writer { + bt.mutex.Lock() + defer bt.mutex.Unlock() + + return bt.table[index] +} diff --git a/magic.go b/magic.go new file mode 100644 index 0000000..b64530e --- /dev/null +++ b/magic.go @@ -0,0 +1,39 @@ +package bsdiff + +import ( + "encoding/binary" + "errors" + "io" +) + +var ( + ErrInvalidMagic = errors.New("Invalid magic") + + sizeEncoding = binary.BigEndian + + magicText = []byte("ENDSLEY/BSDIFF43") +) + +func writeHeader(w io.Writer, size uint64) (err error) { + if _, err = w.Write(magicText); err != nil { + return + } + err = binary.Write(w, sizeEncoding, size) + return +} + +func readHeader(r io.Reader) (size uint64, err error) { + magicBuf := make([]byte, len(magicText)) + n, err := r.Read(magicBuf) + if err != nil { + return + } + if n < len(magicText) { + err = ErrInvalidMagic + return + } + + err = binary.Read(r, sizeEncoding, &size) + + return +} diff --git a/patch.go b/patch.go new file mode 100644 index 0000000..f2a0dde --- /dev/null +++ b/patch.go @@ -0,0 +1,33 @@ +package bsdiff + +import ( + "compress/bzip2" + "io" + "io/ioutil" + "log" + + "github.com/icedream/go-bsdiff/internal/native" +) + +func Patch(oldReader io.Reader, newWriter io.Writer, patchReader io.Reader) (err error) { + oldBytes, err := ioutil.ReadAll(oldReader) + if err != nil { + return + } + + newLen, err := readHeader(patchReader) + if err != nil { + return + } + newBytes := make([]byte, newLen) + + log.Printf("Going to create a file of %d bytes.", newLen) + + // Decompression + bz2Reader := bzip2.NewReader(patchReader) + + err = native.Patch(oldBytes, newBytes, bz2Reader) + + newWriter.Write(newBytes) + return +}