Initial commit (code only without large binaries)

EdgeNode/internal/compressions/cbrotli/cgo.go

@@ -0,0 +1,14 @@
+// Copyright 2017 Google Inc. All Rights Reserved.
+//
+// Distributed under MIT license.
+// See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+
+package cbrotli
+
+// Inform golang build system that it should link brotli libraries.
+
+// #cgo CFLAGS: -O2
+// #cgo LDFLAGS: -lbrotlicommon
+// #cgo LDFLAGS: -lbrotlidec
+// #cgo LDFLAGS: -lbrotlienc
+import "C"

EdgeNode/internal/compressions/cbrotli/reader.go

@@ -0,0 +1,163 @@
+// Copyright 2016 Google Inc. All Rights Reserved.
+//
+// Distributed under MIT license.
+// See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+
+// Package cbrotli compresses and decompresses data with C-Brotli library.
+package cbrotli
+
+/*
+#include <stddef.h>
+#include <stdint.h>
+
+#include <brotli/decode.h>
+
+static BrotliDecoderResult DecompressStream(BrotliDecoderState* s,
+                                            uint8_t* out, size_t out_len,
+                                            const uint8_t* in, size_t in_len,
+                                            size_t* bytes_written,
+                                            size_t* bytes_consumed) {
+  size_t in_remaining = in_len;
+  size_t out_remaining = out_len;
+  BrotliDecoderResult result = BrotliDecoderDecompressStream(
+      s, &in_remaining, &in, &out_remaining, &out, NULL);
+  *bytes_written = out_len - out_remaining;
+  *bytes_consumed = in_len - in_remaining;
+  return result;
+}
+*/
+import "C"
+
+import (
+	"bytes"
+	"errors"
+	"io"
+)
+
+type decodeError C.BrotliDecoderErrorCode
+
+func (err decodeError) Error() string {
+	return "cbrotli: " +
+		C.GoString(C.BrotliDecoderErrorString(C.BrotliDecoderErrorCode(err)))
+}
+
+var errExcessiveInput = errors.New("cbrotli: excessive input")
+var errInvalidState = errors.New("cbrotli: invalid state")
+var errReaderClosed = errors.New("cbrotli: Reader is closed")
+
+// Reader implements io.ReadCloser by reading Brotli-encoded data from an
+// underlying Reader.
+type Reader struct {
+	src   io.Reader
+	state *C.BrotliDecoderState
+	buf   []byte // scratch space for reading from src
+	in    []byte // current chunk to decode; usually aliases buf
+}
+
+// readBufSize is a "good" buffer size that avoids excessive round-trips
+// between C and Go but doesn't waste too much memory on buffering.
+// It is arbitrarily chosen to be equal to the constant used in io.Copy.
+const readBufSize = 32 * 1024
+
+// NewReader initializes new Reader instance.
+// Close MUST be called to free resources.
+func NewReader(src io.Reader) *Reader {
+	return &Reader{
+		src:   src,
+		state: C.BrotliDecoderCreateInstance(nil, nil, nil),
+		buf:   make([]byte, readBufSize),
+	}
+}
+
+// Close implements io.Closer. Close MUST be invoked to free native resources.
+func (r *Reader) Close() error {
+	if r.state == nil {
+		return errReaderClosed
+	}
+	// Close despite the state; i.e. there might be some unread decoded data.
+	C.BrotliDecoderDestroyInstance(r.state)
+	r.state = nil
+	return nil
+}
+
+func (r *Reader) Read(p []byte) (n int, err error) {
+	if r.state == nil {
+		return 0, errReaderClosed
+	}
+	if int(C.BrotliDecoderHasMoreOutput(r.state)) == 0 && len(r.in) == 0 {
+		m, readErr := r.src.Read(r.buf)
+		if m == 0 {
+			// If readErr is `nil`, we just proxy underlying stream behavior.
+			return 0, readErr
+		}
+		r.in = r.buf[:m]
+	}
+
+	if len(p) == 0 {
+		return 0, nil
+	}
+
+	for {
+		var written, consumed C.size_t
+		var data *C.uint8_t
+		if len(r.in) != 0 {
+			data = (*C.uint8_t)(&r.in[0])
+		}
+		result := C.DecompressStream(r.state,
+			(*C.uint8_t)(&p[0]), C.size_t(len(p)),
+			data, C.size_t(len(r.in)),
+			&written, &consumed)
+		r.in = r.in[int(consumed):]
+		n = int(written)
+
+		switch result {
+		case C.BROTLI_DECODER_RESULT_SUCCESS:
+			if len(r.in) > 0 {
+				return n, errExcessiveInput
+			}
+			return n, nil
+		case C.BROTLI_DECODER_RESULT_ERROR:
+			return n, decodeError(C.BrotliDecoderGetErrorCode(r.state))
+		case C.BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT:
+			if n == 0 {
+				return 0, io.ErrShortBuffer
+			}
+			return n, nil
+		case C.BROTLI_DECODER_NEEDS_MORE_INPUT:
+		}
+
+		if len(r.in) != 0 {
+			return 0, errInvalidState
+		}
+
+		// Calling r.src.Read may block. Don't block if we have data to return.
+		if n > 0 {
+			return n, nil
+		}
+
+		// Top off the buffer.
+		encN, err := r.src.Read(r.buf)
+		if encN == 0 {
+			// Not enough data to complete decoding.
+			if err == io.EOF {
+				return 0, io.ErrUnexpectedEOF
+			}
+			return 0, err
+		}
+		r.in = r.buf[:encN]
+	}
+
+	return n, nil
+}
+
+// Decode decodes Brotli encoded data.
+func Decode(encodedData []byte) ([]byte, error) {
+	r := &Reader{
+		src:   bytes.NewReader(nil),
+		state: C.BrotliDecoderCreateInstance(nil, nil, nil),
+		buf:   make([]byte, 4), // arbitrarily small but nonzero so that r.src.Read returns io.EOF
+		in:    encodedData,
+	}
+	defer r.Close()
+	return io.ReadAll(r)
+}

EdgeNode/internal/compressions/cbrotli/writer.go

@@ -0,0 +1,162 @@
+// Copyright 2016 Google Inc. All Rights Reserved.
+//
+// Distributed under MIT license.
+// See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+
+package cbrotli
+
+/*
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+
+#include <brotli/encode.h>
+
+struct CompressStreamResult {
+  size_t bytes_consumed;
+  const uint8_t* output_data;
+  size_t output_data_size;
+  int success;
+  int has_more;
+};
+
+static struct CompressStreamResult CompressStream(
+    BrotliEncoderState* s, BrotliEncoderOperation op,
+    const uint8_t* data, size_t data_size) {
+  struct CompressStreamResult result;
+  size_t available_in = data_size;
+  const uint8_t* next_in = data;
+  size_t available_out = 0;
+  result.success = BrotliEncoderCompressStream(s, op,
+      &available_in, &next_in, &available_out, 0, 0) ? 1 : 0;
+  result.bytes_consumed = data_size - available_in;
+  result.output_data = 0;
+  result.output_data_size = 0;
+  if (result.success) {
+    result.output_data = BrotliEncoderTakeOutput(s, &result.output_data_size);
+  }
+  result.has_more = BrotliEncoderHasMoreOutput(s) ? 1 : 0;
+  return result;
+}
+*/
+import "C"
+
+import (
+	"bytes"
+	"errors"
+	"io"
+	"unsafe"
+)
+
+// WriterOptions configures Writer.
+type WriterOptions struct {
+	// Quality controls the compression-speed vs compression-density trade-offs.
+	// The higher the quality, the slower the compression. Range is 0 to 11.
+	Quality int
+	// LGWin is the base 2 logarithm of the sliding window size.
+	// Range is 10 to 24. 0 indicates automatic configuration based on Quality.
+	LGWin int
+}
+
+// Writer implements io.WriteCloser by writing Brotli-encoded data to an
+// underlying Writer.
+type Writer struct {
+	dst   io.Writer
+	state *C.BrotliEncoderState
+}
+
+var (
+	errEncode       = errors.New("cbrotli: encode error")
+	errWriterClosed = errors.New("cbrotli: Writer is closed")
+)
+
+// NewWriter initializes new Writer instance.
+// Close MUST be called to free resources.
+func NewWriter(dst io.Writer, options WriterOptions) *Writer {
+	state := C.BrotliEncoderCreateInstance(nil, nil, nil)
+	C.BrotliEncoderSetParameter(
+		state, C.BROTLI_PARAM_QUALITY, (C.uint32_t)(options.Quality))
+	if options.LGWin > 0 {
+		C.BrotliEncoderSetParameter(
+			state, C.BROTLI_PARAM_LGWIN, (C.uint32_t)(options.LGWin))
+	}
+	return &Writer{
+		dst:   dst,
+		state: state,
+	}
+}
+
+func (w *Writer) writeChunk(p []byte, op C.BrotliEncoderOperation) (n int, err error) {
+	if w.state == nil {
+		return 0, errWriterClosed
+	}
+
+	for {
+		var data *C.uint8_t
+		if len(p) != 0 {
+			data = (*C.uint8_t)(&p[0])
+		}
+		result := C.CompressStream(w.state, op, data, C.size_t(len(p)))
+		if result.success == 0 {
+			return n, errEncode
+		}
+		p = p[int(result.bytes_consumed):]
+		n += int(result.bytes_consumed)
+
+		length := int(result.output_data_size)
+		if length != 0 {
+			// It is a workaround for non-copying-wrapping of native memory.
+			// C-encoder never pushes output block longer than ((2 << 25) + 502).
+			// TODO(eustas): use natural wrapper, when it becomes available, see
+			//               https://golang.org/issue/13656.
+			output := (*[1 << 30]byte)(unsafe.Pointer(result.output_data))[:length:length]
+			_, err = w.dst.Write(output)
+			if err != nil {
+				return n, err
+			}
+		}
+		if len(p) == 0 && result.has_more == 0 {
+			return n, nil
+		}
+	}
+}
+
+// Flush outputs encoded data for all input provided to Write. The resulting
+// output can be decoded to match all input before Flush, but the stream is
+// not yet complete until after Close.
+// Flush has a negative impact on compression.
+func (w *Writer) Flush() error {
+	_, err := w.writeChunk(nil, C.BROTLI_OPERATION_FLUSH)
+	return err
+}
+
+// Close flushes remaining data to the decorated writer and frees C resources.
+func (w *Writer) Close() error {
+	if w.state == nil {
+		return nil
+	}
+
+	// If stream is already closed, it is reported by `writeChunk`.
+	_, err := w.writeChunk(nil, C.BROTLI_OPERATION_FINISH)
+	// C-Brotli tolerates `nil` pointer here.
+	C.BrotliEncoderDestroyInstance(w.state)
+	w.state = nil
+	return err
+}
+
+// Write implements io.Writer. Flush or Close must be called to ensure that the
+// encoded bytes are actually flushed to the underlying Writer.
+func (w *Writer) Write(p []byte) (n int, err error) {
+	return w.writeChunk(p, C.BROTLI_OPERATION_PROCESS)
+}
+
+// Encode returns content encoded with Brotli.
+func Encode(content []byte, options WriterOptions) ([]byte, error) {
+	var buf bytes.Buffer
+	writer := NewWriter(&buf, options)
+	_, err := writer.Write(content)
+	if closeErr := writer.Close(); err == nil {
+		err = closeErr
+	}
+	return buf.Bytes(), err
+}