From 09cf71869cec48dd79483c65c190f7b79a442da8 Mon Sep 17 00:00:00 2001 From: Quim Rovira Date: Sun, 14 Aug 2016 11:31:10 +0200 Subject: [PATCH] Initial brotli compressor bindings This provides both one-shot and streaming support, but I haven't yet tested this thoroughly in terms of leaks and performance. The code tries to adhere as much as possible to the rest of the module conventions. --- Brotli.xs | 137 +++++++++++++++++++++++++++++++++ lib/IO/Compress/Brotli.pm | 154 +++++++++++++++++++++++++++++++++++++- 2 files changed, 290 insertions(+), 1 deletion(-) diff --git a/Brotli.xs b/Brotli.xs index 4378e97..89167f2 100644 --- a/Brotli.xs +++ b/Brotli.xs @@ -6,6 +6,7 @@ #include "ppport.h" #include +#include #include #define BUFFER_SIZE 1048576 @@ -85,3 +86,139 @@ void BrotliDecoderSetCustomDictionary(state, dict) CODE: data = SvPV(dict, size); BrotliDecoderSetCustomDictionary((BrotliDecoderState*) SvIV(state), size, data); + + +MODULE = IO::Compress::Brotli PACKAGE = IO::Compress::Brotli +PROTOTYPES: ENABLE + +SV* bro(buffer, quality=BROTLI_DEFAULT_QUALITY, lgwin=BROTLI_DEFAULT_WINDOW) + SV* buffer + U32 quality + U32 lgwin + PREINIT: + size_t encoded_size; + STRLEN decoded_size; + uint8_t *encoded_buffer, *decoded_buffer; + BROTLI_BOOL result; + CODE: + if( quality < BROTLI_MIN_QUALITY || quality > BROTLI_MAX_QUALITY ) { + croak("Invalid quality value"); + } + decoded_buffer = (uint8_t*) SvPV(buffer, decoded_size); + encoded_size = BrotliEncoderMaxCompressedSize(decoded_size); + if(!encoded_size){ + croak("Compressed size overflow"); + } + Newx(encoded_buffer, encoded_size+1, uint8_t); + result = BrotliEncoderCompress( quality, + lgwin, + BROTLI_DEFAULT_MODE, + decoded_size, + decoded_buffer, + &encoded_size, + encoded_buffer ); + if(!result){ + croak("Error in BrotliEncoderCompress"); + } + encoded_buffer[encoded_size]=0; + RETVAL = newSV(0); + sv_usepvn_flags(RETVAL, encoded_buffer, encoded_size, SV_SMAGIC | SV_HAS_TRAILING_NUL); + OUTPUT: + RETVAL + +SV* BrotliEncoderCreateInstance() + CODE: + RETVAL = newSViv((IV)BrotliEncoderCreateInstance(NULL, NULL, NULL)); + OUTPUT: + RETVAL + +SV* BrotliEncoderSetWindow(state, window) + SV* state + U32 window + CODE: + if( BrotliEncoderSetParameter((BrotliEncoderState*) SvIV(state), BROTLI_PARAM_LGWIN, window) ) + RETVAL = newSVuv(1); + else + RETVAL = newSVuv(0); + OUTPUT: + RETVAL + +SV* BrotliEncoderSetQuality(state, quality) + SV* state + U32 quality + CODE: + if( quality < BROTLI_MIN_QUALITY || quality > BROTLI_MAX_QUALITY ) { + croak("Invalid quality value"); + } + if( BrotliEncoderSetParameter((BrotliEncoderState*) SvIV(state), BROTLI_PARAM_QUALITY, quality) ) + RETVAL = newSVuv(1); + else + RETVAL = newSVuv(0); + OUTPUT: + RETVAL + +SV* BrotliEncoderSetMode(state, mode) + SV* state + U32 mode + CODE: + if( BrotliEncoderSetParameter((BrotliEncoderState*) SvIV(state), BROTLI_PARAM_MODE, mode) ) + RETVAL = newSVuv(1); + else + RETVAL = newSVuv(0); + OUTPUT: + RETVAL + +SV* BrotliEncoderCompressStream(state, in, op) + SV* state + SV* in + U8 op + PREINIT: + uint8_t *next_in, *next_out, *buffer; + size_t available_in, available_out; + BROTLI_BOOL result; + CODE: + next_in = (uint8_t*) SvPV(in, available_in); + Newx(buffer, BUFFER_SIZE, uint8_t); + RETVAL = newSVpv("", 0); + while(1) { + next_out = buffer; + available_out = BUFFER_SIZE; + result = BrotliEncoderCompressStream( (BrotliEncoderState*) SvIV(state), + (BrotliEncoderOperation) op, + &available_in, + (const uint8_t**) &next_in, + &available_out, + &next_out, + NULL ); + if(!result) { + Safefree(buffer); + croak("Error in BrotliEncoderCompressStream"); + } + + if( available_out != BUFFER_SIZE ) { + sv_catpvn(RETVAL, (const char*)buffer, BUFFER_SIZE-available_out); + } + + if( + BrotliEncoderIsFinished((BrotliEncoderState*) SvIV(state)) || + (!available_in && !BrotliEncoderHasMoreOutput((BrotliEncoderState*) SvIV(state))) + ) break; + } + Safefree(buffer); + OUTPUT: + RETVAL + +void BrotliEncoderDestroyInstance(state) + SV* state + CODE: + BrotliEncoderDestroyInstance((BrotliEncoderState*)SvIV(state)); + +void BrotliEncoderSetCustomDictionary(state, dict) + SV* state + SV* dict + PREINIT: + size_t size; + uint8_t *data; + CODE: + data = SvPV(dict, size); + BrotliEncoderSetCustomDictionary((BrotliEncoderState*) SvIV(state), size, data); diff --git a/lib/IO/Compress/Brotli.pm b/lib/IO/Compress/Brotli.pm index aded86b..9117f70 100644 --- a/lib/IO/Compress/Brotli.pm +++ b/lib/IO/Compress/Brotli.pm @@ -4,8 +4,72 @@ use 5.014000; use strict; use warnings; +use IO::Uncompress::Brotli; + +use parent qw/Exporter/; + +our @EXPORT = qw/bro/; +our @EXPORT_OK = @EXPORT; + our $VERSION = '0.001001'; +sub create { + my ($class) = @_; + my $state = BrotliEncoderCreateInstance(); + bless \$state, $class +} + +sub DESTROY { + my ($self) = @_; + BrotliEncoderDestroyInstance($$self) +} + +sub quality { + my ($self, $quality) = @_; + BrotliEncoderSetQuality($$self, $quality) +} + +sub window { + my ($self, $window) = @_; + BrotliEncoderSetWindow($$self, $window) +} + +my %BROTLI_ENCODER_MODE = ( generic => 0, text => 1, font => 2 ); +sub mode { + my ($self, $mode) = @_; + + die "Invalid encoder mode" + unless $BROTLI_ENCODER_MODE{$mode}; + + BrotliEncoderSetMode($$self, $mode) +} + +use constant { + BROTLI_OPERATION_PROCESS => 0, + BROTLI_OPERATION_FLUSH => 1, + BROTLI_OPERATION_FINISH => 2 +}; +sub compress { + my ($self, $data) = @_; + BrotliEncoderCompressStream($$self, $data, BROTLI_OPERATION_PROCESS ) +} + +sub flush { + my ($self) = @_; + BrotliEncoderCompressStream($$self, '', BROTLI_OPERATION_FLUSH ) +} + +sub finish { + my ($self) = @_; + BrotliEncoderCompressStream($$self, '', BROTLI_OPERATION_FINISH ) +} + +# Untested, probably not working +sub set_dictionary { + my ($self, $dict) = @_; + BrotliEncoderSetCustomDictionary($$self, $dict) +} + 1; __END__ @@ -19,9 +83,97 @@ IO::Compress::Brotli - [Not yet implemented] Write Brotli buffers/streams use IO::Compress::Brotli; + # compress a buffer + my $encoded = bro $encoded; + + # compress a stream + my $bro = IO::Compress::Brotli->create; + while(have_input()) { + my $block = get_input_block(); + my $encoded_block = $bro->compress($block); + handle_output_block($encoded_block); + } + # Need to finish the steam + handle_output_block($bro->finish()); + =head1 DESCRIPTION -IO::Compress::Brotli currently does nothing. +IO::Compress::Brotli is a module that compressed Brotli buffers +and streams. Despite its name, it is not a subclass of +L and does not implement its interface. This +will be rectified in a future release. + +=head2 One-shot interface + +If you have the whole buffer in a Perl scalar use the B +function. + +=over + +=item B(I<$input>) + +Takes a whole uncompressed buffer as input and returns the compressed +data. + +Exported by default. + +=back + +=head2 Streaming interface + +If you want to process the data in blocks use the object oriented +interface. The available methods are: + +=over + +=item IO::Compress::Brotli->B + +Returns a IO::Compress::Brotli instance. Please note that a single +instance cannot be used to decompress multiple streams. + +=item $bro->B(I<$window>) + +Sets the window parameter on the brotli encoder. +Defaults to BROTLI_DEFAULT_WINDOW (22). + +=item $bro->B(I<$quality>) + +Sets the quality paremeter on the brotli encoder. +Defaults to BROTLI_DEFAULT_QUALITY (11). + +=item $bro->B(I<$mode>) + +Sets the brotli encoder mode, which can be any of "generic", +"text" or "font". Defaults to "generic". + +=item $bro->B(I<$block>) + +Takes the a block of uncompressed data and returns a block of +compressed data. Dies on error. + +=item $bro->B() + +Flushes any pending output from the encoder. + +=item $bro->B() + +Tells the encoder to start the finish operation, and flushes +any remaining compressed output. + +Once finish is called, the encoder cannot be used to compress +any more content. + +B: Calling finish is B, or the output might +remain unflushed, and the be missing termination marks. + +=back + +=head1 SEE ALSO + +Brotli Compressed Data Format Internet-Draft: +L + +Brotli source code: L =head1 AUTHOR -- 2.39.2