diff options
Diffstat (limited to 'vendor/github.com/klauspost/compress/zstd/decoder.go')
| -rw-r--r-- | vendor/github.com/klauspost/compress/zstd/decoder.go | 121 |
1 files changed, 59 insertions, 62 deletions
diff --git a/vendor/github.com/klauspost/compress/zstd/decoder.go b/vendor/github.com/klauspost/compress/zstd/decoder.go index 9fcdaac1..286c8f9d 100644 --- a/vendor/github.com/klauspost/compress/zstd/decoder.go +++ b/vendor/github.com/klauspost/compress/zstd/decoder.go @@ -347,18 +347,20 @@ func (d *Decoder) DecodeAll(input, dst []byte) ([]byte, error) { } frame.history.setDict(&dict) } - - if frame.FrameContentSize != fcsUnknown && frame.FrameContentSize > d.o.maxDecodedSize-uint64(len(dst)) { - return dst, ErrDecoderSizeExceeded + if frame.WindowSize > d.o.maxWindowSize { + return dst, ErrWindowSizeExceeded } - if frame.FrameContentSize < 1<<30 { - // Never preallocate more than 1 GB up front. + if frame.FrameContentSize != fcsUnknown { + if frame.FrameContentSize > d.o.maxDecodedSize-uint64(len(dst)) { + return dst, ErrDecoderSizeExceeded + } if cap(dst)-len(dst) < int(frame.FrameContentSize) { - dst2 := make([]byte, len(dst), len(dst)+int(frame.FrameContentSize)) + dst2 := make([]byte, len(dst), len(dst)+int(frame.FrameContentSize)+compressedBlockOverAlloc) copy(dst2, dst) dst = dst2 } } + if cap(dst) == 0 { // Allocate len(input) * 2 by default if nothing is provided // and we didn't get frame content size. @@ -437,7 +439,7 @@ func (d *Decoder) nextBlock(blocking bool) (ok bool) { println("got", len(d.current.b), "bytes, error:", d.current.err, "data crc:", tmp) } - if len(next.b) > 0 { + if !d.o.ignoreChecksum && len(next.b) > 0 { n, err := d.current.crc.Write(next.b) if err == nil { if n != len(next.b) { @@ -449,7 +451,7 @@ func (d *Decoder) nextBlock(blocking bool) (ok bool) { got := d.current.crc.Sum64() var tmp [4]byte binary.LittleEndian.PutUint32(tmp[:], uint32(got)) - if !bytes.Equal(tmp[:], next.d.checkCRC) && !ignoreCRC { + if !d.o.ignoreChecksum && !bytes.Equal(tmp[:], next.d.checkCRC) { if debugDecoder { println("CRC Check Failed:", tmp[:], " (got) !=", next.d.checkCRC, "(on stream)") } @@ -533,9 +535,15 @@ func (d *Decoder) nextBlockSync() (ok bool) { // Update/Check CRC if d.frame.HasCheckSum { - d.frame.crc.Write(d.current.b) + if !d.o.ignoreChecksum { + d.frame.crc.Write(d.current.b) + } if d.current.d.Last { - d.current.err = d.frame.checkCRC() + if !d.o.ignoreChecksum { + d.current.err = d.frame.checkCRC() + } else { + d.current.err = d.frame.consumeCRC() + } if d.current.err != nil { println("CRC error:", d.current.err) return false @@ -629,60 +637,18 @@ func (d *Decoder) startSyncDecoder(r io.Reader) error { // Create Decoder: // ASYNC: -// Spawn 4 go routines. -// 0: Read frames and decode blocks. -// 1: Decode block and literals. Receives hufftree and seqdecs, returns seqdecs and huff tree. -// 2: Wait for recentOffsets if needed. Decode sequences, send recentOffsets. -// 3: Wait for stream history, execute sequences, send stream history. +// Spawn 3 go routines. +// 0: Read frames and decode block literals. +// 1: Decode sequences. +// 2: Execute sequences, send to output. func (d *Decoder) startStreamDecoder(ctx context.Context, r io.Reader, output chan decodeOutput) { defer d.streamWg.Done() br := readerWrapper{r: r} - var seqPrepare = make(chan *blockDec, d.o.concurrent) var seqDecode = make(chan *blockDec, d.o.concurrent) var seqExecute = make(chan *blockDec, d.o.concurrent) - // Async 1: Prepare blocks... - go func() { - var hist history - var hasErr bool - for block := range seqPrepare { - if hasErr { - if block != nil { - seqDecode <- block - } - continue - } - if block.async.newHist != nil { - if debugDecoder { - println("Async 1: new history") - } - hist.reset() - if block.async.newHist.dict != nil { - hist.setDict(block.async.newHist.dict) - } - } - if block.err != nil || block.Type != blockTypeCompressed { - hasErr = block.err != nil - seqDecode <- block - continue - } - - remain, err := block.decodeLiterals(block.data, &hist) - block.err = err - hasErr = block.err != nil - if err == nil { - block.async.literals = hist.decoders.literals - block.async.seqData = remain - } else if debugDecoder { - println("decodeLiterals error:", err) - } - seqDecode <- block - } - close(seqDecode) - }() - - // Async 2: Decode sequences... + // Async 1: Decode sequences... go func() { var hist history var hasErr bool @@ -696,7 +662,7 @@ func (d *Decoder) startStreamDecoder(ctx context.Context, r io.Reader, output ch } if block.async.newHist != nil { if debugDecoder { - println("Async 2: new history, recent:", block.async.newHist.recentOffsets) + println("Async 1: new history, recent:", block.async.newHist.recentOffsets) } hist.decoders = block.async.newHist.decoders hist.recentOffsets = block.async.newHist.recentOffsets @@ -750,7 +716,7 @@ func (d *Decoder) startStreamDecoder(ctx context.Context, r io.Reader, output ch } if block.async.newHist != nil { if debugDecoder { - println("Async 3: new history") + println("Async 2: new history") } hist.windowSize = block.async.newHist.windowSize hist.allocFrameBuffer = block.async.newHist.allocFrameBuffer @@ -837,6 +803,33 @@ func (d *Decoder) startStreamDecoder(ctx context.Context, r io.Reader, output ch decodeStream: for { + var hist history + var hasErr bool + + decodeBlock := func(block *blockDec) { + if hasErr { + if block != nil { + seqDecode <- block + } + return + } + if block.err != nil || block.Type != blockTypeCompressed { + hasErr = block.err != nil + seqDecode <- block + return + } + + remain, err := block.decodeLiterals(block.data, &hist) + block.err = err + hasErr = block.err != nil + if err == nil { + block.async.literals = hist.decoders.literals + block.async.seqData = remain + } else if debugDecoder { + println("decodeLiterals error:", err) + } + seqDecode <- block + } frame := d.frame if debugDecoder { println("New frame...") @@ -863,7 +856,7 @@ decodeStream: case <-ctx.Done(): case dec := <-d.decoders: dec.sendErr(err) - seqPrepare <- dec + decodeBlock(dec) } break decodeStream } @@ -883,6 +876,10 @@ decodeStream: if debugDecoder { println("Alloc History:", h.allocFrameBuffer) } + hist.reset() + if h.dict != nil { + hist.setDict(h.dict) + } dec.async.newHist = &h dec.async.fcs = frame.FrameContentSize historySent = true @@ -909,7 +906,7 @@ decodeStream: } err = dec.err last := dec.Last - seqPrepare <- dec + decodeBlock(dec) if err != nil { break decodeStream } @@ -918,7 +915,7 @@ decodeStream: } } } - close(seqPrepare) + close(seqDecode) wg.Wait() d.frame.history.b = frameHistCache } |