diff options
author | Ivan Krasin <krasin@golang.org> | 2012-06-13 16:24:31 -0400 |
---|---|---|
committer | Russ Cox <rsc@golang.org> | 2012-06-13 16:24:31 -0400 |
commit | 5fc093e3c08f5f19531d5a1efcc088c1f21dc1ea (patch) | |
tree | a1460269376e9686b15d360bb352be7197d97b15 | |
parent | 19af0b28c5bafbcb361ad5ca4b116d9a36a595a3 (diff) | |
download | go-5fc093e3c08f5f19531d5a1efcc088c1f21dc1ea.tar.gz go-5fc093e3c08f5f19531d5a1efcc088c1f21dc1ea.zip |
[release-branch.go1] compress/flate: fix overflow on 2GB input. Reset hashOffset every 16 MB.
««« backport 00a1ca1ea3bd
compress/flate: fix overflow on 2GB input. Reset hashOffset every 16 MB.
This bug has been introduced in the following revision:
changeset: 11404:26dceba5c610
user: Ivan Krasin <krasin@golang.org>
date: Mon Jan 23 09:19:39 2012 -0500
summary: compress/flate: reduce memory pressure at cost of additional arithmetic operation.
This is the review page for that CL: https://golang.org/cl/5555070/
R=rsc, imkrasin
CC=golang-dev
https://golang.org/cl/6249067
»»»
-rw-r--r-- | src/pkg/compress/flate/deflate.go | 20 | ||||
-rw-r--r-- | src/pkg/compress/flate/deflate_test.go | 44 |
2 files changed, 64 insertions, 0 deletions
diff --git a/src/pkg/compress/flate/deflate.go b/src/pkg/compress/flate/deflate.go index 20408409c8..e511b50fd1 100644 --- a/src/pkg/compress/flate/deflate.go +++ b/src/pkg/compress/flate/deflate.go @@ -32,6 +32,7 @@ const ( hashSize = 1 << hashBits hashMask = (1 << hashBits) - 1 hashShift = (hashBits + minMatchLength - 1) / minMatchLength + maxHashOffset = 1 << 24 skipNever = math.MaxInt32 ) @@ -106,6 +107,25 @@ func (d *compressor) fillDeflate(b []byte) int { d.blockStart = math.MaxInt32 } d.hashOffset += windowSize + if d.hashOffset > maxHashOffset { + delta := d.hashOffset - 1 + d.hashOffset -= delta + d.chainHead -= delta + for i, v := range d.hashPrev { + if v > delta { + d.hashPrev[i] -= delta + } else { + d.hashPrev[i] = 0 + } + } + for i, v := range d.hashHead { + if v > delta { + d.hashHead[i] -= delta + } else { + d.hashHead[i] = 0 + } + } + } } n := copy(d.window[d.windowEnd:], b) d.windowEnd += n diff --git a/src/pkg/compress/flate/deflate_test.go b/src/pkg/compress/flate/deflate_test.go index 543c595058..f1e6db2ace 100644 --- a/src/pkg/compress/flate/deflate_test.go +++ b/src/pkg/compress/flate/deflate_test.go @@ -94,6 +94,50 @@ func TestDeflate(t *testing.T) { } } +// A sparseReader returns a stream consisting of 0s followed by 1<<16 1s. +// This tests missing hash references in a very large input. +type sparseReader struct { + l int64 + cur int64 +} + +func (r *sparseReader) Read(b []byte) (n int, err error) { + if r.cur >= r.l { + return 0, io.EOF + } + n = len(b) + cur := r.cur + int64(n) + if cur > r.l { + n -= int(cur - r.l) + cur = r.l + } + for i := range b[0:n] { + if r.cur+int64(i) >= r.l-1<<16 { + b[i] = 1 + } else { + b[i] = 0 + } + } + r.cur = cur + return +} + +func TestVeryLongSparseChunk(t *testing.T) { + if testing.Short() { + t.Logf("skipping sparse chunk during short test") + return + } + w, err := NewWriter(ioutil.Discard, 1) + if err != nil { + t.Errorf("NewWriter: %v", err) + return + } + if _, err = io.Copy(w, &sparseReader{l: 23E8}); err != nil { + t.Errorf("Compress failed: %v", err) + return + } +} + type syncBuffer struct { buf bytes.Buffer mu sync.RWMutex |