Merge branch 'iss#2005'

roaring/fuzz_test.go

@@ -0,0 +1,69 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package roaring
+
+import ( 
+	"testing"
+)
+
+func TestUnmarshalBinary(t *testing.T) {
+	b := NewBitmap()
+	confirmedCrashers := []struct {
+		cr []byte
+		expected string
+	} {
+		{	// Checks for int overflow
+			cr : []byte("<0\x000\x00\x00\x00\x00000000000000" +
+			"0"),														//"<000000000000000"
+			expected : "unmarshaling as pilosa roaring: Maximum operation size exceeded",
+		},		
+		{	// Checks for the zero containers situation
+			cr : []byte(":0\x000\x01\x00\x00\x000000"),					//":000000"
+			expected : "reading roaring header: malformed bitmap, key-cardinality slice overruns buffer at 12",
+		},	
+		{	// The next 5 check for malformed bitmaps
+			cr : []byte("<0\x0000000000000000000" +
+			"\x00\x00\xec\x00\x03\x00\x00\x00\xec000"),					//"<000000000000000000ÏÏ000"
+			expected : "unmarshaling as pilosa roaring: malformed bitmap, key-cardinality not provided for 67372036 containers",
+		},	
+		{
+			cr : []byte("<0\x00\x02\x00\x00\x00\\f\x01\xb5\x8d\x009\v\x01\x00\x00\x00\x00" +
+			"\x00\x00e\x04\x00\x00\x00\x04\xfd\x00\x01\x00"),			//"<0\f
µç9
e˝"
+			expected : "unmarshaling as pilosa roaring: malformed bitmap, key-cardinality not provided for 128625322 containers",
+		},
+		{
+			cr : []byte("<0\x00\x02\x00\x00\x00&x.field safe"),			//"<0&x.field safe"
+			expected : "unmarshaling as pilosa roaring: malformed bitmap, key-cardinality not provided for 53127850 containers",
+		},
+		{
+			cr : []byte("<0\x00\x00\x14\x00\x00\x00\x80\xffp\x05_ 4\x114089" +
+			"\x00\x00\xff\x000\x00\x02\x00\x00\x00\x00\xff\u007f\x00\x00\x01\x10\x00\x00j" +
+			"\x02\x00\x00$\x04_\x00\xff\u007f\xff062616163\x00" +		//"<0ġp_ 44089ˇ0ˇj$_ˇˇ0626161630ø¸ad$j√"
+			"0\x00\x02\x00\x01\xbf\x00\x04\x00\xfcad$\x00\x00j\x10\x00\x00\xc3"),			
+			expected : "unmarshaling as pilosa roaring: malformed bitmap, key-cardinality not provided for 1 containers",
+		},
+		{	// 0 containers because the container is partially formed, but not fully (ie. 3/12 = 0)
+			cr : []byte("<0\x00\x02\x03\x00\x00\x00쳫\v\x00d9\v\x00\x009\v"),		//<0쳫d99	
+			expected : "unmarshaling as pilosa roaring: malformed bitmap, key-cardinality not provided for 0 containers",
+		},
+	}
+
+	for _, crash := range confirmedCrashers {
+		err := b.UnmarshalBinary(crash.cr)
+		if err.Error() != crash.expected {
+			t.Errorf("Expected: %s, Got: %s", crash.expected, err)
+		} 
+	}
+	
+}

roaring/roaring.go

@@ -1102,7 +1102,7 @@ func (b *Bitmap) writeToUnoptimized(w io.Writer) (n int64, err error) {
 // unmarshalPilosaRoaring treats data as being encoded in Pilosa's 64 bit
 // roaring format and decodes it into b.
 func (b *Bitmap) unmarshalPilosaRoaring(data []byte) error {
-	if len(data) < headerBaseSize {
+	if len(data) <= headerBaseSize {
 		return errors.New("data too small")
 	}
 
@@ -1120,6 +1120,9 @@ func (b *Bitmap) unmarshalPilosaRoaring(data []byte) error {
 
 	// Read key count in bytes sizeof(cookie)+sizeof(flag):(sizeof(cookie)+sizeof(uint32)).
 	keyN := binary.LittleEndian.Uint32(data[3+1 : 8])
+	if len(data) < headerBaseSize+int(keyN)*12 {
+		return fmt.Errorf("malformed bitmap, key-cardinality not provided for %d containers", int(keyN)/12)
+	}
 
 	headerSize := headerBaseSize
 	b.Containers.Reset()
@@ -3951,6 +3954,7 @@ func (op *op) WriteTo(w io.Writer) (n int64, err error) {
 }
 
 var minOpSize = 13
+var maxBatchSize = uint64(1<<59)
 
 // UnmarshalBinary decodes data into an op.
 func (op *op) UnmarshalBinary(data []byte) error {
@@ -3968,6 +3972,11 @@ func (op *op) UnmarshalBinary(data []byte) error {
 	_, _ = h.Write(data[0:9])
 
 	if op.typ > 1 {
+		// This ensures that in doing 13+op.value*8, the max int won't be exceeded and a wrap around case
+		// (resulting in a negative value) won't occur in the slice indexing while writing
+		if op.value > maxBatchSize {
+			return fmt.Errorf("Maximum operation size exceeded")
+		}
 		if len(data) < int(13+op.value*8) {
 			return fmt.Errorf("op data truncated - expected %d, got %d", 13+op.value*8, len(data))
 		}
@@ -4460,7 +4469,7 @@ func readOfficialHeader(buf []byte) (size uint32, containerTyper func(index uint
 	}
 
 	// descriptive header
-	if pos+2*2*int(size) > len(buf) {
+	if pos+2*2*int(size) >= len(buf) {
 		err = fmt.Errorf("malformed bitmap, key-cardinality slice overruns buffer at %d", pos+2*2*int(size))
 		return size, containerTyper, header, pos, flags, haveRuns, err
 	}