Merge branch 'master' into 2015RoaringBugs

cluster.go

@@ -1702,22 +1702,24 @@ func confirmNodeDown(uri URI, log logger.Logger) bool {
 		Host:   uri.HostPort(),
 		Path:   "version",
 	}
-	ctx, cancel := context.WithTimeout(context.Background(), confirmDownTimeout*time.Second)
-	defer cancel()
 	req, err := http.NewRequest("GET", u.String(), nil)
 	if err != nil {
 		log.Printf("bad request:%s %s", u.String(), err)
 		return false
 	}
-
 	for i := 0; i < confirmDownRetries; i++ {
+		ctx, cancel := context.WithTimeout(context.Background(), confirmDownTimeout*time.Second)
+		defer cancel()
 		resp, err := http.DefaultClient.Do(req.WithContext(ctx))
+		var bod []byte
 		if err == nil {
+			bod, err = ioutil.ReadAll(resp.Body)
 			if resp.StatusCode == 200 {
 				return false
 			}
 		}
-		log.Printf("NodeLeave Timeout with %s %d", uri.HostPort(), i)
+
+		log.Printf("NodeLeave confirm with %s %d. err: '%v' bod: '%s'", uri.HostPort(), i, err, bod)
 		time.Sleep(confirmDownSleep * time.Second)
 	}
 	return true

roaring/fuzz_test.go

@@ -13,32 +13,57 @@
 // limitations under the License.
 package roaring
 
-import ( 
+import (
 	"testing"
 )
 
 func TestUnmarshalBinary(t *testing.T) {
 	b := NewBitmap()
 	confirmedCrashers := []struct {
-		cr []byte
+		cr       []byte
 		expected string
-	} {
+	}{
+		{ // Checks for the zero containers situation
+			cr:       []byte(":0\x000\x01\x00\x00\x000000"), //":000000"
+			expected: "reading roaring header: malformed bitmap, key-cardinality slice overruns buffer at 12",
+		},
+		{ // Checks for int overflow
+			cr: []byte("<0\x000\x00\x00\x00\x00000000000000" +
+				"0"), //"<000000000000000"
+			expected: "unmarshaling as pilosa roaring: Maximum operation size exceeded",
+		},
+		{ // The next 5 check for malformed bitmaps
+			cr: []byte("<0\x0000000000000000000" +
+				"\x00\x00\xec\x00\x03\x00\x00\x00\xec000"), //"<000000000000000000ÏÏ000"
+			expected: "unmarshaling as pilosa roaring: malformed bitmap, key-cardinality not provided for 67372036 containers",
+		},
 		{
-			cr : []byte(":0\x000\x01\x00\x00\x000000"),					//":000000"
-			expected : "reading roaring header: malformed bitmap, key-cardinality slice overruns buffer at 12",
-		},				
+			cr: []byte("<0\x00\x02\x00\x00\x00\\f\x01\xb5\x8d\x009\v\x01\x00\x00\x00\x00" +
+				"\x00\x00e\x04\x00\x00\x00\x04\xfd\x00\x01\x00"), //"<0\f
µç9
e˝"
+			expected: "unmarshaling as pilosa roaring: malformed bitmap, key-cardinality not provided for 128625322 containers",
+		},
 		{
-			cr : []byte("<0\x000\x00\x00\x00\x00000000000000" +
-			"0"),														//"<000000000000000"
-			expected : "unmarshaling as pilosa roaring: Maximum operation size exceeded",
-		},		
+			cr:       []byte("<0\x00\x02\x00\x00\x00&x.field safe"), //"<0&x.field safe"
+			expected: "unmarshaling as pilosa roaring: malformed bitmap, key-cardinality not provided for 53127850 containers",
+		},
+		{
+			cr: []byte("<0\x00\x00\x14\x00\x00\x00\x80\xffp\x05_ 4\x114089" +
+				"\x00\x00\xff\x000\x00\x02\x00\x00\x00\x00\xff\u007f\x00\x00\x01\x10\x00\x00j" +
+				"\x02\x00\x00$\x04_\x00\xff\u007f\xff062616163\x00" + //"<0ġp_ 44089ˇ0ˇj$_ˇˇ0626161630ø¸ad$j√"
+				"0\x00\x02\x00\x01\xbf\x00\x04\x00\xfcad$\x00\x00j\x10\x00\x00\xc3"),
+			expected: "unmarshaling as pilosa roaring: malformed bitmap, key-cardinality not provided for 1 containers",
+		},
+		{ // 0 containers because the container is partially formed, but not fully (ie. 3/12 = 0)
+			cr:       []byte("<0\x00\x02\x03\x00\x00\x00쳫\v\x00d9\v\x00\x009\v"), //<0쳫d99
+			expected: "unmarshaling as pilosa roaring: malformed bitmap, key-cardinality not provided for 0 containers",
+		},
 	}
 
 	for _, crash := range confirmedCrashers {
 		err := b.UnmarshalBinary(crash.cr)
 		if err.Error() != crash.expected {
 			t.Errorf("Expected: %s, Got: %s", crash.expected, err)
-		} 
+		}
 	}
-	
-}
+
+}

roaring/roaring.go

@@ -1139,6 +1139,9 @@ func (b *Bitmap) unmarshalPilosaRoaring(data []byte) error {
 
 	// Read key count in bytes sizeof(cookie)+sizeof(flag):(sizeof(cookie)+sizeof(uint32)).
 	keyN := binary.LittleEndian.Uint32(data[3+1 : 8])
+	if uint32(len(data)) < headerBaseSize+keyN*12 {
+		return fmt.Errorf("malformed bitmap, key-cardinality not provided for %d containers", int(keyN)/12)
+	}
 
 	headerSize := headerBaseSize
 	b.Containers.Reset()
@@ -4017,7 +4020,7 @@ func (op *op) WriteTo(w io.Writer) (n int64, err error) {
 }
 
 var minOpSize = 13
-var maxBatchSize = uint64(1<<59)
+var maxBatchSize = uint64(1 << 59)
 
 // UnmarshalBinary decodes data into an op.
 func (op *op) UnmarshalBinary(data []byte) error {