Merge branch 'master' into fix-DEGRADED

.circleci/config.yml

@@ -140,46 +140,39 @@ jobs:
           at: .
       - run: sudo pip install awscli
       - run: make prerelease-upload
-  dockerhub-upload:
-    parameters:
-      tag_branch:
-        type: boolean
-        default: true
-      tag_tag:
-        type: boolean
-        default: false
-      tag_latest:
-        type: boolean
-        default: false
-      target_name:
-        type: string
-        default: moleculacorp/pilosa
+  dockerhub-upload-unstable:
     executor:
       name: golang
     steps:
+      - checkout
       - attach_workspace:
           at: .
       - add-github-auth
       - setup_remote_docker
       - run: make docker
       - run: docker login -u $DOCKER_USER -p $DOCKER_PASS
-      - when:
-          condition: << parameters.tag_branch >>
-          steps:
-            - run: make docker-tag-push DOCKER_TARGET=<< parameters.target_name >>:<< pipeline.git.branch >>
-      - when:
-          condition: << parameters.tag_tag >>
-          steps:
-            - run: make docker-tag-push DOCKER_TARGET=<< parameters.target_name >>:$(git describe --tags)
-      - when:
-          condition: << parameters.tag_latest >>
-          steps:
-            - run: make docker-tag-push DOCKER_TARGET=<< parameters.target_name >>:latest
-        
+      - run: make docker-tag-push DOCKER_TARGET=moleculacorp/pilosa:<< pipeline.git.branch >>
+  dockerhub-upload-stable:
+    executor:
+      name: golang
+    steps:
+      - checkout
+      - attach_workspace:
+          at: .
+      - add-github-auth
+      - setup_remote_docker
+      - run: make docker
+      - run: docker login -u $DOCKER_USER -p $DOCKER_PASS
+      - run: make docker-tag-push DOCKER_TARGET=moleculacorp/pilosa:<< pipeline.git.tag >>
+      - run: make docker-tag-push DOCKER_TARGET=moleculacorp/pilosa:latest
+
 workflows:
   build:
     jobs:
-      - setup
+      - setup:
+          filters:
+            tags:
+              only: /^v.*/
       - linter:
           requires:
             - setup
@@ -196,6 +189,9 @@ workflows:
               golang_version: ["1.14", "1.13", "1.12", "1.11"]
           requires:
             - setup
+          filters:
+            tags:
+              only: /^v.*/
       - test:
           name: test-race
           test_make_target: test-race
@@ -220,27 +216,17 @@ workflows:
             - linter
             - check-license-headers
             - test-golang-1.14
-      - dockerhub-upload:
-          name: dockerhub-upload-unstable
-          tag_branch: true
-          tag_tag: false
-          tag_latest: false
+      - dockerhub-upload-unstable:
+          context: molecula
           requires:
-            - linter
-            - check-license-headers
-            - test-golang-1.14
+            - setup
           filters:
             branches:
               only: master
-      - dockerhub-upload:
-          name: dockerhub-upload-stable
-          tag_branch: true
-          tag_tag: true
-          tag_latest: true
+      - dockerhub-upload-stable:
+          context: molecula
           requires:
-            - linter
-            - check-license-headers
-            - test-golang-1.14
+            - setup
           filters:
             tags:
               only: /^v.*/

api.go

@@ -795,11 +795,30 @@ func (api *API) ClusterMessage(ctx context.Context, reqBody io.Reader) error {
 
 	// Forward the message.
 	if err := api.server.receiveMessage(msg); err != nil {
-		return errors.Wrap(err, "receiving message")
+		return MessageProcessingError{err}
 	}
 	return nil
 }
 
+// MessageProcessingError is an error indicating that a cluster message could not be processed.
+type MessageProcessingError struct {
+	Err error
+}
+
+func (err MessageProcessingError) Error() string {
+	return "processing message: " + err.Err.Error()
+}
+
+// Cause allows the error to be unwrapped.
+func (err MessageProcessingError) Cause() error {
+	return err.Err
+}
+
+// Unwrap allows the error to be unwrapped.
+func (err MessageProcessingError) Unwrap() error {
+	return err.Err
+}
+
 // Schema returns information about each index in Pilosa including which fields
 // they contain.
 func (api *API) Schema(ctx context.Context) []*IndexInfo {

executor_test.go

@@ -5719,3 +5719,42 @@ func TestExecutor_Execute_TopNDistinct(t *testing.T) {
 		}
 	})
 }
+
+func TestTimelessClearRegression(t *testing.T) {
+	data, err := ioutil.ReadFile("testdata/timeRegressionSchema.json")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	c := test.MustRunCluster(t, 1)
+	defer c.Close()
+
+	api := c[0].API
+
+	schema := &pilosa.Schema{}
+	if err := json.NewDecoder(bytes.NewReader(data)).Decode(schema); err != nil {
+		t.Fatal(err)
+	}
+	if err := api.ApplySchema(context.TODO(), schema, false); err != nil {
+		t.Fatal(err)
+	}
+
+	idxName := schema.Indexes[0].Name
+
+	setQuery := `Set(511, stargazer=376)`
+	if _, err := api.Query(context.TODO(), &pilosa.QueryRequest{Index: idxName, Query: setQuery}); err != nil {
+		t.Fatal(err)
+	}
+
+	setQuery = `Set(512, stargazer=300, 2017-05-18T00:00)`
+	if _, err := api.Query(context.TODO(), &pilosa.QueryRequest{Index: idxName, Query: setQuery}); err != nil {
+		t.Fatal(err)
+	}
+
+	clearQuery := `Clear(511, stargazer=376)`
+	if res, err := api.Query(context.TODO(), &pilosa.QueryRequest{Index: idxName, Query: clearQuery}); err != nil {
+		t.Fatal(err)
+	} else if res.Results[0] != true {
+		t.Fatal("clear supposedly failed")
+	}
+}

field.go

@@ -1281,15 +1281,14 @@ func (f *Field) ClearBit(rowID, colID uint64) (changed bool, err error) {
 	// Retrieve view. Exit if it doesn't exist.
 	view, present := f.viewMap[viewName]
 	if !present {
-		return changed, errors.Wrap(err, "clearing missing view")
-
+		return false, errors.Wrap(err, "clearing missing view")
 	}
 
 	// Clear non-time bit.
 	if v, err := view.clearBit(rowID, colID); err != nil {
-		return changed, errors.Wrap(err, "clearing on view")
+		return false, errors.Wrap(err, "clearing on view")
 	} else if v {
-		changed = v
+		changed = changed || v
 	}
 	if len(f.viewMap) == 1 { // assuming no time views
 		return changed, nil
@@ -1304,10 +1303,12 @@ func (f *Field) ClearBit(rowID, colID uint64) (changed bool, err error) {
 			level--
 		}
 		if level < skipAbove {
-			if changed, err = view.clearBit(rowID, colID); err != nil {
+			cleared, err := view.clearBit(rowID, colID)
+			changed = changed || cleared
+			if err != nil {
 				return changed, errors.Wrapf(err, "clearing on view %s", view.name)
 			}
-			if !changed {
+			if !cleared {
 				skipAbove = level + 1
 			} else {
 				skipAbove = maxInt

http/handler.go

@@ -252,14 +252,17 @@ func (h *Handler) queryArgValidator(next http.Handler) http.Handler {
 
 		if validator, ok := h.validators[key]; ok {
 			if err := validator.validate(r.URL.Query()); err != nil {
-				// TODO: Return the response depending on the Accept header
-				response := errorResponse{Error: err.Error()}
-				body, err := json.Marshal(response)
-				if err != nil {
-					http.Error(w, err.Error(), http.StatusBadRequest)
-					return
+				errText := err.Error()
+				if validHeaderAcceptJSON(r.Header) {
+					response := errorResponse{Error: errText}
+					data, err := json.Marshal(response)
+					if err != nil {
+						h.logger.Printf("failed to encode error %q as JSON: %v", errText, err)
+					} else {
+						errText = string(data)
+					}
 				}
-				http.Error(w, string(body), http.StatusBadRequest)
+				http.Error(w, errText, http.StatusBadRequest)
 				return
 			}
 		}
@@ -1780,8 +1783,13 @@ func (h *Handler) handlePostClusterMessage(w http.ResponseWriter, r *http.Reques
 	}
 	err := h.api.ClusterMessage(r.Context(), r.Body)
 	if err != nil {
-		// TODO this was the previous behavior, but perhaps not everything is a bad request
-		http.Error(w, err.Error(), http.StatusBadRequest)
+		switch err := err.(type) {
+		case pilosa.MessageProcessingError:
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+		default:
+			http.Error(w, err.Error(), http.StatusBadRequest)
+		}
+		return
 	}
 
 	w.Header().Set("Content-Type", "application/json")
@@ -2114,6 +2122,7 @@ func (h *Handler) handlePostTranslateKeys(w http.ResponseWriter, r *http.Request
 	buf, err := h.api.TranslateKeys(r.Context(), r.Body)
 	if err != nil {
 		http.Error(w, fmt.Sprintf("translate keys: %v", err), http.StatusInternalServerError)
+		return
 	}
 
 	// Write response.

testdata/timeRegressionSchema.json

@@ -0,0 +1,32 @@
+{
+    "indexes": [
+        {
+            "name": "repository",
+            "options": {
+                "keys": false,
+                "trackExistence": true
+            },
+            "fields": [
+                {
+                    "name": "language",
+                    "options": {
+                        "type": "set",
+                        "cacheType": "ranked",
+                        "cacheSize": 50000,
+                        "keys": false
+                    }
+                },
+                {
+                    "name": "stargazer",
+                    "options": {
+                        "type": "time",
+                        "timeQuantum": "YMD",
+                        "keys": false,
+                        "noStandardView": false
+                    }
+                }
+            ],
+            "shardWidth": 1048576
+        }
+    ]
+}