Merge pull request #1391 from alanbernstein/field-cardinality

CORE-92 Add /schema/details endpoint, which includes field cardinality

api.go

@@ -155,14 +155,20 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er
 		return QueryResponse{}, errors.Wrap(err, "validating api method")
 	}
 
+	if !req.Remote {
+		defer api.tracker.Finish(api.tracker.Start(req.Query, req.SQLQuery, api.server.nodeID, req.Index, start))
+	}
+
+	return api.query(ctx, req)
+}
+
+// query provides query functionality for internal use, without tracing, validation, or tracking
+func (api *API) query(ctx context.Context, req *QueryRequest) (QueryResponse, error) {
 	q, err := pql.NewParser(strings.NewReader(req.Query)).Parse()
 	if err != nil {
 		return QueryResponse{}, errors.Wrap(err, "parsing")
 	}
 
-	if !req.Remote {
-		defer api.tracker.Finish(api.tracker.Start(req.Query, req.SQLQuery, api.server.nodeID, req.Index, start))
-	}
 	// TODO can we get rid of exec options and pass the QueryRequest directly to executor?
 	execOpts := &execOptions{
 		Remote:          req.Remote,
@@ -986,7 +992,32 @@ func (err MessageProcessingError) Unwrap() error {
 func (api *API) Schema(ctx context.Context) []*IndexInfo {
 	span, _ := tracing.StartSpanFromContext(ctx, "API.Schema")
 	defer span.Finish()
-	return api.holder.limitedSchema()
+	return api.holder.Schema(false)
+}
+
+// SchemaDetails returns information about each index in Pilosa including which
+// fields they contain, and additional field information such as cardinality
+func (api *API) SchemaDetails(ctx context.Context) ([]*IndexInfo, error) {
+	span, _ := tracing.StartSpanFromContext(ctx, "API.Schema")
+	defer span.Finish()
+	schema := api.holder.Schema(false)
+	for _, index := range schema {
+		for _, field := range index.Fields {
+			q := fmt.Sprintf("Count(Distinct(field=%s))", field.Name)
+			req := QueryRequest{Index: index.Name, Query: q}
+			resp, err := api.query(ctx, &req)
+			if err != nil {
+				return schema, errors.Wrapf(err, "querying cardinality (%s/%s)", index.Name, field.Name)
+			}
+			if len(resp.Results) == 0 {
+				continue
+			}
+			if card, ok := resp.Results[0].(uint64); ok {
+				field.Cardinality = &card
+			}
+		}
+	}
+	return schema, nil
 }
 
 // ApplySchema takes the given schema and applies it across the

cluster.go

@@ -625,7 +625,7 @@ func (c *cluster) unprotectedStatus() *ClusterStatus {
 		ClusterID: c.id,
 		State:     c.state,
 		Nodes:     c.nodes,
-		Schema:    &Schema{Indexes: c.holder.Schema()},
+		Schema:    &Schema{Indexes: c.holder.Schema(true)},
 	}
 }
 
@@ -2372,7 +2372,7 @@ func (c *cluster) nodeLeave(nodeID string) error {
 func (c *cluster) nodeStatus() *NodeStatus {
 	ns := &NodeStatus{
 		Node:   c.Node,
-		Schema: &Schema{Indexes: c.holder.Schema()},
+		Schema: &Schema{Indexes: c.holder.Schema(true)},
 	}
 	var availableShards *roaring.Bitmap
 	for _, idx := range ns.Schema.Indexes {

executor.go

@@ -643,6 +643,41 @@ func (e *executor) preprocessQuery(ctx context.Context, qcx *Qcx, index string,
 	}
 }
 
+type shardSlice []uint64
+
+// String creates a run-length encoded representation of a slice of shard IDs (integers).
+// For example, []uint64{0, 1, 3, 4, 5, 7, 8, 9, 11, 13} is represented as
+// [0-1,3-5,7-9,11,13].
+func (s shardSlice) String() string {
+	if len(s) == 0 {
+		// surely this is impossible
+		return "[]"
+	}
+	runs := make([]string, 0, len(s)/2)
+	start := s[0]
+	end := start
+	for n := 1; n < len(s); n++ {
+		if s[n] == end+1 {
+			end = s[n]
+		} else {
+			repr := fmt.Sprintf("%d", start)
+			if end > start {
+				repr += fmt.Sprintf("-%d", end)
+			}
+			runs = append(runs, repr)
+			start = s[n]
+			end = start
+		}
+	}
+	repr := fmt.Sprintf("%d", start)
+	if end > start {
+		repr += fmt.Sprintf("-%d", end)
+	}
+	runs = append(runs, repr)
+
+	return "[" + strings.Join(runs, ",") + "]"
+}
+
 // executeCall executes a call.
 func (e *executor) executeCall(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *execOptions) (interface{}, error) {
 	span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeCall")
@@ -692,47 +727,47 @@ func (e *executor) executeCall(ctx context.Context, qcx *Qcx, index string, c *p
 	case "Sum":
 		statFn()
 		res, err := e.executeSum(ctx, qcx, index, c, shards, opt)
-		return res, errors.Wrapf(err, "executeSum %v", shards)
+		return res, errors.Wrapf(err, "executeSum %v", shardSlice(shards))
 	case "Min":
 		statFn()
 		res, err := e.executeMin(ctx, qcx, index, c, shards, opt)
-		return res, errors.Wrapf(err, "executeMin %v", shards)
+		return res, errors.Wrapf(err, "executeMin %v", shardSlice(shards))
 	case "Max":
 		statFn()
 		res, err := e.executeMax(ctx, qcx, index, c, shards, opt)
-		return res, errors.Wrapf(err, "executeMax %v", shards)
+		return res, errors.Wrapf(err, "executeMax %v", shardSlice(shards))
 	case "MinRow":
 		statFn()
 		res, err := e.executeMinRow(ctx, qcx, index, c, shards, opt)
-		return res, errors.Wrapf(err, "executeMinRow %v", shards)
+		return res, errors.Wrapf(err, "executeMinRow %v", shardSlice(shards))
 	case "MaxRow":
 		statFn()
 		res, err := e.executeMaxRow(ctx, qcx, index, c, shards, opt)
-		return res, errors.Wrapf(err, "executeMaxRow %v", shards)
+		return res, errors.Wrapf(err, "executeMaxRow %v", shardSlice(shards))
 	case "Clear":
 		statFn()
 		res, err := e.executeClearBit(ctx, qcx, index, c, opt)
-		return res, errors.Wrapf(err, "executeClearBit %v", shards)
+		return res, errors.Wrapf(err, "executeClearBit %v", shardSlice(shards))
 	case "ClearRow":
 		statFn()
 		res, err := e.executeClearRow(ctx, qcx, index, c, shards, opt)
-		return res, errors.Wrapf(err, "executeClearRow %v", shards)
+		return res, errors.Wrapf(err, "executeClearRow %v", shardSlice(shards))
 	case "Distinct":
 		statFn()
 		res, err := e.executeDistinct(ctx, qcx, index, c, shards, opt)
-		return res, errors.Wrapf(err, "executeDistinct %v", shards)
+		return res, errors.Wrapf(err, "executeDistinct %v", shardSlice(shards))
 	case "Store":
 		statFn()
 		res, err := e.executeSetRow(ctx, qcx, index, c, shards, opt)
-		return res, errors.Wrapf(err, "executeSetRow %v", shards)
+		return res, errors.Wrapf(err, "executeSetRow %v", shardSlice(shards))
 	case "Count":
 		statFn()
 		res, err := e.executeCount(ctx, qcx, index, c, shards, opt)
-		return res, errors.Wrapf(err, "executeCount %v", shards)
+		return res, errors.Wrapf(err, "executeCount %v", shardSlice(shards))
 	case "Set":
 		statFn()
 		res, err := e.executeSet(ctx, qcx, index, c, opt)
-		return res, errors.Wrapf(err, "executeSet %v", shards)
+		return res, errors.Wrapf(err, "executeSet %v", shardSlice(shards))
 	case "SetRowAttrs":
 		statFn()
 		return nil, errors.Wrap(e.executeSetRowAttrs(ctx, qcx, index, c, opt), "executeSetRowAttrs")
@@ -742,50 +777,50 @@ func (e *executor) executeCall(ctx context.Context, qcx *Qcx, index string, c *p
 	case "TopK":
 		statFn()
 		res, err := e.executeTopK(ctx, qcx, index, c, shards, opt)
-		return res, errors.Wrapf(err, "executeTopK %v", shards)
+		return res, errors.Wrapf(err, "executeTopK %v", shardSlice(shards))
 	case "TopN":
 		statFn()
 		res, err := e.executeTopN(ctx, qcx, index, c, shards, opt)
-		return res, errors.Wrapf(err, "executeTopN %v", shards)
+		return res, errors.Wrapf(err, "executeTopN %v", shardSlice(shards))
 	case "Rows":
 		statFn()
 		res, err := e.executeRows(ctx, qcx, index, c, shards, opt)
-		return res, errors.Wrapf(err, "executeRows %v", shards)
+		return res, errors.Wrapf(err, "executeRows %v", shardSlice(shards))
 	case "Extract":
 		statFn()
 		res, err := e.executeExtract(ctx, qcx, index, c, shards, opt)
-		return res, errors.Wrapf(err, "executeExtract %v", shards)
+		return res, errors.Wrapf(err, "executeExtract %v", shardSlice(shards))
 	case "GroupBy":
 		statFn()
 		res, err := e.executeGroupBy(ctx, qcx, index, c, shards, opt)
-		return res, errors.Wrapf(err, "executeGroupBy %v", shards)
+		return res, errors.Wrapf(err, "executeGroupBy %v", shardSlice(shards))
 	case "Options":
 		statFn()
 		res, err := e.executeOptionsCall(ctx, qcx, index, c, shards, opt)
-		return res, errors.Wrapf(err, "executeOptionsCall %v", shards)
+		return res, errors.Wrapf(err, "executeOptionsCall %v", shardSlice(shards))
 	case "IncludesColumn":
 		res, err := e.executeIncludesColumnCall(ctx, qcx, index, c, shards, opt)
-		return res, errors.Wrapf(err, "executeIncludesColumnCall %v", shards)
+		return res, errors.Wrapf(err, "executeIncludesColumnCall %v", shardSlice(shards))
 	case "FieldValue":
 		statFn()
 		res, err := e.executeFieldValueCall(ctx, qcx, index, c, shards, opt)
-		return res, errors.Wrapf(err, "executeFieldValueCall %v", shards)
+		return res, errors.Wrapf(err, "executeFieldValueCall %v", shardSlice(shards))
 	case "Precomputed":
 		res, err := e.executePrecomputedCall(ctx, qcx, index, c, shards, opt)
-		return res, errors.Wrapf(err, "executePrecomputedCall %v", shards)
+		return res, errors.Wrapf(err, "executePrecomputedCall %v", shardSlice(shards))
 	case "UnionRows":
 		res, err := e.executeUnionRows(ctx, qcx, index, c, shards, opt)
-		return res, errors.Wrapf(err, "executeUnionRows %v", shards)
+		return res, errors.Wrapf(err, "executeUnionRows %v", shardSlice(shards))
 	case "ConstRow":
 		res, err := e.executeConstRow(ctx, index, c)
-		return res, errors.Wrapf(err, "executeConstRow %v", shards)
+		return res, errors.Wrapf(err, "executeConstRow %v", shardSlice(shards))
 	case "Limit":
 		res, err := e.executeLimitCall(ctx, qcx, index, c, shards, opt)
-		return res, errors.Wrapf(err, "executeLimitCall %v", shards)
+		return res, errors.Wrapf(err, "executeLimitCall %v", shardSlice(shards))
 	default: // e.g. "Row", "Union", "Intersect" or anything that returns a bitmap.
 		statFn()
 		res, err := e.executeBitmapCall(ctx, qcx, index, c, shards, opt)
-		return res, errors.Wrapf(err, "executeBitmapCall %v", shards)
+		return res, errors.Wrapf(err, "executeBitmapCall %v", shardSlice(shards))
 	}
 }
 
@@ -2856,7 +2891,7 @@ func (e *executor) executeGroupBy(ctx context.Context, qcx *Qcx, index string, c
 	// Get full result set.
 	other, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn)
 	if err != nil {
-		return nil, errors.Wrapf(err, "mapReduce shards: %v", shards)
+		return nil, errors.Wrapf(err, "mapReduce shards: %v", shardSlice(shards))
 	}
 	results, _ := other.([]GroupCount)
 
@@ -5595,7 +5630,7 @@ func (e *executor) mapper(ctx context.Context, cancel context.CancelFunc, ch cha
 	// Group shards together by nodes.
 	m, err := e.shardsByNode(nodes, index, shards)
 	if err != nil {
-		return errors.Wrapf(err, "shards by node %v", shards)
+		return errors.Wrapf(err, "shards by node %v", shardSlice(shards))
 	}
 
 	// Execute each node in a separate goroutine.

field.go

@@ -1867,10 +1867,11 @@ func (p fieldSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() }
 
 // FieldInfo represents schema information for a field.
 type FieldInfo struct {
-	Name      string       `json:"name"`
-	CreatedAt int64        `json:"createdAt,omitempty"`
-	Options   FieldOptions `json:"options"`
-	Views     []*ViewInfo  `json:"views,omitempty"`
+	Name        string       `json:"name"`
+	CreatedAt   int64        `json:"createdAt,omitempty"`
+	Options     FieldOptions `json:"options"`
+	Cardinality *uint64      `json:"cardinality,omitempty"`
+	Views       []*ViewInfo  `json:"views,omitempty"`
 }
 
 type fieldInfoSlice []*FieldInfo

holder.go

@@ -847,35 +847,9 @@ func (h *Holder) availableShardsByIndex() map[string]*roaring.Bitmap {
 }
 
 // Schema returns schema information for all indexes, fields, and views.
-func (h *Holder) Schema() []*IndexInfo {
-	var a []*IndexInfo
-	for _, index := range h.Indexes() {
-		di := &IndexInfo{
-			Name:      index.Name(),
-			CreatedAt: index.CreatedAt(),
-			Options:   index.Options(),
-		}
-		for _, field := range index.Fields() {
-			fi := &FieldInfo{
-				Name:      field.Name(),
-				CreatedAt: field.CreatedAt(),
-				Options:   field.Options(),
-			}
-			for _, view := range field.views() {
-				fi.Views = append(fi.Views, &ViewInfo{Name: view.name})
-			}
-			sort.Sort(viewInfoSlice(fi.Views))
-			di.Fields = append(di.Fields, fi)
-		}
-		sort.Sort(fieldInfoSlice(di.Fields))
-		a = append(a, di)
-	}
-	sort.Sort(indexInfoSlice(a))
-	return a
-}
-
-// limitedSchema returns schema information for all indexes and fields.
-func (h *Holder) limitedSchema() []*IndexInfo {
+// If includeHiddenAndViews=true, include fields beginning with "_",
+// as well as view details.
+func (h *Holder) Schema(includeHiddenAndViews bool) []*IndexInfo {
 	var a []*IndexInfo
 	for _, index := range h.Indexes() {
 		di := &IndexInfo{
@@ -883,10 +857,9 @@ func (h *Holder) limitedSchema() []*IndexInfo {
 			CreatedAt:  index.CreatedAt(),
 			Options:    index.Options(),
 			ShardWidth: ShardWidth,
-			Fields:     make([]*FieldInfo, 0, len(index.Fields())),
 		}
 		for _, field := range index.Fields() {
-			if strings.HasPrefix(field.name, "_") {
+			if !includeHiddenAndViews && strings.HasPrefix(field.name, "_") {
 				continue
 			}
 			fi := &FieldInfo{
@@ -894,6 +867,12 @@ func (h *Holder) limitedSchema() []*IndexInfo {
 				CreatedAt: field.CreatedAt(),
 				Options:   field.Options(),
 			}
+			if includeHiddenAndViews {
+				for _, view := range field.views() {
+					fi.Views = append(fi.Views, &ViewInfo{Name: view.name})
+				}
+				sort.Sort(viewInfoSlice(fi.Views))
+			}
 			di.Fields = append(di.Fields, fi)
 		}
 		sort.Sort(fieldInfoSlice(di.Fields))
@@ -1336,7 +1315,7 @@ func (s *holderSyncer) SyncHolder() error {
 	defer s.mu.Unlock()
 	ti := time.Now()
 	// Iterate over schema in sorted order.
-	for _, di := range s.Holder.Schema() {
+	for _, di := range s.Holder.Schema(true) {
 		// Verify syncer has not closed.
 		if s.IsClosing() {
 			return nil

http/handler.go

@@ -392,6 +392,7 @@ func newRouter(handler *Handler) http.Handler {
 	router.HandleFunc("/inspect", handler.handleInspect).Methods("GET").Name("Inspect")
 	router.HandleFunc("/recalculate-caches", handler.handleRecalculateCaches).Methods("POST").Name("RecalculateCaches")
 	router.HandleFunc("/schema", handler.handleGetSchema).Methods("GET").Name("GetSchema")
+	router.HandleFunc("/schema/details", handler.handleGetSchemaDetails).Methods("GET").Name("GetSchemaDetails")
 	router.HandleFunc("/schema", handler.handlePostSchema).Methods("POST").Name("PostSchema")
 	router.HandleFunc("/status", handler.handleGetStatus).Methods("GET").Name("GetStatus")
 	router.HandleFunc("/transaction", handler.handlePostTransaction).Methods("POST").Name("PostTransaction")
@@ -672,6 +673,24 @@ func (h *Handler) handleGetSchema(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
+// handleGetSchema handles GET /schema/details requests.
+func (h *Handler) handleGetSchemaDetails(w http.ResponseWriter, r *http.Request) {
+	if !validHeaderAcceptJSON(r.Header) {
+		http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable)
+		return
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	schema, err := h.api.SchemaDetails(r.Context())
+	if err != nil {
+		h.logger.Printf("error getting detailed schema: %s", err)
+		return
+	}
+	if err := json.NewEncoder(w).Encode(pilosa.Schema{Indexes: schema}); err != nil {
+		h.logger.Printf("write schema response error: %s", err)
+	}
+}
+
 func (h *Handler) handlePostSchema(w http.ResponseWriter, r *http.Request) {
 	q := r.URL.Query()
 	remoteStr := q.Get("remote")

row.go

@@ -461,6 +461,10 @@ func (r *Row) invalidateCount() {
 // Count returns the number of columns in the row.
 func (r *Row) Count() uint64 {
 	var n uint64
+	if r == nil {
+		// Count(Distinct()) on an empty field panics here
+		return n
+	}
 	for i := range r.segments {
 		n += r.segments[i].Count()
 	}

server/handler_test.go

@@ -112,6 +112,19 @@ func TestHandler_Endpoints(t *testing.T) {
 
 	})
 
+	t.Run("SchemaDetailsEmpty", func(t *testing.T) {
+		w := httptest.NewRecorder()
+		h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/schema/details", nil))
+		if w.Code != gohttp.StatusOK {
+			t.Fatalf("unexpected status code: %d", w.Code)
+		}
+		body := w.Body.String()
+		if body != "{\"indexes\":null}\n" {
+			t.Fatalf("unexpected empty schema: '%v'", body)
+		}
+
+	})
+
 	t.Run("PostSchema", func(t *testing.T) {
 		w := httptest.NewRecorder()
 		h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/schema", strings.NewReader(`{"indexes":[{"name":"blah","options":{"keys":false,"trackExistence":true},"fields":[{"name":"f1","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":1048576}]}`)))
@@ -219,9 +232,78 @@ func TestHandler_Endpoints(t *testing.T) {
 		}
 
 		body := strings.TrimSpace(w.Body.String())
-		target := fmt.Sprintf(`{"indexes":[{"name":"i0","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}},{"name":"f1","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":%d},{"name":"i1","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":%[1]d}]}`, pilosa.ShardWidth)
+		target := fmt.Sprintf(`{"indexes":[{"name":"i0","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}},{"name":"f1","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":%[1]d},{"name":"i1","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":%[1]d}]}`, pilosa.ShardWidth)
 		if body != target {
-			t.Fatalf("%s != %s", target, body)
+			t.Fatalf("\n%s\n!=\n%s", target, body)
+		}
+	})
+
+	// i2 is for SchemaDetails
+	i2 := hldr.MustCreateIndexIfNotExists("i2", pilosa.IndexOptions{})
+	tx2, err := holder.BeginTx(true, i2.Index, shard)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer tx2.Rollback()
+	if f, err := i2.CreateFieldIfNotExists("f0", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 1000)); err != nil {
+		t.Fatal(err)
+	} else if _, err := f.SetBit(tx2, 0, 0, nil); err != nil {
+		t.Fatal(err)
+	}
+
+	f, err := i2.CreateFieldIfNotExists("f1", pilosa.OptFieldTypeInt(-100, 100))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for n := 0; n < 4; n++ {
+		if _, err := f.SetValue(tx2, uint64(n), int64(n)); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	f, err = i2.CreateFieldIfNotExists("f2", pilosa.OptFieldTypeDecimal(1, pql.Decimal{Value: -10}, pql.Decimal{Value: 10}))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for n := 0; n < 5; n++ {
+		if _, err := f.SetValue(tx2, uint64(n), int64(n)); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	if f, err := i2.CreateFieldIfNotExists("f3", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"))); err != nil {
+		t.Fatal(err)
+	} else if _, err := f.SetBit(tx2, 0, 0, nil); err != nil {
+		t.Fatal(err)
+	}
+	if f, err := i2.CreateFieldIfNotExists("f4", pilosa.OptFieldTypeMutex(pilosa.CacheTypeRanked, 5000)); err != nil {
+		t.Fatal(err)
+	} else if _, err := f.SetBit(tx2, 0, 0, nil); err != nil {
+		t.Fatal(err)
+	}
+	if f, err := i2.CreateFieldIfNotExists("f5", pilosa.OptFieldTypeBool()); err != nil {
+		t.Fatal(err)
+	} else if _, err := f.SetBit(tx2, 0, 0, nil); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := tx2.Commit(); err != nil {
+		t.Fatal(err)
+	}
+
+	t.Run("SchemaDetails", func(t *testing.T) {
+		w := httptest.NewRecorder()
+		h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/schema/details", nil))
+		if w.Code != gohttp.StatusOK {
+			t.Fatalf("unexpected status code: %d", w.Code)
+		}
+
+		body := strings.TrimSpace(w.Body.String())
+		target := fmt.Sprintf(`{"indexes":[{"name":"i0","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false},"cardinality":0},{"name":"f1","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false},"cardinality":1}],"shardWidth":%[1]d},{"name":"i1","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false},"cardinality":1}],"shardWidth":%[1]d},{"name":"i2","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":1000,"keys":false},"cardinality":1},{"name":"f1","options":{"type":"int","base":0,"bitDepth":2,"min":-100,"max":100,"keys":false,"foreignIndex":""},"cardinality":4},{"name":"f2","options":{"type":"decimal","base":0,"scale":1,"bitDepth":3,"min":-10,"max":10,"keys":false},"cardinality":5},{"name":"f3","options":{"type":"time","timeQuantum":"YMDH","keys":false,"noStandardView":false},"cardinality":1},{"name":"f4","options":{"type":"mutex","cacheType":"ranked","cacheSize":5000,"keys":false},"cardinality":1},{"name":"f5","options":{"type":"bool"},"cardinality":1}],"shardWidth":%[1]d}]}`, pilosa.ShardWidth)
+		if body != target {
+			t.Fatalf("\n%s\n!=\n%s", target, body)
 		}
 	})
 
@@ -398,13 +480,13 @@ func TestHandler_Endpoints(t *testing.T) {
 
 		for _, nodeUsage := range nodeUsages {
 			numIndexes := len(nodeUsage.Disk.IndexUsage)
-			if nodeUsage.Disk.TotalUse < 75000 || nodeUsage.Disk.TotalUse > 300000 {
+			if nodeUsage.Disk.TotalUse < 75000 || nodeUsage.Disk.TotalUse > 500000 {
 				// Usage measurements are not consistent between machines, or
 				// over time, as features and implementations change, so checking
 				// for a range of sizes may be most useful way to test the details of this.
-				t.Fatalf("expected 75k < total < 300k, got %d", nodeUsage.Disk.TotalUse)
+				t.Fatalf("expected 75k < total < 500k, got %d", nodeUsage.Disk.TotalUse)
 			}
-			if numIndexes != 2 {
+			if numIndexes != 3 {
 				t.Fatalf("wrong length index usage list: expected %d, got %d", 2, numIndexes)
 			}
 			numFields := len(nodeUsage.Disk.IndexUsage["i1"].Fields)
@@ -483,7 +565,7 @@ func TestHandler_Endpoints(t *testing.T) {
 		h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/internal/shards/max", nil))
 		if w.Code != gohttp.StatusOK {
 			t.Fatalf("unexpected status code: %d", w.Code)
-		} else if body := w.Body.String(); body != `{"standard":{"i0":3,"i1":0}}`+"\n" {
+		} else if body := w.Body.String(); body != `{"standard":{"i0":3,"i1":0,"i2":0}}`+"\n" {
 			t.Fatalf("unexpected body: %s", body)
 		}
 	})