Eliminate omitempty from /schema/details response type

api.go

@@ -992,15 +992,15 @@ func (err MessageProcessingError) Unwrap() error {
 func (api *API) Schema(ctx context.Context) []*IndexInfo {
 	span, _ := tracing.StartSpanFromContext(ctx, "API.Schema")
 	defer span.Finish()
-	return api.holder.limitedSchema()
+	return api.holder.Schema(false)
 }
 
 // SchemaDetails returns information about each index in Pilosa including which
 // fields they contain, and additional field information such as cardinality
-func (api *API) SchemaDetails(ctx context.Context) []*IndexInfo {
+func (api *API) SchemaDetails(ctx context.Context) []*IndexDetails {
 	span, _ := tracing.StartSpanFromContext(ctx, "API.Schema")
 	defer span.Finish()
-	schema := api.holder.limitedSchema()
+	schema := api.holder.SchemaDetails()
 	for _, index := range schema {
 		for _, field := range index.Fields {
 			q := fmt.Sprintf("Count(Distinct(field=%s))", field.Name)

cluster.go

@@ -625,7 +625,7 @@ func (c *cluster) unprotectedStatus() *ClusterStatus {
 		ClusterID: c.id,
 		State:     c.state,
 		Nodes:     c.nodes,
-		Schema:    &Schema{Indexes: c.holder.Schema()},
+		Schema:    &Schema{Indexes: c.holder.Schema(true)},
 	}
 }
 
@@ -2372,7 +2372,7 @@ func (c *cluster) nodeLeave(nodeID string) error {
 func (c *cluster) nodeStatus() *NodeStatus {
 	ns := &NodeStatus{
 		Node:   c.Node,
-		Schema: &Schema{Indexes: c.holder.Schema()},
+		Schema: &Schema{Indexes: c.holder.Schema(true)},
 	}
 	var availableShards *roaring.Bitmap
 	for _, idx := range ns.Schema.Indexes {
@@ -3134,6 +3134,11 @@ type Schema struct {
 	Indexes []*IndexInfo `json:"indexes"`
 }
 
+// SchemaDetails contains information about indexes and their configuration.
+type SchemaDetails struct {
+	Indexes []*IndexDetails `json:"indexes"`
+}
+
 func encodeTopology(topology *Topology) *internal.Topology {
 	if topology == nil {
 		return nil

field.go

@@ -1867,11 +1867,10 @@ func (p fieldSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() }
 
 // FieldInfo represents schema information for a field.
 type FieldInfo struct {
-	Name        string       `json:"name"`
-	CreatedAt   int64        `json:"createdAt,omitempty"`
-	Options     FieldOptions `json:"options"`
-	Cardinality uint64       `json:"cardinality,omitempty"`
-	Views       []*ViewInfo  `json:"views,omitempty"`
+	Name      string       `json:"name"`
+	CreatedAt int64        `json:"createdAt,omitempty"`
+	Options   FieldOptions `json:"options"`
+	Views     []*ViewInfo  `json:"views,omitempty"`
 }
 
 type fieldInfoSlice []*FieldInfo
@@ -1880,6 +1879,21 @@ func (p fieldInfoSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
 func (p fieldInfoSlice) Len() int           { return len(p) }
 func (p fieldInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name }
 
+// FieldDetails represents detailed schema information for a field.
+type FieldDetails struct {
+	Name        string       `json:"name"`
+	CreatedAt   int64        `json:"createdAt,omitempty"`
+	Options     FieldOptions `json:"options"`
+	Cardinality uint64       `json:"cardinality"`
+	Views       []*ViewInfo  `json:"views,omitempty"`
+}
+
+type fieldDetailsSlice []*FieldDetails
+
+func (p fieldDetailsSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
+func (p fieldDetailsSlice) Len() int           { return len(p) }
+func (p fieldDetailsSlice) Less(i, j int) bool { return p[i].Name < p[j].Name }
+
 // FieldOptions represents options to set when initializing a field.
 type FieldOptions struct {
 	Base           int64       `json:"base,omitempty"`

holder.go

@@ -847,7 +847,9 @@ func (h *Holder) availableShardsByIndex() map[string]*roaring.Bitmap {
 }
 
 // Schema returns schema information for all indexes, fields, and views.
-func (h *Holder) Schema() []*IndexInfo {
+// If includeHiddenAndViews=true, include fields beginning with "_",
+// as well as view details.
+func (h *Holder) Schema(includeHiddenAndViews bool) []*IndexInfo {
 	var a []*IndexInfo
 	for _, index := range h.Indexes() {
 		di := &IndexInfo{
@@ -857,15 +859,20 @@ func (h *Holder) Schema() []*IndexInfo {
 			ShardWidth: ShardWidth,
 		}
 		for _, field := range index.Fields() {
+			if !includeHiddenAndViews && strings.HasPrefix(field.name, "_") {
+				continue
+			}
 			fi := &FieldInfo{
 				Name:      field.Name(),
 				CreatedAt: field.CreatedAt(),
 				Options:   field.Options(),
 			}
-			for _, view := range field.views() {
-				fi.Views = append(fi.Views, &ViewInfo{Name: view.name})
+			if includeHiddenAndViews {
+				for _, view := range field.views() {
+					fi.Views = append(fi.Views, &ViewInfo{Name: view.name})
+				}
+				sort.Sort(viewInfoSlice(fi.Views))
 			}
-			sort.Sort(viewInfoSlice(fi.Views))
 			di.Fields = append(di.Fields, fi)
 		}
 		sort.Sort(fieldInfoSlice(di.Fields))
@@ -875,32 +882,38 @@ func (h *Holder) Schema() []*IndexInfo {
 	return a
 }
 
-// limitedSchema returns schema information for all non-hidden indexes and fields.
-func (h *Holder) limitedSchema() []*IndexInfo {
-	var a []*IndexInfo
+// SchemaDetails returns schema information for all non-hidden indexes and fields,
+// including additional per-field details such as cardinality, actual range of integer data, etc.
+// This function duplicates the logic of Holder.Schema because the FieldDetails struct
+// includes a struct-field for cardinality, with default value 0, so the behavior of omitempty
+// is incompatible between the /schema and /schema/details HTTP endpoints. A value of 0 for
+// cardinality is meaningful, so it should be included when accurate, and not accidentally
+// reported as 0 when the struct-field has not been populated.
+func (h *Holder) SchemaDetails() []*IndexDetails {
+	var a []*IndexDetails
 	for _, index := range h.Indexes() {
-		di := &IndexInfo{
+		di := &IndexDetails{
 			Name:       index.Name(),
 			CreatedAt:  index.CreatedAt(),
 			Options:    index.Options(),
 			ShardWidth: ShardWidth,
-			Fields:     make([]*FieldInfo, 0, len(index.Fields())),
+			Fields:     make([]*FieldDetails, 0, len(index.Fields())),
 		}
 		for _, field := range index.Fields() {
 			if strings.HasPrefix(field.name, "_") {
 				continue
 			}
-			fi := &FieldInfo{
+			fi := &FieldDetails{
 				Name:      field.Name(),
 				CreatedAt: field.CreatedAt(),
 				Options:   field.Options(),
 			}
 			di.Fields = append(di.Fields, fi)
 		}
-		sort.Sort(fieldInfoSlice(di.Fields))
+		sort.Sort(fieldDetailsSlice(di.Fields))
 		a = append(a, di)
 	}
-	sort.Sort(indexInfoSlice(a))
+	sort.Sort(indexDetailsSlice(a))
 	return a
 }
 
@@ -1337,7 +1350,7 @@ func (s *holderSyncer) SyncHolder() error {
 	defer s.mu.Unlock()
 	ti := time.Now()
 	// Iterate over schema in sorted order.
-	for _, di := range s.Holder.Schema() {
+	for _, di := range s.Holder.Schema(true) {
 		// Verify syncer has not closed.
 		if s.IsClosing() {
 			return nil

http/handler.go

@@ -682,7 +682,7 @@ func (h *Handler) handleGetSchemaDetails(w http.ResponseWriter, r *http.Request)
 
 	w.Header().Set("Content-Type", "application/json")
 	schema := h.api.SchemaDetails(r.Context())
-	if err := json.NewEncoder(w).Encode(pilosa.Schema{Indexes: schema}); err != nil {
+	if err := json.NewEncoder(w).Encode(pilosa.SchemaDetails{Indexes: schema}); err != nil {
 		h.logger.Printf("write schema response error: %s", err)
 	}
 }

index.go

@@ -668,6 +668,21 @@ func (p indexInfoSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
 func (p indexInfoSlice) Len() int           { return len(p) }
 func (p indexInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name }
 
+// IndexDetails represents detailed schema information for an index.
+type IndexDetails struct {
+	Name       string          `json:"name"`
+	CreatedAt  int64           `json:"createdAt,omitempty"`
+	Options    IndexOptions    `json:"options"`
+	Fields     []*FieldDetails `json:"fields"`
+	ShardWidth uint64          `json:"shardWidth"`
+}
+
+type indexDetailsSlice []*IndexDetails
+
+func (p indexDetailsSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
+func (p indexDetailsSlice) Len() int           { return len(p) }
+func (p indexDetailsSlice) Less(i, j int) bool { return p[i].Name < p[j].Name }
+
 // IndexOptions represents options to set when initializing an index.
 type IndexOptions struct {
 	Keys           bool `json:"keys"`

server/handler_test.go

@@ -112,6 +112,19 @@ func TestHandler_Endpoints(t *testing.T) {
 
 	})
 
+	t.Run("SchemaDetailsEmpty", func(t *testing.T) {
+		w := httptest.NewRecorder()
+		h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/schema/details", nil))
+		if w.Code != gohttp.StatusOK {
+			t.Fatalf("unexpected status code: %d", w.Code)
+		}
+		body := w.Body.String()
+		if body != "{\"indexes\":null}\n" {
+			t.Fatalf("unexpected empty schema: '%v'", body)
+		}
+
+	})
+
 	t.Run("PostSchema", func(t *testing.T) {
 		w := httptest.NewRecorder()
 		h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/schema", strings.NewReader(`{"indexes":[{"name":"blah","options":{"keys":false,"trackExistence":true},"fields":[{"name":"f1","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":1048576}]}`)))
@@ -211,6 +224,57 @@ func TestHandler_Endpoints(t *testing.T) {
 		t.Fatal(err)
 	}
 
+	// i2 is for SchemaDetails
+	i2 := hldr.MustCreateIndexIfNotExists("i2", pilosa.IndexOptions{})
+	tx2, err := holder.BeginTx(true, i2.Index, shard)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer tx2.Rollback()
+	if f, err := i2.CreateFieldIfNotExists("f0", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 1000)); err != nil {
+		t.Fatal(err)
+	} else if _, err := f.SetBit(tx2, 0, 0, nil); err != nil {
+		t.Fatal(err)
+	}
+	if f, err := i2.CreateFieldIfNotExists("f1", pilosa.OptFieldTypeInt(-100, 100)); err != nil {
+		t.Fatal(err)
+	} else if _, err := f.SetBit(tx2, 0, 0, nil); err != nil {
+		t.Fatal(err)
+	}
+
+	if _, err := cmd.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i2", Query: "Set(10, f1=4) Set(11, f1=5) Set(12, f1=6) Set(13, f1=7)"}); err != nil {
+		t.Fatal(err)
+	}
+
+	if f, err := i2.CreateFieldIfNotExists("f2", pilosa.OptFieldTypeDecimal(1, pql.Decimal{Value: -10}, pql.Decimal{Value: 10})); err != nil {
+		t.Fatal(err)
+	} else if _, err := f.SetBit(tx2, 0, 0, nil); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := cmd.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i2", Query: "Set(10, f2=4) Set(11, f2=5) Set(12, f2=6) Set(13, f2=7) Set(14, f2=8)"}); err != nil {
+		t.Fatal(err)
+	}
+
+	if f, err := i2.CreateFieldIfNotExists("f3", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"))); err != nil {
+		t.Fatal(err)
+	} else if _, err := f.SetBit(tx2, 0, 0, nil); err != nil {
+		t.Fatal(err)
+	}
+	if f, err := i2.CreateFieldIfNotExists("f4", pilosa.OptFieldTypeMutex(pilosa.CacheTypeRanked, 5000)); err != nil {
+		t.Fatal(err)
+	} else if _, err := f.SetBit(tx2, 0, 0, nil); err != nil {
+		t.Fatal(err)
+	}
+	if f, err := i2.CreateFieldIfNotExists("f5", pilosa.OptFieldTypeBool()); err != nil {
+		t.Fatal(err)
+	} else if _, err := f.SetBit(tx2, 0, 0, nil); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := tx2.Commit(); err != nil {
+		t.Fatal(err)
+	}
+
 	t.Run("Schema", func(t *testing.T) {
 		w := httptest.NewRecorder()
 		h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/schema", nil))
@@ -225,6 +289,20 @@ func TestHandler_Endpoints(t *testing.T) {
 		}
 	})
 
+	t.Run("SchemaDetails", func(t *testing.T) {
+		w := httptest.NewRecorder()
+		h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/schema/details", nil))
+		if w.Code != gohttp.StatusOK {
+			t.Fatalf("unexpected status code: %d", w.Code)
+		}
+
+		body := strings.TrimSpace(w.Body.String())
+		target := fmt.Sprintf(`{"indexes":[{"name":"i0","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false},"cardinality":0},{"name":"f1","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false},"cardinality":1}],"shardWidth":1048576},{"name":"i1","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false},"cardinality":1}],"shardWidth":1048576},{"name":"i2","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":1000,"keys":false},"cardinality":1},{"name":"f1","options":{"type":"int","base":0,"bitDepth":3,"min":-100,"max":100,"keys":false,"foreignIndex":""},"cardinality":4},{"name":"f2","options":{"type":"decimal","base":0,"scale":1,"bitDepth":7,"min":-10,"max":10,"keys":false},"cardinality":5},{"name":"f3","options":{"type":"time","timeQuantum":"YMDH","keys":false,"noStandardView":false},"cardinality":1},{"name":"f4","options":{"type":"mutex","cacheType":"ranked","cacheSize":5000,"keys":false},"cardinality":1},{"name":"f5","options":{"type":"bool"},"cardinality":1}],"shardWidth":%[1]d}]}`, pilosa.ShardWidth)
+		if body != target {
+			t.Fatalf("%s\n!=\n%s", target, body)
+		}
+	})
+
 	t.Run("Import", func(t *testing.T) {
 		indexInfo := cmd.API.Schema(context.Background())
 		err := cmd.API.ApplySchema(context.Background(), &pilosa.Schema{Indexes: indexInfo}, false)