diff --git a/api.go b/api.go index 025d37537..6fa885e01 100644 --- a/api.go +++ b/api.go @@ -992,15 +992,15 @@ func (err MessageProcessingError) Unwrap() error { func (api *API) Schema(ctx context.Context) []*IndexInfo { span, _ := tracing.StartSpanFromContext(ctx, "API.Schema") defer span.Finish() - return api.holder.limitedSchema() + return api.holder.Schema(false) } // SchemaDetails returns information about each index in Pilosa including which // fields they contain, and additional field information such as cardinality -func (api *API) SchemaDetails(ctx context.Context) []*IndexInfo { +func (api *API) SchemaDetails(ctx context.Context) []*IndexDetails { span, _ := tracing.StartSpanFromContext(ctx, "API.Schema") defer span.Finish() - schema := api.holder.limitedSchema() + schema := api.holder.SchemaDetails() for _, index := range schema { for _, field := range index.Fields { q := fmt.Sprintf("Count(Distinct(field=%s))", field.Name) diff --git a/cluster.go b/cluster.go index 3a9976e72..45b93ef67 100644 --- a/cluster.go +++ b/cluster.go @@ -625,7 +625,7 @@ func (c *cluster) unprotectedStatus() *ClusterStatus { ClusterID: c.id, State: c.state, Nodes: c.nodes, - Schema: &Schema{Indexes: c.holder.Schema()}, + Schema: &Schema{Indexes: c.holder.Schema(true)}, } } @@ -2372,7 +2372,7 @@ func (c *cluster) nodeLeave(nodeID string) error { func (c *cluster) nodeStatus() *NodeStatus { ns := &NodeStatus{ Node: c.Node, - Schema: &Schema{Indexes: c.holder.Schema()}, + Schema: &Schema{Indexes: c.holder.Schema(true)}, } var availableShards *roaring.Bitmap for _, idx := range ns.Schema.Indexes { @@ -3134,6 +3134,11 @@ type Schema struct { Indexes []*IndexInfo `json:"indexes"` } +// SchemaDetails contains information about indexes and their configuration. +type SchemaDetails struct { + Indexes []*IndexDetails `json:"indexes"` +} + func encodeTopology(topology *Topology) *internal.Topology { if topology == nil { return nil diff --git a/field.go b/field.go index 494057560..f4ab3982b 100644 --- a/field.go +++ b/field.go @@ -1867,11 +1867,10 @@ func (p fieldSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() } // FieldInfo represents schema information for a field. type FieldInfo struct { - Name string `json:"name"` - CreatedAt int64 `json:"createdAt,omitempty"` - Options FieldOptions `json:"options"` - Cardinality uint64 `json:"cardinality,omitempty"` - Views []*ViewInfo `json:"views,omitempty"` + Name string `json:"name"` + CreatedAt int64 `json:"createdAt,omitempty"` + Options FieldOptions `json:"options"` + Views []*ViewInfo `json:"views,omitempty"` } type fieldInfoSlice []*FieldInfo @@ -1880,6 +1879,21 @@ func (p fieldInfoSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } func (p fieldInfoSlice) Len() int { return len(p) } func (p fieldInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name } +// FieldDetails represents detailed schema information for a field. +type FieldDetails struct { + Name string `json:"name"` + CreatedAt int64 `json:"createdAt,omitempty"` + Options FieldOptions `json:"options"` + Cardinality uint64 `json:"cardinality"` + Views []*ViewInfo `json:"views,omitempty"` +} + +type fieldDetailsSlice []*FieldDetails + +func (p fieldDetailsSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } +func (p fieldDetailsSlice) Len() int { return len(p) } +func (p fieldDetailsSlice) Less(i, j int) bool { return p[i].Name < p[j].Name } + // FieldOptions represents options to set when initializing a field. type FieldOptions struct { Base int64 `json:"base,omitempty"` diff --git a/holder.go b/holder.go index 9dc14e6a3..1bbb33984 100644 --- a/holder.go +++ b/holder.go @@ -847,7 +847,9 @@ func (h *Holder) availableShardsByIndex() map[string]*roaring.Bitmap { } // Schema returns schema information for all indexes, fields, and views. -func (h *Holder) Schema() []*IndexInfo { +// If includeHiddenAndViews=true, include fields beginning with "_", +// as well as view details. +func (h *Holder) Schema(includeHiddenAndViews bool) []*IndexInfo { var a []*IndexInfo for _, index := range h.Indexes() { di := &IndexInfo{ @@ -857,15 +859,20 @@ func (h *Holder) Schema() []*IndexInfo { ShardWidth: ShardWidth, } for _, field := range index.Fields() { + if !includeHiddenAndViews && strings.HasPrefix(field.name, "_") { + continue + } fi := &FieldInfo{ Name: field.Name(), CreatedAt: field.CreatedAt(), Options: field.Options(), } - for _, view := range field.views() { - fi.Views = append(fi.Views, &ViewInfo{Name: view.name}) + if includeHiddenAndViews { + for _, view := range field.views() { + fi.Views = append(fi.Views, &ViewInfo{Name: view.name}) + } + sort.Sort(viewInfoSlice(fi.Views)) } - sort.Sort(viewInfoSlice(fi.Views)) di.Fields = append(di.Fields, fi) } sort.Sort(fieldInfoSlice(di.Fields)) @@ -875,32 +882,38 @@ func (h *Holder) Schema() []*IndexInfo { return a } -// limitedSchema returns schema information for all non-hidden indexes and fields. -func (h *Holder) limitedSchema() []*IndexInfo { - var a []*IndexInfo +// SchemaDetails returns schema information for all non-hidden indexes and fields, +// including additional per-field details such as cardinality, actual range of integer data, etc. +// This function duplicates the logic of Holder.Schema because the FieldDetails struct +// includes a struct-field for cardinality, with default value 0, so the behavior of omitempty +// is incompatible between the /schema and /schema/details HTTP endpoints. A value of 0 for +// cardinality is meaningful, so it should be included when accurate, and not accidentally +// reported as 0 when the struct-field has not been populated. +func (h *Holder) SchemaDetails() []*IndexDetails { + var a []*IndexDetails for _, index := range h.Indexes() { - di := &IndexInfo{ + di := &IndexDetails{ Name: index.Name(), CreatedAt: index.CreatedAt(), Options: index.Options(), ShardWidth: ShardWidth, - Fields: make([]*FieldInfo, 0, len(index.Fields())), + Fields: make([]*FieldDetails, 0, len(index.Fields())), } for _, field := range index.Fields() { if strings.HasPrefix(field.name, "_") { continue } - fi := &FieldInfo{ + fi := &FieldDetails{ Name: field.Name(), CreatedAt: field.CreatedAt(), Options: field.Options(), } di.Fields = append(di.Fields, fi) } - sort.Sort(fieldInfoSlice(di.Fields)) + sort.Sort(fieldDetailsSlice(di.Fields)) a = append(a, di) } - sort.Sort(indexInfoSlice(a)) + sort.Sort(indexDetailsSlice(a)) return a } @@ -1337,7 +1350,7 @@ func (s *holderSyncer) SyncHolder() error { defer s.mu.Unlock() ti := time.Now() // Iterate over schema in sorted order. - for _, di := range s.Holder.Schema() { + for _, di := range s.Holder.Schema(true) { // Verify syncer has not closed. if s.IsClosing() { return nil diff --git a/http/handler.go b/http/handler.go index 14ef35fe1..2d98a5ec4 100644 --- a/http/handler.go +++ b/http/handler.go @@ -682,7 +682,7 @@ func (h *Handler) handleGetSchemaDetails(w http.ResponseWriter, r *http.Request) w.Header().Set("Content-Type", "application/json") schema := h.api.SchemaDetails(r.Context()) - if err := json.NewEncoder(w).Encode(pilosa.Schema{Indexes: schema}); err != nil { + if err := json.NewEncoder(w).Encode(pilosa.SchemaDetails{Indexes: schema}); err != nil { h.logger.Printf("write schema response error: %s", err) } } diff --git a/index.go b/index.go index 6129289f5..a290e2d3d 100644 --- a/index.go +++ b/index.go @@ -668,6 +668,21 @@ func (p indexInfoSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } func (p indexInfoSlice) Len() int { return len(p) } func (p indexInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name } +// IndexDetails represents detailed schema information for an index. +type IndexDetails struct { + Name string `json:"name"` + CreatedAt int64 `json:"createdAt,omitempty"` + Options IndexOptions `json:"options"` + Fields []*FieldDetails `json:"fields"` + ShardWidth uint64 `json:"shardWidth"` +} + +type indexDetailsSlice []*IndexDetails + +func (p indexDetailsSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } +func (p indexDetailsSlice) Len() int { return len(p) } +func (p indexDetailsSlice) Less(i, j int) bool { return p[i].Name < p[j].Name } + // IndexOptions represents options to set when initializing an index. type IndexOptions struct { Keys bool `json:"keys"` diff --git a/server/handler_test.go b/server/handler_test.go index 834399587..8c0c83530 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -112,6 +112,19 @@ func TestHandler_Endpoints(t *testing.T) { }) + t.Run("SchemaDetailsEmpty", func(t *testing.T) { + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/schema/details", nil)) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } + body := w.Body.String() + if body != "{\"indexes\":null}\n" { + t.Fatalf("unexpected empty schema: '%v'", body) + } + + }) + t.Run("PostSchema", func(t *testing.T) { w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/schema", strings.NewReader(`{"indexes":[{"name":"blah","options":{"keys":false,"trackExistence":true},"fields":[{"name":"f1","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":1048576}]}`))) @@ -211,6 +224,57 @@ func TestHandler_Endpoints(t *testing.T) { t.Fatal(err) } + // i2 is for SchemaDetails + i2 := hldr.MustCreateIndexIfNotExists("i2", pilosa.IndexOptions{}) + tx2, err := holder.BeginTx(true, i2.Index, shard) + if err != nil { + t.Fatal(err) + } + defer tx2.Rollback() + if f, err := i2.CreateFieldIfNotExists("f0", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 1000)); err != nil { + t.Fatal(err) + } else if _, err := f.SetBit(tx2, 0, 0, nil); err != nil { + t.Fatal(err) + } + if f, err := i2.CreateFieldIfNotExists("f1", pilosa.OptFieldTypeInt(-100, 100)); err != nil { + t.Fatal(err) + } else if _, err := f.SetBit(tx2, 0, 0, nil); err != nil { + t.Fatal(err) + } + + if _, err := cmd.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i2", Query: "Set(10, f1=4) Set(11, f1=5) Set(12, f1=6) Set(13, f1=7)"}); err != nil { + t.Fatal(err) + } + + if f, err := i2.CreateFieldIfNotExists("f2", pilosa.OptFieldTypeDecimal(1, pql.Decimal{Value: -10}, pql.Decimal{Value: 10})); err != nil { + t.Fatal(err) + } else if _, err := f.SetBit(tx2, 0, 0, nil); err != nil { + t.Fatal(err) + } + if _, err := cmd.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i2", Query: "Set(10, f2=4) Set(11, f2=5) Set(12, f2=6) Set(13, f2=7) Set(14, f2=8)"}); err != nil { + t.Fatal(err) + } + + if f, err := i2.CreateFieldIfNotExists("f3", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"))); err != nil { + t.Fatal(err) + } else if _, err := f.SetBit(tx2, 0, 0, nil); err != nil { + t.Fatal(err) + } + if f, err := i2.CreateFieldIfNotExists("f4", pilosa.OptFieldTypeMutex(pilosa.CacheTypeRanked, 5000)); err != nil { + t.Fatal(err) + } else if _, err := f.SetBit(tx2, 0, 0, nil); err != nil { + t.Fatal(err) + } + if f, err := i2.CreateFieldIfNotExists("f5", pilosa.OptFieldTypeBool()); err != nil { + t.Fatal(err) + } else if _, err := f.SetBit(tx2, 0, 0, nil); err != nil { + t.Fatal(err) + } + + if err := tx2.Commit(); err != nil { + t.Fatal(err) + } + t.Run("Schema", func(t *testing.T) { w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/schema", nil)) @@ -225,6 +289,20 @@ func TestHandler_Endpoints(t *testing.T) { } }) + t.Run("SchemaDetails", func(t *testing.T) { + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/schema/details", nil)) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } + + body := strings.TrimSpace(w.Body.String()) + target := fmt.Sprintf(`{"indexes":[{"name":"i0","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false},"cardinality":0},{"name":"f1","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false},"cardinality":1}],"shardWidth":1048576},{"name":"i1","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false},"cardinality":1}],"shardWidth":1048576},{"name":"i2","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":1000,"keys":false},"cardinality":1},{"name":"f1","options":{"type":"int","base":0,"bitDepth":3,"min":-100,"max":100,"keys":false,"foreignIndex":""},"cardinality":4},{"name":"f2","options":{"type":"decimal","base":0,"scale":1,"bitDepth":7,"min":-10,"max":10,"keys":false},"cardinality":5},{"name":"f3","options":{"type":"time","timeQuantum":"YMDH","keys":false,"noStandardView":false},"cardinality":1},{"name":"f4","options":{"type":"mutex","cacheType":"ranked","cacheSize":5000,"keys":false},"cardinality":1},{"name":"f5","options":{"type":"bool"},"cardinality":1}],"shardWidth":%[1]d}]}`, pilosa.ShardWidth) + if body != target { + t.Fatalf("%s\n!=\n%s", target, body) + } + }) + t.Run("Import", func(t *testing.T) { indexInfo := cmd.API.Schema(context.Background()) err := cmd.API.ApplySchema(context.Background(), &pilosa.Schema{Indexes: indexInfo}, false)