From 51744dc77a4e91452c8472132a1596dc66369de0 Mon Sep 17 00:00:00 2001 From: Seebs Date: Thu, 4 Mar 2021 13:32:11 -0600 Subject: [PATCH 01/14] check for nil translate store while reading translate entries If we are using replication, we can be a replica translate store for a partition, which means we start a translate store reader to replicate data for it. The translation logic does not admit *stopping* the translate reader, only "resetting" it (stopping and immediately restarting), so the translate reader just runs until it hits an error and terminates, which it does even if perhaps it shouldn't. Oops. Anyway, one potential failure mode is that if you hit timing just right, you can end up trying to process translation *while* the index is being closed, and the index can close its translation stores, and make them all nil, right before we request a store and try to use it. Another is a similar error, but during the initial startup of the translate store readers. Either way, we want to error out of the process cleanly if this happens. This could also happen during initial creation, perhaps. We're aborting translation sync on these errors, because otherwise we'd continue accepting new keys, and then end up with our highest known key being higher than some keys we missed; this way the next restart will restart from the last key we have. --- holder.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/holder.go b/holder.go index 38a1df3aa..18ae027ec 100644 --- a/holder.go +++ b/holder.go @@ -1883,6 +1883,9 @@ func (s *holderSyncer) initializeIndexTranslateReplication(snap *topology.Cluste } store := index.TranslateStore(partitionID) + if store == nil { + return fmt.Errorf("no store available for index %q, partition %d", index.Name(), partitionID) + } offset, err := store.MaxID() if err != nil { return errors.Wrapf(err, "cannot determine max id for %q", index.Name()) @@ -1974,6 +1977,10 @@ func (s *holderSyncer) readIndexTranslateReader(rd TranslateEntryReader) { // Apply replication to store. store := idx.TranslateStore(snap.KeyToKeyPartition(entry.Index, entry.Key)) + if store == nil { + s.Holder.Logger.Printf("no translate store suitable for index %q, key %q", entry.Index, entry.Key) + return + } if err := store.ForceSet(entry.ID, entry.Key); err != nil { s.Holder.Logger.Printf("cannot force set index translation data: %d=%q", entry.ID, entry.Key) return @@ -1998,6 +2005,10 @@ func (s *holderSyncer) readFieldTranslateReader(rd TranslateEntryReader) { // Apply replication to store. store := f.TranslateStore() + if store == nil { + s.Holder.Logger.Printf("no translate store suitable for index %q, field %q, key %q", entry.Index, entry.Field, entry.Key) + return + } if err := store.ForceSet(entry.ID, entry.Key); err != nil { s.Holder.Logger.Printf("cannot force set field translation data: %d=%q", entry.ID, entry.Key) return From 76e41817404398a22c485abf994be2e601851f7e Mon Sep 17 00:00:00 2001 From: Seebs Date: Fri, 5 Mar 2021 15:56:02 -0600 Subject: [PATCH 02/14] don't reuse sync.Mutex between translate readers The functional option and returned closure combine to result in us using the same sync.Mutex object for every TranslateReader on a given server, which means that if one of them isn't producing anything, we eventually end up waiting on that with all the others blocked waiting for the lock. Use separate locks for each, of the same type as the one initially provided as a template. This does mean that multiple readers can be operating at once, but in theory no two readers should ever be writing to the same stores, we think. --- http/translator.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/http/translator.go b/http/translator.go index 0722dcbf9..a5d1e4152 100644 --- a/http/translator.go +++ b/http/translator.go @@ -22,6 +22,7 @@ import ( "io" "io/ioutil" "net/http" + "reflect" "sync" "github.com/pilosa/pilosa/v2" @@ -33,8 +34,12 @@ func GetOpenTranslateReaderFunc(client *http.Client) pilosa.OpenTranslateReaderF } func GetOpenTranslateReaderWithLockerFunc(client *http.Client, locker sync.Locker) pilosa.OpenTranslateReaderFunc { + lockType := reflect.TypeOf(locker) + if lockType.Kind() == reflect.Ptr { + lockType = lockType.Elem() + } return func(ctx context.Context, nodeURL string, offsets pilosa.TranslateOffsetMap) (pilosa.TranslateEntryReader, error) { - return openTranslateReader(ctx, nodeURL, offsets, client, locker) + return openTranslateReader(ctx, nodeURL, offsets, client, reflect.New(lockType).Interface().(sync.Locker)) } } From 810f840839b7c0b0d3d5d1052d7bd81d29d264ac Mon Sep 17 00:00:00 2001 From: Seebs Date: Fri, 5 Mar 2021 16:00:52 -0600 Subject: [PATCH 03/14] combine field and index translation readers There's no need to have two different translation readers, a single reader can handle both partitions and fields at the same time, so we can combine them. This may not actually change things much but was a useful step in diagnosing a different problem with translate readers, and I think it is a minor improvement so I'm preserving the patch just in case. --- holder.go | 229 +++++++++++++++++++++++++-------------------------- translate.go | 17 ++++ 2 files changed, 131 insertions(+), 115 deletions(-) diff --git a/holder.go b/holder.go index 18ae027ec..052c725ad 100644 --- a/holder.go +++ b/holder.go @@ -1743,16 +1743,11 @@ func (s *holderSyncer) resetTranslationSync() error { // Set read-only flag for all translation stores. s.setTranslateReadOnlyFlags(snap) - // Connect to each node that has a primary for which we are a replica. - if err := s.initializeIndexTranslateReplication(snap); err != nil { - return errors.Wrap(err, "initialize index translate replication") - } - - // Connect to primary to stream field data. - if err := s.initializeFieldTranslateReplication(snap); err != nil { - return errors.Wrap(err, "initialize field translate replication") + if err := s.initializeReplication(snap); err != nil { + return errors.Wrap(err, "initializing translation replication") } return nil + } //////////////////////////////////////////////////////////// @@ -1859,11 +1854,83 @@ func (s *holderSyncer) setTranslateReadOnlyFlags(snap *topology.ClusterSnapshot) s.Cluster.mu.RUnlock() } -// initializeIndexTranslateReplication connects to each node that is the -// primary for a partition that we are a replica of. -func (s *holderSyncer) initializeIndexTranslateReplication(snap *topology.ClusterSnapshot) error { +// initializeReplication builds a map of nodes for which we need to replicate +// any key translation, whether that's field keys (every node replicates these +// from the primary) or index keys (only the replica nodes for each partition +// replicate these from whichever node is primary for that partition). +func (s *holderSyncer) initializeReplication(snap *topology.ClusterSnapshot) error { + nodeMaps := make(map[string]TranslateOffsetMap) + + if snap.ReplicaN > 1 { + if err := s.populateIndexReplication(nodeMaps, snap); err != nil { + return err + } + } + if err := s.populateFieldReplication(nodeMaps, snap); err != nil { + return err + } + + for _, node := range snap.Nodes { + m := nodeMaps[node.ID] + if m.Empty() { + continue + } + + // Connect to remote node and begin streaming. + rd, err := s.Holder.OpenTranslateReader(context.Background(), node.URI.String(), m) + if err != nil { + return err + } + s.readers = append(s.readers, rd) + + s.syncers.Go(func() error { + defer rd.Close() + s.readBothTranslateReader(rd, snap) + return nil + }) + } + return nil +} + +// populateFieldReplication populates a map from node IDs to TranslateOffsetMaps +// to record that we need to translate fields which have key translation +// from the primary node. +func (s *holderSyncer) populateFieldReplication(nodeMaps map[string]TranslateOffsetMap, snap *topology.ClusterSnapshot) error { + // Set up field translation + if !snap.IsPrimaryFieldTranslationNode(s.Cluster.Node.ID) { + primaryID := snap.PrimaryFieldTranslationNode().ID + // Build a map of field key offsets to stream from. + m := nodeMaps[primaryID] + if m == nil { + m = make(TranslateOffsetMap) + nodeMaps[primaryID] = m + } + for _, index := range s.Holder.Indexes() { + for _, field := range index.Fields() { + store := field.TranslateStore() + // I think right now this is supposed to be impossible; + // we use an InMemTranslateStore by default even if + // no translate store is being used or attempted. + if store == nil { + return fmt.Errorf("no translate store for field %q/%q", index.Name(), field.Name()) + } + offset, err := store.MaxID() + if err != nil { + return errors.Wrapf(err, "cannot determine max id for %q/%q", index.Name(), field.Name()) + } + m.SetFieldOffset(index.Name(), field.Name(), offset) + } + } + } + return nil +} + +// populateIndexReplication populates a map of node IDs to TranslateOffsetMaps +// to record which nodes we need to replicate index key translation for. +// That means nodes which are the primary for a partition that we're a +// non-primary replica for. +func (s *holderSyncer) populateIndexReplication(nodeMaps map[string]TranslateOffsetMap, snap *topology.ClusterSnapshot) error { for _, node := range snap.Nodes { - // Skip local node. if node.ID == s.Node.ID { continue } @@ -1898,117 +1965,49 @@ func (s *holderSyncer) initializeIndexTranslateReplication(snap *topology.Cluste if len(m) == 0 { continue } - - // Connect to remote node and begin streaming. - rd, err := s.Holder.OpenTranslateReader(context.Background(), node.URI.String(), m) - if err != nil { - return err - } - s.readers = append(s.readers, rd) - - s.syncers.Go(func() error { - defer rd.Close() - s.readIndexTranslateReader(rd) - return nil - }) + nodeMaps[node.ID] = m } - return nil } -// initializeFieldTranslateReplication connects the primary to stream field data. -func (s *holderSyncer) initializeFieldTranslateReplication(snap *topology.ClusterSnapshot) error { - // Skip if primary. - if snap.IsPrimaryFieldTranslationNode(s.Cluster.Node.ID) { - return nil - } +// readBothTranslateReader reads key translation for field keys or +// index keys from a remote node. Both field and index keys may be sent, +// the distinction is that field keys have a non-empty field name. +func (s *holderSyncer) readBothTranslateReader(rd TranslateEntryReader, snap *topology.ClusterSnapshot) { + for { + var entry TranslateEntry + if err := rd.ReadEntry(&entry); err != nil { + s.Holder.Logger.Printf("cannot read translate entry: %s", err) + return + } - // Build a map of partition offsets to stream from. - m := make(TranslateOffsetMap) - for _, index := range s.Holder.Indexes() { - for _, field := range index.Fields() { - store := field.TranslateStore() - offset, err := store.MaxID() - if err != nil { - return errors.Wrapf(err, "cannot determine max id for %q/%q", index.Name(), field.Name()) + var store TranslateStore + if entry.Field != "" { + // Find appropriate store. + f := s.Holder.Field(entry.Index, entry.Field) + if f == nil { + s.Holder.Logger.Printf("field not found: %s/%s", entry.Index, entry.Field) + return + } + store = f.TranslateStore() + if store == nil { + s.Holder.Logger.Printf("no translate store suitable for index %q, field %q, key %q", entry.Index, entry.Field, entry.Key) + return + } + } else { + // Find appropriate store. + idx := s.Holder.Index(entry.Index) + if idx == nil { + s.Holder.Logger.Printf("index not found: %q", entry.Index) + return + } + store = idx.TranslateStore(snap.KeyToKeyPartition(entry.Index, entry.Key)) + if store == nil { + s.Holder.Logger.Printf("no translate store suitable for index %q, key %q", entry.Index, entry.Key) + return } - m.SetFieldOffset(index.Name(), field.Name(), offset) } - } - - // Skip if no replication required. - if len(m) == 0 { - return nil - } - - // Connect to primary and begin streaming. - primary := snap.PrimaryFieldTranslationNode() - rd, err := s.Holder.OpenTranslateReader(context.Background(), primary.URI.String(), m) - if err != nil { - return err - } - s.readers = append(s.readers, rd) - - s.syncers.Go(func() error { - defer rd.Close() - s.readFieldTranslateReader(rd) - return nil - }) - return nil -} - -func (s *holderSyncer) readIndexTranslateReader(rd TranslateEntryReader) { - // Create a snapshot of the cluster to use for node/partition calculations. - snap := topology.NewClusterSnapshot(s.Cluster.noder, s.Cluster.Hasher, s.Cluster.ReplicaN) - - for { - var entry TranslateEntry - if err := rd.ReadEntry(&entry); err != nil { - s.Holder.Logger.Printf("cannot read index translate entry: %s", err) - return - } - - // Find appropriate store. - idx := s.Holder.Index(entry.Index) - if idx == nil { - s.Holder.Logger.Printf("index not found: %q", entry.Index) - return - } - // Apply replication to store. - store := idx.TranslateStore(snap.KeyToKeyPartition(entry.Index, entry.Key)) - if store == nil { - s.Holder.Logger.Printf("no translate store suitable for index %q, key %q", entry.Index, entry.Key) - return - } - if err := store.ForceSet(entry.ID, entry.Key); err != nil { - s.Holder.Logger.Printf("cannot force set index translation data: %d=%q", entry.ID, entry.Key) - return - } - } -} - -func (s *holderSyncer) readFieldTranslateReader(rd TranslateEntryReader) { - for { - var entry TranslateEntry - if err := rd.ReadEntry(&entry); err != nil { - s.Holder.Logger.Printf("cannot read field translate entry: %s", err) - return - } - - // Find appropriate store. - f := s.Holder.Field(entry.Index, entry.Field) - if f == nil { - s.Holder.Logger.Printf("field not found: %s/%s", entry.Index, entry.Field) - return - } - - // Apply replication to store. - store := f.TranslateStore() - if store == nil { - s.Holder.Logger.Printf("no translate store suitable for index %q, field %q, key %q", entry.Index, entry.Field, entry.Key) - return - } if err := store.ForceSet(entry.ID, entry.Key); err != nil { s.Holder.Logger.Printf("cannot force set field translation data: %d=%q", entry.ID, entry.Key) return diff --git a/translate.go b/translate.go index 96011d24b..948dc1761 100644 --- a/translate.go +++ b/translate.go @@ -304,6 +304,18 @@ func (m TranslateOffsetMap) FieldOffset(index, name string) uint64 { return m[index].Fields[name] } +// Empty reports whether there are any actual entries in the map. This +// is distinct from len(m) == 0 in that an entry in this map which is +// itself empty doesn't count as non-empty. +func (m TranslateOffsetMap) Empty() bool { + for _, sub := range m { + if !sub.Empty() { + return false + } + } + return true +} + // SetFieldOffset sets the offset for the given field. func (m TranslateOffsetMap) SetFieldOffset(index, name string, offset uint64) { if m[index] == nil { @@ -317,6 +329,11 @@ type IndexTranslateOffsetMap struct { Fields map[string]uint64 `json:"fields"` } +// Empty reports whether this map has neither partitions nor fields. +func (i *IndexTranslateOffsetMap) Empty() bool { + return len(i.Partitions) == 0 && len(i.Fields) == 0 +} + func NewIndexTranslateOffsetMap() *IndexTranslateOffsetMap { return &IndexTranslateOffsetMap{ Partitions: make(map[int]uint64), From 47e74f2603dac81beaa65171fbb7400db0888291 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Thu, 11 Mar 2021 11:16:53 -0600 Subject: [PATCH 04/14] Basic pilosa changes for oracle support --- cache.go | 48 +++++++++++++- cmd/random-query/main.go | 59 ++++++++++++----- executor.go | 135 ++++++++++++++++++++++++++++++++++++++- go.mod | 5 +- go.sum | 42 ++---------- handler.go | 51 +++++++++++++++ server/handler_test.go | 2 +- 7 files changed, 283 insertions(+), 59 deletions(-) diff --git a/cache.go b/cache.go index b03e4951c..1c16974f9 100644 --- a/cache.go +++ b/cache.go @@ -369,6 +369,12 @@ type Pair struct { Count uint64 `json:"count"` } +func (pair Pair) MarshalJSON() ([]byte, error) { + buffer := new(bytes.Buffer) + _, err := buffer.WriteString(fmt.Sprintf(`{"count":%v,"id":%v,"key":"%v"}`, pair.Count, pair.ID, pair.Key)) + return buffer.Bytes(), err +} + // PairField is a Pair with its associated field. type PairField struct { Pair Pair @@ -561,7 +567,47 @@ func (p *PairsField) ToRows(callback func(*pb.RowResponse) error) error { // MarshalJSON marshals PairsField into a JSON-encoded byte slice, // excluding `Field`. func (p PairsField) MarshalJSON() ([]byte, error) { - return json.Marshal(p.Pairs) + return json.Marshal(SortedPairs(p.Pairs)) +} + +type SortedPairs []Pair + +func (sortPair SortedPairs) MarshalJSON() ([]byte, error) { + sort.Slice(sortPair, func(i, j int) bool { + if sortPair[j].Count == sortPair[i].Count { + if sortPair[j].Key != "" { + return sortPair[j].Key < sortPair[i].Key + } + return sortPair[j].ID < sortPair[i].ID + } + return sortPair[j].Count < sortPair[i].Count + }) + buffer := new(bytes.Buffer) + _, err := buffer.Write([]byte("[")) + if err != nil { + return nil, err + } + for i, pair := range sortPair { + jsonValue, err := json.Marshal(pair) + if err != nil { + return nil, err + } + if i != 0 { + _, err = buffer.Write([]byte(",")) + if err != nil { + return nil, err + } + } + _, err = buffer.Write(jsonValue) + if err != nil { + return nil, err + } + } + _, err = buffer.Write([]byte("]")) + if err != nil { + return nil, err + } + return buffer.Bytes(), nil } // int64Slice represents a sortable slice of int64 numbers. diff --git a/cmd/random-query/main.go b/cmd/random-query/main.go index ead3ccd9d..5365c3c24 100644 --- a/cmd/random-query/main.go +++ b/cmd/random-query/main.go @@ -212,15 +212,18 @@ NewSetup: pql, err := cfg.GenQuery(index) panicOn(err) - if cfg.Verbose { - fmt.Printf("pql = '%v'\n", pql) - } - // Query node0. res, err := cli.Query(ctx, index, &pilosa.QueryRequest{Index: index, Query: pql}) + colorReset := "\033[0m" if err != nil { - AlwaysPrintf("QUERY FAILED! queries before this=%v; err = '%v', pql='%v'", loops, err, pql) - return err + colorRed := "\033[31m" + colorCyan := "\033[36m" + AlwaysPrintf("\n%vDIFF%v queries before this=%v;\nPQL=%v%v%v\nerr='%v'", colorRed, colorReset, loops, colorCyan, pql, colorReset, err) + } else { + colorYellow := "\033[33m" + if cfg.Verbose { + fmt.Printf("pql = %v%v%v\n", colorYellow, pql, colorReset) + } } if cfg.VeryVerbose { fmt.Printf("success on pql = '%v'; res='%v'\n", pql, res.Results[0]) @@ -374,9 +377,28 @@ func (cfg *RandomQueryConfig) Setup(api API) (err error) { } case "int": foundIntField = true - fallthrough // I bet you thought you'd never see this used + minPQL := fmt.Sprintf("Min(field=%v)", fld.Name) + resp, err := api.Query(ctx, ii.Name, &pilosa.QueryRequest{Index: ii.Name, Query: minPQL}) + if err != nil { + return err + } + min := resp.Results[0] + maxPQL := fmt.Sprintf("Max(field=%v)", fld.Name) + resp, err = api.Query(ctx, ii.Name, &pilosa.QueryRequest{Index: ii.Name, Query: maxPQL}) + if err != nil { + return err + } + max := resp.Results[0] + m := pql.Decimal{} + x := pql.Decimal{} + m.Scale = fld.Options.Scale + m.Value = min.(pilosa.ValCount).Val + x.Value = max.(pilosa.ValCount).Val + x.Scale = fld.Options.Scale + + cfg.AddIntField(ii.Name, fld.Name, m, x, fld.Options.Scale, fld.Options.Type == "decimal") case "decimal": - // we'll ignore row keys and just use value ranges + vv("decimal '%#v'", fld) cfg.AddIntField(ii.Name, fld.Name, fld.Options.Min, fld.Options.Max, fld.Options.Scale, fld.Options.Type == "decimal") default: AlwaysPrintf("ignoring field %q: unhandled type %q\n", fld.Name, fld.Options.Type) @@ -451,10 +473,11 @@ func (cfg *RandomQueryConfig) AddIntField(index, field string, min, max pql.Deci func (cfg *RandomQueryConfig) GenQuery(index string) (pql string, err error) { tree := cfg.GenTree(index, cfg.TreeDepth) - - if cfg.Verbose { - fmt.Printf("%v\n", tree.StringIndent(0)) - } + /* + if cfg.Verbose { + fmt.Printf("%v\n", tree.StringIndent(0)) + } + */ pql = tree.ToPQL() // avoid using too much bandwidth, just count the final bitmap. @@ -508,18 +531,20 @@ func (cfg *RandomQueryConfig) GenTree(index string, depth int) (tr *Tree) { tr = &Tree{S: f} numChild := 2 switch f { - case "Union", "Intersect", "Xor": + case "Union", "Intersect": numChild = cfg.Rnd.Intn(8) + 2 + case "Xor": + numChild = cfg.Rnd.Intn(2) + 1 case "Not": numChild = 1 case "Difference": numChild = 2 case "Distinct": numChild = 1 - // sometimes do a bare distinct without a filter - if cfg.Rnd.Intn(10) == 0 { - numChild = 0 - } + // sometimes do a bare distinct without a filter //TODO (twg) i don't think this is valid + //if cfg.Rnd.Intn(10) == 0 { + //numChild = 0 + // } r = cfg.Rnd.Intn(len(features.Distinctables)) tr.Args = append(tr.Args, fmt.Sprintf("field=%s", features.Distinctables[r].Field)) } diff --git a/executor.go b/executor.go index 0e869ff64..1db8fde7d 100644 --- a/executor.go +++ b/executor.go @@ -15,6 +15,7 @@ package pilosa import ( + "bytes" "context" "encoding/json" "fmt" @@ -3162,6 +3163,16 @@ type FieldRow struct { Value *int64 `json:"value,omitempty"` } +func (p FieldRow) Less(o FieldRow) bool { + if p.Field == o.Field { + if p.RowKey != "" { + return p.RowKey < o.RowKey + } + return p.RowID < o.RowID + } + return p.Field < o.Field +} + func (fr *FieldRow) Clone() (clone *FieldRow) { clone = &FieldRow{ Field: fr.Field, @@ -3328,7 +3339,7 @@ func (g *GroupCounts) ToRows(callback func(*pb.RowResponse) error) error { // customizes the JSON output of the aggregate field label. func (g *GroupCounts) MarshalJSON() ([]byte, error) { groups := g.Groups() - var counts interface{} = groups + var counts interface{} = SortedGroupCount(groups) if len(groups) == 0 { return []byte("[]"), nil @@ -3342,6 +3353,50 @@ func (g *GroupCounts) MarshalJSON() ([]byte, error) { return json.Marshal(counts) } +type SortedGroupCount []GroupCount + +// Provides determanistic ordering for the JSON output +func (sortgroup SortedGroupCount) MarshalJSON() ([]byte, error) { + sort.Slice(sortgroup, func(i, j int) bool { + if sortgroup[j].Count == sortgroup[i].Count { + for x := range sortgroup[j].Group { + if sortgroup[j].Group[x] == sortgroup[i].Group[x] { + continue + } + return sortgroup[j].Group[x].Less(sortgroup[i].Group[x]) + } + + } + return sortgroup[j].Count < sortgroup[i].Count + }) + buffer := new(bytes.Buffer) + _, err := buffer.Write([]byte("[")) + if err != nil { + return nil, err + } + for i, groupCount := range sortgroup { + jsonValue, err := json.Marshal(groupCount) + if err != nil { + return nil, err + } + if i != 0 { + _, err = buffer.Write([]byte(",")) + if err != nil { + return nil, err + } + } + _, err = buffer.Write(jsonValue) + if err != nil { + return nil, err + } + } + _, err = buffer.Write([]byte("]")) + if err != nil { + return nil, err + } + return buffer.Bytes(), nil +} + // GroupCount represents a result item for a group by query. type GroupCount struct { Group []FieldRow `json:"group"` @@ -3349,6 +3404,32 @@ type GroupCount struct { Agg int64 `json:"-"` } +// comparing json structures in the oracle demand that map values have a consistent order +// docs say it should be lexigraphically but experimentation proved otherwise int the +// groupcount state +func (group GroupCount) MarshalJSON() ([]byte, error) { + // TODO(twg) figure out how to access struct tags from code + var buf bytes.Buffer + sort.Slice(group.Group, func(i, j int) bool { + return group.Group[i].Field < group.Group[j].Field + }) + + //write the Counts + buf.WriteString(fmt.Sprintf(`{"count": %v,`, group.Count)) + //write the Groups + buf.WriteString(`"group":[`) + for i, g := range group.Group { + part, _ := g.MarshalJSON() + if i != 0 { + buf.Write([]byte(",")) + } + buf.Write(part) + } + + buf.WriteString("]}") + return buf.Bytes(), nil +} + type groupCountSum struct { Group []FieldRow `json:"group"` Count uint64 `json:"count"` @@ -3845,6 +3926,43 @@ type ExtractedTableColumn struct { Rows []interface{} `json:"rows"` } +func (etc ExtractedTableColumn) MarshalJSON() ([]byte, error) { + buf := new(bytes.Buffer) + buf.WriteString(`{"column":`) + b, _ := etc.Column.MarshalJSON() + buf.Write(b) + buf.WriteString(`,"rows":[`) + for i, row := range etc.Rows { + if i != 0 { + buf.WriteByte(44) //write a comma + } + buf.WriteString(`[`) + switch v := row.(type) { + case []string: + sort.Slice(v, func(i, j int) bool { + return v[i] < v[j] + }) + for si, s := range v { + if si != 0 { + buf.WriteByte(44) //write a comma + } + buf.WriteString(fmt.Sprintf(`"%v"`, s)) + } + case []uint64: + for si, s := range v { + if si != 0 { + buf.WriteByte(44) //write a comma + } + buf.WriteString(fmt.Sprintf(`%v`, s)) + } + } + buf.WriteString(`]`) + + } + buf.WriteString(`]}`) + return buf.Bytes(), nil +} + type ExtractedTable struct { Fields []ExtractedTableField `json:"fields"` Columns []ExtractedTableColumn `json:"columns"` @@ -7407,6 +7525,21 @@ type ValCount struct { Count int64 `json:"count"` } +func (vc ValCount) MarshalJSON() ([]byte, error) { + buf := new(bytes.Buffer) + nullOrVal := "null" + if vc.DecimalVal != nil { + nullOrVal = fmt.Sprintf("%v", vc.DecimalVal) + } + _, err := buf.WriteString(fmt.Sprintf(`{ + "count": %v, + "decimalValue": %v, + "floatValue": %v, + "value":%v + }`, vc.Count, nullOrVal, vc.FloatVal, vc.Val)) + return buf.Bytes(), err +} + func (v *ValCount) Clone() (r *ValCount) { r = &ValCount{ Val: v.Val, diff --git a/go.mod b/go.mod index 671f599af..97f7a944e 100644 --- a/go.mod +++ b/go.mod @@ -34,7 +34,6 @@ require ( github.com/prometheus/client_model v0.1.0 github.com/prometheus/prom2json v1.3.0 github.com/rakyll/statik v0.1.7 - github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 // indirect github.com/rs/cors v1.7.0 // indirect github.com/satori/go.uuid v1.2.0 github.com/shirou/gopsutil/v3 v3.20.11 @@ -52,12 +51,12 @@ require ( golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449 golang.org/x/net v0.0.0-20200822124328-c89045814202 // indirect golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 - golang.org/x/sys v0.0.0-20201214095126-aec9a390925b // indirect + golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect google.golang.org/grpc v1.28.0 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect gopkg.in/yaml.v2 v2.3.0 // indirect - modernc.org/mathutil v1.0.0 + modernc.org/mathutil v1.2.2 modernc.org/strutil v1.0.0 sigs.k8s.io/yaml v1.2.0 // indirect vitess.io/vitess v3.0.0-rc.3.0.20190602171040-12bfde34629c+incompatible diff --git a/go.sum b/go.sum index 7ee30647c..432425473 100644 --- a/go.sum +++ b/go.sum @@ -28,7 +28,6 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/beevik/ntp v0.3.0 h1:xzVrPrE4ziasFXgBVBZJDP0Wg/KpMwk2KHJ4Ba8GrDw= @@ -49,9 +48,7 @@ github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= -github.com/coreos/bbolt v1.3.2 h1:wZwiHHUieZCquLkDL0B8UhzreNWsPHooDAG3q34zk0s= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= -github.com/coreos/etcd v3.3.13+incompatible h1:8F3hqu9fGYLBifCmRCJsicFqDx/D68Rt3q1JMazcgBQ= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= @@ -81,7 +78,6 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= @@ -113,7 +109,6 @@ github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFU github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= @@ -128,7 +123,6 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -152,39 +146,28 @@ github.com/gorilla/mux v1.7.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2z github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 h1:Iju5GlWwrvL6UBg4zJJt3btmonfrMlCDdsejg4CZE7c= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 h1:z53tR0945TRRQO/fLEVPI6SMv7ZflF0TEaTAoU7tOzg= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.9.0 h1:bM6ZAFZmc/wPFaRDi0d5L7hGEZEx/2u+Tmr2evNHDiI= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5 h1:UImYN5qQ8tuGpGE16ZmjvcTtTw24zw1QAp/SlnNrZhI= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= -github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack v0.5.3 h1:zKjpN5BK/P5lMYrLmBHdBULWbJ0XpYR+7NGzqkZzoD4= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= -github.com/hashicorp/go-sockaddr v1.0.0 h1:GeH6tui99pF4NJgfnhp+L6+FfobzVW3Ah46sLo0ICXs= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= -github.com/hashicorp/go-uuid v1.0.0 h1:RS8zrF7PhGwyNPOtxSClXXj9HA8feRnJzgnI1RJCSnM= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= -github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= @@ -198,7 +181,6 @@ github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NH github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= -github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -207,16 +189,13 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= -github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= @@ -230,11 +209,9 @@ github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNx github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/miekg/dns v1.0.14 h1:9jZdLNd/P4+SfEJ0TNyxYpsK8N4GtfylBLqtbYN1sbA= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= @@ -260,7 +237,6 @@ github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c h1:Lgl0gzECD8GnQ5QCWA8o6BtfL6mDH5rQgM4/fX3avOs= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= @@ -294,8 +270,8 @@ github.com/prometheus/prom2json v1.3.0/go.mod h1:rMN7m0ApCowcoDlypBHlkNbp5eJQf/+ github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rakyll/statik v0.1.7 h1:OF3QCZUuyPxuGEP7B4ypUa7sB/iHtqOTDYZXGM8KOdQ= github.com/rakyll/statik v0.1.7/go.mod h1:AlZONWzMtEnMs7W4e/1LURLiI49pIMmp6V9Unghqrcc= -github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 h1:HQagqIiBmr8YXawX/le3+O26N+vPPC1PtjaF3mwnook= -github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk= +github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= @@ -304,7 +280,6 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= -github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/shirou/gopsutil/v3 v3.20.11 h1:NeVf1K0cgxsWz+N3671ojRptdgzvp7BXL3KV21R0JnA= github.com/shirou/gopsutil/v3 v3.20.11/go.mod h1:igHnfak0qnw1biGeI2qKQvu0ZkwvEkUcCLlYhZzdr/4= @@ -338,11 +313,9 @@ github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5q github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk= github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -456,11 +429,10 @@ golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201024232916-9f70ab9862d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201214095126-aec9a390925b h1:tv7/y4pd+sR8bcNb2D6o7BNU6zjWm0VjQLac+w7fNNM= -golang.org/x/sys v0.0.0-20201214095126-aec9a390925b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c h1:VwygUrnw9jn88c4u8GD3rZQbqrP/tgas88tPUbBxQrk= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= @@ -510,7 +482,6 @@ google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a h1:Ob5/580gVHBJZgXnff1cZDbG+xLtMVE5mDRTe+nIsX4= @@ -542,7 +513,6 @@ gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -555,8 +525,8 @@ honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -modernc.org/mathutil v1.0.0 h1:93vKjrJopTPrtTNpZ8XIovER7iCIH1QU7wNbOQXC60I= -modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= +modernc.org/mathutil v1.2.2 h1:+yFk8hBprV+4c0U9GjFtL+dV3N8hOJ8JCituQcMShFY= +modernc.org/mathutil v1.2.2/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= modernc.org/strutil v1.0.0 h1:XVFtQwFVwc02Wk+0L/Z/zDDXO81r5Lhe6iMKmGX3KhE= modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= diff --git a/handler.go b/handler.go index ade7c9c26..c9d6fa1d2 100644 --- a/handler.go +++ b/handler.go @@ -16,6 +16,7 @@ package pilosa import ( "encoding/json" + "sort" "github.com/pilosa/pilosa/v2/tracing" "github.com/pkg/errors" @@ -98,6 +99,56 @@ func (resp *QueryResponse) MarshalJSON() ([]byte, error) { }) } +// used in the oracle, orders keys lexigraphically +func (resp *QueryResponse) Equals(other *QueryResponse) bool { + other.Sort() + resp.Sort() + + a, err := resp.MarshalJSON() + panicOn(err) + b, err := other.MarshalJSON() + panicOn(err) + return string(a) == string(b) + +} + +// SortIfIsRowIdentifiers forces a certain order of rowkeys so that the sauron +// tool gets comparable results +func SortIfIsRowIdentifiers(i interface{}) (bool, interface{}) { + if m, ok := i.(map[string]interface{}); ok { + it, ok := m["keys"] + if ok { + items := it.([]interface{}) + if len(items) > 0 { + sort.Slice(items, func(p, q int) bool { + return items[p].(string) < items[q].(string) + }) + return true, items + //return true, ToGenericArray(sitems) + } + } + + } + return false, nil +} + +// TODO (twg) probably going to get fancy here but hardcode to test idea +func (resp *QueryResponse) Sort() bool { + modified := false + if resp == nil || resp.Results == nil { + return false + } + for i, result := range resp.Results { + sorted, results := SortIfIsRowIdentifiers(result) + if sorted { + resp.Results[i] = results + modified = true + } + } + + return modified +} + // Handler is the interface for the data handler, a wrapper around // Pilosa's data store. type Handler interface { diff --git a/server/handler_test.go b/server/handler_test.go index 0d5ac0974..afc8be8e0 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -790,7 +790,7 @@ func TestHandler_Endpoints(t *testing.T) { h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query", strings.NewReader(`TopN(f0, n=2)`))) if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"results":[[{"id":30,"key":"","count":3},{"id":31,"key":"","count":1}]]}`+"\n" { + } else if body := w.Body.String(); body != `{"results":[[{"count":3,"id":30,"key":""},{"count":1,"id":31,"key":""}]]}`+"\n" { t.Fatalf("unexpected body: %q", body) } }) From 8e123b821d68e26b11833230d2bd7314671cd4bc Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Thu, 11 Mar 2021 11:23:20 -0600 Subject: [PATCH 05/14] Replace null with [] in schema response --- holder.go | 4 ++-- server/handler_test.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/holder.go b/holder.go index 052c725ad..375384271 100644 --- a/holder.go +++ b/holder.go @@ -906,13 +906,13 @@ func (h *Holder) limitedSchema() ([]*IndexInfo, error) { } func (h *Holder) schema(ctx context.Context, includeViews bool) ([]*IndexInfo, error) { - var a []*IndexInfo - schema, err := h.schemator.Schema(ctx) if err != nil { return nil, errors.Wrapf(err, "getting schema via schemator") } + a := make([]*IndexInfo, 0, len(schema)) + for _, index := range schema { cim, err := decodeCreateIndexMessage(h.serializer, index.Data) if err != nil { diff --git a/server/handler_test.go b/server/handler_test.go index 0d5ac0974..ccd3c5b03 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -106,7 +106,7 @@ func TestHandler_Endpoints(t *testing.T) { t.Fatalf("unexpected status code: %d", w.Code) } body := w.Body.String() - if body != "{\"indexes\":null}\n" { + if body != "{\"indexes\":[]}\n" { t.Fatalf("unexpected empty schema: '%v'", body) } @@ -119,7 +119,7 @@ func TestHandler_Endpoints(t *testing.T) { t.Fatalf("unexpected status code: %d", w.Code) } body := w.Body.String() - if body != "{\"indexes\":null}\n" { + if body != "{\"indexes\":[]}\n" { t.Fatalf("unexpected empty schema: '%v'", body) } From a5dceaae11ce5e222928b82ab83bfc2b0bfc326d Mon Sep 17 00:00:00 2001 From: tgruben Date: Thu, 11 Mar 2021 11:26:58 -0600 Subject: [PATCH 06/14] Update main.go --- cmd/random-query/main.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/cmd/random-query/main.go b/cmd/random-query/main.go index 5365c3c24..edd83d07b 100644 --- a/cmd/random-query/main.go +++ b/cmd/random-query/main.go @@ -398,7 +398,6 @@ func (cfg *RandomQueryConfig) Setup(api API) (err error) { cfg.AddIntField(ii.Name, fld.Name, m, x, fld.Options.Scale, fld.Options.Type == "decimal") case "decimal": - vv("decimal '%#v'", fld) cfg.AddIntField(ii.Name, fld.Name, fld.Options.Min, fld.Options.Max, fld.Options.Scale, fld.Options.Type == "decimal") default: AlwaysPrintf("ignoring field %q: unhandled type %q\n", fld.Name, fld.Options.Type) @@ -541,10 +540,6 @@ func (cfg *RandomQueryConfig) GenTree(index string, depth int) (tr *Tree) { numChild = 2 case "Distinct": numChild = 1 - // sometimes do a bare distinct without a filter //TODO (twg) i don't think this is valid - //if cfg.Rnd.Intn(10) == 0 { - //numChild = 0 - // } r = cfg.Rnd.Intn(len(features.Distinctables)) tr.Args = append(tr.Args, fmt.Sprintf("field=%s", features.Distinctables[r].Field)) } From 21910ef321f0da6aa5400eabb9c5a4df21f47b0a Mon Sep 17 00:00:00 2001 From: tgruben Date: Thu, 11 Mar 2021 11:47:57 -0600 Subject: [PATCH 07/14] Update main.go --- cmd/random-query/main.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/cmd/random-query/main.go b/cmd/random-query/main.go index edd83d07b..4bb0962d1 100644 --- a/cmd/random-query/main.go +++ b/cmd/random-query/main.go @@ -472,11 +472,6 @@ func (cfg *RandomQueryConfig) AddIntField(index, field string, min, max pql.Deci func (cfg *RandomQueryConfig) GenQuery(index string) (pql string, err error) { tree := cfg.GenTree(index, cfg.TreeDepth) - /* - if cfg.Verbose { - fmt.Printf("%v\n", tree.StringIndent(0)) - } - */ pql = tree.ToPQL() // avoid using too much bandwidth, just count the final bitmap. From 9e0ea9709a3eaac9374b469d94b4a77a87409f9a Mon Sep 17 00:00:00 2001 From: Travis Date: Thu, 11 Mar 2021 15:49:32 -0600 Subject: [PATCH 08/14] Create the data directory during "log startup" if it doesn't already exist. Prior to using etcd for node membership, the data director was created during the cluster topology setup. Since that no longer exists, we weren't actually creating the data directory before getting to logStartup(). So this change ensure that the data directory exists. --- holder.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/holder.go b/holder.go index 375384271..8df98105d 100644 --- a/holder.go +++ b/holder.go @@ -1478,6 +1478,10 @@ func (h *Holder) logStartup() error { time := time.Now().Format(RFC3339NanoFixedWidth) logLine := fmt.Sprintf("%s\t%s\n", time, Version) + if err := os.MkdirAll(h.path, 0777); err != nil { + return errors.Wrap(err, "creating data directory") + } + f, err := os.OpenFile(h.path+"/startup.log", os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600) if err != nil { return errors.Wrap(err, "opening startup log") From 1045268f012446675849cfb9ab89a667f167369a Mon Sep 17 00:00:00 2001 From: Seebs Date: Wed, 10 Mar 2021 13:37:48 -0600 Subject: [PATCH 09/14] use testhook to ensure temporary files and directories are cleaned up In nearly all cases, we can just switch ioutil.TempDir->testhook.TempDir and similarly for TempFile. There's one case where we can't because we need files to be removed before tests are over. Also in the process give identifiable names to a lot of temporary files and make sure they're being cleaned up, and don't use "/tmp/foo" as a file name in a test that could be running in more than one test process at once. :) --- attr_test.go | 25 +++++++++++++---------- boltdb/translate.go | 2 -- boltdb/translate_test.go | 41 +++++++++++++++++-------------------- cmd/root_test.go | 7 ++++--- cmd/server_test.go | 10 ++++----- ctl/check_test.go | 34 +++++++++++++++---------------- ctl/import_test.go | 33 +++++++++++++++--------------- ctl/inspect_test.go | 5 +++-- dbshard_internal_test.go | 6 +++--- etcd/leasedkv_test.go | 4 ++-- fragment_internal_test.go | 11 ++++++++-- idalloc_test.go | 9 ++------ logger/filewriter_test.go | 43 ++++++++++++++++++--------------------- rbf/db_test.go | 2 +- rbf/ingest_test.go | 6 ++++-- rbf/rbf_test.go | 8 ++++---- rbf/util_test.go | 4 ++-- server/cluster_test.go | 2 +- test/disco.go | 5 ++--- testhook/hook.go | 33 +++++++++++++++++++++++++++++- 20 files changed, 161 insertions(+), 129 deletions(-) diff --git a/attr_test.go b/attr_test.go index 38af27c5a..cc40daf3c 100644 --- a/attr_test.go +++ b/attr_test.go @@ -15,7 +15,6 @@ package pilosa_test import ( - "io/ioutil" "os" "reflect" "runtime" @@ -24,11 +23,12 @@ import ( "github.com/pilosa/pilosa/v2" "github.com/pilosa/pilosa/v2/boltdb" + "github.com/pilosa/pilosa/v2/testhook" ) // Ensure database can set and retrieve column attributes. func TestAttrStore_Attrs(t *testing.T) { - s := MustOpenAttrStore() + s := MustOpenAttrStore(t) defer s.Close() // Set attributes. @@ -57,7 +57,7 @@ func TestAttrStore_Attrs(t *testing.T) { // Ensure database returns a non-nil empty map if unset. func TestAttrStore_Attrs_Empty(t *testing.T) { - s := MustOpenAttrStore() + s := MustOpenAttrStore(t) defer s.Close() if m, err := s.Attrs(100); err != nil { @@ -69,7 +69,7 @@ func TestAttrStore_Attrs_Empty(t *testing.T) { // Ensure database can unset attributes if explicitly set to nil. func TestAttrStore_Attrs_Unset(t *testing.T) { - s := MustOpenAttrStore() + s := MustOpenAttrStore(t) defer s.Close() // Set attributes. @@ -89,7 +89,7 @@ func TestAttrStore_Attrs_Unset(t *testing.T) { // Ensure attribute block checksums can be returned. func TestAttrStore_Blocks(t *testing.T) { - s := MustOpenAttrStore() + s := MustOpenAttrStore(t) defer s.Close() // Set attributes. @@ -135,19 +135,22 @@ type AttrStore struct { } // NewAttrStore returns a new instance of AttrStore. -func NewAttrStore(string) pilosa.AttrStore { - f, err := ioutil.TempFile("", "pilosa-attr-") +func NewAttrStore(tb testing.TB) pilosa.AttrStore { + f, err := testhook.TempFile(tb, "pilosa-attr-") if err != nil { panic(err) } + // Note, even though the file is closed, TempFile will still avoid + // creating the same name again if we leave it existing. The boltdb + // code may already be deleting this, so the TestHook deletion + // may not matter but it's more reliable this way. f.Close() - os.Remove(f.Name()) return &AttrStore{boltdb.NewAttrStore(f.Name())} } func BenchmarkAttrStore_Duplicate(b *testing.B) { - s := MustOpenAttrStore() + s := MustOpenAttrStore(b) defer s.Close() // Set attributes. @@ -186,8 +189,8 @@ func BenchmarkAttrStore_Duplicate(b *testing.B) { } // MustOpenAttrStore returns a new, opened attribute store at a temporary path. Panic on error. -func MustOpenAttrStore() pilosa.AttrStore { - s := NewAttrStore("") +func MustOpenAttrStore(tb testing.TB) pilosa.AttrStore { + s := NewAttrStore(tb) if err := s.Open(); err != nil { panic(err) } diff --git a/boltdb/translate.go b/boltdb/translate.go index 00f0e657c..2df5114b2 100644 --- a/boltdb/translate.go +++ b/boltdb/translate.go @@ -19,7 +19,6 @@ import ( "context" "fmt" "io" - "io/ioutil" "os" "path/filepath" "sync" @@ -32,7 +31,6 @@ import ( "runtime/pprof" ) -var _ = ioutil.TempFile var _ = pprof.StartCPUProfile var ( diff --git a/boltdb/translate_test.go b/boltdb/translate_test.go index 6908da6ac..c016b0e63 100644 --- a/boltdb/translate_test.go +++ b/boltdb/translate_test.go @@ -17,8 +17,6 @@ import ( "bytes" "context" "fmt" - "io/ioutil" - "os" "reflect" "strconv" "testing" @@ -26,13 +24,14 @@ import ( "github.com/pilosa/pilosa/v2" "github.com/pilosa/pilosa/v2/boltdb" + "github.com/pilosa/pilosa/v2/testhook" "github.com/pilosa/pilosa/v2/topology" ) //var vv = pilosa.VV func TestTranslateStore_TranslateKey(t *testing.T) { - s := MustOpenNewTranslateStore() + s := MustOpenNewTranslateStore(t) defer MustCloseTranslateStore(s) // Ensure initial key translates to first ID for shard @@ -57,7 +56,7 @@ func TestTranslateStore_TranslateKey(t *testing.T) { } func TestTranslateStore_TranslateKeys(t *testing.T) { - s := MustOpenNewTranslateStore() + s := MustOpenNewTranslateStore(t) defer MustCloseTranslateStore(s) ids, err := s.TranslateKeys([]string{"abc", "abc"}, true) @@ -97,7 +96,7 @@ func TestTranslateStore_TranslateKeys(t *testing.T) { } func TestTranslateStore_CreateKeys(t *testing.T) { - s := MustOpenNewTranslateStore() + s := MustOpenNewTranslateStore(t) defer MustCloseTranslateStore(s) ids, err := s.CreateKeys("abc", "abc") @@ -137,7 +136,7 @@ func TestTranslateStore_CreateKeys(t *testing.T) { } func TestTranslateStore_ReadKey(t *testing.T) { - s := MustOpenNewTranslateStore() + s := MustOpenNewTranslateStore(t) defer MustCloseTranslateStore(s) id, err := s.TranslateKey("foo", false) @@ -172,7 +171,7 @@ func TestTranslateStore_ReadKey(t *testing.T) { } func TestTranslateStore_ReadKeys(t *testing.T) { - s := MustOpenNewTranslateStore() + s := MustOpenNewTranslateStore(t) defer MustCloseTranslateStore(s) ids, err := s.TranslateKeys([]string{"foo", "bar", "baz", "baz", "bar", "foo"}, false) @@ -200,7 +199,7 @@ func TestTranslateStore_ReadKeys(t *testing.T) { } } func TestTranslateStore_TranslateID(t *testing.T) { - s := MustOpenNewTranslateStore() + s := MustOpenNewTranslateStore(t) defer MustCloseTranslateStore(s) // Setup initial keys. @@ -239,7 +238,7 @@ func TestTranslateStore_TranslateID(t *testing.T) { } func TestTranslateStore_TranslateIDs(t *testing.T) { - s := MustOpenNewTranslateStore() + s := MustOpenNewTranslateStore(t) defer MustCloseTranslateStore(s) // Setup initial keys. @@ -293,7 +292,7 @@ func TestTranslateStore_FindKeys(t *testing.T) { for _, c := range cases { c := c t.Run(c.name, func(t *testing.T) { - s := MustOpenNewTranslateStore() + s := MustOpenNewTranslateStore(t) defer MustCloseTranslateStore(s) var naiveMap map[string]uint64 @@ -339,7 +338,7 @@ func TestTranslateStore_FindKeys(t *testing.T) { } func TestTranslateStore_MaxID(t *testing.T) { - s := MustOpenNewTranslateStore() + s := MustOpenNewTranslateStore(t) defer MustCloseTranslateStore(s) // Generate a bunch of keys. @@ -364,7 +363,7 @@ func TestTranslateStore_MaxID(t *testing.T) { func TestTranslateStore_EntryReader(t *testing.T) { t.Run("OK", func(t *testing.T) { - s := MustOpenNewTranslateStore() + s := MustOpenNewTranslateStore(t) defer MustCloseTranslateStore(s) // Create multiple new keys. @@ -422,7 +421,7 @@ func TestTranslateStore_EntryReader(t *testing.T) { // Ensure reader will read as soon as a new write comes in using WriteNotify(). t.Run("WriteNotify", func(t *testing.T) { - s := MustOpenNewTranslateStore() + s := MustOpenNewTranslateStore(t) defer MustCloseTranslateStore(s) // Start reader from initial position. @@ -465,7 +464,7 @@ func TestTranslateStore_EntryReader(t *testing.T) { // Ensure exits read on close. t.Run("Close", func(t *testing.T) { - s := MustOpenNewTranslateStore() + s := MustOpenNewTranslateStore(t) defer MustCloseTranslateStore(s) // Start reader from initial position. @@ -499,7 +498,7 @@ func TestTranslateStore_EntryReader(t *testing.T) { // Ensure exits read on store close. t.Run("StoreClose", func(t *testing.T) { - s := MustOpenNewTranslateStore() + s := MustOpenNewTranslateStore(t) defer MustCloseTranslateStore(s) // Start reader from initial position. @@ -533,8 +532,8 @@ func TestTranslateStore_EntryReader(t *testing.T) { } // MustNewTranslateStore returns a new TranslateStore with a temporary path. -func MustNewTranslateStore() *boltdb.TranslateStore { - f, err := ioutil.TempFile("", "") +func MustNewTranslateStore(tb testing.TB) *boltdb.TranslateStore { + f, err := testhook.TempFile(tb, "translate-store") if err != nil { panic(err) } else if err := f.Close(); err != nil { @@ -548,7 +547,7 @@ func MustNewTranslateStore() *boltdb.TranslateStore { func TestTranslateStore_ReadWrite(t *testing.T) { t.Run("WriteTo_ReadFrom", func(t *testing.T) { - s := MustOpenNewTranslateStore() + s := MustOpenNewTranslateStore(t) defer MustCloseTranslateStore(s) batch0 := []string{} @@ -612,8 +611,8 @@ func TestTranslateStore_ReadWrite(t *testing.T) { } // MustOpenNewTranslateStore returns a new, opened TranslateStore. -func MustOpenNewTranslateStore() *boltdb.TranslateStore { - s := MustNewTranslateStore() +func MustOpenNewTranslateStore(tb testing.TB) *boltdb.TranslateStore { + s := MustNewTranslateStore(tb) if err := s.Open(); err != nil { panic(err) } @@ -624,7 +623,5 @@ func MustOpenNewTranslateStore() *boltdb.TranslateStore { func MustCloseTranslateStore(s *boltdb.TranslateStore) { if err := s.Close(); err != nil { panic(err) - } else if err := os.Remove(s.Path); err != nil { - panic(err) } } diff --git a/cmd/root_test.go b/cmd/root_test.go index ec998d346..26da96799 100644 --- a/cmd/root_test.go +++ b/cmd/root_test.go @@ -26,6 +26,7 @@ import ( "time" "github.com/pilosa/pilosa/v2/cmd" + "github.com/pilosa/pilosa/v2/testhook" "github.com/spf13/cobra" ) @@ -135,7 +136,7 @@ func executeDry(t *testing.T, tests []commandTest) { // a temp config file with the cfgFileContent string as its content. func (ct *commandTest) setupCommand(t *testing.T) *cobra.Command { // make config file - cfgFile, err := ioutil.TempFile("", "") + cfgFile, err := testhook.TempFile(t, "cmdconf") failErr(t, err, "making temp file") _, err = cfgFile.WriteString(ct.cfgFileContent) failErr(t, err, "writing config to temp file") @@ -180,9 +181,9 @@ func TestRootCommand(t *testing.T) { } func TestRootCommand_Config(t *testing.T) { - file, err := ioutil.TempFile("", "test.conf") + file, err := testhook.TempFile(t, "test.conf") if err != nil { - panic(err) + t.Fatalf("creating config file: %v", err) } config := `data-dir = "/tmp/pil5_0" bind = "127.0.0.1:10101" diff --git a/cmd/server_test.go b/cmd/server_test.go index f698b9e20..452d1db18 100644 --- a/cmd/server_test.go +++ b/cmd/server_test.go @@ -16,13 +16,13 @@ package cmd_test import ( "fmt" - "io/ioutil" "strings" "testing" "time" "github.com/pilosa/pilosa/v2/cmd" _ "github.com/pilosa/pilosa/v2/test" + "github.com/pilosa/pilosa/v2/testhook" "github.com/pilosa/pilosa/v2/toml" "github.com/pkg/errors" ) @@ -42,9 +42,9 @@ func nextPort() string { //nolint:unused func TestServerConfig(t *testing.T) { t.Skip("pilosa hosts config (cmd.Server.Config.Cluster.Hosts and brethren) is test only and will go away with high probability. skip for now.") - actualDataDir, err := ioutil.TempDir("", "") + actualDataDir, err := testhook.TempDir(t, "") failErr(t, err, "making data dir") - logFile, err := ioutil.TempFile("", "") + logFile, err := testhook.TempFile(t, "") failErr(t, err, "making log file") tests := []commandTest{ // TEST 0 @@ -64,7 +64,7 @@ func TestServerConfig(t *testing.T) { bind-grpc = ` + nextPort() + ` max-writes-per-request = 3000 long-query-time = "1m10s" - + [cluster] replicas = 2 long-query-time = "1m10s" @@ -189,7 +189,7 @@ func TestServerConfig(t *testing.T) { } func TestServerConfig_DeprecateLongQueryTime(t *testing.T) { t.Skip("pilosa hosts config (cmd.Server.Config.Cluster.Hosts and brethren) is test only and will go away with high probability. skip for now.") - actualDataDir, err := ioutil.TempDir("", "") + actualDataDir, err := testhook.TempDir(t, "") failErr(t, err, "making data dir") tests := []commandTest{ diff --git a/ctl/check_test.go b/ctl/check_test.go index ab5ef9279..f5f466fd6 100644 --- a/ctl/check_test.go +++ b/ctl/check_test.go @@ -16,20 +16,22 @@ package ctl import ( "bytes" - "encoding/hex" "io" - "io/ioutil" - "math/rand" "os" - "path/filepath" "strings" "testing" "context" + + "github.com/pilosa/pilosa/v2/testhook" ) func TestCheckCommand_RunCacheFile(t *testing.T) { - cacheFile := TempFileName("test", ".cache") + fi, err := testhook.TempFile(t, "test*.cache") + if err != nil { + t.Fatalf("creating test file: %v", err) + } + cacheFile := fi.Name() rder := []byte{} stdin := bytes.NewReader(rder) @@ -37,7 +39,7 @@ func TestCheckCommand_RunCacheFile(t *testing.T) { cm := NewCheckCommand(stdin, w, w) cm.Paths = []string{cacheFile} - err := cm.Run(context.Background()) + err = cm.Run(context.Background()) w.Close() var buf bytes.Buffer if _, err := io.Copy(&buf, r); err != nil { @@ -50,7 +52,11 @@ func TestCheckCommand_RunCacheFile(t *testing.T) { } func TestCheckCommand_RunSnapshot(t *testing.T) { - snapshotFile := TempFileName("test", ".snapshotting") + fi, err := testhook.TempFile(t, "test*.snapshotting") + if err != nil { + t.Fatalf("creating test file: %v", err) + } + snapshotFile := fi.Name() rder := []byte{} stdin := bytes.NewReader(rder) @@ -58,7 +64,7 @@ func TestCheckCommand_RunSnapshot(t *testing.T) { cm := NewCheckCommand(stdin, w, w) cm.Paths = []string{snapshotFile} - err := cm.Run(context.Background()) + err = cm.Run(context.Background()) w.Close() var buf bytes.Buffer if _, err := io.Copy(&buf, r); err != nil { @@ -71,10 +77,11 @@ func TestCheckCommand_RunSnapshot(t *testing.T) { } func TestCheckCommand_Run(t *testing.T) { - file, err := ioutil.TempFile("", "") + file, err := testhook.TempFile(t, "run-command") if err != nil { t.Fatal(err) } + fname := file.Name() if _, err := file.Write([]byte("1234,1223")); err != nil { t.Fatalf("writing to temp file: %v", err) } @@ -84,7 +91,7 @@ func TestCheckCommand_Run(t *testing.T) { stdin := bytes.NewReader(rder) r, w, _ := os.Pipe() cm := NewCheckCommand(stdin, w, w) - cm.Paths = []string{file.Name()} + cm.Paths = []string{fname} err = cm.Run(context.Background()) w.Close() @@ -99,10 +106,3 @@ func TestCheckCommand_Run(t *testing.T) { } // Todo: need correct roaring file for happy path } - -// TempFileName generates a temporary filename with extension -func TempFileName(prefix, suffix string) string { - randBytes := make([]byte, 16) - rand.Read(randBytes) - return filepath.Join(os.TempDir(), prefix+hex.EncodeToString(randBytes)+suffix) -} diff --git a/ctl/import_test.go b/ctl/import_test.go index ff5db6295..8713e83ca 100644 --- a/ctl/import_test.go +++ b/ctl/import_test.go @@ -29,6 +29,7 @@ import ( "github.com/pilosa/pilosa/v2" "github.com/pilosa/pilosa/v2/test" + "github.com/pilosa/pilosa/v2/testhook" ) func TestImportCommand_Validation(t *testing.T) { @@ -58,7 +59,7 @@ func TestImportCommand_Basic(t *testing.T) { buf := bytes.Buffer{} stdin, stdout, stderr := GetIO(buf) cm := NewImportCommand(stdin, stdout, stderr) - file, err := ioutil.TempFile("", "import.csv") + file, err := testhook.TempFile(t, "import.csv") if err != nil { t.Fatalf("creating tempfile: %v", err) } @@ -90,7 +91,7 @@ func TestImportCommand_Basic(t *testing.T) { buf := bytes.Buffer{} stdin, stdout, stderr := GetIO(buf) cm := NewImportCommand(stdin, stdout, stderr) - file, err := ioutil.TempFile("", "import.csv") + file, err := testhook.TempFile(t, "import.csv") if err != nil { t.Fatalf("creating tempfile: %v", err) } @@ -123,7 +124,7 @@ func TestImportCommand_RunValue(t *testing.T) { buf := bytes.Buffer{} stdin, stdout, stderr := GetIO(buf) cm := NewImportCommand(stdin, stdout, stderr) - file, err := ioutil.TempFile("", "import-value.csv") + file, err := testhook.TempFile(t, "import-value.csv") if err != nil { t.Fatalf("creating tempfile: %v", err) } @@ -162,7 +163,7 @@ func TestImportCommand_RunValue(t *testing.T) { buf := bytes.Buffer{} stdin, stdout, stderr := GetIO(buf) cm := NewImportCommand(stdin, stdout, stderr) - file, err := ioutil.TempFile("", "import-value.csv") + file, err := testhook.TempFile(t, "import-value.csv") if err != nil { t.Fatalf("creating tempfile: %v", err) } @@ -207,7 +208,7 @@ func TestImportCommand_RunKeys(t *testing.T) { buf := bytes.Buffer{} stdin, stdout, stderr := GetIO(buf) cm := NewImportCommand(stdin, stdout, stderr) - file, err := ioutil.TempFile("", "import-key.csv") + file, err := testhook.TempFile(t, "import-key.csv") if err != nil { t.Fatal(err) } @@ -247,7 +248,7 @@ func TestImportCommand_KeyReplication(t *testing.T) { buf := bytes.Buffer{} stdin, stdout, stderr := GetIO(buf) cm := NewImportCommand(stdin, stdout, stderr) - file, err := ioutil.TempFile("", "import-key.csv") + file, err := testhook.TempFile(t, "import-key.csv") if err != nil { t.Fatal(err) } @@ -327,7 +328,7 @@ func TestImportCommand_RunValueKeys(t *testing.T) { buf := bytes.Buffer{} stdin, stdout, stderr := GetIO(buf) cm := NewImportCommand(stdin, stdout, stderr) - file, err := ioutil.TempFile("", "import-key.csv") + file, err := testhook.TempFile(t, "import-key.csv") if err != nil { t.Fatal(err) } @@ -373,7 +374,7 @@ func TestImportCommand_InvalidFile(t *testing.T) { cm.Host = cmd.API.Node().URI.HostPort() cm.Index = "i" cm.Field = "f" - file, err := ioutil.TempFile("", "import.csv") + file, err := testhook.TempFile(t, "import.csv") if err != nil { t.Fatalf("creating tempfile: %v", err) } @@ -387,7 +388,7 @@ func TestImportCommand_InvalidFile(t *testing.T) { t.Fatalf("expect error: invalid row id on row, actual: %s", err) } - file, err = ioutil.TempFile("", "import1.csv") + file, err = testhook.TempFile(t, "import1.csv") if err != nil { t.Fatalf("creating tempfile: %v", err) } @@ -401,7 +402,7 @@ func TestImportCommand_InvalidFile(t *testing.T) { t.Fatalf("expect error: invalid column id on row, actual: %s", err) } - file, err = ioutil.TempFile("", "import1.csv") + file, err = testhook.TempFile(t, "import1.csv") if err != nil { t.Fatal(err) } @@ -415,7 +416,7 @@ func TestImportCommand_InvalidFile(t *testing.T) { t.Fatalf("expect error: invalid timestamp on row, actual: %s", err) } - file, err = ioutil.TempFile("", "import1.csv") + file, err = testhook.TempFile(t, "import1.csv") if err != nil { t.Fatal(err) } @@ -458,7 +459,7 @@ func TestImportCommand_BugOverwriteValue(t *testing.T) { buf := bytes.Buffer{} stdin, stdout, stderr := GetIO(buf) cm := NewImportCommand(stdin, stdout, stderr) - file, err := ioutil.TempFile("", "import-value.csv") + file, err := testhook.TempFile(t, "import-value.csv") if err != nil { t.Fatal(err) } @@ -490,7 +491,7 @@ func TestImportCommand_BugOverwriteValue(t *testing.T) { } file.Close() - file, err = ioutil.TempFile("", "import-value2.csv") + file, err = testhook.TempFile(t, "import-value2.csv") if err != nil { t.Fatalf("Error creating tempfile: %s", err) } @@ -505,7 +506,7 @@ func TestImportCommand_BugOverwriteValue(t *testing.T) { } file.Close() - file, err = ioutil.TempFile("", "import-value3.csv") + file, err = testhook.TempFile(t, "import-value3.csv") if err != nil { t.Fatalf("Error creating tempfile: %s", err) } @@ -547,7 +548,7 @@ func TestImportCommand_RunBool(t *testing.T) { cm.Field = "f" t.Run("Valid", func(t *testing.T) { - file, err := ioutil.TempFile("", "import-bool.csv") + file, err := testhook.TempFile(t, "import-bool.csv") if err != nil { t.Fatal(err) } @@ -565,7 +566,7 @@ func TestImportCommand_RunBool(t *testing.T) { // Ensure that invalid bool values return an error. t.Run("Invalid", func(t *testing.T) { - file, err := ioutil.TempFile("", "import-invalid-bool.csv") + file, err := testhook.TempFile(t, "import-invalid-bool.csv") if err != nil { t.Fatal(err) } diff --git a/ctl/inspect_test.go b/ctl/inspect_test.go index 2698dcc03..97599dd68 100644 --- a/ctl/inspect_test.go +++ b/ctl/inspect_test.go @@ -18,10 +18,11 @@ import ( "bytes" "context" "io" - "io/ioutil" "os" "strings" "testing" + + "github.com/pilosa/pilosa/v2/testhook" ) func TestInspectCommand_Run(t *testing.T) { @@ -30,7 +31,7 @@ func TestInspectCommand_Run(t *testing.T) { r, w, _ := os.Pipe() cm := NewInspectCommand(stdin, w, w) - file, err := ioutil.TempFile("", "inspectTest") + file, err := testhook.TempFile(t, "inspectTest") if err != nil { t.Fatalf("Error creating tempfile: %s", err) } diff --git a/dbshard_internal_test.go b/dbshard_internal_test.go index 398ed7569..e56554319 100644 --- a/dbshard_internal_test.go +++ b/dbshard_internal_test.go @@ -16,7 +16,6 @@ package pilosa import ( "fmt" - "io/ioutil" "os" "path/filepath" "strings" @@ -25,6 +24,7 @@ import ( "github.com/pilosa/pilosa/v2/rbf" "github.com/pilosa/pilosa/v2/shardwidth" txkey "github.com/pilosa/pilosa/v2/short_txkey" + "github.com/pilosa/pilosa/v2/testhook" ) // Shard per db evaluation @@ -70,7 +70,7 @@ func TestShardPerDB_SetBit(t *testing.T) { // test that we find all *local* shards func Test_DBPerShard_GetShardsForIndex_LocalOnly(t *testing.T) { - tmpdir, err := ioutil.TempDir("", "Test_DBPerShard_GetShardsForIndex_LocalOnly") + tmpdir, err := testhook.TempDir(t, "Test_DBPerShard_GetShardsForIndex_LocalOnly") panicOn(err) defer os.RemoveAll(tmpdir) @@ -324,7 +324,7 @@ func makeTxTestDBWithViewsShards(holder *Holder, idx *Index, exp *FieldView2Shar // test that rbf can give us a map[view]*shardSet func Test_DBPerShard_GetFieldView2Shards_map_from_RBF(t *testing.T) { - tmpdir, err := ioutil.TempDir("", "Test_DBPerShard_GetFieldView2Shards_map_from_RBF") + tmpdir, err := testhook.TempDir(t, "Test_DBPerShard_GetFieldView2Shards_map_from_RBF") panicOn(err) defer os.RemoveAll(tmpdir) diff --git a/etcd/leasedkv_test.go b/etcd/leasedkv_test.go index 9c7965c2d..fb54fab93 100644 --- a/etcd/leasedkv_test.go +++ b/etcd/leasedkv_test.go @@ -17,12 +17,12 @@ package etcd import ( "context" "errors" - "io/ioutil" "os" "testing" "time" "github.com/pilosa/pilosa/v2/disco" + "github.com/pilosa/pilosa/v2/testhook" "go.etcd.io/etcd/embed" "go.etcd.io/etcd/etcdserver/api/v3client" ) @@ -33,7 +33,7 @@ const newVal = "newValue" func TestLeasedKv(t *testing.T) { cfg := embed.NewConfig() - dir, err := ioutil.TempDir("", "leasedkv-*") + dir, err := testhook.TempDir(t, "leasedkv-*") if err != nil { t.Fatal(err) } diff --git a/fragment_internal_test.go b/fragment_internal_test.go index 2d096640f..8471e0f28 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -3166,7 +3166,7 @@ func BenchmarkImportIntoLargeFragment(b *testing.B) { if err != nil { b.Fatalf("opening frag file: %v", err) } - fi, err := ioutil.TempFile(*TempDir, "") + fi, err := testhook.TempFileInDir(b, *TempDir, "") if err != nil { b.Fatalf("getting temp file: %v", err) } @@ -3216,7 +3216,7 @@ func BenchmarkImportRoaringIntoLargeFragment(b *testing.B) { if err != nil { b.Fatalf("opening frag file: %v", err) } - fi, err := ioutil.TempFile(*TempDir, "") + fi, err := testhook.TempFileInDir(b, *TempDir, "") if err != nil { b.Fatalf("getting temp file: %v", err) } @@ -3472,6 +3472,10 @@ func BenchmarkFileWrite(b *testing.B) { b.Run(fmt.Sprintf("Rows%d", numRows), func(b *testing.B) { b.StopTimer() for i := 0; i < b.N; i++ { + // DO NOT CONVERT THIS ONE TO USE TESTHOOK. + // We're deleting these files as we go because + // otherwise the benchmark could fill up + // $TMPDIR before it finishes running. f, err := ioutil.TempFile(*TempDir, "") if err != nil { b.Fatalf("getting temp file: %v", err) @@ -3479,14 +3483,17 @@ func BenchmarkFileWrite(b *testing.B) { b.StartTimer() _, err = f.Write(data) if err != nil { + os.Remove(f.Name()) b.Fatal(err) } err = f.Sync() if err != nil { + os.Remove(f.Name()) b.Fatal(err) } err = f.Close() if err != nil { + os.Remove(f.Name()) b.Fatal(err) } b.StopTimer() diff --git a/idalloc_test.go b/idalloc_test.go index 5c6d4103c..cc2ee49af 100644 --- a/idalloc_test.go +++ b/idalloc_test.go @@ -17,18 +17,17 @@ package pilosa import ( "crypto/rand" "io" - "io/ioutil" - "os" "reflect" "testing" "time" + "github.com/pilosa/pilosa/v2/testhook" bolt "go.etcd.io/bbolt" ) func TestIDAlloc(t *testing.T) { // Acquire a temporary file. - f, err := ioutil.TempFile("", "") + f, err := testhook.TempFile(t, "idalloc") if err != nil { t.Errorf("acquiring temporary file: %v", err) return @@ -42,10 +41,6 @@ func TestIDAlloc(t *testing.T) { // Open bolt. db, err := bolt.Open(f.Name(), 0666, &bolt.Options{Timeout: 1 * time.Second}) - if rerr := os.Remove(f.Name()); rerr != nil { - t.Errorf("removing temporary file: %v", rerr) - return - } if err != nil { t.Errorf("opening bolt: %v", err) return diff --git a/logger/filewriter_test.go b/logger/filewriter_test.go index 8351e580d..837a36be4 100644 --- a/logger/filewriter_test.go +++ b/logger/filewriter_test.go @@ -29,6 +29,8 @@ import ( "io/ioutil" "os" "testing" + + "github.com/pilosa/pilosa/v2/testhook" ) // TestReopenAppend -- make sure we always append to an existing file @@ -42,16 +44,12 @@ import ( // 5. read file, make sure it contains line0,line1,line2 // func TestReopenAppend(t *testing.T) { - // TODO fix - // (travis) I have no idea what this TODO is asking for. - // Perhaps use `ioutil.TempFile()`? - var fname = "/tmp/foo" - - // Step 1 -- Create a sample file using normal means - forig, err := os.Create(fname) + forig, err := testhook.TempFile(t, "logger-reopen") if err != nil { - t.Fatalf("Unable to create initial file %s: %s", fname, err) + t.Fatalf("unable to create initial file: %v", err) } + fname := forig.Name() + _, err = forig.Write([]byte("line0\n")) if err != nil { t.Fatalf("Unable to write initial line %s: %s", fname, err) @@ -96,26 +94,23 @@ func TestReopenAppend(t *testing.T) { } } -// Test that reopen works when Inode is swapped out -// 1. Create a sample file using normal means -// 2. Open a ioreopen.File -// write line 1 -// 3. call Reopen -// write line 2 -// 4. close file -// 5. read file, make sure it contains line0,line1,line2 +// Test that reopen works when Inode is swapped out. That is to say, +// if the previous file has been renamed, we want to get a new file +// that isn't the old one, not keep writing to the old file. // +// 1. Create a sample file using normal means +// 2. Write to it. +// 3. Rename it. +// 4. Call reopen. +// 5. Write line 2. +// 6. Read file, expecting to see only line 2. func TestChangeInode(t *testing.T) { - // TODO fix - // (travis) I have no idea what this TODO is asking for. - // Perhaps use `ioutil.TempFile()`? - var fname = "/tmp/foo" - // Step 1 -- Create a empty sample file - forig, err := os.Create(fname) + forig, err := testhook.TempFile(t, "changeInode") if err != nil { - t.Fatalf("Unable to create initial file %s: %s", fname, err) + t.Fatalf("Unable to create initial file: %s", err) } + fname := forig.Name() err = forig.Close() if err != nil { t.Fatalf("Unable to close initial file: %s", err) @@ -136,6 +131,8 @@ func TestChangeInode(t *testing.T) { if err != nil { t.Errorf("Renaming error: %s", err) } + // remove the scratch file + defer os.Remove(fname + ".orig") _, err = f.Write([]byte("after1\n")) if err != nil { t.Errorf("Write error: %s", err) diff --git a/rbf/db_test.go b/rbf/db_test.go index 54a2d1c39..431062a84 100644 --- a/rbf/db_test.go +++ b/rbf/db_test.go @@ -32,7 +32,7 @@ import ( ) func TestDB_Open(t *testing.T) { - db := NewDB() + db := NewDB(t) if err := db.Open(); err != nil { t.Fatal(err) } else if err := db.Close(); err != nil { diff --git a/rbf/ingest_test.go b/rbf/ingest_test.go index 6dcfdc9e6..966caead7 100644 --- a/rbf/ingest_test.go +++ b/rbf/ingest_test.go @@ -16,17 +16,19 @@ package rbf import ( "fmt" - "io/ioutil" "os" "path/filepath" "runtime" "runtime/pprof" "strings" "testing" + //"time" "github.com/pilosa/pilosa/v2/rbf/cfg" "github.com/pilosa/pilosa/v2/roaring" + "github.com/pilosa/pilosa/v2/testhook" + // "github.com/pilosa/pilosa/v2/txkey" txkey "github.com/pilosa/pilosa/v2/short_txkey" ) @@ -71,7 +73,7 @@ func TestIngest_lots_of_views(t *testing.T) { //vv("m1.TotalAlloc = %v", m1.TotalAlloc) }() - path, err := ioutil.TempDir("", "rbf_ingest_lots_of_views") + path, err := testhook.TempDir(t, "rbf_ingest_lots_of_views") panicOn(err) defer os.Remove(path) diff --git a/rbf/rbf_test.go b/rbf/rbf_test.go index 082ad4294..f6b1264df 100644 --- a/rbf/rbf_test.go +++ b/rbf/rbf_test.go @@ -18,7 +18,6 @@ import ( "flag" "fmt" "io" - "io/ioutil" "math/rand" "os" "runtime" @@ -27,6 +26,7 @@ import ( "github.com/pilosa/pilosa/v2/rbf" rbfcfg "github.com/pilosa/pilosa/v2/rbf/cfg" + "github.com/pilosa/pilosa/v2/testhook" ) var quickCheckN *int = flag.Int("quickchecks", 10, "The number of iterations for each quickcheck") @@ -61,8 +61,8 @@ func TestReadWriteRootRecord(t *testing.T) { } // NewDB returns a new instance of DB with a temporary path. -func NewDB(cfg ...*rbfcfg.Config) *rbf.DB { - path, err := ioutil.TempDir("", "") +func NewDB(tb testing.TB, cfg ...*rbfcfg.Config) *rbf.DB { + path, err := testhook.TempDir(tb, "rbfdb") if err != nil { panic(err) } @@ -78,7 +78,7 @@ func NewDB(cfg ...*rbfcfg.Config) *rbf.DB { // MustOpenDB returns a db opened on a temporary file. On error, fail test. func MustOpenDB(tb testing.TB, cfg ...*rbfcfg.Config) *rbf.DB { tb.Helper() - db := NewDB(cfg...) + db := NewDB(tb, cfg...) if err := db.Open(); err != nil { tb.Fatal(err) } diff --git a/rbf/util_test.go b/rbf/util_test.go index 10aff5a50..1af2d02ce 100644 --- a/rbf/util_test.go +++ b/rbf/util_test.go @@ -16,12 +16,12 @@ package rbf import ( "fmt" "io" - "io/ioutil" "os" "testing" rbfcfg "github.com/pilosa/pilosa/v2/rbf/cfg" "github.com/pilosa/pilosa/v2/roaring" + "github.com/pilosa/pilosa/v2/testhook" ) // util_test adds reusable utilities for testing. @@ -144,7 +144,7 @@ func verifyElemNBitN(tx *Tx, lc leafCell) { func testHelperMustOpenNewDB(tb testing.TB, cfg ...*rbfcfg.Config) *DB { tb.Helper() - path, err := ioutil.TempDir("", "") + path, err := testhook.TempDir(tb, "rbfdb") if err != nil { panic(err) } diff --git a/server/cluster_test.go b/server/cluster_test.go index 469c9302d..df346fc84 100644 --- a/server/cluster_test.go +++ b/server/cluster_test.go @@ -192,7 +192,7 @@ func TestClusterResize_AddNode(t *testing.T) { } lsns[i] = l.(*net.TCPListener) } - portsCfg := test.GenPortsConfig(test.NewPorts(lsns)) + portsCfg := test.GenPortsConfig(t, test.NewPorts(lsns)) m1.Config.Etcd = portsCfg[0].Etcd m1.Config.Name = portsCfg[0].Name diff --git a/test/disco.go b/test/disco.go index 9b9a8fe5a..ade6a2b2a 100644 --- a/test/disco.go +++ b/test/disco.go @@ -16,7 +16,6 @@ package test import ( "fmt" - "io/ioutil" "net" "strings" "testing" @@ -121,7 +120,7 @@ func GetPortsGenConfigs(tb testing.TB, nodes []*Command) error { } //GenPortsConfig creates specific configuration for etcd. -func GenPortsConfig(ports []Ports) []*server.Config { +func GenPortsConfig(tb testing.TB, ports []Ports) []*server.Config { cfgs := make([]*server.Config, len(ports)) clusterURLs := make([]string, len(ports)) for i := range cfgs { @@ -134,7 +133,7 @@ func GenPortsConfig(ports []Ports) []*server.Config { lPeerURL := fmt.Sprintf("http://localhost:%d", portP) discoDir := "" - if d, err := ioutil.TempDir("", "disco."); err == nil { + if d, err := testhook.TempDir(tb, "disco."); err == nil { discoDir = d } diff --git a/testhook/hook.go b/testhook/hook.go index a478a7cbb..afa036f84 100644 --- a/testhook/hook.go +++ b/testhook/hook.go @@ -87,12 +87,27 @@ func TempDir(tb testing.TB, pattern string) (path string, err error) { if err == nil { Cleanup(tb, func() { os.RemoveAll(path) - fmt.Println("--- testhook:", path, tb.Name()) + // fmt.Println("--- testhook: cleaning up dir", path, tb.Name()) }) } return path, err } +// TempFile creates a temp file that will be automatically deleted when +// this test completes, using go1.14's [TB].Cleanup() if available. +func TempFile(tb testing.TB, pattern string) (file *os.File, err error) { + file, err = ioutil.TempFile("", pattern) + if err == nil { + path := file.Name() + Cleanup(tb, func() { + file.Close() + os.Remove(path) + // fmt.Println("--- testhook: cleaning up file", path, tb.Name()) + }) + } + return file, err +} + // TempDirInDir creates a temp directory that will be automatically deleted when // this test completes, using go1.14's [TB].Cleanup(), but with a specified // path instead of the default Go TMPDIR. Only some tests use this, which is @@ -106,3 +121,19 @@ func TempDirInDir(tb testing.TB, dir string, pattern string) (path string, err e } return path, err } + +// TempFileInDir creates a temp file that will be automatically deleted when +// this test completes, using go1.14's [TB].Cleanup(), but with a specified +// path instead of the default Go TMPDIR. Only some tests use this, which is +// possibly an error... +func TempFileInDir(tb testing.TB, dir string, pattern string) (file *os.File, err error) { + file, err = ioutil.TempFile(dir, pattern) + if err == nil { + path := file.Name() + Cleanup(tb, func() { + file.Close() + os.Remove(path) + }) + } + return file, err +} From ef8dd91077f05cabaef0e40f6d53e318fb1b88ae Mon Sep 17 00:00:00 2001 From: Seebs Date: Wed, 10 Mar 2021 16:43:02 -0600 Subject: [PATCH 10/14] report errors more clearly when trying to recreate leases There's some loose ends here because really we probably want to be using the top-level server logger, and we should fix that, but in the mean time, let's not swallow the errors as much, because the last line printed doesn't actually show what the error was, but it could. To do this, we distinguish between the current error (which might be a wrapper around DeadlineExceeded) and a previous error which we might prefer to return, if one exists, since it's more likely the "real" cause. --- etcd/leasedkv.go | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/etcd/leasedkv.go b/etcd/leasedkv.go index 3f0703b64..858c000d8 100644 --- a/etcd/leasedkv.go +++ b/etcd/leasedkv.go @@ -100,7 +100,7 @@ func (l *leasedKV) consumeLease(ch <-chan *clientv3.LeaseKeepAliveResponse) { return } - if ok := retry(1*time.Second, func() error { + if e := retry("consumeLease", 1*time.Second, func() error { kaChann, err := l.create(l.value) if err != nil { return err @@ -108,13 +108,13 @@ func (l *leasedKV) consumeLease(ch <-chan *clientv3.LeaseKeepAliveResponse) { go l.consumeLease(kaChann) return nil - }); !ok { - log.Println("lease cannot be recreated. Key:", l.key) + }); e != nil { + log.Printf("lease %q cannot be recreated: %v", l.key, e) l.mu.Unlock() return } - log.Println("lease recreated after a problem. Key:", l.key) + log.Printf("lease %q recreated after a problem", l.key) l.mu.Unlock() return } @@ -172,20 +172,21 @@ func (l *leasedKV) Get(ctx context.Context) (string, error) { return l.value, nil } -func retry(sleep time.Duration, f func() error) bool { +func retry(desc string, sleep time.Duration, f func() error) (err error) { for { - err := f() - if err == nil { - return true + lastErr := f() + if lastErr == nil { + return lastErr } - - // sometimes the element in charge of stopping the lease renewal doesn't do it, causing context errors. - if errors.Is(err, context.DeadlineExceeded) { - return false + if errors.Is(lastErr, context.DeadlineExceeded) { + if err != nil { + return err + } else { + return lastErr + } } - + log.Printf("%s: got error %v, retrying", desc, lastErr) + err = lastErr time.Sleep(sleep) - - log.Println("retrying after error:", err) } } From ddc924de0ccffcfc511c3cbc7435383a72141f4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kuba=20Podg=C3=B3rski?= Date: Thu, 11 Mar 2021 15:32:15 +0100 Subject: [PATCH 11/14] go mod tidy --- cache.go | 48 +++++++++++++- cmd/random-query/main.go | 49 +++++++++----- executor.go | 135 ++++++++++++++++++++++++++++++++++++++- go.mod | 7 +- go.sum | 57 ++--------------- handler.go | 51 +++++++++++++++ holder.go | 4 +- server/handler_test.go | 6 +- 8 files changed, 278 insertions(+), 79 deletions(-) diff --git a/cache.go b/cache.go index b03e4951c..1c16974f9 100644 --- a/cache.go +++ b/cache.go @@ -369,6 +369,12 @@ type Pair struct { Count uint64 `json:"count"` } +func (pair Pair) MarshalJSON() ([]byte, error) { + buffer := new(bytes.Buffer) + _, err := buffer.WriteString(fmt.Sprintf(`{"count":%v,"id":%v,"key":"%v"}`, pair.Count, pair.ID, pair.Key)) + return buffer.Bytes(), err +} + // PairField is a Pair with its associated field. type PairField struct { Pair Pair @@ -561,7 +567,47 @@ func (p *PairsField) ToRows(callback func(*pb.RowResponse) error) error { // MarshalJSON marshals PairsField into a JSON-encoded byte slice, // excluding `Field`. func (p PairsField) MarshalJSON() ([]byte, error) { - return json.Marshal(p.Pairs) + return json.Marshal(SortedPairs(p.Pairs)) +} + +type SortedPairs []Pair + +func (sortPair SortedPairs) MarshalJSON() ([]byte, error) { + sort.Slice(sortPair, func(i, j int) bool { + if sortPair[j].Count == sortPair[i].Count { + if sortPair[j].Key != "" { + return sortPair[j].Key < sortPair[i].Key + } + return sortPair[j].ID < sortPair[i].ID + } + return sortPair[j].Count < sortPair[i].Count + }) + buffer := new(bytes.Buffer) + _, err := buffer.Write([]byte("[")) + if err != nil { + return nil, err + } + for i, pair := range sortPair { + jsonValue, err := json.Marshal(pair) + if err != nil { + return nil, err + } + if i != 0 { + _, err = buffer.Write([]byte(",")) + if err != nil { + return nil, err + } + } + _, err = buffer.Write(jsonValue) + if err != nil { + return nil, err + } + } + _, err = buffer.Write([]byte("]")) + if err != nil { + return nil, err + } + return buffer.Bytes(), nil } // int64Slice represents a sortable slice of int64 numbers. diff --git a/cmd/random-query/main.go b/cmd/random-query/main.go index ead3ccd9d..4bb0962d1 100644 --- a/cmd/random-query/main.go +++ b/cmd/random-query/main.go @@ -212,15 +212,18 @@ NewSetup: pql, err := cfg.GenQuery(index) panicOn(err) - if cfg.Verbose { - fmt.Printf("pql = '%v'\n", pql) - } - // Query node0. res, err := cli.Query(ctx, index, &pilosa.QueryRequest{Index: index, Query: pql}) + colorReset := "\033[0m" if err != nil { - AlwaysPrintf("QUERY FAILED! queries before this=%v; err = '%v', pql='%v'", loops, err, pql) - return err + colorRed := "\033[31m" + colorCyan := "\033[36m" + AlwaysPrintf("\n%vDIFF%v queries before this=%v;\nPQL=%v%v%v\nerr='%v'", colorRed, colorReset, loops, colorCyan, pql, colorReset, err) + } else { + colorYellow := "\033[33m" + if cfg.Verbose { + fmt.Printf("pql = %v%v%v\n", colorYellow, pql, colorReset) + } } if cfg.VeryVerbose { fmt.Printf("success on pql = '%v'; res='%v'\n", pql, res.Results[0]) @@ -374,9 +377,27 @@ func (cfg *RandomQueryConfig) Setup(api API) (err error) { } case "int": foundIntField = true - fallthrough // I bet you thought you'd never see this used + minPQL := fmt.Sprintf("Min(field=%v)", fld.Name) + resp, err := api.Query(ctx, ii.Name, &pilosa.QueryRequest{Index: ii.Name, Query: minPQL}) + if err != nil { + return err + } + min := resp.Results[0] + maxPQL := fmt.Sprintf("Max(field=%v)", fld.Name) + resp, err = api.Query(ctx, ii.Name, &pilosa.QueryRequest{Index: ii.Name, Query: maxPQL}) + if err != nil { + return err + } + max := resp.Results[0] + m := pql.Decimal{} + x := pql.Decimal{} + m.Scale = fld.Options.Scale + m.Value = min.(pilosa.ValCount).Val + x.Value = max.(pilosa.ValCount).Val + x.Scale = fld.Options.Scale + + cfg.AddIntField(ii.Name, fld.Name, m, x, fld.Options.Scale, fld.Options.Type == "decimal") case "decimal": - // we'll ignore row keys and just use value ranges cfg.AddIntField(ii.Name, fld.Name, fld.Options.Min, fld.Options.Max, fld.Options.Scale, fld.Options.Type == "decimal") default: AlwaysPrintf("ignoring field %q: unhandled type %q\n", fld.Name, fld.Options.Type) @@ -451,10 +472,6 @@ func (cfg *RandomQueryConfig) AddIntField(index, field string, min, max pql.Deci func (cfg *RandomQueryConfig) GenQuery(index string) (pql string, err error) { tree := cfg.GenTree(index, cfg.TreeDepth) - - if cfg.Verbose { - fmt.Printf("%v\n", tree.StringIndent(0)) - } pql = tree.ToPQL() // avoid using too much bandwidth, just count the final bitmap. @@ -508,18 +525,16 @@ func (cfg *RandomQueryConfig) GenTree(index string, depth int) (tr *Tree) { tr = &Tree{S: f} numChild := 2 switch f { - case "Union", "Intersect", "Xor": + case "Union", "Intersect": numChild = cfg.Rnd.Intn(8) + 2 + case "Xor": + numChild = cfg.Rnd.Intn(2) + 1 case "Not": numChild = 1 case "Difference": numChild = 2 case "Distinct": numChild = 1 - // sometimes do a bare distinct without a filter - if cfg.Rnd.Intn(10) == 0 { - numChild = 0 - } r = cfg.Rnd.Intn(len(features.Distinctables)) tr.Args = append(tr.Args, fmt.Sprintf("field=%s", features.Distinctables[r].Field)) } diff --git a/executor.go b/executor.go index 0e869ff64..1db8fde7d 100644 --- a/executor.go +++ b/executor.go @@ -15,6 +15,7 @@ package pilosa import ( + "bytes" "context" "encoding/json" "fmt" @@ -3162,6 +3163,16 @@ type FieldRow struct { Value *int64 `json:"value,omitempty"` } +func (p FieldRow) Less(o FieldRow) bool { + if p.Field == o.Field { + if p.RowKey != "" { + return p.RowKey < o.RowKey + } + return p.RowID < o.RowID + } + return p.Field < o.Field +} + func (fr *FieldRow) Clone() (clone *FieldRow) { clone = &FieldRow{ Field: fr.Field, @@ -3328,7 +3339,7 @@ func (g *GroupCounts) ToRows(callback func(*pb.RowResponse) error) error { // customizes the JSON output of the aggregate field label. func (g *GroupCounts) MarshalJSON() ([]byte, error) { groups := g.Groups() - var counts interface{} = groups + var counts interface{} = SortedGroupCount(groups) if len(groups) == 0 { return []byte("[]"), nil @@ -3342,6 +3353,50 @@ func (g *GroupCounts) MarshalJSON() ([]byte, error) { return json.Marshal(counts) } +type SortedGroupCount []GroupCount + +// Provides determanistic ordering for the JSON output +func (sortgroup SortedGroupCount) MarshalJSON() ([]byte, error) { + sort.Slice(sortgroup, func(i, j int) bool { + if sortgroup[j].Count == sortgroup[i].Count { + for x := range sortgroup[j].Group { + if sortgroup[j].Group[x] == sortgroup[i].Group[x] { + continue + } + return sortgroup[j].Group[x].Less(sortgroup[i].Group[x]) + } + + } + return sortgroup[j].Count < sortgroup[i].Count + }) + buffer := new(bytes.Buffer) + _, err := buffer.Write([]byte("[")) + if err != nil { + return nil, err + } + for i, groupCount := range sortgroup { + jsonValue, err := json.Marshal(groupCount) + if err != nil { + return nil, err + } + if i != 0 { + _, err = buffer.Write([]byte(",")) + if err != nil { + return nil, err + } + } + _, err = buffer.Write(jsonValue) + if err != nil { + return nil, err + } + } + _, err = buffer.Write([]byte("]")) + if err != nil { + return nil, err + } + return buffer.Bytes(), nil +} + // GroupCount represents a result item for a group by query. type GroupCount struct { Group []FieldRow `json:"group"` @@ -3349,6 +3404,32 @@ type GroupCount struct { Agg int64 `json:"-"` } +// comparing json structures in the oracle demand that map values have a consistent order +// docs say it should be lexigraphically but experimentation proved otherwise int the +// groupcount state +func (group GroupCount) MarshalJSON() ([]byte, error) { + // TODO(twg) figure out how to access struct tags from code + var buf bytes.Buffer + sort.Slice(group.Group, func(i, j int) bool { + return group.Group[i].Field < group.Group[j].Field + }) + + //write the Counts + buf.WriteString(fmt.Sprintf(`{"count": %v,`, group.Count)) + //write the Groups + buf.WriteString(`"group":[`) + for i, g := range group.Group { + part, _ := g.MarshalJSON() + if i != 0 { + buf.Write([]byte(",")) + } + buf.Write(part) + } + + buf.WriteString("]}") + return buf.Bytes(), nil +} + type groupCountSum struct { Group []FieldRow `json:"group"` Count uint64 `json:"count"` @@ -3845,6 +3926,43 @@ type ExtractedTableColumn struct { Rows []interface{} `json:"rows"` } +func (etc ExtractedTableColumn) MarshalJSON() ([]byte, error) { + buf := new(bytes.Buffer) + buf.WriteString(`{"column":`) + b, _ := etc.Column.MarshalJSON() + buf.Write(b) + buf.WriteString(`,"rows":[`) + for i, row := range etc.Rows { + if i != 0 { + buf.WriteByte(44) //write a comma + } + buf.WriteString(`[`) + switch v := row.(type) { + case []string: + sort.Slice(v, func(i, j int) bool { + return v[i] < v[j] + }) + for si, s := range v { + if si != 0 { + buf.WriteByte(44) //write a comma + } + buf.WriteString(fmt.Sprintf(`"%v"`, s)) + } + case []uint64: + for si, s := range v { + if si != 0 { + buf.WriteByte(44) //write a comma + } + buf.WriteString(fmt.Sprintf(`%v`, s)) + } + } + buf.WriteString(`]`) + + } + buf.WriteString(`]}`) + return buf.Bytes(), nil +} + type ExtractedTable struct { Fields []ExtractedTableField `json:"fields"` Columns []ExtractedTableColumn `json:"columns"` @@ -7407,6 +7525,21 @@ type ValCount struct { Count int64 `json:"count"` } +func (vc ValCount) MarshalJSON() ([]byte, error) { + buf := new(bytes.Buffer) + nullOrVal := "null" + if vc.DecimalVal != nil { + nullOrVal = fmt.Sprintf("%v", vc.DecimalVal) + } + _, err := buf.WriteString(fmt.Sprintf(`{ + "count": %v, + "decimalValue": %v, + "floatValue": %v, + "value":%v + }`, vc.Count, nullOrVal, vc.FloatVal, vc.Val)) + return buf.Bytes(), err +} + func (v *ValCount) Clone() (r *ValCount) { r = &ValCount{ Val: v.Val, diff --git a/go.mod b/go.mod index 671f599af..0ecb42c4a 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31 // indirect github.com/glycerine/idem v0.0.0-20190127113923-7a8083893311 github.com/gogo/protobuf v1.2.1 - github.com/golang/protobuf v1.4.2 + github.com/golang/protobuf v1.3.3 github.com/google/go-cmp v0.5.2 github.com/google/uuid v1.1.4 // indirect github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 // indirect @@ -34,7 +34,6 @@ require ( github.com/prometheus/client_model v0.1.0 github.com/prometheus/prom2json v1.3.0 github.com/rakyll/statik v0.1.7 - github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 // indirect github.com/rs/cors v1.7.0 // indirect github.com/satori/go.uuid v1.2.0 github.com/shirou/gopsutil/v3 v3.20.11 @@ -52,12 +51,12 @@ require ( golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449 golang.org/x/net v0.0.0-20200822124328-c89045814202 // indirect golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 - golang.org/x/sys v0.0.0-20201214095126-aec9a390925b // indirect + golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect google.golang.org/grpc v1.28.0 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect gopkg.in/yaml.v2 v2.3.0 // indirect - modernc.org/mathutil v1.0.0 + modernc.org/mathutil v1.2.2 modernc.org/strutil v1.0.0 sigs.k8s.io/yaml v1.2.0 // indirect vitess.io/vitess v3.0.0-rc.3.0.20190602171040-12bfde34629c+incompatible diff --git a/go.sum b/go.sum index 7ee30647c..f36d9b043 100644 --- a/go.sum +++ b/go.sum @@ -28,7 +28,6 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/beevik/ntp v0.3.0 h1:xzVrPrE4ziasFXgBVBZJDP0Wg/KpMwk2KHJ4Ba8GrDw= @@ -49,9 +48,7 @@ github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= -github.com/coreos/bbolt v1.3.2 h1:wZwiHHUieZCquLkDL0B8UhzreNWsPHooDAG3q34zk0s= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= -github.com/coreos/etcd v3.3.13+incompatible h1:8F3hqu9fGYLBifCmRCJsicFqDx/D68Rt3q1JMazcgBQ= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= @@ -81,7 +78,6 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= @@ -115,21 +111,11 @@ github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -152,39 +138,28 @@ github.com/gorilla/mux v1.7.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2z github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 h1:Iju5GlWwrvL6UBg4zJJt3btmonfrMlCDdsejg4CZE7c= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 h1:z53tR0945TRRQO/fLEVPI6SMv7ZflF0TEaTAoU7tOzg= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.9.0 h1:bM6ZAFZmc/wPFaRDi0d5L7hGEZEx/2u+Tmr2evNHDiI= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5 h1:UImYN5qQ8tuGpGE16ZmjvcTtTw24zw1QAp/SlnNrZhI= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= -github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack v0.5.3 h1:zKjpN5BK/P5lMYrLmBHdBULWbJ0XpYR+7NGzqkZzoD4= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= -github.com/hashicorp/go-sockaddr v1.0.0 h1:GeH6tui99pF4NJgfnhp+L6+FfobzVW3Ah46sLo0ICXs= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= -github.com/hashicorp/go-uuid v1.0.0 h1:RS8zrF7PhGwyNPOtxSClXXj9HA8feRnJzgnI1RJCSnM= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= -github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= @@ -198,7 +173,6 @@ github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NH github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= -github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -207,16 +181,13 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= -github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= @@ -230,11 +201,9 @@ github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNx github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/miekg/dns v1.0.14 h1:9jZdLNd/P4+SfEJ0TNyxYpsK8N4GtfylBLqtbYN1sbA= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= @@ -260,7 +229,6 @@ github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c h1:Lgl0gzECD8GnQ5QCWA8o6BtfL6mDH5rQgM4/fX3avOs= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= @@ -294,8 +262,8 @@ github.com/prometheus/prom2json v1.3.0/go.mod h1:rMN7m0ApCowcoDlypBHlkNbp5eJQf/+ github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rakyll/statik v0.1.7 h1:OF3QCZUuyPxuGEP7B4ypUa7sB/iHtqOTDYZXGM8KOdQ= github.com/rakyll/statik v0.1.7/go.mod h1:AlZONWzMtEnMs7W4e/1LURLiI49pIMmp6V9Unghqrcc= -github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 h1:HQagqIiBmr8YXawX/le3+O26N+vPPC1PtjaF3mwnook= -github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk= +github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= @@ -304,7 +272,6 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= -github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/shirou/gopsutil/v3 v3.20.11 h1:NeVf1K0cgxsWz+N3671ojRptdgzvp7BXL3KV21R0JnA= github.com/shirou/gopsutil/v3 v3.20.11/go.mod h1:igHnfak0qnw1biGeI2qKQvu0ZkwvEkUcCLlYhZzdr/4= @@ -338,11 +305,9 @@ github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5q github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk= github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -456,11 +421,10 @@ golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201024232916-9f70ab9862d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201214095126-aec9a390925b h1:tv7/y4pd+sR8bcNb2D6o7BNU6zjWm0VjQLac+w7fNNM= -golang.org/x/sys v0.0.0-20201214095126-aec9a390925b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c h1:VwygUrnw9jn88c4u8GD3rZQbqrP/tgas88tPUbBxQrk= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= @@ -510,7 +474,6 @@ google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a h1:Ob5/580gVHBJZgXnff1cZDbG+xLtMVE5mDRTe+nIsX4= @@ -523,13 +486,6 @@ google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQ google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.28.0 h1:bO/TA4OxCOummhSf10siHuG7vJOiwh7SpRpFZDkOgl4= google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -542,7 +498,6 @@ gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -555,8 +510,8 @@ honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -modernc.org/mathutil v1.0.0 h1:93vKjrJopTPrtTNpZ8XIovER7iCIH1QU7wNbOQXC60I= -modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= +modernc.org/mathutil v1.2.2 h1:+yFk8hBprV+4c0U9GjFtL+dV3N8hOJ8JCituQcMShFY= +modernc.org/mathutil v1.2.2/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= modernc.org/strutil v1.0.0 h1:XVFtQwFVwc02Wk+0L/Z/zDDXO81r5Lhe6iMKmGX3KhE= modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= diff --git a/handler.go b/handler.go index ade7c9c26..c9d6fa1d2 100644 --- a/handler.go +++ b/handler.go @@ -16,6 +16,7 @@ package pilosa import ( "encoding/json" + "sort" "github.com/pilosa/pilosa/v2/tracing" "github.com/pkg/errors" @@ -98,6 +99,56 @@ func (resp *QueryResponse) MarshalJSON() ([]byte, error) { }) } +// used in the oracle, orders keys lexigraphically +func (resp *QueryResponse) Equals(other *QueryResponse) bool { + other.Sort() + resp.Sort() + + a, err := resp.MarshalJSON() + panicOn(err) + b, err := other.MarshalJSON() + panicOn(err) + return string(a) == string(b) + +} + +// SortIfIsRowIdentifiers forces a certain order of rowkeys so that the sauron +// tool gets comparable results +func SortIfIsRowIdentifiers(i interface{}) (bool, interface{}) { + if m, ok := i.(map[string]interface{}); ok { + it, ok := m["keys"] + if ok { + items := it.([]interface{}) + if len(items) > 0 { + sort.Slice(items, func(p, q int) bool { + return items[p].(string) < items[q].(string) + }) + return true, items + //return true, ToGenericArray(sitems) + } + } + + } + return false, nil +} + +// TODO (twg) probably going to get fancy here but hardcode to test idea +func (resp *QueryResponse) Sort() bool { + modified := false + if resp == nil || resp.Results == nil { + return false + } + for i, result := range resp.Results { + sorted, results := SortIfIsRowIdentifiers(result) + if sorted { + resp.Results[i] = results + modified = true + } + } + + return modified +} + // Handler is the interface for the data handler, a wrapper around // Pilosa's data store. type Handler interface { diff --git a/holder.go b/holder.go index 052c725ad..375384271 100644 --- a/holder.go +++ b/holder.go @@ -906,13 +906,13 @@ func (h *Holder) limitedSchema() ([]*IndexInfo, error) { } func (h *Holder) schema(ctx context.Context, includeViews bool) ([]*IndexInfo, error) { - var a []*IndexInfo - schema, err := h.schemator.Schema(ctx) if err != nil { return nil, errors.Wrapf(err, "getting schema via schemator") } + a := make([]*IndexInfo, 0, len(schema)) + for _, index := range schema { cim, err := decodeCreateIndexMessage(h.serializer, index.Data) if err != nil { diff --git a/server/handler_test.go b/server/handler_test.go index 0d5ac0974..1bf507452 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -106,7 +106,7 @@ func TestHandler_Endpoints(t *testing.T) { t.Fatalf("unexpected status code: %d", w.Code) } body := w.Body.String() - if body != "{\"indexes\":null}\n" { + if body != "{\"indexes\":[]}\n" { t.Fatalf("unexpected empty schema: '%v'", body) } @@ -119,7 +119,7 @@ func TestHandler_Endpoints(t *testing.T) { t.Fatalf("unexpected status code: %d", w.Code) } body := w.Body.String() - if body != "{\"indexes\":null}\n" { + if body != "{\"indexes\":[]}\n" { t.Fatalf("unexpected empty schema: '%v'", body) } @@ -790,7 +790,7 @@ func TestHandler_Endpoints(t *testing.T) { h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query", strings.NewReader(`TopN(f0, n=2)`))) if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"results":[[{"id":30,"key":"","count":3},{"id":31,"key":"","count":1}]]}`+"\n" { + } else if body := w.Body.String(); body != `{"results":[[{"count":3,"id":30,"key":""},{"count":1,"id":31,"key":""}]]}`+"\n" { t.Fatalf("unexpected body: %q", body) } }) From 1ba52dbca98cc46621333daca254910b318b3e5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kuba=20Podg=C3=B3rski?= Date: Fri, 12 Mar 2021 13:06:23 +0100 Subject: [PATCH 12/14] Update go.sum --- go.sum | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/go.sum b/go.sum index c7b25d4a7..f36d9b043 100644 --- a/go.sum +++ b/go.sum @@ -109,14 +109,13 @@ github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFU github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= From 94d7b295eaeca7effbccca989a6900444bf71af8 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Fri, 12 Mar 2021 10:37:40 -0600 Subject: [PATCH 13/14] added more graceful error handling for corrupt containers --- rbf.go | 24 ++++++++++++++++++------ rbf/rbf.go | 15 +++++++++------ 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/rbf.go b/rbf.go index 449c35abe..d4a3d357d 100644 --- a/rbf.go +++ b/rbf.go @@ -284,15 +284,21 @@ func (tx *RBFTx) addOrRemove(index, field, view string, shard uint64, batched, r // not first time through, write what we got. if remove && (rc == nil || rc.N() == 0) { err = tx.RemoveContainer(index, field, view, shard, lastHi) - panicOn(err) + if err != nil { + return 0, rbf.ErrTxFailedToRemoveContainer + } } else { err = tx.PutContainer(index, field, view, shard, lastHi, rc) - panicOn(err) + if err != nil { + return 0, rbf.ErrTxFailedToPutContainer + } } } // get the next container rc, err = tx.Container(index, field, view, shard, hi) - panicOn(err) + if err != nil { + return 0, rbf.ErrTxFailedToRetrieveContainer + } } // else same container, keep adding bits to rct. chng := false // rc can be nil before, and nil after, in both Remove/Add below. @@ -311,17 +317,23 @@ func (tx *RBFTx) addOrRemove(index, field, view string, shard uint64, batched, r if remove { if rc == nil || rc.N() == 0 { err = tx.RemoveContainer(index, field, view, shard, hi) - panicOn(err) + if err != nil { + return 0, rbf.ErrTxFailedToRemoveContainer + } } else { err = tx.PutContainer(index, field, view, shard, hi, rc) - panicOn(err) + if err != nil { + return 0, rbf.ErrTxFailedToPutContainer + } } } else { if rc == nil || rc.N() == 0 { panic("there should be no way to have an empty bitmap AFTER an Add() operation") } err = tx.PutContainer(index, field, view, shard, hi, rc) - panicOn(err) + if err != nil { + return 0, rbf.ErrTxFailedToPutContainer + } } return } diff --git a/rbf/rbf.go b/rbf/rbf.go index 365907121..31d5a0f31 100644 --- a/rbf/rbf.go +++ b/rbf/rbf.go @@ -111,12 +111,15 @@ const ( ) var ( - ErrTxClosed = errors.New("transaction closed") - ErrTxNotWritable = errors.New("transaction not writable") - ErrBitmapNameRequired = errors.New("bitmap name required") - ErrBitmapNotFound = errors.New("bitmap not found") - ErrBitmapExists = errors.New("bitmap already exists") - ErrTxTooLarge = errors.New("rbf tx too large") + ErrTxClosed = errors.New("transaction closed") + ErrTxNotWritable = errors.New("transaction not writable") + ErrBitmapNameRequired = errors.New("bitmap name required") + ErrBitmapNotFound = errors.New("bitmap not found") + ErrBitmapExists = errors.New("bitmap already exists") + ErrTxTooLarge = errors.New("rbf tx too large") + ErrTxFailedToRemoveContainer = errors.New("rbf tx failed to remove container") + ErrTxFailedToPutContainer = errors.New("rbf tx failed to put container") + ErrTxFailedToRetrieveContainer = errors.New("rbf tx failed to put container") ) // Debug is just a temporary flag used for debugging. From 2711bf321fc1c60a0fac20a68bee8c7bc3876d97 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Fri, 12 Mar 2021 13:27:28 -0600 Subject: [PATCH 14/14] replace error with wrap --- rbf.go | 12 ++++++------ rbf/rbf.go | 15 ++++++--------- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/rbf.go b/rbf.go index d4a3d357d..4da6341be 100644 --- a/rbf.go +++ b/rbf.go @@ -285,19 +285,19 @@ func (tx *RBFTx) addOrRemove(index, field, view string, shard uint64, batched, r if remove && (rc == nil || rc.N() == 0) { err = tx.RemoveContainer(index, field, view, shard, lastHi) if err != nil { - return 0, rbf.ErrTxFailedToRemoveContainer + return 0, errors.Wrap(err, "failed to remove container") } } else { err = tx.PutContainer(index, field, view, shard, lastHi, rc) if err != nil { - return 0, rbf.ErrTxFailedToPutContainer + return 0, errors.Wrap(err, "failed to put container") } } } // get the next container rc, err = tx.Container(index, field, view, shard, hi) if err != nil { - return 0, rbf.ErrTxFailedToRetrieveContainer + return 0, errors.Wrap(err, "failed to retrieve container") } } // else same container, keep adding bits to rct. chng := false @@ -318,12 +318,12 @@ func (tx *RBFTx) addOrRemove(index, field, view string, shard uint64, batched, r if rc == nil || rc.N() == 0 { err = tx.RemoveContainer(index, field, view, shard, hi) if err != nil { - return 0, rbf.ErrTxFailedToRemoveContainer + return 0, errors.Wrap(err, "failed to remove container") } } else { err = tx.PutContainer(index, field, view, shard, hi, rc) if err != nil { - return 0, rbf.ErrTxFailedToPutContainer + return 0, errors.Wrap(err, "failed to put container") } } } else { @@ -332,7 +332,7 @@ func (tx *RBFTx) addOrRemove(index, field, view string, shard uint64, batched, r } err = tx.PutContainer(index, field, view, shard, hi, rc) if err != nil { - return 0, rbf.ErrTxFailedToPutContainer + return 0, errors.Wrap(err, "failed to put container") } } return diff --git a/rbf/rbf.go b/rbf/rbf.go index 31d5a0f31..365907121 100644 --- a/rbf/rbf.go +++ b/rbf/rbf.go @@ -111,15 +111,12 @@ const ( ) var ( - ErrTxClosed = errors.New("transaction closed") - ErrTxNotWritable = errors.New("transaction not writable") - ErrBitmapNameRequired = errors.New("bitmap name required") - ErrBitmapNotFound = errors.New("bitmap not found") - ErrBitmapExists = errors.New("bitmap already exists") - ErrTxTooLarge = errors.New("rbf tx too large") - ErrTxFailedToRemoveContainer = errors.New("rbf tx failed to remove container") - ErrTxFailedToPutContainer = errors.New("rbf tx failed to put container") - ErrTxFailedToRetrieveContainer = errors.New("rbf tx failed to put container") + ErrTxClosed = errors.New("transaction closed") + ErrTxNotWritable = errors.New("transaction not writable") + ErrBitmapNameRequired = errors.New("bitmap name required") + ErrBitmapNotFound = errors.New("bitmap not found") + ErrBitmapExists = errors.New("bitmap already exists") + ErrTxTooLarge = errors.New("rbf tx too large") ) // Debug is just a temporary flag used for debugging.