From 96dfe845d334cd415e7eb4c04104727a372d385b Mon Sep 17 00:00:00 2001 From: reesporte Date: Mon, 15 Nov 2021 15:04:27 -0600 Subject: [PATCH 01/15] fix presentation of timestamps from a distinct pql call --- executor.go | 19 +++++++++++++++++++ server/pg.go | 21 ++++++++++++++++++++- 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/executor.go b/executor.go index f99a0f667..6523ce925 100644 --- a/executor.go +++ b/executor.go @@ -322,6 +322,8 @@ func (e *executor) safeCopy(resp QueryResponse) (out QueryResponse) { safe[i] = v.Clone() } out.Results = append(out.Results, safe) + case DistinctTimestamp: + out.Results = append(out.Results, x) default: panic(fmt.Sprintf("handle %T here", v)) } @@ -1498,6 +1500,7 @@ func (e *executor) executeDistinctShard(ctx context.Context, qcx *Qcx, index str if field == nil { return nil, ErrFieldNotFound } + bsig := field.bsiGroup(fieldName) if bsig == nil { result = &Row{ @@ -1534,9 +1537,25 @@ func (e *executor) executeDistinctShard(ctx context.Context, qcx *Qcx, index str if bsig == nil { return executeDistinctShardSet(ctx, qcx, idx, fieldName, shard, filterBitmap) } + if field.Options().Type == FieldTypeTimestamp { + r, err := executeDistinctShardBSI(ctx, qcx, idx, fieldName, shard, bsig, filterBitmap) + if err != nil { + return nil, err + } + results := make([]string, len(r.Pos.Columns())) + for i, val := range r.Pos.Columns() { + results[i] = time.Unix(0, (int64(val)+int64(bsig.Base))*TimeUnitNanos(field.options.TimeUnit)).UTC().Format(time.RFC3339Nano) + } + return DistinctTimestamp{Name: fieldName, Values: results}, nil + } return executeDistinctShardBSI(ctx, qcx, idx, fieldName, shard, bsig, filterBitmap) } +type DistinctTimestamp struct { + Values []string + Name string +} + func executeDistinctShardSet(ctx context.Context, qcx *Qcx, idx *Index, fieldName string, shard uint64, filterBitmap *roaring.Bitmap) (result *Row, err0 error) { index := idx.Name() tx, finisher, err := qcx.GetTx(Txo{Write: !writable, Index: idx, Shard: shard}) diff --git a/server/pg.go b/server/pg.go index c2522f286..2a55bc3e1 100644 --- a/server/pg.go +++ b/server/pg.go @@ -155,6 +155,24 @@ type PilosaQueryHandler struct { sqlVersion SqlVersion } +func pgWriteDistinctTimestamp(w pg.QueryResultWriter, val pilosa.DistinctTimestamp) error { + err := w.WriteHeader(pg.ColumnInfo{ + Name: val.Name, + Type: pg.TypeCharoid, + }) + if err != nil { + return errors.Wrap(err, "writing result header") + } + + for _, k := range val.Values { + err = w.WriteRowText(k) + if err != nil { + return errors.Wrap(err, "writing key") + } + } + return nil +} + func pgWriteRow(w pg.QueryResultWriter, row *pilosa.Row) error { err := w.WriteHeader(pg.ColumnInfo{ Name: "_id", @@ -551,7 +569,8 @@ func pgWriteResult(w pg.QueryResultWriter, result interface{}) error { } return nil - + case pilosa.DistinctTimestamp: + return pgWriteDistinctTimestamp(w, result) case nil: return nil From 9e570c9e84424b8af694f65d929e92f09f5c2dfb Mon Sep 17 00:00:00 2001 From: reesporte Date: Mon, 15 Nov 2021 15:59:51 -0600 Subject: [PATCH 02/15] fix presentation of timestamps from a groupby pql call --- executor.go | 17 ++++++++++++----- server/pg.go | 6 +++++- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/executor.go b/executor.go index 6523ce925..686379609 100644 --- a/executor.go +++ b/executor.go @@ -3064,10 +3064,11 @@ func applyLimitAndOffsetToGroupByResult(c *pql.Call, results []GroupCount) ([]Gr // FieldRow is used to distinguish rows in a group by result. type FieldRow struct { - Field string `json:"field"` - RowID uint64 `json:"rowID"` - RowKey string `json:"rowKey,omitempty"` - Value *int64 `json:"value,omitempty"` + Field string `json:"field"` + RowID uint64 `json:"rowID"` + RowKey string `json:"rowKey,omitempty"` + Value *int64 `json:"value,omitempty"` + FieldOptions *FieldOptions `json:"-"` } func (fr *FieldRow) Clone() (clone *FieldRow) { @@ -3081,6 +3082,11 @@ func (fr *FieldRow) Clone() (clone *FieldRow) { v := *fr.Value clone.Value = &v } + if fr.FieldOptions != nil { + // deep copy, for Extra Safety + v := *fr.FieldOptions + clone.FieldOptions = &v + } return } @@ -7738,6 +7744,8 @@ func newGroupByIterator(executor *executor, qcx *Qcx, rowIDs []RowIDs, children return nil, newNotFoundError(ErrFieldNotFound, fieldName) } gbi.fields[i].Field = fieldName + options := field.Options() + gbi.fields[i].FieldOptions = &options switch field.Type() { case FieldTypeSet, FieldTypeMutex, FieldTypeBool: @@ -7971,7 +7979,6 @@ func (gbi *groupByIterator) Next(ctx context.Context) (ret GroupCount, done bool ret.Group = make([]FieldRow, len(gbi.rows)) copy(ret.Group, gbi.fields) for i, r := range gbi.rows { - ret.Group[i].RowID = r.id ret.Group[i].Value = r.value } diff --git a/server/pg.go b/server/pg.go index 2a55bc3e1..da6dfd633 100644 --- a/server/pg.go +++ b/server/pg.go @@ -331,7 +331,11 @@ func pgWriteGroupCount(w pg.QueryResultWriter, counts *pilosa.GroupCounts) error var v string switch { case g.Value != nil: - v = strconv.FormatInt(*g.Value, 10) + if g.FieldOptions.Type == pilosa.FieldTypeTimestamp { + v = time.Unix(0, (int64(*g.Value)+int64(g.FieldOptions.Base))*pilosa.TimeUnitNanos(g.FieldOptions.TimeUnit)).UTC().Format(time.RFC3339Nano) + } else { + v = strconv.FormatInt(*g.Value, 10) + } case g.RowKey != "": v = g.RowKey default: From 269526348e7326eae24e81b486ffd42ec38c98cd Mon Sep 17 00:00:00 2001 From: reesporte Date: Mon, 15 Nov 2021 17:03:40 -0600 Subject: [PATCH 03/15] update test to ignore FieldOptions field --- test/pilosa.go | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/test/pilosa.go b/test/pilosa.go index 925213cd1..0703e394c 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -26,7 +26,7 @@ import ( "testing" "time" - "github.com/molecula/featurebase/v2" + pilosa "github.com/molecula/featurebase/v2" "github.com/molecula/featurebase/v2/disco" "github.com/molecula/featurebase/v2/encoding/proto" "github.com/molecula/featurebase/v2/http" @@ -315,7 +315,27 @@ func CheckGroupBy(t *testing.T, expected, results []pilosa.GroupCount) { t.Fatalf("number of groupings mismatch:\n got:%+v\nwant:%+v\n", results, expected) } for i, result := range results { - if !reflect.DeepEqual(expected[i], result) { + // have to check each field Row individually because FieldOptions is getting set + for j := range expected[i].Group { + // Field:"ppa", RowID:0x3, RowKey:"", Value:(*int64)(nil), FieldOptions: + if !reflect.DeepEqual(expected[i].Group[j].Field, result.Group[j].Field) { + t.Fatalf("unexpected result at %d: \n got:%+v\nwant:%+v\n", i, result, expected[i]) + } + if !reflect.DeepEqual(expected[i].Group[j].RowKey, result.Group[j].RowKey) { + t.Fatalf("unexpected result at %d: \n got:%+v\nwant:%+v\n", i, result, expected[i]) + } + if !reflect.DeepEqual(expected[i].Group[j].Value, result.Group[j].Value) { + t.Fatalf("unexpected result at %d: \n got:%+v\nwant:%+v\n", i, result, expected[i]) + } + } + + if !reflect.DeepEqual(expected[i].Count, result.Count) { + t.Fatalf("unexpected result at %d: \n got:%+v\nwant:%+v\n", i, result, expected[i]) + } + if !reflect.DeepEqual(expected[i].Agg, result.Agg) { + t.Fatalf("unexpected result at %d: \n got:%+v\nwant:%+v\n", i, result, expected[i]) + } + if !reflect.DeepEqual(expected[i].DecimalAgg, result.DecimalAgg) { t.Fatalf("unexpected result at %d: \n got:%+v\nwant:%+v\n", i, result, expected[i]) } } From 8770ce245effd6dcf61ee441d38954efc2131f12 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 15 Nov 2021 14:51:57 -0600 Subject: [PATCH 04/15] ingest and delete for samsung workflow --- .gitlab/ingestWorkload.sh | 88 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100755 .gitlab/ingestWorkload.sh diff --git a/.gitlab/ingestWorkload.sh b/.gitlab/ingestWorkload.sh new file mode 100755 index 000000000..2502dba2a --- /dev/null +++ b/.gitlab/ingestWorkload.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash + +# To run: +# ./ingestWorkload.sh {Path for featurebase binary} {Path for directory with csv files} {initialize flag} + +function delete_field { + if (($INITIALIZE == 0)); + then + curl -XDELETE $HOST/index/$INDEX/field/$FIELD + fi +} + +# Script to replicate samsung workload of deleting and re-ingesting fields every night +# outline delete and re-ingest workload +function ingest_int_field { + delete_field + curl -XPOST $HOST/index/$INDEX/field/$FIELD -d '{"options": {"type": "int", "min": 0, "max":'$MAX'}}' + $FEATUREBASE_PATH/featurebase import --host $HOST -i $INDEX -f $FIELD $CSV_FILE +} + +function ingest_time_field { + delete_field + curl -XPOST $HOST/index/$INDEX/field/$FIELD -d '{"options": {"keys": true, "type": "time", "timeQuantum": "YMD"}}' + $FEATUREBASE_PATH/featurebase import --host $HOST -i $INDEX -f $FIELD $CSV_FILE +} + +function ingest_set_field { + delete_field + curl -XPOST $HOST/index/$INDEX/field/$FIELD -d '{"options": {"keys": true}}' + $FEATUREBASE_PATH/featurebase import --host $HOST -i $INDEX -f $FIELD $CSV_FILE +} + +# path for featurebase binary +FEATUREBASE_PATH=$1 +shift + +# path for directory with csv directory files for all fields to be ingested +CSV_DIR_PATH=$1 +shift + +# intialize flag - 0:disabled, 1:enabled - creates the index and fields for testing +INITIALIZE=$1 +shift + +# get a list of csv files in the directory +CSV_FILES=`ls $CSV_DIR_PATH/*.csv` + +# featurebase host +HOST="localhost:10101" +# assign index name +INDEX="samsung" +if (($INITIALIZE == 1)); +then + curl -XPOST $HOST/index/$INDEX +fi + +# perform delete and re-ingest for all fields +for CSV_FILE in ${CSV_FILES[@]} + do + # get field name from csv file path + FIELD="$(basename $CSV_FILE .csv)" + if [[ "$FIELD" == *"age"* ]]; + then + MAX=100 + echo $CSV_FILE $MAX + ingest_int_field + elif [[ "$FIELD" == *"identifier"* ]]; + then + MAX=$((2**63 - 1)) # compute max value for 64bit + echo $CSV_FILE $MAX + ingest_int_field + elif [[ "$FIELD" == *"ip"* ]]; + then + MAX=$((2**31 - 1)) # compute max value for 32bit + echo $CSV_FILE $MAX + ingest_int_field + elif [[ "$FIELD" == *"time"* ]]; + then + echo $CSV_FILE + ingest_time_field + else + echo $CSV_FILE + ingest_set_field + fi + done + + + From 6413ed3228b5e19f342f94adffaba7b8a86f62fe Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Mon, 15 Nov 2021 10:54:13 -0600 Subject: [PATCH 05/15] added protection against trailing slash --- cmd/roaring-migrate/main.go | 3 +++ cover-everything.sh | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/cmd/roaring-migrate/main.go b/cmd/roaring-migrate/main.go index 627d56ef2..e12348e56 100644 --- a/cmd/roaring-migrate/main.go +++ b/cmd/roaring-migrate/main.go @@ -243,6 +243,9 @@ func copyFile(src, dest string) error { } func Migrate(dataDir, backupPath string) error { + if strings.HasSuffix(dataDir, "/") { + dataDir = dataDir[:len(dataDir)-1] + } err := os.MkdirAll(backupPath, 0777) if err != nil { return err diff --git a/cover-everything.sh b/cover-everything.sh index f34640cfb..500c9013b 100755 --- a/cover-everything.sh +++ b/cover-everything.sh @@ -2,7 +2,8 @@ # actually get test coverage for every single package and subpackage # very slow but oh well what are you gonna do, not test things? +# note it skips the roaring migrate echo "mode: atomic" > coverage.out -for pkg in $(go list all | grep featurebase); do +for pkg in $(go list all | grep featurebase | grep -v roaring-migrate); do go test -coverprofile=pkgcoverage.out -covermode=atomic $pkg; tail -n +2 pkgcoverage.out >> coverage.out; done From f5afb7a3eda222ac50f165cf2c95b370b8116fc4 Mon Sep 17 00:00:00 2001 From: reesporte Date: Mon, 15 Nov 2021 17:03:40 -0600 Subject: [PATCH 06/15] update test to ignore FieldOptions field --- test/pilosa.go | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/test/pilosa.go b/test/pilosa.go index 925213cd1..0703e394c 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -26,7 +26,7 @@ import ( "testing" "time" - "github.com/molecula/featurebase/v2" + pilosa "github.com/molecula/featurebase/v2" "github.com/molecula/featurebase/v2/disco" "github.com/molecula/featurebase/v2/encoding/proto" "github.com/molecula/featurebase/v2/http" @@ -315,7 +315,27 @@ func CheckGroupBy(t *testing.T, expected, results []pilosa.GroupCount) { t.Fatalf("number of groupings mismatch:\n got:%+v\nwant:%+v\n", results, expected) } for i, result := range results { - if !reflect.DeepEqual(expected[i], result) { + // have to check each field Row individually because FieldOptions is getting set + for j := range expected[i].Group { + // Field:"ppa", RowID:0x3, RowKey:"", Value:(*int64)(nil), FieldOptions: + if !reflect.DeepEqual(expected[i].Group[j].Field, result.Group[j].Field) { + t.Fatalf("unexpected result at %d: \n got:%+v\nwant:%+v\n", i, result, expected[i]) + } + if !reflect.DeepEqual(expected[i].Group[j].RowKey, result.Group[j].RowKey) { + t.Fatalf("unexpected result at %d: \n got:%+v\nwant:%+v\n", i, result, expected[i]) + } + if !reflect.DeepEqual(expected[i].Group[j].Value, result.Group[j].Value) { + t.Fatalf("unexpected result at %d: \n got:%+v\nwant:%+v\n", i, result, expected[i]) + } + } + + if !reflect.DeepEqual(expected[i].Count, result.Count) { + t.Fatalf("unexpected result at %d: \n got:%+v\nwant:%+v\n", i, result, expected[i]) + } + if !reflect.DeepEqual(expected[i].Agg, result.Agg) { + t.Fatalf("unexpected result at %d: \n got:%+v\nwant:%+v\n", i, result, expected[i]) + } + if !reflect.DeepEqual(expected[i].DecimalAgg, result.DecimalAgg) { t.Fatalf("unexpected result at %d: \n got:%+v\nwant:%+v\n", i, result, expected[i]) } } From 6d6cf7e51f53abaabcee8c6b396e54f8d5eed3ec Mon Sep 17 00:00:00 2001 From: reesporte Date: Mon, 15 Nov 2021 15:04:27 -0600 Subject: [PATCH 07/15] fix presentation of timestamps from a distinct pql call --- executor.go | 19 +++++++++++++++++++ server/pg.go | 21 ++++++++++++++++++++- 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/executor.go b/executor.go index f99a0f667..6523ce925 100644 --- a/executor.go +++ b/executor.go @@ -322,6 +322,8 @@ func (e *executor) safeCopy(resp QueryResponse) (out QueryResponse) { safe[i] = v.Clone() } out.Results = append(out.Results, safe) + case DistinctTimestamp: + out.Results = append(out.Results, x) default: panic(fmt.Sprintf("handle %T here", v)) } @@ -1498,6 +1500,7 @@ func (e *executor) executeDistinctShard(ctx context.Context, qcx *Qcx, index str if field == nil { return nil, ErrFieldNotFound } + bsig := field.bsiGroup(fieldName) if bsig == nil { result = &Row{ @@ -1534,9 +1537,25 @@ func (e *executor) executeDistinctShard(ctx context.Context, qcx *Qcx, index str if bsig == nil { return executeDistinctShardSet(ctx, qcx, idx, fieldName, shard, filterBitmap) } + if field.Options().Type == FieldTypeTimestamp { + r, err := executeDistinctShardBSI(ctx, qcx, idx, fieldName, shard, bsig, filterBitmap) + if err != nil { + return nil, err + } + results := make([]string, len(r.Pos.Columns())) + for i, val := range r.Pos.Columns() { + results[i] = time.Unix(0, (int64(val)+int64(bsig.Base))*TimeUnitNanos(field.options.TimeUnit)).UTC().Format(time.RFC3339Nano) + } + return DistinctTimestamp{Name: fieldName, Values: results}, nil + } return executeDistinctShardBSI(ctx, qcx, idx, fieldName, shard, bsig, filterBitmap) } +type DistinctTimestamp struct { + Values []string + Name string +} + func executeDistinctShardSet(ctx context.Context, qcx *Qcx, idx *Index, fieldName string, shard uint64, filterBitmap *roaring.Bitmap) (result *Row, err0 error) { index := idx.Name() tx, finisher, err := qcx.GetTx(Txo{Write: !writable, Index: idx, Shard: shard}) diff --git a/server/pg.go b/server/pg.go index c2522f286..2a55bc3e1 100644 --- a/server/pg.go +++ b/server/pg.go @@ -155,6 +155,24 @@ type PilosaQueryHandler struct { sqlVersion SqlVersion } +func pgWriteDistinctTimestamp(w pg.QueryResultWriter, val pilosa.DistinctTimestamp) error { + err := w.WriteHeader(pg.ColumnInfo{ + Name: val.Name, + Type: pg.TypeCharoid, + }) + if err != nil { + return errors.Wrap(err, "writing result header") + } + + for _, k := range val.Values { + err = w.WriteRowText(k) + if err != nil { + return errors.Wrap(err, "writing key") + } + } + return nil +} + func pgWriteRow(w pg.QueryResultWriter, row *pilosa.Row) error { err := w.WriteHeader(pg.ColumnInfo{ Name: "_id", @@ -551,7 +569,8 @@ func pgWriteResult(w pg.QueryResultWriter, result interface{}) error { } return nil - + case pilosa.DistinctTimestamp: + return pgWriteDistinctTimestamp(w, result) case nil: return nil From b3b536505dad34a943982571a288ddc939627e3c Mon Sep 17 00:00:00 2001 From: reesporte Date: Mon, 15 Nov 2021 15:59:51 -0600 Subject: [PATCH 08/15] fix presentation of timestamps from a groupby pql call --- executor.go | 17 ++++++++++++----- server/pg.go | 6 +++++- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/executor.go b/executor.go index 6523ce925..686379609 100644 --- a/executor.go +++ b/executor.go @@ -3064,10 +3064,11 @@ func applyLimitAndOffsetToGroupByResult(c *pql.Call, results []GroupCount) ([]Gr // FieldRow is used to distinguish rows in a group by result. type FieldRow struct { - Field string `json:"field"` - RowID uint64 `json:"rowID"` - RowKey string `json:"rowKey,omitempty"` - Value *int64 `json:"value,omitempty"` + Field string `json:"field"` + RowID uint64 `json:"rowID"` + RowKey string `json:"rowKey,omitempty"` + Value *int64 `json:"value,omitempty"` + FieldOptions *FieldOptions `json:"-"` } func (fr *FieldRow) Clone() (clone *FieldRow) { @@ -3081,6 +3082,11 @@ func (fr *FieldRow) Clone() (clone *FieldRow) { v := *fr.Value clone.Value = &v } + if fr.FieldOptions != nil { + // deep copy, for Extra Safety + v := *fr.FieldOptions + clone.FieldOptions = &v + } return } @@ -7738,6 +7744,8 @@ func newGroupByIterator(executor *executor, qcx *Qcx, rowIDs []RowIDs, children return nil, newNotFoundError(ErrFieldNotFound, fieldName) } gbi.fields[i].Field = fieldName + options := field.Options() + gbi.fields[i].FieldOptions = &options switch field.Type() { case FieldTypeSet, FieldTypeMutex, FieldTypeBool: @@ -7971,7 +7979,6 @@ func (gbi *groupByIterator) Next(ctx context.Context) (ret GroupCount, done bool ret.Group = make([]FieldRow, len(gbi.rows)) copy(ret.Group, gbi.fields) for i, r := range gbi.rows { - ret.Group[i].RowID = r.id ret.Group[i].Value = r.value } diff --git a/server/pg.go b/server/pg.go index 2a55bc3e1..da6dfd633 100644 --- a/server/pg.go +++ b/server/pg.go @@ -331,7 +331,11 @@ func pgWriteGroupCount(w pg.QueryResultWriter, counts *pilosa.GroupCounts) error var v string switch { case g.Value != nil: - v = strconv.FormatInt(*g.Value, 10) + if g.FieldOptions.Type == pilosa.FieldTypeTimestamp { + v = time.Unix(0, (int64(*g.Value)+int64(g.FieldOptions.Base))*pilosa.TimeUnitNanos(g.FieldOptions.TimeUnit)).UTC().Format(time.RFC3339Nano) + } else { + v = strconv.FormatInt(*g.Value, 10) + } case g.RowKey != "": v = g.RowKey default: From 9bbd946ac5b1ec56f3540940d92d07c4634c7433 Mon Sep 17 00:00:00 2001 From: reesporte Date: Mon, 15 Nov 2021 17:03:40 -0600 Subject: [PATCH 09/15] update test to ignore FieldOptions field --- test/pilosa.go | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/test/pilosa.go b/test/pilosa.go index 925213cd1..0703e394c 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -26,7 +26,7 @@ import ( "testing" "time" - "github.com/molecula/featurebase/v2" + pilosa "github.com/molecula/featurebase/v2" "github.com/molecula/featurebase/v2/disco" "github.com/molecula/featurebase/v2/encoding/proto" "github.com/molecula/featurebase/v2/http" @@ -315,7 +315,27 @@ func CheckGroupBy(t *testing.T, expected, results []pilosa.GroupCount) { t.Fatalf("number of groupings mismatch:\n got:%+v\nwant:%+v\n", results, expected) } for i, result := range results { - if !reflect.DeepEqual(expected[i], result) { + // have to check each field Row individually because FieldOptions is getting set + for j := range expected[i].Group { + // Field:"ppa", RowID:0x3, RowKey:"", Value:(*int64)(nil), FieldOptions: + if !reflect.DeepEqual(expected[i].Group[j].Field, result.Group[j].Field) { + t.Fatalf("unexpected result at %d: \n got:%+v\nwant:%+v\n", i, result, expected[i]) + } + if !reflect.DeepEqual(expected[i].Group[j].RowKey, result.Group[j].RowKey) { + t.Fatalf("unexpected result at %d: \n got:%+v\nwant:%+v\n", i, result, expected[i]) + } + if !reflect.DeepEqual(expected[i].Group[j].Value, result.Group[j].Value) { + t.Fatalf("unexpected result at %d: \n got:%+v\nwant:%+v\n", i, result, expected[i]) + } + } + + if !reflect.DeepEqual(expected[i].Count, result.Count) { + t.Fatalf("unexpected result at %d: \n got:%+v\nwant:%+v\n", i, result, expected[i]) + } + if !reflect.DeepEqual(expected[i].Agg, result.Agg) { + t.Fatalf("unexpected result at %d: \n got:%+v\nwant:%+v\n", i, result, expected[i]) + } + if !reflect.DeepEqual(expected[i].DecimalAgg, result.DecimalAgg) { t.Fatalf("unexpected result at %d: \n got:%+v\nwant:%+v\n", i, result, expected[i]) } } From da783d97efc1ee289d773d2d2f31a38398bd3f5f Mon Sep 17 00:00:00 2001 From: reesporte Date: Tue, 16 Nov 2021 09:14:15 -0600 Subject: [PATCH 10/15] add test for pgWriteDistinctTimestamp --- server/pg_internal_test.go | 53 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 server/pg_internal_test.go diff --git a/server/pg_internal_test.go b/server/pg_internal_test.go new file mode 100644 index 000000000..3b83d032e --- /dev/null +++ b/server/pg_internal_test.go @@ -0,0 +1,53 @@ +package server + +import ( + "testing" + + pilosa "github.com/molecula/featurebase/v2" + "github.com/molecula/featurebase/v2/pg" +) + +// pg_internal_test.go tests unexported methods from server/pg.go + +// TestQueryResultWriter implements the QueryResultWriter interface for testing +type TestQueryResultWriter struct { + Header []pg.ColumnInfo + RowText []string + TagTag string +} + +func (t *TestQueryResultWriter) WriteHeader(headers ...pg.ColumnInfo) error { + for _, header := range headers { + t.Header = append(t.Header, header) + } + return nil +} + +func (t *TestQueryResultWriter) WriteRowText(rowTexts ...string) error { + for _, rowText := range rowTexts { + t.RowText = append(t.RowText, rowText) + } + return nil +} + +func (t *TestQueryResultWriter) Tag(tag string) { + t.TagTag = tag +} + +func TestPgWriteDistinctTimestamp(t *testing.T) { + w := TestQueryResultWriter{} + expected := pilosa.DistinctTimestamp{Name: "test", Values: []string{"date1", "date2", "date3"}} + pgWriteDistinctTimestamp(&w, expected) + if w.Header[0].Name != expected.Name { + t.Fatalf("Header Name is wrong. got %v, want %v", w.Header[0], expected.Name) + } + if w.Header[0].Type != pg.TypeCharoid { + t.Fatalf("Header Type is wrong. got %v, want %v", w.Header[0].Type, pg.TypeCharoid) + } + for i, value := range w.RowText { + if value != expected.Values[i] { + t.Fatalf("Value not written properly. got %v, want %v", value, expected.Values[i]) + } + } + +} From 9e041803fb16f6e240e154ac16b17750eeddf4ca Mon Sep 17 00:00:00 2001 From: reesporte Date: Tue, 16 Nov 2021 09:20:44 -0600 Subject: [PATCH 11/15] add license --- server/pg_internal_test.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/server/pg_internal_test.go b/server/pg_internal_test.go index 3b83d032e..75024d509 100644 --- a/server/pg_internal_test.go +++ b/server/pg_internal_test.go @@ -1,3 +1,17 @@ +// Copyright 2021 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package server import ( From 1cc6a87d2077c52b951ea18ed55edd9ddccf7a14 Mon Sep 17 00:00:00 2001 From: reesporte Date: Tue, 16 Nov 2021 14:53:30 -0600 Subject: [PATCH 12/15] refactor and add test --- server/pg.go | 2 +- util.go | 11 +++++++++-- util_test.go | 18 ++++++++++++++++++ 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/server/pg.go b/server/pg.go index da6dfd633..169f2e747 100644 --- a/server/pg.go +++ b/server/pg.go @@ -332,7 +332,7 @@ func pgWriteGroupCount(w pg.QueryResultWriter, counts *pilosa.GroupCounts) error switch { case g.Value != nil: if g.FieldOptions.Type == pilosa.FieldTypeTimestamp { - v = time.Unix(0, (int64(*g.Value)+int64(g.FieldOptions.Base))*pilosa.TimeUnitNanos(g.FieldOptions.TimeUnit)).UTC().Format(time.RFC3339Nano) + v = pilosa.FormatTimestampNano(int64(*g.Value), g.FieldOptions.Base, g.FieldOptions.TimeUnit) } else { v = strconv.FormatInt(*g.Value, 10) } diff --git a/util.go b/util.go index 6d4b8e09e..1db0c74fc 100644 --- a/util.go +++ b/util.go @@ -115,8 +115,9 @@ func roaringFragmentHasData(path string, index, field, view string, shard uint64 return } -// GetLoopProgress returns the estimated remaining time to iterate through some items -// as well as the loop completion percentage with the following parameters: +// GetLoopProgress returns the estimated remaining time to iterate through some +// items as well as the loop completion percentage with the following +// parameters: // the start time, the current time, the iteration, and the number of items func GetLoopProgress(start time.Time, now time.Time, iteration uint, total uint) (remaining time.Duration, pctDone float64) { itemsLeft := total - (iteration + 1) @@ -124,3 +125,9 @@ func GetLoopProgress(start time.Time, now time.Time, iteration uint, total uint) pctDone = (float64(iteration+1) / float64(total)) * 100 return time.Duration(avgItemTime * float64(itemsLeft)), pctDone } + +// FormatTimestampNano returns the string representation of a timestamp given: +// an epoch value, base, and time unit +func FormatTimestampNano(value, base int64, timeUnit string) string { + return time.Unix(0, (value+base)*TimeUnitNanos(timeUnit)).UTC().Format(time.RFC3339Nano) +} diff --git a/util_test.go b/util_test.go index c659053c5..14ed78c36 100644 --- a/util_test.go +++ b/util_test.go @@ -85,3 +85,21 @@ func TestGetLoopProgress(t *testing.T) { }) } } + +func TestFormatTimestampNano(t *testing.T) { + if FormatTimestampNano(0, 69, "s") != "1970-01-01T00:01:09Z" { + t.Fatal("Timestamp not formatted properly") + } + if FormatTimestampNano(0, 420, "ms") != "1970-01-01T00:00:00.42Z" { + t.Fatal("Timestamp not formatted properly") + } + if FormatTimestampNano(420, 0, "μs") != "1970-01-01T00:00:00.00000042Z" { + t.Fatal("Timestamp not formatted properly") + } + if FormatTimestampNano(420, 69, "us") != "1970-01-01T00:00:00.000489Z" { + t.Fatal("Timestamp not formatted properly") + } + if FormatTimestampNano(69, 420, "ns") != "1970-01-01T00:00:00.000000489Z" { + t.Fatal("Timestamp not formatted properly") + } +} From f8e93871c036802c090d72640b4d4d5a3fd30792 Mon Sep 17 00:00:00 2001 From: reesporte Date: Wed, 17 Nov 2021 08:51:28 -0600 Subject: [PATCH 13/15] refactor safeCopy to pure function and add unit test --- executor.go | 4 ++-- executor_internal_test.go | 12 ++++++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/executor.go b/executor.go index 686379609..3668f9279 100644 --- a/executor.go +++ b/executor.go @@ -254,7 +254,7 @@ func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shar } } // Must copy out of Tx data before Commiting, because it will become invalid afterwards. - respSafeNoTxData := e.safeCopy(resp) + respSafeNoTxData := safeCopy(resp) // Commit transactions if writing; else let the defer grp.Abort do the rollbacks. if needWriteTxn { @@ -267,7 +267,7 @@ func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shar // safeCopy copies everything in resp that has Bitmap material, // to avoid anything coming from the mmap-ed Tx storage. -func (e *executor) safeCopy(resp QueryResponse) (out QueryResponse) { +func safeCopy(resp QueryResponse) (out QueryResponse) { out = QueryResponse{ Err: resp.Err, // error Profile: resp.Profile, // *tracing.Profile diff --git a/executor_internal_test.go b/executor_internal_test.go index b752957ef..ce860c46c 100644 --- a/executor_internal_test.go +++ b/executor_internal_test.go @@ -475,3 +475,15 @@ func TestGetSorter(t *testing.T) { }) } } + +func TestExecutorSafeCopyDistinctTimestamp(t *testing.T) { + result := DistinctTimestamp{Values: []string{"test", "test"}, Name: "test"} + results := make([]interface{}, 1) + results[0] = result + + response := QueryResponse{Results: results, Err: nil, Profile: nil} + copied := safeCopy(response) + if !reflect.DeepEqual(copied.Results, response.Results) { + t.Fatalf("Did not copy results. got %+v, want %+v", copied.Results, response.Results) + } +} From 6419a87797bcce03261eb7b2a85e110591d5c9da Mon Sep 17 00:00:00 2001 From: reesporte Date: Wed, 17 Nov 2021 09:07:08 -0600 Subject: [PATCH 14/15] use util function for formatting timestamp --- executor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/executor.go b/executor.go index 3668f9279..0d738f72b 100644 --- a/executor.go +++ b/executor.go @@ -1544,7 +1544,7 @@ func (e *executor) executeDistinctShard(ctx context.Context, qcx *Qcx, index str } results := make([]string, len(r.Pos.Columns())) for i, val := range r.Pos.Columns() { - results[i] = time.Unix(0, (int64(val)+int64(bsig.Base))*TimeUnitNanos(field.options.TimeUnit)).UTC().Format(time.RFC3339Nano) + results[i] = FormatTimeStampNano(int64(val), bsig.Base, field.options.TimeUnit) } return DistinctTimestamp{Name: fieldName, Values: results}, nil } From ea96c10114915b91cb30ef3d7dc024afb364e477 Mon Sep 17 00:00:00 2001 From: reesporte Date: Wed, 17 Nov 2021 09:10:50 -0600 Subject: [PATCH 15/15] fix typo --- executor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/executor.go b/executor.go index 0d738f72b..74e565aed 100644 --- a/executor.go +++ b/executor.go @@ -1544,7 +1544,7 @@ func (e *executor) executeDistinctShard(ctx context.Context, qcx *Qcx, index str } results := make([]string, len(r.Pos.Columns())) for i, val := range r.Pos.Columns() { - results[i] = FormatTimeStampNano(int64(val), bsig.Base, field.options.TimeUnit) + results[i] = FormatTimestampNano(int64(val), bsig.Base, field.options.TimeUnit) } return DistinctTimestamp{Name: fieldName, Values: results}, nil }