From 539d1ffe6b591d5f6c5d357a4c44b94f16571859 Mon Sep 17 00:00:00 2001 From: Nia Weiss Date: Thu, 10 Jun 2021 09:41:59 -0400 Subject: [PATCH 1/4] fix batch import of time fields with key translation Time fields were not listed as a type which could accept key translation, causing the translation code to fail. This also adds tests for keyed time and mutex fields. --- client/batch.go | 2 +- client/batch_test.go | 26 ++++++++++++++------------ 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/client/batch.go b/client/batch.go index 56fb1d0c2..9ec51d519 100644 --- a/client/batch.go +++ b/client/batch.go @@ -852,7 +852,7 @@ func (b *Batch) doTranslation() error { } switch ftype := field.Opts().Type(); ftype { - case FieldTypeSet, FieldTypeMutex: + case FieldTypeSet, FieldTypeMutex, FieldTypeTime: // Fill out missing IDs in local batch records with translated IDs. rows := b.rowIDs[i] for key, idxs := range tt { diff --git a/client/batch_test.go b/client/batch_test.go index 69cb6d247..70a5d5dcb 100644 --- a/client/batch_test.go +++ b/client/batch_test.go @@ -993,8 +993,10 @@ func TestBatchesStringIDs(t *testing.T) { client := DefaultClient() schema := NewSchema() idx := schema.Index("gopilosatest-blah", OptIndexKeys(true)) - fields := make([]*Field, 1) + fields := make([]*Field, 3) fields[0] = idx.Field("zero", OptFieldKeys(true)) + fields[1] = idx.Field("one", OptFieldTypeMutex(CacheTypeNone, 0), OptFieldKeys(true)) + fields[2] = idx.Field("two", OptFieldTypeTime("YMDH"), OptFieldKeys(true)) err := client.SyncSchema(schema) if err != nil { t.Fatalf("syncing schema: %v", err) @@ -1011,14 +1013,21 @@ func TestBatchesStringIDs(t *testing.T) { t.Fatalf("getting new batch: %v", err) } - r := Row{Values: make([]interface{}, 1)} + r := Row{Values: make([]interface{}, 3)} + r.Time.Set(time.Date(2019, time.January, 2, 15, 45, 0, 0, time.UTC)) for i := 0; i < 3; i++ { r.ID = strconv.Itoa(i) if i%2 == 0 { r.Values[0] = "a" + r.Values[1] = "b" + r.Values[2] = "c" + r.Time.SetMonth("01") } else { r.Values[0] = "x" + r.Values[1] = "y" + r.Values[2] = "z" + r.Time.SetMonth("02") } err := b.Add(r) if err != nil && err != ErrBatchNowFull { @@ -1040,19 +1049,12 @@ func TestBatchesStringIDs(t *testing.T) { t.Fatalf("translating: %v", err) } - // the ids are based off what the strings hash to, and are at the - // very beginning of a few different shards. this could change if - // Pilosa's hashing algorithm changes. - if err := isPermutationOfInt(b.ids, []uint64{44040193, 45088769, 41943041}); err != nil { - t.Fatalf("wrong ids: %v. exp/got:\n%v\n%v", err, []uint64{44040193, 45088769, 41943041}, b.ids) - } - err = b.Import() if err != nil { t.Fatalf("importing: %v", err) } - resp, err := client.Query(idx.BatchQuery(fields[0].Row("a"), fields[0].Row("x"))) + resp, err := client.Query(idx.BatchQuery(fields[0].Row("a"), fields[0].Row("x"), fields[1].Row("b"), fields[1].Row("y"), fields[2].Row("c"), fields[2].Row("z"))) if err != nil { t.Fatalf("querying: %v", err) } @@ -1060,10 +1062,10 @@ func TestBatchesStringIDs(t *testing.T) { results := resp.Results() for i, res := range results { cols := res.Row().Keys - if i == 0 && !reflect.DeepEqual(cols, []string{"0", "2"}) && !reflect.DeepEqual(cols, []string{"2", "0"}) { + if i%2 == 0 && !reflect.DeepEqual(cols, []string{"0", "2"}) && !reflect.DeepEqual(cols, []string{"2", "0"}) { t.Fatalf("unexpected columns: %v", cols) } - if i == 1 && !reflect.DeepEqual(cols, []string{"1"}) { + if i%2 == 1 && !reflect.DeepEqual(cols, []string{"1"}) { t.Fatalf("unexpected columns: %v", cols) } } From 1bd872e1f44da2ebf4854080b3e1c436b84469ad Mon Sep 17 00:00:00 2001 From: Stephanie Yang Date: Tue, 8 Jun 2021 10:54:54 -0500 Subject: [PATCH 2/4] make keys optional on RowCallType --- lattice/src/App/QueryBuilder/rowTypes.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lattice/src/App/QueryBuilder/rowTypes.ts b/lattice/src/App/QueryBuilder/rowTypes.ts index b4808cd0d..7ed00293f 100644 --- a/lattice/src/App/QueryBuilder/rowTypes.ts +++ b/lattice/src/App/QueryBuilder/rowTypes.ts @@ -13,7 +13,7 @@ export type RowCallType = { rowOperator: string; value: string; type: string; - keys: boolean; + keys?: boolean; }; export type RowsCallType = { From db018e5f41ca1da0a5d47a5fc7ca1ac3d616185b Mon Sep 17 00:00:00 2001 From: Stephanie Yang Date: Wed, 9 Jun 2021 15:27:19 -0500 Subject: [PATCH 3/4] Add support for single codepoint keys for like queries --- .../App/QueryBuilder/stringifyRowData.test.ts | 42 +++++++++++++++++-- .../src/App/QueryBuilder/stringifyRowData.ts | 2 +- 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/lattice/src/App/QueryBuilder/stringifyRowData.test.ts b/lattice/src/App/QueryBuilder/stringifyRowData.test.ts index d6cb4b586..22f439e82 100644 --- a/lattice/src/App/QueryBuilder/stringifyRowData.test.ts +++ b/lattice/src/App/QueryBuilder/stringifyRowData.test.ts @@ -225,6 +225,18 @@ describe('mutex (keys) types', () => { }])).toEqual({ error: false, query: 'Not(Union(Row(fieldName="one"), Row(fieldName="two")))' }) }); + it('stringifies like operator with a single-codepoint key', () => { + expect(stringifyRowData([{ + row: [{ + "field": "fieldName", + "rowOperator": "like", + "value": "_alue", + "type": "mutex", + "keys": true + }] + }])).toEqual({ error: false, query: 'UnionRows(Rows(field=fieldName, like="_alue"))' }) + }); + it('stringifies like operator with a wildcard', () => { expect(stringifyRowData([{ row: [{ @@ -237,7 +249,7 @@ describe('mutex (keys) types', () => { }])).toEqual({ error: false, query: 'UnionRows(Rows(field=fieldName, like="value%"))' }) }); - it('stringifies like operator without a wildcard', () => { + it('stringifies like operator without a single-codepoint key or wildcard', () => { expect(stringifyRowData([{ row: [{ "field": "fieldName", @@ -374,6 +386,18 @@ describe('set (keys) types', () => { }])).toEqual({ error: false, query: 'Not(Union(Row(fieldName="one"), Row(fieldName="two")))' }) }); + it('stringifies like operator with a single-codepoint key', () => { + expect(stringifyRowData([{ + row: [{ + "field": "fieldName", + "rowOperator": "like", + "value": "_alue", + "type": "set", + "keys": true + }] + }])).toEqual({ error: false, query: 'UnionRows(Rows(field=fieldName, like="_alue"))' }) + }); + it('stringifies like operator with a wildcard', () => { expect(stringifyRowData([{ row: [{ @@ -386,7 +410,7 @@ describe('set (keys) types', () => { }])).toEqual({ error: false, query: 'UnionRows(Rows(field=fieldName, like="value%"))' }) }); - it('stringifies like operator without a wildcard', () => { + it('stringifies like operator without a single-codepoint key or wildcard', () => { expect(stringifyRowData([{ row: [{ "field": "fieldName", @@ -488,6 +512,18 @@ describe('time (keys) types', () => { }])).toEqual({ error: false, query: 'Not(Row(fieldName="1234"))' }) }); + it('stringifies like operators with a single-codepoint key', () => { + expect(stringifyRowData([{ + row: [{ + "field": "fieldName", + "rowOperator": "like", + "value": "123_", + "type": "time", + "keys": true + }] + }])).toEqual({ error: false, query: 'UnionRows(Rows(field=fieldName, like="123_"))' }) + }); + it('stringifies like operators with a wildcard', () => { expect(stringifyRowData([{ row: [{ @@ -500,7 +536,7 @@ describe('time (keys) types', () => { }])).toEqual({ error: false, query: 'UnionRows(Rows(field=fieldName, like="%1234"))' }) }); - it('stringifies like operators without a wildcard', () => { + it('stringifies like operators without a single-codepoint key or wildcard', () => { expect(stringifyRowData([{ row: [{ "field": "fieldName", diff --git a/lattice/src/App/QueryBuilder/stringifyRowData.ts b/lattice/src/App/QueryBuilder/stringifyRowData.ts index 5b06c88c9..0faa642c5 100644 --- a/lattice/src/App/QueryBuilder/stringifyRowData.ts +++ b/lattice/src/App/QueryBuilder/stringifyRowData.ts @@ -30,7 +30,7 @@ export const stringifyRowData = (rowData: RowGrouping[], operator?: Operator) => return { error: true, query: error.message }; } } else if (rowOperator === 'like') { - if (value.includes('%')) { + if (value.includes('%') || value.includes('_')) { rowString = `UnionRows(Rows(field=${field}, like="${value}"))`; } else { rowString = `UnionRows(Rows(field=${field}, like="%${value}%"))`; From b218901faea731f9ba8b9f97dd399f5e258503c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kuba=20Podg=C3=B3rski?= Date: Thu, 10 Jun 2021 21:58:08 +0200 Subject: [PATCH 4/4] Add support for timestamps in pg writer --- server/pg.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/pg.go b/server/pg.go index 1db60ecca..281c48c17 100644 --- a/server/pg.go +++ b/server/pg.go @@ -367,6 +367,8 @@ func pgWriteRowser(w pg.QueryResultWriter, result pb.ToRowser) error { case *pb.ColumnResponse_Uint64ArrayVal: data, _ := json.Marshal(col.Uint64ArrayVal.Vals) v = string(data) + case *pb.ColumnResponse_TimestampVal: + v = col.TimestampVal default: return errors.Errorf("unable to process value of type %T", col) }