From 45cd48c1b7a6c7a5bbfe65c9c4cf3554fbf78734 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Mon, 17 Dec 2018 15:44:50 -0600 Subject: [PATCH 1/2] change exists field to _exists --- executor_test.go | 12 ++++++------ holder.go | 2 +- http/client_test.go | 4 ++-- pilosa.go | 5 ----- pilosa_internal_test.go | 5 ++--- 5 files changed, 11 insertions(+), 17 deletions(-) diff --git a/executor_test.go b/executor_test.go index 3c54fbefb..141533d7f 100644 --- a/executor_test.go +++ b/executor_test.go @@ -2163,10 +2163,10 @@ func TestExecutor_Execute_Existence(t *testing.T) { t.Fatalf("unexpected columns: %+v", bits) } - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(exists=0)`}); err != nil { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Not(Row(f=10))`}); err != nil { t.Fatal(err) - } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth + 1, ShardWidth + 2}) { - t.Fatalf("unexpected existence columns: %+v", bits) + } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{ShardWidth + 2}) { + t.Fatalf("unexpected columns after Not: %+v", bits) } // Reopen cluster to ensure existence field is reloaded. @@ -2174,10 +2174,10 @@ func TestExecutor_Execute_Existence(t *testing.T) { t.Fatal(err) } - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(exists=0)`}); err != nil { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Not(Row(f=10))`}); err != nil { t.Fatal(err) - } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth + 1, ShardWidth + 2}) { - t.Fatalf("unexpected existence columns after reopen: %+v", bits) + } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{ShardWidth + 2}) { + t.Fatalf("unexpected columns after reopen: %+v", bits) } }) } diff --git a/holder.go b/holder.go index 9e58e8057..39fef3595 100644 --- a/holder.go +++ b/holder.go @@ -43,7 +43,7 @@ const ( fileLimit = 262144 // (512^2) // existenceFieldName is the name of the internal field used to store existence values. - existenceFieldName = "exists" + existenceFieldName = "_exists" ) // Holder represents a container for indexes. diff --git a/http/client_test.go b/http/client_test.go index fee3754df..79aa2a2ea 100644 --- a/http/client_test.go +++ b/http/client_test.go @@ -900,7 +900,7 @@ func TestClient_ImportExistence(t *testing.T) { } // Verify existence. - if a := hldr.ReadRow(idxName, "exists", 0).Columns(); !reflect.DeepEqual(a, []uint64{1, 5, 6}) { + if a := hldr.ReadRow(idxName, "_exists", 0).Columns(); !reflect.DeepEqual(a, []uint64{1, 5, 6}) { t.Fatalf("unexpected existence columns: %+v", a) } }) @@ -935,7 +935,7 @@ func TestClient_ImportExistence(t *testing.T) { } // Verify existence. - if a := hldr.ReadRow(idxName, "exists", 0).Columns(); !reflect.DeepEqual(a, []uint64{1, 2, 3}) { + if a := hldr.ReadRow(idxName, "_exists", 0).Columns(); !reflect.DeepEqual(a, []uint64{1, 2, 3}) { t.Fatalf("unexpected existence columns: %+v", a) } }) diff --git a/pilosa.go b/pilosa.go index 56e8e0f9e..fe61d8969 100644 --- a/pilosa.go +++ b/pilosa.go @@ -49,8 +49,6 @@ var ( ErrName = errors.New("invalid index or field name, must match [a-z0-9_-]") ErrLabel = errors.New("invalid row or column label, must match [A-Za-z0-9_-]") - ErrReservedName = errors.New("reserved index or field name") - // ErrFragmentNotFound is returned when a fragment does not exist. ErrFragmentNotFound = errors.New("fragment not found") ErrQueryRequired = errors.New("query required") @@ -131,9 +129,6 @@ const TimeFormat = "2006-01-02T15:04" // validateName ensures that the name is a valid format. func validateName(name string) error { - if name == existenceFieldName { - return ErrReservedName - } if !nameRegexp.Match([]byte(name)) { return ErrName } diff --git a/pilosa_internal_test.go b/pilosa_internal_test.go index e1bea3199..40e33ac4b 100644 --- a/pilosa_internal_test.go +++ b/pilosa_internal_test.go @@ -20,7 +20,7 @@ import ( func TestValidateName(t *testing.T) { names := []string{ - "a", "ab", "ab1", "b-c", "d_e", + "a", "ab", "ab1", "b-c", "d_e", "exists", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", } for _, name := range names { @@ -33,8 +33,7 @@ func TestValidateName(t *testing.T) { func TestValidateNameInvalid(t *testing.T) { names := []string{ "", "'", "^", "/", "\\", "A", "*", "a:b", "valid?no", "yüce", "1", "_", "-", - "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1", - "exists", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1", "_exists", } for _, name := range names { if validateName(name) == nil { From ef7f04c09d01d3bcfe298bbdaa5beb63d86cc688 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Mon, 17 Dec 2018 16:01:32 -0600 Subject: [PATCH 2/2] schema endpoint doesn't return internal fields --- api_test.go | 19 ++++++++++++++++++- holder.go | 3 +++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/api_test.go b/api_test.go index e569896f0..ecb4265ed 100644 --- a/api_test.go +++ b/api_test.go @@ -18,6 +18,7 @@ import ( "context" "fmt" "reflect" + "strings" "testing" "github.com/pilosa/pilosa" @@ -48,7 +49,7 @@ func TestAPI_Import(t *testing.T) { index := "rick" field := "f" - _, err := m0.API.CreateIndex(ctx, index, pilosa.IndexOptions{Keys: true}) + _, err := m0.API.CreateIndex(ctx, index, pilosa.IndexOptions{Keys: true, TrackExistence: true}) if err != nil { t.Fatalf("creating index: %v", err) } @@ -99,6 +100,22 @@ func TestAPI_Import(t *testing.T) { } else if keys := res.Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, colKeys) { t.Fatalf("unexpected column keys: %+v", keys) } + + }) + + // Relies on the previous test creating an index with TrackExistence and + // adding some data. + t.Run("SchemaHasNoExists", func(t *testing.T) { + schema := m1.API.Schema(context.Background()) + for _, f := range schema[0].Fields { + if f.Name == "_exists" { + t.Fatalf("found _exists field in schema") + } + if strings.HasPrefix(f.Name, "_") { + t.Fatalf("found internal field '%s' in schema output", f.Name) + } + } + }) t.Run("RowKeyColumnID", func(t *testing.T) { diff --git a/holder.go b/holder.go index 39fef3595..471d4bd3e 100644 --- a/holder.go +++ b/holder.go @@ -289,6 +289,9 @@ func (h *Holder) limitedSchema() []*IndexInfo { for _, index := range h.Indexes() { di := &IndexInfo{Name: index.Name(), Options: index.Options()} for _, field := range index.Fields() { + if strings.HasPrefix(field.name, "_") { + continue + } fi := &FieldInfo{Name: field.Name(), Options: field.Options()} di.Fields = append(di.Fields, fi) }