From 231cf683f0cd4a6093211ac8ff0e0373e0c96fb2 Mon Sep 17 00:00:00 2001 From: Travis Date: Mon, 1 May 2017 23:07:41 -0500 Subject: [PATCH] Adjust tests to work with new `DefaultPartitionN`. This mainly involved running `RecalculateCache()` on all fragments in `TopN` tests. It seems these tests have been running as LRU caches, and once we made `ranked` the default, they stopped working. Some tests were affected by the change in partition number and therefore the change in fragment to node mapping. --- client_test.go | 32 +++++++++++++++++--------------- executor_test.go | 8 ++++++++ handler_test.go | 2 +- server/server_test.go | 16 ++++++++-------- 4 files changed, 34 insertions(+), 24 deletions(-) diff --git a/client_test.go b/client_test.go index 3cca9ac9e..24cc0185f 100644 --- a/client_test.go +++ b/client_test.go @@ -93,6 +93,14 @@ func TestClient_MultiNode(t *testing.T) { baseBit0 := SliceWidth * sliceNums[0] baseBit1 := SliceWidth * sliceNums[1] baseBit2 := SliceWidth * sliceNums[2] + + maxSlice := uint64(0) + for _, x := range sliceNums { + if x > maxSlice { + maxSlice = x + } + } + hldr[0].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(100, baseBit0+10) hldr[0].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(4, baseBit0+10, baseBit0+11, baseBit0+12) hldr[0].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(4, baseBit0+10, baseBit0+11, baseBit0+12, baseBit0+13, baseBit0+14, baseBit0+15) @@ -100,13 +108,13 @@ func TestClient_MultiNode(t *testing.T) { hldr[0].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(3, baseBit0+1, baseBit0+2, baseBit0+3, baseBit0+4, baseBit0+5) hldr[0].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(22, baseBit0+1, baseBit0+2, baseBit0+10) - hldr[2].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(24, baseBit2+10, baseBit2+11, baseBit2+12, baseBit2+13, baseBit2+14) hldr[1].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetBits(99, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4) hldr[1].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetBits(100, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5, baseBit1+6, baseBit1+7, baseBit1+8, baseBit1+9, baseBit1+10) hldr[1].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetBits(98, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5, baseBit1+6) hldr[1].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetBits(1, baseBit1+4) hldr[1].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetBits(22, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5) + hldr[2].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(24, baseBit2+10, baseBit2+11, baseBit2+12, baseBit2+13, baseBit2+14) hldr[2].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(20, baseBit2+10, baseBit2+11, baseBit2+12, baseBit2+13) hldr[2].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(21, baseBit2+10) hldr[2].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(100, baseBit2+10) @@ -117,21 +125,15 @@ func TestClient_MultiNode(t *testing.T) { // Rebuild the RankCache. // We have to do this to avoid the 10-second cache invalidation delay // built into cache.Invalidate() - // hldr[0].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).RecalculateCache() - // hldr[0].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).RecalculateCache() - // hldr[0].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).RecalculateCache() - // hldr[1].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).RecalculateCache() + hldr[0].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).RecalculateCache() hldr[1].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).RecalculateCache() - //hldr[1].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).RecalculateCache() - //hldr[2].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).RecalculateCache() - //hldr[2].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).RecalculateCache() - //hldr[2].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).RecalculateCache() + hldr[2].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).RecalculateCache() // Connect to each node to compare results. client := make([]*Client, 3) client[0] = MustNewClient(s[0].Host()) - client[1] = MustNewClient(s[0].Host()) - client[2] = MustNewClient(s[0].Host()) + client[1] = MustNewClient(s[1].Host()) + client[2] = MustNewClient(s[2].Host()) topN := 4 q := fmt.Sprintf(`TopN(frame="%s", n=%d)`, "f", topN) @@ -144,15 +146,15 @@ func TestClient_MultiNode(t *testing.T) { // Check the results before every node has the correct max slice value. pairs := result.(internal.QueryResponse).Results[0].Pairs for _, pair := range pairs { - if pair.Key == 22 && pair.Count != 11 { + if pair.Key == 22 && pair.Count != 3 { t.Fatalf("Invalid Cluster wide MaxSlice prevents accurate calculation of %s", pair) } } // Set max slice to correct value. - hldr[0].Index("i").SetRemoteMaxSlice(10) - hldr[1].Index("i").SetRemoteMaxSlice(10) - hldr[2].Index("i").SetRemoteMaxSlice(10) + hldr[0].Index("i").SetRemoteMaxSlice(maxSlice) + hldr[1].Index("i").SetRemoteMaxSlice(maxSlice) + hldr[2].Index("i").SetRemoteMaxSlice(maxSlice) result, err = client[0].ExecuteQuery(context.Background(), "i", q, true) if err != nil { diff --git a/executor_test.go b/executor_test.go index 084511fb0..c33cbd15b 100644 --- a/executor_test.go +++ b/executor_test.go @@ -286,6 +286,10 @@ func TestExecutor_Execute_TopN(t *testing.T) { hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth) hldr.MustCreateFragmentIfNotExists("i", "other", pilosa.ViewStandard, 0).SetBit(0, 0) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).RecalculateCache() + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).RecalculateCache() + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 5).RecalculateCache() + // Execute query. e := NewExecutor(hldr.Holder, NewCluster(1)) if result, err := e.Execute(context.Background(), "i", MustParse(`TopN(frame=f, n=2)`), nil, nil); err != nil { @@ -374,6 +378,10 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) { hldr.MustCreateFragmentIfNotExists("i", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth+1) hldr.MustCreateFragmentIfNotExists("i", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth+2) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).RecalculateCache() + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).RecalculateCache() + hldr.MustCreateFragmentIfNotExists("i", "other", pilosa.ViewStandard, 1).RecalculateCache() + // Execute query. e := NewExecutor(hldr.Holder, NewCluster(1)) if result, err := e.Execute(context.Background(), "i", MustParse(`TopN(Bitmap(rowID=100, frame=other), frame=f, n=3)`), nil, nil); err != nil { diff --git a/handler_test.go b/handler_test.go index c2aa50927..6744e74ea 100644 --- a/handler_test.go +++ b/handler_test.go @@ -777,7 +777,7 @@ func TestHandler_Fragment_Nodes(t *testing.T) { h.ServeHTTP(w, r) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if w.Body.String() != `[{"host":"host1","internalHost":""},{"host":"host2","internalHost":""}]`+"\n" { + } else if w.Body.String() != `[{"host":"host2","internalHost":""},{"host":"host0","internalHost":""}]`+"\n" { t.Fatalf("unexpected body: %q", w.Body.String()) } } diff --git a/server/server_test.go b/server/server_test.go index d981a4cf3..d677569e0 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -291,14 +291,14 @@ func TestMain_FrameRestore(t *testing.T) { // Create frames. client := m0.Client() - if err := client.CreateIndex(context.Background(), "x", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { + if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { t.Fatal(err) - } else if err := client.CreateFrame(context.Background(), "x", "f", pilosa.FrameOptions{}); err != nil { + } else if err := client.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil { t.Fatal(err) } // Write data on first cluster. - if _, err := m0.Query("x", "", ` + if _, err := m0.Query("i", "", ` SetBit(rowID=1, frame="f", columnID=100) SetBit(rowID=1, frame="f", columnID=1000) SetBit(rowID=1, frame="f", columnID=100000) @@ -311,7 +311,7 @@ func TestMain_FrameRestore(t *testing.T) { } // Query row on first cluster. - if res, err := m0.Query("x", "", `Bitmap(rowID=1, frame="f")`); err != nil { + if res, err := m0.Query("i", "", `Bitmap(rowID=1, frame="f")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{},"bits":[100,1000,100000,200000,400000,600000,800000]}]}`+"\n" { t.Fatalf("unexpected result: %s", res) @@ -325,16 +325,16 @@ func TestMain_FrameRestore(t *testing.T) { client, err := pilosa.NewClient(m2.Server.Host) if err != nil { t.Fatal(err) - } else if err := m2.Client().CreateIndex(context.Background(), "x", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { + } else if err := m2.Client().CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { t.Fatal(err) - } else if err := m2.Client().CreateFrame(context.Background(), "x", "f", pilosa.FrameOptions{}); err != nil { + } else if err := m2.Client().CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil { t.Fatal(err) - } else if err := client.RestoreFrame(context.Background(), m0.Server.Host, "x", "f"); err != nil { + } else if err := client.RestoreFrame(context.Background(), m0.Server.Host, "i", "f"); err != nil { t.Fatal(err) } // Query row on second cluster. - if res, err := m2.Query("x", "", `Bitmap(rowID=1, frame="f")`); err != nil { + if res, err := m2.Query("i", "", `Bitmap(rowID=1, frame="f")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{},"bits":[100,1000,100000,200000,400000,600000,800000]}]}`+"\n" { t.Fatalf("unexpected result: %s", res)