From 9647d9b4bbb799d3c25a9dcc08daf608d1948c3b Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Fri, 27 Dec 2019 15:50:13 -0700 Subject: [PATCH] fixing additional tests --- boltdb/translate.go | 2 +- boltdb/translate_test.go | 48 ++++++++++++++++++++++------------------ ctl/import_test.go | 2 +- http/client_test.go | 48 ++++++++++++++++++++++++++-------------- 4 files changed, 59 insertions(+), 41 deletions(-) diff --git a/boltdb/translate.go b/boltdb/translate.go index dbdf5a867..0c67e74bc 100644 --- a/boltdb/translate.go +++ b/boltdb/translate.go @@ -169,7 +169,7 @@ func (s *TranslateStore) TranslateKey(key string) (id uint64, _ error) { return nil } - id := pilosa.GenerateNextPartitionedID(s.index, maxID(tx), s.partitionID, s.partitionN) + id = pilosa.GenerateNextPartitionedID(s.index, maxID(tx), s.partitionID, s.partitionN) if err := bkt.Put([]byte(key), u64tob(id)); err != nil { return err } else if err := tx.Bucket([]byte("ids")).Put(u64tob(id), []byte(key)); err != nil { diff --git a/boltdb/translate_test.go b/boltdb/translate_test.go index 7c3d0e3d5..219885324 100644 --- a/boltdb/translate_test.go +++ b/boltdb/translate_test.go @@ -28,24 +28,24 @@ func TestTranslateStore_TranslateKey(t *testing.T) { s := MustOpenNewTranslateStore() defer MustCloseTranslateStore(s) - // Ensure initial key translates to ID 1. + // Ensure initial key translates to first ID for shard if id, err := s.TranslateKey("foo"); err != nil { t.Fatal(err) - } else if got, want := id, uint64(1); got != want { + } else if got, want := id, uint64(247463937); got != want { t.Fatalf("TranslateKey()=%d, want %d", got, want) } // Ensure next key autoincrements. if id, err := s.TranslateKey("bar"); err != nil { t.Fatal(err) - } else if got, want := id, uint64(2); got != want { + } else if got, want := id, uint64(247463938); got != want { t.Fatalf("TranslateKey()=%d, want %d", got, want) } // Ensure retranslating existing key returns original ID. if id, err := s.TranslateKey("foo"); err != nil { t.Fatal(err) - } else if got, want := id, uint64(1); got != want { + } else if got, want := id, uint64(247463937); got != want { t.Fatalf("TranslateKey()=%d, want %d", got, want) } } @@ -57,29 +57,29 @@ func TestTranslateStore_TranslateKeys(t *testing.T) { // Ensure initial keys translate to incrementing IDs. if ids, err := s.TranslateKeys([]string{"foo", "bar"}); err != nil { t.Fatal(err) - } else if got, want := ids[0], uint64(1); got != want { + } else if got, want := ids[0], uint64(247463937); got != want { t.Fatalf("TranslateKeys()[0]=%d, want %d", got, want) - } else if got, want := ids[1], uint64(2); got != want { + } else if got, want := ids[1], uint64(247463938); got != want { t.Fatalf("TranslateKeys()[1]=%d, want %d", got, want) } // Ensure retranslation returns original IDs. if ids, err := s.TranslateKeys([]string{"foo", "bar"}); err != nil { t.Fatal(err) - } else if got, want := ids[0], uint64(1); got != want { + } else if got, want := ids[0], uint64(247463937); got != want { t.Fatalf("TranslateKeys()[0]=%d, want %d", got, want) - } else if got, want := ids[1], uint64(2); got != want { + } else if got, want := ids[1], uint64(247463938); got != want { t.Fatalf("TranslateKeys()[1]=%d, want %d", got, want) } // Ensure retranslating with existing and non-existing keys returns correctly. if ids, err := s.TranslateKeys([]string{"foo", "baz", "bar"}); err != nil { t.Fatal(err) - } else if got, want := ids[0], uint64(1); got != want { + } else if got, want := ids[0], uint64(247463937); got != want { t.Fatalf("TranslateKeys()[0]=%d, want %d", got, want) - } else if got, want := ids[1], uint64(3); got != want { + } else if got, want := ids[1], uint64(247463939); got != want { t.Fatalf("TranslateKeys()[1]=%d, want %d", got, want) - } else if got, want := ids[2], uint64(2); got != want { + } else if got, want := ids[2], uint64(247463938); got != want { t.Fatalf("TranslateKeys()[2]=%d, want %d", got, want) } } @@ -89,20 +89,23 @@ func TestTranslateStore_TranslateID(t *testing.T) { defer MustCloseTranslateStore(s) // Setup initial keys. - if _, err := s.TranslateKey("foo"); err != nil { + id1, err := s.TranslateKey("foo") + if err != nil { t.Fatal(err) - } else if _, err := s.TranslateKey("bar"); err != nil { + } + id2, err := s.TranslateKey("bar") + if err != nil { t.Fatal(err) } // Ensure IDs can be translated back to keys. - if key, err := s.TranslateID(1); err != nil { + if key, err := s.TranslateID(id1); err != nil { t.Fatal(err) } else if got, want := key, "foo"; got != want { t.Fatalf("TranslateID()=%s, want %s", got, want) } - if key, err := s.TranslateID(2); err != nil { + if key, err := s.TranslateID(id2); err != nil { t.Fatal(err) } else if got, want := key, "bar"; got != want { t.Fatalf("TranslateID()=%s, want %s", got, want) @@ -114,12 +117,13 @@ func TestTranslateStore_TranslateIDs(t *testing.T) { defer MustCloseTranslateStore(s) // Setup initial keys. - if _, err := s.TranslateKeys([]string{"foo", "bar"}); err != nil { + ids, err := s.TranslateKeys([]string{"foo", "bar"}) + if err != nil { t.Fatal(err) } // Ensure IDs can be translated back to keys. - if keys, err := s.TranslateIDs([]uint64{1, 2, 3}); err != nil { + if keys, err := s.TranslateIDs([]uint64{ids[0], ids[1], 1}); err != nil { t.Fatal(err) } else if got, want := keys[0], "foo"; got != want { t.Fatalf("TranslateIDs()[0]=%s, want %s", got, want) @@ -151,7 +155,7 @@ func TestTranslateStore_EntryReader(t *testing.T) { // Read first entry. if err := r.ReadEntry(&entry); err != nil { t.Fatal(err) - } else if got, want := entry.ID, uint64(1); got != want { + } else if got, want := entry.ID, uint64(247463937); got != want { t.Fatalf("ReadEntry() ID=%d, want %d", got, want) } else if got, want := entry.Key, "foo"; got != want { t.Fatalf("ReadEntry() Key=%s, want %s", got, want) @@ -160,7 +164,7 @@ func TestTranslateStore_EntryReader(t *testing.T) { // Read next entry. if err := r.ReadEntry(&entry); err != nil { t.Fatal(err) - } else if got, want := entry.ID, uint64(2); got != want { + } else if got, want := entry.ID, uint64(247463938); got != want { t.Fatalf("ReadEntry() ID=%d, want %d", got, want) } else if got, want := entry.Key, "bar"; got != want { t.Fatalf("ReadEntry() Key=%s, want %s", got, want) @@ -174,7 +178,7 @@ func TestTranslateStore_EntryReader(t *testing.T) { // Read newly created entry. if err := r.ReadEntry(&entry); err != nil { t.Fatal(err) - } else if got, want := entry.ID, uint64(3); got != want { + } else if got, want := entry.ID, uint64(247463939); got != want { t.Fatalf("ReadEntry() ID=%d, want %d", got, want) } else if got, want := entry.Key, "baz"; got != want { t.Fatalf("ReadEntry() Key=%s, want %s", got, want) @@ -211,7 +215,7 @@ func TestTranslateStore_EntryReader(t *testing.T) { var entry pilosa.TranslateEntry if err := r.ReadEntry(&entry); err != nil { t.Fatal(err) - } else if got, want := entry.ID, uint64(1); got != want { + } else if got, want := entry.ID, uint64(247463937); got != want { t.Fatalf("ReadEntry() ID=%d, want %d", got, want) } else if got, want := entry.Key, "foo"; got != want { t.Fatalf("ReadEntry() Key=%s, want %s", got, want) @@ -302,7 +306,7 @@ func MustNewTranslateStore() *boltdb.TranslateStore { panic(err) } - s := boltdb.NewTranslateStore("I", "F") + s := boltdb.NewTranslateStore("I", "F", 0, pilosa.DefaultPartitionN) s.Path = f.Name() return s } diff --git a/ctl/import_test.go b/ctl/import_test.go index ec8ed7666..62b9b3c33 100644 --- a/ctl/import_test.go +++ b/ctl/import_test.go @@ -310,7 +310,7 @@ func TestImportCommand_KeyReplication(t *testing.T) { if body, err := ioutil.ReadAll(resp.Body); err != nil { t.Fatalf("reading: %s", err) } else if !reflect.DeepEqual(body, []byte(exp)) { - t.Fatalf("expected: %s, but got: %s", exp, body) + t.Errorf("expected: %s, but got: %s", exp, body) } } } diff --git a/http/client_test.go b/http/client_test.go index 077a11ff4..9405c0230 100644 --- a/http/client_test.go +++ b/http/client_test.go @@ -294,22 +294,29 @@ func TestClient_Export(t *testing.T) { buf := bytes.NewBuffer(nil) bw := bufio.NewWriter(buf) - // Send export request. - if err := c.ExportCSV(context.Background(), "keyed", "unkeyedf", 0, bw); err != nil { - t.Fatal(err) + // Send export request for every partition. + for i := 0; i < pilosa.DefaultPartitionN; i++ { + if err := c.ExportCSV(context.Background(), "keyed", "unkeyedf", uint64(i), bw); err != nil { + t.Fatal(err) + } } got := buf.String() - // Expected output. - exp := "" - for _, bit := range data { - exp += fmt.Sprintf("%d,%s\n", bit.RowID, bit.ColumnKey) - } + // Expected output is not sorted because of key sharding. + exp := "" + + "2,col200\n" + + "2,col201\n" + + "2,col202\n" + + "2,col203\n" + + "1,col103\n" + + "1,col102\n" + + "1,col101\n" + + "1,col100\n" // Verify data. if got != exp { - t.Fatalf("unexpected export data: %s", got) + t.Fatalf("unexpected export data: %q, expected %q", got, exp) } }) @@ -329,21 +336,28 @@ func TestClient_Export(t *testing.T) { bw := bufio.NewWriter(buf) // Send export request. - if err := c.ExportCSV(context.Background(), "keyed", "keyedf", 0, bw); err != nil { - t.Fatal(err) + for i := 0; i < pilosa.DefaultPartitionN; i++ { + if err := c.ExportCSV(context.Background(), "keyed", "keyedf", uint64(i), bw); err != nil { + t.Fatal(err) + } } got := buf.String() - // Expected output. - exp := "" - for _, bit := range data { - exp += fmt.Sprintf("%s,%s\n", bit.RowKey, bit.ColumnKey) - } + // Expected output is unsorted because of key sharding. + exp := "" + + "row2,col200\n" + + "row2,col201\n" + + "row2,col202\n" + + "row2,col203\n" + + "row1,col103\n" + + "row1,col102\n" + + "row1,col101\n" + + "row1,col100\n" // Verify data. if got != exp { - t.Fatalf("unexpected export data: %s", got) + t.Fatalf("unexpected export data: %q, expected %q", got, exp) } }) }