diff --git a/api.go b/api.go index 7736b12e5..a8783f502 100644 --- a/api.go +++ b/api.go @@ -732,7 +732,7 @@ func (api *API) Import(_ context.Context, req *ImportRequest) error { if ts == 0 { continue } - t := time.Unix(0, ts) + t := time.Unix(0, ts).UTC() timestamps[i] = &t } diff --git a/roaring/roaring.go b/roaring/roaring.go index 5e82364b1..4a5eea5fc 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -46,7 +46,7 @@ const ( // at the beginning of every serialized run container. runCountHeaderSize = 2 - // interval32Size is the size of a single run in a container.runs. + // interval16Size is the size of a single run in a container.runs. interval16Size = 4 // bitmapN is the number of values in a container.bitmap. @@ -167,7 +167,8 @@ func (b *Bitmap) Add(a ...uint64) (changed bool, err error) { return changed, nil } -func (b *Bitmap) add(v uint64) bool { +// DirectAdd adds a value to the bitmap by bypassing the op log. +func (b *Bitmap) DirectAdd(v uint64) bool { cont := b.Containers.GetOrCreate(highbits(v)) return cont.add(lowbits(v)) } @@ -771,22 +772,22 @@ func (b *Bitmap) Flip(start, end uint64) *Bitmap { v, eof := itr.Next() //copy over previous bits. for v < start && !eof { - result.add(v) + result.DirectAdd(v) v, eof = itr.Next() } //flip bits in range . for i := start; i <= end; i++ { if eof { - result.add(i) + result.DirectAdd(i) } else if v == i { v, eof = itr.Next() } else { - result.add(i) + result.DirectAdd(i) } } //add remaining. for !eof { - result.add(v) + result.DirectAdd(v) v, eof = itr.Next() } return result @@ -1697,7 +1698,7 @@ func (c *Container) arrayWriteTo(w io.Writer) (n int64, err error) { // assert(lowbits(uint64(v)) == v, "cannot write array value out of range: %d", v) //} - // Write sizeof(uint32) * cardinality bytes. + // Write sizeof(uint16) * cardinality bytes. nn, err := w.Write((*[0xFFFFFFF]byte)(unsafe.Pointer(&c.array[0]))[:2*c.n]) return int64(nn), err } @@ -2954,7 +2955,7 @@ type op struct { func (op *op) apply(b *Bitmap) bool { switch op.typ { case opTypeAdd: - return b.add(op.value) + return b.DirectAdd(op.value) case opTypeRemove: return b.remove(op.value) default: diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go index d29f1226f..64cb45e81 100644 --- a/roaring/roaring_test.go +++ b/roaring/roaring_test.go @@ -331,6 +331,22 @@ func TestBitmap_ArrayCountRange(t *testing.T) { } } +func TestBitmap_DirectAdd(t *testing.T) { + bits := []uint64{0, 1, 2, 3, 4, 5, 12, 13, 14, 15, 16, 17, 1000000, 1000002, 1000003, 1000004, 1000005, 1000006, 1000010, 1000011, 1000012, 1000013, 1000014} + bm := roaring.NewBitmap() + for _, b := range []uint64{0, 1, 2, 3, 4, 5, 12, 13, 14, 15, 16, 17, 1000000, 1000002, 1000003, 1000004, 1000005, 1000006, 1000010, 1000011, 1000012, 1000013, 1000014} { + bm.DirectAdd(b) + } + if len(bits) != int(bm.Count()) { + t.Fatalf("count %d != %d", len(bits), bm.Count()) + } + for _, bit := range bits { + if !bm.Contains(bit) { + t.Fatalf("%d should be in the bitmap", bit) + } + } +} + func TestBitmap_RunCountRange(t *testing.T) { bm0 := roaring.NewFileBitmap(0, 1, 2, 3, 4, 5, 12, 13, 14, 15, 16, 17, 1000000, 1000002, 1000003, 1000004, 1000005, 1000006, 1000010, 1000011, 1000012, 1000013, 1000014) bm0.Optimize() // convert to runs diff --git a/server/server_test.go b/server/server_test.go index ab860dc1b..67440c21e 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -18,6 +18,7 @@ import ( "context" "encoding/json" "fmt" + "io/ioutil" "math/rand" "reflect" "sort" @@ -564,4 +565,57 @@ func TestRemoveNodeAfterItDies(t *testing.T) { } } +// Ensure program imports timestamps as UTC. +func TestMain_ImportTimestamp(t *testing.T) { + m := test.MustRunCommand() + defer m.Close() + + indexName := "i" + fieldName := "f" + + // Create index. + if _, err := m.API.CreateIndex(context.Background(), indexName, pilosa.IndexOptions{}); err != nil { + t.Fatal(err) + } + + // Create field. + if _, err := m.API.CreateField(context.Background(), indexName, fieldName, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMD"))); err != nil { + t.Fatal(err) + } + + data := pilosa.ImportRequest{ + Index: indexName, + Field: fieldName, + Shard: 0, + RowIDs: []uint64{1, 2}, + ColumnIDs: []uint64{1, 2}, + Timestamps: []int64{1514764800000000000, 1577833200000000000}, // 2018-01-01T00:00, 2019-12-31T23:00 + } + + // Import data. + if err := m.API.Import(context.Background(), &data); err != nil { + t.Fatal(err) + } + + // Ensure the correct views were created. + dir := fmt.Sprintf("%s/%s/%s/views", m.Config.DataDir, indexName, fieldName) + files, err := ioutil.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + + exp := []string{ + "standard", "standard_2018", "standard_201801", "standard_20180101", + "standard_2019", "standard_201912", "standard_20191231", + } + got := []string{} + for _, f := range files { + got = append(got, f.Name()) + } + + if !reflect.DeepEqual(got, exp) { + t.Fatalf("expected %v, but got %v", exp, got) + } +} + // TODO: confirm that things keep working if a node is hard-closed (no nodeLeave event) and immediately restarted with a different address.