Merge branch 'master' into bounds-check

api.go

@@ -732,7 +732,7 @@ func (api *API) Import(_ context.Context, req *ImportRequest) error {
 		if ts == 0 {
 			continue
 		}
-		t := time.Unix(0, ts)
+		t := time.Unix(0, ts).UTC()
 		timestamps[i] = &t
 	}
 

roaring/roaring.go

@@ -46,7 +46,7 @@ const (
 	// at the beginning of every serialized run container.
 	runCountHeaderSize = 2
 
-	// interval32Size is the size of a single run in a container.runs.
+	// interval16Size is the size of a single run in a container.runs.
 	interval16Size = 4
 
 	// bitmapN is the number of values in a container.bitmap.
@@ -167,7 +167,8 @@ func (b *Bitmap) Add(a ...uint64) (changed bool, err error) {
 	return changed, nil
 }
 
-func (b *Bitmap) add(v uint64) bool {
+// DirectAdd adds a value to the bitmap by bypassing the op log.
+func (b *Bitmap) DirectAdd(v uint64) bool {
 	cont := b.Containers.GetOrCreate(highbits(v))
 	return cont.add(lowbits(v))
 }
@@ -771,22 +772,22 @@ func (b *Bitmap) Flip(start, end uint64) *Bitmap {
 	v, eof := itr.Next()
 	//copy over previous bits.
 	for v < start && !eof {
-		result.add(v)
+		result.DirectAdd(v)
 		v, eof = itr.Next()
 	}
 	//flip bits in range .
 	for i := start; i <= end; i++ {
 		if eof {
-			result.add(i)
+			result.DirectAdd(i)
 		} else if v == i {
 			v, eof = itr.Next()
 		} else {
-			result.add(i)
+			result.DirectAdd(i)
 		}
 	}
 	//add remaining.
 	for !eof {
-		result.add(v)
+		result.DirectAdd(v)
 		v, eof = itr.Next()
 	}
 	return result
@@ -1697,7 +1698,7 @@ func (c *Container) arrayWriteTo(w io.Writer) (n int64, err error) {
 	//	assert(lowbits(uint64(v)) == v, "cannot write array value out of range: %d", v)
 	//}
 
-	// Write sizeof(uint32) * cardinality bytes.
+	// Write sizeof(uint16) * cardinality bytes.
 	nn, err := w.Write((*[0xFFFFFFF]byte)(unsafe.Pointer(&c.array[0]))[:2*c.n])
 	return int64(nn), err
 }
@@ -2954,7 +2955,7 @@ type op struct {
 func (op *op) apply(b *Bitmap) bool {
 	switch op.typ {
 	case opTypeAdd:
-		return b.add(op.value)
+		return b.DirectAdd(op.value)
 	case opTypeRemove:
 		return b.remove(op.value)
 	default:

roaring/roaring_test.go

@@ -331,6 +331,22 @@ func TestBitmap_ArrayCountRange(t *testing.T) {
 	}
 }
 
+func TestBitmap_DirectAdd(t *testing.T) {
+	bits := []uint64{0, 1, 2, 3, 4, 5, 12, 13, 14, 15, 16, 17, 1000000, 1000002, 1000003, 1000004, 1000005, 1000006, 1000010, 1000011, 1000012, 1000013, 1000014}
+	bm := roaring.NewBitmap()
+	for _, b := range []uint64{0, 1, 2, 3, 4, 5, 12, 13, 14, 15, 16, 17, 1000000, 1000002, 1000003, 1000004, 1000005, 1000006, 1000010, 1000011, 1000012, 1000013, 1000014} {
+		bm.DirectAdd(b)
+	}
+	if len(bits) != int(bm.Count()) {
+		t.Fatalf("count %d != %d", len(bits), bm.Count())
+	}
+	for _, bit := range bits {
+		if !bm.Contains(bit) {
+			t.Fatalf("%d should be in the bitmap", bit)
+		}
+	}
+}
+
 func TestBitmap_RunCountRange(t *testing.T) {
 	bm0 := roaring.NewFileBitmap(0, 1, 2, 3, 4, 5, 12, 13, 14, 15, 16, 17, 1000000, 1000002, 1000003, 1000004, 1000005, 1000006, 1000010, 1000011, 1000012, 1000013, 1000014)
 	bm0.Optimize() // convert to runs

server/server_test.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"math/rand"
 	"reflect"
 	"sort"
@@ -564,4 +565,57 @@ func TestRemoveNodeAfterItDies(t *testing.T) {
 	}
 }
 
+// Ensure program imports timestamps as UTC.
+func TestMain_ImportTimestamp(t *testing.T) {
+	m := test.MustRunCommand()
+	defer m.Close()
+
+	indexName := "i"
+	fieldName := "f"
+
+	// Create index.
+	if _, err := m.API.CreateIndex(context.Background(), indexName, pilosa.IndexOptions{}); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create field.
+	if _, err := m.API.CreateField(context.Background(), indexName, fieldName, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMD"))); err != nil {
+		t.Fatal(err)
+	}
+
+	data := pilosa.ImportRequest{
+		Index:      indexName,
+		Field:      fieldName,
+		Shard:      0,
+		RowIDs:     []uint64{1, 2},
+		ColumnIDs:  []uint64{1, 2},
+		Timestamps: []int64{1514764800000000000, 1577833200000000000}, // 2018-01-01T00:00, 2019-12-31T23:00
+	}
+
+	// Import data.
+	if err := m.API.Import(context.Background(), &data); err != nil {
+		t.Fatal(err)
+	}
+
+	// Ensure the correct views were created.
+	dir := fmt.Sprintf("%s/%s/%s/views", m.Config.DataDir, indexName, fieldName)
+	files, err := ioutil.ReadDir(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	exp := []string{
+		"standard", "standard_2018", "standard_201801", "standard_20180101",
+		"standard_2019", "standard_201912", "standard_20191231",
+	}
+	got := []string{}
+	for _, f := range files {
+		got = append(got, f.Name())
+	}
+
+	if !reflect.DeepEqual(got, exp) {
+		t.Fatalf("expected %v, but got %v", exp, got)
+	}
+}
+
 // TODO: confirm that things keep working if a node is hard-closed (no nodeLeave event) and immediately restarted with a different address.