scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-10-06 10:57:51 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #1571 from molecula/timestamp-epoch

Browse source 
Switch timestamp field to use epoch instead of min/max
This commit is contained in:
Ben Johnson 2021-04-14 12:34:35 -06:00 • committed by GitHub
commit 3eca5944d4
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 63 additions and 59 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
api_test.go
View file

@ -454,7 +454,7 @@ func TestAPI_ImportValue(t *testing.T) {
if err != nil {
t.Fatalf("creating index: %v", err)
}
_, err = m1.API.CreateField(ctx, index, field, pilosa.OptFieldTypeTimestamp(pilosa.MinTimestamp, pilosa.MaxTimestamp, pilosa.TimeUnitSeconds))
_, err = m1.API.CreateField(ctx, index, field, pilosa.OptFieldTypeTimestamp(pilosa.DefaultEpoch, pilosa.TimeUnitSeconds))
if err != nil {
t.Fatalf("creating field: %v", err)
}

2
client/batch.go
View file

@ -240,7 +240,7 @@ func NewBatch(client *Client, size int, index *Index, fields []*Field, opts ...B
ttSets[field.Name()] = make(map[string][]int)
}
hasTime = typ == FieldTypeTime || hasTime
case FieldTypeInt, FieldTypeDecimal:
case FieldTypeInt, FieldTypeDecimal, FieldTypeTimestamp:
// tt line only needed if int field is string foreign key
tt[i] = make(map[string][]int)
values[field.Name()] = make([]int64, 0, size)

3
client/client.go
View file

@ -901,7 +901,6 @@ func (c *Client) doRequest(host *pnet.URI, method, path string, headers map[stri
sleepTime time.Duration
rand = rand.New(rand.NewSource(time.Now().UnixNano()))
)
for retry := 0; ; {
if req, err = buildRequest(host, method, path, headers, data); err != nil {
return 0, nil, errors.Wrap(err, "building request")
@ -1723,6 +1722,7 @@ type SchemaOptions struct {
Keys bool `json:"keys"`
NoStandardView bool `json:"noStandardView"`
TrackExistence bool `json:"trackExistence"`
TimeUnit string `json:"timeUnit"`
}
func (so SchemaOptions) asIndexOptions() *IndexOptions {
@ -1745,6 +1745,7 @@ func (so SchemaOptions) asFieldOptions() *FieldOptions {
scale: so.Scale,
keys: so.Keys,
noStandardView: so.NoStandardView,
timeUnit: so.TimeUnit,
}
}

34
client/orm.go
View file

@ -794,6 +794,7 @@ type FieldOptions struct {
noStandardView bool
foreignIndex string
timeUnit string
base int64
}
// Type returns the type of the field. Currently "set", "int", or "time".
@ -969,15 +970,36 @@ func OptFieldTypeTime(quantum TimeQuantum, opts ...bool) FieldOption {
}
}
func OptFieldTypeTimestamp(min, max time.Time, timeUnit string) FieldOption {
// Timestamp field range.
var (
DefaultEpoch = time.Unix(0, 0).UTC() // 1970-01-01T00:00:00Z
MinTimestamp = time.Unix(-1<<32, 0).UTC() // 1833-11-24T17:31:44Z
MaxTimestamp = time.Unix(1<<32, 0).UTC() // 2106-02-07T06:28:16Z
)
// TimeUnitNanos returns the number of nanoseconds in unit.
func TimeUnitNanos(unit string) int64 {
switch unit {
case TimeUnitSeconds:
return int64(time.Second)
case TimeUnitMilliseconds:
return int64(time.Millisecond)
case TimeUnitMicroseconds:
return int64(time.Microsecond)
default:
return int64(time.Nanosecond)
}
}
func OptFieldTypeTimestamp(epoch time.Time, timeUnit string) FieldOption {
return func(fo *FieldOptions) {
minNano := min.UnixNano()
maxNano := max.UnixNano()
epochValue := epoch.UnixNano() / TimeUnitNanos(timeUnit)
fo.fieldType = FieldTypeTimestamp
fo.timeUnit = timeUnit
fo.min = pql.NewDecimal(minNano, 0)
fo.max = pql.NewDecimal(maxNano, 0)
// fo.Base = bsiBase(minNano, maxNano)
fo.min = pql.NewDecimal(MinTimestamp.UnixNano()/TimeUnitNanos(timeUnit), 0)
fo.max = pql.NewDecimal(MaxTimestamp.UnixNano()/TimeUnitNanos(timeUnit), 0)
fo.base = epochValue
}
}

14
client/orm_test.go
View file

@ -26,6 +26,7 @@ import (
"testing"
"time"
"github.com/pilosa/pilosa/v2"
"github.com/pilosa/pilosa/v2/pql"
"github.com/pkg/errors"
)
@ -1195,22 +1196,25 @@ func TestORM(t *testing.T) {
})
t.Run("TimestampFieldOptions", func(t *testing.T) {
field := sampleIndex.Field("timestamp-field", OptFieldTypeTimestamp(time.Unix(0, 2), time.Unix(0, 5000), "s"))
jsonString := field.options.String()
field := sampleIndex.Field("timestamp-field", OptFieldTypeTimestamp(pilosa.DefaultEpoch, pilosa.TimeUnitSeconds))
/*jsonString := field.options.String()
targetString := `{"options":{"type":"timestamp","timeUnit":"s","max":5000,"min":2}}`
if sortedString(targetString) != sortedString(jsonString) {
t.Fatalf("`%s` != `%s`", targetString, jsonString)
}
*/
compareFieldOptions(t,
field.Options(),
FieldTypeTimestamp,
TimeQuantumNone,
CacheTypeDefault,
0,
pql.NewDecimal(2, 0),
pql.NewDecimal(5000, 0),
pql.NewDecimal(MinTimestamp.UnixNano()/TimeUnitNanos(pilosa.TimeUnitSeconds), 0),
pql.NewDecimal(MaxTimestamp.UnixNano()/TimeUnitNanos(pilosa.TimeUnitSeconds), 0),
"",
"s")
pilosa.TimeUnitSeconds)
})
t.Run("EncodeMapPanicsOnMarshalFailure", func(t *testing.T) {

12
executor_test.go
View file

@ -1777,7 +1777,7 @@ func TestExecutor_Execute_SetValue(t *testing.T) {
// Create fields.
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeTimestamp(pilosa.MinTimestamp, pilosa.MaxTimestamp, pilosa.TimeUnitSeconds)); err != nil {
if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeTimestamp(pilosa.DefaultEpoch, pilosa.TimeUnitSeconds)); err != nil {
t.Fatal(err)
} else if _, err := index.CreateFieldIfNotExists("xxx", pilosa.OptFieldTypeDefault()); err != nil {
t.Fatal(err)
@ -2570,20 +2570,18 @@ func TestExecutor_Execute_MinMax(t *testing.T) {
}
tests := []struct {
min time.Time
max time.Time
set time.Time
epoch time.Time
set time.Time
}{
{
time.Date(2000, time.January, 10, 0, 0, 0, 0, time.UTC),
time.Date(2000, time.January, 20, 0, 0, 0, 0, time.UTC),
time.Date(2000, time.January, 11, 0, 0, 0, 0, time.UTC),
},
}
for i, test := range tests {
fld := fmt.Sprintf("f%d", i)
t.Run("MinMaxField_"+fld, func(t *testing.T) {
if _, err := idx.CreateField(fld, pilosa.OptFieldTypeTimestamp(test.min, test.max, pilosa.TimeUnitSeconds)); err != nil {
if _, err := idx.CreateField(fld, pilosa.OptFieldTypeTimestamp(test.epoch, pilosa.TimeUnitSeconds)); err != nil {
t.Fatal(err)
} else if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: fmt.Sprintf(`Set(10, %s="%s")`, fld, test.set.Format(time.RFC3339))}); err != nil {
t.Fatal(err)
@ -4989,7 +4987,7 @@ func TestExecutor_Execute_Extract(t *testing.T) {
Set(3, bsidecimal=-1.01)
`)
c.CreateField(t, "i", pilosa.IndexOptions{TrackExistence: true}, "timestamp", pilosa.OptFieldTypeTimestamp(pilosa.MinTimestamp, pilosa.MaxTimestamp, pilosa.TimeUnitSeconds))
c.CreateField(t, "i", pilosa.IndexOptions{TrackExistence: true}, "timestamp", pilosa.OptFieldTypeTimestamp(pilosa.DefaultEpoch, pilosa.TimeUnitSeconds))
c.Query(t, "i", `
Set(0, timestamp='2000-01-01T00:00:00Z')
Set(1, timestamp='2000-01-01T00:00:01Z')

36
field.go
View file

@ -208,10 +208,9 @@ func OptFieldTypeInt(min, max int64) FieldOption {
// OptFieldTypeTimestamp is a functional option on FieldOptions
// used to specify the field as being type `timestamp` and to
// provide any respective configuration values.
func OptFieldTypeTimestamp(min, max time.Time, timeUnit string) FieldOption {
func OptFieldTypeTimestamp(epoch time.Time, timeUnit string) FieldOption {
return func(fo *FieldOptions) error {
minValue := min.UnixNano() / TimeUnitNanos(timeUnit)
maxValue := max.UnixNano() / TimeUnitNanos(timeUnit)
epochValue := epoch.UnixNano() / TimeUnitNanos(timeUnit)
if fo.Type != "" {
return errors.Errorf("field type is already set to: %s", fo.Type)
}
@ -220,18 +219,11 @@ func OptFieldTypeTimestamp(min, max time.Time, timeUnit string) FieldOption {
} else if !IsValidTimeUnit(timeUnit) {
return errors.Errorf("invalid time unit: %q", fo.TimeUnit)
}
if min.Before(MinTimestamp) {
return errors.New("timestamp field min is too low")
} else if max.After(MaxTimestamp) {
return errors.New("timestamp field max is too high")
} else if min.After(max) {
return errors.New("timestamp field min cannot be greater than max")
}
fo.Type = FieldTypeTimestamp
fo.TimeUnit = timeUnit
fo.Min = pql.NewDecimal(minValue, 0)
fo.Max = pql.NewDecimal(maxValue, 0)
fo.Base = bsiBase(minValue, maxValue)
fo.Min = pql.NewDecimal(MinTimestamp.UnixNano()/TimeUnitNanos(timeUnit), 0)
fo.Max = pql.NewDecimal(MaxTimestamp.UnixNano()/TimeUnitNanos(timeUnit), 0)
fo.Base = epochValue
return nil
}
}
@ -1925,23 +1917,15 @@ func (o *FieldOptions) MarshalJSON() ([]byte, error) {
})
case FieldTypeTimestamp:
return json.Marshal(struct {
Type string `json:"type"`
BaseTimestamp time.Time `json:"baseTimestamp"`
BitDepth uint64 `json:"bitDepth"`
MinTimestamp time.Time `json:"minTimestamp"`
MaxTimestamp time.Time `json:"maxTimestamp"`
Keys bool `json:"keys"`
TimeUnit string `json:"timeUnit"`
ForeignIndex string `json:"foreignIndex"`
Type string `json:"type"`
Epoch time.Time `json:"epoch"`
BitDepth uint64 `json:"bitDepth"`
TimeUnit string `json:"timeUnit"`
}{
o.Type,
time.Unix(0, o.Base*TimeUnitNanos(o.TimeUnit)).UTC(),
o.BitDepth,
time.Unix(0, o.Min.Value*TimeUnitNanos(o.TimeUnit)).UTC(),
time.Unix(0, o.Max.Value*TimeUnitNanos(o.TimeUnit)).UTC(),
o.Keys,
o.TimeUnit,
o.ForeignIndex,
})
case FieldTypeTime:
return json.Marshal(struct {
@ -2154,6 +2138,8 @@ func (f *Field) persistView(ctx context.Context, cvm *CreateViewMessage) error {
// Timestamp field range.
var (
DefaultEpoch = time.Unix(0, 0).UTC() // 1970-01-01T00:00:00Z
MinTimestamp = time.Unix(-1<<32, 0).UTC() // 1833-11-24T17:31:44Z
MaxTimestamp = time.Unix(1<<32, 0).UTC() // 2106-02-07T06:28:16Z
)

17
http/handler.go
View file

@ -1341,19 +1341,11 @@ func (h *Handler) handlePostField(w http.ResponseWriter, r *http.Request) {
}
fos = append(fos, pilosa.OptFieldTypeDecimal(scale, minmax...))
case pilosa.FieldTypeTimestamp:
if req.Options.Min == nil {
min := pql.NewDecimal(pilosa.MinTimestamp.UnixNano()/pilosa.TimeUnitNanos(*req.Options.TimeUnit), 0)
req.Options.Min = &min
if req.Options.Epoch == nil {
epoch := pilosa.DefaultEpoch
req.Options.Epoch = &epoch
}
if req.Options.Max == nil {
max := pql.NewDecimal(pilosa.MaxTimestamp.UnixNano()/pilosa.TimeUnitNanos(*req.Options.TimeUnit), 0)
req.Options.Max = &max
}
fos = append(fos, pilosa.OptFieldTypeTimestamp(
time.Unix(0, req.Options.Min.ToInt64(0)*pilosa.TimeUnitNanos(*req.Options.TimeUnit)).UTC(),
time.Unix(0, req.Options.Max.ToInt64(0)*pilosa.TimeUnitNanos(*req.Options.TimeUnit)).UTC(),
*req.Options.TimeUnit,
))
fos = append(fos, pilosa.OptFieldTypeTimestamp(req.Options.Epoch.UTC(), *req.Options.TimeUnit))
case pilosa.FieldTypeTime:
fos = append(fos, pilosa.OptFieldTypeTime(*req.Options.TimeQuantum, req.Options.NoStandardView))
case pilosa.FieldTypeMutex:
@ -1398,6 +1390,7 @@ type fieldOptions struct {
Min *pql.Decimal `json:"min,omitempty"`
Max *pql.Decimal `json:"max,omitempty"`
Scale *int64 `json:"scale,omitempty"`
Epoch *time.Time `json:"epoch,omitempty"`
TimeUnit *string `json:"timeUnit,omitempty"`
TimeQuantum *pilosa.TimeQuantum `json:"timeQuantum,omitempty"`
Keys *bool `json:"keys,omitempty"`

2
index_test.go
View file

@ -113,7 +113,7 @@ func TestIndex_CreateField(t *testing.T) {
defer index.Close()
// Create field with schema and verify it exists.
if f, err := index.CreateField("f", pilosa.OptFieldTypeTimestamp(pilosa.MinTimestamp, pilosa.MaxTimestamp, pilosa.TimeUnitSeconds)); err != nil {
if f, err := index.CreateField("f", pilosa.OptFieldTypeTimestamp(pilosa.DefaultEpoch, pilosa.TimeUnitSeconds)); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(f.Type(), pilosa.FieldTypeTimestamp) {
t.Fatalf("unexpected type: %#v", f.Type())