Merge pull request #1954 from molecula/fb-1188-ttl

FB-1188 - Added TTL field option
This commit is contained in:
hphamMolecula 2022-03-08 03:59:43 -06:00 • committed by GitHub
commit f7639e4a0b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
30 changed files with 777 additions and 192 deletions

View file

@ -555,7 +555,7 @@ func TestAPI_Ingest(t *testing.T) {
if err != nil {
t.Fatalf("creating field: %v", err)
}
_, err = coord.API.CreateField(ctx, index, timeField, pilosa.OptFieldTypeTime("YMD"))
_, err = coord.API.CreateField(ctx, index, timeField, pilosa.OptFieldTypeTime("YMD", "0"))
if err != nil {
t.Fatalf("creating field: %v", err)
}
@ -678,7 +678,7 @@ func BenchmarkIngest(b *testing.B) {
if err != nil {
b.Fatalf("creating field: %v", err)
}
_, err = coord.API.CreateField(ctx, index, tqField, pilosa.OptFieldTypeTime("YMDH"))
_, err = coord.API.CreateField(ctx, index, tqField, pilosa.OptFieldTypeTime("YMDH", "0"))
if err != nil {
b.Fatalf("creating field: %v", err)
}

View file

@ -43,11 +43,15 @@ func _() {
_ = x[apiIDReserve-32]
_ = x[apiIDCommit-33]
_ = x[apiIDReset-34]
_ = x[apiPartitionNodes-35]
_ = x[apiIngestOperations-36]
_ = x[apiIngestNodeOperations-37]
_ = x[apiMutexCheck-38]
}
const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteAvailableShardapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFragmentDataapiTranslateDataapiFieldTranslateDataapiFieldapiImportapiImportValueapiIndexapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiSchemaapiShardNodesapiStateapiViewsapiApplySchemaapiStartTransactionapiFinishTransactionapiTransactionsapiGetTransactionapiActiveQueriesapiPastQueriesapiIDReserveapiIDCommitapiIDReset"
const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteAvailableShardapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFragmentDataapiTranslateDataapiFieldTranslateDataapiFieldapiImportapiImportValueapiIndexapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiSchemaapiShardNodesapiStateapiViewsapiApplySchemaapiStartTransactionapiFinishTransactionapiTransactionsapiGetTransactionapiActiveQueriesapiPastQueriesapiIDReserveapiIDCommitapiIDResetapiPartitionNodesapiIngestOperationsapiIngestNodeOperationsapiMutexCheck"
var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 82, 96, 109, 121, 141, 158, 173, 189, 210, 218, 227, 241, 249, 257, 277, 290, 304, 313, 326, 334, 342, 356, 375, 395, 410, 427, 443, 457, 469, 480, 490}
var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 82, 96, 109, 121, 141, 158, 173, 189, 210, 218, 227, 241, 249, 257, 277, 290, 304, 313, 326, 334, 342, 356, 375, 395, 410, 427, 443, 457, 469, 480, 490, 507, 526, 549, 562}
func (i apiMethod) String() string {
if i < 0 || i >= apiMethod(len(_apiMethod_index)-1) {

View file

@ -1674,17 +1674,18 @@ type SchemaField struct {
// SchemaOptions contains options for a field or an index.
type SchemaOptions struct {
FieldType FieldType `json:"type"`
CacheType string `json:"cacheType"`
CacheSize uint `json:"cacheSize"`
TimeQuantum string `json:"timeQuantum"`
Min pql.Decimal `json:"min"`
Max pql.Decimal `json:"max"`
Scale int64 `json:"scale"`
Keys bool `json:"keys"`
NoStandardView bool `json:"noStandardView"`
TrackExistence bool `json:"trackExistence"`
TimeUnit string `json:"timeUnit"`
FieldType FieldType `json:"type"`
CacheType string `json:"cacheType"`
CacheSize uint `json:"cacheSize"`
TimeQuantum string `json:"timeQuantum"`
Ttl time.Duration `json:"ttl"`
Min pql.Decimal `json:"min"`
Max pql.Decimal `json:"max"`
Scale int64 `json:"scale"`
Keys bool `json:"keys"`
NoStandardView bool `json:"noStandardView"`
TrackExistence bool `json:"trackExistence"`
TimeUnit string `json:"timeUnit"`
}
func (so SchemaOptions) asIndexOptions() *IndexOptions {
@ -1702,6 +1703,7 @@ func (so SchemaOptions) asFieldOptions() *FieldOptions {
cacheSize: int(so.CacheSize),
cacheType: CacheType(so.CacheType),
timeQuantum: TimeQuantum(so.TimeQuantum),
ttl: so.Ttl,
min: so.Min,
max: so.Max,
scale: so.Scale,

View file

@ -728,6 +728,7 @@ type FieldInfo struct {
type FieldOptions struct {
fieldType FieldType
timeQuantum TimeQuantum
ttl time.Duration
cacheType CacheType
cacheSize int
min pql.Decimal
@ -751,6 +752,11 @@ func (fo FieldOptions) TimeQuantum() TimeQuantum {
return fo.timeQuantum
}
// Ttl returns the configured ttl for a time field.
func (fo FieldOptions) Ttl() time.Duration {
return fo.ttl
}
// CacheType returns the configured cache type for a "set" field. Empty string
// otherwise.
func (fo FieldOptions) CacheType() CacheType {
@ -826,6 +832,7 @@ func (fo FieldOptions) String() string {
case FieldTypeTime:
mopt["timeQuantum"] = string(fo.timeQuantum)
mopt["noStandardView"] = fo.noStandardView
mopt["ttl"] = fo.ttl.String()
case FieldTypeTimestamp:
mopt["min"] = fo.min
mopt["max"] = fo.max
@ -913,6 +920,12 @@ func OptFieldTypeTime(quantum TimeQuantum, opts ...bool) FieldOption {
}
}
func OptFieldTtl(dur time.Duration) FieldOption {
return func(options *FieldOptions) {
options.ttl = dur
}
}
// Timestamp field range.
var (
DefaultEpoch = time.Unix(0, 0).UTC() // 1970-01-01T00:00:00Z

View file

@ -925,7 +925,8 @@ func TestORM(t *testing.T) {
pql.NewDecimal(0, 0),
pql.NewDecimal(0, 0),
"",
"")
"",
0)
})
t.Run("IntFieldOptions", func(t *testing.T) {
@ -944,7 +945,8 @@ func TestORM(t *testing.T) {
pql.NewDecimal(-10, 0),
pql.NewDecimal(100, 0),
"",
"")
"",
0)
field = sampleIndex.Field("int-field2", OptFieldTypeInt(-10))
jsonString = field.options.String()
@ -962,7 +964,8 @@ func TestORM(t *testing.T) {
pql.NewDecimal(-10, 0),
pql.NewDecimal(math.MaxInt64, 0),
"",
"")
"",
0)
field = sampleIndex.Field("int-field3", OptFieldTypeInt())
jsonString = field.options.String()
targetString = fmt.Sprintf(`{"options":{"type":"int","min":%d,"max":%d}}`, math.MinInt64, math.MaxInt64)
@ -978,7 +981,8 @@ func TestORM(t *testing.T) {
pql.NewDecimal(math.MinInt64, 0),
pql.NewDecimal(math.MaxInt64, 0),
"",
"")
"",
0)
field = sampleIndex.Field("int-field4", OptFieldTypeInt(), OptFieldForeignIndex("blerg"))
jsonString = field.options.String()
@ -995,7 +999,8 @@ func TestORM(t *testing.T) {
pql.NewDecimal(math.MinInt64, 0),
pql.NewDecimal(math.MaxInt64, 0),
"blerg",
"")
"",
0)
})
t.Run("TimeFieldOptions", func(t *testing.T) {
@ -1004,7 +1009,7 @@ func TestORM(t *testing.T) {
t.Fatalf("field noStandardView %v != %v", true, field.Opts().NoStandardView())
}
jsonString := field.options.String()
targetString := `{"options":{"noStandardView":true,"type":"time","timeQuantum":"DH"}}`
targetString := `{"options":{"noStandardView":true,"type":"time","timeQuantum":"DH","ttl":"0s"}}`
if sortedString(targetString) != sortedString(jsonString) {
t.Fatalf("`%s` != `%s`", targetString, jsonString)
}
@ -1017,7 +1022,31 @@ func TestORM(t *testing.T) {
pql.NewDecimal(0, 0),
pql.NewDecimal(0, 0),
"",
"")
"",
0)
})
t.Run("TtlOptions", func(t *testing.T) {
field := sampleIndex.Field("ttl-field", OptFieldTypeTime(TimeQuantumDayHour, true), OptFieldTtl(0))
if true != field.Opts().NoStandardView() {
t.Fatalf("field noStandardView %v != %v", true, field.Opts().NoStandardView())
}
jsonString := field.options.String()
targetString := `{"options":{"noStandardView":true,"type":"time","timeQuantum":"DH","ttl":"0s"}}`
if sortedString(targetString) != sortedString(jsonString) {
t.Fatalf("`%s` != `%s`", targetString, jsonString)
}
compareFieldOptions(t,
field.Options(),
FieldTypeTime,
TimeQuantumDayHour,
CacheTypeDefault,
0,
pql.NewDecimal(0, 0),
pql.NewDecimal(0, 0),
"",
"",
0)
})
t.Run("MutexFieldOptions", func(t *testing.T) {
@ -1036,7 +1065,8 @@ func TestORM(t *testing.T) {
pql.NewDecimal(0, 0),
pql.NewDecimal(0, 0),
"",
"")
"",
0)
})
t.Run("BoolFieldOptions", func(t *testing.T) {
@ -1055,7 +1085,8 @@ func TestORM(t *testing.T) {
pql.NewDecimal(0, 0),
pql.NewDecimal(0, 0),
"",
"")
"",
0)
})
t.Run("DecimalFieldOptions", func(t *testing.T) {
@ -1074,7 +1105,8 @@ func TestORM(t *testing.T) {
pql.NewDecimal(7, 3),
pql.NewDecimal(999, 3),
"",
"")
"",
0)
})
t.Run("DecimalFieldOptions", func(t *testing.T) {
@ -1093,7 +1125,8 @@ func TestORM(t *testing.T) {
pql.NewDecimal(7, 3),
pql.NewDecimal(999, 3),
"",
"")
"",
0)
})
t.Run("TimestampFieldOptions", func(t *testing.T) {
@ -1114,7 +1147,8 @@ func TestORM(t *testing.T) {
pql.NewDecimal(MinTimestamp.UnixNano()/TimeUnitNanos(pilosa.TimeUnitSeconds), 0),
pql.NewDecimal(MaxTimestamp.UnixNano()/TimeUnitNanos(pilosa.TimeUnitSeconds), 0),
"",
pilosa.TimeUnitSeconds)
pilosa.TimeUnitSeconds,
0)
})
@ -1163,7 +1197,7 @@ func comparePQL(t *testing.T, target string, q PQLQuery) {
}
}
func compareFieldOptions(t *testing.T, opts *FieldOptions, fieldType FieldType, timeQuantum TimeQuantum, cacheType CacheType, cacheSize int, min pql.Decimal, max pql.Decimal, foreignIndex string, timeUnit string) {
func compareFieldOptions(t *testing.T, opts *FieldOptions, fieldType FieldType, timeQuantum TimeQuantum, cacheType CacheType, cacheSize int, min pql.Decimal, max pql.Decimal, foreignIndex string, timeUnit string, ttl time.Duration) {
if fieldType != opts.Type() {
t.Fatalf("%s != %s", fieldType, opts.Type())
}
@ -1188,6 +1222,9 @@ func compareFieldOptions(t *testing.T, opts *FieldOptions, fieldType FieldType,
if timeUnit != opts.TimeUnit() {
t.Fatalf("%s != %s", timeUnit, opts.TimeUnit())
}
if ttl != opts.Ttl() {
t.Fatalf("%s != %s", ttl, opts.Ttl())
}
}
func sortedString(s string) string {

View file

@ -46,6 +46,7 @@ omitted. If it is present then its format should be YYYY-MM-DDTHH:MM.
flags.StringVar(&Importer.FieldOptions.CacheType, "field-cache-type", pilosa.CacheTypeRanked, "Specify the cache type for a set field on creation. One of: none, lru, ranked")
flags.Uint32Var(&Importer.FieldOptions.CacheSize, "field-cache-size", 50000, "Specify the cache size for a set field on creation")
flags.Var(&Importer.FieldOptions.TimeQuantum, "field-time-quantum", "Specify the time quantum for a time field on creation. One of: D, DH, H, M, MD, MDH, Y, YM, YMD, YMDH")
flags.DurationVarP(&Importer.FieldOptions.Ttl, "time-to-live", "t", 0, "Specify the time to live for views created by time quantum. Supported time unit: \"s\", \"m\", \"h\"") // \"ns\", \"us\" (or \"µs\"), \"ms\" also supported but ommitted for simplicity
flags.IntVarP(&Importer.BufferSize, "buffer-size", "s", 10000000, "Number of bits to buffer/sort before importing.")
flags.BoolVarP(&Importer.Sort, "sort", "", false, "Enables sorting before import.")
flags.BoolVarP(&Importer.CreateSchema, "create", "e", false, "Create the schema if it does not exist before import.")

View file

@ -34,7 +34,7 @@ func TestExecutor_DeleteRecords(t *testing.T) {
{ID: 0, Val: 4},
{ID: 2, Val: 8},
})
c.CreateField(t, indexName, pilosa.IndexOptions{TrackExistence: true}, "timefield", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH")))
c.CreateField(t, indexName, pilosa.IndexOptions{TrackExistence: true}, "timefield", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"), "0"))
c.ImportBitsWithTimestamp(t, indexName, "timefield", [][2]uint64{
{0, 0},
{0, 1},
@ -67,7 +67,7 @@ func TestExecutor_DeleteRecords(t *testing.T) {
setupKeys := func(t *testing.T, r *require.Assertions, c *test.Cluster) {
t.Helper()
c.CreateField(t, indexName, pilosa.IndexOptions{Keys: true, TrackExistence: true}, "timefield", pilosa.OptFieldKeys(), pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH")))
c.CreateField(t, indexName, pilosa.IndexOptions{Keys: true, TrackExistence: true}, "timefield", pilosa.OptFieldKeys(), pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"), "0"))
c.ImportTimeQuantumKey(t, indexName, "timefield", []test.TimeQuantumKey{
{RowKey: "fish", ColKey: "one", Ts: time.Date(2019, time.January, 2, 17, 45, 0, 0, time.UTC).Unix()},
{RowKey: "fish", ColKey: "one", Ts: time.Date(2020, time.January, 2, 17, 45, 0, 0, time.UTC).Unix()},

View file

@ -669,6 +669,7 @@ func (s Serializer) encodeFieldOptions(o *pilosa.FieldOptions) *pb.FieldOptions
Scale: o.Scale,
BitDepth: uint64(o.BitDepth),
TimeQuantum: string(o.TimeQuantum),
Ttl: o.Ttl.String(),
TimeUnit: string(o.TimeUnit),
Keys: o.Keys,
ForeignIndex: o.ForeignIndex,
@ -1072,6 +1073,11 @@ func (s Serializer) decodeFieldOptions(options *pb.FieldOptions, m *pilosa.Field
m.Scale = options.Scale
m.BitDepth = uint64(options.BitDepth)
m.TimeQuantum = pilosa.TimeQuantum(options.TimeQuantum)
ttlValue, err := time.ParseDuration(options.Ttl)
if err != nil {
ttlValue = 0
}
m.Ttl = ttlValue
m.TimeUnit = options.TimeUnit
m.Keys = options.Keys
m.ForeignIndex = options.ForeignIndex

View file

@ -490,7 +490,7 @@ func TestExecutor(t *testing.T) {
`Row(f=1, from=1999-12-31T00:00, to=2002-01-01T03:00)`,
}
responses := runCallTest(c, t, writeQuery, readQueries,
nil, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH")))
nil, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"), "0"))
t.Run("Standard", func(t *testing.T) {
if columns := responses[0].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6, 7}) {
@ -536,7 +536,7 @@ func TestExecutor(t *testing.T) {
}
responses := runCallTest(c, t, writeQuery, readQueries,
&pilosa.IndexOptions{Keys: true},
pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH")))
pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"), "0"))
t.Run("Standard", func(t *testing.T) {
if keys := responses[0].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"six", "four", "five", "seven", "two", "three"}) {
@ -570,7 +570,7 @@ func TestExecutor(t *testing.T) {
}
responses := runCallTest(c, t, writeQuery, readQueries,
nil,
pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH")),
pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"), "0"),
pilosa.OptFieldKeys())
t.Run("Standard", func(t *testing.T) {
@ -605,7 +605,7 @@ func TestExecutor(t *testing.T) {
}
responses := runCallTest(c, t, writeQuery, readQueries,
&pilosa.IndexOptions{Keys: true},
pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH")),
pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"), "0"),
pilosa.OptFieldKeys())
t.Run("Standard", func(t *testing.T) {
@ -639,7 +639,7 @@ func TestExecutor(t *testing.T) {
`Row(f=1, from=946598400, to=1009854000)`,
}
responses := runCallTest(c, t, writeQuery, readQueries,
nil, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH")))
nil, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"), "0"))
t.Run("Standard", func(t *testing.T) {
if columns := responses[0].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6, 7}) {
@ -674,7 +674,7 @@ func TestExecutor(t *testing.T) {
`Range(f=1, from=1999-12-31T00:00, to=2002-01-01T03:00)`,
}
responses := runCallTest(c, t, writeQuery, readQueries,
nil, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH")))
nil, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"), "0"))
t.Run("Standard", func(t *testing.T) {
if columns := responses[0].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6, 7}) {
@ -693,7 +693,7 @@ func TestExecutor(t *testing.T) {
`Range(f=1, 1999-12-31T00:00, 2002-01-01T03:00)`,
}
responses = runCallTest(c, t, writeQuery, rq2,
nil, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH")))
nil, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"), "0"))
t.Run("OldRange", func(t *testing.T) {
if columns := responses[0].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6, 7}) {
t.Fatalf("unexpected columns: %+v", columns)
@ -721,7 +721,7 @@ func TestExecutor(t *testing.T) {
}
responses := runCallTest(c, t, writeQuery, readQueries,
&pilosa.IndexOptions{Keys: true},
pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH")))
pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"), "0"))
t.Run("Standard", func(t *testing.T) {
if keys := responses[0].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"two", "three", "seven", "four", "five", "six"}) {
@ -755,7 +755,7 @@ func TestExecutor(t *testing.T) {
}
responses := runCallTest(c, t, writeQuery, readQueries,
nil,
pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH")),
pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"), "0"),
pilosa.OptFieldKeys())
t.Run("Standard", func(t *testing.T) {
@ -790,7 +790,7 @@ func TestExecutor(t *testing.T) {
}
responses := runCallTest(c, t, writeQuery, readQueries,
&pilosa.IndexOptions{Keys: true},
pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH")),
pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"), "0"),
pilosa.OptFieldKeys())
t.Run("Standard", func(t *testing.T) {
@ -990,7 +990,7 @@ func TestExecutor(t *testing.T) {
}
responses := runCallTest(c, t, writeQuery, readQueries,
&pilosa.IndexOptions{TrackExistence: true},
pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMD")))
pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMD"), "0"))
if columns := responses[0].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6, 7}) {
t.Fatalf("unexpected columns: %+v", columns)
}
@ -1059,7 +1059,7 @@ func TestExecutor(t *testing.T) {
}
responses := runCallTest(c, t, writeQuery, readQueries,
nil, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMD"), true))
nil, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMD"), "0", true))
for i := range responses {
t.Run(fmt.Sprintf("response-%d", i), func(t *testing.T) {
@ -1730,7 +1730,7 @@ func TestExecutor_Execute_TopK_Time(t *testing.T) {
defer c.Close()
// Load some test data into a time field.
c.CreateField(t, "i", pilosa.IndexOptions{TrackExistence: true}, "f", pilosa.OptFieldTypeTime("YMD", true))
c.CreateField(t, "i", pilosa.IndexOptions{TrackExistence: true}, "f", pilosa.OptFieldTypeTime("YMD", "0", true))
c.Query(t, "i", `
Set(0, f=0, 2016-01-02T00:00)
Set(0, f=1, 2016-01-02T00:00)
@ -3413,7 +3413,7 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) {
})
t.Run("remote with timestamp", func(t *testing.T) {
_, err = c.GetPrimary().API.CreateField(context.Background(), "i", "z", pilosa.OptFieldTypeTime("Y"))
_, err = c.GetPrimary().API.CreateField(context.Background(), "i", "z", pilosa.OptFieldTypeTime("Y", "0"))
if err != nil {
t.Fatalf("creating field: %v", err)
}
@ -3746,7 +3746,7 @@ func TestExecutor_Time_Clear_Quantums(t *testing.T) {
indexName := strings.ToLower(string(tt.quantum))
index := hldr.MustCreateIndexIfNotExists(indexName, pilosa.IndexOptions{})
// Create field.
if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeTime(tt.quantum)); err != nil {
if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeTime(tt.quantum, "0")); err != nil {
t.Fatal(err)
}
// Populate
@ -4691,14 +4691,14 @@ func TestExecutor_Execute_Extract(t *testing.T) {
Set(3, keymutex="plugh")
`)
c.CreateField(t, "i", pilosa.IndexOptions{TrackExistence: true}, "time", pilosa.OptFieldTypeTime("YMDH"))
c.CreateField(t, "i", pilosa.IndexOptions{TrackExistence: true}, "time", pilosa.OptFieldTypeTime("YMDH", "0"))
c.Query(t, "i", `
Set(0, time=1, 2016-01-01T00:00)
Set(1, time=2, 2017-01-01T00:00)
Set(3, time=3, 2018-01-01T00:00)
`)
c.CreateField(t, "i", pilosa.IndexOptions{TrackExistence: true}, "keytime", pilosa.OptFieldKeys(), pilosa.OptFieldTypeTime("YMDH"))
c.CreateField(t, "i", pilosa.IndexOptions{TrackExistence: true}, "keytime", pilosa.OptFieldKeys(), pilosa.OptFieldTypeTime("YMDH", "0"))
c.Query(t, "i", `
Set(0, keytime="h", 2016-01-01T00:00)
Set(1, keytime="xyzzy", 2017-01-01T00:00)
@ -5087,7 +5087,7 @@ func TestExecutor_Execute_RowsTime(t *testing.T) {
func TestExecutor_Execute_RowsTimeEmpty(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()
c.CreateField(t, "i", pilosa.IndexOptions{}, "x", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMD"), true))
c.CreateField(t, "i", pilosa.IndexOptions{}, "x", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMD"), "0", true))
rows := c.Query(t, "i", `Rows(x, from=1999-12-31T00:00, to=2002-01-01T03:00)`).Results[0].(pilosa.RowIdentifiers).Rows
if !reflect.DeepEqual(rows, []uint64{}) {
t.Fatalf("unexpected rows: %+v", rows)
@ -7356,7 +7356,7 @@ func variousQueriesOnTimeFields(t *testing.T, c *test.Cluster) {
// generic index
// worth noting, since we are using YMDH resolution, both C4 & C5
// get binned to the same hour
c.CreateField(t, "t_index", pilosa.IndexOptions{Keys: true, TrackExistence: true}, "f1", pilosa.OptFieldKeys(), pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH")))
c.CreateField(t, "t_index", pilosa.IndexOptions{Keys: true, TrackExistence: true}, "f1", pilosa.OptFieldKeys(), pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"), "0"))
c.ImportTimeQuantumKey(t, "t_index", "f1", []test.TimeQuantumKey{
// from edge cases
{ColKey: "C1", RowKey: "R1", Ts: ts(time.Date(2019, 1, 1, 0, 0, 0, 0, time.UTC))},
@ -7372,7 +7372,7 @@ func variousQueriesOnTimeFields(t *testing.T, c *test.Cluster) {
})
// in this field, all columns have the same row value to simplify test queries for Row
c.CreateField(t, "t_index", pilosa.IndexOptions{Keys: true, TrackExistence: true}, "f2", pilosa.OptFieldKeys(), pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH")))
c.CreateField(t, "t_index", pilosa.IndexOptions{Keys: true, TrackExistence: true}, "f2", pilosa.OptFieldKeys(), pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"), "0"))
c.ImportTimeQuantumKey(t, "t_index", "f2", []test.TimeQuantumKey{
// from
{ColKey: "C1", RowKey: "R", Ts: ts(time.Date(2019, 1, 1, 0, 0, 0, 0, time.UTC))},
@ -7522,7 +7522,7 @@ func populateTestData(t *testing.T, c *test.Cluster) {
})
// Create and populate "places_visited" time field.
c.CreateField(t, usersIndex, pilosa.IndexOptions{Keys: true, TrackExistence: true}, "places_visited", pilosa.OptFieldKeys(), pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YM")))
c.CreateField(t, usersIndex, pilosa.IndexOptions{Keys: true, TrackExistence: true}, "places_visited", pilosa.OptFieldKeys(), pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YM"), "0"))
ts2019Jan01 := int64(1546300800) * 1e+9 // 2019 January 1st 0:00:00
ts2019Aug01 := int64(1564617600) * 1e+9 // 2019 August 1st 0:00:00
ts2020Jan01 := int64(1577836800) * 1e+9 // 2020 January 1st 0:00:00

View file

@ -286,7 +286,7 @@ func OptFieldTypeDecimal(scale int64, minmax ...pql.Decimal) FieldOption {
// used to specify the field as being type `time` and to
// provide any respective configuration values.
// Pass true to skip creation of the standard view.
func OptFieldTypeTime(timeQuantum TimeQuantum, opt ...bool) FieldOption {
func OptFieldTypeTime(timeQuantum TimeQuantum, ttl string, opt ...bool) FieldOption {
return func(fo *FieldOptions) error {
if fo.Type != "" {
return errors.Errorf("field type is already set to: %s", fo.Type)
@ -296,6 +296,11 @@ func OptFieldTypeTime(timeQuantum TimeQuantum, opt ...bool) FieldOption {
}
fo.Type = FieldTypeTime
fo.TimeQuantum = timeQuantum
ttlParsed, err := time.ParseDuration(ttl)
if err != nil {
return errors.Errorf("cannot parse ttl: %s", ttl)
}
fo.Ttl = ttlParsed
fo.NoStandardView = len(opt) >= 1 && opt[0]
return nil
}
@ -674,6 +679,11 @@ func (f *Field) ForeignIndex() string {
return f.options.ForeignIndex
}
// Ttl returns the ttl of the field.
func (f *Field) Ttl() time.Duration {
return f.options.Ttl
}
func (f *Field) bitDepth() (uint64, error) {
var maxBitDepth uint64
@ -763,6 +773,7 @@ func (f *Field) applyOptions(opt FieldOptions) error {
f.options.Base = 0
f.options.BitDepth = 0
f.options.TimeQuantum = ""
f.options.Ttl = 0
f.options.Keys = opt.Keys
f.options.ForeignIndex = opt.ForeignIndex
case FieldTypeInt, FieldTypeDecimal, FieldTypeTimestamp:
@ -776,6 +787,7 @@ func (f *Field) applyOptions(opt FieldOptions) error {
f.options.BitDepth = opt.BitDepth
f.options.TimeUnit = opt.TimeUnit
f.options.TimeQuantum = ""
f.options.Ttl = 0
f.options.Keys = opt.Keys
f.options.ForeignIndex = opt.ForeignIndex
@ -809,6 +821,7 @@ func (f *Field) applyOptions(opt FieldOptions) error {
return ErrInvalidTimeQuantum
}
f.options.TimeQuantum = opt.TimeQuantum
f.options.Ttl = opt.Ttl
f.options.ForeignIndex = opt.ForeignIndex
case FieldTypeBool:
f.options.Type = FieldTypeBool
@ -819,6 +832,7 @@ func (f *Field) applyOptions(opt FieldOptions) error {
f.options.Base = 0
f.options.BitDepth = 0
f.options.TimeQuantum = ""
f.options.Ttl = 0
f.options.Keys = false
f.options.ForeignIndex = ""
default:
@ -1831,19 +1845,20 @@ func (p fieldInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name }
// FieldOptions represents options to set when initializing a field.
type FieldOptions struct {
Base int64 `json:"base,omitempty"`
BitDepth uint64 `json:"bitDepth,omitempty"`
Min pql.Decimal `json:"min,omitempty"`
Max pql.Decimal `json:"max,omitempty"`
Scale int64 `json:"scale,omitempty"`
Keys bool `json:"keys"`
NoStandardView bool `json:"noStandardView,omitempty"`
CacheSize uint32 `json:"cacheSize,omitempty"`
CacheType string `json:"cacheType,omitempty"`
Type string `json:"type,omitempty"`
TimeUnit string `json:"timeUnit,omitempty"`
TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"`
ForeignIndex string `json:"foreignIndex"`
Base int64 `json:"base,omitempty"`
BitDepth uint64 `json:"bitDepth,omitempty"`
Min pql.Decimal `json:"min,omitempty"`
Max pql.Decimal `json:"max,omitempty"`
Scale int64 `json:"scale,omitempty"`
Keys bool `json:"keys"`
NoStandardView bool `json:"noStandardView,omitempty"`
CacheSize uint32 `json:"cacheSize,omitempty"`
CacheType string `json:"cacheType,omitempty"`
Type string `json:"type,omitempty"`
TimeUnit string `json:"timeUnit,omitempty"`
TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"`
ForeignIndex string `json:"foreignIndex"`
Ttl time.Duration `json:"ttl,omitempty"`
}
// newFieldOptions returns a new instance of FieldOptions
@ -1954,15 +1969,17 @@ func (o *FieldOptions) MarshalJSON() ([]byte, error) {
})
case FieldTypeTime:
return json.Marshal(struct {
Type string `json:"type"`
TimeQuantum TimeQuantum `json:"timeQuantum"`
Keys bool `json:"keys"`
NoStandardView bool `json:"noStandardView"`
Type string `json:"type"`
TimeQuantum TimeQuantum `json:"timeQuantum"`
Keys bool `json:"keys"`
NoStandardView bool `json:"noStandardView"`
Ttl time.Duration `json:"Ttl"`
}{
o.Type,
o.TimeQuantum,
o.Keys,
o.NoStandardView,
o.Ttl,
})
case FieldTypeMutex:
return json.Marshal(struct {

View file

@ -343,7 +343,7 @@ func TestField_CreateViewIfNotExists(t *testing.T) {
}
func TestField_SetTimeQuantum(t *testing.T) {
f := OpenField(t, OptFieldTypeTime(TimeQuantum("YMDH")))
f := OpenField(t, OptFieldTypeTime(TimeQuantum("YMDH"), "0"))
defer f.Close()
// Retrieve time quantum.
@ -360,7 +360,7 @@ func TestField_SetTimeQuantum(t *testing.T) {
}
func TestField_RowTime(t *testing.T) {
f := OpenField(t, OptFieldTypeTime(TimeQuantum("YMDH")))
f := OpenField(t, OptFieldTypeTime(TimeQuantum("YMDH"), "0"))
defer f.Close()
// Obtain transaction.

5
go.mod
View file

@ -54,11 +54,12 @@ require (
go.etcd.io/etcd v0.0.0-20201125193152-8a03d2e9614b
golang.org/x/crypto v0.0.0-20201217014255-9d1352758620 // indirect
golang.org/x/exp v0.0.0-20201008143054-e3b2a7f2fdc7
golang.org/x/mod v0.4.2
golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d // indirect
golang.org/x/mod v0.5.1
golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f // indirect
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
golang.org/x/sys v0.0.0-20220111092808-5a964db01320 // indirect
golang.org/x/text v0.3.7 // indirect
google.golang.org/grpc v1.28.0
gopkg.in/yaml.v2 v2.4.0
modernc.org/mathutil v1.0.0

11
go.sum
View file

@ -444,8 +444,8 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.5.1 h1:OJxoQ/rynoF0dcCdI7cLPktw/hR2cueqYfjm43oqK38=
golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@ -465,8 +465,8 @@ golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLL
golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d h1:20cMwl2fHAzkJMEA+8J4JgqBQcQGzbisXo31MIeenXI=
golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f h1:OfiFi4JbukWwe3lzw+xunroH1mnC1e2Gy5cxNJApiSY=
golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0=
@ -515,8 +515,9 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ=

View file

@ -2,6 +2,8 @@
package pilosa
import (
"time"
"github.com/gogo/protobuf/proto"
"github.com/molecula/featurebase/v3/pb"
"github.com/molecula/featurebase/v3/pql"
@ -57,6 +59,11 @@ func UnmarshalFieldOptions(name string, createdAt int64, buf []byte) (*FieldInfo
fi.Options.Base = pbi.Base
fi.Options.BitDepth = pbi.BitDepth
fi.Options.TimeQuantum = TimeQuantum(pbi.TimeQuantum)
ttlValue, err := time.ParseDuration(pbi.Ttl)
if err != nil {
ttlValue = 0
}
fi.Options.Ttl = ttlValue
fi.Options.Keys = pbi.Keys
fi.Options.NoStandardView = pbi.NoStandardView

View file

@ -453,7 +453,7 @@ func TestHolderSyncer_TimeQuantum(t *testing.T) {
if err != nil {
t.Fatalf("creating index i: %v", err)
}
_, err = c.GetNode(0).API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeTime(pilosa.TimeQuantum(quantum)))
_, err = c.GetNode(0).API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeTime(pilosa.TimeQuantum(quantum), "0"))
if err != nil {
t.Fatalf("creating field f: %v", err)
}

View file

@ -429,6 +429,8 @@ func newRouter(handler *Handler) http.Handler {
//router.HandleFunc("/index/{index}/field", handler.chkAuthZ(handler.handleGetFields, authz.Read)).Methods("GET") // Not implemented.
router.HandleFunc("/index/{index}/field", handler.chkAuthZ(handler.handlePostField, authz.Write)).Methods("POST").Name("PostField")
router.HandleFunc("/index/{index}/field/", handler.chkAuthZ(handler.handlePostField, authz.Write)).Methods("POST").Name("PostField")
router.HandleFunc("/index/{index}/field/{field}/view", handler.chkAuthZ(handler.handleGetView, authz.Admin)).Methods("GET")
router.HandleFunc("/index/{index}/field/{field}/view/{view}", handler.chkAuthZ(handler.handleDeleteView, authz.Admin)).Methods("DELETE").Name("DeleteView")
router.HandleFunc("/index/{index}/field/{field}", handler.chkAuthZ(handler.handlePostField, authz.Write)).Methods("POST").Name("PostField")
router.HandleFunc("/index/{index}/field/{field}", handler.chkAuthZ(handler.handleDeleteField, authz.Write)).Methods("DELETE").Name("DeleteField")
router.HandleFunc("/index/{index}/field/{field}/import", handler.chkAuthZ(handler.handlePostImport, authz.Write)).Methods("POST").Name("PostImport")
@ -1281,6 +1283,62 @@ func (h *Handler) handleGetIndex(w http.ResponseWriter, r *http.Request) {
http.Error(w, fmt.Sprintf("Index %s Not Found", indexName), http.StatusNotFound)
}
// handleGetView handles GET /index/<indexname>/field/<fieldname>/view requests.
func (h *Handler) handleGetView(w http.ResponseWriter, r *http.Request) {
if !validHeaderAcceptJSON(r.Header) {
http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable)
return
}
indexName := mux.Vars(r)["index"]
fieldName := mux.Vars(r)["field"]
index, err := h.api.Index(r.Context(), indexName)
if err != nil {
http.Error(w, fmt.Sprintf("Index %s Not Found", indexName), http.StatusNotFound)
return
} else {
w.Header().Set("Content-Type", "application/json")
var viewsList []viewReponse
for _, field := range index.fields {
if field.name == fieldName {
for _, view := range field.views() {
viewsList = append(viewsList, viewReponse{Name: view.name, Type: view.fieldType, Field: view.field, Index: view.index})
}
if err := json.NewEncoder(w).Encode(viewsList); err != nil {
h.logger.Errorf("write response error: %s", err)
}
return
}
}
}
http.Error(w, fmt.Sprintf("Field %s Not Found", fieldName), http.StatusNotFound)
}
type viewReponse struct {
Name string `json:"name"`
Type string `json:"type"`
Field string `json:"field"`
Index string `json:"index"`
}
// handleDeleteIndex handles DELETE /index/<indexname>/field/<fieldname>/view/<viewname> request.
func (h *Handler) handleDeleteView(w http.ResponseWriter, r *http.Request) {
if !validHeaderAcceptJSON(r.Header) {
http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable)
return
}
indexName := mux.Vars(r)["index"]
fieldName := mux.Vars(r)["field"]
viewName := mux.Vars(r)["view"]
resp := successResponse{h: h}
err := h.api.DeleteView(r.Context(), indexName, fieldName, viewName)
resp.write(w, err)
}
type postIndexRequest struct {
Options IndexOptions `json:"options"`
}
@ -1524,7 +1582,11 @@ func fieldOptionsToFunctionalOpts(opt fieldOptions) []FieldOption {
}
fos = append(fos, OptFieldTypeTimestamp(opt.Epoch.UTC(), *opt.TimeUnit))
case FieldTypeTime:
fos = append(fos, OptFieldTypeTime(*opt.TimeQuantum, opt.NoStandardView))
if opt.Ttl != nil {
fos = append(fos, OptFieldTypeTime(*opt.TimeQuantum, *opt.Ttl, opt.NoStandardView))
} else {
fos = append(fos, OptFieldTypeTime(*opt.TimeQuantum, "0", opt.NoStandardView))
}
case FieldTypeMutex:
fos = append(fos, OptFieldTypeMutex(*opt.CacheType, *opt.CacheSize))
case FieldTypeBool:
@ -1646,6 +1708,7 @@ type fieldOptionSpec struct {
Epoch *time.Time `json:"epoch"`
Unit *string `json:"unit"`
TimeQuantum *string `json:"time-quantum"`
Ttl *string `json:"ttl"`
}
func fieldSpecToFieldOption(fSpec fieldSpec) fieldOptions {
@ -1679,6 +1742,7 @@ func fieldSpecToFieldOption(fSpec fieldSpec) fieldOptions {
timeQuantumVal := TimeQuantum(*fSpec.FieldOptions.TimeQuantum)
opt.TimeQuantum = &timeQuantumVal
}
opt.Ttl = fSpec.FieldOptions.Ttl
return opt
}
@ -1918,6 +1982,7 @@ type fieldOptions struct {
Keys *bool `json:"keys,omitempty"`
NoStandardView bool `json:"noStandardView,omitempty"`
ForeignIndex *string `json:"foreignIndex,omitempty"`
Ttl *string `json:"ttl,omitempty"`
}
func (o *fieldOptions) validate() error {
@ -1945,6 +2010,8 @@ func (o *fieldOptions) validate() error {
return NewBadRequestError(errors.New("max does not apply to field type set"))
} else if o.TimeQuantum != nil {
return NewBadRequestError(errors.New("timeQuantum does not apply to field type set"))
} else if o.Ttl != nil {
return NewBadRequestError(errors.New("ttl does not apply to field type set"))
}
case FieldTypeInt:
if o.CacheType != nil {
@ -1953,6 +2020,8 @@ func (o *fieldOptions) validate() error {
return NewBadRequestError(errors.New("cacheSize does not apply to field type int"))
} else if o.TimeQuantum != nil {
return NewBadRequestError(errors.New("timeQuantum does not apply to field type int"))
} else if o.Ttl != nil {
return NewBadRequestError(errors.New("ttl does not apply to field type int"))
}
case FieldTypeDecimal:
if o.Scale == nil {
@ -1963,6 +2032,8 @@ func (o *fieldOptions) validate() error {
return NewBadRequestError(errors.New("cacheSize does not apply to field type int"))
} else if o.TimeQuantum != nil {
return NewBadRequestError(errors.New("timeQuantum does not apply to field type int"))
} else if o.Ttl != nil {
return NewBadRequestError(errors.New("ttl does not apply to field type int"))
} else if o.ForeignIndex != nil && o.Type == FieldTypeDecimal {
return NewBadRequestError(errors.New("decimal field cannot be a foreign key"))
}
@ -1977,6 +2048,8 @@ func (o *fieldOptions) validate() error {
return NewBadRequestError(errors.New("cacheSize does not apply to field type timestamp"))
} else if o.TimeQuantum != nil {
return NewBadRequestError(errors.New("timeQuantum does not apply to field type timestamp"))
} else if o.Ttl != nil {
return NewBadRequestError(errors.New("ttl does not apply to field type timestamp"))
} else if o.ForeignIndex != nil {
return NewBadRequestError(errors.New("timestamp field cannot be a foreign key"))
}
@ -2005,6 +2078,8 @@ func (o *fieldOptions) validate() error {
return NewBadRequestError(errors.New("max does not apply to field type mutex"))
} else if o.TimeQuantum != nil {
return NewBadRequestError(errors.New("timeQuantum does not apply to field type mutex"))
} else if o.Ttl != nil {
return NewBadRequestError(errors.New("ttl does not apply to field type mutex"))
}
case FieldTypeBool:
if o.CacheType != nil {
@ -2019,6 +2094,8 @@ func (o *fieldOptions) validate() error {
return NewBadRequestError(errors.New("timeQuantum does not apply to field type bool"))
} else if o.Keys != nil {
return NewBadRequestError(errors.New("keys does not apply to field type bool"))
} else if o.Ttl != nil {
return NewBadRequestError(errors.New("ttl does not apply to field type bool"))
} else if o.ForeignIndex != nil {
return NewBadRequestError(errors.New("bool field cannot be a foreign key"))
}

View file

@ -131,6 +131,7 @@ func TestFieldOptionValidation(t *testing.T) {
{json: `{"options": {"type": "set", "min": 0}}`, err: "min does not apply to field type set"},
{json: `{"options": {"type": "set", "max": 100}}`, err: "max does not apply to field type set"},
{json: `{"options": {"type": "set", "timeQuantum": "YMD"}}`, err: "timeQuantum does not apply to field type set"},
{json: `{"options": {"type": "set", "ttl": "1h"}}`, err: "ttl does not apply to field type set"},
// FieldType: Int
{json: `{"options": {"type": "int"}}`, err: "min is required for field type int"},
@ -143,6 +144,7 @@ func TestFieldOptionValidation(t *testing.T) {
{json: `{"options": {"type": "int", "min": 0, "max": 1000, "cacheType": "ranked"}}`, err: "cacheType does not apply to field type int"},
{json: `{"options": {"type": "int", "min": 0, "max": 1000, "cacheSize": 1000}}`, err: "cacheSize does not apply to field type int"},
{json: `{"options": {"type": "int", "min": 0, "max": 1000, "timeQuantum": "YMD"}}`, err: "timeQuantum does not apply to field type int"},
{json: `{"options": {"type": "int", "min": 0, "max": 1000, "ttl": "1h"}}`, err: "ttl does not apply to field type int"},
// FieldType: Time
{json: `{"options": {"type": "time"}}`, err: "timeQuantum is required for field type time"},

View file

@ -6,6 +6,8 @@ import (
"fmt"
"net"
gohttp "net/http"
"reflect"
"sort"
"strings"
"testing"
@ -170,6 +172,166 @@ func TestIngestSchemaHandler(t *testing.T) {
}
}
func TestPostFieldWithTtl(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()
schema := `
{
"index-name": "example",
"primary-key-type": "string",
"index-action": "create",
"fields":[]
}
`
m := c.GetPrimary()
schemaURL := fmt.Sprintf("%s/internal/schema", m.URL())
resp := test.Do(t, "POST", schemaURL, string(schema))
if resp.StatusCode != gohttp.StatusOK {
t.Errorf("invalid status: %d, body=%s", resp.StatusCode, resp.Body)
}
postFieldTtlUrl := fmt.Sprintf("%s/index/example/field/with_ttl", m.URL())
// Create new field with ttl but in invalid format
fieldOptionInvalidTtl := `
{ "options": {"timeQuantum":"YMDH","type":"time","ttl":"24hour" }}
`
respField := test.Do(t, "POST", postFieldTtlUrl, string(fieldOptionInvalidTtl))
if (respField.StatusCode != gohttp.StatusBadRequest) &&
(respField.Body != "applying option: cannot parse ttl: 24hour") {
t.Errorf("expected ttl parse error, got status: %d, body=%s", respField.StatusCode, respField.Body)
}
// Create new field with ttl in invalid format
fieldOptionValidTtl := `
{ "options": {"timeQuantum":"YMDH","type":"time","ttl":"24h" }}
`
respField = test.Do(t, "POST", postFieldTtlUrl, string(fieldOptionValidTtl))
if resp.StatusCode != gohttp.StatusOK {
t.Errorf("creating field with ttl, got status: %d, body=%s", respField.StatusCode, respField.Body)
}
// Create new field without ttl
postFieldNoTtlUrl := fmt.Sprintf("%s/index/example/field/no_ttl", m.URL())
fieldOptionNoTtl := `
{ "options": {"timeQuantum":"YMDH","type":"time" }}
`
respField = test.Do(t, "POST", postFieldNoTtlUrl, string(fieldOptionNoTtl))
if resp.StatusCode != gohttp.StatusOK {
t.Errorf("creating field without ttl, status: %d, body=%s", respField.StatusCode, respField.Body)
}
}
func TestGetViewAndDelete(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()
schema := `
{
"index-name": "example",
"primary-key-type": "string",
"index-action": "create",
"fields": [
{
"field-name": "test_view",
"field-type": "time",
"field-options": {
"time-quantum": "YMDH"
}
}
]
}
`
m := c.GetPrimary()
schemaURL := fmt.Sprintf("%s/internal/schema", m.URL())
resp := test.Do(t, "POST", schemaURL, string(schema))
if resp.StatusCode != gohttp.StatusOK {
t.Errorf("invalid status: %d, body=%s", resp.StatusCode, resp.Body)
}
// Send sample data
postQueryUrl := fmt.Sprintf("%s/index/example/query", m.URL())
queryOption := `
Set(1,test_view=1,2001-02-03T04:05)
`
respQuery := test.Do(t, "POST", postQueryUrl, string(queryOption))
if respQuery.StatusCode != gohttp.StatusOK {
t.Errorf("posting query, status: %d, body=%s", respQuery.StatusCode, respQuery.Body)
}
// The above sample data should create these views:
expectedViewNames := []string{
"standard",
"standard_2001",
"standard_200102",
"standard_20010203",
"standard_2001020304",
}
// Call view to get data
viewUrl := fmt.Sprintf("%s/index/example/field/test_view/view", m.URL())
respView := test.Do(t, "GET", viewUrl, "")
if respView.StatusCode != gohttp.StatusOK {
t.Errorf("view handler, status: %d, body=%s", respView.StatusCode, respView.Body)
}
type viewReponse struct {
Name string `json:"name"`
Type string `json:"type"`
Field string `json:"field"`
Index string `json:"index"`
}
var parsedViews []viewReponse
if err := json.Unmarshal([]byte(respView.Body), &parsedViews); err != nil {
t.Errorf("parsing view, err: %s", err)
}
// check if data from view matches with expectedViewNames
parseViewNames := []string{}
for _, view := range parsedViews {
parseViewNames = append(parseViewNames, view.Name)
}
sort.Strings(parseViewNames)
if !reflect.DeepEqual(expectedViewNames, parseViewNames) {
t.Fatalf("expected %v, but got %v", expectedViewNames, parseViewNames)
}
// call delete on view standard_2001020304
deleteViewUrl := fmt.Sprintf("%s/index/example/field/test_view/view/standard_2001020304", m.URL())
respDelete := test.Do(t, "DELETE", deleteViewUrl, "")
if respDelete.StatusCode != gohttp.StatusOK {
t.Errorf("delete handler, status: %d, body=%s", respDelete.StatusCode, respDelete.Body)
}
// remove view that was deleted (standard_2001020304) from expectedViewNames
expectedViewNames = expectedViewNames[:len(expectedViewNames)-1]
// call view again
viewUrl = fmt.Sprintf("%s/index/example/field/test_view/view", m.URL())
respView = test.Do(t, "GET", viewUrl, "")
if respView.StatusCode != gohttp.StatusOK {
t.Errorf("view handler after delete, status: %d, body=%s", respView.StatusCode, respView.Body)
}
if err := json.Unmarshal([]byte(respView.Body), &parsedViews); err != nil {
t.Errorf("parsing view, err: %s", err)
}
// check if data from view matches with expectedViewNames
parseViewNames = []string{}
for _, view := range parsedViews {
parseViewNames = append(parseViewNames, view.Name)
}
sort.Strings(parseViewNames)
if !reflect.DeepEqual(expectedViewNames, parseViewNames) {
t.Fatalf("after delete, expected %v, but got %v", expectedViewNames, parseViewNames)
}
}
func TestTranslationHandlers(t *testing.T) {
// reusable data for the tests
nameBytes, err := json.Marshal([]string{"a", "b", "c"})

View file

@ -55,7 +55,7 @@ func TestIndex_CreateField(t *testing.T) {
defer index.Close()
// Create field with explicit quantum.
f, err := index.CreateField("f", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH")))
f, err := index.CreateField("f", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"), "0"))
if err != nil {
t.Fatal(err)
} else if q := f.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") {
@ -71,7 +71,7 @@ func TestIndex_CreateField(t *testing.T) {
defer index.Close()
// Create field with explicit quantum with no standard view
f, err := index.CreateField("f", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"), true))
f, err := index.CreateField("f", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"), "0", true))
if err != nil {
t.Fatal(err)
} else if q := f.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") {

View file

@ -1094,6 +1094,8 @@ func (c *InternalClient) CreateFieldWithOptions(ctx context.Context, index, fiel
fieldOpt.Max = &opt.Max
case FieldTypeTime:
fieldOpt.TimeQuantum = &opt.TimeQuantum
ttlString := opt.Ttl.String()
fieldOpt.Ttl = &ttlString
case FieldTypeBool:
// pass
case FieldTypeDecimal:

View file

@ -1175,6 +1175,46 @@ func TestClient_FragmentBlocks(t *testing.T) {
}
}
func TestClient_CreateTimeField(t *testing.T) {
cluster := test.MustRunCluster(t, 1)
defer cluster.Close()
cmd := cluster.GetNode(0)
c := MustNewClient(cmd.URL(), pilosa.GetHTTPClient(nil))
index := "cdf"
err := c.CreateIndex(context.Background(), index, pilosa.IndexOptions{})
if err != nil {
t.Fatalf("creating index: %v", err)
}
field := "field"
err = c.CreateFieldWithOptions(context.Background(), index, field, pilosa.FieldOptions{Type: pilosa.FieldTypeTime, TimeQuantum: "YMDH"})
if err != nil {
t.Fatalf("creating field: %v", err)
}
fld, err := cmd.API.Field(context.Background(), index, field)
if err != nil {
t.Fatalf("getting field: %v", err)
}
if fld.Ttl() != 0 {
t.Fatalf("expected Ttl to be 0, got: %+v", fld.Options().Ttl.String())
}
fieldTtl := "field_ttl"
err = c.CreateFieldWithOptions(context.Background(), index, fieldTtl, pilosa.FieldOptions{Type: pilosa.FieldTypeTime, TimeQuantum: "YMDH", Ttl: time.Hour})
if err != nil {
t.Fatalf("creating field: %v", err)
}
fldTtl, err := cmd.API.Field(context.Background(), index, fieldTtl)
if err != nil {
t.Fatalf("getting field: %v", err)
}
if fldTtl.Ttl() != time.Hour {
t.Fatalf("expected Ttl 1 hour, got: %+v", fldTtl.Ttl().String())
}
}
func TestClient_CreateDecimalField(t *testing.T) {
cluster := test.MustRunCluster(t, 1)
defer cluster.Close()

View file

@ -93,6 +93,7 @@ type FieldOptions struct {
Min *Decimal `protobuf:"bytes,17,opt,name=Min,proto3" json:"Min,omitempty"`
Max *Decimal `protobuf:"bytes,18,opt,name=Max,proto3" json:"Max,omitempty"`
TimeUnit string `protobuf:"bytes,19,opt,name=TimeUnit,proto3" json:"TimeUnit,omitempty"`
Ttl string `protobuf:"bytes,20,opt,name=Ttl,proto3" json:"Ttl,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
@ -236,6 +237,13 @@ func (m *FieldOptions) GetTimeUnit() string {
return ""
}
func (m *FieldOptions) GetTtl() string {
if m != nil {
return m.Ttl
}
return ""
}
type ImportResponse struct {
Err string `protobuf:"bytes,1,opt,name=Err,proto3" json:"Err,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
@ -2759,110 +2767,111 @@ func init() {
func init() { proto.RegisterFile("private.proto", fileDescriptor_d2a91b51c7bdc125) }
var fileDescriptor_d2a91b51c7bdc125 = []byte{
// 1639 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0xdf, 0x6e, 0x1b, 0x45,
0x17, 0xff, 0x76, 0xd7, 0x8e, 0xed, 0xe3, 0x38, 0x71, 0xa6, 0xf9, 0xfa, 0x6d, 0xd2, 0x7e, 0x91,
0x33, 0xa0, 0xd6, 0x44, 0x22, 0x88, 0xf4, 0xa2, 0x08, 0x6e, 0x9a, 0xd8, 0x69, 0x31, 0x25, 0x6d,
0x3a, 0x49, 0x73, 0x09, 0x9a, 0xd8, 0xa3, 0x64, 0x95, 0xf5, 0xae, 0xd9, 0x5d, 0xa7, 0x76, 0x2f,
0x90, 0x40, 0x20, 0xb8, 0xe1, 0x9e, 0x2b, 0x9e, 0x81, 0x1b, 0xde, 0x81, 0x1b, 0x24, 0x1e, 0x01,
0x95, 0x17, 0x41, 0x73, 0x66, 0x66, 0x77, 0xed, 0x3a, 0x35, 0x44, 0xdc, 0xed, 0xf9, 0x9d, 0x99,
0xf3, 0x7f, 0xce, 0x9c, 0x59, 0xa8, 0x0d, 0x22, 0xef, 0x92, 0x27, 0x62, 0x7b, 0x10, 0x85, 0x49,
0x48, 0xec, 0xc1, 0xe9, 0xfa, 0xe2, 0x60, 0x78, 0xea, 0x7b, 0x5d, 0x85, 0xd0, 0x47, 0x50, 0xe9,
0x04, 0x3d, 0x31, 0x3a, 0x10, 0x09, 0x27, 0x04, 0x0a, 0x8f, 0xc5, 0x38, 0x76, 0x9d, 0x86, 0xd5,
0x2c, 0x33, 0xfc, 0x26, 0x77, 0x60, 0xe9, 0x38, 0xe2, 0xdd, 0x8b, 0xfd, 0x91, 0x17, 0x27, 0x22,
0xe8, 0x0a, 0xb7, 0x80, 0xdc, 0x29, 0x94, 0xfe, 0xec, 0xc0, 0xe2, 0x43, 0x4f, 0xf8, 0xbd, 0xa7,
0x83, 0xc4, 0x0b, 0x83, 0x58, 0x0a, 0x3b, 0x1e, 0x0f, 0x84, 0x5b, 0x6e, 0x58, 0xcd, 0x0a, 0xc3,
0x6f, 0x72, 0x1b, 0x2a, 0x2d, 0xde, 0x3d, 0x17, 0xc8, 0x70, 0x90, 0x91, 0x01, 0x29, 0xf7, 0xc8,
0x7b, 0xa9, 0xb4, 0xd4, 0x58, 0x06, 0x90, 0x06, 0x54, 0x8f, 0xbd, 0xbe, 0x78, 0x36, 0xe4, 0x41,
0x32, 0xec, 0xbb, 0x45, 0xdc, 0x9d, 0x87, 0xc8, 0x4d, 0x58, 0x78, 0xea, 0xf7, 0x0e, 0xbc, 0xc0,
0xad, 0x34, 0xac, 0xa6, 0xc3, 0x34, 0x65, 0x70, 0x3e, 0x72, 0x21, 0xc3, 0xf9, 0x28, 0x75, 0xb7,
0x3a, 0xe9, 0xee, 0x93, 0xf0, 0x28, 0xe1, 0x41, 0x8f, 0x47, 0xbd, 0x13, 0x4f, 0xbc, 0x70, 0x17,
0x95, 0xbb, 0x93, 0xa8, 0xdc, 0xbb, 0xc7, 0x63, 0xe1, 0xd6, 0x50, 0x22, 0x7e, 0x93, 0x75, 0x28,
0xef, 0x79, 0x49, 0x5b, 0x0c, 0x92, 0x73, 0x77, 0xa9, 0x61, 0x35, 0x0b, 0x2c, 0xa5, 0xc9, 0x2a,
0x14, 0x8f, 0xba, 0xdc, 0x17, 0xee, 0x32, 0x6e, 0x50, 0x04, 0xa1, 0xb0, 0xf8, 0x30, 0x8c, 0x84,
0x77, 0x16, 0x60, 0x12, 0xdc, 0x3a, 0x3a, 0x35, 0x81, 0x91, 0xff, 0x83, 0x23, 0x5d, 0x5a, 0x69,
0x58, 0xcd, 0xea, 0x4e, 0x75, 0x7b, 0x70, 0xba, 0xdd, 0x16, 0x5d, 0xaf, 0xcf, 0x7d, 0x26, 0x71,
0x64, 0xf3, 0x91, 0x4b, 0x66, 0xb1, 0xf9, 0x48, 0xda, 0x24, 0x43, 0xf4, 0x3c, 0xf0, 0x12, 0xf7,
0x06, 0x4a, 0x4f, 0x69, 0x4a, 0x61, 0xa9, 0xd3, 0x1f, 0x84, 0x51, 0xc2, 0x44, 0x3c, 0x08, 0x83,
0x58, 0x90, 0x3a, 0x38, 0xfb, 0x51, 0xe4, 0x5a, 0xb8, 0x50, 0x7e, 0xd2, 0x2f, 0xa1, 0xbe, 0xe7,
0x87, 0xdd, 0x8b, 0x36, 0x4f, 0x38, 0x13, 0x5f, 0x0c, 0x45, 0x9c, 0x48, 0x5f, 0x94, 0xb9, 0x6a,
0x9d, 0x22, 0x24, 0x8a, 0xf9, 0x77, 0x6d, 0x85, 0x22, 0x21, 0xe3, 0x84, 0x51, 0x54, 0xe9, 0xc2,
0x6f, 0x8c, 0xc5, 0x39, 0x8f, 0x7a, 0x98, 0xe3, 0x02, 0x53, 0x84, 0x44, 0x51, 0x13, 0xd6, 0x45,
0x81, 0x29, 0x82, 0x76, 0x60, 0x25, 0xa7, 0x5f, 0x9b, 0x79, 0x13, 0x16, 0x58, 0xf8, 0xa2, 0xd3,
0x8e, 0x5d, 0xab, 0xe1, 0x34, 0x0b, 0x4c, 0x53, 0x58, 0x40, 0xa1, 0x3f, 0xec, 0x07, 0x92, 0x65,
0x23, 0x2b, 0x03, 0xe8, 0x1a, 0x14, 0xb1, 0x9a, 0xa4, 0x97, 0xd9, 0x5e, 0xf9, 0x49, 0xbf, 0xb2,
0xa0, 0x72, 0xc0, 0x47, 0x68, 0x48, 0x4c, 0xee, 0x43, 0xd9, 0xe4, 0x1a, 0x17, 0x55, 0x77, 0x6e,
0xc9, 0xb8, 0xa6, 0x0b, 0xb6, 0x0d, 0x77, 0x3f, 0x48, 0xa2, 0x31, 0x4b, 0x17, 0xaf, 0x7f, 0x04,
0xb5, 0x09, 0x96, 0xd4, 0x74, 0x21, 0xc6, 0x26, 0x9e, 0x17, 0x62, 0x2c, 0xbd, 0xbc, 0xe4, 0xfe,
0x50, 0x60, 0x94, 0x0a, 0x4c, 0x11, 0x1f, 0xda, 0x1f, 0x58, 0xf4, 0x04, 0x48, 0x2b, 0x12, 0x3c,
0x11, 0xa8, 0xe4, 0x40, 0xc4, 0x31, 0x3f, 0x13, 0xf3, 0x62, 0xed, 0xe4, 0x63, 0x9d, 0xc6, 0xd5,
0xce, 0xc5, 0x95, 0x6e, 0x01, 0x69, 0x0b, 0x5f, 0x24, 0x42, 0x9f, 0xf3, 0x37, 0xc8, 0xa5, 0x17,
0xc6, 0x86, 0xf9, 0x6b, 0xc9, 0x26, 0x14, 0x64, 0xd3, 0x40, 0x65, 0xd5, 0x9d, 0x9a, 0x8c, 0x50,
0xda, 0x49, 0x18, 0xb2, 0x30, 0x1f, 0x28, 0xae, 0xb7, 0x9b, 0xa0, 0xa9, 0x0e, 0xcb, 0x00, 0xfa,
0x8d, 0x65, 0xb4, 0xa1, 0xf9, 0x7f, 0xd3, 0xe3, 0x89, 0xea, 0x7a, 0x5b, 0xdb, 0xe0, 0xa0, 0x0d,
0x75, 0x69, 0x43, 0xbe, 0x07, 0xcd, 0x32, 0xa3, 0x30, 0x6d, 0xc6, 0x03, 0x13, 0x9f, 0xeb, 0x5a,
0x41, 0xbb, 0x70, 0x4b, 0x49, 0xd8, 0xbd, 0xe4, 0x9e, 0xcf, 0x4f, 0xfd, 0x7f, 0x94, 0xc2, 0x09,
0x87, 0x5c, 0x28, 0xe1, 0xde, 0x4e, 0x5b, 0x1f, 0x03, 0x43, 0xd2, 0x21, 0x64, 0x27, 0xea, 0x09,
0xef, 0x0b, 0x2d, 0x0d, 0xbf, 0xd3, 0x38, 0xd8, 0x6f, 0x8c, 0xc3, 0x2a, 0x14, 0xe5, 0xf9, 0x93,
0xfd, 0xdd, 0x91, 0x2a, 0x91, 0x98, 0x13, 0x9d, 0x77, 0x61, 0xe1, 0xa8, 0x7b, 0x2e, 0xfa, 0x9c,
0xbc, 0x05, 0x25, 0xb4, 0x5c, 0xc4, 0xfa, 0x50, 0x54, 0xd2, 0x94, 0x33, 0xc3, 0xa1, 0xdf, 0x5a,
0xda, 0xd9, 0x99, 0x66, 0x4e, 0xa8, 0xb2, 0xa7, 0x54, 0x91, 0xbb, 0x50, 0xd2, 0xf6, 0x62, 0xb7,
0x78, 0xad, 0xa6, 0x0c, 0x97, 0x6c, 0xc2, 0x02, 0x7a, 0x17, 0xbb, 0x85, 0xcc, 0x10, 0x44, 0x98,
0x66, 0xd0, 0x7d, 0x70, 0x9e, 0xb3, 0x8e, 0x6c, 0x14, 0x68, 0xbd, 0x31, 0x43, 0x53, 0xd2, 0xb8,
0x8f, 0xc3, 0x38, 0xd1, 0xb1, 0xc7, 0x6f, 0x89, 0x1d, 0x86, 0x91, 0xaa, 0xd3, 0x1a, 0xc3, 0x6f,
0xfa, 0xbd, 0x05, 0x85, 0x27, 0x61, 0x4f, 0x90, 0x25, 0xb0, 0x3b, 0x6d, 0x2d, 0xc4, 0xee, 0xb4,
0xc9, 0x1a, 0xca, 0xd7, 0xf1, 0x2e, 0x49, 0xfd, 0xcf, 0x59, 0x87, 0xa1, 0xce, 0xdb, 0x50, 0xe9,
0xc4, 0x87, 0x91, 0xd7, 0xe7, 0xd1, 0x58, 0xdf, 0xa4, 0x19, 0x80, 0x67, 0x34, 0xe1, 0x89, 0xba,
0xdf, 0x2a, 0x4c, 0x11, 0x64, 0x13, 0x4a, 0x8f, 0xd8, 0x61, 0x4b, 0x8a, 0x2c, 0x4e, 0x8a, 0x34,
0x38, 0x7d, 0x00, 0x75, 0x69, 0x09, 0xae, 0x37, 0x95, 0x75, 0x13, 0x16, 0x24, 0x96, 0x5a, 0xa6,
0xa9, 0x4c, 0x89, 0x9d, 0x53, 0x42, 0x1f, 0x2a, 0x09, 0xfb, 0x97, 0x22, 0x48, 0x72, 0xb5, 0x89,
0x34, 0x0a, 0xa8, 0x31, 0x45, 0x90, 0xdb, 0xca, 0x6b, 0xed, 0x5e, 0x59, 0xda, 0x22, 0x69, 0x86,
0x28, 0x1d, 0x03, 0x18, 0x4b, 0x86, 0x71, 0xba, 0xd6, 0x9a, 0xb5, 0x96, 0x50, 0x53, 0x3e, 0xfa,
0x88, 0x82, 0xe4, 0x2b, 0x84, 0x99, 0xc2, 0x7a, 0x27, 0x2b, 0x2c, 0x95, 0xcf, 0xe5, 0x34, 0xef,
0x4a, 0x47, 0x56, 0x5e, 0xe7, 0x50, 0xcd, 0xe1, 0x33, 0x6b, 0xec, 0x6e, 0x5a, 0x1c, 0x76, 0x26,
0x0c, 0x11, 0x2d, 0x4c, 0xb3, 0xe7, 0x34, 0x27, 0x0f, 0xaa, 0xb9, 0x4d, 0x33, 0x35, 0x35, 0x61,
0x79, 0xf2, 0xc0, 0x9b, 0x3b, 0x67, 0x1a, 0x9e, 0xa3, 0xea, 0x3b, 0x0b, 0x6a, 0x2d, 0x7f, 0x18,
0x27, 0x22, 0x4a, 0x63, 0x5a, 0xd1, 0x40, 0x9a, 0xda, 0x0c, 0x98, 0x9d, 0x5d, 0xb2, 0x01, 0x45,
0x19, 0x71, 0x75, 0xb8, 0xf3, 0x89, 0x50, 0x70, 0x2e, 0x13, 0x85, 0xab, 0x32, 0x41, 0x4f, 0xa0,
0xbc, 0x77, 0xd4, 0x79, 0x14, 0x85, 0xc3, 0xc1, 0x4c, 0x8f, 0xcd, 0x48, 0x67, 0xe7, 0x46, 0xba,
0xba, 0x1a, 0x4f, 0x94, 0x57, 0x38, 0x91, 0xd4, 0xd5, 0x44, 0x52, 0xd0, 0x08, 0x1f, 0xd1, 0x23,
0x58, 0x51, 0xee, 0xca, 0x8e, 0x73, 0x9d, 0xb6, 0x68, 0xa6, 0x08, 0x27, 0x9b, 0x22, 0xa4, 0x50,
0xd5, 0x75, 0xff, 0x4d, 0xa1, 0xbf, 0xd9, 0xb0, 0xc2, 0x44, 0xec, 0xbd, 0x14, 0x9d, 0x20, 0x4e,
0xa2, 0x61, 0x57, 0x76, 0x1c, 0xb9, 0xff, 0x93, 0xf0, 0x54, 0xe7, 0xc2, 0x61, 0x8a, 0x78, 0xf3,
0x29, 0x21, 0x14, 0x4a, 0xf9, 0x26, 0x90, 0x5f, 0x60, 0x18, 0x64, 0x0b, 0x4a, 0x47, 0xe1, 0x30,
0xea, 0xa6, 0x95, 0x8f, 0x9d, 0x5b, 0xe9, 0x57, 0x0c, 0x66, 0x16, 0x90, 0xc7, 0x40, 0x8e, 0x23,
0x1e, 0xc4, 0x3e, 0x97, 0x26, 0x99, 0x6d, 0xe5, 0x6c, 0x3c, 0xc9, 0x71, 0x27, 0x24, 0xcc, 0xd8,
0x46, 0xb6, 0xf3, 0x47, 0xd8, 0x2d, 0xa1, 0x7d, 0x4b, 0xc6, 0x3e, 0x7d, 0x4e, 0xf2, 0x87, 0xfc,
0xfe, 0x54, 0x85, 0xba, 0x0b, 0xb8, 0x65, 0x45, 0x6e, 0x99, 0x60, 0xb0, 0xc9, 0x75, 0xf4, 0x6b,
0x0b, 0x16, 0xf3, 0xd6, 0xcc, 0x69, 0x17, 0x69, 0xfa, 0xec, 0xf9, 0xd3, 0x8e, 0x49, 0x5f, 0x61,
0xd6, 0x64, 0x59, 0xcc, 0x4f, 0x40, 0x21, 0xfc, 0xef, 0x8a, 0xe0, 0x5c, 0xcb, 0x9c, 0x06, 0x54,
0x0f, 0x79, 0x94, 0x78, 0x52, 0x98, 0xbe, 0xa7, 0x8b, 0x2c, 0x0f, 0x51, 0x01, 0x6b, 0xaf, 0x15,
0x51, 0x2b, 0xec, 0x0f, 0x64, 0xb5, 0x5e, 0xab, 0x98, 0x64, 0x9b, 0x8e, 0xa2, 0x30, 0x32, 0x11,
0x40, 0x82, 0xee, 0x41, 0xf9, 0x38, 0x1c, 0x84, 0x7e, 0x78, 0x36, 0x9e, 0xd3, 0x32, 0x5c, 0x28,
0xa9, 0xab, 0x41, 0xb5, 0xa8, 0x0a, 0x33, 0x24, 0xbd, 0x21, 0xeb, 0xbd, 0xcb, 0xfd, 0xee, 0xd0,
0xe7, 0x89, 0xc0, 0xf9, 0x18, 0xc1, 0x4f, 0x43, 0xde, 0x53, 0x5d, 0x41, 0x1f, 0x2d, 0xfa, 0xb9,
0x2e, 0x40, 0x8e, 0xee, 0xe4, 0xae, 0xa0, 0x5d, 0x04, 0xcc, 0x15, 0xa4, 0x28, 0xf2, 0x3e, 0x54,
0x73, 0xab, 0xb5, 0x5b, 0xcb, 0x69, 0x9d, 0x2a, 0x98, 0xe5, 0xd7, 0xd0, 0x5f, 0xac, 0x89, 0x3d,
0xaf, 0xdd, 0xb9, 0x5a, 0xd5, 0xa5, 0x0a, 0x52, 0x99, 0x69, 0x4a, 0xba, 0xbe, 0x3f, 0xea, 0xfa,
0xc3, 0x58, 0xb2, 0xf4, 0x85, 0x9b, 0x02, 0xd2, 0x75, 0xf9, 0xe0, 0x09, 0x87, 0x66, 0xb8, 0x31,
0xa4, 0x7c, 0x1a, 0xb5, 0x05, 0xef, 0xf9, 0x5e, 0x20, 0xb0, 0x5e, 0x1c, 0x96, 0xd2, 0x64, 0x4b,
0xf5, 0x58, 0x53, 0xe8, 0xab, 0x53, 0x86, 0x23, 0x4f, 0x75, 0xde, 0x98, 0x12, 0xa8, 0x4f, 0xb3,
0xe8, 0x2a, 0x10, 0x55, 0x01, 0xbb, 0xa7, 0x61, 0x64, 0x6e, 0x5b, 0xda, 0x32, 0xcd, 0x45, 0x46,
0x7f, 0xde, 0x25, 0x9e, 0x45, 0xd6, 0xce, 0x47, 0x96, 0x7e, 0x06, 0x4b, 0x7a, 0xb6, 0x13, 0x11,
0x16, 0xb4, 0x0c, 0x00, 0x13, 0xdd, 0x50, 0x8e, 0x89, 0xe6, 0x55, 0x93, 0x01, 0x52, 0xce, 0x89,
0x7c, 0x64, 0x98, 0xdb, 0x49, 0x53, 0x38, 0x1b, 0x79, 0x67, 0x81, 0xe8, 0xe1, 0x8d, 0xe1, 0x30,
0x4d, 0xd1, 0x1f, 0x6c, 0x58, 0x55, 0x43, 0x67, 0x70, 0x26, 0xe2, 0x24, 0x53, 0x23, 0x9f, 0xd1,
0x03, 0xec, 0xff, 0xda, 0x50, 0x45, 0xc9, 0x27, 0x73, 0xcb, 0x17, 0x3c, 0xca, 0x6c, 0x50, 0x8a,
0xa6, 0x50, 0x79, 0x6e, 0x10, 0xd1, 0xd7, 0xb3, 0x1a, 0x42, 0xf3, 0x10, 0xd9, 0x83, 0xb2, 0x76,
0xcd, 0x34, 0xc4, 0x3b, 0x78, 0x4b, 0xcd, 0xb0, 0xc6, 0xcc, 0xb7, 0xb1, 0x7e, 0x83, 0x19, 0x72,
0xfd, 0x29, 0xd4, 0x26, 0x58, 0x33, 0xde, 0x60, 0xcd, 0xfc, 0x1b, 0xac, 0xba, 0x43, 0x72, 0xe3,
0xb2, 0x96, 0x9e, 0x7f, 0x97, 0xb5, 0xe0, 0xbf, 0xb3, 0x0c, 0x88, 0xc9, 0x16, 0x38, 0xd2, 0x50,
0x35, 0x0c, 0xbb, 0x57, 0x19, 0xca, 0xe4, 0x22, 0xfa, 0x93, 0xa5, 0x83, 0x2a, 0x34, 0xdf, 0xbc,
0xa5, 0xef, 0xe5, 0x85, 0x6c, 0xa6, 0x42, 0xa6, 0x96, 0x6d, 0xa7, 0x8e, 0xca, 0xd5, 0xeb, 0xcf,
0xa0, 0x3c, 0xcb, 0xbd, 0x82, 0x72, 0xef, 0xbd, 0x49, 0xf7, 0xd6, 0xae, 0xb2, 0x2c, 0xce, 0x79,
0xb9, 0x57, 0xff, 0xf5, 0xd5, 0x86, 0xf5, 0xfb, 0xab, 0x0d, 0xeb, 0x8f, 0x57, 0x1b, 0xd6, 0x8f,
0x7f, 0x6e, 0xfc, 0xe7, 0x74, 0x01, 0x7f, 0x10, 0xdd, 0xfb, 0x2b, 0x00, 0x00, 0xff, 0xff, 0x38,
0x55, 0x86, 0x89, 0x43, 0x12, 0x00, 0x00,
// 1650 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0xcf, 0x72, 0x1b, 0x45,
0x13, 0xff, 0x76, 0x57, 0xb2, 0xa4, 0x96, 0x65, 0xcb, 0x13, 0x7f, 0xf9, 0xd6, 0x4e, 0x3e, 0x97,
0x3c, 0x50, 0x89, 0x70, 0x15, 0xa6, 0x70, 0x0e, 0xa1, 0xe0, 0x12, 0x5b, 0x72, 0x82, 0x08, 0x4e,
0x9c, 0xb1, 0xe3, 0x23, 0xd4, 0x58, 0x9a, 0xb2, 0xb7, 0xbc, 0xda, 0x15, 0xbb, 0x2b, 0x47, 0xca,
0x81, 0x2a, 0x28, 0x28, 0xb8, 0x70, 0xe7, 0xc4, 0x5b, 0xf0, 0x02, 0x9c, 0xb8, 0x50, 0xc5, 0x23,
0x50, 0xe1, 0x45, 0xa8, 0xe9, 0x99, 0xd9, 0x5d, 0x29, 0x72, 0x04, 0x2e, 0x6e, 0xdb, 0xbf, 0xee,
0xe9, 0x7f, 0xd3, 0xd3, 0xd3, 0xb3, 0x50, 0x1b, 0x44, 0xde, 0x25, 0x4f, 0xc4, 0xf6, 0x20, 0x0a,
0x93, 0x90, 0xd8, 0x83, 0xd3, 0xf5, 0xc5, 0xc1, 0xf0, 0xd4, 0xf7, 0xba, 0x0a, 0xa1, 0x8f, 0xa0,
0xd2, 0x09, 0x7a, 0x62, 0x74, 0x20, 0x12, 0x4e, 0x08, 0x14, 0x1e, 0x8b, 0x71, 0xec, 0x3a, 0x0d,
0xab, 0x59, 0x66, 0xf8, 0x4d, 0xee, 0xc0, 0xd2, 0x71, 0xc4, 0xbb, 0x17, 0xfb, 0x23, 0x2f, 0x4e,
0x44, 0xd0, 0x15, 0x6e, 0x01, 0xb9, 0x53, 0x28, 0xfd, 0xc5, 0x81, 0xc5, 0x87, 0x9e, 0xf0, 0x7b,
0x4f, 0x07, 0x89, 0x17, 0x06, 0xb1, 0x54, 0x76, 0x3c, 0x1e, 0x08, 0xb7, 0xdc, 0xb0, 0x9a, 0x15,
0x86, 0xdf, 0xe4, 0x36, 0x54, 0x5a, 0xbc, 0x7b, 0x2e, 0x90, 0xe1, 0x20, 0x23, 0x03, 0x52, 0xee,
0x91, 0xf7, 0x52, 0x59, 0xa9, 0xb1, 0x0c, 0x20, 0x0d, 0xa8, 0x1e, 0x7b, 0x7d, 0xf1, 0x6c, 0xc8,
0x83, 0x64, 0xd8, 0x77, 0x8b, 0xb8, 0x3a, 0x0f, 0x91, 0x9b, 0xb0, 0xf0, 0xd4, 0xef, 0x1d, 0x78,
0x81, 0x5b, 0x69, 0x58, 0x4d, 0x87, 0x69, 0xca, 0xe0, 0x7c, 0xe4, 0x42, 0x86, 0xf3, 0x51, 0x1a,
0x6e, 0x75, 0x32, 0xdc, 0x27, 0xe1, 0x51, 0xc2, 0x83, 0x1e, 0x8f, 0x7a, 0x27, 0x9e, 0x78, 0xe1,
0x2e, 0xaa, 0x70, 0x27, 0x51, 0xb9, 0x76, 0x8f, 0xc7, 0xc2, 0xad, 0xa1, 0x46, 0xfc, 0x26, 0xeb,
0x50, 0xde, 0xf3, 0x92, 0xb6, 0x18, 0x24, 0xe7, 0xee, 0x52, 0xc3, 0x6a, 0x16, 0x58, 0x4a, 0x93,
0x55, 0x28, 0x1e, 0x75, 0xb9, 0x2f, 0xdc, 0x65, 0x5c, 0xa0, 0x08, 0x42, 0x61, 0xf1, 0x61, 0x18,
0x09, 0xef, 0x2c, 0xc0, 0x4d, 0x70, 0xeb, 0x18, 0xd4, 0x04, 0x46, 0xfe, 0x0f, 0x8e, 0x0c, 0x69,
0xa5, 0x61, 0x35, 0xab, 0x3b, 0xd5, 0xed, 0xc1, 0xe9, 0x76, 0x5b, 0x74, 0xbd, 0x3e, 0xf7, 0x99,
0xc4, 0x91, 0xcd, 0x47, 0x2e, 0x99, 0xc5, 0xe6, 0x23, 0xe9, 0x93, 0x4c, 0xd1, 0xf3, 0xc0, 0x4b,
0xdc, 0x1b, 0xa8, 0x3d, 0xa5, 0x49, 0x1d, 0x9c, 0xe3, 0xc4, 0x77, 0x57, 0x11, 0x96, 0x9f, 0x94,
0xc2, 0x52, 0xa7, 0x3f, 0x08, 0xa3, 0x84, 0x89, 0x78, 0x10, 0x06, 0xb1, 0x90, 0x32, 0xfb, 0x51,
0xe4, 0x5a, 0x4a, 0x66, 0x3f, 0x8a, 0xe8, 0x97, 0x50, 0xdf, 0xf3, 0xc3, 0xee, 0x45, 0x9b, 0x27,
0x9c, 0x89, 0x2f, 0x86, 0x22, 0x4e, 0x64, 0x74, 0x2a, 0x00, 0x25, 0xa7, 0x08, 0x89, 0x62, 0x45,
0xb8, 0xb6, 0x42, 0x91, 0x90, 0x99, 0xc3, 0xbc, 0xaa, 0x0d, 0xc4, 0x6f, 0xcc, 0xce, 0x39, 0x8f,
0x7a, 0xb8, 0xeb, 0x05, 0xa6, 0x08, 0x89, 0xa2, 0x25, 0xac, 0x94, 0x02, 0x53, 0x04, 0xed, 0xc0,
0x4a, 0xce, 0xbe, 0x76, 0xf3, 0x26, 0x2c, 0xb0, 0xf0, 0x45, 0xa7, 0x1d, 0xbb, 0x56, 0xc3, 0x69,
0x16, 0x98, 0xa6, 0xb0, 0xa4, 0x42, 0x7f, 0xd8, 0x0f, 0x24, 0xcb, 0x46, 0x56, 0x06, 0xd0, 0x35,
0x28, 0x62, 0x7d, 0xc9, 0x28, 0xb3, 0xb5, 0xf2, 0x93, 0x7e, 0x65, 0x41, 0xe5, 0x80, 0x8f, 0xd0,
0x91, 0x98, 0xdc, 0x87, 0xb2, 0xd9, 0x7d, 0x14, 0xaa, 0xee, 0xdc, 0x92, 0x99, 0x4e, 0x05, 0xb6,
0x0d, 0x77, 0x3f, 0x48, 0xa2, 0x31, 0x4b, 0x85, 0xd7, 0x3f, 0x82, 0xda, 0x04, 0x4b, 0x5a, 0xba,
0x10, 0x63, 0x93, 0xcf, 0x0b, 0x31, 0x96, 0x51, 0x5e, 0x72, 0x7f, 0x28, 0x30, 0x4b, 0x05, 0xa6,
0x88, 0x0f, 0xed, 0x0f, 0x2c, 0x7a, 0x02, 0xa4, 0x15, 0x09, 0x9e, 0x08, 0x34, 0x72, 0x20, 0xe2,
0x98, 0x9f, 0x89, 0x79, 0xb9, 0x76, 0xf2, 0xb9, 0x4e, 0xf3, 0x6a, 0xe7, 0xf2, 0x4a, 0xb7, 0x80,
0xb4, 0x85, 0x2f, 0x12, 0xa1, 0x4f, 0xfe, 0x1b, 0xf4, 0xd2, 0x0b, 0xe3, 0xc3, 0x7c, 0x59, 0xb2,
0x09, 0x05, 0xd9, 0x46, 0xd0, 0x58, 0x75, 0xa7, 0x26, 0x33, 0x94, 0xf6, 0x16, 0x86, 0x2c, 0xdc,
0x0f, 0x54, 0xd7, 0xdb, 0x4d, 0xd0, 0x55, 0x87, 0x65, 0x00, 0xfd, 0xc6, 0x32, 0xd6, 0xd0, 0xfd,
0xbf, 0x19, 0xf1, 0x44, 0x75, 0xbd, 0xad, 0x7d, 0x70, 0xd0, 0x87, 0xba, 0xf4, 0x21, 0xdf, 0x95,
0x66, 0xb9, 0x51, 0x98, 0x76, 0xe3, 0x81, 0xc9, 0xcf, 0x75, 0xbd, 0xa0, 0x5d, 0xb8, 0xa5, 0x34,
0xec, 0x5e, 0x72, 0xcf, 0xe7, 0xa7, 0xfe, 0x3f, 0xda, 0xc2, 0x89, 0x80, 0x5c, 0x28, 0xe1, 0xda,
0x4e, 0x5b, 0x1f, 0x03, 0x43, 0xd2, 0x21, 0x64, 0x27, 0xea, 0x09, 0xef, 0x0b, 0xad, 0x0d, 0xbf,
0xd3, 0x3c, 0xd8, 0x6f, 0xcc, 0xc3, 0x2a, 0x14, 0xe5, 0xf9, 0x93, 0x1d, 0xdf, 0x91, 0x26, 0x91,
0x98, 0x93, 0x9d, 0x77, 0x61, 0xe1, 0xa8, 0x7b, 0x2e, 0xfa, 0x9c, 0xbc, 0x05, 0x25, 0xf4, 0x5c,
0xc4, 0xfa, 0x50, 0x54, 0xd2, 0x2d, 0x67, 0x86, 0x43, 0xbf, 0xb5, 0x74, 0xb0, 0x33, 0xdd, 0x9c,
0x30, 0x65, 0x4f, 0x99, 0x22, 0x77, 0xa1, 0xa4, 0xfd, 0xc5, 0x6e, 0xf1, 0x5a, 0x4d, 0x19, 0x2e,
0xd9, 0x84, 0x05, 0x8c, 0x2e, 0x76, 0x0b, 0x99, 0x23, 0x88, 0x30, 0xcd, 0xa0, 0xfb, 0xe0, 0x3c,
0x67, 0x1d, 0xd9, 0x28, 0xd0, 0x7b, 0xe3, 0x86, 0xa6, 0xa4, 0x73, 0x1f, 0x87, 0x71, 0xa2, 0x73,
0x8f, 0xdf, 0x12, 0x3b, 0x0c, 0x23, 0x55, 0xa7, 0x35, 0x86, 0xdf, 0xf4, 0x7b, 0x0b, 0x0a, 0x4f,
0xc2, 0x9e, 0x20, 0x4b, 0x60, 0x77, 0xda, 0x5a, 0x89, 0xdd, 0x69, 0x93, 0x35, 0xd4, 0xaf, 0xf3,
0x5d, 0x92, 0xf6, 0x9f, 0xb3, 0x0e, 0x43, 0x9b, 0xb7, 0xa1, 0xd2, 0x89, 0x0f, 0x23, 0xaf, 0xcf,
0xa3, 0xb1, 0xbe, 0x5b, 0x33, 0x00, 0xcf, 0x68, 0xc2, 0x13, 0x75, 0xe3, 0x55, 0x98, 0x22, 0xc8,
0x26, 0x94, 0x1e, 0xb1, 0xc3, 0x96, 0x54, 0x59, 0x9c, 0x54, 0x69, 0x70, 0xfa, 0x00, 0xea, 0xd2,
0x13, 0x94, 0x37, 0x95, 0x75, 0x13, 0x16, 0x24, 0x96, 0x7a, 0xa6, 0xa9, 0xcc, 0x88, 0x9d, 0x33,
0x42, 0x1f, 0x2a, 0x0d, 0xfb, 0x97, 0x22, 0x48, 0x72, 0xb5, 0x89, 0x34, 0x2a, 0xa8, 0x31, 0x45,
0x90, 0xdb, 0x2a, 0x6a, 0x1d, 0x5e, 0x59, 0xfa, 0x22, 0x69, 0x86, 0x28, 0x1d, 0x03, 0x18, 0x4f,
0x86, 0x71, 0x2a, 0x6b, 0xcd, 0x92, 0x25, 0xd4, 0x94, 0x8f, 0x3e, 0xa2, 0x20, 0xf9, 0x0a, 0x61,
0xa6, 0xb0, 0xde, 0xc9, 0x0a, 0x4b, 0xed, 0xe7, 0x72, 0xba, 0xef, 0xca, 0x46, 0x56, 0x5e, 0xe7,
0x50, 0xcd, 0xe1, 0x33, 0x6b, 0xec, 0x6e, 0x5a, 0x1c, 0x76, 0xa6, 0x0c, 0x11, 0xad, 0x4c, 0xb3,
0xe7, 0x34, 0x27, 0x0f, 0xaa, 0xb9, 0x45, 0x33, 0x2d, 0x35, 0x61, 0x79, 0xf2, 0xc0, 0x9b, 0x3b,
0x67, 0x1a, 0x9e, 0x63, 0xea, 0x3b, 0x0b, 0x6a, 0x2d, 0x7f, 0x18, 0x27, 0x22, 0x4a, 0x73, 0x5a,
0xd1, 0x40, 0xba, 0xb5, 0x19, 0x30, 0x7b, 0x77, 0xc9, 0x06, 0x14, 0x65, 0xc6, 0xd5, 0xe1, 0xce,
0x6f, 0x84, 0x82, 0x73, 0x3b, 0x51, 0xb8, 0x6a, 0x27, 0xe8, 0x09, 0x94, 0xf7, 0x8e, 0x3a, 0x8f,
0xa2, 0x70, 0x38, 0x98, 0x19, 0xb1, 0x19, 0xf2, 0xec, 0xdc, 0x90, 0x57, 0x57, 0x03, 0x8b, 0x8a,
0x0a, 0x67, 0x94, 0xba, 0x9a, 0x51, 0x0a, 0x1a, 0xe1, 0x23, 0x7a, 0x04, 0x2b, 0x2a, 0x5c, 0xd9,
0x71, 0xae, 0xd3, 0x16, 0xcd, 0x14, 0xe1, 0x64, 0x53, 0x84, 0x54, 0xaa, 0xba, 0xee, 0xbf, 0xa9,
0xf4, 0x37, 0x1b, 0x56, 0x98, 0x88, 0xbd, 0x97, 0xa2, 0x13, 0xc4, 0x49, 0x34, 0xec, 0xca, 0x8e,
0x23, 0xd7, 0x7f, 0x12, 0x9e, 0xea, 0xbd, 0x70, 0x98, 0x22, 0xde, 0x7c, 0x4a, 0x08, 0x85, 0x52,
0xbe, 0x09, 0xe4, 0x05, 0x0c, 0x83, 0x6c, 0x41, 0xe9, 0x28, 0x1c, 0x46, 0xdd, 0xb4, 0xf2, 0xb1,
0x73, 0x2b, 0xfb, 0x8a, 0xc1, 0x8c, 0x00, 0x79, 0x0c, 0xe4, 0x38, 0xe2, 0x41, 0xec, 0x73, 0xe9,
0x92, 0x59, 0x56, 0xce, 0xc6, 0x93, 0x1c, 0x77, 0x42, 0xc3, 0x8c, 0x65, 0x64, 0x3b, 0x7f, 0x84,
0xdd, 0x12, 0xfa, 0xb7, 0x64, 0xfc, 0xd3, 0xe7, 0x24, 0x7f, 0xc8, 0xef, 0x4f, 0x55, 0xa8, 0xbb,
0x80, 0x4b, 0x56, 0xe4, 0x92, 0x09, 0x06, 0x9b, 0x94, 0xa3, 0x5f, 0x5b, 0xb0, 0x98, 0xf7, 0x66,
0x4e, 0xbb, 0x48, 0xb7, 0xcf, 0x9e, 0x3f, 0xed, 0x98, 0xed, 0x2b, 0xcc, 0x9a, 0x2c, 0x8b, 0xf9,
0x09, 0x28, 0x84, 0xff, 0x5d, 0x91, 0x9c, 0x6b, 0xb9, 0xd3, 0x80, 0xea, 0x21, 0x8f, 0x12, 0x4f,
0x2a, 0xd3, 0xf7, 0x74, 0x91, 0xe5, 0x21, 0x2a, 0x60, 0xed, 0xb5, 0x22, 0x6a, 0x85, 0xfd, 0x81,
0xac, 0xd6, 0x6b, 0x15, 0x93, 0x6c, 0xd3, 0x51, 0x14, 0x46, 0x26, 0x03, 0x48, 0xd0, 0x3d, 0x28,
0x1f, 0x87, 0x83, 0xd0, 0x0f, 0xcf, 0xc6, 0x73, 0x5a, 0x86, 0x0b, 0x25, 0x75, 0x35, 0xa8, 0x16,
0x55, 0x61, 0x86, 0xa4, 0x37, 0x64, 0xbd, 0x77, 0xb9, 0xdf, 0x1d, 0xfa, 0x3c, 0x11, 0x38, 0x1f,
0x23, 0xf8, 0x69, 0xc8, 0x7b, 0xaa, 0x2b, 0xe8, 0xa3, 0x45, 0x3f, 0xd7, 0x05, 0xc8, 0x31, 0x9c,
0xdc, 0x15, 0xb4, 0x8b, 0x80, 0xb9, 0x82, 0x14, 0x45, 0xde, 0x87, 0x6a, 0x4e, 0x5a, 0x87, 0xb5,
0x9c, 0xd6, 0xa9, 0x82, 0x59, 0x5e, 0x86, 0xfe, 0x6c, 0x4d, 0xac, 0x79, 0xed, 0xce, 0xd5, 0xa6,
0x2e, 0x55, 0x92, 0xca, 0x4c, 0x53, 0x32, 0xf4, 0xfd, 0x51, 0xd7, 0x1f, 0xc6, 0x92, 0xa5, 0x2f,
0xdc, 0x14, 0x90, 0xa1, 0xcb, 0x27, 0x50, 0x38, 0x34, 0xc3, 0x8d, 0x21, 0xe5, 0x63, 0xa9, 0x2d,
0x78, 0xcf, 0xf7, 0x02, 0x81, 0xf5, 0xe2, 0xb0, 0x94, 0x26, 0x5b, 0xaa, 0xc7, 0x9a, 0x42, 0x5f,
0x9d, 0x72, 0x1c, 0x79, 0xaa, 0xf3, 0xc6, 0x94, 0x40, 0x7d, 0x9a, 0x45, 0x57, 0x81, 0xa8, 0x0a,
0xd8, 0x3d, 0x0d, 0x23, 0x73, 0xdb, 0xd2, 0x96, 0x69, 0x2e, 0x32, 0xfb, 0xf3, 0x2e, 0xf1, 0x2c,
0xb3, 0x76, 0x3e, 0xb3, 0xf4, 0x33, 0x58, 0xd2, 0xb3, 0x9d, 0x88, 0xb0, 0xa0, 0x65, 0x02, 0x98,
0xe8, 0x86, 0x72, 0x4c, 0x34, 0xaf, 0x9a, 0x0c, 0x90, 0x7a, 0x4e, 0xe4, 0x23, 0xc3, 0xdc, 0x4e,
0x9a, 0xc2, 0xd9, 0xc8, 0x3b, 0x0b, 0x44, 0x0f, 0x6f, 0x0c, 0x87, 0x69, 0x8a, 0xfe, 0x60, 0xc3,
0xaa, 0x1a, 0x3a, 0x83, 0x33, 0x11, 0x27, 0x99, 0x19, 0xf9, 0xb0, 0x1e, 0x60, 0xff, 0xd7, 0x8e,
0x2a, 0x4a, 0x3e, 0xa2, 0x5b, 0xbe, 0xe0, 0x51, 0xe6, 0x83, 0x32, 0x34, 0x85, 0xca, 0x73, 0x83,
0x88, 0xbe, 0x9e, 0xd5, 0x10, 0x9a, 0x87, 0xc8, 0x1e, 0x94, 0x75, 0x68, 0xa6, 0x21, 0xde, 0xc1,
0x5b, 0x6a, 0x86, 0x37, 0x66, 0xbe, 0x8d, 0xf5, 0x1b, 0xcc, 0x90, 0xeb, 0x4f, 0xa1, 0x36, 0xc1,
0x9a, 0xf1, 0x06, 0x6b, 0xe6, 0xdf, 0x60, 0xd5, 0x1d, 0x92, 0x1b, 0x97, 0xb5, 0xf6, 0xfc, 0xbb,
0xac, 0x05, 0xff, 0x9d, 0xe5, 0x40, 0x4c, 0xb6, 0xc0, 0x91, 0x8e, 0xaa, 0x61, 0xd8, 0xbd, 0xca,
0x51, 0x26, 0x85, 0xe8, 0x4f, 0x96, 0x4e, 0xaa, 0xd0, 0x7c, 0xf3, 0x96, 0xbe, 0x97, 0x57, 0xb2,
0x99, 0x2a, 0x99, 0x12, 0xdb, 0x4e, 0x03, 0x95, 0xd2, 0xeb, 0xcf, 0xa0, 0x3c, 0x2b, 0xbc, 0x82,
0x0a, 0xef, 0xbd, 0xc9, 0xf0, 0xd6, 0xae, 0xf2, 0x2c, 0xce, 0x45, 0xb9, 0x57, 0xff, 0xf5, 0xd5,
0x86, 0xf5, 0xfb, 0xab, 0x0d, 0xeb, 0x8f, 0x57, 0x1b, 0xd6, 0x8f, 0x7f, 0x6e, 0xfc, 0xe7, 0x74,
0x01, 0x7f, 0x19, 0xdd, 0xfb, 0x2b, 0x00, 0x00, 0xff, 0xff, 0xcf, 0xdb, 0x63, 0xcd, 0x55, 0x12,
0x00, 0x00,
}
func (m *IndexMeta) Marshal() (dAtA []byte, err error) {
@ -2936,6 +2945,15 @@ func (m *FieldOptions) MarshalToSizedBuffer(dAtA []byte) (int, error) {
i -= len(m.XXX_unrecognized)
copy(dAtA[i:], m.XXX_unrecognized)
}
if len(m.Ttl) > 0 {
i -= len(m.Ttl)
copy(dAtA[i:], m.Ttl)
i = encodeVarintPrivate(dAtA, i, uint64(len(m.Ttl)))
i--
dAtA[i] = 0x1
i--
dAtA[i] = 0xa2
}
if len(m.TimeUnit) > 0 {
i -= len(m.TimeUnit)
copy(dAtA[i:], m.TimeUnit)
@ -5274,6 +5292,10 @@ func (m *FieldOptions) Size() (n int) {
if l > 0 {
n += 2 + l + sovPrivate(uint64(l))
}
l = len(m.Ttl)
if l > 0 {
n += 2 + l + sovPrivate(uint64(l))
}
if m.XXX_unrecognized != nil {
n += len(m.XXX_unrecognized)
}
@ -6765,6 +6787,38 @@ func (m *FieldOptions) Unmarshal(dAtA []byte) error {
}
m.TimeUnit = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
case 20:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Ttl", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
stringLen |= uint64(b&0x7F) << shift
if b < 0x80 {
break
}
}
intStringLen := int(stringLen)
if intStringLen < 0 {
return ErrInvalidLengthPrivate
}
postIndex := iNdEx + intStringLen
if postIndex < 0 {
return ErrInvalidLengthPrivate
}
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.Ttl = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipPrivate(dAtA[iNdEx:])

View file

@ -25,6 +25,7 @@ message FieldOptions {
Decimal Min = 17;
Decimal Max = 18;
string TimeUnit = 19;
string Ttl = 20;
}
message ImportResponse {

View file

@ -76,6 +76,7 @@ type Server struct { // nolint: maligned
antiEntropyInterval time.Duration
metricInterval time.Duration
diagnosticInterval time.Duration
ttlRemovalInterval time.Duration
maxWritesPerRequest int
confirmDownSleep time.Duration
confirmDownRetries int
@ -167,6 +168,15 @@ func OptServerAntiEntropyInterval(interval time.Duration) ServerOption {
}
}
// OptServerTtlRemovalInterval is a functional option on Server
// used to set the ttl removal interval.
func OptServerTtlRemovalInterval(interval time.Duration) ServerOption {
return func(s *Server) error {
s.ttlRemovalInterval = interval
return nil
}
}
// OptServerLongQueryTime is a functional option on Server
// used to set long query duration.
func OptServerLongQueryTime(dur time.Duration) ServerOption {
@ -433,6 +443,7 @@ func NewServer(opts ...ServerOption) (*Server, error) {
antiEntropyInterval: 0,
metricInterval: 0,
diagnosticInterval: 0,
ttlRemovalInterval: time.Hour,
disCo: disco.NopDisCo,
stator: disco.NopStator,
@ -641,12 +652,13 @@ func (s *Server) Open() error {
return errors.Wrap(err, "setting nodeState")
}
if ok := s.addToWaitGroup(3); !ok {
if ok := s.addToWaitGroup(4); !ok {
return fmt.Errorf("closing server while opening server is NOT allowed")
}
go func() { defer s.wg.Done(); s.monitorAntiEntropy() }()
go func() { defer s.wg.Done(); s.monitorRuntime() }()
go func() { defer s.wg.Done(); s.monitorDiagnostics() }()
go func() { defer s.wg.Done(); s.monitorTtl() }()
toSend := func() []Message {
s.holder.startMsgsMu.Lock()
@ -830,6 +842,53 @@ func (s *Server) monitorResetTranslationSync() {
}
}
func (s *Server) monitorTtl() {
ctx := context.Background()
ticker := time.NewTicker(s.ttlRemovalInterval)
for {
select {
case <-s.closing:
return
case <-ticker.C:
s.TtlRemoval(ctx)
}
}
}
func (s *Server) TtlRemoval(ctx context.Context) {
for _, index := range s.holder.Indexes() {
for _, field := range index.Fields() {
if field.Options().Type == "time" {
if field.Options().Ttl > 0 {
for _, view := range field.views() {
viewNames := strings.Split(view.name, "_")
if len(viewNames) >= 2 {
viewTime, err := timeOfView(view.name, false)
if err != nil {
s.logger.Printf("ttl parse view time: %s", err)
continue
}
timeSince := time.Since(viewTime)
if timeSince >= field.Options().Ttl {
for _, shard := range field.AvailableShards(true).Slice() {
s.holder.txf.DeleteFragmentFromStore(index.Name(), field.Name(), view.name, shard, nil)
}
err := s.defaultClient.api.DeleteView(ctx, index.Name(), field.Name(), view.name)
if err != nil {
s.logger.Errorf("ttl delete view: %s", err)
}
s.logger.Infof("ttl deleted view: %s", view.name)
}
}
}
}
}
}
}
}
func (s *Server) monitorAntiEntropy() {
if s.antiEntropyInterval == 0 || s.cluster.ReplicaN <= 1 {
return // anti entropy disabled

View file

@ -274,7 +274,7 @@ func TestHandler_Endpoints(t *testing.T) {
}
}
if f, err := i2.CreateFieldIfNotExists("f3", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"))); err != nil {
if f, err := i2.CreateFieldIfNotExists("f3", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"), "0")); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(tx2, 0, 0, nil); err != nil {
t.Fatal(err)

View file

@ -29,7 +29,7 @@ func TestPostgresHandler(t *testing.T) {
m.MustCreateField(t, "i", "keymutex", pilosa.OptFieldKeys(), pilosa.OptFieldTypeMutex(pilosa.CacheTypeNone, 0))
m.MustCreateField(t, "i", "int", pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64))
m.MustCreateField(t, "i", "decimal", pilosa.OptFieldTypeDecimal(2))
m.MustCreateField(t, "i", "time", pilosa.OptFieldTypeTime("YMDH"))
m.MustCreateField(t, "i", "time", pilosa.OptFieldTypeTime("YMDH", "0"))
m.MustCreateField(t, "i", "bool", pilosa.OptFieldTypeBool())
m.MustCreateIndex(t, "j", pilosa.IndexOptions{TrackExistence: true, Keys: true})

View file

@ -760,7 +760,7 @@ func TestMain_ImportTimestamp(t *testing.T) {
}
// Create field.
if _, err := m.API.CreateField(context.Background(), indexName, fieldName, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMD"))); err != nil {
if _, err := m.API.CreateField(context.Background(), indexName, fieldName, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMD"), "0")); err != nil {
t.Fatal(err)
}
@ -815,7 +815,7 @@ func TestMain_ImportTimestampNoStandardView(t *testing.T) {
}
// Create field.
if _, err := m.API.CreateField(context.Background(), indexName, fieldName, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMD"), true)); err != nil {
if _, err := m.API.CreateField(context.Background(), indexName, fieldName, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMD"), "0", true)); err != nil {
t.Fatal(err)
}

95
server_test.go Normal file
View file

@ -0,0 +1,95 @@
// Copyright 2022 Molecula Corp. All rights reserved.
package pilosa_test
import (
"context"
"fmt"
"reflect"
"sort"
"testing"
"time"
pilosa "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/test"
)
func TestTtlRemoval(t *testing.T) {
cluster := test.MustRunCluster(t, 1)
node := cluster.GetNode(0)
defer cluster.Close()
// Create a client
client := node.Client()
indexName := "i"
fieldName := "f"
// Create indexes and field with ttl lasting 24 hours
if err := client.CreateIndex(context.Background(), indexName, pilosa.IndexOptions{TrackExistence: true}); err != nil && err != pilosa.ErrIndexExists {
t.Fatalf("creating index, err: %v", err)
} else if err := client.CreateFieldWithOptions(context.Background(), indexName, fieldName, pilosa.FieldOptions{Ttl: time.Hour * 24, Type: pilosa.FieldTypeTime, TimeQuantum: "YMDH"}); err != nil {
t.Fatalf("creating field, err: %v", err)
}
/* Set sample data 1 using this date: '2001-02-03T04:05', this will create these views:
- standard
- standard_2001
- standard_200102
- standard_20010203
- standard_2001020304
Since the sample date here is over 24 hours, all views except "standard" should get deleted
*/
_, err := client.Query(context.Background(), indexName, &pilosa.QueryRequest{Index: indexName, Query: "Set(1, f=1, 2001-02-03T04:05)"})
if err != nil {
t.Fatalf("setting sample data 1, err: %v", err)
}
dateNow := time.Now()
dateNowString := fmt.Sprintf("%d-%02d-%02dT%02d:%02d", dateNow.Year(), dateNow.Month(), dateNow.Day(), dateNow.Hour(), dateNow.Minute())
/* Set sample data 2 using current time.
For example: current time is 2022-03-03T15:17 (also when the 24 hrs ttl countdown starts) will generate these views:
- standard_2022 -> gets converted to 2022_01_01, over 24 hours for ttl -> deleted
- standard_202203 -> gets converted to 2022_03_01, over 24 hours for ttl -> deleted
- standard_20220303 -> gets converted to 2022_03_03, within 24 hours -> keep
- standard_2022030315 -> gets converted to 2022_03_03 15:00, within 24 hours -> keep
*/
_, err = client.Query(context.Background(), indexName, &pilosa.QueryRequest{Index: indexName, Query: "Set(2, f=2, " + dateNowString + ")"})
if err != nil {
t.Fatalf("setting sample data 2, err: %v", err)
}
/* Set sample data 3 using yesterday's date
All views generated from this date should be deleted
*/
dateYesterday := time.Now().Add(-24*time.Hour + -1*time.Nanosecond)
dateYesterdayString := fmt.Sprintf("%d-%02d-%02dT%02d:%02d", dateYesterday.Year(), dateYesterday.Month(), dateYesterday.Day(), dateYesterday.Hour(), dateYesterday.Minute())
_, err = client.Query(context.Background(), indexName, &pilosa.QueryRequest{Index: indexName, Query: "Set(3, f=3, " + dateYesterdayString + ")"})
if err != nil {
t.Fatalf("setting sample data 3, err: %v", err)
}
node.Server.TtlRemoval(context.Background())
// Get all the views for given index + field
views, err := node.API.Views(context.Background(), indexName, fieldName)
if err != nil {
t.Fatal(err)
}
expectedViewNames := []string{
"standard",
"standard_" + fmt.Sprintf("%d%02d%02d", dateNow.Year(), dateNow.Month(), dateNow.Day()),
"standard_" + fmt.Sprintf("%d%02d%02d%02d", dateNow.Year(), dateNow.Month(), dateNow.Day(), dateNow.Hour()),
}
var viewNames []string
for _, view := range views {
viewNames = append(viewNames, view.Name())
}
sort.Strings(viewNames)
if !reflect.DeepEqual(expectedViewNames, viewNames) {
t.Fatalf("after ttl removal, expected %v, but got %v", expectedViewNames, viewNames)
}
}

View file

@ -450,7 +450,7 @@ func timeOfView(v string, adj bool) (time.Time, error) {
return time.Time{}, nil
}
layout := "2006010203"
layout := "2006010215"
timePart := viewTimePart(v)
switch len(timePart) {

View file

@ -307,6 +307,10 @@ func (v *view) recalculateCaches() {
}
}
func (v *view) Name() string {
return v.name
}
// CreateFragmentIfNotExists returns a fragment in the view by shard.
func (v *view) CreateFragmentIfNotExists(shard uint64) (*fragment, error) {
v.mu.Lock()