diff --git a/api_test.go b/api_test.go index 3b4836fab..0ebf16b5f 100644 --- a/api_test.go +++ b/api_test.go @@ -555,7 +555,7 @@ func TestAPI_Ingest(t *testing.T) { if err != nil { t.Fatalf("creating field: %v", err) } - _, err = coord.API.CreateField(ctx, index, timeField, pilosa.OptFieldTypeTime("YMD")) + _, err = coord.API.CreateField(ctx, index, timeField, pilosa.OptFieldTypeTime("YMD", "0")) if err != nil { t.Fatalf("creating field: %v", err) } @@ -678,7 +678,7 @@ func BenchmarkIngest(b *testing.B) { if err != nil { b.Fatalf("creating field: %v", err) } - _, err = coord.API.CreateField(ctx, index, tqField, pilosa.OptFieldTypeTime("YMDH")) + _, err = coord.API.CreateField(ctx, index, tqField, pilosa.OptFieldTypeTime("YMDH", "0")) if err != nil { b.Fatalf("creating field: %v", err) } diff --git a/apimethod_string.go b/apimethod_string.go index 11ed5a916..24703b05a 100644 --- a/apimethod_string.go +++ b/apimethod_string.go @@ -43,11 +43,15 @@ func _() { _ = x[apiIDReserve-32] _ = x[apiIDCommit-33] _ = x[apiIDReset-34] + _ = x[apiPartitionNodes-35] + _ = x[apiIngestOperations-36] + _ = x[apiIngestNodeOperations-37] + _ = x[apiMutexCheck-38] } -const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteAvailableShardapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFragmentDataapiTranslateDataapiFieldTranslateDataapiFieldapiImportapiImportValueapiIndexapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiSchemaapiShardNodesapiStateapiViewsapiApplySchemaapiStartTransactionapiFinishTransactionapiTransactionsapiGetTransactionapiActiveQueriesapiPastQueriesapiIDReserveapiIDCommitapiIDReset" +const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteAvailableShardapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFragmentDataapiTranslateDataapiFieldTranslateDataapiFieldapiImportapiImportValueapiIndexapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiSchemaapiShardNodesapiStateapiViewsapiApplySchemaapiStartTransactionapiFinishTransactionapiTransactionsapiGetTransactionapiActiveQueriesapiPastQueriesapiIDReserveapiIDCommitapiIDResetapiPartitionNodesapiIngestOperationsapiIngestNodeOperationsapiMutexCheck" -var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 82, 96, 109, 121, 141, 158, 173, 189, 210, 218, 227, 241, 249, 257, 277, 290, 304, 313, 326, 334, 342, 356, 375, 395, 410, 427, 443, 457, 469, 480, 490} +var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 82, 96, 109, 121, 141, 158, 173, 189, 210, 218, 227, 241, 249, 257, 277, 290, 304, 313, 326, 334, 342, 356, 375, 395, 410, 427, 443, 457, 469, 480, 490, 507, 526, 549, 562} func (i apiMethod) String() string { if i < 0 || i >= apiMethod(len(_apiMethod_index)-1) { diff --git a/client/client.go b/client/client.go index 19d532e50..a023f9c29 100644 --- a/client/client.go +++ b/client/client.go @@ -1674,17 +1674,18 @@ type SchemaField struct { // SchemaOptions contains options for a field or an index. type SchemaOptions struct { - FieldType FieldType `json:"type"` - CacheType string `json:"cacheType"` - CacheSize uint `json:"cacheSize"` - TimeQuantum string `json:"timeQuantum"` - Min pql.Decimal `json:"min"` - Max pql.Decimal `json:"max"` - Scale int64 `json:"scale"` - Keys bool `json:"keys"` - NoStandardView bool `json:"noStandardView"` - TrackExistence bool `json:"trackExistence"` - TimeUnit string `json:"timeUnit"` + FieldType FieldType `json:"type"` + CacheType string `json:"cacheType"` + CacheSize uint `json:"cacheSize"` + TimeQuantum string `json:"timeQuantum"` + Ttl time.Duration `json:"ttl"` + Min pql.Decimal `json:"min"` + Max pql.Decimal `json:"max"` + Scale int64 `json:"scale"` + Keys bool `json:"keys"` + NoStandardView bool `json:"noStandardView"` + TrackExistence bool `json:"trackExistence"` + TimeUnit string `json:"timeUnit"` } func (so SchemaOptions) asIndexOptions() *IndexOptions { @@ -1702,6 +1703,7 @@ func (so SchemaOptions) asFieldOptions() *FieldOptions { cacheSize: int(so.CacheSize), cacheType: CacheType(so.CacheType), timeQuantum: TimeQuantum(so.TimeQuantum), + ttl: so.Ttl, min: so.Min, max: so.Max, scale: so.Scale, diff --git a/client/orm.go b/client/orm.go index ddb77bbd4..5035cc59c 100644 --- a/client/orm.go +++ b/client/orm.go @@ -728,6 +728,7 @@ type FieldInfo struct { type FieldOptions struct { fieldType FieldType timeQuantum TimeQuantum + ttl time.Duration cacheType CacheType cacheSize int min pql.Decimal @@ -751,6 +752,11 @@ func (fo FieldOptions) TimeQuantum() TimeQuantum { return fo.timeQuantum } +// Ttl returns the configured ttl for a time field. +func (fo FieldOptions) Ttl() time.Duration { + return fo.ttl +} + // CacheType returns the configured cache type for a "set" field. Empty string // otherwise. func (fo FieldOptions) CacheType() CacheType { @@ -826,6 +832,7 @@ func (fo FieldOptions) String() string { case FieldTypeTime: mopt["timeQuantum"] = string(fo.timeQuantum) mopt["noStandardView"] = fo.noStandardView + mopt["ttl"] = fo.ttl.String() case FieldTypeTimestamp: mopt["min"] = fo.min mopt["max"] = fo.max @@ -913,6 +920,12 @@ func OptFieldTypeTime(quantum TimeQuantum, opts ...bool) FieldOption { } } +func OptFieldTtl(dur time.Duration) FieldOption { + return func(options *FieldOptions) { + options.ttl = dur + } +} + // Timestamp field range. var ( DefaultEpoch = time.Unix(0, 0).UTC() // 1970-01-01T00:00:00Z diff --git a/client/orm_test.go b/client/orm_test.go index 650d113c3..29256a23c 100644 --- a/client/orm_test.go +++ b/client/orm_test.go @@ -925,7 +925,8 @@ func TestORM(t *testing.T) { pql.NewDecimal(0, 0), pql.NewDecimal(0, 0), "", - "") + "", + 0) }) t.Run("IntFieldOptions", func(t *testing.T) { @@ -944,7 +945,8 @@ func TestORM(t *testing.T) { pql.NewDecimal(-10, 0), pql.NewDecimal(100, 0), "", - "") + "", + 0) field = sampleIndex.Field("int-field2", OptFieldTypeInt(-10)) jsonString = field.options.String() @@ -962,7 +964,8 @@ func TestORM(t *testing.T) { pql.NewDecimal(-10, 0), pql.NewDecimal(math.MaxInt64, 0), "", - "") + "", + 0) field = sampleIndex.Field("int-field3", OptFieldTypeInt()) jsonString = field.options.String() targetString = fmt.Sprintf(`{"options":{"type":"int","min":%d,"max":%d}}`, math.MinInt64, math.MaxInt64) @@ -978,7 +981,8 @@ func TestORM(t *testing.T) { pql.NewDecimal(math.MinInt64, 0), pql.NewDecimal(math.MaxInt64, 0), "", - "") + "", + 0) field = sampleIndex.Field("int-field4", OptFieldTypeInt(), OptFieldForeignIndex("blerg")) jsonString = field.options.String() @@ -995,7 +999,8 @@ func TestORM(t *testing.T) { pql.NewDecimal(math.MinInt64, 0), pql.NewDecimal(math.MaxInt64, 0), "blerg", - "") + "", + 0) }) t.Run("TimeFieldOptions", func(t *testing.T) { @@ -1004,7 +1009,7 @@ func TestORM(t *testing.T) { t.Fatalf("field noStandardView %v != %v", true, field.Opts().NoStandardView()) } jsonString := field.options.String() - targetString := `{"options":{"noStandardView":true,"type":"time","timeQuantum":"DH"}}` + targetString := `{"options":{"noStandardView":true,"type":"time","timeQuantum":"DH","ttl":"0s"}}` if sortedString(targetString) != sortedString(jsonString) { t.Fatalf("`%s` != `%s`", targetString, jsonString) } @@ -1017,7 +1022,31 @@ func TestORM(t *testing.T) { pql.NewDecimal(0, 0), pql.NewDecimal(0, 0), "", - "") + "", + 0) + }) + + t.Run("TtlOptions", func(t *testing.T) { + field := sampleIndex.Field("ttl-field", OptFieldTypeTime(TimeQuantumDayHour, true), OptFieldTtl(0)) + if true != field.Opts().NoStandardView() { + t.Fatalf("field noStandardView %v != %v", true, field.Opts().NoStandardView()) + } + jsonString := field.options.String() + targetString := `{"options":{"noStandardView":true,"type":"time","timeQuantum":"DH","ttl":"0s"}}` + if sortedString(targetString) != sortedString(jsonString) { + t.Fatalf("`%s` != `%s`", targetString, jsonString) + } + compareFieldOptions(t, + field.Options(), + FieldTypeTime, + TimeQuantumDayHour, + CacheTypeDefault, + 0, + pql.NewDecimal(0, 0), + pql.NewDecimal(0, 0), + "", + "", + 0) }) t.Run("MutexFieldOptions", func(t *testing.T) { @@ -1036,7 +1065,8 @@ func TestORM(t *testing.T) { pql.NewDecimal(0, 0), pql.NewDecimal(0, 0), "", - "") + "", + 0) }) t.Run("BoolFieldOptions", func(t *testing.T) { @@ -1055,7 +1085,8 @@ func TestORM(t *testing.T) { pql.NewDecimal(0, 0), pql.NewDecimal(0, 0), "", - "") + "", + 0) }) t.Run("DecimalFieldOptions", func(t *testing.T) { @@ -1074,7 +1105,8 @@ func TestORM(t *testing.T) { pql.NewDecimal(7, 3), pql.NewDecimal(999, 3), "", - "") + "", + 0) }) t.Run("DecimalFieldOptions", func(t *testing.T) { @@ -1093,7 +1125,8 @@ func TestORM(t *testing.T) { pql.NewDecimal(7, 3), pql.NewDecimal(999, 3), "", - "") + "", + 0) }) t.Run("TimestampFieldOptions", func(t *testing.T) { @@ -1114,7 +1147,8 @@ func TestORM(t *testing.T) { pql.NewDecimal(MinTimestamp.UnixNano()/TimeUnitNanos(pilosa.TimeUnitSeconds), 0), pql.NewDecimal(MaxTimestamp.UnixNano()/TimeUnitNanos(pilosa.TimeUnitSeconds), 0), "", - pilosa.TimeUnitSeconds) + pilosa.TimeUnitSeconds, + 0) }) @@ -1163,7 +1197,7 @@ func comparePQL(t *testing.T, target string, q PQLQuery) { } } -func compareFieldOptions(t *testing.T, opts *FieldOptions, fieldType FieldType, timeQuantum TimeQuantum, cacheType CacheType, cacheSize int, min pql.Decimal, max pql.Decimal, foreignIndex string, timeUnit string) { +func compareFieldOptions(t *testing.T, opts *FieldOptions, fieldType FieldType, timeQuantum TimeQuantum, cacheType CacheType, cacheSize int, min pql.Decimal, max pql.Decimal, foreignIndex string, timeUnit string, ttl time.Duration) { if fieldType != opts.Type() { t.Fatalf("%s != %s", fieldType, opts.Type()) } @@ -1188,6 +1222,9 @@ func compareFieldOptions(t *testing.T, opts *FieldOptions, fieldType FieldType, if timeUnit != opts.TimeUnit() { t.Fatalf("%s != %s", timeUnit, opts.TimeUnit()) } + if ttl != opts.Ttl() { + t.Fatalf("%s != %s", ttl, opts.Ttl()) + } } func sortedString(s string) string { diff --git a/cmd/import.go b/cmd/import.go index d3f50dd27..77dfbb7f0 100644 --- a/cmd/import.go +++ b/cmd/import.go @@ -46,6 +46,7 @@ omitted. If it is present then its format should be YYYY-MM-DDTHH:MM. flags.StringVar(&Importer.FieldOptions.CacheType, "field-cache-type", pilosa.CacheTypeRanked, "Specify the cache type for a set field on creation. One of: none, lru, ranked") flags.Uint32Var(&Importer.FieldOptions.CacheSize, "field-cache-size", 50000, "Specify the cache size for a set field on creation") flags.Var(&Importer.FieldOptions.TimeQuantum, "field-time-quantum", "Specify the time quantum for a time field on creation. One of: D, DH, H, M, MD, MDH, Y, YM, YMD, YMDH") + flags.DurationVarP(&Importer.FieldOptions.Ttl, "time-to-live", "t", 0, "Specify the time to live for views created by time quantum. Supported time unit: \"s\", \"m\", \"h\"") // \"ns\", \"us\" (or \"µs\"), \"ms\" also supported but ommitted for simplicity flags.IntVarP(&Importer.BufferSize, "buffer-size", "s", 10000000, "Number of bits to buffer/sort before importing.") flags.BoolVarP(&Importer.Sort, "sort", "", false, "Enables sorting before import.") flags.BoolVarP(&Importer.CreateSchema, "create", "e", false, "Create the schema if it does not exist before import.") diff --git a/delete_test.go b/delete_test.go index 63c77be34..62d39a789 100644 --- a/delete_test.go +++ b/delete_test.go @@ -34,7 +34,7 @@ func TestExecutor_DeleteRecords(t *testing.T) { {ID: 0, Val: 4}, {ID: 2, Val: 8}, }) - c.CreateField(t, indexName, pilosa.IndexOptions{TrackExistence: true}, "timefield", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"))) + c.CreateField(t, indexName, pilosa.IndexOptions{TrackExistence: true}, "timefield", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"), "0")) c.ImportBitsWithTimestamp(t, indexName, "timefield", [][2]uint64{ {0, 0}, {0, 1}, @@ -67,7 +67,7 @@ func TestExecutor_DeleteRecords(t *testing.T) { setupKeys := func(t *testing.T, r *require.Assertions, c *test.Cluster) { t.Helper() - c.CreateField(t, indexName, pilosa.IndexOptions{Keys: true, TrackExistence: true}, "timefield", pilosa.OptFieldKeys(), pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"))) + c.CreateField(t, indexName, pilosa.IndexOptions{Keys: true, TrackExistence: true}, "timefield", pilosa.OptFieldKeys(), pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"), "0")) c.ImportTimeQuantumKey(t, indexName, "timefield", []test.TimeQuantumKey{ {RowKey: "fish", ColKey: "one", Ts: time.Date(2019, time.January, 2, 17, 45, 0, 0, time.UTC).Unix()}, {RowKey: "fish", ColKey: "one", Ts: time.Date(2020, time.January, 2, 17, 45, 0, 0, time.UTC).Unix()}, diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index b9f826938..d0e894cfe 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -669,6 +669,7 @@ func (s Serializer) encodeFieldOptions(o *pilosa.FieldOptions) *pb.FieldOptions Scale: o.Scale, BitDepth: uint64(o.BitDepth), TimeQuantum: string(o.TimeQuantum), + Ttl: o.Ttl.String(), TimeUnit: string(o.TimeUnit), Keys: o.Keys, ForeignIndex: o.ForeignIndex, @@ -1072,6 +1073,11 @@ func (s Serializer) decodeFieldOptions(options *pb.FieldOptions, m *pilosa.Field m.Scale = options.Scale m.BitDepth = uint64(options.BitDepth) m.TimeQuantum = pilosa.TimeQuantum(options.TimeQuantum) + ttlValue, err := time.ParseDuration(options.Ttl) + if err != nil { + ttlValue = 0 + } + m.Ttl = ttlValue m.TimeUnit = options.TimeUnit m.Keys = options.Keys m.ForeignIndex = options.ForeignIndex diff --git a/executor_test.go b/executor_test.go index f580f76a6..8064a2e5d 100644 --- a/executor_test.go +++ b/executor_test.go @@ -490,7 +490,7 @@ func TestExecutor(t *testing.T) { `Row(f=1, from=1999-12-31T00:00, to=2002-01-01T03:00)`, } responses := runCallTest(c, t, writeQuery, readQueries, - nil, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"))) + nil, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"), "0")) t.Run("Standard", func(t *testing.T) { if columns := responses[0].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6, 7}) { @@ -536,7 +536,7 @@ func TestExecutor(t *testing.T) { } responses := runCallTest(c, t, writeQuery, readQueries, &pilosa.IndexOptions{Keys: true}, - pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"))) + pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"), "0")) t.Run("Standard", func(t *testing.T) { if keys := responses[0].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"six", "four", "five", "seven", "two", "three"}) { @@ -570,7 +570,7 @@ func TestExecutor(t *testing.T) { } responses := runCallTest(c, t, writeQuery, readQueries, nil, - pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH")), + pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"), "0"), pilosa.OptFieldKeys()) t.Run("Standard", func(t *testing.T) { @@ -605,7 +605,7 @@ func TestExecutor(t *testing.T) { } responses := runCallTest(c, t, writeQuery, readQueries, &pilosa.IndexOptions{Keys: true}, - pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH")), + pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"), "0"), pilosa.OptFieldKeys()) t.Run("Standard", func(t *testing.T) { @@ -639,7 +639,7 @@ func TestExecutor(t *testing.T) { `Row(f=1, from=946598400, to=1009854000)`, } responses := runCallTest(c, t, writeQuery, readQueries, - nil, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"))) + nil, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"), "0")) t.Run("Standard", func(t *testing.T) { if columns := responses[0].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6, 7}) { @@ -674,7 +674,7 @@ func TestExecutor(t *testing.T) { `Range(f=1, from=1999-12-31T00:00, to=2002-01-01T03:00)`, } responses := runCallTest(c, t, writeQuery, readQueries, - nil, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"))) + nil, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"), "0")) t.Run("Standard", func(t *testing.T) { if columns := responses[0].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6, 7}) { @@ -693,7 +693,7 @@ func TestExecutor(t *testing.T) { `Range(f=1, 1999-12-31T00:00, 2002-01-01T03:00)`, } responses = runCallTest(c, t, writeQuery, rq2, - nil, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"))) + nil, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"), "0")) t.Run("OldRange", func(t *testing.T) { if columns := responses[0].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6, 7}) { t.Fatalf("unexpected columns: %+v", columns) @@ -721,7 +721,7 @@ func TestExecutor(t *testing.T) { } responses := runCallTest(c, t, writeQuery, readQueries, &pilosa.IndexOptions{Keys: true}, - pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"))) + pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"), "0")) t.Run("Standard", func(t *testing.T) { if keys := responses[0].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"two", "three", "seven", "four", "five", "six"}) { @@ -755,7 +755,7 @@ func TestExecutor(t *testing.T) { } responses := runCallTest(c, t, writeQuery, readQueries, nil, - pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH")), + pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"), "0"), pilosa.OptFieldKeys()) t.Run("Standard", func(t *testing.T) { @@ -790,7 +790,7 @@ func TestExecutor(t *testing.T) { } responses := runCallTest(c, t, writeQuery, readQueries, &pilosa.IndexOptions{Keys: true}, - pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH")), + pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"), "0"), pilosa.OptFieldKeys()) t.Run("Standard", func(t *testing.T) { @@ -990,7 +990,7 @@ func TestExecutor(t *testing.T) { } responses := runCallTest(c, t, writeQuery, readQueries, &pilosa.IndexOptions{TrackExistence: true}, - pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMD"))) + pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMD"), "0")) if columns := responses[0].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6, 7}) { t.Fatalf("unexpected columns: %+v", columns) } @@ -1059,7 +1059,7 @@ func TestExecutor(t *testing.T) { } responses := runCallTest(c, t, writeQuery, readQueries, - nil, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMD"), true)) + nil, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMD"), "0", true)) for i := range responses { t.Run(fmt.Sprintf("response-%d", i), func(t *testing.T) { @@ -1730,7 +1730,7 @@ func TestExecutor_Execute_TopK_Time(t *testing.T) { defer c.Close() // Load some test data into a time field. - c.CreateField(t, "i", pilosa.IndexOptions{TrackExistence: true}, "f", pilosa.OptFieldTypeTime("YMD", true)) + c.CreateField(t, "i", pilosa.IndexOptions{TrackExistence: true}, "f", pilosa.OptFieldTypeTime("YMD", "0", true)) c.Query(t, "i", ` Set(0, f=0, 2016-01-02T00:00) Set(0, f=1, 2016-01-02T00:00) @@ -3413,7 +3413,7 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) { }) t.Run("remote with timestamp", func(t *testing.T) { - _, err = c.GetPrimary().API.CreateField(context.Background(), "i", "z", pilosa.OptFieldTypeTime("Y")) + _, err = c.GetPrimary().API.CreateField(context.Background(), "i", "z", pilosa.OptFieldTypeTime("Y", "0")) if err != nil { t.Fatalf("creating field: %v", err) } @@ -3746,7 +3746,7 @@ func TestExecutor_Time_Clear_Quantums(t *testing.T) { indexName := strings.ToLower(string(tt.quantum)) index := hldr.MustCreateIndexIfNotExists(indexName, pilosa.IndexOptions{}) // Create field. - if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeTime(tt.quantum)); err != nil { + if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeTime(tt.quantum, "0")); err != nil { t.Fatal(err) } // Populate @@ -4691,14 +4691,14 @@ func TestExecutor_Execute_Extract(t *testing.T) { Set(3, keymutex="plugh") `) - c.CreateField(t, "i", pilosa.IndexOptions{TrackExistence: true}, "time", pilosa.OptFieldTypeTime("YMDH")) + c.CreateField(t, "i", pilosa.IndexOptions{TrackExistence: true}, "time", pilosa.OptFieldTypeTime("YMDH", "0")) c.Query(t, "i", ` Set(0, time=1, 2016-01-01T00:00) Set(1, time=2, 2017-01-01T00:00) Set(3, time=3, 2018-01-01T00:00) `) - c.CreateField(t, "i", pilosa.IndexOptions{TrackExistence: true}, "keytime", pilosa.OptFieldKeys(), pilosa.OptFieldTypeTime("YMDH")) + c.CreateField(t, "i", pilosa.IndexOptions{TrackExistence: true}, "keytime", pilosa.OptFieldKeys(), pilosa.OptFieldTypeTime("YMDH", "0")) c.Query(t, "i", ` Set(0, keytime="h", 2016-01-01T00:00) Set(1, keytime="xyzzy", 2017-01-01T00:00) @@ -5087,7 +5087,7 @@ func TestExecutor_Execute_RowsTime(t *testing.T) { func TestExecutor_Execute_RowsTimeEmpty(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - c.CreateField(t, "i", pilosa.IndexOptions{}, "x", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMD"), true)) + c.CreateField(t, "i", pilosa.IndexOptions{}, "x", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMD"), "0", true)) rows := c.Query(t, "i", `Rows(x, from=1999-12-31T00:00, to=2002-01-01T03:00)`).Results[0].(pilosa.RowIdentifiers).Rows if !reflect.DeepEqual(rows, []uint64{}) { t.Fatalf("unexpected rows: %+v", rows) @@ -7356,7 +7356,7 @@ func variousQueriesOnTimeFields(t *testing.T, c *test.Cluster) { // generic index // worth noting, since we are using YMDH resolution, both C4 & C5 // get binned to the same hour - c.CreateField(t, "t_index", pilosa.IndexOptions{Keys: true, TrackExistence: true}, "f1", pilosa.OptFieldKeys(), pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"))) + c.CreateField(t, "t_index", pilosa.IndexOptions{Keys: true, TrackExistence: true}, "f1", pilosa.OptFieldKeys(), pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"), "0")) c.ImportTimeQuantumKey(t, "t_index", "f1", []test.TimeQuantumKey{ // from edge cases {ColKey: "C1", RowKey: "R1", Ts: ts(time.Date(2019, 1, 1, 0, 0, 0, 0, time.UTC))}, @@ -7372,7 +7372,7 @@ func variousQueriesOnTimeFields(t *testing.T, c *test.Cluster) { }) // in this field, all columns have the same row value to simplify test queries for Row - c.CreateField(t, "t_index", pilosa.IndexOptions{Keys: true, TrackExistence: true}, "f2", pilosa.OptFieldKeys(), pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"))) + c.CreateField(t, "t_index", pilosa.IndexOptions{Keys: true, TrackExistence: true}, "f2", pilosa.OptFieldKeys(), pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"), "0")) c.ImportTimeQuantumKey(t, "t_index", "f2", []test.TimeQuantumKey{ // from {ColKey: "C1", RowKey: "R", Ts: ts(time.Date(2019, 1, 1, 0, 0, 0, 0, time.UTC))}, @@ -7522,7 +7522,7 @@ func populateTestData(t *testing.T, c *test.Cluster) { }) // Create and populate "places_visited" time field. - c.CreateField(t, usersIndex, pilosa.IndexOptions{Keys: true, TrackExistence: true}, "places_visited", pilosa.OptFieldKeys(), pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YM"))) + c.CreateField(t, usersIndex, pilosa.IndexOptions{Keys: true, TrackExistence: true}, "places_visited", pilosa.OptFieldKeys(), pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YM"), "0")) ts2019Jan01 := int64(1546300800) * 1e+9 // 2019 January 1st 0:00:00 ts2019Aug01 := int64(1564617600) * 1e+9 // 2019 August 1st 0:00:00 ts2020Jan01 := int64(1577836800) * 1e+9 // 2020 January 1st 0:00:00 diff --git a/field.go b/field.go index 5beb2983a..228f18c8a 100644 --- a/field.go +++ b/field.go @@ -286,7 +286,7 @@ func OptFieldTypeDecimal(scale int64, minmax ...pql.Decimal) FieldOption { // used to specify the field as being type `time` and to // provide any respective configuration values. // Pass true to skip creation of the standard view. -func OptFieldTypeTime(timeQuantum TimeQuantum, opt ...bool) FieldOption { +func OptFieldTypeTime(timeQuantum TimeQuantum, ttl string, opt ...bool) FieldOption { return func(fo *FieldOptions) error { if fo.Type != "" { return errors.Errorf("field type is already set to: %s", fo.Type) @@ -296,6 +296,11 @@ func OptFieldTypeTime(timeQuantum TimeQuantum, opt ...bool) FieldOption { } fo.Type = FieldTypeTime fo.TimeQuantum = timeQuantum + ttlParsed, err := time.ParseDuration(ttl) + if err != nil { + return errors.Errorf("cannot parse ttl: %s", ttl) + } + fo.Ttl = ttlParsed fo.NoStandardView = len(opt) >= 1 && opt[0] return nil } @@ -674,6 +679,11 @@ func (f *Field) ForeignIndex() string { return f.options.ForeignIndex } +// Ttl returns the ttl of the field. +func (f *Field) Ttl() time.Duration { + return f.options.Ttl +} + func (f *Field) bitDepth() (uint64, error) { var maxBitDepth uint64 @@ -763,6 +773,7 @@ func (f *Field) applyOptions(opt FieldOptions) error { f.options.Base = 0 f.options.BitDepth = 0 f.options.TimeQuantum = "" + f.options.Ttl = 0 f.options.Keys = opt.Keys f.options.ForeignIndex = opt.ForeignIndex case FieldTypeInt, FieldTypeDecimal, FieldTypeTimestamp: @@ -776,6 +787,7 @@ func (f *Field) applyOptions(opt FieldOptions) error { f.options.BitDepth = opt.BitDepth f.options.TimeUnit = opt.TimeUnit f.options.TimeQuantum = "" + f.options.Ttl = 0 f.options.Keys = opt.Keys f.options.ForeignIndex = opt.ForeignIndex @@ -809,6 +821,7 @@ func (f *Field) applyOptions(opt FieldOptions) error { return ErrInvalidTimeQuantum } f.options.TimeQuantum = opt.TimeQuantum + f.options.Ttl = opt.Ttl f.options.ForeignIndex = opt.ForeignIndex case FieldTypeBool: f.options.Type = FieldTypeBool @@ -819,6 +832,7 @@ func (f *Field) applyOptions(opt FieldOptions) error { f.options.Base = 0 f.options.BitDepth = 0 f.options.TimeQuantum = "" + f.options.Ttl = 0 f.options.Keys = false f.options.ForeignIndex = "" default: @@ -1831,19 +1845,20 @@ func (p fieldInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name } // FieldOptions represents options to set when initializing a field. type FieldOptions struct { - Base int64 `json:"base,omitempty"` - BitDepth uint64 `json:"bitDepth,omitempty"` - Min pql.Decimal `json:"min,omitempty"` - Max pql.Decimal `json:"max,omitempty"` - Scale int64 `json:"scale,omitempty"` - Keys bool `json:"keys"` - NoStandardView bool `json:"noStandardView,omitempty"` - CacheSize uint32 `json:"cacheSize,omitempty"` - CacheType string `json:"cacheType,omitempty"` - Type string `json:"type,omitempty"` - TimeUnit string `json:"timeUnit,omitempty"` - TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"` - ForeignIndex string `json:"foreignIndex"` + Base int64 `json:"base,omitempty"` + BitDepth uint64 `json:"bitDepth,omitempty"` + Min pql.Decimal `json:"min,omitempty"` + Max pql.Decimal `json:"max,omitempty"` + Scale int64 `json:"scale,omitempty"` + Keys bool `json:"keys"` + NoStandardView bool `json:"noStandardView,omitempty"` + CacheSize uint32 `json:"cacheSize,omitempty"` + CacheType string `json:"cacheType,omitempty"` + Type string `json:"type,omitempty"` + TimeUnit string `json:"timeUnit,omitempty"` + TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"` + ForeignIndex string `json:"foreignIndex"` + Ttl time.Duration `json:"ttl,omitempty"` } // newFieldOptions returns a new instance of FieldOptions @@ -1954,15 +1969,17 @@ func (o *FieldOptions) MarshalJSON() ([]byte, error) { }) case FieldTypeTime: return json.Marshal(struct { - Type string `json:"type"` - TimeQuantum TimeQuantum `json:"timeQuantum"` - Keys bool `json:"keys"` - NoStandardView bool `json:"noStandardView"` + Type string `json:"type"` + TimeQuantum TimeQuantum `json:"timeQuantum"` + Keys bool `json:"keys"` + NoStandardView bool `json:"noStandardView"` + Ttl time.Duration `json:"Ttl"` }{ o.Type, o.TimeQuantum, o.Keys, o.NoStandardView, + o.Ttl, }) case FieldTypeMutex: return json.Marshal(struct { diff --git a/field_internal_test.go b/field_internal_test.go index 98dea321b..5f0f7d013 100644 --- a/field_internal_test.go +++ b/field_internal_test.go @@ -343,7 +343,7 @@ func TestField_CreateViewIfNotExists(t *testing.T) { } func TestField_SetTimeQuantum(t *testing.T) { - f := OpenField(t, OptFieldTypeTime(TimeQuantum("YMDH"))) + f := OpenField(t, OptFieldTypeTime(TimeQuantum("YMDH"), "0")) defer f.Close() // Retrieve time quantum. @@ -360,7 +360,7 @@ func TestField_SetTimeQuantum(t *testing.T) { } func TestField_RowTime(t *testing.T) { - f := OpenField(t, OptFieldTypeTime(TimeQuantum("YMDH"))) + f := OpenField(t, OptFieldTypeTime(TimeQuantum("YMDH"), "0")) defer f.Close() // Obtain transaction. diff --git a/go.mod b/go.mod index 529f2a29d..4d1f6361d 100644 --- a/go.mod +++ b/go.mod @@ -54,11 +54,12 @@ require ( go.etcd.io/etcd v0.0.0-20201125193152-8a03d2e9614b golang.org/x/crypto v0.0.0-20201217014255-9d1352758620 // indirect golang.org/x/exp v0.0.0-20201008143054-e3b2a7f2fdc7 - golang.org/x/mod v0.4.2 - golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d // indirect + golang.org/x/mod v0.5.1 + golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f // indirect golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c golang.org/x/sys v0.0.0-20220111092808-5a964db01320 // indirect + golang.org/x/text v0.3.7 // indirect google.golang.org/grpc v1.28.0 gopkg.in/yaml.v2 v2.4.0 modernc.org/mathutil v1.0.0 diff --git a/go.sum b/go.sum index 79fc9f1f6..c89a96884 100644 --- a/go.sum +++ b/go.sum @@ -444,8 +444,8 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.5.1 h1:OJxoQ/rynoF0dcCdI7cLPktw/hR2cueqYfjm43oqK38= +golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -465,8 +465,8 @@ golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d h1:20cMwl2fHAzkJMEA+8J4JgqBQcQGzbisXo31MIeenXI= -golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f h1:OfiFi4JbukWwe3lzw+xunroH1mnC1e2Gy5cxNJApiSY= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= @@ -515,8 +515,9 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= diff --git a/hack.go b/hack.go index 4f8a8a460..eb1f5787b 100644 --- a/hack.go +++ b/hack.go @@ -2,6 +2,8 @@ package pilosa import ( + "time" + "github.com/gogo/protobuf/proto" "github.com/molecula/featurebase/v3/pb" "github.com/molecula/featurebase/v3/pql" @@ -57,6 +59,11 @@ func UnmarshalFieldOptions(name string, createdAt int64, buf []byte) (*FieldInfo fi.Options.Base = pbi.Base fi.Options.BitDepth = pbi.BitDepth fi.Options.TimeQuantum = TimeQuantum(pbi.TimeQuantum) + ttlValue, err := time.ParseDuration(pbi.Ttl) + if err != nil { + ttlValue = 0 + } + fi.Options.Ttl = ttlValue fi.Options.Keys = pbi.Keys fi.Options.NoStandardView = pbi.NoStandardView diff --git a/holder_test.go b/holder_test.go index 1485f59fc..bd5edd5d7 100644 --- a/holder_test.go +++ b/holder_test.go @@ -453,7 +453,7 @@ func TestHolderSyncer_TimeQuantum(t *testing.T) { if err != nil { t.Fatalf("creating index i: %v", err) } - _, err = c.GetNode(0).API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeTime(pilosa.TimeQuantum(quantum))) + _, err = c.GetNode(0).API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeTime(pilosa.TimeQuantum(quantum), "0")) if err != nil { t.Fatalf("creating field f: %v", err) } diff --git a/http_handler.go b/http_handler.go index 712994fe2..4756b0fab 100644 --- a/http_handler.go +++ b/http_handler.go @@ -429,6 +429,8 @@ func newRouter(handler *Handler) http.Handler { //router.HandleFunc("/index/{index}/field", handler.chkAuthZ(handler.handleGetFields, authz.Read)).Methods("GET") // Not implemented. router.HandleFunc("/index/{index}/field", handler.chkAuthZ(handler.handlePostField, authz.Write)).Methods("POST").Name("PostField") router.HandleFunc("/index/{index}/field/", handler.chkAuthZ(handler.handlePostField, authz.Write)).Methods("POST").Name("PostField") + router.HandleFunc("/index/{index}/field/{field}/view", handler.chkAuthZ(handler.handleGetView, authz.Admin)).Methods("GET") + router.HandleFunc("/index/{index}/field/{field}/view/{view}", handler.chkAuthZ(handler.handleDeleteView, authz.Admin)).Methods("DELETE").Name("DeleteView") router.HandleFunc("/index/{index}/field/{field}", handler.chkAuthZ(handler.handlePostField, authz.Write)).Methods("POST").Name("PostField") router.HandleFunc("/index/{index}/field/{field}", handler.chkAuthZ(handler.handleDeleteField, authz.Write)).Methods("DELETE").Name("DeleteField") router.HandleFunc("/index/{index}/field/{field}/import", handler.chkAuthZ(handler.handlePostImport, authz.Write)).Methods("POST").Name("PostImport") @@ -1281,6 +1283,62 @@ func (h *Handler) handleGetIndex(w http.ResponseWriter, r *http.Request) { http.Error(w, fmt.Sprintf("Index %s Not Found", indexName), http.StatusNotFound) } +// handleGetView handles GET /index//field//view requests. +func (h *Handler) handleGetView(w http.ResponseWriter, r *http.Request) { + if !validHeaderAcceptJSON(r.Header) { + http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable) + return + } + + indexName := mux.Vars(r)["index"] + fieldName := mux.Vars(r)["field"] + index, err := h.api.Index(r.Context(), indexName) + + if err != nil { + http.Error(w, fmt.Sprintf("Index %s Not Found", indexName), http.StatusNotFound) + return + } else { + w.Header().Set("Content-Type", "application/json") + var viewsList []viewReponse + for _, field := range index.fields { + if field.name == fieldName { + for _, view := range field.views() { + viewsList = append(viewsList, viewReponse{Name: view.name, Type: view.fieldType, Field: view.field, Index: view.index}) + } + + if err := json.NewEncoder(w).Encode(viewsList); err != nil { + h.logger.Errorf("write response error: %s", err) + } + return + } + } + } + http.Error(w, fmt.Sprintf("Field %s Not Found", fieldName), http.StatusNotFound) +} + +type viewReponse struct { + Name string `json:"name"` + Type string `json:"type"` + Field string `json:"field"` + Index string `json:"index"` +} + +// handleDeleteIndex handles DELETE /index//field//view/ request. +func (h *Handler) handleDeleteView(w http.ResponseWriter, r *http.Request) { + if !validHeaderAcceptJSON(r.Header) { + http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable) + return + } + + indexName := mux.Vars(r)["index"] + fieldName := mux.Vars(r)["field"] + viewName := mux.Vars(r)["view"] + + resp := successResponse{h: h} + err := h.api.DeleteView(r.Context(), indexName, fieldName, viewName) + resp.write(w, err) +} + type postIndexRequest struct { Options IndexOptions `json:"options"` } @@ -1524,7 +1582,11 @@ func fieldOptionsToFunctionalOpts(opt fieldOptions) []FieldOption { } fos = append(fos, OptFieldTypeTimestamp(opt.Epoch.UTC(), *opt.TimeUnit)) case FieldTypeTime: - fos = append(fos, OptFieldTypeTime(*opt.TimeQuantum, opt.NoStandardView)) + if opt.Ttl != nil { + fos = append(fos, OptFieldTypeTime(*opt.TimeQuantum, *opt.Ttl, opt.NoStandardView)) + } else { + fos = append(fos, OptFieldTypeTime(*opt.TimeQuantum, "0", opt.NoStandardView)) + } case FieldTypeMutex: fos = append(fos, OptFieldTypeMutex(*opt.CacheType, *opt.CacheSize)) case FieldTypeBool: @@ -1646,6 +1708,7 @@ type fieldOptionSpec struct { Epoch *time.Time `json:"epoch"` Unit *string `json:"unit"` TimeQuantum *string `json:"time-quantum"` + Ttl *string `json:"ttl"` } func fieldSpecToFieldOption(fSpec fieldSpec) fieldOptions { @@ -1679,6 +1742,7 @@ func fieldSpecToFieldOption(fSpec fieldSpec) fieldOptions { timeQuantumVal := TimeQuantum(*fSpec.FieldOptions.TimeQuantum) opt.TimeQuantum = &timeQuantumVal } + opt.Ttl = fSpec.FieldOptions.Ttl return opt } @@ -1918,6 +1982,7 @@ type fieldOptions struct { Keys *bool `json:"keys,omitempty"` NoStandardView bool `json:"noStandardView,omitempty"` ForeignIndex *string `json:"foreignIndex,omitempty"` + Ttl *string `json:"ttl,omitempty"` } func (o *fieldOptions) validate() error { @@ -1945,6 +2010,8 @@ func (o *fieldOptions) validate() error { return NewBadRequestError(errors.New("max does not apply to field type set")) } else if o.TimeQuantum != nil { return NewBadRequestError(errors.New("timeQuantum does not apply to field type set")) + } else if o.Ttl != nil { + return NewBadRequestError(errors.New("ttl does not apply to field type set")) } case FieldTypeInt: if o.CacheType != nil { @@ -1953,6 +2020,8 @@ func (o *fieldOptions) validate() error { return NewBadRequestError(errors.New("cacheSize does not apply to field type int")) } else if o.TimeQuantum != nil { return NewBadRequestError(errors.New("timeQuantum does not apply to field type int")) + } else if o.Ttl != nil { + return NewBadRequestError(errors.New("ttl does not apply to field type int")) } case FieldTypeDecimal: if o.Scale == nil { @@ -1963,6 +2032,8 @@ func (o *fieldOptions) validate() error { return NewBadRequestError(errors.New("cacheSize does not apply to field type int")) } else if o.TimeQuantum != nil { return NewBadRequestError(errors.New("timeQuantum does not apply to field type int")) + } else if o.Ttl != nil { + return NewBadRequestError(errors.New("ttl does not apply to field type int")) } else if o.ForeignIndex != nil && o.Type == FieldTypeDecimal { return NewBadRequestError(errors.New("decimal field cannot be a foreign key")) } @@ -1977,6 +2048,8 @@ func (o *fieldOptions) validate() error { return NewBadRequestError(errors.New("cacheSize does not apply to field type timestamp")) } else if o.TimeQuantum != nil { return NewBadRequestError(errors.New("timeQuantum does not apply to field type timestamp")) + } else if o.Ttl != nil { + return NewBadRequestError(errors.New("ttl does not apply to field type timestamp")) } else if o.ForeignIndex != nil { return NewBadRequestError(errors.New("timestamp field cannot be a foreign key")) } @@ -2005,6 +2078,8 @@ func (o *fieldOptions) validate() error { return NewBadRequestError(errors.New("max does not apply to field type mutex")) } else if o.TimeQuantum != nil { return NewBadRequestError(errors.New("timeQuantum does not apply to field type mutex")) + } else if o.Ttl != nil { + return NewBadRequestError(errors.New("ttl does not apply to field type mutex")) } case FieldTypeBool: if o.CacheType != nil { @@ -2019,6 +2094,8 @@ func (o *fieldOptions) validate() error { return NewBadRequestError(errors.New("timeQuantum does not apply to field type bool")) } else if o.Keys != nil { return NewBadRequestError(errors.New("keys does not apply to field type bool")) + } else if o.Ttl != nil { + return NewBadRequestError(errors.New("ttl does not apply to field type bool")) } else if o.ForeignIndex != nil { return NewBadRequestError(errors.New("bool field cannot be a foreign key")) } diff --git a/http_handler_internal_test.go b/http_handler_internal_test.go index 59ae88e66..30bf3a782 100644 --- a/http_handler_internal_test.go +++ b/http_handler_internal_test.go @@ -131,6 +131,7 @@ func TestFieldOptionValidation(t *testing.T) { {json: `{"options": {"type": "set", "min": 0}}`, err: "min does not apply to field type set"}, {json: `{"options": {"type": "set", "max": 100}}`, err: "max does not apply to field type set"}, {json: `{"options": {"type": "set", "timeQuantum": "YMD"}}`, err: "timeQuantum does not apply to field type set"}, + {json: `{"options": {"type": "set", "ttl": "1h"}}`, err: "ttl does not apply to field type set"}, // FieldType: Int {json: `{"options": {"type": "int"}}`, err: "min is required for field type int"}, @@ -143,6 +144,7 @@ func TestFieldOptionValidation(t *testing.T) { {json: `{"options": {"type": "int", "min": 0, "max": 1000, "cacheType": "ranked"}}`, err: "cacheType does not apply to field type int"}, {json: `{"options": {"type": "int", "min": 0, "max": 1000, "cacheSize": 1000}}`, err: "cacheSize does not apply to field type int"}, {json: `{"options": {"type": "int", "min": 0, "max": 1000, "timeQuantum": "YMD"}}`, err: "timeQuantum does not apply to field type int"}, + {json: `{"options": {"type": "int", "min": 0, "max": 1000, "ttl": "1h"}}`, err: "ttl does not apply to field type int"}, // FieldType: Time {json: `{"options": {"type": "time"}}`, err: "timeQuantum is required for field type time"}, diff --git a/http_handler_test.go b/http_handler_test.go index a692a8435..2a213642b 100644 --- a/http_handler_test.go +++ b/http_handler_test.go @@ -6,6 +6,8 @@ import ( "fmt" "net" gohttp "net/http" + "reflect" + "sort" "strings" "testing" @@ -170,6 +172,166 @@ func TestIngestSchemaHandler(t *testing.T) { } } +func TestPostFieldWithTtl(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + + schema := ` + { + "index-name": "example", + "primary-key-type": "string", + "index-action": "create", + "fields":[] + } + ` + m := c.GetPrimary() + schemaURL := fmt.Sprintf("%s/internal/schema", m.URL()) + resp := test.Do(t, "POST", schemaURL, string(schema)) + if resp.StatusCode != gohttp.StatusOK { + t.Errorf("invalid status: %d, body=%s", resp.StatusCode, resp.Body) + } + + postFieldTtlUrl := fmt.Sprintf("%s/index/example/field/with_ttl", m.URL()) + + // Create new field with ttl but in invalid format + fieldOptionInvalidTtl := ` + { "options": {"timeQuantum":"YMDH","type":"time","ttl":"24hour" }} + ` + respField := test.Do(t, "POST", postFieldTtlUrl, string(fieldOptionInvalidTtl)) + if (respField.StatusCode != gohttp.StatusBadRequest) && + (respField.Body != "applying option: cannot parse ttl: 24hour") { + t.Errorf("expected ttl parse error, got status: %d, body=%s", respField.StatusCode, respField.Body) + } + + // Create new field with ttl in invalid format + fieldOptionValidTtl := ` + { "options": {"timeQuantum":"YMDH","type":"time","ttl":"24h" }} + ` + respField = test.Do(t, "POST", postFieldTtlUrl, string(fieldOptionValidTtl)) + if resp.StatusCode != gohttp.StatusOK { + t.Errorf("creating field with ttl, got status: %d, body=%s", respField.StatusCode, respField.Body) + } + + // Create new field without ttl + postFieldNoTtlUrl := fmt.Sprintf("%s/index/example/field/no_ttl", m.URL()) + fieldOptionNoTtl := ` + { "options": {"timeQuantum":"YMDH","type":"time" }} + ` + respField = test.Do(t, "POST", postFieldNoTtlUrl, string(fieldOptionNoTtl)) + if resp.StatusCode != gohttp.StatusOK { + t.Errorf("creating field without ttl, status: %d, body=%s", respField.StatusCode, respField.Body) + } +} + +func TestGetViewAndDelete(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + + schema := ` + { + "index-name": "example", + "primary-key-type": "string", + "index-action": "create", + "fields": [ + { + "field-name": "test_view", + "field-type": "time", + "field-options": { + "time-quantum": "YMDH" + } + } + ] + } + ` + m := c.GetPrimary() + schemaURL := fmt.Sprintf("%s/internal/schema", m.URL()) + resp := test.Do(t, "POST", schemaURL, string(schema)) + if resp.StatusCode != gohttp.StatusOK { + t.Errorf("invalid status: %d, body=%s", resp.StatusCode, resp.Body) + } + + // Send sample data + postQueryUrl := fmt.Sprintf("%s/index/example/query", m.URL()) + queryOption := ` + Set(1,test_view=1,2001-02-03T04:05) + ` + respQuery := test.Do(t, "POST", postQueryUrl, string(queryOption)) + if respQuery.StatusCode != gohttp.StatusOK { + t.Errorf("posting query, status: %d, body=%s", respQuery.StatusCode, respQuery.Body) + } + + // The above sample data should create these views: + expectedViewNames := []string{ + "standard", + "standard_2001", + "standard_200102", + "standard_20010203", + "standard_2001020304", + } + + // Call view to get data + viewUrl := fmt.Sprintf("%s/index/example/field/test_view/view", m.URL()) + respView := test.Do(t, "GET", viewUrl, "") + if respView.StatusCode != gohttp.StatusOK { + t.Errorf("view handler, status: %d, body=%s", respView.StatusCode, respView.Body) + } + + type viewReponse struct { + Name string `json:"name"` + Type string `json:"type"` + Field string `json:"field"` + Index string `json:"index"` + } + + var parsedViews []viewReponse + if err := json.Unmarshal([]byte(respView.Body), &parsedViews); err != nil { + t.Errorf("parsing view, err: %s", err) + } + + // check if data from view matches with expectedViewNames + parseViewNames := []string{} + for _, view := range parsedViews { + parseViewNames = append(parseViewNames, view.Name) + } + sort.Strings(parseViewNames) + + if !reflect.DeepEqual(expectedViewNames, parseViewNames) { + t.Fatalf("expected %v, but got %v", expectedViewNames, parseViewNames) + } + + // call delete on view standard_2001020304 + deleteViewUrl := fmt.Sprintf("%s/index/example/field/test_view/view/standard_2001020304", m.URL()) + respDelete := test.Do(t, "DELETE", deleteViewUrl, "") + if respDelete.StatusCode != gohttp.StatusOK { + t.Errorf("delete handler, status: %d, body=%s", respDelete.StatusCode, respDelete.Body) + } + + // remove view that was deleted (standard_2001020304) from expectedViewNames + expectedViewNames = expectedViewNames[:len(expectedViewNames)-1] + + // call view again + viewUrl = fmt.Sprintf("%s/index/example/field/test_view/view", m.URL()) + respView = test.Do(t, "GET", viewUrl, "") + if respView.StatusCode != gohttp.StatusOK { + t.Errorf("view handler after delete, status: %d, body=%s", respView.StatusCode, respView.Body) + } + + if err := json.Unmarshal([]byte(respView.Body), &parsedViews); err != nil { + t.Errorf("parsing view, err: %s", err) + } + + // check if data from view matches with expectedViewNames + parseViewNames = []string{} + for _, view := range parsedViews { + parseViewNames = append(parseViewNames, view.Name) + } + sort.Strings(parseViewNames) + + if !reflect.DeepEqual(expectedViewNames, parseViewNames) { + t.Fatalf("after delete, expected %v, but got %v", expectedViewNames, parseViewNames) + } +} + func TestTranslationHandlers(t *testing.T) { // reusable data for the tests nameBytes, err := json.Marshal([]string{"a", "b", "c"}) diff --git a/index_test.go b/index_test.go index 67d69ecbe..7d63f8639 100644 --- a/index_test.go +++ b/index_test.go @@ -55,7 +55,7 @@ func TestIndex_CreateField(t *testing.T) { defer index.Close() // Create field with explicit quantum. - f, err := index.CreateField("f", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"))) + f, err := index.CreateField("f", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"), "0")) if err != nil { t.Fatal(err) } else if q := f.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") { @@ -71,7 +71,7 @@ func TestIndex_CreateField(t *testing.T) { defer index.Close() // Create field with explicit quantum with no standard view - f, err := index.CreateField("f", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"), true)) + f, err := index.CreateField("f", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"), "0", true)) if err != nil { t.Fatal(err) } else if q := f.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") { diff --git a/internal_client.go b/internal_client.go index 70d2205bb..1bff1e90a 100644 --- a/internal_client.go +++ b/internal_client.go @@ -1094,6 +1094,8 @@ func (c *InternalClient) CreateFieldWithOptions(ctx context.Context, index, fiel fieldOpt.Max = &opt.Max case FieldTypeTime: fieldOpt.TimeQuantum = &opt.TimeQuantum + ttlString := opt.Ttl.String() + fieldOpt.Ttl = &ttlString case FieldTypeBool: // pass case FieldTypeDecimal: diff --git a/internal_client_test.go b/internal_client_test.go index 8d1531137..204a85cae 100644 --- a/internal_client_test.go +++ b/internal_client_test.go @@ -1175,6 +1175,46 @@ func TestClient_FragmentBlocks(t *testing.T) { } } +func TestClient_CreateTimeField(t *testing.T) { + cluster := test.MustRunCluster(t, 1) + defer cluster.Close() + cmd := cluster.GetNode(0) + + c := MustNewClient(cmd.URL(), pilosa.GetHTTPClient(nil)) + + index := "cdf" + err := c.CreateIndex(context.Background(), index, pilosa.IndexOptions{}) + if err != nil { + t.Fatalf("creating index: %v", err) + } + + field := "field" + err = c.CreateFieldWithOptions(context.Background(), index, field, pilosa.FieldOptions{Type: pilosa.FieldTypeTime, TimeQuantum: "YMDH"}) + if err != nil { + t.Fatalf("creating field: %v", err) + } + fld, err := cmd.API.Field(context.Background(), index, field) + if err != nil { + t.Fatalf("getting field: %v", err) + } + if fld.Ttl() != 0 { + t.Fatalf("expected Ttl to be 0, got: %+v", fld.Options().Ttl.String()) + } + + fieldTtl := "field_ttl" + err = c.CreateFieldWithOptions(context.Background(), index, fieldTtl, pilosa.FieldOptions{Type: pilosa.FieldTypeTime, TimeQuantum: "YMDH", Ttl: time.Hour}) + if err != nil { + t.Fatalf("creating field: %v", err) + } + fldTtl, err := cmd.API.Field(context.Background(), index, fieldTtl) + if err != nil { + t.Fatalf("getting field: %v", err) + } + if fldTtl.Ttl() != time.Hour { + t.Fatalf("expected Ttl 1 hour, got: %+v", fldTtl.Ttl().String()) + } +} + func TestClient_CreateDecimalField(t *testing.T) { cluster := test.MustRunCluster(t, 1) defer cluster.Close() diff --git a/pb/private.pb.go b/pb/private.pb.go index 3a2807420..21d27da12 100644 --- a/pb/private.pb.go +++ b/pb/private.pb.go @@ -93,6 +93,7 @@ type FieldOptions struct { Min *Decimal `protobuf:"bytes,17,opt,name=Min,proto3" json:"Min,omitempty"` Max *Decimal `protobuf:"bytes,18,opt,name=Max,proto3" json:"Max,omitempty"` TimeUnit string `protobuf:"bytes,19,opt,name=TimeUnit,proto3" json:"TimeUnit,omitempty"` + Ttl string `protobuf:"bytes,20,opt,name=Ttl,proto3" json:"Ttl,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -236,6 +237,13 @@ func (m *FieldOptions) GetTimeUnit() string { return "" } +func (m *FieldOptions) GetTtl() string { + if m != nil { + return m.Ttl + } + return "" +} + type ImportResponse struct { Err string `protobuf:"bytes,1,opt,name=Err,proto3" json:"Err,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` @@ -2759,110 +2767,111 @@ func init() { func init() { proto.RegisterFile("private.proto", fileDescriptor_d2a91b51c7bdc125) } var fileDescriptor_d2a91b51c7bdc125 = []byte{ - // 1639 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0xdf, 0x6e, 0x1b, 0x45, - 0x17, 0xff, 0x76, 0xd7, 0x8e, 0xed, 0xe3, 0x38, 0x71, 0xa6, 0xf9, 0xfa, 0x6d, 0xd2, 0x7e, 0x91, - 0x33, 0xa0, 0xd6, 0x44, 0x22, 0x88, 0xf4, 0xa2, 0x08, 0x6e, 0x9a, 0xd8, 0x69, 0x31, 0x25, 0x6d, - 0x3a, 0x49, 0x73, 0x09, 0x9a, 0xd8, 0xa3, 0x64, 0x95, 0xf5, 0xae, 0xd9, 0x5d, 0xa7, 0x76, 0x2f, - 0x90, 0x40, 0x20, 0xb8, 0xe1, 0x9e, 0x2b, 0x9e, 0x81, 0x1b, 0xde, 0x81, 0x1b, 0x24, 0x1e, 0x01, - 0x95, 0x17, 0x41, 0x73, 0x66, 0x66, 0x77, 0xed, 0x3a, 0x35, 0x44, 0xdc, 0xed, 0xf9, 0x9d, 0x99, - 0xf3, 0x7f, 0xce, 0x9c, 0x59, 0xa8, 0x0d, 0x22, 0xef, 0x92, 0x27, 0x62, 0x7b, 0x10, 0x85, 0x49, - 0x48, 0xec, 0xc1, 0xe9, 0xfa, 0xe2, 0x60, 0x78, 0xea, 0x7b, 0x5d, 0x85, 0xd0, 0x47, 0x50, 0xe9, - 0x04, 0x3d, 0x31, 0x3a, 0x10, 0x09, 0x27, 0x04, 0x0a, 0x8f, 0xc5, 0x38, 0x76, 0x9d, 0x86, 0xd5, - 0x2c, 0x33, 0xfc, 0x26, 0x77, 0x60, 0xe9, 0x38, 0xe2, 0xdd, 0x8b, 0xfd, 0x91, 0x17, 0x27, 0x22, - 0xe8, 0x0a, 0xb7, 0x80, 0xdc, 0x29, 0x94, 0xfe, 0xec, 0xc0, 0xe2, 0x43, 0x4f, 0xf8, 0xbd, 0xa7, - 0x83, 0xc4, 0x0b, 0x83, 0x58, 0x0a, 0x3b, 0x1e, 0x0f, 0x84, 0x5b, 0x6e, 0x58, 0xcd, 0x0a, 0xc3, - 0x6f, 0x72, 0x1b, 0x2a, 0x2d, 0xde, 0x3d, 0x17, 0xc8, 0x70, 0x90, 0x91, 0x01, 0x29, 0xf7, 0xc8, - 0x7b, 0xa9, 0xb4, 0xd4, 0x58, 0x06, 0x90, 0x06, 0x54, 0x8f, 0xbd, 0xbe, 0x78, 0x36, 0xe4, 0x41, - 0x32, 0xec, 0xbb, 0x45, 0xdc, 0x9d, 0x87, 0xc8, 0x4d, 0x58, 0x78, 0xea, 0xf7, 0x0e, 0xbc, 0xc0, - 0xad, 0x34, 0xac, 0xa6, 0xc3, 0x34, 0x65, 0x70, 0x3e, 0x72, 0x21, 0xc3, 0xf9, 0x28, 0x75, 0xb7, - 0x3a, 0xe9, 0xee, 0x93, 0xf0, 0x28, 0xe1, 0x41, 0x8f, 0x47, 0xbd, 0x13, 0x4f, 0xbc, 0x70, 0x17, - 0x95, 0xbb, 0x93, 0xa8, 0xdc, 0xbb, 0xc7, 0x63, 0xe1, 0xd6, 0x50, 0x22, 0x7e, 0x93, 0x75, 0x28, - 0xef, 0x79, 0x49, 0x5b, 0x0c, 0x92, 0x73, 0x77, 0xa9, 0x61, 0x35, 0x0b, 0x2c, 0xa5, 0xc9, 0x2a, - 0x14, 0x8f, 0xba, 0xdc, 0x17, 0xee, 0x32, 0x6e, 0x50, 0x04, 0xa1, 0xb0, 0xf8, 0x30, 0x8c, 0x84, - 0x77, 0x16, 0x60, 0x12, 0xdc, 0x3a, 0x3a, 0x35, 0x81, 0x91, 0xff, 0x83, 0x23, 0x5d, 0x5a, 0x69, - 0x58, 0xcd, 0xea, 0x4e, 0x75, 0x7b, 0x70, 0xba, 0xdd, 0x16, 0x5d, 0xaf, 0xcf, 0x7d, 0x26, 0x71, - 0x64, 0xf3, 0x91, 0x4b, 0x66, 0xb1, 0xf9, 0x48, 0xda, 0x24, 0x43, 0xf4, 0x3c, 0xf0, 0x12, 0xf7, - 0x06, 0x4a, 0x4f, 0x69, 0x4a, 0x61, 0xa9, 0xd3, 0x1f, 0x84, 0x51, 0xc2, 0x44, 0x3c, 0x08, 0x83, - 0x58, 0x90, 0x3a, 0x38, 0xfb, 0x51, 0xe4, 0x5a, 0xb8, 0x50, 0x7e, 0xd2, 0x2f, 0xa1, 0xbe, 0xe7, - 0x87, 0xdd, 0x8b, 0x36, 0x4f, 0x38, 0x13, 0x5f, 0x0c, 0x45, 0x9c, 0x48, 0x5f, 0x94, 0xb9, 0x6a, - 0x9d, 0x22, 0x24, 0x8a, 0xf9, 0x77, 0x6d, 0x85, 0x22, 0x21, 0xe3, 0x84, 0x51, 0x54, 0xe9, 0xc2, - 0x6f, 0x8c, 0xc5, 0x39, 0x8f, 0x7a, 0x98, 0xe3, 0x02, 0x53, 0x84, 0x44, 0x51, 0x13, 0xd6, 0x45, - 0x81, 0x29, 0x82, 0x76, 0x60, 0x25, 0xa7, 0x5f, 0x9b, 0x79, 0x13, 0x16, 0x58, 0xf8, 0xa2, 0xd3, - 0x8e, 0x5d, 0xab, 0xe1, 0x34, 0x0b, 0x4c, 0x53, 0x58, 0x40, 0xa1, 0x3f, 0xec, 0x07, 0x92, 0x65, - 0x23, 0x2b, 0x03, 0xe8, 0x1a, 0x14, 0xb1, 0x9a, 0xa4, 0x97, 0xd9, 0x5e, 0xf9, 0x49, 0xbf, 0xb2, - 0xa0, 0x72, 0xc0, 0x47, 0x68, 0x48, 0x4c, 0xee, 0x43, 0xd9, 0xe4, 0x1a, 0x17, 0x55, 0x77, 0x6e, - 0xc9, 0xb8, 0xa6, 0x0b, 0xb6, 0x0d, 0x77, 0x3f, 0x48, 0xa2, 0x31, 0x4b, 0x17, 0xaf, 0x7f, 0x04, - 0xb5, 0x09, 0x96, 0xd4, 0x74, 0x21, 0xc6, 0x26, 0x9e, 0x17, 0x62, 0x2c, 0xbd, 0xbc, 0xe4, 0xfe, - 0x50, 0x60, 0x94, 0x0a, 0x4c, 0x11, 0x1f, 0xda, 0x1f, 0x58, 0xf4, 0x04, 0x48, 0x2b, 0x12, 0x3c, - 0x11, 0xa8, 0xe4, 0x40, 0xc4, 0x31, 0x3f, 0x13, 0xf3, 0x62, 0xed, 0xe4, 0x63, 0x9d, 0xc6, 0xd5, - 0xce, 0xc5, 0x95, 0x6e, 0x01, 0x69, 0x0b, 0x5f, 0x24, 0x42, 0x9f, 0xf3, 0x37, 0xc8, 0xa5, 0x17, - 0xc6, 0x86, 0xf9, 0x6b, 0xc9, 0x26, 0x14, 0x64, 0xd3, 0x40, 0x65, 0xd5, 0x9d, 0x9a, 0x8c, 0x50, - 0xda, 0x49, 0x18, 0xb2, 0x30, 0x1f, 0x28, 0xae, 0xb7, 0x9b, 0xa0, 0xa9, 0x0e, 0xcb, 0x00, 0xfa, - 0x8d, 0x65, 0xb4, 0xa1, 0xf9, 0x7f, 0xd3, 0xe3, 0x89, 0xea, 0x7a, 0x5b, 0xdb, 0xe0, 0xa0, 0x0d, - 0x75, 0x69, 0x43, 0xbe, 0x07, 0xcd, 0x32, 0xa3, 0x30, 0x6d, 0xc6, 0x03, 0x13, 0x9f, 0xeb, 0x5a, - 0x41, 0xbb, 0x70, 0x4b, 0x49, 0xd8, 0xbd, 0xe4, 0x9e, 0xcf, 0x4f, 0xfd, 0x7f, 0x94, 0xc2, 0x09, - 0x87, 0x5c, 0x28, 0xe1, 0xde, 0x4e, 0x5b, 0x1f, 0x03, 0x43, 0xd2, 0x21, 0x64, 0x27, 0xea, 0x09, - 0xef, 0x0b, 0x2d, 0x0d, 0xbf, 0xd3, 0x38, 0xd8, 0x6f, 0x8c, 0xc3, 0x2a, 0x14, 0xe5, 0xf9, 0x93, - 0xfd, 0xdd, 0x91, 0x2a, 0x91, 0x98, 0x13, 0x9d, 0x77, 0x61, 0xe1, 0xa8, 0x7b, 0x2e, 0xfa, 0x9c, - 0xbc, 0x05, 0x25, 0xb4, 0x5c, 0xc4, 0xfa, 0x50, 0x54, 0xd2, 0x94, 0x33, 0xc3, 0xa1, 0xdf, 0x5a, - 0xda, 0xd9, 0x99, 0x66, 0x4e, 0xa8, 0xb2, 0xa7, 0x54, 0x91, 0xbb, 0x50, 0xd2, 0xf6, 0x62, 0xb7, - 0x78, 0xad, 0xa6, 0x0c, 0x97, 0x6c, 0xc2, 0x02, 0x7a, 0x17, 0xbb, 0x85, 0xcc, 0x10, 0x44, 0x98, - 0x66, 0xd0, 0x7d, 0x70, 0x9e, 0xb3, 0x8e, 0x6c, 0x14, 0x68, 0xbd, 0x31, 0x43, 0x53, 0xd2, 0xb8, - 0x8f, 0xc3, 0x38, 0xd1, 0xb1, 0xc7, 0x6f, 0x89, 0x1d, 0x86, 0x91, 0xaa, 0xd3, 0x1a, 0xc3, 0x6f, - 0xfa, 0xbd, 0x05, 0x85, 0x27, 0x61, 0x4f, 0x90, 0x25, 0xb0, 0x3b, 0x6d, 0x2d, 0xc4, 0xee, 0xb4, - 0xc9, 0x1a, 0xca, 0xd7, 0xf1, 0x2e, 0x49, 0xfd, 0xcf, 0x59, 0x87, 0xa1, 0xce, 0xdb, 0x50, 0xe9, - 0xc4, 0x87, 0x91, 0xd7, 0xe7, 0xd1, 0x58, 0xdf, 0xa4, 0x19, 0x80, 0x67, 0x34, 0xe1, 0x89, 0xba, - 0xdf, 0x2a, 0x4c, 0x11, 0x64, 0x13, 0x4a, 0x8f, 0xd8, 0x61, 0x4b, 0x8a, 0x2c, 0x4e, 0x8a, 0x34, - 0x38, 0x7d, 0x00, 0x75, 0x69, 0x09, 0xae, 0x37, 0x95, 0x75, 0x13, 0x16, 0x24, 0x96, 0x5a, 0xa6, - 0xa9, 0x4c, 0x89, 0x9d, 0x53, 0x42, 0x1f, 0x2a, 0x09, 0xfb, 0x97, 0x22, 0x48, 0x72, 0xb5, 0x89, - 0x34, 0x0a, 0xa8, 0x31, 0x45, 0x90, 0xdb, 0xca, 0x6b, 0xed, 0x5e, 0x59, 0xda, 0x22, 0x69, 0x86, - 0x28, 0x1d, 0x03, 0x18, 0x4b, 0x86, 0x71, 0xba, 0xd6, 0x9a, 0xb5, 0x96, 0x50, 0x53, 0x3e, 0xfa, - 0x88, 0x82, 0xe4, 0x2b, 0x84, 0x99, 0xc2, 0x7a, 0x27, 0x2b, 0x2c, 0x95, 0xcf, 0xe5, 0x34, 0xef, - 0x4a, 0x47, 0x56, 0x5e, 0xe7, 0x50, 0xcd, 0xe1, 0x33, 0x6b, 0xec, 0x6e, 0x5a, 0x1c, 0x76, 0x26, - 0x0c, 0x11, 0x2d, 0x4c, 0xb3, 0xe7, 0x34, 0x27, 0x0f, 0xaa, 0xb9, 0x4d, 0x33, 0x35, 0x35, 0x61, - 0x79, 0xf2, 0xc0, 0x9b, 0x3b, 0x67, 0x1a, 0x9e, 0xa3, 0xea, 0x3b, 0x0b, 0x6a, 0x2d, 0x7f, 0x18, - 0x27, 0x22, 0x4a, 0x63, 0x5a, 0xd1, 0x40, 0x9a, 0xda, 0x0c, 0x98, 0x9d, 0x5d, 0xb2, 0x01, 0x45, - 0x19, 0x71, 0x75, 0xb8, 0xf3, 0x89, 0x50, 0x70, 0x2e, 0x13, 0x85, 0xab, 0x32, 0x41, 0x4f, 0xa0, - 0xbc, 0x77, 0xd4, 0x79, 0x14, 0x85, 0xc3, 0xc1, 0x4c, 0x8f, 0xcd, 0x48, 0x67, 0xe7, 0x46, 0xba, - 0xba, 0x1a, 0x4f, 0x94, 0x57, 0x38, 0x91, 0xd4, 0xd5, 0x44, 0x52, 0xd0, 0x08, 0x1f, 0xd1, 0x23, - 0x58, 0x51, 0xee, 0xca, 0x8e, 0x73, 0x9d, 0xb6, 0x68, 0xa6, 0x08, 0x27, 0x9b, 0x22, 0xa4, 0x50, - 0xd5, 0x75, 0xff, 0x4d, 0xa1, 0xbf, 0xd9, 0xb0, 0xc2, 0x44, 0xec, 0xbd, 0x14, 0x9d, 0x20, 0x4e, - 0xa2, 0x61, 0x57, 0x76, 0x1c, 0xb9, 0xff, 0x93, 0xf0, 0x54, 0xe7, 0xc2, 0x61, 0x8a, 0x78, 0xf3, - 0x29, 0x21, 0x14, 0x4a, 0xf9, 0x26, 0x90, 0x5f, 0x60, 0x18, 0x64, 0x0b, 0x4a, 0x47, 0xe1, 0x30, - 0xea, 0xa6, 0x95, 0x8f, 0x9d, 0x5b, 0xe9, 0x57, 0x0c, 0x66, 0x16, 0x90, 0xc7, 0x40, 0x8e, 0x23, - 0x1e, 0xc4, 0x3e, 0x97, 0x26, 0x99, 0x6d, 0xe5, 0x6c, 0x3c, 0xc9, 0x71, 0x27, 0x24, 0xcc, 0xd8, - 0x46, 0xb6, 0xf3, 0x47, 0xd8, 0x2d, 0xa1, 0x7d, 0x4b, 0xc6, 0x3e, 0x7d, 0x4e, 0xf2, 0x87, 0xfc, - 0xfe, 0x54, 0x85, 0xba, 0x0b, 0xb8, 0x65, 0x45, 0x6e, 0x99, 0x60, 0xb0, 0xc9, 0x75, 0xf4, 0x6b, - 0x0b, 0x16, 0xf3, 0xd6, 0xcc, 0x69, 0x17, 0x69, 0xfa, 0xec, 0xf9, 0xd3, 0x8e, 0x49, 0x5f, 0x61, - 0xd6, 0x64, 0x59, 0xcc, 0x4f, 0x40, 0x21, 0xfc, 0xef, 0x8a, 0xe0, 0x5c, 0xcb, 0x9c, 0x06, 0x54, - 0x0f, 0x79, 0x94, 0x78, 0x52, 0x98, 0xbe, 0xa7, 0x8b, 0x2c, 0x0f, 0x51, 0x01, 0x6b, 0xaf, 0x15, - 0x51, 0x2b, 0xec, 0x0f, 0x64, 0xb5, 0x5e, 0xab, 0x98, 0x64, 0x9b, 0x8e, 0xa2, 0x30, 0x32, 0x11, - 0x40, 0x82, 0xee, 0x41, 0xf9, 0x38, 0x1c, 0x84, 0x7e, 0x78, 0x36, 0x9e, 0xd3, 0x32, 0x5c, 0x28, - 0xa9, 0xab, 0x41, 0xb5, 0xa8, 0x0a, 0x33, 0x24, 0xbd, 0x21, 0xeb, 0xbd, 0xcb, 0xfd, 0xee, 0xd0, - 0xe7, 0x89, 0xc0, 0xf9, 0x18, 0xc1, 0x4f, 0x43, 0xde, 0x53, 0x5d, 0x41, 0x1f, 0x2d, 0xfa, 0xb9, - 0x2e, 0x40, 0x8e, 0xee, 0xe4, 0xae, 0xa0, 0x5d, 0x04, 0xcc, 0x15, 0xa4, 0x28, 0xf2, 0x3e, 0x54, - 0x73, 0xab, 0xb5, 0x5b, 0xcb, 0x69, 0x9d, 0x2a, 0x98, 0xe5, 0xd7, 0xd0, 0x5f, 0xac, 0x89, 0x3d, - 0xaf, 0xdd, 0xb9, 0x5a, 0xd5, 0xa5, 0x0a, 0x52, 0x99, 0x69, 0x4a, 0xba, 0xbe, 0x3f, 0xea, 0xfa, - 0xc3, 0x58, 0xb2, 0xf4, 0x85, 0x9b, 0x02, 0xd2, 0x75, 0xf9, 0xe0, 0x09, 0x87, 0x66, 0xb8, 0x31, - 0xa4, 0x7c, 0x1a, 0xb5, 0x05, 0xef, 0xf9, 0x5e, 0x20, 0xb0, 0x5e, 0x1c, 0x96, 0xd2, 0x64, 0x4b, - 0xf5, 0x58, 0x53, 0xe8, 0xab, 0x53, 0x86, 0x23, 0x4f, 0x75, 0xde, 0x98, 0x12, 0xa8, 0x4f, 0xb3, - 0xe8, 0x2a, 0x10, 0x55, 0x01, 0xbb, 0xa7, 0x61, 0x64, 0x6e, 0x5b, 0xda, 0x32, 0xcd, 0x45, 0x46, - 0x7f, 0xde, 0x25, 0x9e, 0x45, 0xd6, 0xce, 0x47, 0x96, 0x7e, 0x06, 0x4b, 0x7a, 0xb6, 0x13, 0x11, - 0x16, 0xb4, 0x0c, 0x00, 0x13, 0xdd, 0x50, 0x8e, 0x89, 0xe6, 0x55, 0x93, 0x01, 0x52, 0xce, 0x89, - 0x7c, 0x64, 0x98, 0xdb, 0x49, 0x53, 0x38, 0x1b, 0x79, 0x67, 0x81, 0xe8, 0xe1, 0x8d, 0xe1, 0x30, - 0x4d, 0xd1, 0x1f, 0x6c, 0x58, 0x55, 0x43, 0x67, 0x70, 0x26, 0xe2, 0x24, 0x53, 0x23, 0x9f, 0xd1, - 0x03, 0xec, 0xff, 0xda, 0x50, 0x45, 0xc9, 0x27, 0x73, 0xcb, 0x17, 0x3c, 0xca, 0x6c, 0x50, 0x8a, - 0xa6, 0x50, 0x79, 0x6e, 0x10, 0xd1, 0xd7, 0xb3, 0x1a, 0x42, 0xf3, 0x10, 0xd9, 0x83, 0xb2, 0x76, - 0xcd, 0x34, 0xc4, 0x3b, 0x78, 0x4b, 0xcd, 0xb0, 0xc6, 0xcc, 0xb7, 0xb1, 0x7e, 0x83, 0x19, 0x72, - 0xfd, 0x29, 0xd4, 0x26, 0x58, 0x33, 0xde, 0x60, 0xcd, 0xfc, 0x1b, 0xac, 0xba, 0x43, 0x72, 0xe3, - 0xb2, 0x96, 0x9e, 0x7f, 0x97, 0xb5, 0xe0, 0xbf, 0xb3, 0x0c, 0x88, 0xc9, 0x16, 0x38, 0xd2, 0x50, - 0x35, 0x0c, 0xbb, 0x57, 0x19, 0xca, 0xe4, 0x22, 0xfa, 0x93, 0xa5, 0x83, 0x2a, 0x34, 0xdf, 0xbc, - 0xa5, 0xef, 0xe5, 0x85, 0x6c, 0xa6, 0x42, 0xa6, 0x96, 0x6d, 0xa7, 0x8e, 0xca, 0xd5, 0xeb, 0xcf, - 0xa0, 0x3c, 0xcb, 0xbd, 0x82, 0x72, 0xef, 0xbd, 0x49, 0xf7, 0xd6, 0xae, 0xb2, 0x2c, 0xce, 0x79, - 0xb9, 0x57, 0xff, 0xf5, 0xd5, 0x86, 0xf5, 0xfb, 0xab, 0x0d, 0xeb, 0x8f, 0x57, 0x1b, 0xd6, 0x8f, - 0x7f, 0x6e, 0xfc, 0xe7, 0x74, 0x01, 0x7f, 0x10, 0xdd, 0xfb, 0x2b, 0x00, 0x00, 0xff, 0xff, 0x38, - 0x55, 0x86, 0x89, 0x43, 0x12, 0x00, 0x00, + // 1650 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0xcf, 0x72, 0x1b, 0x45, + 0x13, 0xff, 0x76, 0x57, 0xb2, 0xa4, 0x96, 0x65, 0xcb, 0x13, 0x7f, 0xf9, 0xd6, 0x4e, 0x3e, 0x97, + 0x3c, 0x50, 0x89, 0x70, 0x15, 0xa6, 0x70, 0x0e, 0xa1, 0xe0, 0x12, 0x5b, 0x72, 0x82, 0x08, 0x4e, + 0x9c, 0xb1, 0xe3, 0x23, 0xd4, 0x58, 0x9a, 0xb2, 0xb7, 0xbc, 0xda, 0x15, 0xbb, 0x2b, 0x47, 0xca, + 0x81, 0x2a, 0x28, 0x28, 0xb8, 0x70, 0xe7, 0xc4, 0x5b, 0xf0, 0x02, 0x9c, 0xb8, 0x50, 0xc5, 0x23, + 0x50, 0xe1, 0x45, 0xa8, 0xe9, 0x99, 0xd9, 0x5d, 0x29, 0x72, 0x04, 0x2e, 0x6e, 0xdb, 0xbf, 0xee, + 0xe9, 0x7f, 0xd3, 0xd3, 0xd3, 0xb3, 0x50, 0x1b, 0x44, 0xde, 0x25, 0x4f, 0xc4, 0xf6, 0x20, 0x0a, + 0x93, 0x90, 0xd8, 0x83, 0xd3, 0xf5, 0xc5, 0xc1, 0xf0, 0xd4, 0xf7, 0xba, 0x0a, 0xa1, 0x8f, 0xa0, + 0xd2, 0x09, 0x7a, 0x62, 0x74, 0x20, 0x12, 0x4e, 0x08, 0x14, 0x1e, 0x8b, 0x71, 0xec, 0x3a, 0x0d, + 0xab, 0x59, 0x66, 0xf8, 0x4d, 0xee, 0xc0, 0xd2, 0x71, 0xc4, 0xbb, 0x17, 0xfb, 0x23, 0x2f, 0x4e, + 0x44, 0xd0, 0x15, 0x6e, 0x01, 0xb9, 0x53, 0x28, 0xfd, 0xc5, 0x81, 0xc5, 0x87, 0x9e, 0xf0, 0x7b, + 0x4f, 0x07, 0x89, 0x17, 0x06, 0xb1, 0x54, 0x76, 0x3c, 0x1e, 0x08, 0xb7, 0xdc, 0xb0, 0x9a, 0x15, + 0x86, 0xdf, 0xe4, 0x36, 0x54, 0x5a, 0xbc, 0x7b, 0x2e, 0x90, 0xe1, 0x20, 0x23, 0x03, 0x52, 0xee, + 0x91, 0xf7, 0x52, 0x59, 0xa9, 0xb1, 0x0c, 0x20, 0x0d, 0xa8, 0x1e, 0x7b, 0x7d, 0xf1, 0x6c, 0xc8, + 0x83, 0x64, 0xd8, 0x77, 0x8b, 0xb8, 0x3a, 0x0f, 0x91, 0x9b, 0xb0, 0xf0, 0xd4, 0xef, 0x1d, 0x78, + 0x81, 0x5b, 0x69, 0x58, 0x4d, 0x87, 0x69, 0xca, 0xe0, 0x7c, 0xe4, 0x42, 0x86, 0xf3, 0x51, 0x1a, + 0x6e, 0x75, 0x32, 0xdc, 0x27, 0xe1, 0x51, 0xc2, 0x83, 0x1e, 0x8f, 0x7a, 0x27, 0x9e, 0x78, 0xe1, + 0x2e, 0xaa, 0x70, 0x27, 0x51, 0xb9, 0x76, 0x8f, 0xc7, 0xc2, 0xad, 0xa1, 0x46, 0xfc, 0x26, 0xeb, + 0x50, 0xde, 0xf3, 0x92, 0xb6, 0x18, 0x24, 0xe7, 0xee, 0x52, 0xc3, 0x6a, 0x16, 0x58, 0x4a, 0x93, + 0x55, 0x28, 0x1e, 0x75, 0xb9, 0x2f, 0xdc, 0x65, 0x5c, 0xa0, 0x08, 0x42, 0x61, 0xf1, 0x61, 0x18, + 0x09, 0xef, 0x2c, 0xc0, 0x4d, 0x70, 0xeb, 0x18, 0xd4, 0x04, 0x46, 0xfe, 0x0f, 0x8e, 0x0c, 0x69, + 0xa5, 0x61, 0x35, 0xab, 0x3b, 0xd5, 0xed, 0xc1, 0xe9, 0x76, 0x5b, 0x74, 0xbd, 0x3e, 0xf7, 0x99, + 0xc4, 0x91, 0xcd, 0x47, 0x2e, 0x99, 0xc5, 0xe6, 0x23, 0xe9, 0x93, 0x4c, 0xd1, 0xf3, 0xc0, 0x4b, + 0xdc, 0x1b, 0xa8, 0x3d, 0xa5, 0x49, 0x1d, 0x9c, 0xe3, 0xc4, 0x77, 0x57, 0x11, 0x96, 0x9f, 0x94, + 0xc2, 0x52, 0xa7, 0x3f, 0x08, 0xa3, 0x84, 0x89, 0x78, 0x10, 0x06, 0xb1, 0x90, 0x32, 0xfb, 0x51, + 0xe4, 0x5a, 0x4a, 0x66, 0x3f, 0x8a, 0xe8, 0x97, 0x50, 0xdf, 0xf3, 0xc3, 0xee, 0x45, 0x9b, 0x27, + 0x9c, 0x89, 0x2f, 0x86, 0x22, 0x4e, 0x64, 0x74, 0x2a, 0x00, 0x25, 0xa7, 0x08, 0x89, 0x62, 0x45, + 0xb8, 0xb6, 0x42, 0x91, 0x90, 0x99, 0xc3, 0xbc, 0xaa, 0x0d, 0xc4, 0x6f, 0xcc, 0xce, 0x39, 0x8f, + 0x7a, 0xb8, 0xeb, 0x05, 0xa6, 0x08, 0x89, 0xa2, 0x25, 0xac, 0x94, 0x02, 0x53, 0x04, 0xed, 0xc0, + 0x4a, 0xce, 0xbe, 0x76, 0xf3, 0x26, 0x2c, 0xb0, 0xf0, 0x45, 0xa7, 0x1d, 0xbb, 0x56, 0xc3, 0x69, + 0x16, 0x98, 0xa6, 0xb0, 0xa4, 0x42, 0x7f, 0xd8, 0x0f, 0x24, 0xcb, 0x46, 0x56, 0x06, 0xd0, 0x35, + 0x28, 0x62, 0x7d, 0xc9, 0x28, 0xb3, 0xb5, 0xf2, 0x93, 0x7e, 0x65, 0x41, 0xe5, 0x80, 0x8f, 0xd0, + 0x91, 0x98, 0xdc, 0x87, 0xb2, 0xd9, 0x7d, 0x14, 0xaa, 0xee, 0xdc, 0x92, 0x99, 0x4e, 0x05, 0xb6, + 0x0d, 0x77, 0x3f, 0x48, 0xa2, 0x31, 0x4b, 0x85, 0xd7, 0x3f, 0x82, 0xda, 0x04, 0x4b, 0x5a, 0xba, + 0x10, 0x63, 0x93, 0xcf, 0x0b, 0x31, 0x96, 0x51, 0x5e, 0x72, 0x7f, 0x28, 0x30, 0x4b, 0x05, 0xa6, + 0x88, 0x0f, 0xed, 0x0f, 0x2c, 0x7a, 0x02, 0xa4, 0x15, 0x09, 0x9e, 0x08, 0x34, 0x72, 0x20, 0xe2, + 0x98, 0x9f, 0x89, 0x79, 0xb9, 0x76, 0xf2, 0xb9, 0x4e, 0xf3, 0x6a, 0xe7, 0xf2, 0x4a, 0xb7, 0x80, + 0xb4, 0x85, 0x2f, 0x12, 0xa1, 0x4f, 0xfe, 0x1b, 0xf4, 0xd2, 0x0b, 0xe3, 0xc3, 0x7c, 0x59, 0xb2, + 0x09, 0x05, 0xd9, 0x46, 0xd0, 0x58, 0x75, 0xa7, 0x26, 0x33, 0x94, 0xf6, 0x16, 0x86, 0x2c, 0xdc, + 0x0f, 0x54, 0xd7, 0xdb, 0x4d, 0xd0, 0x55, 0x87, 0x65, 0x00, 0xfd, 0xc6, 0x32, 0xd6, 0xd0, 0xfd, + 0xbf, 0x19, 0xf1, 0x44, 0x75, 0xbd, 0xad, 0x7d, 0x70, 0xd0, 0x87, 0xba, 0xf4, 0x21, 0xdf, 0x95, + 0x66, 0xb9, 0x51, 0x98, 0x76, 0xe3, 0x81, 0xc9, 0xcf, 0x75, 0xbd, 0xa0, 0x5d, 0xb8, 0xa5, 0x34, + 0xec, 0x5e, 0x72, 0xcf, 0xe7, 0xa7, 0xfe, 0x3f, 0xda, 0xc2, 0x89, 0x80, 0x5c, 0x28, 0xe1, 0xda, + 0x4e, 0x5b, 0x1f, 0x03, 0x43, 0xd2, 0x21, 0x64, 0x27, 0xea, 0x09, 0xef, 0x0b, 0xad, 0x0d, 0xbf, + 0xd3, 0x3c, 0xd8, 0x6f, 0xcc, 0xc3, 0x2a, 0x14, 0xe5, 0xf9, 0x93, 0x1d, 0xdf, 0x91, 0x26, 0x91, + 0x98, 0x93, 0x9d, 0x77, 0x61, 0xe1, 0xa8, 0x7b, 0x2e, 0xfa, 0x9c, 0xbc, 0x05, 0x25, 0xf4, 0x5c, + 0xc4, 0xfa, 0x50, 0x54, 0xd2, 0x2d, 0x67, 0x86, 0x43, 0xbf, 0xb5, 0x74, 0xb0, 0x33, 0xdd, 0x9c, + 0x30, 0x65, 0x4f, 0x99, 0x22, 0x77, 0xa1, 0xa4, 0xfd, 0xc5, 0x6e, 0xf1, 0x5a, 0x4d, 0x19, 0x2e, + 0xd9, 0x84, 0x05, 0x8c, 0x2e, 0x76, 0x0b, 0x99, 0x23, 0x88, 0x30, 0xcd, 0xa0, 0xfb, 0xe0, 0x3c, + 0x67, 0x1d, 0xd9, 0x28, 0xd0, 0x7b, 0xe3, 0x86, 0xa6, 0xa4, 0x73, 0x1f, 0x87, 0x71, 0xa2, 0x73, + 0x8f, 0xdf, 0x12, 0x3b, 0x0c, 0x23, 0x55, 0xa7, 0x35, 0x86, 0xdf, 0xf4, 0x7b, 0x0b, 0x0a, 0x4f, + 0xc2, 0x9e, 0x20, 0x4b, 0x60, 0x77, 0xda, 0x5a, 0x89, 0xdd, 0x69, 0x93, 0x35, 0xd4, 0xaf, 0xf3, + 0x5d, 0x92, 0xf6, 0x9f, 0xb3, 0x0e, 0x43, 0x9b, 0xb7, 0xa1, 0xd2, 0x89, 0x0f, 0x23, 0xaf, 0xcf, + 0xa3, 0xb1, 0xbe, 0x5b, 0x33, 0x00, 0xcf, 0x68, 0xc2, 0x13, 0x75, 0xe3, 0x55, 0x98, 0x22, 0xc8, + 0x26, 0x94, 0x1e, 0xb1, 0xc3, 0x96, 0x54, 0x59, 0x9c, 0x54, 0x69, 0x70, 0xfa, 0x00, 0xea, 0xd2, + 0x13, 0x94, 0x37, 0x95, 0x75, 0x13, 0x16, 0x24, 0x96, 0x7a, 0xa6, 0xa9, 0xcc, 0x88, 0x9d, 0x33, + 0x42, 0x1f, 0x2a, 0x0d, 0xfb, 0x97, 0x22, 0x48, 0x72, 0xb5, 0x89, 0x34, 0x2a, 0xa8, 0x31, 0x45, + 0x90, 0xdb, 0x2a, 0x6a, 0x1d, 0x5e, 0x59, 0xfa, 0x22, 0x69, 0x86, 0x28, 0x1d, 0x03, 0x18, 0x4f, + 0x86, 0x71, 0x2a, 0x6b, 0xcd, 0x92, 0x25, 0xd4, 0x94, 0x8f, 0x3e, 0xa2, 0x20, 0xf9, 0x0a, 0x61, + 0xa6, 0xb0, 0xde, 0xc9, 0x0a, 0x4b, 0xed, 0xe7, 0x72, 0xba, 0xef, 0xca, 0x46, 0x56, 0x5e, 0xe7, + 0x50, 0xcd, 0xe1, 0x33, 0x6b, 0xec, 0x6e, 0x5a, 0x1c, 0x76, 0xa6, 0x0c, 0x11, 0xad, 0x4c, 0xb3, + 0xe7, 0x34, 0x27, 0x0f, 0xaa, 0xb9, 0x45, 0x33, 0x2d, 0x35, 0x61, 0x79, 0xf2, 0xc0, 0x9b, 0x3b, + 0x67, 0x1a, 0x9e, 0x63, 0xea, 0x3b, 0x0b, 0x6a, 0x2d, 0x7f, 0x18, 0x27, 0x22, 0x4a, 0x73, 0x5a, + 0xd1, 0x40, 0xba, 0xb5, 0x19, 0x30, 0x7b, 0x77, 0xc9, 0x06, 0x14, 0x65, 0xc6, 0xd5, 0xe1, 0xce, + 0x6f, 0x84, 0x82, 0x73, 0x3b, 0x51, 0xb8, 0x6a, 0x27, 0xe8, 0x09, 0x94, 0xf7, 0x8e, 0x3a, 0x8f, + 0xa2, 0x70, 0x38, 0x98, 0x19, 0xb1, 0x19, 0xf2, 0xec, 0xdc, 0x90, 0x57, 0x57, 0x03, 0x8b, 0x8a, + 0x0a, 0x67, 0x94, 0xba, 0x9a, 0x51, 0x0a, 0x1a, 0xe1, 0x23, 0x7a, 0x04, 0x2b, 0x2a, 0x5c, 0xd9, + 0x71, 0xae, 0xd3, 0x16, 0xcd, 0x14, 0xe1, 0x64, 0x53, 0x84, 0x54, 0xaa, 0xba, 0xee, 0xbf, 0xa9, + 0xf4, 0x37, 0x1b, 0x56, 0x98, 0x88, 0xbd, 0x97, 0xa2, 0x13, 0xc4, 0x49, 0x34, 0xec, 0xca, 0x8e, + 0x23, 0xd7, 0x7f, 0x12, 0x9e, 0xea, 0xbd, 0x70, 0x98, 0x22, 0xde, 0x7c, 0x4a, 0x08, 0x85, 0x52, + 0xbe, 0x09, 0xe4, 0x05, 0x0c, 0x83, 0x6c, 0x41, 0xe9, 0x28, 0x1c, 0x46, 0xdd, 0xb4, 0xf2, 0xb1, + 0x73, 0x2b, 0xfb, 0x8a, 0xc1, 0x8c, 0x00, 0x79, 0x0c, 0xe4, 0x38, 0xe2, 0x41, 0xec, 0x73, 0xe9, + 0x92, 0x59, 0x56, 0xce, 0xc6, 0x93, 0x1c, 0x77, 0x42, 0xc3, 0x8c, 0x65, 0x64, 0x3b, 0x7f, 0x84, + 0xdd, 0x12, 0xfa, 0xb7, 0x64, 0xfc, 0xd3, 0xe7, 0x24, 0x7f, 0xc8, 0xef, 0x4f, 0x55, 0xa8, 0xbb, + 0x80, 0x4b, 0x56, 0xe4, 0x92, 0x09, 0x06, 0x9b, 0x94, 0xa3, 0x5f, 0x5b, 0xb0, 0x98, 0xf7, 0x66, + 0x4e, 0xbb, 0x48, 0xb7, 0xcf, 0x9e, 0x3f, 0xed, 0x98, 0xed, 0x2b, 0xcc, 0x9a, 0x2c, 0x8b, 0xf9, + 0x09, 0x28, 0x84, 0xff, 0x5d, 0x91, 0x9c, 0x6b, 0xb9, 0xd3, 0x80, 0xea, 0x21, 0x8f, 0x12, 0x4f, + 0x2a, 0xd3, 0xf7, 0x74, 0x91, 0xe5, 0x21, 0x2a, 0x60, 0xed, 0xb5, 0x22, 0x6a, 0x85, 0xfd, 0x81, + 0xac, 0xd6, 0x6b, 0x15, 0x93, 0x6c, 0xd3, 0x51, 0x14, 0x46, 0x26, 0x03, 0x48, 0xd0, 0x3d, 0x28, + 0x1f, 0x87, 0x83, 0xd0, 0x0f, 0xcf, 0xc6, 0x73, 0x5a, 0x86, 0x0b, 0x25, 0x75, 0x35, 0xa8, 0x16, + 0x55, 0x61, 0x86, 0xa4, 0x37, 0x64, 0xbd, 0x77, 0xb9, 0xdf, 0x1d, 0xfa, 0x3c, 0x11, 0x38, 0x1f, + 0x23, 0xf8, 0x69, 0xc8, 0x7b, 0xaa, 0x2b, 0xe8, 0xa3, 0x45, 0x3f, 0xd7, 0x05, 0xc8, 0x31, 0x9c, + 0xdc, 0x15, 0xb4, 0x8b, 0x80, 0xb9, 0x82, 0x14, 0x45, 0xde, 0x87, 0x6a, 0x4e, 0x5a, 0x87, 0xb5, + 0x9c, 0xd6, 0xa9, 0x82, 0x59, 0x5e, 0x86, 0xfe, 0x6c, 0x4d, 0xac, 0x79, 0xed, 0xce, 0xd5, 0xa6, + 0x2e, 0x55, 0x92, 0xca, 0x4c, 0x53, 0x32, 0xf4, 0xfd, 0x51, 0xd7, 0x1f, 0xc6, 0x92, 0xa5, 0x2f, + 0xdc, 0x14, 0x90, 0xa1, 0xcb, 0x27, 0x50, 0x38, 0x34, 0xc3, 0x8d, 0x21, 0xe5, 0x63, 0xa9, 0x2d, + 0x78, 0xcf, 0xf7, 0x02, 0x81, 0xf5, 0xe2, 0xb0, 0x94, 0x26, 0x5b, 0xaa, 0xc7, 0x9a, 0x42, 0x5f, + 0x9d, 0x72, 0x1c, 0x79, 0xaa, 0xf3, 0xc6, 0x94, 0x40, 0x7d, 0x9a, 0x45, 0x57, 0x81, 0xa8, 0x0a, + 0xd8, 0x3d, 0x0d, 0x23, 0x73, 0xdb, 0xd2, 0x96, 0x69, 0x2e, 0x32, 0xfb, 0xf3, 0x2e, 0xf1, 0x2c, + 0xb3, 0x76, 0x3e, 0xb3, 0xf4, 0x33, 0x58, 0xd2, 0xb3, 0x9d, 0x88, 0xb0, 0xa0, 0x65, 0x02, 0x98, + 0xe8, 0x86, 0x72, 0x4c, 0x34, 0xaf, 0x9a, 0x0c, 0x90, 0x7a, 0x4e, 0xe4, 0x23, 0xc3, 0xdc, 0x4e, + 0x9a, 0xc2, 0xd9, 0xc8, 0x3b, 0x0b, 0x44, 0x0f, 0x6f, 0x0c, 0x87, 0x69, 0x8a, 0xfe, 0x60, 0xc3, + 0xaa, 0x1a, 0x3a, 0x83, 0x33, 0x11, 0x27, 0x99, 0x19, 0xf9, 0xb0, 0x1e, 0x60, 0xff, 0xd7, 0x8e, + 0x2a, 0x4a, 0x3e, 0xa2, 0x5b, 0xbe, 0xe0, 0x51, 0xe6, 0x83, 0x32, 0x34, 0x85, 0xca, 0x73, 0x83, + 0x88, 0xbe, 0x9e, 0xd5, 0x10, 0x9a, 0x87, 0xc8, 0x1e, 0x94, 0x75, 0x68, 0xa6, 0x21, 0xde, 0xc1, + 0x5b, 0x6a, 0x86, 0x37, 0x66, 0xbe, 0x8d, 0xf5, 0x1b, 0xcc, 0x90, 0xeb, 0x4f, 0xa1, 0x36, 0xc1, + 0x9a, 0xf1, 0x06, 0x6b, 0xe6, 0xdf, 0x60, 0xd5, 0x1d, 0x92, 0x1b, 0x97, 0xb5, 0xf6, 0xfc, 0xbb, + 0xac, 0x05, 0xff, 0x9d, 0xe5, 0x40, 0x4c, 0xb6, 0xc0, 0x91, 0x8e, 0xaa, 0x61, 0xd8, 0xbd, 0xca, + 0x51, 0x26, 0x85, 0xe8, 0x4f, 0x96, 0x4e, 0xaa, 0xd0, 0x7c, 0xf3, 0x96, 0xbe, 0x97, 0x57, 0xb2, + 0x99, 0x2a, 0x99, 0x12, 0xdb, 0x4e, 0x03, 0x95, 0xd2, 0xeb, 0xcf, 0xa0, 0x3c, 0x2b, 0xbc, 0x82, + 0x0a, 0xef, 0xbd, 0xc9, 0xf0, 0xd6, 0xae, 0xf2, 0x2c, 0xce, 0x45, 0xb9, 0x57, 0xff, 0xf5, 0xd5, + 0x86, 0xf5, 0xfb, 0xab, 0x0d, 0xeb, 0x8f, 0x57, 0x1b, 0xd6, 0x8f, 0x7f, 0x6e, 0xfc, 0xe7, 0x74, + 0x01, 0x7f, 0x19, 0xdd, 0xfb, 0x2b, 0x00, 0x00, 0xff, 0xff, 0xcf, 0xdb, 0x63, 0xcd, 0x55, 0x12, + 0x00, 0x00, } func (m *IndexMeta) Marshal() (dAtA []byte, err error) { @@ -2936,6 +2945,15 @@ func (m *FieldOptions) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if len(m.Ttl) > 0 { + i -= len(m.Ttl) + copy(dAtA[i:], m.Ttl) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Ttl))) + i-- + dAtA[i] = 0x1 + i-- + dAtA[i] = 0xa2 + } if len(m.TimeUnit) > 0 { i -= len(m.TimeUnit) copy(dAtA[i:], m.TimeUnit) @@ -5274,6 +5292,10 @@ func (m *FieldOptions) Size() (n int) { if l > 0 { n += 2 + l + sovPrivate(uint64(l)) } + l = len(m.Ttl) + if l > 0 { + n += 2 + l + sovPrivate(uint64(l)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -6765,6 +6787,38 @@ func (m *FieldOptions) Unmarshal(dAtA []byte) error { } m.TimeUnit = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 20: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Ttl", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Ttl = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) diff --git a/pb/private.proto b/pb/private.proto index 9f15939c7..9c4c37f25 100644 --- a/pb/private.proto +++ b/pb/private.proto @@ -25,6 +25,7 @@ message FieldOptions { Decimal Min = 17; Decimal Max = 18; string TimeUnit = 19; + string Ttl = 20; } message ImportResponse { diff --git a/server.go b/server.go index b58b1b714..d68b0c40d 100644 --- a/server.go +++ b/server.go @@ -76,6 +76,7 @@ type Server struct { // nolint: maligned antiEntropyInterval time.Duration metricInterval time.Duration diagnosticInterval time.Duration + ttlRemovalInterval time.Duration maxWritesPerRequest int confirmDownSleep time.Duration confirmDownRetries int @@ -167,6 +168,15 @@ func OptServerAntiEntropyInterval(interval time.Duration) ServerOption { } } +// OptServerTtlRemovalInterval is a functional option on Server +// used to set the ttl removal interval. +func OptServerTtlRemovalInterval(interval time.Duration) ServerOption { + return func(s *Server) error { + s.ttlRemovalInterval = interval + return nil + } +} + // OptServerLongQueryTime is a functional option on Server // used to set long query duration. func OptServerLongQueryTime(dur time.Duration) ServerOption { @@ -433,6 +443,7 @@ func NewServer(opts ...ServerOption) (*Server, error) { antiEntropyInterval: 0, metricInterval: 0, diagnosticInterval: 0, + ttlRemovalInterval: time.Hour, disCo: disco.NopDisCo, stator: disco.NopStator, @@ -641,12 +652,13 @@ func (s *Server) Open() error { return errors.Wrap(err, "setting nodeState") } - if ok := s.addToWaitGroup(3); !ok { + if ok := s.addToWaitGroup(4); !ok { return fmt.Errorf("closing server while opening server is NOT allowed") } go func() { defer s.wg.Done(); s.monitorAntiEntropy() }() go func() { defer s.wg.Done(); s.monitorRuntime() }() go func() { defer s.wg.Done(); s.monitorDiagnostics() }() + go func() { defer s.wg.Done(); s.monitorTtl() }() toSend := func() []Message { s.holder.startMsgsMu.Lock() @@ -830,6 +842,53 @@ func (s *Server) monitorResetTranslationSync() { } } +func (s *Server) monitorTtl() { + ctx := context.Background() + ticker := time.NewTicker(s.ttlRemovalInterval) + for { + select { + case <-s.closing: + return + case <-ticker.C: + s.TtlRemoval(ctx) + } + } +} + +func (s *Server) TtlRemoval(ctx context.Context) { + for _, index := range s.holder.Indexes() { + for _, field := range index.Fields() { + if field.Options().Type == "time" { + if field.Options().Ttl > 0 { + for _, view := range field.views() { + viewNames := strings.Split(view.name, "_") + if len(viewNames) >= 2 { + viewTime, err := timeOfView(view.name, false) + if err != nil { + s.logger.Printf("ttl parse view time: %s", err) + continue + } + timeSince := time.Since(viewTime) + + if timeSince >= field.Options().Ttl { + for _, shard := range field.AvailableShards(true).Slice() { + s.holder.txf.DeleteFragmentFromStore(index.Name(), field.Name(), view.name, shard, nil) + } + + err := s.defaultClient.api.DeleteView(ctx, index.Name(), field.Name(), view.name) + if err != nil { + s.logger.Errorf("ttl delete view: %s", err) + } + s.logger.Infof("ttl deleted view: %s", view.name) + } + } + } + } + } + } + } +} + func (s *Server) monitorAntiEntropy() { if s.antiEntropyInterval == 0 || s.cluster.ReplicaN <= 1 { return // anti entropy disabled diff --git a/server/handler_test.go b/server/handler_test.go index 92d500238..2b17d7bd3 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -274,7 +274,7 @@ func TestHandler_Endpoints(t *testing.T) { } } - if f, err := i2.CreateFieldIfNotExists("f3", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"))); err != nil { + if f, err := i2.CreateFieldIfNotExists("f3", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"), "0")); err != nil { t.Fatal(err) } else if _, err := f.SetBit(tx2, 0, 0, nil); err != nil { t.Fatal(err) diff --git a/server/pg_test.go b/server/pg_test.go index 7250c6f7e..53f177191 100644 --- a/server/pg_test.go +++ b/server/pg_test.go @@ -29,7 +29,7 @@ func TestPostgresHandler(t *testing.T) { m.MustCreateField(t, "i", "keymutex", pilosa.OptFieldKeys(), pilosa.OptFieldTypeMutex(pilosa.CacheTypeNone, 0)) m.MustCreateField(t, "i", "int", pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64)) m.MustCreateField(t, "i", "decimal", pilosa.OptFieldTypeDecimal(2)) - m.MustCreateField(t, "i", "time", pilosa.OptFieldTypeTime("YMDH")) + m.MustCreateField(t, "i", "time", pilosa.OptFieldTypeTime("YMDH", "0")) m.MustCreateField(t, "i", "bool", pilosa.OptFieldTypeBool()) m.MustCreateIndex(t, "j", pilosa.IndexOptions{TrackExistence: true, Keys: true}) diff --git a/server/server_test.go b/server/server_test.go index c1efbc8a7..093e919b5 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -760,7 +760,7 @@ func TestMain_ImportTimestamp(t *testing.T) { } // Create field. - if _, err := m.API.CreateField(context.Background(), indexName, fieldName, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMD"))); err != nil { + if _, err := m.API.CreateField(context.Background(), indexName, fieldName, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMD"), "0")); err != nil { t.Fatal(err) } @@ -815,7 +815,7 @@ func TestMain_ImportTimestampNoStandardView(t *testing.T) { } // Create field. - if _, err := m.API.CreateField(context.Background(), indexName, fieldName, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMD"), true)); err != nil { + if _, err := m.API.CreateField(context.Background(), indexName, fieldName, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMD"), "0", true)); err != nil { t.Fatal(err) } diff --git a/server_test.go b/server_test.go new file mode 100644 index 000000000..487a39286 --- /dev/null +++ b/server_test.go @@ -0,0 +1,95 @@ +// Copyright 2022 Molecula Corp. All rights reserved. +package pilosa_test + +import ( + "context" + "fmt" + "reflect" + "sort" + "testing" + "time" + + pilosa "github.com/molecula/featurebase/v3" + "github.com/molecula/featurebase/v3/test" +) + +func TestTtlRemoval(t *testing.T) { + + cluster := test.MustRunCluster(t, 1) + node := cluster.GetNode(0) + defer cluster.Close() + + // Create a client + client := node.Client() + + indexName := "i" + fieldName := "f" + + // Create indexes and field with ttl lasting 24 hours + if err := client.CreateIndex(context.Background(), indexName, pilosa.IndexOptions{TrackExistence: true}); err != nil && err != pilosa.ErrIndexExists { + t.Fatalf("creating index, err: %v", err) + } else if err := client.CreateFieldWithOptions(context.Background(), indexName, fieldName, pilosa.FieldOptions{Ttl: time.Hour * 24, Type: pilosa.FieldTypeTime, TimeQuantum: "YMDH"}); err != nil { + t.Fatalf("creating field, err: %v", err) + } + + /* Set sample data 1 using this date: '2001-02-03T04:05', this will create these views: + - standard + - standard_2001 + - standard_200102 + - standard_20010203 + - standard_2001020304 + Since the sample date here is over 24 hours, all views except "standard" should get deleted + */ + _, err := client.Query(context.Background(), indexName, &pilosa.QueryRequest{Index: indexName, Query: "Set(1, f=1, 2001-02-03T04:05)"}) + if err != nil { + t.Fatalf("setting sample data 1, err: %v", err) + } + + dateNow := time.Now() + dateNowString := fmt.Sprintf("%d-%02d-%02dT%02d:%02d", dateNow.Year(), dateNow.Month(), dateNow.Day(), dateNow.Hour(), dateNow.Minute()) + /* Set sample data 2 using current time. + For example: current time is 2022-03-03T15:17 (also when the 24 hrs ttl countdown starts) will generate these views: + - standard_2022 -> gets converted to 2022_01_01, over 24 hours for ttl -> deleted + - standard_202203 -> gets converted to 2022_03_01, over 24 hours for ttl -> deleted + - standard_20220303 -> gets converted to 2022_03_03, within 24 hours -> keep + - standard_2022030315 -> gets converted to 2022_03_03 15:00, within 24 hours -> keep + */ + _, err = client.Query(context.Background(), indexName, &pilosa.QueryRequest{Index: indexName, Query: "Set(2, f=2, " + dateNowString + ")"}) + if err != nil { + t.Fatalf("setting sample data 2, err: %v", err) + } + + /* Set sample data 3 using yesterday's date + All views generated from this date should be deleted + */ + dateYesterday := time.Now().Add(-24*time.Hour + -1*time.Nanosecond) + dateYesterdayString := fmt.Sprintf("%d-%02d-%02dT%02d:%02d", dateYesterday.Year(), dateYesterday.Month(), dateYesterday.Day(), dateYesterday.Hour(), dateYesterday.Minute()) + _, err = client.Query(context.Background(), indexName, &pilosa.QueryRequest{Index: indexName, Query: "Set(3, f=3, " + dateYesterdayString + ")"}) + if err != nil { + t.Fatalf("setting sample data 3, err: %v", err) + } + + node.Server.TtlRemoval(context.Background()) + + // Get all the views for given index + field + views, err := node.API.Views(context.Background(), indexName, fieldName) + if err != nil { + t.Fatal(err) + } + + expectedViewNames := []string{ + "standard", + "standard_" + fmt.Sprintf("%d%02d%02d", dateNow.Year(), dateNow.Month(), dateNow.Day()), + "standard_" + fmt.Sprintf("%d%02d%02d%02d", dateNow.Year(), dateNow.Month(), dateNow.Day(), dateNow.Hour()), + } + + var viewNames []string + for _, view := range views { + viewNames = append(viewNames, view.Name()) + } + sort.Strings(viewNames) + + if !reflect.DeepEqual(expectedViewNames, viewNames) { + t.Fatalf("after ttl removal, expected %v, but got %v", expectedViewNames, viewNames) + } +} diff --git a/time.go b/time.go index 1c479c996..945d87106 100644 --- a/time.go +++ b/time.go @@ -450,7 +450,7 @@ func timeOfView(v string, adj bool) (time.Time, error) { return time.Time{}, nil } - layout := "2006010203" + layout := "2006010215" timePart := viewTimePart(v) switch len(timePart) { diff --git a/view.go b/view.go index d5e408810..927490a9f 100644 --- a/view.go +++ b/view.go @@ -307,6 +307,10 @@ func (v *view) recalculateCaches() { } } +func (v *view) Name() string { + return v.name +} + // CreateFragmentIfNotExists returns a fragment in the view by shard. func (v *view) CreateFragmentIfNotExists(shard uint64) (*fragment, error) { v.mu.Lock()