From e755fecf637cf81a51f6b66b70ccaedec0a83614 Mon Sep 17 00:00:00 2001 From: Pat Okeeffe <85502298+paddyjok@users.noreply.github.com> Date: Fri, 17 Feb 2023 13:35:36 -0600 Subject: [PATCH 01/19] ORDER BY ....what now!? (fb-1954) (#2257) * can now order by columns not in the select list * added testing coverage --- sql3/planner/compileselect.go | 171 ++++++++++++++++++++------ sql3/planner/expression.go | 52 ++++++-- sql3/planner/expressionanalyzer.go | 64 +++++----- sql3/planner/expressiontypes.go | 12 +- sql3/planner/inbuiltfunctionstable.go | 2 +- sql3/planner/oporderby.go | 58 +++++++-- sql3/planner/planoptimizer.go | 41 +++++- sql3/test/defs/defs_orderby.go | 130 +++++++++++++++++++- 8 files changed, 434 insertions(+), 96 deletions(-) diff --git a/sql3/planner/compileselect.go b/sql3/planner/compileselect.go index 9a03bda0f..940aff215 100644 --- a/sql3/planner/compileselect.go +++ b/sql3/planner/compileselect.go @@ -11,7 +11,6 @@ import ( "github.com/featurebasedb/featurebase/v3/sql3" "github.com/featurebasedb/featurebase/v3/sql3/parser" "github.com/featurebasedb/featurebase/v3/sql3/planner/types" - "github.com/pkg/errors" ) // compileSelectStatment compiles a parser.SelectStatment AST into a PlanOperator @@ -20,12 +19,12 @@ func (p *ExecutionPlanner) compileSelectStatement(stmt *parser.SelectStatement, aggregates := make([]types.PlanExpression, 0) - // handle projections + // compile select list and generate a list of projections projections := make([]types.PlanExpression, 0) for _, c := range stmt.Columns { planExpr, err := p.compileExpr(c.Expr) if err != nil { - return nil, errors.Wrap(err, "planning select column expression") + return nil, err } if c.Alias != nil { planExpr = newAliasPlanExpression(c.Alias.Name, planExpr) @@ -34,7 +33,7 @@ func (p *ExecutionPlanner) compileSelectStatement(stmt *parser.SelectStatement, aggregates = p.gatherExprAggregates(planExpr, aggregates) } - // group by clause. + // compile group by clause and generate a list of group by expressions groupByExprs := make([]types.PlanExpression, 0) for _, expr := range stmt.GroupByExprs { switch expr := expr.(type) { @@ -44,32 +43,32 @@ func (p *ExecutionPlanner) compileSelectStatement(stmt *parser.SelectStatement, return nil, sql3.NewErrInternalf("unsupported expression type in GROUP BY clause: %T", expr) } } - var err error - // handle the where clause + // compile the where clause where, err := p.compileExpr(stmt.WhereExpr) if err != nil { return nil, err } - // source expression + // compile source expression source, err := p.compileSource(query, stmt.Source) if err != nil { return nil, err } - // if we did have a where, insert the filter op + // if we did have a where, insert the filter op after source if where != nil { aggregates = p.gatherExprAggregates(where, aggregates) source = NewPlanOpFilter(p, where, source) } - // handle the having clause + // compile the having clause having, err := p.compileExpr(stmt.HavingExpr) if err != nil { return nil, err } + // if we have a having, check references if having != nil { // gather aggregates aggregates = p.gatherExprAggregates(having, aggregates) @@ -129,9 +128,49 @@ func (p *ExecutionPlanner) compileSelectStatement(stmt *parser.SelectStatement, } } - // do we have straight projection or a group by? + // compile order by and generate a list of ordering expressions + orderByExprs := make([]*OrderByExpression, 0) + nonReferenceOrderByExpressions := make([]types.PlanExpression, 0) + if len(stmt.OrderingTerms) > 0 { + for _, ot := range stmt.OrderingTerms { + // compile the ordering term + expr, err := p.compileOrderingTermExpr(ot.X, projections, stmt.Source) + if err != nil { + return nil, err + } + + f := &OrderByExpression{ + Expr: expr, + } + f.Order = orderByAsc + if ot.Desc.IsValid() { + f.Order = orderByDesc + } + orderByExprs = append(orderByExprs, f) + } + + // if the expression is just references, we + // can put the sort directly after the source + for _, oe := range orderByExprs { + _, ok := oe.Expr.(*qualifiedRefPlanExpression) + if !ok { + nonReferenceOrderByExpressions = append(nonReferenceOrderByExpressions, oe.Expr) + } + } + + // all the order by expressions are references, so we can put the order by before the + // projection + if len(nonReferenceOrderByExpressions) == 0 { + source = NewPlanOpOrderBy(orderByExprs, source) + } + } + var compiledOp types.PlanOperator + + // do we have straight projection or a group by? if len(aggregates) > 0 { + // we have a group by + //check that any projections that are not aggregates are in the group by list nonAggregateReferences := make([]*qualifiedRefPlanExpression, 0) for _, expr := range projections { @@ -172,37 +211,98 @@ func (p *ExecutionPlanner) compileSelectStatement(stmt *parser.SelectStatement, } compiledOp = NewPlanOpProjection(projections, groupByOp) } else { + // no group by, just a straight projection compiledOp = NewPlanOpProjection(projections, source) } - // handle order by - if len(stmt.OrderingTerms) > 0 { - orderByFields := make([]*OrderByExpression, 0) - for _, ot := range stmt.OrderingTerms { - index, err := p.compileOrderingTermExpr(ot.X) - if err != nil { - return nil, err - } - // get the data type from the projection - projDataType := projections[index].Type() + // handle the case where we have order by expressions and they are not references + // in this case we need to put the order by after the projection + if len(orderByExprs) > 0 && len(nonReferenceOrderByExpressions) > 0 { - // don't let a sort happen on something unsortable right now - switch projDataType.(type) { - case *parser.DataTypeStringSet, *parser.DataTypeIDSet: - return nil, sql3.NewErrExpectedSortableExpression(0, 0, projDataType.TypeDescription()) - } + // if the order by expressions contain a reference not in the projection list, + // we have to create a new projection, add references to current projection, + // and place the new order by in between - f := &OrderByExpression{ - Index: index, - ExprType: projDataType, + // get a list of all the refs for the order by exprs + orderByRefs := make(map[string]*qualifiedRefPlanExpression) + for _, oe := range orderByExprs { + ex, ok := oe.Expr.(*qualifiedRefPlanExpression) + if ok { + orderByRefs[ex.String()] = ex } - f.Order = orderByAsc - if ot.Desc.IsValid() { - f.Order = orderByDesc - } - orderByFields = append(orderByFields, f) } - compiledOp = NewPlanOpOrderBy(orderByFields, compiledOp) + + // get a list of all the projection refs + projRefs := make(map[string]*qualifiedRefPlanExpression) + for _, p := range projections { + InspectExpression(p, func(expr types.PlanExpression) bool { + switch ex := expr.(type) { + case *qualifiedRefPlanExpression: + projRefs[ex.String()] = ex + return false + } + return true + }) + } + + // iterate the order by terms, make a list of the ones not projected + unprojectedRefs := make([]*qualifiedRefPlanExpression, 0) + for kobr, obr := range orderByRefs { + _, found := projRefs[kobr] + if !found { + unprojectedRefs = append(unprojectedRefs, obr) + } + } + + // sigh - ok. If we have unprojected refs, we need to insert a projection + if len(unprojectedRefs) > 0 { + // create the final projection list - this will go before the order by + newProjections := make([]types.PlanExpression, len(projections)) + for i, p := range projections { + switch pe := p.(type) { + case *aliasPlanExpression: + newProjections[i] = newQualifiedRefPlanExpression("", pe.aliasName, i, pe.Type()) + case *qualifiedRefPlanExpression: + newProjections[i] = newQualifiedRefPlanExpression(pe.tableName, pe.columnName, i, pe.Type()) + default: + newProjections[i] = newQualifiedRefPlanExpression("", p.String(), i, p.Type()) + } + } + + // add the unprojected refs to the existing projection op + projectionOp, ok := compiledOp.(*PlanOpProjection) + if !ok { + return nil, sql3.NewErrInternalf("unexpected compiledOp type '%T'", compiledOp) + } + for _, uref := range unprojectedRefs { + projectionOp.Projections = append(projectionOp.Projections, uref) + } + + // add the order by on top of this + // rewrite all the order by expressions that are not qualified refs to be qualified + // refs referring to the expression + for i, oe := range orderByExprs { + _, ok := oe.Expr.(*qualifiedRefPlanExpression) + if !ok { + orderByExprs[i].Expr = newQualifiedRefPlanExpression("", oe.Expr.String(), 0, oe.Expr.Type()) + } + } + compiledOp = NewPlanOpOrderBy(orderByExprs, compiledOp) + + // add the final projection on top of this + compiledOp = NewPlanOpProjection(newProjections, compiledOp) + + } else { + // rewrite all the order by expressions that are not qualified refs to be qualified + // refs referring to the expression + for i, oe := range orderByExprs { + _, ok := oe.Expr.(*qualifiedRefPlanExpression) + if !ok { + orderByExprs[i].Expr = newQualifiedRefPlanExpression("", oe.Expr.String(), 0, oe.Expr.Type()) + } + } + compiledOp = NewPlanOpOrderBy(orderByExprs, compiledOp) + } } // insert the top operator if it exists @@ -583,11 +683,10 @@ func (p *ExecutionPlanner) analyzeSelectStatement(ctx context.Context, stmt *par } for _, term := range stmt.OrderingTerms { - expr, err := p.analyzeOrderingTermExpression(term.X, stmt) + err := p.analyzeOrderingTermExpression(term.X, stmt) if err != nil { return nil, err } - term.X = expr } return stmt, nil diff --git a/sql3/planner/expression.go b/sql3/planner/expression.go index 292e0716d..c014fe140 100644 --- a/sql3/planner/expression.go +++ b/sql3/planner/expression.go @@ -2817,26 +2817,62 @@ func (p *ExecutionPlanner) compileCallExpr(expr *parser.Call) (_ types.PlanExpre } } -func (p *ExecutionPlanner) compileOrderingTermExpr(expr parser.Expr) (index int, err error) { +func (p *ExecutionPlanner) compileOrderingTermExpr(expr parser.Expr, projections []types.PlanExpression, source parser.Source) (types.PlanExpression, error) { if expr == nil { - return 0, nil + return nil, nil } switch thisExpr := expr.(type) { - case *parser.QualifiedRef: - return thisExpr.ColumnIndex, nil + case *parser.Ident: + for _, proj := range projections { + switch p := proj.(type) { + case *qualifiedRefPlanExpression: + if strings.EqualFold(thisExpr.Name, p.columnName) { + if !typeCanBeSortedOn(p.Type()) { + return nil, sql3.NewErrExpectedSortableExpression(0, 0, p.Type().TypeDescription()) + } + return p, nil + } + case *aliasPlanExpression: + if strings.EqualFold(thisExpr.Name, p.aliasName) { + if !typeCanBeSortedOn(p.expr.Type()) { + return nil, sql3.NewErrExpectedSortableExpression(0, 0, p.expr.Type().TypeDescription()) + } + return p.expr, nil + } + + } + } + + // we didn't find in projection list so go look in the source columns + for _, col := range source.PossibleOutputColumns() { + if strings.EqualFold(thisExpr.Name, col.ColumnName) { + orderExpr := newQualifiedRefPlanExpression(col.TableName, col.ColumnName, col.ColumnIndex, col.Datatype) + if !typeCanBeSortedOn(orderExpr.Type()) { + return nil, sql3.NewErrExpectedSortableExpression(0, 0, orderExpr.Type().TypeDescription()) + } + return orderExpr, nil + } + } + + return nil, sql3.NewErrColumnNotFound(thisExpr.NamePos.Line, thisExpr.NamePos.Column, thisExpr.Name) case *parser.IntegerLit: val, err := strconv.ParseInt(thisExpr.Value, 10, 64) if err != nil { - return 0, err + return nil, err } // subtract one because ordering terms are 1 based, not 0 based - return int(val - 1), nil + index := int(val - 1) + // get the expr from the projection + orderExpr := projections[index] + if !typeCanBeSortedOn(orderExpr.Type()) { + return nil, sql3.NewErrExpectedSortableExpression(0, 0, orderExpr.Type().TypeDescription()) + } + return orderExpr, nil default: - return 0, sql3.NewErrInternalf("unexpected ordering expression type: %T", expr) - + return nil, sql3.NewErrInternalf("unexpected ordering expression type: %T", expr) } } diff --git a/sql3/planner/expressionanalyzer.go b/sql3/planner/expressionanalyzer.go index 1214bd05a..78aa5d5b7 100644 --- a/sql3/planner/expressionanalyzer.go +++ b/sql3/planner/expressionanalyzer.go @@ -831,60 +831,54 @@ func (p *ExecutionPlanner) analyzeCaseBlockExpression(ctx context.Context, expr return expr, nil } -func (p *ExecutionPlanner) analyzeOrderingTermExpression(expr parser.Expr, scope parser.Statement) (parser.Expr, error) { +func (p *ExecutionPlanner) analyzeOrderingTermExpression(expr parser.Expr, scope parser.Statement) error { if expr == nil { - return nil, nil + return nil } - // ordering terms need to be either a column name, an alias name or an integer literal representing - // position of column in the select list + // ordering terms can be: + // 1. a *parser.Ident reference to either a column name in the source, or a reference to a a column or alias name in the projection list + // 2. a *parser.IntegerLit representing position of column in the projection list switch thisExpr := expr.(type) { case *parser.Ident: switch sc := scope.(type) { case *parser.SelectStatement: - // go find the first ident in the projection list that matches - columnIndex := 0 - found := false - for idx, proj := range sc.Columns { + // go look for the first ident in the projection list that matches + foundInProjectionList := false + for _, proj := range sc.Columns { // if the expression is a qualified ref, check the name colExpr, ok := proj.Expr.(*parser.QualifiedRef) if ok && strings.EqualFold(thisExpr.Name, colExpr.Column.Name) { - columnIndex = idx - found = true + foundInProjectionList = true break } // try the alias is there is one if proj.Alias != nil && strings.EqualFold(thisExpr.Name, proj.Alias.Name) { - columnIndex = idx - found = true + foundInProjectionList = true break } } - if !found { - return nil, sql3.NewErrColumnNotFound(thisExpr.NamePos.Line, thisExpr.NamePos.Column, thisExpr.Name) - } + if !foundInProjectionList { + // we didn't find in projection list so go look in the source columns + foundInSource := false + for _, col := range sc.Source.PossibleOutputColumns() { + if strings.EqualFold(thisExpr.Name, col.ColumnName) { + foundInSource = true + break + } + } - // turn *parser.Ident into *parser.QualifiedRef - ident := &parser.QualifiedRef{ - Table: &parser.Ident{ - Name: "", - NamePos: parser.Pos{Line: 0, Column: 0}, - }, - Column: &parser.Ident{ - Name: thisExpr.Name, - NamePos: thisExpr.NamePos, - }, - ColumnIndex: columnIndex, - // since this is a ordring term, we don't care about the type - RefDataType: parser.NewDataTypeVoid(), + if !foundInSource { + return sql3.NewErrColumnNotFound(thisExpr.NamePos.Line, thisExpr.NamePos.Column, thisExpr.Name) + } } - return ident, nil + return nil default: - return nil, sql3.NewErrInternalf("unhandled scope type '%T'", sc) + return sql3.NewErrInternalf("unhandled scope type '%T'", sc) } case *parser.IntegerLit: @@ -893,17 +887,17 @@ func (p *ExecutionPlanner) analyzeOrderingTermExpression(expr parser.Expr, scope // check to see if the offset is in the range value, err := strconv.ParseInt(thisExpr.Value, 10, 64) if err != nil { - return nil, sql3.NewErrInternalf("unexpected integer literal value") + return sql3.NewErrInternalf("unexpected integer literal value") } if value < 1 || value > int64(len(sc.Columns)) { - return nil, sql3.NewErrExpectedSortExpressionReference(0, 0) + return sql3.NewErrExpectedSortExpressionReference(0, 0) } default: - return nil, sql3.NewErrInternalf("unhandled scope type '%T'", sc) + return sql3.NewErrInternalf("unhandled scope type '%T'", sc) } + return nil default: - return nil, sql3.NewErrExpectedSortExpressionReference(expr.Pos().Line, expr.Pos().Column) + return sql3.NewErrExpectedSortExpressionReference(expr.Pos().Line, expr.Pos().Column) } - return expr, nil } diff --git a/sql3/planner/expressiontypes.go b/sql3/planner/expressiontypes.go index 43ff26f3c..242bf287d 100644 --- a/sql3/planner/expressiontypes.go +++ b/sql3/planner/expressiontypes.go @@ -466,7 +466,7 @@ func typeIsTimeQuantum(testType parser.ExprDataType) (bool, parser.ExprDataType) switch testType.(type) { case *parser.DataTypeIDSetQuantum: return true, parser.NewDataTypeIDSet() - case *parser.DataTypeStringSet: + case *parser.DataTypeStringSetQuantum: return true, parser.NewDataTypeStringSet() default: return false, nil @@ -542,6 +542,16 @@ func typeIsBSI(testType parser.ExprDataType) bool { } } +// returns true if we can sort on a type +func typeCanBeSortedOn(testType parser.ExprDataType) bool { + switch testType.(type) { + case *parser.DataTypeStringSet, *parser.DataTypeIDSet: + return false + default: + return true + } +} + // returns true if the types can be compared func typesAreComparable(testTypeL parser.ExprDataType, testTypeR parser.ExprDataType) bool { switch testTypeL.(type) { diff --git a/sql3/planner/inbuiltfunctionstable.go b/sql3/planner/inbuiltfunctionstable.go index 0968e659f..19bc67c93 100644 --- a/sql3/planner/inbuiltfunctionstable.go +++ b/sql3/planner/inbuiltfunctionstable.go @@ -47,7 +47,7 @@ func (p *ExecutionPlanner) analyzeFunctionSubtable(call *parser.Call, scope pars ok, _ := typeIsTimeQuantum(call.Args[0].DataType()) if !ok { // TODO (pok) send back the right error - return nil, sql3.NewErrSetExpressionExpected(call.Args[1].Pos().Line, call.Args[1].Pos().Column) + return nil, sql3.NewErrSetExpressionExpected(call.Args[0].Pos().Line, call.Args[0].Pos().Column) } call.ResultDataType = parser.NewDataTypeSubtable([]*parser.SubtableColumn{ { diff --git a/sql3/planner/oporderby.go b/sql3/planner/oporderby.go index b11c431a1..feced2451 100644 --- a/sql3/planner/oporderby.go +++ b/sql3/planner/oporderby.go @@ -32,8 +32,7 @@ const ( // OrderByExpression is the expression on which an order by can be computed type OrderByExpression struct { - Index int - ExprType parser.ExprDataType + Expr types.PlanExpression Order orderByOrder NullOrdering nullOrdering } @@ -79,6 +78,24 @@ func (n *PlanOpOrderBy) WithChildren(children ...types.PlanOperator) (types.Plan return NewPlanOpOrderBy(n.orderByFields, children[0]), nil } +func (n *PlanOpOrderBy) Expressions() []types.PlanExpression { + res := make([]types.PlanExpression, 0) + for _, e := range n.orderByFields { + res = append(res, e.Expr) + } + return res +} + +func (n *PlanOpOrderBy) WithUpdatedExpressions(exprs ...types.PlanExpression) (types.PlanOperator, error) { + if len(exprs) != len(n.orderByFields) { + return nil, sql3.NewErrInternalf("unexpected number of exprs '%d'", len(exprs)) + } + for i, e := range exprs { + n.orderByFields[i].Expr = e + } + return n, nil +} + func (n *PlanOpOrderBy) String() string { return "" } @@ -92,8 +109,7 @@ func (n *PlanOpOrderBy) Plan() map[string]interface{} { ps := make([]interface{}, 0) for _, e := range n.orderByFields { ps = append(ps, &map[string]interface{}{ - "index": e.Index, - "exprType": e.ExprType.TypeDescription(), + "expr": e.Expr.Plan(), "order": e.Order, "nullOrdering": e.NullOrdering, }) @@ -200,8 +216,20 @@ func (s *OrderBySorter) Less(i, j int) bool { a := s.Rows[i] b := s.Rows[j] for _, sf := range s.SortFields { - av := a[sf.Index] - bv := b[sf.Index] + + var sortIndex int + switch se := sf.Expr.(type) { + case *qualifiedRefPlanExpression: + sortIndex = se.columnIndex + case *intLiteralPlanExpression: + sortIndex = int(se.value) + default: + s.LastError = sql3.NewErrInternalf("unexpected sort field expression type '%T'", se) + return false + } + + av := a[sortIndex] + bv := b[sortIndex] if sf.Order == orderByDesc { av, bv = bv, av @@ -215,8 +243,8 @@ func (s *OrderBySorter) Less(i, j int) bool { return sf.NullOrdering != nullOrderingFirst } - switch sf.ExprType.(type) { - case *parser.DataTypeInt, *parser.DataTypeID: + switch t := sf.Expr.Type().(type) { + case *parser.DataTypeInt: avInt, aok := av.(int64) bvInt, bok := bv.(int64) if !(aok && bok) { @@ -228,6 +256,18 @@ func (s *OrderBySorter) Less(i, j int) bool { } return true + case *parser.DataTypeID: + avInt, aok := av.(uint64) + bvInt, bok := bv.(uint64) + if !(aok && bok) { + s.LastError = sql3.NewErrInternalf("unexpected type conversion result") + return false + } + if avInt > bvInt { + return false + } + return true + case *parser.DataTypeBool: avBool, aok := av.(bool) bvBool, bok := bv.(bool) @@ -277,7 +317,7 @@ func (s *OrderBySorter) Less(i, j int) bool { return true default: - s.LastError = sql3.NewErrInternalf("unhandled data type '%T'", sf.ExprType) + s.LastError = sql3.NewErrInternalf("unhandled data type '%T'", t) return false } } diff --git a/sql3/planner/planoptimizer.go b/sql3/planner/planoptimizer.go index e5c1761ef..114c29922 100644 --- a/sql3/planner/planoptimizer.go +++ b/sql3/planner/planoptimizer.go @@ -16,7 +16,6 @@ import ( //TODO(pok) push filter down into join condition if terms reference either side of join //TODO(pok) push order by down as far as possible -//TODO(pok) handle the case of the order by expressions not being in a projection list //TODO(pok) you can't group by _id in PQL, so we need to not use a PQL group by operator here //TODO(pok) move constant folding to in here @@ -1052,7 +1051,7 @@ func fixProjectionReferences(ctx context.Context, a *ExecutionPlanner, n types.P return thisNode, false, nil // everything else that can be a child of projection - case *PlanOpRelAlias, *PlanOpFilter, *PlanOpPQLTableScan, *PlanOpPQLDistinctScan, *PlanOpNestedLoops: + case *PlanOpRelAlias, *PlanOpFilter, *PlanOpPQLTableScan, *PlanOpPQLDistinctScan, *PlanOpNestedLoops, *PlanOpOrderBy: exprs, same, err := fixFieldRefIndexesOnExpressions(ctx, scope, a, childOp.Schema(), thisNode.Projections...) if err != nil { return thisNode, true, err @@ -1073,6 +1072,44 @@ func fixProjectionReferences(ctx context.Context, a *ExecutionPlanner, n types.P func fixFieldRefs(ctx context.Context, a *ExecutionPlanner, n types.PlanOperator, scope *OptimizerScope) (types.PlanOperator, bool, error) { return TransformPlanOp(n, func(node types.PlanOperator) (types.PlanOperator, bool, error) { switch thisNode := node.(type) { + case *PlanOpOrderBy: + switch childOp := thisNode.ChildOp.(type) { + case *PlanOpProjection: + expressions := thisNode.Expressions() + + for _, ex := range expressions { + ref, ok := ex.(*qualifiedRefPlanExpression) + if !ok { + return nil, true, sql3.NewErrInternalf("unexpected expression type '%T'", ex) + } + for i, proj := range childOp.Projections { + if strings.EqualFold(ref.String(), proj.String()) { + ref.columnIndex = i + break + } + } + } + newNode, err := thisNode.WithUpdatedExpressions(expressions...) + if err != nil { + return nil, true, err + } + return newNode, false, nil + + default: + // fix references for the expressions referenced in the order by list + schema := childOp.Schema() + expressions := thisNode.Expressions() + fixed, same, err := fixFieldRefIndexesOnExpressions(ctx, scope, a, schema, expressions...) + if err != nil { + return nil, true, err + } + newNode, err := thisNode.WithUpdatedExpressions(fixed...) + if err != nil { + return nil, true, err + } + return newNode, same, nil + } + case *PlanOpFilter: // fix references for the expressions referenced in the filter predicate expression schema := thisNode.Schema() diff --git a/sql3/test/defs/defs_orderby.go b/sql3/test/defs/defs_orderby.go index 8d2deb7fa..aded96bb8 100644 --- a/sql3/test/defs/defs_orderby.go +++ b/sql3/test/defs/defs_orderby.go @@ -14,10 +14,10 @@ var orderByTests = TableTest{ srcHdr("a_decimal", fldTypeDecimal2), ), srcRows( - srcRow(int64(1), int64(11), []int64{11, 12, 13}, int64(101), "str1", []string{"a1", "b1", "c1"}, float64(123.45)), - srcRow(int64(2), int64(22), []int64{21, 22, 23}, int64(201), "str2", []string{"a2", "b2", "c2"}, float64(234.56)), - srcRow(int64(3), int64(33), []int64{31, 32, 33}, int64(301), "str3", []string{"a3", "b3", "c3"}, float64(345.67)), - srcRow(int64(4), int64(44), []int64{41, 42, 43}, int64(401), "str4", []string{"a4", "b4", "c4"}, float64(456.78)), + srcRow(int64(1), int64(44), []int64{11, 12, 13}, int64(101), "str1", []string{"a1", "b1", "c1"}, float64(123.45)), + srcRow(int64(2), int64(33), []int64{21, 22, 23}, int64(201), "str2", []string{"a2", "b2", "c2"}, float64(234.56)), + srcRow(int64(3), int64(21), []int64{31, 32, 33}, int64(301), "str3", []string{"a3", "b3", "c3"}, float64(345.67)), + srcRow(int64(4), int64(10), []int64{41, 42, 43}, int64(401), "str4", []string{"a4", "b4", "c4"}, float64(456.78)), ), ), SQLTests: []SQLTest{ @@ -35,5 +35,127 @@ var orderByTests = TableTest{ ), ExpErr: "unable to sort a column of type 'idset'", }, + { + SQLs: sqls( + "select an_int from order_by_test order by an_id asc", + ), + ExpHdrs: hdrs( + hdr("an_int", fldTypeInt), + ), + ExpRows: rows( + row(int64(44)), + row(int64(33)), + row(int64(21)), + row(int64(10)), + ), + Compare: CompareExactOrdered, + }, + { + SQLs: sqls( + "select an_int, an_id from order_by_test order by a_decimal asc", + ), + ExpHdrs: hdrs( + hdr("an_int", fldTypeInt), + hdr("an_id", fldTypeID), + ), + ExpRows: rows( + row(int64(44), int64(101)), + row(int64(33), int64(201)), + row(int64(21), int64(301)), + row(int64(10), int64(401)), + ), + Compare: CompareExactOrdered, + }, + { + SQLs: sqls( + "select an_int + 1 as foo, an_id from order_by_test order by foo asc, a_decimal asc", + ), + ExpHdrs: hdrs( + hdr("foo", fldTypeInt), + hdr("an_id", fldTypeID), + ), + ExpRows: rows( + row(int64(11), int64(401)), + row(int64(22), int64(301)), + row(int64(34), int64(201)), + row(int64(45), int64(101)), + ), + Compare: CompareExactOrdered, + }, + { + SQLs: sqls( + "select an_int from order_by_test order by an_int asc", + ), + ExpHdrs: hdrs( + hdr("an_int", fldTypeInt), + ), + ExpRows: rows( + row(int64(10)), + row(int64(21)), + row(int64(33)), + row(int64(44)), + ), + Compare: CompareExactOrdered, + }, + { + SQLs: sqls( + "select an_int as foo from order_by_test order by foo asc", + ), + ExpHdrs: hdrs( + hdr("foo", fldTypeInt), + ), + ExpRows: rows( + row(int64(10)), + row(int64(21)), + row(int64(33)), + row(int64(44)), + ), + Compare: CompareExactOrdered, + }, + { + SQLs: sqls( + "select an_int as foo from order_by_test order by 1 asc", + ), + ExpHdrs: hdrs( + hdr("foo", fldTypeInt), + ), + ExpRows: rows( + row(int64(10)), + row(int64(21)), + row(int64(33)), + row(int64(44)), + ), + Compare: CompareExactOrdered, + }, + { + SQLs: sqls( + "select an_int + 1 from order_by_test order by 1 asc", + ), + ExpHdrs: hdrs( + hdr("", fldTypeInt), + ), + ExpRows: rows( + row(int64(11)), + row(int64(22)), + row(int64(34)), + row(int64(45)), + ), + Compare: CompareExactOrdered, + }, + { + SQLs: sqls( + "select an_int + 1 as bar from order_by_test order by bar desc", + ), + ExpHdrs: hdrs( + hdr("bar", fldTypeInt), + ), + ExpRows: rows( + row(int64(45)), + row(int64(34)), + row(int64(22)), + row(int64(11)), + ), + Compare: CompareExactOrdered, + }, }, } From d0e40120256a80e72e232e713cb1d2662e83a62d Mon Sep 17 00:00:00 2001 From: Lory Cloutier <118481783+lorycloutier@users.noreply.github.com> Date: Fri, 17 Feb 2023 13:50:45 -0600 Subject: [PATCH 02/19] Fb 1975 (#2254) * Store version-check file in the configured data-directory This also fixes what I think is a bug. It also un-exports everything. I have questions. * Version checking: clean up code, add server flags FB-1975 Cleaned up version checking, removed a race condition, and added error checking. Added server flags for check-in endpoint and UUID storage file. Incorporates Travis's changes to store UUID file in data directory and unexport most of verchk.go. --------- Co-authored-by: Travis Turner Co-authored-by: seebs --- ctl/server.go | 2 + ctl/server_test.go | 6 +++ server.go | 33 +++++++++++++++-- server/config.go | 11 ++++++ server/server.go | 2 + verchk.go | 91 ++++++++++++++++++++++------------------------ 6 files changed, 93 insertions(+), 52 deletions(-) diff --git a/ctl/server.go b/ctl/server.go index 03261ffc2..0c03dca24 100644 --- a/ctl/server.go +++ b/ctl/server.go @@ -49,6 +49,8 @@ func serverFlagSet(srv *server.Config, prefix string) *pflag.FlagSet { flags.DurationVar((*time.Duration)(&srv.LongQueryTime), pre("long-query-time"), time.Duration(srv.LongQueryTime), "Duration that will trigger log and stat messages for slow queries. Zero to disable.") flags.IntVar(&srv.QueryHistoryLength, pre("query-history-length"), srv.QueryHistoryLength, "Number of queries to remember in history.") flags.Int64Var(&srv.MaxQueryMemory, pre("max-query-memory"), srv.MaxQueryMemory, "Maximum memory allowed per Extract() or SELECT query.") + flags.StringVar(&srv.VerChkAddress, pre("verchk-address"), srv.VerChkAddress, "Address to contact to check for latest version.") + flags.StringVar(&srv.UUIDFile, pre("uuid-file"), srv.UUIDFile, "File to store UUID used in checking latest version. If this is a relative path, the file will be stored in the server's data directory.") // TLS SetTLSConfig(flags, pre(""), &srv.TLS.CertificatePath, &srv.TLS.CertificateKeyPath, &srv.TLS.CACertPath, &srv.TLS.SkipVerify, &srv.TLS.EnableClientVerification) diff --git a/ctl/server_test.go b/ctl/server_test.go index 495c9b03c..1bc4cff30 100644 --- a/ctl/server_test.go +++ b/ctl/server_test.go @@ -20,4 +20,10 @@ func TestBuildServerFlags(t *testing.T) { if cm.Flags().Lookup("log-path").Name == "" { t.Fatal("log-path flag is required") } + if cm.Flags().Lookup("verchk-address").Name == "" { + t.Fatal("verchk-address flag is required") + } + if cm.Flags().Lookup("uuid-file").Name == "" { + t.Fatal("uuid-file flag is required") + } } diff --git a/server.go b/server.go index 007cd6add..b04feea7f 100644 --- a/server.go +++ b/server.go @@ -89,6 +89,8 @@ type Server struct { // nolint: maligned defaultClient *InternalClient dataDir string + verChkAddress string + uuidFile string // Threshold for logging long-running queries longQueryTime time.Duration @@ -167,6 +169,24 @@ func OptServerDataDir(dir string) ServerOption { } } +// OptServerVerChkAddress is a functional option on Server +// used to set the address to check for the current version. +func OptServerVerChkAddress(addr string) ServerOption { + return func(s *Server) error { + s.verChkAddress = addr + return nil + } +} + +// OptServerUUIDFile is a functional option on Server +// used to set the file name for storing the checkin UUID. +func OptServerUUIDFile(uf string) ServerOption { + return func(s *Server) error { + s.uuidFile = uf + return nil + } +} + // OptServerViewsRemovalInterval is a functional option on Server // used to set the ttl removal interval. func OptServerViewsRemovalInterval(interval time.Duration) ServerOption { @@ -604,16 +624,21 @@ func (s *Server) Open() error { log.Println(errors.Wrap(err, "logging startup")) } - // Do version check in. This is in a goroutine so that we don't block server startup if the server endpoint is down/having issues. + // Do version check in. This is in a goroutine so that we don't block server + // startup if the server endpoint is down/having issues. go func() { s.logger.Printf("Beginning featurebase version check-in") - vc := VersionChecker{URL: "https://analytics.featurebase.com/v2/featurebase/metrics"} - resp, err := vc.CheckIn() + vc := newVersionChecker(s.cluster.Path, s.verChkAddress, s.uuidFile) + resp, err := vc.checkIn() if err != nil { s.logger.Errorf("doing version checkin. Error was %s", err) return } - s.logger.Printf("Version check-in complete. Latest version is %s", resp.Version) + if resp.Error != "" { + s.logger.Printf("Version check-in failed, endpoint response was %s", resp.Error) + } else { + s.logger.Printf("Version check-in complete. Latest version is %s", resp.Version) + } }() // Start DisCo. diff --git a/server/config.go b/server/config.go index f2d42e8ac..92e86c10d 100644 --- a/server/config.go +++ b/server/config.go @@ -152,6 +152,15 @@ type Config struct { // Limits the total amount of memory to be used by Extract() & SELECT queries. MaxQueryMemory int64 `toml:"max-query-memory"` + // On startup, featurebase server contacts a web server to check the latest version. + // This stores the address for that check + VerChkAddress string `toml:"verchk-address"` + + // When checking version, server sends a UUID so that we can keep track of + // how many unique Featurebase installs are out there. The file is stored in + // the data directory; this stores the filename to use. + UUIDFile string `toml:"uuid-file"` + Cluster struct { ReplicaN int `toml:"replicas"` Name string `toml:"name"` @@ -383,6 +392,8 @@ func NewConfig() *Config { LongQueryTime: toml.Duration(-time.Minute), CheckInInterval: 60 * time.Second, + VerChkAddress: "https://analytics.featurebase.com/v2/featurebase/metrics", + UUIDFile: ".client_id.txt", } // Cluster config. diff --git a/server/server.go b/server/server.go index fb50cb2c8..827d6edcd 100644 --- a/server/server.go +++ b/server/server.go @@ -596,6 +596,8 @@ func (m *Command) setupServer() error { pilosa.OptServerServerlessStorage(m.serverlessStorage), pilosa.OptServerIsDataframeEnabled(m.Config.Dataframe.Enable), pilosa.OptServerDataframeUseParquet(m.Config.Dataframe.UseParquet), + pilosa.OptServerVerChkAddress(m.Config.VerChkAddress), + pilosa.OptServerUUIDFile(m.Config.UUIDFile), } if m.isComputeNode { diff --git a/verchk.go b/verchk.go index 0041a9ca1..1dcf93558 100644 --- a/verchk.go +++ b/verchk.go @@ -6,25 +6,29 @@ import ( "io" "net/http" "os" + "path/filepath" "strings" "github.com/google/uuid" ) -type VersionChecker struct { - URL string +type versionChecker struct { + path string + url string + idfile string } -func NewVersionChecker(endpoint string) *VersionChecker { - v := VersionChecker{ - URL: endpoint, +func newVersionChecker(path, endpoint, idfile string) *versionChecker { + v := versionChecker{ + path: path, + url: endpoint, + idfile: idfile, } return &v } -func (v *VersionChecker) CheckIn() (*VerCheckResponse, error) { - - id, err := v.WriteClientUUID() +func (v *versionChecker) checkIn() (*verCheckResponse, error) { + id, err := v.writeClientUUID() if err != nil { return nil, err } @@ -38,79 +42,70 @@ func (v *VersionChecker) CheckIn() (*VerCheckResponse, error) { if err != nil { return nil, err } - wReq := bytes.NewReader(req) + var jsonResp verCheckResponse + r, err := http.Post(v.url, "application/json", wReq) if err != nil { return nil, err } - var json_resp VerCheckResponse - r, err := http.Post(v.URL, "application/json", wReq) - if err != nil { - return nil, err - } data, err := io.ReadAll(r.Body) - if err != nil { return nil, err } - err = json.Unmarshal(data, &json_resp) + err = json.Unmarshal(data, &jsonResp) if err != nil { return nil, err } - return &json_resp, nil - + return &jsonResp, nil } -func (v *VersionChecker) GenerateClientUUID() (string, error) { +func (v *versionChecker) generateClientUUID() (string, error) { clientUUID := uuid.New() cleanedUUID := strings.Replace(clientUUID.String(), "-", "", -1) return cleanedUUID, nil } -func (v *VersionChecker) WriteClientUUID() (string, error) { - filename := ".client_id.txt" - _, err := os.Stat(filename) - if err != nil { - if os.IsNotExist(err) { - fh, err := os.Create(filename) - if err != nil { - return "", err - } - defer fh.Close() - id, err := v.GenerateClientUUID() - if err != nil { - return "", err - } - - _, err = fh.WriteString(id) - if err != nil { - return "", err - } - - return "", err - } else { - return "", err - } +func (v *versionChecker) writeClientUUID() (string, error) { + var filename string + // if v.idfile starts with a path separator then it's an absolute path + // otherwise it's a relative path and the file goes in the data directory + if v.idfile[0] == os.PathSeparator { + filename = v.idfile + } else { + filename = filepath.Join(v.path, v.idfile) } - - fh, err := os.Open(filename) + fh, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE, 0744) if err != nil { return "", err } defer fh.Close() - buf, err := os.ReadFile(filename) + buf, err := os.ReadFile(filename) if err != nil { return "", err } + // this just checks to see if there was anything at all in the file. + // we should probably check to make sure it's a valid UUID + if string(buf) == "" { + id, err := v.generateClientUUID() + if err != nil { + return "", err + } + _, err = fh.WriteString(id) + if err != nil { + return "", err + } + return id, nil + } + return string(buf), nil - } -type VerCheckResponse struct { +type verCheckResponse struct { Version string `json:"latest_version"` + Error string `json:"error"` } From 7a839f2e8f62db888436750a6ff4303115fd09e3 Mon Sep 17 00:00:00 2001 From: Julio Martinez Date: Tue, 21 Feb 2023 08:55:27 -0800 Subject: [PATCH 03/19] Build target also builds fbsql, fbsql is also packaged. (#2260) Co-authored-by: Julio Martinez --- Makefile | 2 ++ nfpm.yaml | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 2e952d166..2b288acb9 100644 --- a/Makefile +++ b/Makefile @@ -111,6 +111,8 @@ cover-viz: cover # Compile Pilosa build: $(GO) build -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/featurebase + $(GO) build -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/fbsql + package: GOOS=$(GOOS) GOARCH=$(GOARCH) FLAGS="-o featurebase" $(MAKE) build diff --git a/nfpm.yaml b/nfpm.yaml index 2c0c4d254..b4db53b39 100644 --- a/nfpm.yaml +++ b/nfpm.yaml @@ -10,6 +10,8 @@ homepage: "https://molecula.com" contents: - src: ./featurebase dst: /usr/bin/featurebase + - src: ./fbsql + dst: /usr/bin/fbsql - src: ./install/featurebase.conf dst: /etc/featurebase/featurebase.conf type: config|noreplace @@ -22,7 +24,7 @@ contents: - src: ./install/featurebase.debian.service dst: /lib/systemd/system/featurebase.service packager: deb - - dst: /var/log/molecula # We use vendor name on log directory in case other molecula components need it. + - dst: /var/log/molecula # We use vendor name on log directory in case other molecula components need it. type: dir file_info: mode: 0755 From f65f7ffe95b509d6338d56b34696b62120a18fe5 Mon Sep 17 00:00:00 2001 From: Julio Martinez Date: Tue, 21 Feb 2023 10:55:05 -0800 Subject: [PATCH 04/19] Fix bad FLAGS env var passed when building to package. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 2b288acb9..a3ff35bea 100644 --- a/Makefile +++ b/Makefile @@ -115,7 +115,7 @@ build: package: - GOOS=$(GOOS) GOARCH=$(GOARCH) FLAGS="-o featurebase" $(MAKE) build + GOOS=$(GOOS) GOARCH=$(GOARCH) $(MAKE) build GOARCH=$(GOARCH) VERSION=$(VERSION) nfpm package --packager deb --target featurebase.$(VERSION).$(GOARCH).deb GOARCH=$(GOARCH) VERSION=$(VERSION) nfpm package --packager rpm --target featurebase.$(VERSION).$(GOARCH).rpm From bf37dfa9bacf6d80dd22ba99973f1910db060de6 Mon Sep 17 00:00:00 2001 From: jacob Date: Tue, 21 Feb 2023 11:54:17 -0600 Subject: [PATCH 05/19] add name field to recordTime field --- idk/kafka/source.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/idk/kafka/source.go b/idk/kafka/source.go index bcc519d20..003c8d76f 100644 --- a/idk/kafka/source.go +++ b/idk/kafka/source.go @@ -648,7 +648,8 @@ func avroToPDKField(aField *avro.SchemaField) (idk.Field, error) { return nil, errors.Errorf("required property for RecordTimeField: layout, err:%v", err) } return idk.RecordTimeField{ - Layout: layout, + NameVal: aField.Name, + Layout: layout, }, nil } From 864c6ad4e772f7a1d019c8250d85fc42f17eb5b5 Mon Sep 17 00:00:00 2001 From: jacob Date: Tue, 21 Feb 2023 13:23:43 -0600 Subject: [PATCH 06/19] adding test avro to PDK for recordTime --- idk/kafka/source_test.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/idk/kafka/source_test.go b/idk/kafka/source_test.go index 0738bc753..e695cba11 100644 --- a/idk/kafka/source_test.go +++ b/idk/kafka/source_test.go @@ -226,6 +226,19 @@ func TestAvroToPDKField(t *testing.T) { expField: idk.IDField{NameVal: "int-ttl"}, expErr: "nil", }, + { + name: "recordTime", + schemaField: &avro.SchemaField{ + Name: "record-time", + Type: &avro.BytesSchema{}, + Properties: map[string]interface{}{ + "layout": "2006-01-02 15:04:05", + "fieldType": "recordTime", + }, + }, + expField: idk.RecordTimeField{NameVal: "record-time", Layout: "2006-01-02 15:04:05"}, + expErr: "nil", + }, } for _, test := range tests { From 3b2111b31cf80636da2a510808aa11904b451e16 Mon Sep 17 00:00:00 2001 From: jacob Date: Tue, 21 Feb 2023 15:44:01 -0600 Subject: [PATCH 07/19] adding some test data --- idk/kafka/cmd_test.go | 54 +++ idk/kafka/testdata/records/timeQuantum.txt | 500 ++++++++++++++++++++ idk/kafka/testdata/schemas/timeQuantum.json | 26 + 3 files changed, 580 insertions(+) create mode 100644 idk/kafka/testdata/records/timeQuantum.txt create mode 100644 idk/kafka/testdata/schemas/timeQuantum.json diff --git a/idk/kafka/cmd_test.go b/idk/kafka/cmd_test.go index f3cd26477..645db6948 100644 --- a/idk/kafka/cmd_test.go +++ b/idk/kafka/cmd_test.go @@ -638,6 +638,60 @@ func TestCmdSchemaChange(t *testing.T) { } } +func TestTimeQuantums(t *testing.T) { + + /* + at a high level, a test here represents + - an avro schema + - a set of records to ingest to kafka + - an ingest configuration + - query to run to confirm the data was ingest properly + */ + tests := []struct { + name string + autoGenerateID bool + primaryKeyFields string // nil when idField is not nil + idField string // nil when primaryKeyFields is not nil + PilosaHosts []string + RegistryURL string + pathToAvroSchema string + pathToRecords string + topic string + }{ + { + name: "time quantums exist", + autoGenerateID: false, + primaryKeyFields: "device", + idField: nil, + PilosaHosts []string + RegistryURL string + pathToAvroSchema string + pathToRecords string + topic string // don't duplicate + + }, + { + name: "3 primary keys str/str/int TLS", + PrimaryKeyFields: []string{"abc", "db", "user_id"}, + PilosaHosts: []string{pilosaTLSHost}, + TLS: &idk.TLSConfig{ + CertificatePath: certPath + "/theclient.crt", + CertificateKeyPath: certPath + "/theclient.key", + CACertPath: certPath + "/ca.crt", + EnableClientVerification: true, + }, + expRhinoKeys: []string{"2|1|159", "4|3|44", "123456789|q2db_1234|432"}, // "2" + "1" + uint32(159) + + }, + { + name: "IDField int", + IDField: "user_id", + expRhinoCols: []uint64{44, 159, 432}, + }, + } + +} + type sortableCRI []pilosaclient.CountResultItem func (s sortableCRI) Len() int { return len(s) } diff --git a/idk/kafka/testdata/records/timeQuantum.txt b/idk/kafka/testdata/records/timeQuantum.txt new file mode 100644 index 000000000..ba0d8458d --- /dev/null +++ b/idk/kafka/testdata/records/timeQuantum.txt @@ -0,0 +1,500 @@ +{'device': 'XbJ7vwASddz1xBQ4', 'segment_ts': 'fub2', 'time_q': '2023-02-13 18:11:08'} +{'device': 'g3AwJJUDwrN1LYzY', 'segment_ts': 'Qn3I', 'time_q': '2023-02-13 19:39:01'} +{'device': 'DIxrncsISTdUJdsH', 'segment_ts': 'o8Ig', 'time_q': '2023-01-27 09:41:54'} +{'device': 'yR1RUTVnjwUvjWLS', 'segment_ts': 'BUnG', 'time_q': '2023-01-25 06:55:04'} +{'device': 'dpeJtjO0mVsN3wiZ', 'segment_ts': 'Udwh', 'time_q': '2023-01-25 14:34:42'} +{'device': 'hOJrFi9xmOnM5J2B', 'segment_ts': '9jED', 'time_q': '2023-01-27 17:05:19'} +{'device': 'xPLZ3qjA029BWoD8', 'segment_ts': 'PBbR', 'time_q': '2023-02-17 08:47:05'} +{'device': 'sdnPNO80Trlnkfpi', 'segment_ts': 'ACVo', 'time_q': '2023-02-21 09:01:47'} +{'device': 'fcMcP2T8hIBzYzrB', 'segment_ts': 'HppX', 'time_q': '2023-02-18 00:59:36'} +{'device': 'xiRO1wD8bdg6tsy2', 'segment_ts': 'iSQp', 'time_q': '2023-01-28 06:15:59'} +{'device': 'nlsTgLlvTBhQGewH', 'segment_ts': 'CZHg', 'time_q': '2023-02-20 18:53:14'} +{'device': 'bpLtS0dLbzC1t0aM', 'segment_ts': 'oanZ', 'time_q': '2023-02-11 08:25:43'} +{'device': 'RcYtSPPTAQyOE80E', 'segment_ts': '57He', 'time_q': '2023-02-14 06:40:17'} +{'device': 'BLqS1Js4hqpAGKbs', 'segment_ts': '8Esc', 'time_q': '2023-02-13 01:59:32'} +{'device': 'NDJ13gRlrfjSpJul', 'segment_ts': 'fBcm', 'time_q': '2023-01-24 07:05:02'} +{'device': 'CUS2BbP8fJrBuJRB', 'segment_ts': 'kUK0', 'time_q': '2023-02-05 22:33:45'} +{'device': 'lwC1KW3eNMX7kqwS', 'segment_ts': '8s12', 'time_q': '2023-02-15 19:55:08'} +{'device': 'jtEPRgME5UKtmv2O', 'segment_ts': 'Ew1L', 'time_q': '2023-02-10 10:44:56'} +{'device': 'zJJQ58p1vZEVr8pV', 'segment_ts': 'cb9G', 'time_q': '2023-01-31 14:49:08'} +{'device': 'zGLCqlcb9WEVYeTD', 'segment_ts': 'JatF', 'time_q': '2023-02-19 17:37:42'} +{'device': 'PgyMZVp5lPVUAzju', 'segment_ts': 'H4w7', 'time_q': '2023-01-25 01:13:51'} +{'device': 'KS2ftxZpJaobrFcH', 'segment_ts': 'lzde', 'time_q': '2023-02-08 07:29:00'} +{'device': 'pvfQ90NOCNxkZ9qp', 'segment_ts': 'meI7', 'time_q': '2023-02-01 09:29:32'} +{'device': 'et0tVgDi4gQLBbCZ', 'segment_ts': '8roi', 'time_q': '2023-02-17 13:59:05'} +{'device': 'KVpmUGw3YFQWFKUJ', 'segment_ts': 'reQ2', 'time_q': '2023-02-07 05:12:56'} +{'device': '7e62NVroqaZ5KCCG', 'segment_ts': 'mRqe', 'time_q': '2023-01-25 18:55:33'} +{'device': '2jhbxRtroaywIHz0', 'segment_ts': 'vGvz', 'time_q': '2023-01-31 06:31:33'} +{'device': 'i8A3oSUCLIRVI3z6', 'segment_ts': 'bRK7', 'time_q': '2023-01-22 15:45:16'} +{'device': 'w9m2zmvBebNOYm7M', 'segment_ts': 'EUTl', 'time_q': '2023-02-12 17:45:44'} +{'device': '6goOWv6GmSB5SImL', 'segment_ts': 'IcRw', 'time_q': '2023-02-02 19:44:49'} +{'device': '9TdRfZY8fyUv0MDA', 'segment_ts': 'jxeF', 'time_q': '2023-02-15 12:46:23'} +{'device': 'XFT7MaPT04gy9giN', 'segment_ts': 'NDT5', 'time_q': '2023-01-27 02:02:09'} +{'device': 'dYf2esJeNHts76qt', 'segment_ts': 'nD5r', 'time_q': '2023-02-20 19:18:20'} +{'device': 'XO3Mw07kTWxq4S6A', 'segment_ts': 'wu98', 'time_q': '2023-02-19 09:11:04'} +{'device': 'PR20MB3DrKZyxBYN', 'segment_ts': 'VxO2', 'time_q': '2023-02-14 02:48:28'} +{'device': '8HoHoySkF3ONDVMT', 'segment_ts': 'RZW0', 'time_q': '2023-02-17 09:47:36'} +{'device': 'D9EMY6KMiUupVo86', 'segment_ts': 'uKvn', 'time_q': '2023-02-13 20:34:37'} +{'device': '6vw27rBMy4z08u0H', 'segment_ts': 'IhoS', 'time_q': '2023-02-07 21:19:31'} +{'device': '8B3tax9WJNOuceOO', 'segment_ts': '2sCg', 'time_q': '2023-02-13 06:09:00'} +{'device': 'q1kmQxWdgK1fI2Aq', 'segment_ts': 'tpUd', 'time_q': '2023-01-22 22:14:51'} +{'device': 'VPe3DuOwKmbOf9hj', 'segment_ts': 'Q1EB', 'time_q': '2023-02-07 01:49:40'} +{'device': 'bcn6NL9OVcDI9NOE', 'segment_ts': 'pmBn', 'time_q': '2023-02-06 19:24:58'} +{'device': 'xveWOtVYpFmdKoE3', 'segment_ts': '3pUE', 'time_q': '2023-01-29 05:45:06'} +{'device': 'O9p8Ij6SiXU7w9Bo', 'segment_ts': 'wDD7', 'time_q': '2023-02-07 02:09:17'} +{'device': 'JU1KI8rx8qLXtY9Y', 'segment_ts': 'NBSG', 'time_q': '2023-02-15 13:20:17'} +{'device': 'zvc3NHLWCKRbwL4w', 'segment_ts': 'wAqp', 'time_q': '2023-02-06 19:57:45'} +{'device': 'uIKOal9xGdwgmxQZ', 'segment_ts': '3x0X', 'time_q': '2023-02-19 00:21:42'} +{'device': '0Rn9nBxCnr1abKBu', 'segment_ts': 'K3q7', 'time_q': '2023-02-07 03:02:30'} +{'device': 'DcPFdJzT9aCgD7hZ', 'segment_ts': '1WZE', 'time_q': '2023-02-04 22:53:14'} +{'device': 'I33U4ld8p1ZTEo2z', 'segment_ts': 'h327', 'time_q': '2023-02-15 09:02:02'} +{'device': 'HzV2F1u15P6MfTq4', 'segment_ts': 'Oi4K', 'time_q': '2023-02-18 03:07:18'} +{'device': 'Gs8bQMX2wl9AjM5I', 'segment_ts': 'CT5V', 'time_q': '2023-02-16 01:45:58'} +{'device': 'aPv5uNdC2x9Zrs31', 'segment_ts': 'ijS2', 'time_q': '2023-01-29 20:43:50'} +{'device': 'gmBMopHkBCbcGRaS', 'segment_ts': 'nD5r', 'time_q': '2023-02-11 05:05:58'} +{'device': 'g3Ns2caZMQVlOSPr', 'segment_ts': '9vzm', 'time_q': '2023-01-27 00:22:33'} +{'device': 'JWY6hPLwnhy76KGZ', 'segment_ts': '2IS2', 'time_q': '2023-02-11 21:47:10'} +{'device': '0QKtSTqJYXMZWvVe', 'segment_ts': '7R83', 'time_q': '2023-02-17 05:40:55'} +{'device': 'Ximq2ebfKR1eqySD', 'segment_ts': 'Jpup', 'time_q': '2023-01-23 17:34:53'} +{'device': 'YWGLIrHDKwe6UUM1', 'segment_ts': 'Smbg', 'time_q': '2023-02-11 22:29:38'} +{'device': 'i8Hy1Hhy6vbQ7Gnj', 'segment_ts': 'Xjn9', 'time_q': '2023-02-07 05:52:25'} +{'device': 'trQjczSWWWqD3xip', 'segment_ts': 'zSyE', 'time_q': '2023-02-14 03:51:05'} +{'device': 'cRAvzL88YU74zOVR', 'segment_ts': 'FAk3', 'time_q': '2023-02-02 06:26:40'} +{'device': 'H7RUb4moxDU5RY8L', 'segment_ts': '6sUs', 'time_q': '2023-02-06 15:29:02'} +{'device': 'g6uZFROq6hA9VwXU', 'segment_ts': '9v3C', 'time_q': '2023-01-30 05:29:57'} +{'device': 'yIGidxsJbbyaTMJu', 'segment_ts': '5TbD', 'time_q': '2023-01-31 20:18:06'} +{'device': 'DC9B8r6fLUrQEKWd', 'segment_ts': 'VDz6', 'time_q': '2023-02-09 22:19:07'} +{'device': 'rymp2l5MJvVdAEiU', 'segment_ts': 'fyte', 'time_q': '2023-02-02 11:53:13'} +{'device': 'Ua1gF72JZ4WAc8Bd', 'segment_ts': 'v5sp', 'time_q': '2023-01-24 02:06:04'} +{'device': '9qSO5sST0XGp5jMS', 'segment_ts': '7dBw', 'time_q': '2023-02-10 09:35:42'} +{'device': 'p0HAiSpiSNAaCrZB', 'segment_ts': 'a1iQ', 'time_q': '2023-01-31 15:45:28'} +{'device': 'wkYtecM7PdltlJ6U', 'segment_ts': 'tCye', 'time_q': '2023-01-29 14:50:59'} +{'device': 'efTNUHAxzw90hiOb', 'segment_ts': '9Wf1', 'time_q': '2023-02-18 02:29:02'} +{'device': 'fQI3EBaehVoTHSuQ', 'segment_ts': 'zRBT', 'time_q': '2023-02-11 19:06:37'} +{'device': 'udeapVgb384YhUvm', 'segment_ts': 'oL67', 'time_q': '2023-02-05 10:27:36'} +{'device': 'eqbom6mcHS60rV4o', 'segment_ts': 'GNdZ', 'time_q': '2023-02-11 08:44:18'} +{'device': '1wWPwlCsbs9ZBcZC', 'segment_ts': 'zaKC', 'time_q': '2023-02-08 00:49:32'} +{'device': 'ecY2Gsp5o5y8RDEy', 'segment_ts': 'QWzq', 'time_q': '2023-01-29 02:43:04'} +{'device': 'DxqHMz1dx4Bqsobg', 'segment_ts': '2RQv', 'time_q': '2023-02-18 21:50:04'} +{'device': 'XU1b7NOcHSuGzCUv', 'segment_ts': 'Yuoy', 'time_q': '2023-02-13 21:54:55'} +{'device': 'lO2wwnU1eJPm8GXQ', 'segment_ts': 'AP3b', 'time_q': '2023-01-28 14:02:06'} +{'device': 'tLnTsvtbSVqjrYe9', 'segment_ts': 'fFTT', 'time_q': '2023-01-24 23:23:30'} +{'device': 'BD3RZkRDscI24QIW', 'segment_ts': '3Bry', 'time_q': '2023-02-14 17:16:33'} +{'device': 'EJTuSaV3BN56Wj2O', 'segment_ts': 'W4JQ', 'time_q': '2023-02-01 18:56:00'} +{'device': 'TfinqcstHOBc9vgU', 'segment_ts': 'VybU', 'time_q': '2023-01-30 13:41:42'} +{'device': 'oLCaL36jSe58AJzI', 'segment_ts': '28pM', 'time_q': '2023-02-03 07:47:20'} +{'device': 'I85aiLVze3vm3TkQ', 'segment_ts': 'y8y1', 'time_q': '2023-02-19 19:12:25'} +{'device': 'J1Z6A5uaqDbcJCfB', 'segment_ts': 'Wzwh', 'time_q': '2023-02-10 08:51:06'} +{'device': 'DUJWSG7TcQrG1SiU', 'segment_ts': 'qKgC', 'time_q': '2023-02-18 22:00:14'} +{'device': 'oq4StgCLUoilIYHQ', 'segment_ts': 'MNLg', 'time_q': '2023-02-05 10:11:01'} +{'device': 'P44lydZpIGjbmQOy', 'segment_ts': 'uHCR', 'time_q': '2023-01-22 21:38:00'} +{'device': '8lV1uEL9zdwqOuwT', 'segment_ts': '8wB6', 'time_q': '2023-02-05 21:52:08'} +{'device': 'uVGi6rqE7NKl7D0W', 'segment_ts': 'kge3', 'time_q': '2023-02-12 14:45:28'} +{'device': 'fBDDRmw5ifz9Ibk1', 'segment_ts': 'e8wv', 'time_q': '2023-02-12 01:36:43'} +{'device': 'q8q2AXRsgPrtU7MF', 'segment_ts': 'lznG', 'time_q': '2023-01-31 09:08:30'} +{'device': 'rKVwah03kvEN1Xf8', 'segment_ts': 'CIf7', 'time_q': '2023-02-17 15:32:01'} +{'device': '5gqkkJu0HnW8e7Jd', 'segment_ts': '11VE', 'time_q': '2023-02-21 12:50:18'} +{'device': '67aeTL1Lhk7zPniw', 'segment_ts': '7WnQ', 'time_q': '2023-02-15 09:54:03'} +{'device': 'JSqkbIUNs9XsDhyf', 'segment_ts': 'Vef3', 'time_q': '2023-01-28 05:40:03'} +{'device': 'O6pY363lkXtNFnPW', 'segment_ts': 'JPG3', 'time_q': '2023-02-09 07:18:08'} +{'device': 'tpldKtJXhNOJs5is', 'segment_ts': 'x40J', 'time_q': '2023-02-19 03:15:47'} +{'device': 'zJFXIMKFyyMjvHzJ', 'segment_ts': 'ddlX', 'time_q': '2023-02-12 02:18:47'} +{'device': 'o0W2DatNPhPHeUCG', 'segment_ts': '3Lzx', 'time_q': '2023-02-01 03:05:37'} +{'device': '0qNRr7JlwEJo0UZs', 'segment_ts': 'x1R8', 'time_q': '2023-02-04 16:06:55'} +{'device': 'ybv2vBmnJIMXDQ7C', 'segment_ts': 'DyAY', 'time_q': '2023-02-04 14:33:24'} +{'device': '7bkYnhDEWrdlKPzi', 'segment_ts': 'CHYf', 'time_q': '2023-02-18 13:52:19'} +{'device': 'gS5FFLJPIgSpPn2p', 'segment_ts': '73ea', 'time_q': '2023-02-20 13:38:30'} +{'device': 'Iz5J7QbGpOaw9Mc3', 'segment_ts': 'UD8U', 'time_q': '2023-02-19 21:56:03'} +{'device': 'Q4zAccg3hJyiWPQV', 'segment_ts': 'WbDH', 'time_q': '2023-02-19 21:57:53'} +{'device': 'vwowDnNJnQ3Noych', 'segment_ts': 'YyvE', 'time_q': '2023-02-07 21:35:53'} +{'device': 'V4Yg3xtwNAdXT7AC', 'segment_ts': 'OQ1f', 'time_q': '2023-01-27 03:41:03'} +{'device': 'kBcU57h5YJlps0M7', 'segment_ts': 'X9xH', 'time_q': '2023-02-16 21:18:05'} +{'device': 'mTOckeiB1Myp9PzA', 'segment_ts': 'wDD7', 'time_q': '2023-02-16 06:44:48'} +{'device': 'jFqBXPPMyJJaM4VG', 'segment_ts': 'csTs', 'time_q': '2023-02-06 02:52:21'} +{'device': 'vKNH7K6ozve7702k', 'segment_ts': 'SJ25', 'time_q': '2023-02-08 01:27:29'} +{'device': 'oyT2TtgfCba2Hdml', 'segment_ts': 'Tvqb', 'time_q': '2023-02-21 03:13:30'} +{'device': 'rsOlS9C8eEy347rZ', 'segment_ts': 'hr7p', 'time_q': '2023-02-19 02:30:46'} +{'device': '90GjHMWLi1dcpp0y', 'segment_ts': 'W4JQ', 'time_q': '2023-01-25 02:29:48'} +{'device': 'fvudPtjiPgFEJ7bD', 'segment_ts': 'lwno', 'time_q': '2023-01-24 05:44:13'} +{'device': 'VBWAgZvfVhht5RLf', 'segment_ts': 'Ftw2', 'time_q': '2023-01-27 13:08:57'} +{'device': 'sZK0cJqcYBnZXwyj', 'segment_ts': '9G1V', 'time_q': '2023-01-27 21:13:09'} +{'device': 'Nw2Qbh1azig2aS8i', 'segment_ts': 'VPUK', 'time_q': '2023-02-20 06:55:48'} +{'device': 'YrmDmrxR6yVVsmBK', 'segment_ts': 'kEmM', 'time_q': '2023-02-17 04:13:52'} +{'device': 'FJg8emti6uDPkZWb', 'segment_ts': 'cu9S', 'time_q': '2023-01-31 15:15:17'} +{'device': 'D2KVrcXONfBL0DOq', 'segment_ts': 'E7q4', 'time_q': '2023-02-07 05:21:14'} +{'device': 'zpjzEqsoIDSfgkrw', 'segment_ts': 'kuSy', 'time_q': '2023-02-08 21:39:35'} +{'device': 'TyBhOHhPHg3mABfs', 'segment_ts': 'FOLC', 'time_q': '2023-02-05 01:50:35'} +{'device': 'SSsebFMnz2vz0lKT', 'segment_ts': 'qXUp', 'time_q': '2023-02-17 15:12:43'} +{'device': 'oeMORvnHFqNvuqA9', 'segment_ts': 'D16L', 'time_q': '2023-02-17 12:56:55'} +{'device': 'pZGE5Q8ax0QnrKlK', 'segment_ts': '6O2l', 'time_q': '2023-02-17 06:43:06'} +{'device': 'xXUDPhByePkaTvpe', 'segment_ts': 'UqWC', 'time_q': '2023-02-05 01:26:33'} +{'device': '8zOXbhQQFeXPuMgW', 'segment_ts': 'v5s5', 'time_q': '2023-01-28 21:54:59'} +{'device': 'Kvd0ii9qinTBgkEK', 'segment_ts': 'YWRA', 'time_q': '2023-02-15 04:02:15'} +{'device': 'JoC3FaiJFH7nWQbY', 'segment_ts': 'RV1o', 'time_q': '2023-01-23 11:40:04'} +{'device': 'JZtyjzapfAhKTdEL', 'segment_ts': '6Hyt', 'time_q': '2023-02-12 23:31:41'} +{'device': 'lsE2LnvCTkfUv3Hb', 'segment_ts': 'mvNJ', 'time_q': '2023-02-20 04:17:14'} +{'device': 'ZaG5u6BctGV7Phak', 'segment_ts': 'xS0C', 'time_q': '2023-02-12 21:40:07'} +{'device': 'VqkvcCFemWmmdLIN', 'segment_ts': '1vCq', 'time_q': '2023-02-04 13:43:54'} +{'device': 'MnWYYlFI787w7ENs', 'segment_ts': 'YHny', 'time_q': '2023-02-02 04:12:38'} +{'device': '5dFd1cLHS8fjZaOz', 'segment_ts': '0JSK', 'time_q': '2023-02-14 05:41:43'} +{'device': 'G7r2BebALuPczt3b', 'segment_ts': 'nowH', 'time_q': '2023-02-19 08:55:48'} +{'device': '7V6SBGSNrMZUvrw8', 'segment_ts': 'KKuH', 'time_q': '2023-02-07 03:00:14'} +{'device': 'YWmt4ZWofrE68Eg4', 'segment_ts': '9ub1', 'time_q': '2023-01-24 10:32:07'} +{'device': 'RgsF3Paw1CFHmXrz', 'segment_ts': 'GKFG', 'time_q': '2023-02-05 14:23:12'} +{'device': 'uRUd8DnEptcCg051', 'segment_ts': 'w6bQ', 'time_q': '2023-02-12 02:55:44'} +{'device': 'gpJn2OhNJ17yFKfj', 'segment_ts': 'Ms6E', 'time_q': '2023-02-11 12:28:14'} +{'device': 'OS8p3UgcZwCtitJ5', 'segment_ts': 'lUE6', 'time_q': '2023-02-12 12:43:29'} +{'device': 'AaBf2goUzSIOLwKU', 'segment_ts': 'jRhp', 'time_q': '2023-01-28 02:43:57'} +{'device': 'Gi7o9G6zuxIje6TA', 'segment_ts': 'TPjM', 'time_q': '2023-01-22 21:42:14'} +{'device': 'Zgp8HrnLjtoUBstx', 'segment_ts': '669s', 'time_q': '2023-01-23 07:40:22'} +{'device': 'yZ39wYgza1sjjH6a', 'segment_ts': '80La', 'time_q': '2023-01-25 23:58:17'} +{'device': 'gKSePVA4eUpPeQeB', 'segment_ts': 'uPAl', 'time_q': '2023-02-06 04:22:11'} +{'device': 'ZvPnZt45ZdDJf2wV', 'segment_ts': 'eQg8', 'time_q': '2023-01-29 16:01:24'} +{'device': 'RGaReXrDxjLvIN1h', 'segment_ts': 'wV8N', 'time_q': '2023-01-27 01:38:59'} +{'device': 'DBG2BbMzd2AiJ1ra', 'segment_ts': 'or1s', 'time_q': '2023-02-14 19:21:12'} +{'device': 'LaRpvrLjCem5I1iG', 'segment_ts': 'MKyg', 'time_q': '2023-01-22 20:08:37'} +{'device': '6VPMKOpB2Gax57AM', 'segment_ts': 'jWnU', 'time_q': '2023-02-02 08:27:23'} +{'device': 'WWmDiCgEhcFaRoxd', 'segment_ts': 'IcRw', 'time_q': '2023-01-28 20:36:01'} +{'device': 'g8tKyCuvwcmnOleP', 'segment_ts': 'l57C', 'time_q': '2023-02-21 12:15:51'} +{'device': 'mGmZoqlgLWwRHhWd', 'segment_ts': 'MJWQ', 'time_q': '2023-01-26 00:54:26'} +{'device': '9vYzND4LhGIpLRPs', 'segment_ts': 'X8An', 'time_q': '2023-02-17 17:12:20'} +{'device': 'TqAci7xzsz9HKRwX', 'segment_ts': 'HQvb', 'time_q': '2023-02-07 19:26:19'} +{'device': 'jXeyesYOxpErf0Wr', 'segment_ts': '9Pah', 'time_q': '2023-01-29 07:43:44'} +{'device': 'PKpsammUcBhHIj4d', 'segment_ts': 'SJ96', 'time_q': '2023-02-01 11:22:01'} +{'device': 'd43Y87nTdkx4nw0N', 'segment_ts': 'qt7V', 'time_q': '2023-02-13 07:35:55'} +{'device': '3wSdpuLVAqb2DqwQ', 'segment_ts': '1uul', 'time_q': '2023-02-20 17:20:48'} +{'device': 'b86HMaTJskEFSegv', 'segment_ts': 'e0pl', 'time_q': '2023-02-20 04:23:59'} +{'device': 'q7XDusNiRySmmaRj', 'segment_ts': '7dBw', 'time_q': '2023-01-29 01:47:01'} +{'device': 'LrWDgsZruGK3akSe', 'segment_ts': '0vLc', 'time_q': '2023-02-20 13:19:00'} +{'device': 'JJOo2fsdKEG6QCRu', 'segment_ts': 'LN5W', 'time_q': '2023-02-12 01:36:02'} +{'device': 'iXc2nZTT1U0iWf2W', 'segment_ts': '3ftI', 'time_q': '2023-02-21 10:12:08'} +{'device': 'YJ4TjVpH6ZQhMvrH', 'segment_ts': 'IVww', 'time_q': '2023-01-24 12:14:46'} +{'device': '4cBW0bSimlW8ygs0', 'segment_ts': 'tGGE', 'time_q': '2023-02-13 11:54:22'} +{'device': 'Ye2BuE77YVbChRNA', 'segment_ts': 'SRVM', 'time_q': '2023-01-25 16:55:03'} +{'device': 'BD4sQkvrmTWcfgdd', 'segment_ts': 'uqv1', 'time_q': '2023-02-17 14:21:23'} +{'device': 'kB9K8PKJpQnkJTcd', 'segment_ts': 'WCit', 'time_q': '2023-02-20 20:54:19'} +{'device': 'uSAbl2Nyu6nK3ddO', 'segment_ts': '0cq0', 'time_q': '2023-01-28 17:04:49'} +{'device': 'uJEVeGx0jSYrxfsx', 'segment_ts': 'GL95', 'time_q': '2023-02-19 21:43:48'} +{'device': 'wowb2b399vUqVxWH', 'segment_ts': 'YcDM', 'time_q': '2023-02-03 10:03:34'} +{'device': 'Yt42EPAk8TRsKsha', 'segment_ts': '9vzm', 'time_q': '2023-02-06 16:06:01'} +{'device': 'nuQgvAgPYUF1ML4s', 'segment_ts': 'SOUQ', 'time_q': '2023-02-02 10:23:33'} +{'device': 'LEuEmNLEgjoCu3p1', 'segment_ts': 'Mhpw', 'time_q': '2023-01-31 19:35:49'} +{'device': '4H1WoLSdS2qs5jOM', 'segment_ts': 'II21', 'time_q': '2023-02-10 14:24:09'} +{'device': 'dN5oTZXijbMTz8OP', 'segment_ts': 'NOvo', 'time_q': '2023-01-27 11:27:12'} +{'device': 'i2ITUxoWBCEdW8Bs', 'segment_ts': 'FqD3', 'time_q': '2023-01-30 06:33:32'} +{'device': 'NyZi9ibptktAZTjI', 'segment_ts': 'C24z', 'time_q': '2023-02-13 18:48:37'} +{'device': '5MMCow4pOwRmSIlc', 'segment_ts': 'NuYD', 'time_q': '2023-02-03 15:50:35'} +{'device': 'lK52wLIAUCcXP4AH', 'segment_ts': 'geC5', 'time_q': '2023-02-14 07:33:08'} +{'device': 'kdEACkrL59EG8vBX', 'segment_ts': '44WL', 'time_q': '2023-02-15 16:46:30'} +{'device': 'BZhrV8EAlPBft7XX', 'segment_ts': 'w04k', 'time_q': '2023-02-16 03:47:55'} +{'device': '8mKdbrORBbygBkPb', 'segment_ts': '3x0X', 'time_q': '2023-01-27 04:58:59'} +{'device': 'C3ue47cXeXukj7rU', 'segment_ts': '7AXG', 'time_q': '2023-02-14 15:59:38'} +{'device': 'PFZtQxNxLhvhEuil', 'segment_ts': 'mCKg', 'time_q': '2023-02-12 16:29:55'} +{'device': 'Qfa7tEckTSssilkU', 'segment_ts': '3Epz', 'time_q': '2023-01-30 04:49:42'} +{'device': 'OZGXqiDoUPo7codg', 'segment_ts': 'k234', 'time_q': '2023-02-21 02:14:45'} +{'device': 'VcXbfEsURAyiOlXz', 'segment_ts': 'XXHG', 'time_q': '2023-02-06 21:09:36'} +{'device': 'd9jY3TNnU9v39DOE', 'segment_ts': 'JBAB', 'time_q': '2023-02-13 05:44:52'} +{'device': 'bY1aO1ktexSWKAjB', 'segment_ts': 'OUWG', 'time_q': '2023-02-19 16:34:05'} +{'device': '4J0KbbaKXbT17iAy', 'segment_ts': 'THLZ', 'time_q': '2023-02-19 16:01:05'} +{'device': 'TdfnSWLdZuoHDmT4', 'segment_ts': '9bQC', 'time_q': '2023-02-13 07:47:03'} +{'device': 'vKvuSLFrmEElACFv', 'segment_ts': 'WgTc', 'time_q': '2023-02-01 05:41:50'} +{'device': 'EgUuFOflBQ4zUJz2', 'segment_ts': 'dfJl', 'time_q': '2023-02-11 07:30:24'} +{'device': '3cWfkAf3Ma8T8KkO', 'segment_ts': 'PDJS', 'time_q': '2023-02-05 03:09:15'} +{'device': 'xoCum1PFgRMFFAtd', 'segment_ts': 'bdMN', 'time_q': '2023-02-10 21:10:02'} +{'device': 'G10WOxXP7vDSrqG2', 'segment_ts': '9vHF', 'time_q': '2023-01-25 10:44:21'} +{'device': 'GyDiESn1pQ7C2eOq', 'segment_ts': 'X5WH', 'time_q': '2023-02-10 05:32:32'} +{'device': 'Ihum52kryw40U2Pq', 'segment_ts': 'YQGo', 'time_q': '2023-01-29 03:27:14'} +{'device': '9DeVCz49iKTNFPyr', 'segment_ts': '2rzd', 'time_q': '2023-01-24 10:45:49'} +{'device': '7mW6uwngb7RD2lEp', 'segment_ts': '5jBK', 'time_q': '2023-02-11 22:13:48'} +{'device': 'xuW4yuuFgbdd4a6p', 'segment_ts': 'lkBB', 'time_q': '2023-01-31 18:54:23'} +{'device': '5kzp9NXHYWNPgYoh', 'segment_ts': 'fioj', 'time_q': '2023-01-26 03:08:34'} +{'device': 'gmWHoGjwiWF82jHM', 'segment_ts': 'kiO6', 'time_q': '2023-01-22 18:57:36'} +{'device': 'zFyepAAJnlCoeqEU', 'segment_ts': 'tJnv', 'time_q': '2023-02-01 00:26:55'} +{'device': 'EglsUmauQATFBwu6', 'segment_ts': 'fGsr', 'time_q': '2023-01-30 09:02:11'} +{'device': 'qUd53Kl4mymteZxS', 'segment_ts': 'TdgJ', 'time_q': '2023-01-27 05:11:33'} +{'device': 'F5zBVO0bkWwMjrQ3', 'segment_ts': 'yEsz', 'time_q': '2023-02-17 00:44:53'} +{'device': 'HDgL5w5o1AyW8KNq', 'segment_ts': 'WMDV', 'time_q': '2023-02-20 04:25:58'} +{'device': 'b080loiW7g8hJYZN', 'segment_ts': 's8vj', 'time_q': '2023-01-26 23:15:47'} +{'device': 'PLB1THGiEIKtVnuq', 'segment_ts': 'eqHY', 'time_q': '2023-02-17 07:42:59'} +{'device': 'Y90nqduLp4v38lVD', 'segment_ts': 'hZVY', 'time_q': '2023-02-15 14:26:35'} +{'device': 'fh35hFi6oIEaA5Df', 'segment_ts': 'TJ6P', 'time_q': '2023-01-28 02:34:05'} +{'device': 'RBLsLZ4eIt3nrLPH', 'segment_ts': '7WHK', 'time_q': '2023-01-29 20:05:29'} +{'device': '9Db2XAc0xDm9Mbmx', 'segment_ts': 'ls7g', 'time_q': '2023-02-18 08:15:15'} +{'device': 'M1Ih4aY0dYWKWegA', 'segment_ts': 'sgfr', 'time_q': '2023-02-04 00:43:41'} +{'device': '0YtmJi8QdEGnWaHz', 'segment_ts': '5HQX', 'time_q': '2023-01-25 06:09:08'} +{'device': 'LjuzkroRYOCdQSz5', 'segment_ts': 'dzph', 'time_q': '2023-01-24 00:27:56'} +{'device': 'K0y5JORnnCPqA3PL', 'segment_ts': '6jsI', 'time_q': '2023-02-06 10:18:08'} +{'device': '75xJYP1muHt9LVvt', 'segment_ts': '2PcN', 'time_q': '2023-01-25 23:49:07'} +{'device': 'aa15VzCz3SZrbBAl', 'segment_ts': 'FfVa', 'time_q': '2023-02-07 04:37:48'} +{'device': '65lf7rvFyN74HbTL', 'segment_ts': 'q9o4', 'time_q': '2023-02-10 16:57:52'} +{'device': 'BkATNfc6INXfprd4', 'segment_ts': 'QW7U', 'time_q': '2023-02-18 04:08:48'} +{'device': 'HSBPAi87vKtuhabY', 'segment_ts': 'Ft0J', 'time_q': '2023-01-29 04:20:36'} +{'device': 'Rb23K4sxSmDFCiK2', 'segment_ts': 'CZHg', 'time_q': '2023-02-04 22:53:44'} +{'device': 'VzfrrwW2GG8Au619', 'segment_ts': 'qmhZ', 'time_q': '2023-01-29 07:18:18'} +{'device': 'UW5oYzVVoP9VUNvi', 'segment_ts': 'FD5Q', 'time_q': '2023-02-13 03:55:02'} +{'device': '7ZqKm1mzPqTjKAK8', 'segment_ts': 'yAxf', 'time_q': '2023-02-03 08:04:47'} +{'device': 'RKs79WJe0pmc30Ji', 'segment_ts': 'iQ7P', 'time_q': '2023-02-14 20:17:09'} +{'device': 'fawEL3nd7h4bsCSA', 'segment_ts': '5JQi', 'time_q': '2023-02-09 09:47:37'} +{'device': '1cK5y0fv2oasCsEg', 'segment_ts': 'FZfm', 'time_q': '2023-01-24 23:15:26'} +{'device': 'fZlV1wQfgN0slT78', 'segment_ts': 'QjWp', 'time_q': '2023-01-23 18:30:49'} +{'device': 'ReYnQoZKQD7FhiPY', 'segment_ts': 'uTjn', 'time_q': '2023-01-25 01:24:16'} +{'device': 'NPuU31MUQXT5txEw', 'segment_ts': 'clSh', 'time_q': '2023-02-04 23:56:25'} +{'device': 'uu9ToklvVI6lAa9s', 'segment_ts': '9v3C', 'time_q': '2023-01-23 19:15:43'} +{'device': 'iVXaaRjEBBkz0hz3', 'segment_ts': 'aV1U', 'time_q': '2023-02-05 20:11:12'} +{'device': 'xM0S07A9cCsd9Yfn', 'segment_ts': 'MeDE', 'time_q': '2023-01-24 04:13:01'} +{'device': 'nQHMNXhbyFZ6UoDn', 'segment_ts': 'K4Ze', 'time_q': '2023-02-10 16:22:30'} +{'device': 'fXaNQOwz30KmlHrP', 'segment_ts': '36ap', 'time_q': '2023-02-14 12:08:13'} +{'device': 'dw2ya1E1U6tjsd7T', 'segment_ts': 'Jb8U', 'time_q': '2023-02-14 03:53:01'} +{'device': 'GZ0h5bYPi6E6ohn4', 'segment_ts': '44WL', 'time_q': '2023-01-29 13:46:23'} +{'device': 'wNZt7rqhrv3YtKuz', 'segment_ts': 'zaKC', 'time_q': '2023-01-25 03:03:32'} +{'device': 'RpROmDVcSKPVt0C8', 'segment_ts': 'lvES', 'time_q': '2023-02-17 21:12:47'} +{'device': 'jbZJ7XFo3s3aLnjJ', 'segment_ts': 'cvD6', 'time_q': '2023-02-17 14:55:37'} +{'device': '4Qw6ei7d9Ux7cTDd', 'segment_ts': 'gwio', 'time_q': '2023-02-01 08:26:39'} +{'device': 'JLYhTM9PRqiptyZD', 'segment_ts': 'QrPy', 'time_q': '2023-01-26 09:58:00'} +{'device': '6FO0vI0i8EqKQhd6', 'segment_ts': 'hH9B', 'time_q': '2023-01-31 23:43:46'} +{'device': 'iGeoTrx9INiZGRHX', 'segment_ts': 'X2bk', 'time_q': '2023-01-30 16:07:24'} +{'device': '5pRaiBzTt31JZkve', 'segment_ts': '8Auu', 'time_q': '2023-02-05 09:10:17'} +{'device': 'OvWvtocpkdNLqrny', 'segment_ts': 'Inti', 'time_q': '2023-02-12 21:28:35'} +{'device': 'bYwqcLrv6vNbGiov', 'segment_ts': 'ysSv', 'time_q': '2023-02-14 22:07:17'} +{'device': 'o9lqa5RcsnYdxzPx', 'segment_ts': 'R8bW', 'time_q': '2023-01-31 19:56:44'} +{'device': 'WmUqr1hnmvlRw6YO', 'segment_ts': '5viW', 'time_q': '2023-01-28 20:23:04'} +{'device': 'JhzC3ZigvlLK1seo', 'segment_ts': 'J5sY', 'time_q': '2023-02-12 20:59:49'} +{'device': 'evKPQwjRV6cBQvjy', 'segment_ts': 'dDZY', 'time_q': '2023-01-31 04:46:11'} +{'device': 'sLMrZiBFi2lonYwg', 'segment_ts': 'unUy', 'time_q': '2023-02-15 08:51:34'} +{'device': 'o4lT6snRzZnQa1HO', 'segment_ts': 'Z7Ft', 'time_q': '2023-02-03 21:15:32'} +{'device': 'hqZWBbZog5N1kD9V', 'segment_ts': 'WBtx', 'time_q': '2023-02-19 18:01:06'} +{'device': 'xXzLPLm0OsnDJwUz', 'segment_ts': 'K1Yr', 'time_q': '2023-01-28 23:13:12'} +{'device': 'KXSUj8l7UVFARe7n', 'segment_ts': 'BvbW', 'time_q': '2023-01-29 21:44:04'} +{'device': 'UvZA7BW1fWlt0GbE', 'segment_ts': 'ajjJ', 'time_q': '2023-01-24 03:31:50'} +{'device': 'cQGXvK61LboETbwo', 'segment_ts': 'Ey3g', 'time_q': '2023-01-23 21:46:37'} +{'device': 'CQ101gI3Mx4eORJw', 'segment_ts': 'Jb8U', 'time_q': '2023-01-28 03:33:22'} +{'device': 'ed2KkpZOCurKwUW1', 'segment_ts': 'ZWmw', 'time_q': '2023-02-15 06:18:12'} +{'device': 'cmsyEAqBqINzHOdc', 'segment_ts': 'MXoi', 'time_q': '2023-02-10 07:11:49'} +{'device': 'HrEkRR54fFMG0LDI', 'segment_ts': 'siac', 'time_q': '2023-02-07 23:36:13'} +{'device': 'wxGldUWtgrjazC1G', 'segment_ts': 'tvS0', 'time_q': '2023-02-04 05:46:44'} +{'device': 'RxPCG6v5f4sqllhf', 'segment_ts': 'Ysq9', 'time_q': '2023-02-12 12:54:14'} +{'device': 'r7xQr423rEko5eka', 'segment_ts': 'XPPE', 'time_q': '2023-02-07 03:44:00'} +{'device': 'wxuOz2ANcqk4RoaO', 'segment_ts': 'lDol', 'time_q': '2023-02-16 15:04:43'} +{'device': 'PlSBAhofMJFpB2uV', 'segment_ts': 'RFaV', 'time_q': '2023-02-11 22:02:11'} +{'device': 'hZ1tbXscywB9ibyA', 'segment_ts': 'xnZO', 'time_q': '2023-02-20 15:51:50'} +{'device': 'KbFXsiA8Q0SqfSF2', 'segment_ts': 'glBE', 'time_q': '2023-02-18 23:04:11'} +{'device': 'MRPK2r2JhCir2u2l', 'segment_ts': 'Med8', 'time_q': '2023-01-24 09:27:31'} +{'device': 'cEFmg5V2xJHPyqp5', 'segment_ts': 'Kd02', 'time_q': '2023-02-10 09:41:59'} +{'device': '1lwjNCCV5WEJzXT7', 'segment_ts': '5CQq', 'time_q': '2023-01-27 01:07:21'} +{'device': 'VvOulNjY9xQCblv7', 'segment_ts': 'C8F7', 'time_q': '2023-01-27 21:04:34'} +{'device': 'dF1H0PGvmSbpRyiI', 'segment_ts': 'eD44', 'time_q': '2023-01-31 13:38:45'} +{'device': 'NkDgwYVx0XveN5Dt', 'segment_ts': 'WxCz', 'time_q': '2023-02-21 07:16:46'} +{'device': 'LBsuQUNvWKpyd89V', 'segment_ts': 'V9Fm', 'time_q': '2023-02-10 12:25:41'} +{'device': 'n1KoKQk8oZ8lUPtV', 'segment_ts': 'J7YR', 'time_q': '2023-02-16 05:30:48'} +{'device': 'YDqHeGZquVwEjqQc', 'segment_ts': '2GHM', 'time_q': '2023-02-15 00:35:09'} +{'device': 'RgQGcEnSZcPxClfB', 'segment_ts': 'hvML', 'time_q': '2023-01-23 08:49:45'} +{'device': 'jSvUQlxGgJWX0n7M', 'segment_ts': 'HMSz', 'time_q': '2023-02-12 06:13:19'} +{'device': 'K7lIlpxjv71aoW9n', 'segment_ts': 'rkkj', 'time_q': '2023-02-02 15:11:54'} +{'device': 'tlcCO6MF2RYl0g96', 'segment_ts': '10GK', 'time_q': '2023-02-17 18:09:56'} +{'device': 'p4o8sJpApQwJOiox', 'segment_ts': 'XAUS', 'time_q': '2023-01-29 10:34:46'} +{'device': '525KZ78kEncXH9VI', 'segment_ts': 'ugYO', 'time_q': '2023-02-03 20:14:10'} +{'device': '7arB9WqfPZVvkzAs', 'segment_ts': 'Jydo', 'time_q': '2023-02-09 13:03:19'} +{'device': 'GsqKaz0OuOy0p8D5', 'segment_ts': 's3Xs', 'time_q': '2023-02-01 17:04:58'} +{'device': 'NJRTnXytfyhbSltj', 'segment_ts': 'pa4b', 'time_q': '2023-01-22 23:47:33'} +{'device': 'V6BcYDFUslTaeKz2', 'segment_ts': 'c3ip', 'time_q': '2023-01-24 12:24:16'} +{'device': 'Lali4GnxbDXiICfd', 'segment_ts': '5HQX', 'time_q': '2023-02-08 07:21:08'} +{'device': '4G67CFMQFfw5nKhH', 'segment_ts': 'HfZ3', 'time_q': '2023-01-27 13:08:26'} +{'device': 'VOU1PEANNqFpksPD', 'segment_ts': 'UCFz', 'time_q': '2023-02-02 17:55:58'} +{'device': 'NQkRwi0QBjL9OUfe', 'segment_ts': 'peAr', 'time_q': '2023-01-31 09:44:07'} +{'device': 'C1Nzu0HNqUqhiv4v', 'segment_ts': 'ido8', 'time_q': '2023-02-20 11:23:58'} +{'device': 'jc76NMsTRXg6yFT8', 'segment_ts': 'IlVL', 'time_q': '2023-02-04 08:44:26'} +{'device': 'mkPyfZXT9VoyITp1', 'segment_ts': 'AElH', 'time_q': '2023-02-01 22:49:56'} +{'device': 'aODXvK25bBgLB89w', 'segment_ts': 'UyMT', 'time_q': '2023-02-06 18:46:10'} +{'device': 'bIRJnUdmTWC4bQjs', 'segment_ts': 'q6P2', 'time_q': '2023-01-25 10:01:19'} +{'device': 'NwthNY2x1PiG4NXu', 'segment_ts': 'ziKj', 'time_q': '2023-01-26 00:26:01'} +{'device': 'Y0Jwgj0CtaaVPGLU', 'segment_ts': 'uRmO', 'time_q': '2023-02-13 01:29:07'} +{'device': 'P59OfLIdsNz1u2bf', 'segment_ts': 'hm6H', 'time_q': '2023-01-25 21:43:28'} +{'device': 'dHwliEj94uGOjy4z', 'segment_ts': 'bgF8', 'time_q': '2023-02-16 14:27:07'} +{'device': 'Tp85tKVvwtOGxCsN', 'segment_ts': 'z871', 'time_q': '2023-02-03 03:24:04'} +{'device': 'Nu5VHjVpGjotVjHh', 'segment_ts': 'u8t8', 'time_q': '2023-02-14 04:32:01'} +{'device': 'rWcINdZOiu9BHzkL', 'segment_ts': 'CVvt', 'time_q': '2023-02-07 13:15:38'} +{'device': 'yGILVrj3iinOAS4U', 'segment_ts': '0fQb', 'time_q': '2023-02-01 08:06:25'} +{'device': 'CkVe5GZWBIuoCDh9', 'segment_ts': '0JA6', 'time_q': '2023-02-14 22:41:17'} +{'device': 'gGG9uQjtROdibp0O', 'segment_ts': 'mwXi', 'time_q': '2023-02-18 08:50:46'} +{'device': 'GljeFNPy2bmdxmEz', 'segment_ts': '7Jts', 'time_q': '2023-02-06 12:45:43'} +{'device': '3xlhVhGTJmURk3MX', 'segment_ts': 'NZxR', 'time_q': '2023-02-02 14:51:14'} +{'device': '47ujFWpbmPFKdX1g', 'segment_ts': 'KNqJ', 'time_q': '2023-01-25 06:27:30'} +{'device': 'gNDkMVt00vJT84jK', 'segment_ts': 'WX3F', 'time_q': '2023-02-09 12:57:48'} +{'device': 'LCaDtdl6EqAmPlGU', 'segment_ts': 'AVAC', 'time_q': '2023-02-14 13:38:47'} +{'device': 'RMS3UOMf4mt5Nlfx', 'segment_ts': 'dDQ0', 'time_q': '2023-02-15 20:39:12'} +{'device': 'mCCyHPx2EyQkjDI6', 'segment_ts': 'mvNJ', 'time_q': '2023-02-13 15:16:22'} +{'device': 'TWoLnaxEPgoNH3Fk', 'segment_ts': 'bwdO', 'time_q': '2023-02-11 09:10:07'} +{'device': '2U0w1oZWqt42ge69', 'segment_ts': 'OQPD', 'time_q': '2023-02-08 05:38:25'} +{'device': 'Wr1pqW7ogHm1l57C', 'segment_ts': 'yEsz', 'time_q': '2023-02-19 15:25:54'} +{'device': 'X2x7ETjtNN7XU6wE', 'segment_ts': 'R4fq', 'time_q': '2023-02-09 09:30:45'} +{'device': 'JJsmrj8eGnKEpkgT', 'segment_ts': 'jHFs', 'time_q': '2023-01-28 19:05:13'} +{'device': 'jQOGpX4X5gohJjgf', 'segment_ts': 'NDT5', 'time_q': '2023-01-28 07:12:42'} +{'device': 'CktBzqEuXCvd6ThV', 'segment_ts': 'ItFE', 'time_q': '2023-01-24 21:16:41'} +{'device': 'o9JPJauPfRraIlLu', 'segment_ts': '76j0', 'time_q': '2023-01-23 20:11:55'} +{'device': 'QdHyZk5P5vaBYTak', 'segment_ts': 'mfIG', 'time_q': '2023-02-08 05:08:49'} +{'device': 'u4QrAiPlcTouwbT5', 'segment_ts': 'YZHZ', 'time_q': '2023-02-14 11:46:14'} +{'device': '65dIzY2Z9TJWgbhv', 'segment_ts': 'EqdZ', 'time_q': '2023-01-28 12:55:23'} +{'device': 'kT4R6IGOTSqzw7uB', 'segment_ts': 'SGHa', 'time_q': '2023-02-11 03:18:04'} +{'device': '66KWZA8L7G89WDZI', 'segment_ts': 'wTC5', 'time_q': '2023-02-21 14:23:50'} +{'device': 'LRxsGXs41oqHtRqM', 'segment_ts': 'u7El', 'time_q': '2023-02-09 14:03:09'} +{'device': 'hgyESiq7XGtiQCql', 'segment_ts': 'hxmr', 'time_q': '2023-01-26 12:29:40'} +{'device': '0qbdQklTFNy7ISCH', 'segment_ts': 'R9B9', 'time_q': '2023-01-29 15:23:30'} +{'device': 'xYAjqmLYwZPiLGsd', 'segment_ts': 'lIlA', 'time_q': '2023-02-17 07:43:40'} +{'device': 'BzgAUk3wTrBt2JWM', 'segment_ts': 'sfqh', 'time_q': '2023-02-16 04:38:46'} +{'device': 'UrSOAFfBipxV2iV0', 'segment_ts': 'Il3r', 'time_q': '2023-02-13 10:27:30'} +{'device': 'ALr9hJYdlrw6oNcJ', 'segment_ts': 'Keh8', 'time_q': '2023-01-23 20:17:27'} +{'device': 'GVDWDnyS3E5WQkFK', 'segment_ts': 'uW7S', 'time_q': '2023-02-11 17:04:10'} +{'device': '2OKxK3JW7y96EHEa', 'segment_ts': 'KFml', 'time_q': '2023-01-28 00:05:53'} +{'device': 'FPkKren0hQTn7grY', 'segment_ts': 'eShQ', 'time_q': '2023-02-04 04:45:29'} +{'device': 'FWyRyqknGMhkVpo3', 'segment_ts': 'ZdoH', 'time_q': '2023-02-15 23:28:49'} +{'device': 'JK21TkVFQLsvKUG0', 'segment_ts': '1MVW', 'time_q': '2023-02-06 15:20:46'} +{'device': 'cjo9bUnDXUQHlnCa', 'segment_ts': 'PT3c', 'time_q': '2023-01-25 10:30:12'} +{'device': '2a0x5y7y1dd0WSHe', 'segment_ts': 'hrIh', 'time_q': '2023-02-06 13:52:24'} +{'device': 'k0TSwJgBwwVCZLlv', 'segment_ts': 'vRB7', 'time_q': '2023-02-01 22:19:25'} +{'device': 'j5LFK6TZErFfQ1VZ', 'segment_ts': 'tTr2', 'time_q': '2023-01-30 20:49:48'} +{'device': 'JGYeTG9SlkPY6wFa', 'segment_ts': 'Z9Pu', 'time_q': '2023-02-06 20:31:31'} +{'device': 'Ymh6ayRgp1H60AFc', 'segment_ts': 'Bwf8', 'time_q': '2023-02-06 00:29:22'} +{'device': 'Td0EpJOxCHYWUbrR', 'segment_ts': 'v25d', 'time_q': '2023-01-28 05:25:00'} +{'device': 'mS6Iy8Qa2mmoqYQu', 'segment_ts': '9q3k', 'time_q': '2023-01-30 00:18:20'} +{'device': 'fc6guCFHdG8VzC3p', 'segment_ts': 'QjWp', 'time_q': '2023-02-08 11:44:52'} +{'device': 'ItkmsLuHy98vfLUQ', 'segment_ts': 'JoNb', 'time_q': '2023-02-04 16:39:25'} +{'device': 'zEKKWQ45k04JGcCV', 'segment_ts': 'Tpwm', 'time_q': '2023-02-05 19:39:55'} +{'device': 'JPsTfxDVBAtWfW8r', 'segment_ts': '6sUs', 'time_q': '2023-01-29 20:29:27'} +{'device': 'yIeSaXpU3rkCbxUr', 'segment_ts': 'yZ7s', 'time_q': '2023-02-18 01:23:16'} +{'device': 'ekMZsH5A0zvUgkOr', 'segment_ts': 'KSlf', 'time_q': '2023-01-30 07:30:21'} +{'device': '4c1W84L1nNUfHkZV', 'segment_ts': 'vvM0', 'time_q': '2023-02-01 17:47:15'} +{'device': 'RBN3tJA4BYV71FBJ', 'segment_ts': 'eQg8', 'time_q': '2023-02-21 13:02:27'} +{'device': '9DS3pYWFruFTvJZS', 'segment_ts': 'rRuc', 'time_q': '2023-01-25 06:18:46'} +{'device': 'Oge628HPmV0XJxLE', 'segment_ts': 'XDA1', 'time_q': '2023-02-20 09:45:25'} +{'device': 'qHZcNOXzeJFDOsbi', 'segment_ts': 'X3q3', 'time_q': '2023-02-03 12:14:04'} +{'device': 'Z4TwRqUaxlM4nWox', 'segment_ts': 'G8xC', 'time_q': '2023-01-25 21:21:40'} +{'device': 'cN2qypm0e5KXaM4f', 'segment_ts': 'DPPO', 'time_q': '2023-02-20 05:51:54'} +{'device': 'G1umjEcxzSJpDeVf', 'segment_ts': 'ivk4', 'time_q': '2023-02-04 14:30:20'} +{'device': 'Y4YHyYf0eIA0dQT2', 'segment_ts': 'w9bo', 'time_q': '2023-01-30 04:27:41'} +{'device': 'sybCGj8bJcuWVkMH', 'segment_ts': 'n7r6', 'time_q': '2023-02-10 08:55:38'} +{'device': 'G9qtN4oPylocAhLo', 'segment_ts': 'ygKY', 'time_q': '2023-02-11 15:58:38'} +{'device': 'dqbOQmU1nCEM1tsb', 'segment_ts': 'snFa', 'time_q': '2023-01-26 05:18:19'} +{'device': 'PvXh8nHFVEKnfyw5', 'segment_ts': 'es0t', 'time_q': '2023-02-03 04:33:00'} +{'device': '9evTiKiWziX3THWt', 'segment_ts': 'mrI5', 'time_q': '2023-02-09 12:39:43'} +{'device': '8vWA3DPg0sCubmHt', 'segment_ts': 'VyRj', 'time_q': '2023-01-30 17:09:05'} +{'device': 'JBh1VD2mJSzS93VN', 'segment_ts': 'A2E6', 'time_q': '2023-01-23 22:53:01'} +{'device': 'AVTuFDv6MFEeDkkg', 'segment_ts': 'FQxa', 'time_q': '2023-02-16 15:15:49'} +{'device': 'jEnRgZZOuDplk7CK', 'segment_ts': 'McUm', 'time_q': '2023-02-13 16:58:42'} +{'device': 'w9eyvBTQMP0TlFPg', 'segment_ts': '061H', 'time_q': '2023-02-14 07:07:22'} +{'device': 'HxBZsLSXkTOwIj7U', 'segment_ts': 'Yny1', 'time_q': '2023-02-16 18:30:09'} +{'device': 'OOp6NIdrHxTZgb1a', 'segment_ts': '3gVe', 'time_q': '2023-01-27 03:18:26'} +{'device': 'XQeEiBgEVGX3xRyy', 'segment_ts': 'iK88', 'time_q': '2023-01-31 16:16:42'} +{'device': '7XXbvAMS3NbVIuSb', 'segment_ts': 'ikv3', 'time_q': '2023-01-27 17:58:57'} +{'device': 'Z77uepdP4uNOMFEm', 'segment_ts': 'aNrW', 'time_q': '2023-01-25 19:27:06'} +{'device': 'x7Ru0sut9cCNKErD', 'segment_ts': 'fSS4', 'time_q': '2023-02-16 13:17:10'} +{'device': 'tfF7ARhQav7Vs1sW', 'segment_ts': 'yFy4', 'time_q': '2023-02-18 08:18:49'} +{'device': 'IezPnCOr2g6m2iq3', 'segment_ts': 'YOF3', 'time_q': '2023-02-09 18:31:03'} +{'device': 'VOA5cvCyvFZ7bnzk', 'segment_ts': 'Viph', 'time_q': '2023-01-24 18:36:49'} +{'device': 'p0On4aZWYDoxDWHI', 'segment_ts': 'FTU0', 'time_q': '2023-02-07 02:13:54'} +{'device': 'FSpOgwOaHynPz0zH', 'segment_ts': 'WwVR', 'time_q': '2023-02-03 21:12:09'} +{'device': 'PVSlKZkOPl5hFKDc', 'segment_ts': 'AZRz', 'time_q': '2023-02-09 05:04:38'} +{'device': 'hXJ4b5HgGPtZxZDG', 'segment_ts': 'eNW4', 'time_q': '2023-02-06 20:20:25'} +{'device': 'iyM6IB8Kyie9rXTk', 'segment_ts': 'WbDH', 'time_q': '2023-01-25 02:38:32'} +{'device': 'ZHDFyhQFjT36sYEX', 'segment_ts': 'RrPC', 'time_q': '2023-01-24 13:30:11'} +{'device': 'FAmkVIvvqiBv3J2v', 'segment_ts': 'BLc4', 'time_q': '2023-02-05 01:57:00'} +{'device': 'uUlaL9nusB5K9wah', 'segment_ts': '6NJ4', 'time_q': '2023-02-04 05:09:45'} +{'device': 'uq7LIwGy8Cs8VS8S', 'segment_ts': 'Ksz1', 'time_q': '2023-01-31 14:34:35'} +{'device': 'Sljf8wt6YxMJlQ9B', 'segment_ts': '3LOG', 'time_q': '2023-02-08 01:47:29'} +{'device': '8tnuglv62PYooaqm', 'segment_ts': 'vXhF', 'time_q': '2023-02-02 02:09:46'} +{'device': 'B00CXFzWHZ7T6vw5', 'segment_ts': '993f', 'time_q': '2023-01-28 01:31:50'} +{'device': 'FsKiFfXLePj7r3xn', 'segment_ts': 'uR2f', 'time_q': '2023-02-16 05:49:48'} +{'device': 'P2Pssg8wIUhJfJge', 'segment_ts': 'SACW', 'time_q': '2023-01-23 19:12:29'} +{'device': 'F83dj5jXAaNW08tN', 'segment_ts': 'NI7t', 'time_q': '2023-02-03 06:42:10'} +{'device': 'tz2BamJyhAvZSmKL', 'segment_ts': 'WFNu', 'time_q': '2023-02-05 21:04:43'} +{'device': 'od15jkcehIGTLhka', 'segment_ts': 'pY0b', 'time_q': '2023-02-11 09:23:34'} +{'device': 'o4J3jbqm2n2VMxAV', 'segment_ts': '4G0L', 'time_q': '2023-01-24 13:22:35'} +{'device': 'uyBW48TE0tyYNDtB', 'segment_ts': 'RhKk', 'time_q': '2023-01-25 13:14:11'} +{'device': 'ANOQhJ0WObAmU99M', 'segment_ts': 'NyJ1', 'time_q': '2023-02-10 22:25:45'} +{'device': 'vDQzeXOS56cwm5Ha', 'segment_ts': '3G8I', 'time_q': '2023-02-11 19:06:38'} +{'device': 'YqyX9Xk6Kw1DXOad', 'segment_ts': 'Qk0n', 'time_q': '2023-02-10 21:58:02'} +{'device': '9neVj3Sy1VeKQxzF', 'segment_ts': 'j592', 'time_q': '2023-01-28 02:02:51'} +{'device': 'I1750EUbKfHl2yT0', 'segment_ts': 'Zztj', 'time_q': '2023-02-06 23:04:30'} +{'device': 'cdId56I5EVFYECEn', 'segment_ts': 'Trhf', 'time_q': '2023-02-05 18:35:57'} +{'device': 'mnuDFIng7Ffw33N7', 'segment_ts': '7dQn', 'time_q': '2023-01-31 14:53:16'} +{'device': 'x6GzEx5oAis07KM2', 'segment_ts': 'XcKm', 'time_q': '2023-01-23 06:33:10'} +{'device': 'k27SrvBpgJ43XRj8', 'segment_ts': 'zKI7', 'time_q': '2023-02-14 21:54:58'} +{'device': 'lhj3vbaLb1F2K4lH', 'segment_ts': 'EQLa', 'time_q': '2023-02-18 03:17:10'} +{'device': 'TRlW7EbIXGs2Bpy8', 'segment_ts': 'iOm2', 'time_q': '2023-02-04 11:54:32'} +{'device': 'klEC6kMcr7NP6sNg', 'segment_ts': 'bRK7', 'time_q': '2023-02-19 00:37:12'} +{'device': 'TEemRDFbCJdjyNsQ', 'segment_ts': 'D16L', 'time_q': '2023-02-17 11:29:57'} +{'device': 'jptd7AJiFEAk4tSw', 'segment_ts': 'IAOL', 'time_q': '2023-01-31 08:48:08'} +{'device': 'L45DvspGDQF2fOyi', 'segment_ts': 'sALF', 'time_q': '2023-02-05 03:25:00'} +{'device': 'R2NkbrNMNldSBJUG', 'segment_ts': '609r', 'time_q': '2023-02-02 09:49:49'} +{'device': '8ZqVW4kT45fPZy5a', 'segment_ts': 'l00H', 'time_q': '2023-02-11 13:05:33'} +{'device': 'jHbO5YUIUxVvdzJA', 'segment_ts': '8dJN', 'time_q': '2023-02-17 02:03:26'} +{'device': 'ktIY7e9VGCKMjL7P', 'segment_ts': 'Mxrj', 'time_q': '2023-02-20 23:12:59'} +{'device': 'uuOZC7aaCy9sUiT5', 'segment_ts': 'EUD1', 'time_q': '2023-02-01 23:31:46'} +{'device': '3RtgCZcjgxxCmg5T', 'segment_ts': '5JQi', 'time_q': '2023-01-23 12:12:05'} +{'device': 'YnI4T3vi1Lxyt5wa', 'segment_ts': 'rXIU', 'time_q': '2023-02-12 23:58:24'} +{'device': 'ev9IcNjTEZekqALB', 'segment_ts': 'GAxJ', 'time_q': '2023-01-24 12:16:20'} +{'device': 'vp5qi7TRC8KQWk7d', 'segment_ts': 'TJ6P', 'time_q': '2023-02-07 17:28:23'} +{'device': 'bSoAX2L1JHmPXODo', 'segment_ts': 'fasF', 'time_q': '2023-02-06 15:59:59'} +{'device': '3XauXuq6ws73yQla', 'segment_ts': 'SsnB', 'time_q': '2023-01-26 16:11:14'} +{'device': 'TyYnqrqHKPeqZ82q', 'segment_ts': 'Wzwh', 'time_q': '2023-02-16 20:02:19'} +{'device': 'O8hOzOOgxw2iQiHx', 'segment_ts': 'Yj0D', 'time_q': '2023-02-14 00:19:14'} +{'device': 'u4Pxs4Cyj00rtA60', 'segment_ts': '1vCq', 'time_q': '2023-01-31 16:09:15'} +{'device': 'jo8xd3EdVQRVKZG0', 'segment_ts': 'ltBU', 'time_q': '2023-01-31 07:14:48'} +{'device': 'W06dd1sDTy0apH3Y', 'segment_ts': 'ygKY', 'time_q': '2023-02-09 21:21:10'} +{'device': 'Sh7dZSxHMgFlutjC', 'segment_ts': 'MbqF', 'time_q': '2023-02-11 23:49:25'} +{'device': 'o5CLa2P1xlbs3BYb', 'segment_ts': 'v0BW', 'time_q': '2023-02-01 11:00:06'} +{'device': '4wegF6Zxl955mSik', 'segment_ts': '281a', 'time_q': '2023-01-29 11:24:20'} +{'device': 'bx31bZeyk6v3ZATS', 'segment_ts': 'wT1j', 'time_q': '2023-02-03 22:21:54'} +{'device': 'kfBaygAngrkZHyhR', 'segment_ts': 'KFml', 'time_q': '2023-02-11 02:35:20'} +{'device': 'IENwCndPscOSssCf', 'segment_ts': 'tGZF', 'time_q': '2023-02-05 15:36:28'} +{'device': 'JU1KqBsrNFR4vOqo', 'segment_ts': 'U1XA', 'time_q': '2023-01-31 18:39:08'} +{'device': '8qqyRLZQv8IAoQgP', 'segment_ts': '1WZE', 'time_q': '2023-02-03 06:48:42'} +{'device': 'rBCNpDGLi9AodGdX', 'segment_ts': '3J3I', 'time_q': '2023-02-17 06:28:51'} +{'device': 'x1VmoWLCRvrSmkEh', 'segment_ts': 'lqnk', 'time_q': '2023-01-24 18:01:20'} +{'device': 'iL3Swp4VajwaNf3O', 'segment_ts': 'gtdx', 'time_q': '2023-01-23 11:19:34'} +{'device': 'g7iVs7QsRXDubZvV', 'segment_ts': 'Oyvx', 'time_q': '2023-01-29 11:33:47'} +{'device': '4iCnj0wRJoMFKwx4', 'segment_ts': 'XU65', 'time_q': '2023-01-26 13:56:11'} +{'device': 'WWBQzqhhgRMwMVYH', 'segment_ts': 'MwvV', 'time_q': '2023-01-26 21:09:06'} +{'device': 'ZjwDtsLC8jMJIsQ6', 'segment_ts': 'NhT6', 'time_q': '2023-02-10 05:06:17'} +{'device': 'YkI6KyY3TgyviASv', 'segment_ts': 'CZdM', 'time_q': '2023-02-04 03:49:11'} +{'device': 'kemopcvgaWvww445', 'segment_ts': '9EjB', 'time_q': '2023-01-25 18:58:42'} +{'device': 'Nfuv9QZRaTdtZ1j1', 'segment_ts': 'nyoq', 'time_q': '2023-02-08 15:39:28'} +{'device': 'IDNpRkPQqBYG8rIL', 'segment_ts': 'nrGc', 'time_q': '2023-02-19 09:20:51'} +{'device': 'woj5K9AYcOPi8wbH', 'segment_ts': 'q9o4', 'time_q': '2023-01-30 00:47:23'} +{'device': 'A3Q592GiKPtRBifW', 'segment_ts': '479s', 'time_q': '2023-02-03 13:21:56'} +{'device': 't0tSKqKvPJsSLKek', 'segment_ts': 'PzBM', 'time_q': '2023-02-09 19:16:28'} +{'device': 'xvu9RL3gVCt6YIqd', 'segment_ts': 'OE7b', 'time_q': '2023-02-10 13:47:41'} +{'device': 'WMwNdlIc5a3s0o4d', 'segment_ts': 'WFNu', 'time_q': '2023-02-10 20:06:12'} +{'device': 'kkvalugYPF7CeARB', 'segment_ts': 'xOmt', 'time_q': '2023-01-27 20:04:35'} +{'device': 'O2Pxt9eTcZ8DXb7Z', 'segment_ts': '3sv4', 'time_q': '2023-02-10 23:46:23'} +{'device': 'n67yYlHFaWmgsEIn', 'segment_ts': 'MbqF', 'time_q': '2023-01-30 20:38:08'} +{'device': '4vHRlcIj4fA6nyem', 'segment_ts': 'vroI', 'time_q': '2023-02-14 16:20:29'} +{'device': 'TMbltseSls9UOzxq', 'segment_ts': 'v5gi', 'time_q': '2023-02-20 09:03:38'} +{'device': 'Jj6eg1CNcUpAtY4b', 'segment_ts': 'xd2m', 'time_q': '2023-02-16 06:33:10'} +{'device': 'ipuU6YIHXJVKYBb5', 'segment_ts': 'LwNw', 'time_q': '2023-01-28 00:19:52'} +{'device': 't66AJJAWxgfr54og', 'segment_ts': 'q74O', 'time_q': '2023-01-30 06:53:54'} +{'device': '72tpVrLi7idQ711R', 'segment_ts': 'TuWP', 'time_q': '2023-02-19 22:09:35'} +{'device': 'Hvq7OAQ6pXafCIaP', 'segment_ts': 'OdFT', 'time_q': '2023-02-17 20:46:42'} +{'device': 'pme4dFY8cxRdM0iD', 'segment_ts': '7exz', 'time_q': '2023-02-16 00:45:20'} +{'device': 'urddNEFtldtGx3WR', 'segment_ts': 'eZiz', 'time_q': '2023-02-10 06:35:13'} +{'device': 'uZvYbARAUvBgqqiC', 'segment_ts': 'AwpA', 'time_q': '2023-02-05 08:54:00'} +{'device': 'qynlelnsdXES0L3o', 'segment_ts': 'qe17', 'time_q': '2023-02-04 03:25:06'} +{'device': 'VvuWbHbURvKgsqUF', 'segment_ts': '7Sxg', 'time_q': '2023-02-14 10:47:24'} +{'device': 'TlnhHfBk12CQiQ0K', 'segment_ts': 'QVU0', 'time_q': '2023-02-11 01:01:08'} +{'device': 'pPUdA3tMUudFfuu8', 'segment_ts': 'Ubci', 'time_q': '2023-02-20 07:54:52'} +{'device': 'zgZbzsxqUD8EYtTt', 'segment_ts': 'Oyvx', 'time_q': '2023-02-16 23:14:46'} +{'device': 'mJ4o0n4HIVfcYlOo', 'segment_ts': '1Xba', 'time_q': '2023-02-09 23:33:48'} +{'device': 'aqh9qEavdsDiY54R', 'segment_ts': 'O951', 'time_q': '2023-01-23 21:18:26'} +{'device': 'baPCRewYf20VOMzV', 'segment_ts': '7QZg', 'time_q': '2023-01-30 03:32:06'} +{'device': 'JGqBYR14WAkGBwFM', 'segment_ts': '3ES4', 'time_q': '2023-02-06 06:31:37'} +{'device': 'k5Jpfk62razi9ZgC', 'segment_ts': 'jefZ', 'time_q': '2023-02-21 06:43:37'} +{'device': 'J4R3L5UMS3DjpooS', 'segment_ts': 'J8fh', 'time_q': '2023-02-12 03:19:39'} +{'device': 'mKaEAZkAqmHVYbYx', 'segment_ts': 'NZxR', 'time_q': '2023-02-20 12:07:35'} +{'device': 'mnFNrh7YGDVK0LF0', 'segment_ts': '1yCJ', 'time_q': '2023-01-29 11:05:00'} +{'device': '15x8g2YTRH5wxg8D', 'segment_ts': 'hOB3', 'time_q': '2023-02-15 09:54:07'} +{'device': 'MDhSc461Le45qIOD', 'segment_ts': 'Oqnw', 'time_q': '2023-02-13 12:55:24'} +{'device': '2wtPLtzsT0nw72Ki', 'segment_ts': 'iC7M', 'time_q': '2023-02-13 08:02:54'} +{'device': 'EJN36J2EWxJO6d3A', 'segment_ts': 'aCKQ', 'time_q': '2023-01-26 18:11:35'} +{'device': '4Cibygd0vko8tblj', 'segment_ts': 'xF4p', 'time_q': '2023-01-27 09:25:50'} +{'device': '1EryLdrZkHz9TBM2', 'segment_ts': 'M97m', 'time_q': '2023-02-06 08:09:39'} +{'device': 'NUp95xg6x9K07uJw', 'segment_ts': 'DuKl', 'time_q': '2023-02-15 14:42:37'} +{'device': 'QTvOJ3fHPnsnFksM', 'segment_ts': 'PNnd', 'time_q': '2023-01-23 22:31:31'} \ No newline at end of file diff --git a/idk/kafka/testdata/schemas/timeQuantum.json b/idk/kafka/testdata/schemas/timeQuantum.json new file mode 100644 index 000000000..a3565666e --- /dev/null +++ b/idk/kafka/testdata/schemas/timeQuantum.json @@ -0,0 +1,26 @@ +{ + "doc": "Records for testing time quantums", + "fields": [ + { + "doc": "A device ID", + "name": "device", + "type": "string" + }, + { + "doc": "The segment the device belongs to which is related to time", + "name": "segment_ts", + "type": "string", + "quantum": "YMD" + }, + { + "doc": "The timestamp associated with segment_ts", + "name": "time_q", + "type": "bytes", + "fieldType": "recordTime", + "layout": "2006-01-02 15:04:05" + } + ], + "name": "tq_record", + "namespace": "com.featurebase.tq", + "type": "record" +} \ No newline at end of file From 67d247a479d068add1b90150267150008addd44b Mon Sep 17 00:00:00 2001 From: jacob Date: Tue, 21 Feb 2023 21:01:10 -0600 Subject: [PATCH 08/19] adding time quantum testing --- idk/kafka/cmd_test.go | 174 +++++-- idk/kafka/testdata/records/timeQuantum.json | 500 ++++++++++++++++++++ idk/kafka/testdata/records/timeQuantum.txt | 500 -------------------- 3 files changed, 633 insertions(+), 541 deletions(-) create mode 100644 idk/kafka/testdata/records/timeQuantum.json delete mode 100644 idk/kafka/testdata/records/timeQuantum.txt diff --git a/idk/kafka/cmd_test.go b/idk/kafka/cmd_test.go index 645db6948..397a2e6de 100644 --- a/idk/kafka/cmd_test.go +++ b/idk/kafka/cmd_test.go @@ -4,6 +4,8 @@ package kafka import ( + "bufio" + "encoding/json" "fmt" "io" "math/rand" @@ -639,57 +641,147 @@ func TestCmdSchemaChange(t *testing.T) { } func TestTimeQuantums(t *testing.T) { - - /* - at a high level, a test here represents - - an avro schema - - a set of records to ingest to kafka - - an ingest configuration - - query to run to confirm the data was ingest properly + t.Parallel() + /* + at a high level, a test here represents + - an avro schema + - a set of records to ingest to kafka + - an ingest configuration + - query to run to confirm the data was ingest properly */ tests := []struct { name string - autoGenerateID bool - primaryKeyFields string // nil when idField is not nil - idField string // nil when primaryKeyFields is not nil - PilosaHosts []string - RegistryURL string pathToAvroSchema string pathToRecords string - topic string + idType string // must be "generated", "id", or "string" + keyField string // field used for "id" or "string" record keys + pilosaHosts string + kafkaHost string + registryURL string + topic string + index string + queries []string + expectedResults []string }{ - { - name: "time quantums exist", - autoGenerateID: false, - primaryKeyFields: "device", - idField: nil, - PilosaHosts []string - RegistryURL string - pathToAvroSchema string - pathToRecords string - topic string // don't duplicate - - }, - { - name: "3 primary keys str/str/int TLS", - PrimaryKeyFields: []string{"abc", "db", "user_id"}, - PilosaHosts: []string{pilosaTLSHost}, - TLS: &idk.TLSConfig{ - CertificatePath: certPath + "/theclient.crt", - CertificateKeyPath: certPath + "/theclient.key", - CACertPath: certPath + "/ca.crt", - EnableClientVerification: true, + { // confirm time quantums are being ingested + name: "time quantums exist", + pathToAvroSchema: "timeQuantum.json", + pathToRecords: "./testdata/records/timeQuantum.json", + idType: "string", + keyField: "device", + pilosaHosts: pilosaHost, + registryURL: registryHost, + kafkaHost: kafkaHost, + topic: "timequantums", + index: "timequantums", + queries: []string{ + "Row(segment_ts='7R83')", + "Row(segment_ts='7R83', from=\"2023-02-17T00:00\", to=\"2023-02-18T00:00\")", + "Row(segment_ts='7R83', from=\"2023-02-16T00:00\", to=\"2023-02-17T00:00\")", + }, + expectedResults: []string{ + "{\"results\":[{\"columns\":[],\"keys\":[\"0QKtSTqJYXMZWvVe\"]}]}\n", + "{\"results\":[{\"columns\":[],\"keys\":[\"0QKtSTqJYXMZWvVe\"]}]}\n", + "{\"results\":[{\"columns\":[]}]}\n", }, - expRhinoKeys: []string{"2|1|159", "4|3|44", "123456789|q2db_1234|432"}, // "2" + "1" + uint32(159) - - }, - { - name: "IDField int", - IDField: "user_id", - expRhinoCols: []uint64{44, 159, 432}, }, } + fmt.Printf("created tests") + + for _, test := range tests { + + fmt.Printf("starting test") + // define some vars + now := time.Now().UnixNano() + index := fmt.Sprintf("%s_%d", test.index, now) + topic := fmt.Sprintf("%s_%d", test.topic, now) + + // read in records + var records []map[string]interface{} + var data map[string]interface{} + recordsFile, err := os.Open(test.pathToRecords) + if err != nil { + t.Errorf("opening records file") + } + defer recordsFile.Close() + + s := bufio.NewScanner(recordsFile) + for s.Scan() { + err := json.Unmarshal(s.Bytes(), &data) + if err != nil { + t.Errorf("unmarshal json: %s", err) + } + records = append(records, data) + data = make(map[string]interface{}) + } + + /* + records, err := ioutil.ReadFile(test.pathToRecords) + if err != nil { + t.Errorf("issue reading records file") + } + err = json.Unmarshal(records, &data) + if err != nil { + t.Errorf("unmarshal json: %s", err) + } + */ + fmt.Printf("finished reading records") + + // configure the consumer + consumer, err := NewMain() + if err != nil { + t.Fatalf("creating main %v", err) + } + configureTestFlags(consumer) + consumer.Index = index + consumer.Topics = []string{topic} + //consumer.KafkaBootstrapServers = []string{test.kafkaHost} + //consumer.SchemaRegistryURL = test.registryURL + switch test.idType { + case "id": + consumer.IDField = test.keyField + case "string": + consumer.PrimaryKeyFields = []string{test.keyField} + case "generate": + consumer.AutoGenerate = true + consumer.ExternalGenerate = true + default: + t.Errorf("incorrect idType supplied") + } + consumer.MaxMsgs = uint64(len(records)) + + fmt.Println("finished configuring the consumer") + + // load schema registry, create produce, topic and run consumer data + licodec := liDecodeTestSchema(t, test.pathToAvroSchema) + schemaID := postSchema(t, test.pathToAvroSchema, fmt.Sprintf("%s_id", topic), consumer.SchemaRegistryURL, nil) + p, err := confluent.NewProducer(&confluent.ConfigMap{ + "bootstrap.servers": kafkaHost, + }) + if err != nil { + t.Fatalf("Failed to create producer: %s", err) + } + defer p.Close() + tCreateTopic(t, topic, p) + tPutRecordsKafka(t, p, topic, schemaID, licodec, "akey", records...) + err = consumer.Run() + if err != nil { + t.Fatalf("running consumer: %v", err) + } + + // now run queries and confirm the data is as expected + client := consumer.PilosaClient() + for i, q := range test.queries { + status, body, err := client.HTTPRequest("POST", fmt.Sprintf("/index/%s/query", index), []byte(q), nil) + if err != nil { + t.Fatalf("querying featurebase: status: %d, response: %s, error: %s", status, body, err) + } + if string(body[:]) != test.expectedResults[i] { + t.Fatalf("running query: %s... expected %s but got %s", q, test.expectedResults[i], body) + } + } + } } type sortableCRI []pilosaclient.CountResultItem diff --git a/idk/kafka/testdata/records/timeQuantum.json b/idk/kafka/testdata/records/timeQuantum.json new file mode 100644 index 000000000..7404128f8 --- /dev/null +++ b/idk/kafka/testdata/records/timeQuantum.json @@ -0,0 +1,500 @@ +{"device": "XbJ7vwASddz1xBQ4", "segment_ts": "fub2", "time_q": "2023-02-13 18:11:08"} +{"device": "g3AwJJUDwrN1LYzY", "segment_ts": "Qn3I", "time_q": "2023-02-13 19:39:01"} +{"device": "DIxrncsISTdUJdsH", "segment_ts": "o8Ig", "time_q": "2023-01-27 09:41:54"} +{"device": "yR1RUTVnjwUvjWLS", "segment_ts": "BUnG", "time_q": "2023-01-25 06:55:04"} +{"device": "dpeJtjO0mVsN3wiZ", "segment_ts": "Udwh", "time_q": "2023-01-25 14:34:42"} +{"device": "hOJrFi9xmOnM5J2B", "segment_ts": "9jED", "time_q": "2023-01-27 17:05:19"} +{"device": "xPLZ3qjA029BWoD8", "segment_ts": "PBbR", "time_q": "2023-02-17 08:47:05"} +{"device": "sdnPNO80Trlnkfpi", "segment_ts": "ACVo", "time_q": "2023-02-21 09:01:47"} +{"device": "fcMcP2T8hIBzYzrB", "segment_ts": "HppX", "time_q": "2023-02-18 00:59:36"} +{"device": "xiRO1wD8bdg6tsy2", "segment_ts": "iSQp", "time_q": "2023-01-28 06:15:59"} +{"device": "nlsTgLlvTBhQGewH", "segment_ts": "CZHg", "time_q": "2023-02-20 18:53:14"} +{"device": "bpLtS0dLbzC1t0aM", "segment_ts": "oanZ", "time_q": "2023-02-11 08:25:43"} +{"device": "RcYtSPPTAQyOE80E", "segment_ts": "57He", "time_q": "2023-02-14 06:40:17"} +{"device": "BLqS1Js4hqpAGKbs", "segment_ts": "8Esc", "time_q": "2023-02-13 01:59:32"} +{"device": "NDJ13gRlrfjSpJul", "segment_ts": "fBcm", "time_q": "2023-01-24 07:05:02"} +{"device": "CUS2BbP8fJrBuJRB", "segment_ts": "kUK0", "time_q": "2023-02-05 22:33:45"} +{"device": "lwC1KW3eNMX7kqwS", "segment_ts": "8s12", "time_q": "2023-02-15 19:55:08"} +{"device": "jtEPRgME5UKtmv2O", "segment_ts": "Ew1L", "time_q": "2023-02-10 10:44:56"} +{"device": "zJJQ58p1vZEVr8pV", "segment_ts": "cb9G", "time_q": "2023-01-31 14:49:08"} +{"device": "zGLCqlcb9WEVYeTD", "segment_ts": "JatF", "time_q": "2023-02-19 17:37:42"} +{"device": "PgyMZVp5lPVUAzju", "segment_ts": "H4w7", "time_q": "2023-01-25 01:13:51"} +{"device": "KS2ftxZpJaobrFcH", "segment_ts": "lzde", "time_q": "2023-02-08 07:29:00"} +{"device": "pvfQ90NOCNxkZ9qp", "segment_ts": "meI7", "time_q": "2023-02-01 09:29:32"} +{"device": "et0tVgDi4gQLBbCZ", "segment_ts": "8roi", "time_q": "2023-02-17 13:59:05"} +{"device": "KVpmUGw3YFQWFKUJ", "segment_ts": "reQ2", "time_q": "2023-02-07 05:12:56"} +{"device": "7e62NVroqaZ5KCCG", "segment_ts": "mRqe", "time_q": "2023-01-25 18:55:33"} +{"device": "2jhbxRtroaywIHz0", "segment_ts": "vGvz", "time_q": "2023-01-31 06:31:33"} +{"device": "i8A3oSUCLIRVI3z6", "segment_ts": "bRK7", "time_q": "2023-01-22 15:45:16"} +{"device": "w9m2zmvBebNOYm7M", "segment_ts": "EUTl", "time_q": "2023-02-12 17:45:44"} +{"device": "6goOWv6GmSB5SImL", "segment_ts": "IcRw", "time_q": "2023-02-02 19:44:49"} +{"device": "9TdRfZY8fyUv0MDA", "segment_ts": "jxeF", "time_q": "2023-02-15 12:46:23"} +{"device": "XFT7MaPT04gy9giN", "segment_ts": "NDT5", "time_q": "2023-01-27 02:02:09"} +{"device": "dYf2esJeNHts76qt", "segment_ts": "nD5r", "time_q": "2023-02-20 19:18:20"} +{"device": "XO3Mw07kTWxq4S6A", "segment_ts": "wu98", "time_q": "2023-02-19 09:11:04"} +{"device": "PR20MB3DrKZyxBYN", "segment_ts": "VxO2", "time_q": "2023-02-14 02:48:28"} +{"device": "8HoHoySkF3ONDVMT", "segment_ts": "RZW0", "time_q": "2023-02-17 09:47:36"} +{"device": "D9EMY6KMiUupVo86", "segment_ts": "uKvn", "time_q": "2023-02-13 20:34:37"} +{"device": "6vw27rBMy4z08u0H", "segment_ts": "IhoS", "time_q": "2023-02-07 21:19:31"} +{"device": "8B3tax9WJNOuceOO", "segment_ts": "2sCg", "time_q": "2023-02-13 06:09:00"} +{"device": "q1kmQxWdgK1fI2Aq", "segment_ts": "tpUd", "time_q": "2023-01-22 22:14:51"} +{"device": "VPe3DuOwKmbOf9hj", "segment_ts": "Q1EB", "time_q": "2023-02-07 01:49:40"} +{"device": "bcn6NL9OVcDI9NOE", "segment_ts": "pmBn", "time_q": "2023-02-06 19:24:58"} +{"device": "xveWOtVYpFmdKoE3", "segment_ts": "3pUE", "time_q": "2023-01-29 05:45:06"} +{"device": "O9p8Ij6SiXU7w9Bo", "segment_ts": "wDD7", "time_q": "2023-02-07 02:09:17"} +{"device": "JU1KI8rx8qLXtY9Y", "segment_ts": "NBSG", "time_q": "2023-02-15 13:20:17"} +{"device": "zvc3NHLWCKRbwL4w", "segment_ts": "wAqp", "time_q": "2023-02-06 19:57:45"} +{"device": "uIKOal9xGdwgmxQZ", "segment_ts": "3x0X", "time_q": "2023-02-19 00:21:42"} +{"device": "0Rn9nBxCnr1abKBu", "segment_ts": "K3q7", "time_q": "2023-02-07 03:02:30"} +{"device": "DcPFdJzT9aCgD7hZ", "segment_ts": "1WZE", "time_q": "2023-02-04 22:53:14"} +{"device": "I33U4ld8p1ZTEo2z", "segment_ts": "h327", "time_q": "2023-02-15 09:02:02"} +{"device": "HzV2F1u15P6MfTq4", "segment_ts": "Oi4K", "time_q": "2023-02-18 03:07:18"} +{"device": "Gs8bQMX2wl9AjM5I", "segment_ts": "CT5V", "time_q": "2023-02-16 01:45:58"} +{"device": "aPv5uNdC2x9Zrs31", "segment_ts": "ijS2", "time_q": "2023-01-29 20:43:50"} +{"device": "gmBMopHkBCbcGRaS", "segment_ts": "nD5r", "time_q": "2023-02-11 05:05:58"} +{"device": "g3Ns2caZMQVlOSPr", "segment_ts": "9vzm", "time_q": "2023-01-27 00:22:33"} +{"device": "JWY6hPLwnhy76KGZ", "segment_ts": "2IS2", "time_q": "2023-02-11 21:47:10"} +{"device": "0QKtSTqJYXMZWvVe", "segment_ts": "7R83", "time_q": "2023-02-17 05:40:55"} +{"device": "Ximq2ebfKR1eqySD", "segment_ts": "Jpup", "time_q": "2023-01-23 17:34:53"} +{"device": "YWGLIrHDKwe6UUM1", "segment_ts": "Smbg", "time_q": "2023-02-11 22:29:38"} +{"device": "i8Hy1Hhy6vbQ7Gnj", "segment_ts": "Xjn9", "time_q": "2023-02-07 05:52:25"} +{"device": "trQjczSWWWqD3xip", "segment_ts": "zSyE", "time_q": "2023-02-14 03:51:05"} +{"device": "cRAvzL88YU74zOVR", "segment_ts": "FAk3", "time_q": "2023-02-02 06:26:40"} +{"device": "H7RUb4moxDU5RY8L", "segment_ts": "6sUs", "time_q": "2023-02-06 15:29:02"} +{"device": "g6uZFROq6hA9VwXU", "segment_ts": "9v3C", "time_q": "2023-01-30 05:29:57"} +{"device": "yIGidxsJbbyaTMJu", "segment_ts": "5TbD", "time_q": "2023-01-31 20:18:06"} +{"device": "DC9B8r6fLUrQEKWd", "segment_ts": "VDz6", "time_q": "2023-02-09 22:19:07"} +{"device": "rymp2l5MJvVdAEiU", "segment_ts": "fyte", "time_q": "2023-02-02 11:53:13"} +{"device": "Ua1gF72JZ4WAc8Bd", "segment_ts": "v5sp", "time_q": "2023-01-24 02:06:04"} +{"device": "9qSO5sST0XGp5jMS", "segment_ts": "7dBw", "time_q": "2023-02-10 09:35:42"} +{"device": "p0HAiSpiSNAaCrZB", "segment_ts": "a1iQ", "time_q": "2023-01-31 15:45:28"} +{"device": "wkYtecM7PdltlJ6U", "segment_ts": "tCye", "time_q": "2023-01-29 14:50:59"} +{"device": "efTNUHAxzw90hiOb", "segment_ts": "9Wf1", "time_q": "2023-02-18 02:29:02"} +{"device": "fQI3EBaehVoTHSuQ", "segment_ts": "zRBT", "time_q": "2023-02-11 19:06:37"} +{"device": "udeapVgb384YhUvm", "segment_ts": "oL67", "time_q": "2023-02-05 10:27:36"} +{"device": "eqbom6mcHS60rV4o", "segment_ts": "GNdZ", "time_q": "2023-02-11 08:44:18"} +{"device": "1wWPwlCsbs9ZBcZC", "segment_ts": "zaKC", "time_q": "2023-02-08 00:49:32"} +{"device": "ecY2Gsp5o5y8RDEy", "segment_ts": "QWzq", "time_q": "2023-01-29 02:43:04"} +{"device": "DxqHMz1dx4Bqsobg", "segment_ts": "2RQv", "time_q": "2023-02-18 21:50:04"} +{"device": "XU1b7NOcHSuGzCUv", "segment_ts": "Yuoy", "time_q": "2023-02-13 21:54:55"} +{"device": "lO2wwnU1eJPm8GXQ", "segment_ts": "AP3b", "time_q": "2023-01-28 14:02:06"} +{"device": "tLnTsvtbSVqjrYe9", "segment_ts": "fFTT", "time_q": "2023-01-24 23:23:30"} +{"device": "BD3RZkRDscI24QIW", "segment_ts": "3Bry", "time_q": "2023-02-14 17:16:33"} +{"device": "EJTuSaV3BN56Wj2O", "segment_ts": "W4JQ", "time_q": "2023-02-01 18:56:00"} +{"device": "TfinqcstHOBc9vgU", "segment_ts": "VybU", "time_q": "2023-01-30 13:41:42"} +{"device": "oLCaL36jSe58AJzI", "segment_ts": "28pM", "time_q": "2023-02-03 07:47:20"} +{"device": "I85aiLVze3vm3TkQ", "segment_ts": "y8y1", "time_q": "2023-02-19 19:12:25"} +{"device": "J1Z6A5uaqDbcJCfB", "segment_ts": "Wzwh", "time_q": "2023-02-10 08:51:06"} +{"device": "DUJWSG7TcQrG1SiU", "segment_ts": "qKgC", "time_q": "2023-02-18 22:00:14"} +{"device": "oq4StgCLUoilIYHQ", "segment_ts": "MNLg", "time_q": "2023-02-05 10:11:01"} +{"device": "P44lydZpIGjbmQOy", "segment_ts": "uHCR", "time_q": "2023-01-22 21:38:00"} +{"device": "8lV1uEL9zdwqOuwT", "segment_ts": "8wB6", "time_q": "2023-02-05 21:52:08"} +{"device": "uVGi6rqE7NKl7D0W", "segment_ts": "kge3", "time_q": "2023-02-12 14:45:28"} +{"device": "fBDDRmw5ifz9Ibk1", "segment_ts": "e8wv", "time_q": "2023-02-12 01:36:43"} +{"device": "q8q2AXRsgPrtU7MF", "segment_ts": "lznG", "time_q": "2023-01-31 09:08:30"} +{"device": "rKVwah03kvEN1Xf8", "segment_ts": "CIf7", "time_q": "2023-02-17 15:32:01"} +{"device": "5gqkkJu0HnW8e7Jd", "segment_ts": "11VE", "time_q": "2023-02-21 12:50:18"} +{"device": "67aeTL1Lhk7zPniw", "segment_ts": "7WnQ", "time_q": "2023-02-15 09:54:03"} +{"device": "JSqkbIUNs9XsDhyf", "segment_ts": "Vef3", "time_q": "2023-01-28 05:40:03"} +{"device": "O6pY363lkXtNFnPW", "segment_ts": "JPG3", "time_q": "2023-02-09 07:18:08"} +{"device": "tpldKtJXhNOJs5is", "segment_ts": "x40J", "time_q": "2023-02-19 03:15:47"} +{"device": "zJFXIMKFyyMjvHzJ", "segment_ts": "ddlX", "time_q": "2023-02-12 02:18:47"} +{"device": "o0W2DatNPhPHeUCG", "segment_ts": "3Lzx", "time_q": "2023-02-01 03:05:37"} +{"device": "0qNRr7JlwEJo0UZs", "segment_ts": "x1R8", "time_q": "2023-02-04 16:06:55"} +{"device": "ybv2vBmnJIMXDQ7C", "segment_ts": "DyAY", "time_q": "2023-02-04 14:33:24"} +{"device": "7bkYnhDEWrdlKPzi", "segment_ts": "CHYf", "time_q": "2023-02-18 13:52:19"} +{"device": "gS5FFLJPIgSpPn2p", "segment_ts": "73ea", "time_q": "2023-02-20 13:38:30"} +{"device": "Iz5J7QbGpOaw9Mc3", "segment_ts": "UD8U", "time_q": "2023-02-19 21:56:03"} +{"device": "Q4zAccg3hJyiWPQV", "segment_ts": "WbDH", "time_q": "2023-02-19 21:57:53"} +{"device": "vwowDnNJnQ3Noych", "segment_ts": "YyvE", "time_q": "2023-02-07 21:35:53"} +{"device": "V4Yg3xtwNAdXT7AC", "segment_ts": "OQ1f", "time_q": "2023-01-27 03:41:03"} +{"device": "kBcU57h5YJlps0M7", "segment_ts": "X9xH", "time_q": "2023-02-16 21:18:05"} +{"device": "mTOckeiB1Myp9PzA", "segment_ts": "wDD7", "time_q": "2023-02-16 06:44:48"} +{"device": "jFqBXPPMyJJaM4VG", "segment_ts": "csTs", "time_q": "2023-02-06 02:52:21"} +{"device": "vKNH7K6ozve7702k", "segment_ts": "SJ25", "time_q": "2023-02-08 01:27:29"} +{"device": "oyT2TtgfCba2Hdml", "segment_ts": "Tvqb", "time_q": "2023-02-21 03:13:30"} +{"device": "rsOlS9C8eEy347rZ", "segment_ts": "hr7p", "time_q": "2023-02-19 02:30:46"} +{"device": "90GjHMWLi1dcpp0y", "segment_ts": "W4JQ", "time_q": "2023-01-25 02:29:48"} +{"device": "fvudPtjiPgFEJ7bD", "segment_ts": "lwno", "time_q": "2023-01-24 05:44:13"} +{"device": "VBWAgZvfVhht5RLf", "segment_ts": "Ftw2", "time_q": "2023-01-27 13:08:57"} +{"device": "sZK0cJqcYBnZXwyj", "segment_ts": "9G1V", "time_q": "2023-01-27 21:13:09"} +{"device": "Nw2Qbh1azig2aS8i", "segment_ts": "VPUK", "time_q": "2023-02-20 06:55:48"} +{"device": "YrmDmrxR6yVVsmBK", "segment_ts": "kEmM", "time_q": "2023-02-17 04:13:52"} +{"device": "FJg8emti6uDPkZWb", "segment_ts": "cu9S", "time_q": "2023-01-31 15:15:17"} +{"device": "D2KVrcXONfBL0DOq", "segment_ts": "E7q4", "time_q": "2023-02-07 05:21:14"} +{"device": "zpjzEqsoIDSfgkrw", "segment_ts": "kuSy", "time_q": "2023-02-08 21:39:35"} +{"device": "TyBhOHhPHg3mABfs", "segment_ts": "FOLC", "time_q": "2023-02-05 01:50:35"} +{"device": "SSsebFMnz2vz0lKT", "segment_ts": "qXUp", "time_q": "2023-02-17 15:12:43"} +{"device": "oeMORvnHFqNvuqA9", "segment_ts": "D16L", "time_q": "2023-02-17 12:56:55"} +{"device": "pZGE5Q8ax0QnrKlK", "segment_ts": "6O2l", "time_q": "2023-02-17 06:43:06"} +{"device": "xXUDPhByePkaTvpe", "segment_ts": "UqWC", "time_q": "2023-02-05 01:26:33"} +{"device": "8zOXbhQQFeXPuMgW", "segment_ts": "v5s5", "time_q": "2023-01-28 21:54:59"} +{"device": "Kvd0ii9qinTBgkEK", "segment_ts": "YWRA", "time_q": "2023-02-15 04:02:15"} +{"device": "JoC3FaiJFH7nWQbY", "segment_ts": "RV1o", "time_q": "2023-01-23 11:40:04"} +{"device": "JZtyjzapfAhKTdEL", "segment_ts": "6Hyt", "time_q": "2023-02-12 23:31:41"} +{"device": "lsE2LnvCTkfUv3Hb", "segment_ts": "mvNJ", "time_q": "2023-02-20 04:17:14"} +{"device": "ZaG5u6BctGV7Phak", "segment_ts": "xS0C", "time_q": "2023-02-12 21:40:07"} +{"device": "VqkvcCFemWmmdLIN", "segment_ts": "1vCq", "time_q": "2023-02-04 13:43:54"} +{"device": "MnWYYlFI787w7ENs", "segment_ts": "YHny", "time_q": "2023-02-02 04:12:38"} +{"device": "5dFd1cLHS8fjZaOz", "segment_ts": "0JSK", "time_q": "2023-02-14 05:41:43"} +{"device": "G7r2BebALuPczt3b", "segment_ts": "nowH", "time_q": "2023-02-19 08:55:48"} +{"device": "7V6SBGSNrMZUvrw8", "segment_ts": "KKuH", "time_q": "2023-02-07 03:00:14"} +{"device": "YWmt4ZWofrE68Eg4", "segment_ts": "9ub1", "time_q": "2023-01-24 10:32:07"} +{"device": "RgsF3Paw1CFHmXrz", "segment_ts": "GKFG", "time_q": "2023-02-05 14:23:12"} +{"device": "uRUd8DnEptcCg051", "segment_ts": "w6bQ", "time_q": "2023-02-12 02:55:44"} +{"device": "gpJn2OhNJ17yFKfj", "segment_ts": "Ms6E", "time_q": "2023-02-11 12:28:14"} +{"device": "OS8p3UgcZwCtitJ5", "segment_ts": "lUE6", "time_q": "2023-02-12 12:43:29"} +{"device": "AaBf2goUzSIOLwKU", "segment_ts": "jRhp", "time_q": "2023-01-28 02:43:57"} +{"device": "Gi7o9G6zuxIje6TA", "segment_ts": "TPjM", "time_q": "2023-01-22 21:42:14"} +{"device": "Zgp8HrnLjtoUBstx", "segment_ts": "669s", "time_q": "2023-01-23 07:40:22"} +{"device": "yZ39wYgza1sjjH6a", "segment_ts": "80La", "time_q": "2023-01-25 23:58:17"} +{"device": "gKSePVA4eUpPeQeB", "segment_ts": "uPAl", "time_q": "2023-02-06 04:22:11"} +{"device": "ZvPnZt45ZdDJf2wV", "segment_ts": "eQg8", "time_q": "2023-01-29 16:01:24"} +{"device": "RGaReXrDxjLvIN1h", "segment_ts": "wV8N", "time_q": "2023-01-27 01:38:59"} +{"device": "DBG2BbMzd2AiJ1ra", "segment_ts": "or1s", "time_q": "2023-02-14 19:21:12"} +{"device": "LaRpvrLjCem5I1iG", "segment_ts": "MKyg", "time_q": "2023-01-22 20:08:37"} +{"device": "6VPMKOpB2Gax57AM", "segment_ts": "jWnU", "time_q": "2023-02-02 08:27:23"} +{"device": "WWmDiCgEhcFaRoxd", "segment_ts": "IcRw", "time_q": "2023-01-28 20:36:01"} +{"device": "g8tKyCuvwcmnOleP", "segment_ts": "l57C", "time_q": "2023-02-21 12:15:51"} +{"device": "mGmZoqlgLWwRHhWd", "segment_ts": "MJWQ", "time_q": "2023-01-26 00:54:26"} +{"device": "9vYzND4LhGIpLRPs", "segment_ts": "X8An", "time_q": "2023-02-17 17:12:20"} +{"device": "TqAci7xzsz9HKRwX", "segment_ts": "HQvb", "time_q": "2023-02-07 19:26:19"} +{"device": "jXeyesYOxpErf0Wr", "segment_ts": "9Pah", "time_q": "2023-01-29 07:43:44"} +{"device": "PKpsammUcBhHIj4d", "segment_ts": "SJ96", "time_q": "2023-02-01 11:22:01"} +{"device": "d43Y87nTdkx4nw0N", "segment_ts": "qt7V", "time_q": "2023-02-13 07:35:55"} +{"device": "3wSdpuLVAqb2DqwQ", "segment_ts": "1uul", "time_q": "2023-02-20 17:20:48"} +{"device": "b86HMaTJskEFSegv", "segment_ts": "e0pl", "time_q": "2023-02-20 04:23:59"} +{"device": "q7XDusNiRySmmaRj", "segment_ts": "7dBw", "time_q": "2023-01-29 01:47:01"} +{"device": "LrWDgsZruGK3akSe", "segment_ts": "0vLc", "time_q": "2023-02-20 13:19:00"} +{"device": "JJOo2fsdKEG6QCRu", "segment_ts": "LN5W", "time_q": "2023-02-12 01:36:02"} +{"device": "iXc2nZTT1U0iWf2W", "segment_ts": "3ftI", "time_q": "2023-02-21 10:12:08"} +{"device": "YJ4TjVpH6ZQhMvrH", "segment_ts": "IVww", "time_q": "2023-01-24 12:14:46"} +{"device": "4cBW0bSimlW8ygs0", "segment_ts": "tGGE", "time_q": "2023-02-13 11:54:22"} +{"device": "Ye2BuE77YVbChRNA", "segment_ts": "SRVM", "time_q": "2023-01-25 16:55:03"} +{"device": "BD4sQkvrmTWcfgdd", "segment_ts": "uqv1", "time_q": "2023-02-17 14:21:23"} +{"device": "kB9K8PKJpQnkJTcd", "segment_ts": "WCit", "time_q": "2023-02-20 20:54:19"} +{"device": "uSAbl2Nyu6nK3ddO", "segment_ts": "0cq0", "time_q": "2023-01-28 17:04:49"} +{"device": "uJEVeGx0jSYrxfsx", "segment_ts": "GL95", "time_q": "2023-02-19 21:43:48"} +{"device": "wowb2b399vUqVxWH", "segment_ts": "YcDM", "time_q": "2023-02-03 10:03:34"} +{"device": "Yt42EPAk8TRsKsha", "segment_ts": "9vzm", "time_q": "2023-02-06 16:06:01"} +{"device": "nuQgvAgPYUF1ML4s", "segment_ts": "SOUQ", "time_q": "2023-02-02 10:23:33"} +{"device": "LEuEmNLEgjoCu3p1", "segment_ts": "Mhpw", "time_q": "2023-01-31 19:35:49"} +{"device": "4H1WoLSdS2qs5jOM", "segment_ts": "II21", "time_q": "2023-02-10 14:24:09"} +{"device": "dN5oTZXijbMTz8OP", "segment_ts": "NOvo", "time_q": "2023-01-27 11:27:12"} +{"device": "i2ITUxoWBCEdW8Bs", "segment_ts": "FqD3", "time_q": "2023-01-30 06:33:32"} +{"device": "NyZi9ibptktAZTjI", "segment_ts": "C24z", "time_q": "2023-02-13 18:48:37"} +{"device": "5MMCow4pOwRmSIlc", "segment_ts": "NuYD", "time_q": "2023-02-03 15:50:35"} +{"device": "lK52wLIAUCcXP4AH", "segment_ts": "geC5", "time_q": "2023-02-14 07:33:08"} +{"device": "kdEACkrL59EG8vBX", "segment_ts": "44WL", "time_q": "2023-02-15 16:46:30"} +{"device": "BZhrV8EAlPBft7XX", "segment_ts": "w04k", "time_q": "2023-02-16 03:47:55"} +{"device": "8mKdbrORBbygBkPb", "segment_ts": "3x0X", "time_q": "2023-01-27 04:58:59"} +{"device": "C3ue47cXeXukj7rU", "segment_ts": "7AXG", "time_q": "2023-02-14 15:59:38"} +{"device": "PFZtQxNxLhvhEuil", "segment_ts": "mCKg", "time_q": "2023-02-12 16:29:55"} +{"device": "Qfa7tEckTSssilkU", "segment_ts": "3Epz", "time_q": "2023-01-30 04:49:42"} +{"device": "OZGXqiDoUPo7codg", "segment_ts": "k234", "time_q": "2023-02-21 02:14:45"} +{"device": "VcXbfEsURAyiOlXz", "segment_ts": "XXHG", "time_q": "2023-02-06 21:09:36"} +{"device": "d9jY3TNnU9v39DOE", "segment_ts": "JBAB", "time_q": "2023-02-13 05:44:52"} +{"device": "bY1aO1ktexSWKAjB", "segment_ts": "OUWG", "time_q": "2023-02-19 16:34:05"} +{"device": "4J0KbbaKXbT17iAy", "segment_ts": "THLZ", "time_q": "2023-02-19 16:01:05"} +{"device": "TdfnSWLdZuoHDmT4", "segment_ts": "9bQC", "time_q": "2023-02-13 07:47:03"} +{"device": "vKvuSLFrmEElACFv", "segment_ts": "WgTc", "time_q": "2023-02-01 05:41:50"} +{"device": "EgUuFOflBQ4zUJz2", "segment_ts": "dfJl", "time_q": "2023-02-11 07:30:24"} +{"device": "3cWfkAf3Ma8T8KkO", "segment_ts": "PDJS", "time_q": "2023-02-05 03:09:15"} +{"device": "xoCum1PFgRMFFAtd", "segment_ts": "bdMN", "time_q": "2023-02-10 21:10:02"} +{"device": "G10WOxXP7vDSrqG2", "segment_ts": "9vHF", "time_q": "2023-01-25 10:44:21"} +{"device": "GyDiESn1pQ7C2eOq", "segment_ts": "X5WH", "time_q": "2023-02-10 05:32:32"} +{"device": "Ihum52kryw40U2Pq", "segment_ts": "YQGo", "time_q": "2023-01-29 03:27:14"} +{"device": "9DeVCz49iKTNFPyr", "segment_ts": "2rzd", "time_q": "2023-01-24 10:45:49"} +{"device": "7mW6uwngb7RD2lEp", "segment_ts": "5jBK", "time_q": "2023-02-11 22:13:48"} +{"device": "xuW4yuuFgbdd4a6p", "segment_ts": "lkBB", "time_q": "2023-01-31 18:54:23"} +{"device": "5kzp9NXHYWNPgYoh", "segment_ts": "fioj", "time_q": "2023-01-26 03:08:34"} +{"device": "gmWHoGjwiWF82jHM", "segment_ts": "kiO6", "time_q": "2023-01-22 18:57:36"} +{"device": "zFyepAAJnlCoeqEU", "segment_ts": "tJnv", "time_q": "2023-02-01 00:26:55"} +{"device": "EglsUmauQATFBwu6", "segment_ts": "fGsr", "time_q": "2023-01-30 09:02:11"} +{"device": "qUd53Kl4mymteZxS", "segment_ts": "TdgJ", "time_q": "2023-01-27 05:11:33"} +{"device": "F5zBVO0bkWwMjrQ3", "segment_ts": "yEsz", "time_q": "2023-02-17 00:44:53"} +{"device": "HDgL5w5o1AyW8KNq", "segment_ts": "WMDV", "time_q": "2023-02-20 04:25:58"} +{"device": "b080loiW7g8hJYZN", "segment_ts": "s8vj", "time_q": "2023-01-26 23:15:47"} +{"device": "PLB1THGiEIKtVnuq", "segment_ts": "eqHY", "time_q": "2023-02-17 07:42:59"} +{"device": "Y90nqduLp4v38lVD", "segment_ts": "hZVY", "time_q": "2023-02-15 14:26:35"} +{"device": "fh35hFi6oIEaA5Df", "segment_ts": "TJ6P", "time_q": "2023-01-28 02:34:05"} +{"device": "RBLsLZ4eIt3nrLPH", "segment_ts": "7WHK", "time_q": "2023-01-29 20:05:29"} +{"device": "9Db2XAc0xDm9Mbmx", "segment_ts": "ls7g", "time_q": "2023-02-18 08:15:15"} +{"device": "M1Ih4aY0dYWKWegA", "segment_ts": "sgfr", "time_q": "2023-02-04 00:43:41"} +{"device": "0YtmJi8QdEGnWaHz", "segment_ts": "5HQX", "time_q": "2023-01-25 06:09:08"} +{"device": "LjuzkroRYOCdQSz5", "segment_ts": "dzph", "time_q": "2023-01-24 00:27:56"} +{"device": "K0y5JORnnCPqA3PL", "segment_ts": "6jsI", "time_q": "2023-02-06 10:18:08"} +{"device": "75xJYP1muHt9LVvt", "segment_ts": "2PcN", "time_q": "2023-01-25 23:49:07"} +{"device": "aa15VzCz3SZrbBAl", "segment_ts": "FfVa", "time_q": "2023-02-07 04:37:48"} +{"device": "65lf7rvFyN74HbTL", "segment_ts": "q9o4", "time_q": "2023-02-10 16:57:52"} +{"device": "BkATNfc6INXfprd4", "segment_ts": "QW7U", "time_q": "2023-02-18 04:08:48"} +{"device": "HSBPAi87vKtuhabY", "segment_ts": "Ft0J", "time_q": "2023-01-29 04:20:36"} +{"device": "Rb23K4sxSmDFCiK2", "segment_ts": "CZHg", "time_q": "2023-02-04 22:53:44"} +{"device": "VzfrrwW2GG8Au619", "segment_ts": "qmhZ", "time_q": "2023-01-29 07:18:18"} +{"device": "UW5oYzVVoP9VUNvi", "segment_ts": "FD5Q", "time_q": "2023-02-13 03:55:02"} +{"device": "7ZqKm1mzPqTjKAK8", "segment_ts": "yAxf", "time_q": "2023-02-03 08:04:47"} +{"device": "RKs79WJe0pmc30Ji", "segment_ts": "iQ7P", "time_q": "2023-02-14 20:17:09"} +{"device": "fawEL3nd7h4bsCSA", "segment_ts": "5JQi", "time_q": "2023-02-09 09:47:37"} +{"device": "1cK5y0fv2oasCsEg", "segment_ts": "FZfm", "time_q": "2023-01-24 23:15:26"} +{"device": "fZlV1wQfgN0slT78", "segment_ts": "QjWp", "time_q": "2023-01-23 18:30:49"} +{"device": "ReYnQoZKQD7FhiPY", "segment_ts": "uTjn", "time_q": "2023-01-25 01:24:16"} +{"device": "NPuU31MUQXT5txEw", "segment_ts": "clSh", "time_q": "2023-02-04 23:56:25"} +{"device": "uu9ToklvVI6lAa9s", "segment_ts": "9v3C", "time_q": "2023-01-23 19:15:43"} +{"device": "iVXaaRjEBBkz0hz3", "segment_ts": "aV1U", "time_q": "2023-02-05 20:11:12"} +{"device": "xM0S07A9cCsd9Yfn", "segment_ts": "MeDE", "time_q": "2023-01-24 04:13:01"} +{"device": "nQHMNXhbyFZ6UoDn", "segment_ts": "K4Ze", "time_q": "2023-02-10 16:22:30"} +{"device": "fXaNQOwz30KmlHrP", "segment_ts": "36ap", "time_q": "2023-02-14 12:08:13"} +{"device": "dw2ya1E1U6tjsd7T", "segment_ts": "Jb8U", "time_q": "2023-02-14 03:53:01"} +{"device": "GZ0h5bYPi6E6ohn4", "segment_ts": "44WL", "time_q": "2023-01-29 13:46:23"} +{"device": "wNZt7rqhrv3YtKuz", "segment_ts": "zaKC", "time_q": "2023-01-25 03:03:32"} +{"device": "RpROmDVcSKPVt0C8", "segment_ts": "lvES", "time_q": "2023-02-17 21:12:47"} +{"device": "jbZJ7XFo3s3aLnjJ", "segment_ts": "cvD6", "time_q": "2023-02-17 14:55:37"} +{"device": "4Qw6ei7d9Ux7cTDd", "segment_ts": "gwio", "time_q": "2023-02-01 08:26:39"} +{"device": "JLYhTM9PRqiptyZD", "segment_ts": "QrPy", "time_q": "2023-01-26 09:58:00"} +{"device": "6FO0vI0i8EqKQhd6", "segment_ts": "hH9B", "time_q": "2023-01-31 23:43:46"} +{"device": "iGeoTrx9INiZGRHX", "segment_ts": "X2bk", "time_q": "2023-01-30 16:07:24"} +{"device": "5pRaiBzTt31JZkve", "segment_ts": "8Auu", "time_q": "2023-02-05 09:10:17"} +{"device": "OvWvtocpkdNLqrny", "segment_ts": "Inti", "time_q": "2023-02-12 21:28:35"} +{"device": "bYwqcLrv6vNbGiov", "segment_ts": "ysSv", "time_q": "2023-02-14 22:07:17"} +{"device": "o9lqa5RcsnYdxzPx", "segment_ts": "R8bW", "time_q": "2023-01-31 19:56:44"} +{"device": "WmUqr1hnmvlRw6YO", "segment_ts": "5viW", "time_q": "2023-01-28 20:23:04"} +{"device": "JhzC3ZigvlLK1seo", "segment_ts": "J5sY", "time_q": "2023-02-12 20:59:49"} +{"device": "evKPQwjRV6cBQvjy", "segment_ts": "dDZY", "time_q": "2023-01-31 04:46:11"} +{"device": "sLMrZiBFi2lonYwg", "segment_ts": "unUy", "time_q": "2023-02-15 08:51:34"} +{"device": "o4lT6snRzZnQa1HO", "segment_ts": "Z7Ft", "time_q": "2023-02-03 21:15:32"} +{"device": "hqZWBbZog5N1kD9V", "segment_ts": "WBtx", "time_q": "2023-02-19 18:01:06"} +{"device": "xXzLPLm0OsnDJwUz", "segment_ts": "K1Yr", "time_q": "2023-01-28 23:13:12"} +{"device": "KXSUj8l7UVFARe7n", "segment_ts": "BvbW", "time_q": "2023-01-29 21:44:04"} +{"device": "UvZA7BW1fWlt0GbE", "segment_ts": "ajjJ", "time_q": "2023-01-24 03:31:50"} +{"device": "cQGXvK61LboETbwo", "segment_ts": "Ey3g", "time_q": "2023-01-23 21:46:37"} +{"device": "CQ101gI3Mx4eORJw", "segment_ts": "Jb8U", "time_q": "2023-01-28 03:33:22"} +{"device": "ed2KkpZOCurKwUW1", "segment_ts": "ZWmw", "time_q": "2023-02-15 06:18:12"} +{"device": "cmsyEAqBqINzHOdc", "segment_ts": "MXoi", "time_q": "2023-02-10 07:11:49"} +{"device": "HrEkRR54fFMG0LDI", "segment_ts": "siac", "time_q": "2023-02-07 23:36:13"} +{"device": "wxGldUWtgrjazC1G", "segment_ts": "tvS0", "time_q": "2023-02-04 05:46:44"} +{"device": "RxPCG6v5f4sqllhf", "segment_ts": "Ysq9", "time_q": "2023-02-12 12:54:14"} +{"device": "r7xQr423rEko5eka", "segment_ts": "XPPE", "time_q": "2023-02-07 03:44:00"} +{"device": "wxuOz2ANcqk4RoaO", "segment_ts": "lDol", "time_q": "2023-02-16 15:04:43"} +{"device": "PlSBAhofMJFpB2uV", "segment_ts": "RFaV", "time_q": "2023-02-11 22:02:11"} +{"device": "hZ1tbXscywB9ibyA", "segment_ts": "xnZO", "time_q": "2023-02-20 15:51:50"} +{"device": "KbFXsiA8Q0SqfSF2", "segment_ts": "glBE", "time_q": "2023-02-18 23:04:11"} +{"device": "MRPK2r2JhCir2u2l", "segment_ts": "Med8", "time_q": "2023-01-24 09:27:31"} +{"device": "cEFmg5V2xJHPyqp5", "segment_ts": "Kd02", "time_q": "2023-02-10 09:41:59"} +{"device": "1lwjNCCV5WEJzXT7", "segment_ts": "5CQq", "time_q": "2023-01-27 01:07:21"} +{"device": "VvOulNjY9xQCblv7", "segment_ts": "C8F7", "time_q": "2023-01-27 21:04:34"} +{"device": "dF1H0PGvmSbpRyiI", "segment_ts": "eD44", "time_q": "2023-01-31 13:38:45"} +{"device": "NkDgwYVx0XveN5Dt", "segment_ts": "WxCz", "time_q": "2023-02-21 07:16:46"} +{"device": "LBsuQUNvWKpyd89V", "segment_ts": "V9Fm", "time_q": "2023-02-10 12:25:41"} +{"device": "n1KoKQk8oZ8lUPtV", "segment_ts": "J7YR", "time_q": "2023-02-16 05:30:48"} +{"device": "YDqHeGZquVwEjqQc", "segment_ts": "2GHM", "time_q": "2023-02-15 00:35:09"} +{"device": "RgQGcEnSZcPxClfB", "segment_ts": "hvML", "time_q": "2023-01-23 08:49:45"} +{"device": "jSvUQlxGgJWX0n7M", "segment_ts": "HMSz", "time_q": "2023-02-12 06:13:19"} +{"device": "K7lIlpxjv71aoW9n", "segment_ts": "rkkj", "time_q": "2023-02-02 15:11:54"} +{"device": "tlcCO6MF2RYl0g96", "segment_ts": "10GK", "time_q": "2023-02-17 18:09:56"} +{"device": "p4o8sJpApQwJOiox", "segment_ts": "XAUS", "time_q": "2023-01-29 10:34:46"} +{"device": "525KZ78kEncXH9VI", "segment_ts": "ugYO", "time_q": "2023-02-03 20:14:10"} +{"device": "7arB9WqfPZVvkzAs", "segment_ts": "Jydo", "time_q": "2023-02-09 13:03:19"} +{"device": "GsqKaz0OuOy0p8D5", "segment_ts": "s3Xs", "time_q": "2023-02-01 17:04:58"} +{"device": "NJRTnXytfyhbSltj", "segment_ts": "pa4b", "time_q": "2023-01-22 23:47:33"} +{"device": "V6BcYDFUslTaeKz2", "segment_ts": "c3ip", "time_q": "2023-01-24 12:24:16"} +{"device": "Lali4GnxbDXiICfd", "segment_ts": "5HQX", "time_q": "2023-02-08 07:21:08"} +{"device": "4G67CFMQFfw5nKhH", "segment_ts": "HfZ3", "time_q": "2023-01-27 13:08:26"} +{"device": "VOU1PEANNqFpksPD", "segment_ts": "UCFz", "time_q": "2023-02-02 17:55:58"} +{"device": "NQkRwi0QBjL9OUfe", "segment_ts": "peAr", "time_q": "2023-01-31 09:44:07"} +{"device": "C1Nzu0HNqUqhiv4v", "segment_ts": "ido8", "time_q": "2023-02-20 11:23:58"} +{"device": "jc76NMsTRXg6yFT8", "segment_ts": "IlVL", "time_q": "2023-02-04 08:44:26"} +{"device": "mkPyfZXT9VoyITp1", "segment_ts": "AElH", "time_q": "2023-02-01 22:49:56"} +{"device": "aODXvK25bBgLB89w", "segment_ts": "UyMT", "time_q": "2023-02-06 18:46:10"} +{"device": "bIRJnUdmTWC4bQjs", "segment_ts": "q6P2", "time_q": "2023-01-25 10:01:19"} +{"device": "NwthNY2x1PiG4NXu", "segment_ts": "ziKj", "time_q": "2023-01-26 00:26:01"} +{"device": "Y0Jwgj0CtaaVPGLU", "segment_ts": "uRmO", "time_q": "2023-02-13 01:29:07"} +{"device": "P59OfLIdsNz1u2bf", "segment_ts": "hm6H", "time_q": "2023-01-25 21:43:28"} +{"device": "dHwliEj94uGOjy4z", "segment_ts": "bgF8", "time_q": "2023-02-16 14:27:07"} +{"device": "Tp85tKVvwtOGxCsN", "segment_ts": "z871", "time_q": "2023-02-03 03:24:04"} +{"device": "Nu5VHjVpGjotVjHh", "segment_ts": "u8t8", "time_q": "2023-02-14 04:32:01"} +{"device": "rWcINdZOiu9BHzkL", "segment_ts": "CVvt", "time_q": "2023-02-07 13:15:38"} +{"device": "yGILVrj3iinOAS4U", "segment_ts": "0fQb", "time_q": "2023-02-01 08:06:25"} +{"device": "CkVe5GZWBIuoCDh9", "segment_ts": "0JA6", "time_q": "2023-02-14 22:41:17"} +{"device": "gGG9uQjtROdibp0O", "segment_ts": "mwXi", "time_q": "2023-02-18 08:50:46"} +{"device": "GljeFNPy2bmdxmEz", "segment_ts": "7Jts", "time_q": "2023-02-06 12:45:43"} +{"device": "3xlhVhGTJmURk3MX", "segment_ts": "NZxR", "time_q": "2023-02-02 14:51:14"} +{"device": "47ujFWpbmPFKdX1g", "segment_ts": "KNqJ", "time_q": "2023-01-25 06:27:30"} +{"device": "gNDkMVt00vJT84jK", "segment_ts": "WX3F", "time_q": "2023-02-09 12:57:48"} +{"device": "LCaDtdl6EqAmPlGU", "segment_ts": "AVAC", "time_q": "2023-02-14 13:38:47"} +{"device": "RMS3UOMf4mt5Nlfx", "segment_ts": "dDQ0", "time_q": "2023-02-15 20:39:12"} +{"device": "mCCyHPx2EyQkjDI6", "segment_ts": "mvNJ", "time_q": "2023-02-13 15:16:22"} +{"device": "TWoLnaxEPgoNH3Fk", "segment_ts": "bwdO", "time_q": "2023-02-11 09:10:07"} +{"device": "2U0w1oZWqt42ge69", "segment_ts": "OQPD", "time_q": "2023-02-08 05:38:25"} +{"device": "Wr1pqW7ogHm1l57C", "segment_ts": "yEsz", "time_q": "2023-02-19 15:25:54"} +{"device": "X2x7ETjtNN7XU6wE", "segment_ts": "R4fq", "time_q": "2023-02-09 09:30:45"} +{"device": "JJsmrj8eGnKEpkgT", "segment_ts": "jHFs", "time_q": "2023-01-28 19:05:13"} +{"device": "jQOGpX4X5gohJjgf", "segment_ts": "NDT5", "time_q": "2023-01-28 07:12:42"} +{"device": "CktBzqEuXCvd6ThV", "segment_ts": "ItFE", "time_q": "2023-01-24 21:16:41"} +{"device": "o9JPJauPfRraIlLu", "segment_ts": "76j0", "time_q": "2023-01-23 20:11:55"} +{"device": "QdHyZk5P5vaBYTak", "segment_ts": "mfIG", "time_q": "2023-02-08 05:08:49"} +{"device": "u4QrAiPlcTouwbT5", "segment_ts": "YZHZ", "time_q": "2023-02-14 11:46:14"} +{"device": "65dIzY2Z9TJWgbhv", "segment_ts": "EqdZ", "time_q": "2023-01-28 12:55:23"} +{"device": "kT4R6IGOTSqzw7uB", "segment_ts": "SGHa", "time_q": "2023-02-11 03:18:04"} +{"device": "66KWZA8L7G89WDZI", "segment_ts": "wTC5", "time_q": "2023-02-21 14:23:50"} +{"device": "LRxsGXs41oqHtRqM", "segment_ts": "u7El", "time_q": "2023-02-09 14:03:09"} +{"device": "hgyESiq7XGtiQCql", "segment_ts": "hxmr", "time_q": "2023-01-26 12:29:40"} +{"device": "0qbdQklTFNy7ISCH", "segment_ts": "R9B9", "time_q": "2023-01-29 15:23:30"} +{"device": "xYAjqmLYwZPiLGsd", "segment_ts": "lIlA", "time_q": "2023-02-17 07:43:40"} +{"device": "BzgAUk3wTrBt2JWM", "segment_ts": "sfqh", "time_q": "2023-02-16 04:38:46"} +{"device": "UrSOAFfBipxV2iV0", "segment_ts": "Il3r", "time_q": "2023-02-13 10:27:30"} +{"device": "ALr9hJYdlrw6oNcJ", "segment_ts": "Keh8", "time_q": "2023-01-23 20:17:27"} +{"device": "GVDWDnyS3E5WQkFK", "segment_ts": "uW7S", "time_q": "2023-02-11 17:04:10"} +{"device": "2OKxK3JW7y96EHEa", "segment_ts": "KFml", "time_q": "2023-01-28 00:05:53"} +{"device": "FPkKren0hQTn7grY", "segment_ts": "eShQ", "time_q": "2023-02-04 04:45:29"} +{"device": "FWyRyqknGMhkVpo3", "segment_ts": "ZdoH", "time_q": "2023-02-15 23:28:49"} +{"device": "JK21TkVFQLsvKUG0", "segment_ts": "1MVW", "time_q": "2023-02-06 15:20:46"} +{"device": "cjo9bUnDXUQHlnCa", "segment_ts": "PT3c", "time_q": "2023-01-25 10:30:12"} +{"device": "2a0x5y7y1dd0WSHe", "segment_ts": "hrIh", "time_q": "2023-02-06 13:52:24"} +{"device": "k0TSwJgBwwVCZLlv", "segment_ts": "vRB7", "time_q": "2023-02-01 22:19:25"} +{"device": "j5LFK6TZErFfQ1VZ", "segment_ts": "tTr2", "time_q": "2023-01-30 20:49:48"} +{"device": "JGYeTG9SlkPY6wFa", "segment_ts": "Z9Pu", "time_q": "2023-02-06 20:31:31"} +{"device": "Ymh6ayRgp1H60AFc", "segment_ts": "Bwf8", "time_q": "2023-02-06 00:29:22"} +{"device": "Td0EpJOxCHYWUbrR", "segment_ts": "v25d", "time_q": "2023-01-28 05:25:00"} +{"device": "mS6Iy8Qa2mmoqYQu", "segment_ts": "9q3k", "time_q": "2023-01-30 00:18:20"} +{"device": "fc6guCFHdG8VzC3p", "segment_ts": "QjWp", "time_q": "2023-02-08 11:44:52"} +{"device": "ItkmsLuHy98vfLUQ", "segment_ts": "JoNb", "time_q": "2023-02-04 16:39:25"} +{"device": "zEKKWQ45k04JGcCV", "segment_ts": "Tpwm", "time_q": "2023-02-05 19:39:55"} +{"device": "JPsTfxDVBAtWfW8r", "segment_ts": "6sUs", "time_q": "2023-01-29 20:29:27"} +{"device": "yIeSaXpU3rkCbxUr", "segment_ts": "yZ7s", "time_q": "2023-02-18 01:23:16"} +{"device": "ekMZsH5A0zvUgkOr", "segment_ts": "KSlf", "time_q": "2023-01-30 07:30:21"} +{"device": "4c1W84L1nNUfHkZV", "segment_ts": "vvM0", "time_q": "2023-02-01 17:47:15"} +{"device": "RBN3tJA4BYV71FBJ", "segment_ts": "eQg8", "time_q": "2023-02-21 13:02:27"} +{"device": "9DS3pYWFruFTvJZS", "segment_ts": "rRuc", "time_q": "2023-01-25 06:18:46"} +{"device": "Oge628HPmV0XJxLE", "segment_ts": "XDA1", "time_q": "2023-02-20 09:45:25"} +{"device": "qHZcNOXzeJFDOsbi", "segment_ts": "X3q3", "time_q": "2023-02-03 12:14:04"} +{"device": "Z4TwRqUaxlM4nWox", "segment_ts": "G8xC", "time_q": "2023-01-25 21:21:40"} +{"device": "cN2qypm0e5KXaM4f", "segment_ts": "DPPO", "time_q": "2023-02-20 05:51:54"} +{"device": "G1umjEcxzSJpDeVf", "segment_ts": "ivk4", "time_q": "2023-02-04 14:30:20"} +{"device": "Y4YHyYf0eIA0dQT2", "segment_ts": "w9bo", "time_q": "2023-01-30 04:27:41"} +{"device": "sybCGj8bJcuWVkMH", "segment_ts": "n7r6", "time_q": "2023-02-10 08:55:38"} +{"device": "G9qtN4oPylocAhLo", "segment_ts": "ygKY", "time_q": "2023-02-11 15:58:38"} +{"device": "dqbOQmU1nCEM1tsb", "segment_ts": "snFa", "time_q": "2023-01-26 05:18:19"} +{"device": "PvXh8nHFVEKnfyw5", "segment_ts": "es0t", "time_q": "2023-02-03 04:33:00"} +{"device": "9evTiKiWziX3THWt", "segment_ts": "mrI5", "time_q": "2023-02-09 12:39:43"} +{"device": "8vWA3DPg0sCubmHt", "segment_ts": "VyRj", "time_q": "2023-01-30 17:09:05"} +{"device": "JBh1VD2mJSzS93VN", "segment_ts": "A2E6", "time_q": "2023-01-23 22:53:01"} +{"device": "AVTuFDv6MFEeDkkg", "segment_ts": "FQxa", "time_q": "2023-02-16 15:15:49"} +{"device": "jEnRgZZOuDplk7CK", "segment_ts": "McUm", "time_q": "2023-02-13 16:58:42"} +{"device": "w9eyvBTQMP0TlFPg", "segment_ts": "061H", "time_q": "2023-02-14 07:07:22"} +{"device": "HxBZsLSXkTOwIj7U", "segment_ts": "Yny1", "time_q": "2023-02-16 18:30:09"} +{"device": "OOp6NIdrHxTZgb1a", "segment_ts": "3gVe", "time_q": "2023-01-27 03:18:26"} +{"device": "XQeEiBgEVGX3xRyy", "segment_ts": "iK88", "time_q": "2023-01-31 16:16:42"} +{"device": "7XXbvAMS3NbVIuSb", "segment_ts": "ikv3", "time_q": "2023-01-27 17:58:57"} +{"device": "Z77uepdP4uNOMFEm", "segment_ts": "aNrW", "time_q": "2023-01-25 19:27:06"} +{"device": "x7Ru0sut9cCNKErD", "segment_ts": "fSS4", "time_q": "2023-02-16 13:17:10"} +{"device": "tfF7ARhQav7Vs1sW", "segment_ts": "yFy4", "time_q": "2023-02-18 08:18:49"} +{"device": "IezPnCOr2g6m2iq3", "segment_ts": "YOF3", "time_q": "2023-02-09 18:31:03"} +{"device": "VOA5cvCyvFZ7bnzk", "segment_ts": "Viph", "time_q": "2023-01-24 18:36:49"} +{"device": "p0On4aZWYDoxDWHI", "segment_ts": "FTU0", "time_q": "2023-02-07 02:13:54"} +{"device": "FSpOgwOaHynPz0zH", "segment_ts": "WwVR", "time_q": "2023-02-03 21:12:09"} +{"device": "PVSlKZkOPl5hFKDc", "segment_ts": "AZRz", "time_q": "2023-02-09 05:04:38"} +{"device": "hXJ4b5HgGPtZxZDG", "segment_ts": "eNW4", "time_q": "2023-02-06 20:20:25"} +{"device": "iyM6IB8Kyie9rXTk", "segment_ts": "WbDH", "time_q": "2023-01-25 02:38:32"} +{"device": "ZHDFyhQFjT36sYEX", "segment_ts": "RrPC", "time_q": "2023-01-24 13:30:11"} +{"device": "FAmkVIvvqiBv3J2v", "segment_ts": "BLc4", "time_q": "2023-02-05 01:57:00"} +{"device": "uUlaL9nusB5K9wah", "segment_ts": "6NJ4", "time_q": "2023-02-04 05:09:45"} +{"device": "uq7LIwGy8Cs8VS8S", "segment_ts": "Ksz1", "time_q": "2023-01-31 14:34:35"} +{"device": "Sljf8wt6YxMJlQ9B", "segment_ts": "3LOG", "time_q": "2023-02-08 01:47:29"} +{"device": "8tnuglv62PYooaqm", "segment_ts": "vXhF", "time_q": "2023-02-02 02:09:46"} +{"device": "B00CXFzWHZ7T6vw5", "segment_ts": "993f", "time_q": "2023-01-28 01:31:50"} +{"device": "FsKiFfXLePj7r3xn", "segment_ts": "uR2f", "time_q": "2023-02-16 05:49:48"} +{"device": "P2Pssg8wIUhJfJge", "segment_ts": "SACW", "time_q": "2023-01-23 19:12:29"} +{"device": "F83dj5jXAaNW08tN", "segment_ts": "NI7t", "time_q": "2023-02-03 06:42:10"} +{"device": "tz2BamJyhAvZSmKL", "segment_ts": "WFNu", "time_q": "2023-02-05 21:04:43"} +{"device": "od15jkcehIGTLhka", "segment_ts": "pY0b", "time_q": "2023-02-11 09:23:34"} +{"device": "o4J3jbqm2n2VMxAV", "segment_ts": "4G0L", "time_q": "2023-01-24 13:22:35"} +{"device": "uyBW48TE0tyYNDtB", "segment_ts": "RhKk", "time_q": "2023-01-25 13:14:11"} +{"device": "ANOQhJ0WObAmU99M", "segment_ts": "NyJ1", "time_q": "2023-02-10 22:25:45"} +{"device": "vDQzeXOS56cwm5Ha", "segment_ts": "3G8I", "time_q": "2023-02-11 19:06:38"} +{"device": "YqyX9Xk6Kw1DXOad", "segment_ts": "Qk0n", "time_q": "2023-02-10 21:58:02"} +{"device": "9neVj3Sy1VeKQxzF", "segment_ts": "j592", "time_q": "2023-01-28 02:02:51"} +{"device": "I1750EUbKfHl2yT0", "segment_ts": "Zztj", "time_q": "2023-02-06 23:04:30"} +{"device": "cdId56I5EVFYECEn", "segment_ts": "Trhf", "time_q": "2023-02-05 18:35:57"} +{"device": "mnuDFIng7Ffw33N7", "segment_ts": "7dQn", "time_q": "2023-01-31 14:53:16"} +{"device": "x6GzEx5oAis07KM2", "segment_ts": "XcKm", "time_q": "2023-01-23 06:33:10"} +{"device": "k27SrvBpgJ43XRj8", "segment_ts": "zKI7", "time_q": "2023-02-14 21:54:58"} +{"device": "lhj3vbaLb1F2K4lH", "segment_ts": "EQLa", "time_q": "2023-02-18 03:17:10"} +{"device": "TRlW7EbIXGs2Bpy8", "segment_ts": "iOm2", "time_q": "2023-02-04 11:54:32"} +{"device": "klEC6kMcr7NP6sNg", "segment_ts": "bRK7", "time_q": "2023-02-19 00:37:12"} +{"device": "TEemRDFbCJdjyNsQ", "segment_ts": "D16L", "time_q": "2023-02-17 11:29:57"} +{"device": "jptd7AJiFEAk4tSw", "segment_ts": "IAOL", "time_q": "2023-01-31 08:48:08"} +{"device": "L45DvspGDQF2fOyi", "segment_ts": "sALF", "time_q": "2023-02-05 03:25:00"} +{"device": "R2NkbrNMNldSBJUG", "segment_ts": "609r", "time_q": "2023-02-02 09:49:49"} +{"device": "8ZqVW4kT45fPZy5a", "segment_ts": "l00H", "time_q": "2023-02-11 13:05:33"} +{"device": "jHbO5YUIUxVvdzJA", "segment_ts": "8dJN", "time_q": "2023-02-17 02:03:26"} +{"device": "ktIY7e9VGCKMjL7P", "segment_ts": "Mxrj", "time_q": "2023-02-20 23:12:59"} +{"device": "uuOZC7aaCy9sUiT5", "segment_ts": "EUD1", "time_q": "2023-02-01 23:31:46"} +{"device": "3RtgCZcjgxxCmg5T", "segment_ts": "5JQi", "time_q": "2023-01-23 12:12:05"} +{"device": "YnI4T3vi1Lxyt5wa", "segment_ts": "rXIU", "time_q": "2023-02-12 23:58:24"} +{"device": "ev9IcNjTEZekqALB", "segment_ts": "GAxJ", "time_q": "2023-01-24 12:16:20"} +{"device": "vp5qi7TRC8KQWk7d", "segment_ts": "TJ6P", "time_q": "2023-02-07 17:28:23"} +{"device": "bSoAX2L1JHmPXODo", "segment_ts": "fasF", "time_q": "2023-02-06 15:59:59"} +{"device": "3XauXuq6ws73yQla", "segment_ts": "SsnB", "time_q": "2023-01-26 16:11:14"} +{"device": "TyYnqrqHKPeqZ82q", "segment_ts": "Wzwh", "time_q": "2023-02-16 20:02:19"} +{"device": "O8hOzOOgxw2iQiHx", "segment_ts": "Yj0D", "time_q": "2023-02-14 00:19:14"} +{"device": "u4Pxs4Cyj00rtA60", "segment_ts": "1vCq", "time_q": "2023-01-31 16:09:15"} +{"device": "jo8xd3EdVQRVKZG0", "segment_ts": "ltBU", "time_q": "2023-01-31 07:14:48"} +{"device": "W06dd1sDTy0apH3Y", "segment_ts": "ygKY", "time_q": "2023-02-09 21:21:10"} +{"device": "Sh7dZSxHMgFlutjC", "segment_ts": "MbqF", "time_q": "2023-02-11 23:49:25"} +{"device": "o5CLa2P1xlbs3BYb", "segment_ts": "v0BW", "time_q": "2023-02-01 11:00:06"} +{"device": "4wegF6Zxl955mSik", "segment_ts": "281a", "time_q": "2023-01-29 11:24:20"} +{"device": "bx31bZeyk6v3ZATS", "segment_ts": "wT1j", "time_q": "2023-02-03 22:21:54"} +{"device": "kfBaygAngrkZHyhR", "segment_ts": "KFml", "time_q": "2023-02-11 02:35:20"} +{"device": "IENwCndPscOSssCf", "segment_ts": "tGZF", "time_q": "2023-02-05 15:36:28"} +{"device": "JU1KqBsrNFR4vOqo", "segment_ts": "U1XA", "time_q": "2023-01-31 18:39:08"} +{"device": "8qqyRLZQv8IAoQgP", "segment_ts": "1WZE", "time_q": "2023-02-03 06:48:42"} +{"device": "rBCNpDGLi9AodGdX", "segment_ts": "3J3I", "time_q": "2023-02-17 06:28:51"} +{"device": "x1VmoWLCRvrSmkEh", "segment_ts": "lqnk", "time_q": "2023-01-24 18:01:20"} +{"device": "iL3Swp4VajwaNf3O", "segment_ts": "gtdx", "time_q": "2023-01-23 11:19:34"} +{"device": "g7iVs7QsRXDubZvV", "segment_ts": "Oyvx", "time_q": "2023-01-29 11:33:47"} +{"device": "4iCnj0wRJoMFKwx4", "segment_ts": "XU65", "time_q": "2023-01-26 13:56:11"} +{"device": "WWBQzqhhgRMwMVYH", "segment_ts": "MwvV", "time_q": "2023-01-26 21:09:06"} +{"device": "ZjwDtsLC8jMJIsQ6", "segment_ts": "NhT6", "time_q": "2023-02-10 05:06:17"} +{"device": "YkI6KyY3TgyviASv", "segment_ts": "CZdM", "time_q": "2023-02-04 03:49:11"} +{"device": "kemopcvgaWvww445", "segment_ts": "9EjB", "time_q": "2023-01-25 18:58:42"} +{"device": "Nfuv9QZRaTdtZ1j1", "segment_ts": "nyoq", "time_q": "2023-02-08 15:39:28"} +{"device": "IDNpRkPQqBYG8rIL", "segment_ts": "nrGc", "time_q": "2023-02-19 09:20:51"} +{"device": "woj5K9AYcOPi8wbH", "segment_ts": "q9o4", "time_q": "2023-01-30 00:47:23"} +{"device": "A3Q592GiKPtRBifW", "segment_ts": "479s", "time_q": "2023-02-03 13:21:56"} +{"device": "t0tSKqKvPJsSLKek", "segment_ts": "PzBM", "time_q": "2023-02-09 19:16:28"} +{"device": "xvu9RL3gVCt6YIqd", "segment_ts": "OE7b", "time_q": "2023-02-10 13:47:41"} +{"device": "WMwNdlIc5a3s0o4d", "segment_ts": "WFNu", "time_q": "2023-02-10 20:06:12"} +{"device": "kkvalugYPF7CeARB", "segment_ts": "xOmt", "time_q": "2023-01-27 20:04:35"} +{"device": "O2Pxt9eTcZ8DXb7Z", "segment_ts": "3sv4", "time_q": "2023-02-10 23:46:23"} +{"device": "n67yYlHFaWmgsEIn", "segment_ts": "MbqF", "time_q": "2023-01-30 20:38:08"} +{"device": "4vHRlcIj4fA6nyem", "segment_ts": "vroI", "time_q": "2023-02-14 16:20:29"} +{"device": "TMbltseSls9UOzxq", "segment_ts": "v5gi", "time_q": "2023-02-20 09:03:38"} +{"device": "Jj6eg1CNcUpAtY4b", "segment_ts": "xd2m", "time_q": "2023-02-16 06:33:10"} +{"device": "ipuU6YIHXJVKYBb5", "segment_ts": "LwNw", "time_q": "2023-01-28 00:19:52"} +{"device": "t66AJJAWxgfr54og", "segment_ts": "q74O", "time_q": "2023-01-30 06:53:54"} +{"device": "72tpVrLi7idQ711R", "segment_ts": "TuWP", "time_q": "2023-02-19 22:09:35"} +{"device": "Hvq7OAQ6pXafCIaP", "segment_ts": "OdFT", "time_q": "2023-02-17 20:46:42"} +{"device": "pme4dFY8cxRdM0iD", "segment_ts": "7exz", "time_q": "2023-02-16 00:45:20"} +{"device": "urddNEFtldtGx3WR", "segment_ts": "eZiz", "time_q": "2023-02-10 06:35:13"} +{"device": "uZvYbARAUvBgqqiC", "segment_ts": "AwpA", "time_q": "2023-02-05 08:54:00"} +{"device": "qynlelnsdXES0L3o", "segment_ts": "qe17", "time_q": "2023-02-04 03:25:06"} +{"device": "VvuWbHbURvKgsqUF", "segment_ts": "7Sxg", "time_q": "2023-02-14 10:47:24"} +{"device": "TlnhHfBk12CQiQ0K", "segment_ts": "QVU0", "time_q": "2023-02-11 01:01:08"} +{"device": "pPUdA3tMUudFfuu8", "segment_ts": "Ubci", "time_q": "2023-02-20 07:54:52"} +{"device": "zgZbzsxqUD8EYtTt", "segment_ts": "Oyvx", "time_q": "2023-02-16 23:14:46"} +{"device": "mJ4o0n4HIVfcYlOo", "segment_ts": "1Xba", "time_q": "2023-02-09 23:33:48"} +{"device": "aqh9qEavdsDiY54R", "segment_ts": "O951", "time_q": "2023-01-23 21:18:26"} +{"device": "baPCRewYf20VOMzV", "segment_ts": "7QZg", "time_q": "2023-01-30 03:32:06"} +{"device": "JGqBYR14WAkGBwFM", "segment_ts": "3ES4", "time_q": "2023-02-06 06:31:37"} +{"device": "k5Jpfk62razi9ZgC", "segment_ts": "jefZ", "time_q": "2023-02-21 06:43:37"} +{"device": "J4R3L5UMS3DjpooS", "segment_ts": "J8fh", "time_q": "2023-02-12 03:19:39"} +{"device": "mKaEAZkAqmHVYbYx", "segment_ts": "NZxR", "time_q": "2023-02-20 12:07:35"} +{"device": "mnFNrh7YGDVK0LF0", "segment_ts": "1yCJ", "time_q": "2023-01-29 11:05:00"} +{"device": "15x8g2YTRH5wxg8D", "segment_ts": "hOB3", "time_q": "2023-02-15 09:54:07"} +{"device": "MDhSc461Le45qIOD", "segment_ts": "Oqnw", "time_q": "2023-02-13 12:55:24"} +{"device": "2wtPLtzsT0nw72Ki", "segment_ts": "iC7M", "time_q": "2023-02-13 08:02:54"} +{"device": "EJN36J2EWxJO6d3A", "segment_ts": "aCKQ", "time_q": "2023-01-26 18:11:35"} +{"device": "4Cibygd0vko8tblj", "segment_ts": "xF4p", "time_q": "2023-01-27 09:25:50"} +{"device": "1EryLdrZkHz9TBM2", "segment_ts": "M97m", "time_q": "2023-02-06 08:09:39"} +{"device": "NUp95xg6x9K07uJw", "segment_ts": "DuKl", "time_q": "2023-02-15 14:42:37"} +{"device": "QTvOJ3fHPnsnFksM", "segment_ts": "PNnd", "time_q": "2023-01-23 22:31:31"} \ No newline at end of file diff --git a/idk/kafka/testdata/records/timeQuantum.txt b/idk/kafka/testdata/records/timeQuantum.txt deleted file mode 100644 index ba0d8458d..000000000 --- a/idk/kafka/testdata/records/timeQuantum.txt +++ /dev/null @@ -1,500 +0,0 @@ -{'device': 'XbJ7vwASddz1xBQ4', 'segment_ts': 'fub2', 'time_q': '2023-02-13 18:11:08'} -{'device': 'g3AwJJUDwrN1LYzY', 'segment_ts': 'Qn3I', 'time_q': '2023-02-13 19:39:01'} -{'device': 'DIxrncsISTdUJdsH', 'segment_ts': 'o8Ig', 'time_q': '2023-01-27 09:41:54'} -{'device': 'yR1RUTVnjwUvjWLS', 'segment_ts': 'BUnG', 'time_q': '2023-01-25 06:55:04'} -{'device': 'dpeJtjO0mVsN3wiZ', 'segment_ts': 'Udwh', 'time_q': '2023-01-25 14:34:42'} -{'device': 'hOJrFi9xmOnM5J2B', 'segment_ts': '9jED', 'time_q': '2023-01-27 17:05:19'} -{'device': 'xPLZ3qjA029BWoD8', 'segment_ts': 'PBbR', 'time_q': '2023-02-17 08:47:05'} -{'device': 'sdnPNO80Trlnkfpi', 'segment_ts': 'ACVo', 'time_q': '2023-02-21 09:01:47'} -{'device': 'fcMcP2T8hIBzYzrB', 'segment_ts': 'HppX', 'time_q': '2023-02-18 00:59:36'} -{'device': 'xiRO1wD8bdg6tsy2', 'segment_ts': 'iSQp', 'time_q': '2023-01-28 06:15:59'} -{'device': 'nlsTgLlvTBhQGewH', 'segment_ts': 'CZHg', 'time_q': '2023-02-20 18:53:14'} -{'device': 'bpLtS0dLbzC1t0aM', 'segment_ts': 'oanZ', 'time_q': '2023-02-11 08:25:43'} -{'device': 'RcYtSPPTAQyOE80E', 'segment_ts': '57He', 'time_q': '2023-02-14 06:40:17'} -{'device': 'BLqS1Js4hqpAGKbs', 'segment_ts': '8Esc', 'time_q': '2023-02-13 01:59:32'} -{'device': 'NDJ13gRlrfjSpJul', 'segment_ts': 'fBcm', 'time_q': '2023-01-24 07:05:02'} -{'device': 'CUS2BbP8fJrBuJRB', 'segment_ts': 'kUK0', 'time_q': '2023-02-05 22:33:45'} -{'device': 'lwC1KW3eNMX7kqwS', 'segment_ts': '8s12', 'time_q': '2023-02-15 19:55:08'} -{'device': 'jtEPRgME5UKtmv2O', 'segment_ts': 'Ew1L', 'time_q': '2023-02-10 10:44:56'} -{'device': 'zJJQ58p1vZEVr8pV', 'segment_ts': 'cb9G', 'time_q': '2023-01-31 14:49:08'} -{'device': 'zGLCqlcb9WEVYeTD', 'segment_ts': 'JatF', 'time_q': '2023-02-19 17:37:42'} -{'device': 'PgyMZVp5lPVUAzju', 'segment_ts': 'H4w7', 'time_q': '2023-01-25 01:13:51'} -{'device': 'KS2ftxZpJaobrFcH', 'segment_ts': 'lzde', 'time_q': '2023-02-08 07:29:00'} -{'device': 'pvfQ90NOCNxkZ9qp', 'segment_ts': 'meI7', 'time_q': '2023-02-01 09:29:32'} -{'device': 'et0tVgDi4gQLBbCZ', 'segment_ts': '8roi', 'time_q': '2023-02-17 13:59:05'} -{'device': 'KVpmUGw3YFQWFKUJ', 'segment_ts': 'reQ2', 'time_q': '2023-02-07 05:12:56'} -{'device': '7e62NVroqaZ5KCCG', 'segment_ts': 'mRqe', 'time_q': '2023-01-25 18:55:33'} -{'device': '2jhbxRtroaywIHz0', 'segment_ts': 'vGvz', 'time_q': '2023-01-31 06:31:33'} -{'device': 'i8A3oSUCLIRVI3z6', 'segment_ts': 'bRK7', 'time_q': '2023-01-22 15:45:16'} -{'device': 'w9m2zmvBebNOYm7M', 'segment_ts': 'EUTl', 'time_q': '2023-02-12 17:45:44'} -{'device': '6goOWv6GmSB5SImL', 'segment_ts': 'IcRw', 'time_q': '2023-02-02 19:44:49'} -{'device': '9TdRfZY8fyUv0MDA', 'segment_ts': 'jxeF', 'time_q': '2023-02-15 12:46:23'} -{'device': 'XFT7MaPT04gy9giN', 'segment_ts': 'NDT5', 'time_q': '2023-01-27 02:02:09'} -{'device': 'dYf2esJeNHts76qt', 'segment_ts': 'nD5r', 'time_q': '2023-02-20 19:18:20'} -{'device': 'XO3Mw07kTWxq4S6A', 'segment_ts': 'wu98', 'time_q': '2023-02-19 09:11:04'} -{'device': 'PR20MB3DrKZyxBYN', 'segment_ts': 'VxO2', 'time_q': '2023-02-14 02:48:28'} -{'device': '8HoHoySkF3ONDVMT', 'segment_ts': 'RZW0', 'time_q': '2023-02-17 09:47:36'} -{'device': 'D9EMY6KMiUupVo86', 'segment_ts': 'uKvn', 'time_q': '2023-02-13 20:34:37'} -{'device': '6vw27rBMy4z08u0H', 'segment_ts': 'IhoS', 'time_q': '2023-02-07 21:19:31'} -{'device': '8B3tax9WJNOuceOO', 'segment_ts': '2sCg', 'time_q': '2023-02-13 06:09:00'} -{'device': 'q1kmQxWdgK1fI2Aq', 'segment_ts': 'tpUd', 'time_q': '2023-01-22 22:14:51'} -{'device': 'VPe3DuOwKmbOf9hj', 'segment_ts': 'Q1EB', 'time_q': '2023-02-07 01:49:40'} -{'device': 'bcn6NL9OVcDI9NOE', 'segment_ts': 'pmBn', 'time_q': '2023-02-06 19:24:58'} -{'device': 'xveWOtVYpFmdKoE3', 'segment_ts': '3pUE', 'time_q': '2023-01-29 05:45:06'} -{'device': 'O9p8Ij6SiXU7w9Bo', 'segment_ts': 'wDD7', 'time_q': '2023-02-07 02:09:17'} -{'device': 'JU1KI8rx8qLXtY9Y', 'segment_ts': 'NBSG', 'time_q': '2023-02-15 13:20:17'} -{'device': 'zvc3NHLWCKRbwL4w', 'segment_ts': 'wAqp', 'time_q': '2023-02-06 19:57:45'} -{'device': 'uIKOal9xGdwgmxQZ', 'segment_ts': '3x0X', 'time_q': '2023-02-19 00:21:42'} -{'device': '0Rn9nBxCnr1abKBu', 'segment_ts': 'K3q7', 'time_q': '2023-02-07 03:02:30'} -{'device': 'DcPFdJzT9aCgD7hZ', 'segment_ts': '1WZE', 'time_q': '2023-02-04 22:53:14'} -{'device': 'I33U4ld8p1ZTEo2z', 'segment_ts': 'h327', 'time_q': '2023-02-15 09:02:02'} -{'device': 'HzV2F1u15P6MfTq4', 'segment_ts': 'Oi4K', 'time_q': '2023-02-18 03:07:18'} -{'device': 'Gs8bQMX2wl9AjM5I', 'segment_ts': 'CT5V', 'time_q': '2023-02-16 01:45:58'} -{'device': 'aPv5uNdC2x9Zrs31', 'segment_ts': 'ijS2', 'time_q': '2023-01-29 20:43:50'} -{'device': 'gmBMopHkBCbcGRaS', 'segment_ts': 'nD5r', 'time_q': '2023-02-11 05:05:58'} -{'device': 'g3Ns2caZMQVlOSPr', 'segment_ts': '9vzm', 'time_q': '2023-01-27 00:22:33'} -{'device': 'JWY6hPLwnhy76KGZ', 'segment_ts': '2IS2', 'time_q': '2023-02-11 21:47:10'} -{'device': '0QKtSTqJYXMZWvVe', 'segment_ts': '7R83', 'time_q': '2023-02-17 05:40:55'} -{'device': 'Ximq2ebfKR1eqySD', 'segment_ts': 'Jpup', 'time_q': '2023-01-23 17:34:53'} -{'device': 'YWGLIrHDKwe6UUM1', 'segment_ts': 'Smbg', 'time_q': '2023-02-11 22:29:38'} -{'device': 'i8Hy1Hhy6vbQ7Gnj', 'segment_ts': 'Xjn9', 'time_q': '2023-02-07 05:52:25'} -{'device': 'trQjczSWWWqD3xip', 'segment_ts': 'zSyE', 'time_q': '2023-02-14 03:51:05'} -{'device': 'cRAvzL88YU74zOVR', 'segment_ts': 'FAk3', 'time_q': '2023-02-02 06:26:40'} -{'device': 'H7RUb4moxDU5RY8L', 'segment_ts': '6sUs', 'time_q': '2023-02-06 15:29:02'} -{'device': 'g6uZFROq6hA9VwXU', 'segment_ts': '9v3C', 'time_q': '2023-01-30 05:29:57'} -{'device': 'yIGidxsJbbyaTMJu', 'segment_ts': '5TbD', 'time_q': '2023-01-31 20:18:06'} -{'device': 'DC9B8r6fLUrQEKWd', 'segment_ts': 'VDz6', 'time_q': '2023-02-09 22:19:07'} -{'device': 'rymp2l5MJvVdAEiU', 'segment_ts': 'fyte', 'time_q': '2023-02-02 11:53:13'} -{'device': 'Ua1gF72JZ4WAc8Bd', 'segment_ts': 'v5sp', 'time_q': '2023-01-24 02:06:04'} -{'device': '9qSO5sST0XGp5jMS', 'segment_ts': '7dBw', 'time_q': '2023-02-10 09:35:42'} -{'device': 'p0HAiSpiSNAaCrZB', 'segment_ts': 'a1iQ', 'time_q': '2023-01-31 15:45:28'} -{'device': 'wkYtecM7PdltlJ6U', 'segment_ts': 'tCye', 'time_q': '2023-01-29 14:50:59'} -{'device': 'efTNUHAxzw90hiOb', 'segment_ts': '9Wf1', 'time_q': '2023-02-18 02:29:02'} -{'device': 'fQI3EBaehVoTHSuQ', 'segment_ts': 'zRBT', 'time_q': '2023-02-11 19:06:37'} -{'device': 'udeapVgb384YhUvm', 'segment_ts': 'oL67', 'time_q': '2023-02-05 10:27:36'} -{'device': 'eqbom6mcHS60rV4o', 'segment_ts': 'GNdZ', 'time_q': '2023-02-11 08:44:18'} -{'device': '1wWPwlCsbs9ZBcZC', 'segment_ts': 'zaKC', 'time_q': '2023-02-08 00:49:32'} -{'device': 'ecY2Gsp5o5y8RDEy', 'segment_ts': 'QWzq', 'time_q': '2023-01-29 02:43:04'} -{'device': 'DxqHMz1dx4Bqsobg', 'segment_ts': '2RQv', 'time_q': '2023-02-18 21:50:04'} -{'device': 'XU1b7NOcHSuGzCUv', 'segment_ts': 'Yuoy', 'time_q': '2023-02-13 21:54:55'} -{'device': 'lO2wwnU1eJPm8GXQ', 'segment_ts': 'AP3b', 'time_q': '2023-01-28 14:02:06'} -{'device': 'tLnTsvtbSVqjrYe9', 'segment_ts': 'fFTT', 'time_q': '2023-01-24 23:23:30'} -{'device': 'BD3RZkRDscI24QIW', 'segment_ts': '3Bry', 'time_q': '2023-02-14 17:16:33'} -{'device': 'EJTuSaV3BN56Wj2O', 'segment_ts': 'W4JQ', 'time_q': '2023-02-01 18:56:00'} -{'device': 'TfinqcstHOBc9vgU', 'segment_ts': 'VybU', 'time_q': '2023-01-30 13:41:42'} -{'device': 'oLCaL36jSe58AJzI', 'segment_ts': '28pM', 'time_q': '2023-02-03 07:47:20'} -{'device': 'I85aiLVze3vm3TkQ', 'segment_ts': 'y8y1', 'time_q': '2023-02-19 19:12:25'} -{'device': 'J1Z6A5uaqDbcJCfB', 'segment_ts': 'Wzwh', 'time_q': '2023-02-10 08:51:06'} -{'device': 'DUJWSG7TcQrG1SiU', 'segment_ts': 'qKgC', 'time_q': '2023-02-18 22:00:14'} -{'device': 'oq4StgCLUoilIYHQ', 'segment_ts': 'MNLg', 'time_q': '2023-02-05 10:11:01'} -{'device': 'P44lydZpIGjbmQOy', 'segment_ts': 'uHCR', 'time_q': '2023-01-22 21:38:00'} -{'device': '8lV1uEL9zdwqOuwT', 'segment_ts': '8wB6', 'time_q': '2023-02-05 21:52:08'} -{'device': 'uVGi6rqE7NKl7D0W', 'segment_ts': 'kge3', 'time_q': '2023-02-12 14:45:28'} -{'device': 'fBDDRmw5ifz9Ibk1', 'segment_ts': 'e8wv', 'time_q': '2023-02-12 01:36:43'} -{'device': 'q8q2AXRsgPrtU7MF', 'segment_ts': 'lznG', 'time_q': '2023-01-31 09:08:30'} -{'device': 'rKVwah03kvEN1Xf8', 'segment_ts': 'CIf7', 'time_q': '2023-02-17 15:32:01'} -{'device': '5gqkkJu0HnW8e7Jd', 'segment_ts': '11VE', 'time_q': '2023-02-21 12:50:18'} -{'device': '67aeTL1Lhk7zPniw', 'segment_ts': '7WnQ', 'time_q': '2023-02-15 09:54:03'} -{'device': 'JSqkbIUNs9XsDhyf', 'segment_ts': 'Vef3', 'time_q': '2023-01-28 05:40:03'} -{'device': 'O6pY363lkXtNFnPW', 'segment_ts': 'JPG3', 'time_q': '2023-02-09 07:18:08'} -{'device': 'tpldKtJXhNOJs5is', 'segment_ts': 'x40J', 'time_q': '2023-02-19 03:15:47'} -{'device': 'zJFXIMKFyyMjvHzJ', 'segment_ts': 'ddlX', 'time_q': '2023-02-12 02:18:47'} -{'device': 'o0W2DatNPhPHeUCG', 'segment_ts': '3Lzx', 'time_q': '2023-02-01 03:05:37'} -{'device': '0qNRr7JlwEJo0UZs', 'segment_ts': 'x1R8', 'time_q': '2023-02-04 16:06:55'} -{'device': 'ybv2vBmnJIMXDQ7C', 'segment_ts': 'DyAY', 'time_q': '2023-02-04 14:33:24'} -{'device': '7bkYnhDEWrdlKPzi', 'segment_ts': 'CHYf', 'time_q': '2023-02-18 13:52:19'} -{'device': 'gS5FFLJPIgSpPn2p', 'segment_ts': '73ea', 'time_q': '2023-02-20 13:38:30'} -{'device': 'Iz5J7QbGpOaw9Mc3', 'segment_ts': 'UD8U', 'time_q': '2023-02-19 21:56:03'} -{'device': 'Q4zAccg3hJyiWPQV', 'segment_ts': 'WbDH', 'time_q': '2023-02-19 21:57:53'} -{'device': 'vwowDnNJnQ3Noych', 'segment_ts': 'YyvE', 'time_q': '2023-02-07 21:35:53'} -{'device': 'V4Yg3xtwNAdXT7AC', 'segment_ts': 'OQ1f', 'time_q': '2023-01-27 03:41:03'} -{'device': 'kBcU57h5YJlps0M7', 'segment_ts': 'X9xH', 'time_q': '2023-02-16 21:18:05'} -{'device': 'mTOckeiB1Myp9PzA', 'segment_ts': 'wDD7', 'time_q': '2023-02-16 06:44:48'} -{'device': 'jFqBXPPMyJJaM4VG', 'segment_ts': 'csTs', 'time_q': '2023-02-06 02:52:21'} -{'device': 'vKNH7K6ozve7702k', 'segment_ts': 'SJ25', 'time_q': '2023-02-08 01:27:29'} -{'device': 'oyT2TtgfCba2Hdml', 'segment_ts': 'Tvqb', 'time_q': '2023-02-21 03:13:30'} -{'device': 'rsOlS9C8eEy347rZ', 'segment_ts': 'hr7p', 'time_q': '2023-02-19 02:30:46'} -{'device': '90GjHMWLi1dcpp0y', 'segment_ts': 'W4JQ', 'time_q': '2023-01-25 02:29:48'} -{'device': 'fvudPtjiPgFEJ7bD', 'segment_ts': 'lwno', 'time_q': '2023-01-24 05:44:13'} -{'device': 'VBWAgZvfVhht5RLf', 'segment_ts': 'Ftw2', 'time_q': '2023-01-27 13:08:57'} -{'device': 'sZK0cJqcYBnZXwyj', 'segment_ts': '9G1V', 'time_q': '2023-01-27 21:13:09'} -{'device': 'Nw2Qbh1azig2aS8i', 'segment_ts': 'VPUK', 'time_q': '2023-02-20 06:55:48'} -{'device': 'YrmDmrxR6yVVsmBK', 'segment_ts': 'kEmM', 'time_q': '2023-02-17 04:13:52'} -{'device': 'FJg8emti6uDPkZWb', 'segment_ts': 'cu9S', 'time_q': '2023-01-31 15:15:17'} -{'device': 'D2KVrcXONfBL0DOq', 'segment_ts': 'E7q4', 'time_q': '2023-02-07 05:21:14'} -{'device': 'zpjzEqsoIDSfgkrw', 'segment_ts': 'kuSy', 'time_q': '2023-02-08 21:39:35'} -{'device': 'TyBhOHhPHg3mABfs', 'segment_ts': 'FOLC', 'time_q': '2023-02-05 01:50:35'} -{'device': 'SSsebFMnz2vz0lKT', 'segment_ts': 'qXUp', 'time_q': '2023-02-17 15:12:43'} -{'device': 'oeMORvnHFqNvuqA9', 'segment_ts': 'D16L', 'time_q': '2023-02-17 12:56:55'} -{'device': 'pZGE5Q8ax0QnrKlK', 'segment_ts': '6O2l', 'time_q': '2023-02-17 06:43:06'} -{'device': 'xXUDPhByePkaTvpe', 'segment_ts': 'UqWC', 'time_q': '2023-02-05 01:26:33'} -{'device': '8zOXbhQQFeXPuMgW', 'segment_ts': 'v5s5', 'time_q': '2023-01-28 21:54:59'} -{'device': 'Kvd0ii9qinTBgkEK', 'segment_ts': 'YWRA', 'time_q': '2023-02-15 04:02:15'} -{'device': 'JoC3FaiJFH7nWQbY', 'segment_ts': 'RV1o', 'time_q': '2023-01-23 11:40:04'} -{'device': 'JZtyjzapfAhKTdEL', 'segment_ts': '6Hyt', 'time_q': '2023-02-12 23:31:41'} -{'device': 'lsE2LnvCTkfUv3Hb', 'segment_ts': 'mvNJ', 'time_q': '2023-02-20 04:17:14'} -{'device': 'ZaG5u6BctGV7Phak', 'segment_ts': 'xS0C', 'time_q': '2023-02-12 21:40:07'} -{'device': 'VqkvcCFemWmmdLIN', 'segment_ts': '1vCq', 'time_q': '2023-02-04 13:43:54'} -{'device': 'MnWYYlFI787w7ENs', 'segment_ts': 'YHny', 'time_q': '2023-02-02 04:12:38'} -{'device': '5dFd1cLHS8fjZaOz', 'segment_ts': '0JSK', 'time_q': '2023-02-14 05:41:43'} -{'device': 'G7r2BebALuPczt3b', 'segment_ts': 'nowH', 'time_q': '2023-02-19 08:55:48'} -{'device': '7V6SBGSNrMZUvrw8', 'segment_ts': 'KKuH', 'time_q': '2023-02-07 03:00:14'} -{'device': 'YWmt4ZWofrE68Eg4', 'segment_ts': '9ub1', 'time_q': '2023-01-24 10:32:07'} -{'device': 'RgsF3Paw1CFHmXrz', 'segment_ts': 'GKFG', 'time_q': '2023-02-05 14:23:12'} -{'device': 'uRUd8DnEptcCg051', 'segment_ts': 'w6bQ', 'time_q': '2023-02-12 02:55:44'} -{'device': 'gpJn2OhNJ17yFKfj', 'segment_ts': 'Ms6E', 'time_q': '2023-02-11 12:28:14'} -{'device': 'OS8p3UgcZwCtitJ5', 'segment_ts': 'lUE6', 'time_q': '2023-02-12 12:43:29'} -{'device': 'AaBf2goUzSIOLwKU', 'segment_ts': 'jRhp', 'time_q': '2023-01-28 02:43:57'} -{'device': 'Gi7o9G6zuxIje6TA', 'segment_ts': 'TPjM', 'time_q': '2023-01-22 21:42:14'} -{'device': 'Zgp8HrnLjtoUBstx', 'segment_ts': '669s', 'time_q': '2023-01-23 07:40:22'} -{'device': 'yZ39wYgza1sjjH6a', 'segment_ts': '80La', 'time_q': '2023-01-25 23:58:17'} -{'device': 'gKSePVA4eUpPeQeB', 'segment_ts': 'uPAl', 'time_q': '2023-02-06 04:22:11'} -{'device': 'ZvPnZt45ZdDJf2wV', 'segment_ts': 'eQg8', 'time_q': '2023-01-29 16:01:24'} -{'device': 'RGaReXrDxjLvIN1h', 'segment_ts': 'wV8N', 'time_q': '2023-01-27 01:38:59'} -{'device': 'DBG2BbMzd2AiJ1ra', 'segment_ts': 'or1s', 'time_q': '2023-02-14 19:21:12'} -{'device': 'LaRpvrLjCem5I1iG', 'segment_ts': 'MKyg', 'time_q': '2023-01-22 20:08:37'} -{'device': '6VPMKOpB2Gax57AM', 'segment_ts': 'jWnU', 'time_q': '2023-02-02 08:27:23'} -{'device': 'WWmDiCgEhcFaRoxd', 'segment_ts': 'IcRw', 'time_q': '2023-01-28 20:36:01'} -{'device': 'g8tKyCuvwcmnOleP', 'segment_ts': 'l57C', 'time_q': '2023-02-21 12:15:51'} -{'device': 'mGmZoqlgLWwRHhWd', 'segment_ts': 'MJWQ', 'time_q': '2023-01-26 00:54:26'} -{'device': '9vYzND4LhGIpLRPs', 'segment_ts': 'X8An', 'time_q': '2023-02-17 17:12:20'} -{'device': 'TqAci7xzsz9HKRwX', 'segment_ts': 'HQvb', 'time_q': '2023-02-07 19:26:19'} -{'device': 'jXeyesYOxpErf0Wr', 'segment_ts': '9Pah', 'time_q': '2023-01-29 07:43:44'} -{'device': 'PKpsammUcBhHIj4d', 'segment_ts': 'SJ96', 'time_q': '2023-02-01 11:22:01'} -{'device': 'd43Y87nTdkx4nw0N', 'segment_ts': 'qt7V', 'time_q': '2023-02-13 07:35:55'} -{'device': '3wSdpuLVAqb2DqwQ', 'segment_ts': '1uul', 'time_q': '2023-02-20 17:20:48'} -{'device': 'b86HMaTJskEFSegv', 'segment_ts': 'e0pl', 'time_q': '2023-02-20 04:23:59'} -{'device': 'q7XDusNiRySmmaRj', 'segment_ts': '7dBw', 'time_q': '2023-01-29 01:47:01'} -{'device': 'LrWDgsZruGK3akSe', 'segment_ts': '0vLc', 'time_q': '2023-02-20 13:19:00'} -{'device': 'JJOo2fsdKEG6QCRu', 'segment_ts': 'LN5W', 'time_q': '2023-02-12 01:36:02'} -{'device': 'iXc2nZTT1U0iWf2W', 'segment_ts': '3ftI', 'time_q': '2023-02-21 10:12:08'} -{'device': 'YJ4TjVpH6ZQhMvrH', 'segment_ts': 'IVww', 'time_q': '2023-01-24 12:14:46'} -{'device': '4cBW0bSimlW8ygs0', 'segment_ts': 'tGGE', 'time_q': '2023-02-13 11:54:22'} -{'device': 'Ye2BuE77YVbChRNA', 'segment_ts': 'SRVM', 'time_q': '2023-01-25 16:55:03'} -{'device': 'BD4sQkvrmTWcfgdd', 'segment_ts': 'uqv1', 'time_q': '2023-02-17 14:21:23'} -{'device': 'kB9K8PKJpQnkJTcd', 'segment_ts': 'WCit', 'time_q': '2023-02-20 20:54:19'} -{'device': 'uSAbl2Nyu6nK3ddO', 'segment_ts': '0cq0', 'time_q': '2023-01-28 17:04:49'} -{'device': 'uJEVeGx0jSYrxfsx', 'segment_ts': 'GL95', 'time_q': '2023-02-19 21:43:48'} -{'device': 'wowb2b399vUqVxWH', 'segment_ts': 'YcDM', 'time_q': '2023-02-03 10:03:34'} -{'device': 'Yt42EPAk8TRsKsha', 'segment_ts': '9vzm', 'time_q': '2023-02-06 16:06:01'} -{'device': 'nuQgvAgPYUF1ML4s', 'segment_ts': 'SOUQ', 'time_q': '2023-02-02 10:23:33'} -{'device': 'LEuEmNLEgjoCu3p1', 'segment_ts': 'Mhpw', 'time_q': '2023-01-31 19:35:49'} -{'device': '4H1WoLSdS2qs5jOM', 'segment_ts': 'II21', 'time_q': '2023-02-10 14:24:09'} -{'device': 'dN5oTZXijbMTz8OP', 'segment_ts': 'NOvo', 'time_q': '2023-01-27 11:27:12'} -{'device': 'i2ITUxoWBCEdW8Bs', 'segment_ts': 'FqD3', 'time_q': '2023-01-30 06:33:32'} -{'device': 'NyZi9ibptktAZTjI', 'segment_ts': 'C24z', 'time_q': '2023-02-13 18:48:37'} -{'device': '5MMCow4pOwRmSIlc', 'segment_ts': 'NuYD', 'time_q': '2023-02-03 15:50:35'} -{'device': 'lK52wLIAUCcXP4AH', 'segment_ts': 'geC5', 'time_q': '2023-02-14 07:33:08'} -{'device': 'kdEACkrL59EG8vBX', 'segment_ts': '44WL', 'time_q': '2023-02-15 16:46:30'} -{'device': 'BZhrV8EAlPBft7XX', 'segment_ts': 'w04k', 'time_q': '2023-02-16 03:47:55'} -{'device': '8mKdbrORBbygBkPb', 'segment_ts': '3x0X', 'time_q': '2023-01-27 04:58:59'} -{'device': 'C3ue47cXeXukj7rU', 'segment_ts': '7AXG', 'time_q': '2023-02-14 15:59:38'} -{'device': 'PFZtQxNxLhvhEuil', 'segment_ts': 'mCKg', 'time_q': '2023-02-12 16:29:55'} -{'device': 'Qfa7tEckTSssilkU', 'segment_ts': '3Epz', 'time_q': '2023-01-30 04:49:42'} -{'device': 'OZGXqiDoUPo7codg', 'segment_ts': 'k234', 'time_q': '2023-02-21 02:14:45'} -{'device': 'VcXbfEsURAyiOlXz', 'segment_ts': 'XXHG', 'time_q': '2023-02-06 21:09:36'} -{'device': 'd9jY3TNnU9v39DOE', 'segment_ts': 'JBAB', 'time_q': '2023-02-13 05:44:52'} -{'device': 'bY1aO1ktexSWKAjB', 'segment_ts': 'OUWG', 'time_q': '2023-02-19 16:34:05'} -{'device': '4J0KbbaKXbT17iAy', 'segment_ts': 'THLZ', 'time_q': '2023-02-19 16:01:05'} -{'device': 'TdfnSWLdZuoHDmT4', 'segment_ts': '9bQC', 'time_q': '2023-02-13 07:47:03'} -{'device': 'vKvuSLFrmEElACFv', 'segment_ts': 'WgTc', 'time_q': '2023-02-01 05:41:50'} -{'device': 'EgUuFOflBQ4zUJz2', 'segment_ts': 'dfJl', 'time_q': '2023-02-11 07:30:24'} -{'device': '3cWfkAf3Ma8T8KkO', 'segment_ts': 'PDJS', 'time_q': '2023-02-05 03:09:15'} -{'device': 'xoCum1PFgRMFFAtd', 'segment_ts': 'bdMN', 'time_q': '2023-02-10 21:10:02'} -{'device': 'G10WOxXP7vDSrqG2', 'segment_ts': '9vHF', 'time_q': '2023-01-25 10:44:21'} -{'device': 'GyDiESn1pQ7C2eOq', 'segment_ts': 'X5WH', 'time_q': '2023-02-10 05:32:32'} -{'device': 'Ihum52kryw40U2Pq', 'segment_ts': 'YQGo', 'time_q': '2023-01-29 03:27:14'} -{'device': '9DeVCz49iKTNFPyr', 'segment_ts': '2rzd', 'time_q': '2023-01-24 10:45:49'} -{'device': '7mW6uwngb7RD2lEp', 'segment_ts': '5jBK', 'time_q': '2023-02-11 22:13:48'} -{'device': 'xuW4yuuFgbdd4a6p', 'segment_ts': 'lkBB', 'time_q': '2023-01-31 18:54:23'} -{'device': '5kzp9NXHYWNPgYoh', 'segment_ts': 'fioj', 'time_q': '2023-01-26 03:08:34'} -{'device': 'gmWHoGjwiWF82jHM', 'segment_ts': 'kiO6', 'time_q': '2023-01-22 18:57:36'} -{'device': 'zFyepAAJnlCoeqEU', 'segment_ts': 'tJnv', 'time_q': '2023-02-01 00:26:55'} -{'device': 'EglsUmauQATFBwu6', 'segment_ts': 'fGsr', 'time_q': '2023-01-30 09:02:11'} -{'device': 'qUd53Kl4mymteZxS', 'segment_ts': 'TdgJ', 'time_q': '2023-01-27 05:11:33'} -{'device': 'F5zBVO0bkWwMjrQ3', 'segment_ts': 'yEsz', 'time_q': '2023-02-17 00:44:53'} -{'device': 'HDgL5w5o1AyW8KNq', 'segment_ts': 'WMDV', 'time_q': '2023-02-20 04:25:58'} -{'device': 'b080loiW7g8hJYZN', 'segment_ts': 's8vj', 'time_q': '2023-01-26 23:15:47'} -{'device': 'PLB1THGiEIKtVnuq', 'segment_ts': 'eqHY', 'time_q': '2023-02-17 07:42:59'} -{'device': 'Y90nqduLp4v38lVD', 'segment_ts': 'hZVY', 'time_q': '2023-02-15 14:26:35'} -{'device': 'fh35hFi6oIEaA5Df', 'segment_ts': 'TJ6P', 'time_q': '2023-01-28 02:34:05'} -{'device': 'RBLsLZ4eIt3nrLPH', 'segment_ts': '7WHK', 'time_q': '2023-01-29 20:05:29'} -{'device': '9Db2XAc0xDm9Mbmx', 'segment_ts': 'ls7g', 'time_q': '2023-02-18 08:15:15'} -{'device': 'M1Ih4aY0dYWKWegA', 'segment_ts': 'sgfr', 'time_q': '2023-02-04 00:43:41'} -{'device': '0YtmJi8QdEGnWaHz', 'segment_ts': '5HQX', 'time_q': '2023-01-25 06:09:08'} -{'device': 'LjuzkroRYOCdQSz5', 'segment_ts': 'dzph', 'time_q': '2023-01-24 00:27:56'} -{'device': 'K0y5JORnnCPqA3PL', 'segment_ts': '6jsI', 'time_q': '2023-02-06 10:18:08'} -{'device': '75xJYP1muHt9LVvt', 'segment_ts': '2PcN', 'time_q': '2023-01-25 23:49:07'} -{'device': 'aa15VzCz3SZrbBAl', 'segment_ts': 'FfVa', 'time_q': '2023-02-07 04:37:48'} -{'device': '65lf7rvFyN74HbTL', 'segment_ts': 'q9o4', 'time_q': '2023-02-10 16:57:52'} -{'device': 'BkATNfc6INXfprd4', 'segment_ts': 'QW7U', 'time_q': '2023-02-18 04:08:48'} -{'device': 'HSBPAi87vKtuhabY', 'segment_ts': 'Ft0J', 'time_q': '2023-01-29 04:20:36'} -{'device': 'Rb23K4sxSmDFCiK2', 'segment_ts': 'CZHg', 'time_q': '2023-02-04 22:53:44'} -{'device': 'VzfrrwW2GG8Au619', 'segment_ts': 'qmhZ', 'time_q': '2023-01-29 07:18:18'} -{'device': 'UW5oYzVVoP9VUNvi', 'segment_ts': 'FD5Q', 'time_q': '2023-02-13 03:55:02'} -{'device': '7ZqKm1mzPqTjKAK8', 'segment_ts': 'yAxf', 'time_q': '2023-02-03 08:04:47'} -{'device': 'RKs79WJe0pmc30Ji', 'segment_ts': 'iQ7P', 'time_q': '2023-02-14 20:17:09'} -{'device': 'fawEL3nd7h4bsCSA', 'segment_ts': '5JQi', 'time_q': '2023-02-09 09:47:37'} -{'device': '1cK5y0fv2oasCsEg', 'segment_ts': 'FZfm', 'time_q': '2023-01-24 23:15:26'} -{'device': 'fZlV1wQfgN0slT78', 'segment_ts': 'QjWp', 'time_q': '2023-01-23 18:30:49'} -{'device': 'ReYnQoZKQD7FhiPY', 'segment_ts': 'uTjn', 'time_q': '2023-01-25 01:24:16'} -{'device': 'NPuU31MUQXT5txEw', 'segment_ts': 'clSh', 'time_q': '2023-02-04 23:56:25'} -{'device': 'uu9ToklvVI6lAa9s', 'segment_ts': '9v3C', 'time_q': '2023-01-23 19:15:43'} -{'device': 'iVXaaRjEBBkz0hz3', 'segment_ts': 'aV1U', 'time_q': '2023-02-05 20:11:12'} -{'device': 'xM0S07A9cCsd9Yfn', 'segment_ts': 'MeDE', 'time_q': '2023-01-24 04:13:01'} -{'device': 'nQHMNXhbyFZ6UoDn', 'segment_ts': 'K4Ze', 'time_q': '2023-02-10 16:22:30'} -{'device': 'fXaNQOwz30KmlHrP', 'segment_ts': '36ap', 'time_q': '2023-02-14 12:08:13'} -{'device': 'dw2ya1E1U6tjsd7T', 'segment_ts': 'Jb8U', 'time_q': '2023-02-14 03:53:01'} -{'device': 'GZ0h5bYPi6E6ohn4', 'segment_ts': '44WL', 'time_q': '2023-01-29 13:46:23'} -{'device': 'wNZt7rqhrv3YtKuz', 'segment_ts': 'zaKC', 'time_q': '2023-01-25 03:03:32'} -{'device': 'RpROmDVcSKPVt0C8', 'segment_ts': 'lvES', 'time_q': '2023-02-17 21:12:47'} -{'device': 'jbZJ7XFo3s3aLnjJ', 'segment_ts': 'cvD6', 'time_q': '2023-02-17 14:55:37'} -{'device': '4Qw6ei7d9Ux7cTDd', 'segment_ts': 'gwio', 'time_q': '2023-02-01 08:26:39'} -{'device': 'JLYhTM9PRqiptyZD', 'segment_ts': 'QrPy', 'time_q': '2023-01-26 09:58:00'} -{'device': '6FO0vI0i8EqKQhd6', 'segment_ts': 'hH9B', 'time_q': '2023-01-31 23:43:46'} -{'device': 'iGeoTrx9INiZGRHX', 'segment_ts': 'X2bk', 'time_q': '2023-01-30 16:07:24'} -{'device': '5pRaiBzTt31JZkve', 'segment_ts': '8Auu', 'time_q': '2023-02-05 09:10:17'} -{'device': 'OvWvtocpkdNLqrny', 'segment_ts': 'Inti', 'time_q': '2023-02-12 21:28:35'} -{'device': 'bYwqcLrv6vNbGiov', 'segment_ts': 'ysSv', 'time_q': '2023-02-14 22:07:17'} -{'device': 'o9lqa5RcsnYdxzPx', 'segment_ts': 'R8bW', 'time_q': '2023-01-31 19:56:44'} -{'device': 'WmUqr1hnmvlRw6YO', 'segment_ts': '5viW', 'time_q': '2023-01-28 20:23:04'} -{'device': 'JhzC3ZigvlLK1seo', 'segment_ts': 'J5sY', 'time_q': '2023-02-12 20:59:49'} -{'device': 'evKPQwjRV6cBQvjy', 'segment_ts': 'dDZY', 'time_q': '2023-01-31 04:46:11'} -{'device': 'sLMrZiBFi2lonYwg', 'segment_ts': 'unUy', 'time_q': '2023-02-15 08:51:34'} -{'device': 'o4lT6snRzZnQa1HO', 'segment_ts': 'Z7Ft', 'time_q': '2023-02-03 21:15:32'} -{'device': 'hqZWBbZog5N1kD9V', 'segment_ts': 'WBtx', 'time_q': '2023-02-19 18:01:06'} -{'device': 'xXzLPLm0OsnDJwUz', 'segment_ts': 'K1Yr', 'time_q': '2023-01-28 23:13:12'} -{'device': 'KXSUj8l7UVFARe7n', 'segment_ts': 'BvbW', 'time_q': '2023-01-29 21:44:04'} -{'device': 'UvZA7BW1fWlt0GbE', 'segment_ts': 'ajjJ', 'time_q': '2023-01-24 03:31:50'} -{'device': 'cQGXvK61LboETbwo', 'segment_ts': 'Ey3g', 'time_q': '2023-01-23 21:46:37'} -{'device': 'CQ101gI3Mx4eORJw', 'segment_ts': 'Jb8U', 'time_q': '2023-01-28 03:33:22'} -{'device': 'ed2KkpZOCurKwUW1', 'segment_ts': 'ZWmw', 'time_q': '2023-02-15 06:18:12'} -{'device': 'cmsyEAqBqINzHOdc', 'segment_ts': 'MXoi', 'time_q': '2023-02-10 07:11:49'} -{'device': 'HrEkRR54fFMG0LDI', 'segment_ts': 'siac', 'time_q': '2023-02-07 23:36:13'} -{'device': 'wxGldUWtgrjazC1G', 'segment_ts': 'tvS0', 'time_q': '2023-02-04 05:46:44'} -{'device': 'RxPCG6v5f4sqllhf', 'segment_ts': 'Ysq9', 'time_q': '2023-02-12 12:54:14'} -{'device': 'r7xQr423rEko5eka', 'segment_ts': 'XPPE', 'time_q': '2023-02-07 03:44:00'} -{'device': 'wxuOz2ANcqk4RoaO', 'segment_ts': 'lDol', 'time_q': '2023-02-16 15:04:43'} -{'device': 'PlSBAhofMJFpB2uV', 'segment_ts': 'RFaV', 'time_q': '2023-02-11 22:02:11'} -{'device': 'hZ1tbXscywB9ibyA', 'segment_ts': 'xnZO', 'time_q': '2023-02-20 15:51:50'} -{'device': 'KbFXsiA8Q0SqfSF2', 'segment_ts': 'glBE', 'time_q': '2023-02-18 23:04:11'} -{'device': 'MRPK2r2JhCir2u2l', 'segment_ts': 'Med8', 'time_q': '2023-01-24 09:27:31'} -{'device': 'cEFmg5V2xJHPyqp5', 'segment_ts': 'Kd02', 'time_q': '2023-02-10 09:41:59'} -{'device': '1lwjNCCV5WEJzXT7', 'segment_ts': '5CQq', 'time_q': '2023-01-27 01:07:21'} -{'device': 'VvOulNjY9xQCblv7', 'segment_ts': 'C8F7', 'time_q': '2023-01-27 21:04:34'} -{'device': 'dF1H0PGvmSbpRyiI', 'segment_ts': 'eD44', 'time_q': '2023-01-31 13:38:45'} -{'device': 'NkDgwYVx0XveN5Dt', 'segment_ts': 'WxCz', 'time_q': '2023-02-21 07:16:46'} -{'device': 'LBsuQUNvWKpyd89V', 'segment_ts': 'V9Fm', 'time_q': '2023-02-10 12:25:41'} -{'device': 'n1KoKQk8oZ8lUPtV', 'segment_ts': 'J7YR', 'time_q': '2023-02-16 05:30:48'} -{'device': 'YDqHeGZquVwEjqQc', 'segment_ts': '2GHM', 'time_q': '2023-02-15 00:35:09'} -{'device': 'RgQGcEnSZcPxClfB', 'segment_ts': 'hvML', 'time_q': '2023-01-23 08:49:45'} -{'device': 'jSvUQlxGgJWX0n7M', 'segment_ts': 'HMSz', 'time_q': '2023-02-12 06:13:19'} -{'device': 'K7lIlpxjv71aoW9n', 'segment_ts': 'rkkj', 'time_q': '2023-02-02 15:11:54'} -{'device': 'tlcCO6MF2RYl0g96', 'segment_ts': '10GK', 'time_q': '2023-02-17 18:09:56'} -{'device': 'p4o8sJpApQwJOiox', 'segment_ts': 'XAUS', 'time_q': '2023-01-29 10:34:46'} -{'device': '525KZ78kEncXH9VI', 'segment_ts': 'ugYO', 'time_q': '2023-02-03 20:14:10'} -{'device': '7arB9WqfPZVvkzAs', 'segment_ts': 'Jydo', 'time_q': '2023-02-09 13:03:19'} -{'device': 'GsqKaz0OuOy0p8D5', 'segment_ts': 's3Xs', 'time_q': '2023-02-01 17:04:58'} -{'device': 'NJRTnXytfyhbSltj', 'segment_ts': 'pa4b', 'time_q': '2023-01-22 23:47:33'} -{'device': 'V6BcYDFUslTaeKz2', 'segment_ts': 'c3ip', 'time_q': '2023-01-24 12:24:16'} -{'device': 'Lali4GnxbDXiICfd', 'segment_ts': '5HQX', 'time_q': '2023-02-08 07:21:08'} -{'device': '4G67CFMQFfw5nKhH', 'segment_ts': 'HfZ3', 'time_q': '2023-01-27 13:08:26'} -{'device': 'VOU1PEANNqFpksPD', 'segment_ts': 'UCFz', 'time_q': '2023-02-02 17:55:58'} -{'device': 'NQkRwi0QBjL9OUfe', 'segment_ts': 'peAr', 'time_q': '2023-01-31 09:44:07'} -{'device': 'C1Nzu0HNqUqhiv4v', 'segment_ts': 'ido8', 'time_q': '2023-02-20 11:23:58'} -{'device': 'jc76NMsTRXg6yFT8', 'segment_ts': 'IlVL', 'time_q': '2023-02-04 08:44:26'} -{'device': 'mkPyfZXT9VoyITp1', 'segment_ts': 'AElH', 'time_q': '2023-02-01 22:49:56'} -{'device': 'aODXvK25bBgLB89w', 'segment_ts': 'UyMT', 'time_q': '2023-02-06 18:46:10'} -{'device': 'bIRJnUdmTWC4bQjs', 'segment_ts': 'q6P2', 'time_q': '2023-01-25 10:01:19'} -{'device': 'NwthNY2x1PiG4NXu', 'segment_ts': 'ziKj', 'time_q': '2023-01-26 00:26:01'} -{'device': 'Y0Jwgj0CtaaVPGLU', 'segment_ts': 'uRmO', 'time_q': '2023-02-13 01:29:07'} -{'device': 'P59OfLIdsNz1u2bf', 'segment_ts': 'hm6H', 'time_q': '2023-01-25 21:43:28'} -{'device': 'dHwliEj94uGOjy4z', 'segment_ts': 'bgF8', 'time_q': '2023-02-16 14:27:07'} -{'device': 'Tp85tKVvwtOGxCsN', 'segment_ts': 'z871', 'time_q': '2023-02-03 03:24:04'} -{'device': 'Nu5VHjVpGjotVjHh', 'segment_ts': 'u8t8', 'time_q': '2023-02-14 04:32:01'} -{'device': 'rWcINdZOiu9BHzkL', 'segment_ts': 'CVvt', 'time_q': '2023-02-07 13:15:38'} -{'device': 'yGILVrj3iinOAS4U', 'segment_ts': '0fQb', 'time_q': '2023-02-01 08:06:25'} -{'device': 'CkVe5GZWBIuoCDh9', 'segment_ts': '0JA6', 'time_q': '2023-02-14 22:41:17'} -{'device': 'gGG9uQjtROdibp0O', 'segment_ts': 'mwXi', 'time_q': '2023-02-18 08:50:46'} -{'device': 'GljeFNPy2bmdxmEz', 'segment_ts': '7Jts', 'time_q': '2023-02-06 12:45:43'} -{'device': '3xlhVhGTJmURk3MX', 'segment_ts': 'NZxR', 'time_q': '2023-02-02 14:51:14'} -{'device': '47ujFWpbmPFKdX1g', 'segment_ts': 'KNqJ', 'time_q': '2023-01-25 06:27:30'} -{'device': 'gNDkMVt00vJT84jK', 'segment_ts': 'WX3F', 'time_q': '2023-02-09 12:57:48'} -{'device': 'LCaDtdl6EqAmPlGU', 'segment_ts': 'AVAC', 'time_q': '2023-02-14 13:38:47'} -{'device': 'RMS3UOMf4mt5Nlfx', 'segment_ts': 'dDQ0', 'time_q': '2023-02-15 20:39:12'} -{'device': 'mCCyHPx2EyQkjDI6', 'segment_ts': 'mvNJ', 'time_q': '2023-02-13 15:16:22'} -{'device': 'TWoLnaxEPgoNH3Fk', 'segment_ts': 'bwdO', 'time_q': '2023-02-11 09:10:07'} -{'device': '2U0w1oZWqt42ge69', 'segment_ts': 'OQPD', 'time_q': '2023-02-08 05:38:25'} -{'device': 'Wr1pqW7ogHm1l57C', 'segment_ts': 'yEsz', 'time_q': '2023-02-19 15:25:54'} -{'device': 'X2x7ETjtNN7XU6wE', 'segment_ts': 'R4fq', 'time_q': '2023-02-09 09:30:45'} -{'device': 'JJsmrj8eGnKEpkgT', 'segment_ts': 'jHFs', 'time_q': '2023-01-28 19:05:13'} -{'device': 'jQOGpX4X5gohJjgf', 'segment_ts': 'NDT5', 'time_q': '2023-01-28 07:12:42'} -{'device': 'CktBzqEuXCvd6ThV', 'segment_ts': 'ItFE', 'time_q': '2023-01-24 21:16:41'} -{'device': 'o9JPJauPfRraIlLu', 'segment_ts': '76j0', 'time_q': '2023-01-23 20:11:55'} -{'device': 'QdHyZk5P5vaBYTak', 'segment_ts': 'mfIG', 'time_q': '2023-02-08 05:08:49'} -{'device': 'u4QrAiPlcTouwbT5', 'segment_ts': 'YZHZ', 'time_q': '2023-02-14 11:46:14'} -{'device': '65dIzY2Z9TJWgbhv', 'segment_ts': 'EqdZ', 'time_q': '2023-01-28 12:55:23'} -{'device': 'kT4R6IGOTSqzw7uB', 'segment_ts': 'SGHa', 'time_q': '2023-02-11 03:18:04'} -{'device': '66KWZA8L7G89WDZI', 'segment_ts': 'wTC5', 'time_q': '2023-02-21 14:23:50'} -{'device': 'LRxsGXs41oqHtRqM', 'segment_ts': 'u7El', 'time_q': '2023-02-09 14:03:09'} -{'device': 'hgyESiq7XGtiQCql', 'segment_ts': 'hxmr', 'time_q': '2023-01-26 12:29:40'} -{'device': '0qbdQklTFNy7ISCH', 'segment_ts': 'R9B9', 'time_q': '2023-01-29 15:23:30'} -{'device': 'xYAjqmLYwZPiLGsd', 'segment_ts': 'lIlA', 'time_q': '2023-02-17 07:43:40'} -{'device': 'BzgAUk3wTrBt2JWM', 'segment_ts': 'sfqh', 'time_q': '2023-02-16 04:38:46'} -{'device': 'UrSOAFfBipxV2iV0', 'segment_ts': 'Il3r', 'time_q': '2023-02-13 10:27:30'} -{'device': 'ALr9hJYdlrw6oNcJ', 'segment_ts': 'Keh8', 'time_q': '2023-01-23 20:17:27'} -{'device': 'GVDWDnyS3E5WQkFK', 'segment_ts': 'uW7S', 'time_q': '2023-02-11 17:04:10'} -{'device': '2OKxK3JW7y96EHEa', 'segment_ts': 'KFml', 'time_q': '2023-01-28 00:05:53'} -{'device': 'FPkKren0hQTn7grY', 'segment_ts': 'eShQ', 'time_q': '2023-02-04 04:45:29'} -{'device': 'FWyRyqknGMhkVpo3', 'segment_ts': 'ZdoH', 'time_q': '2023-02-15 23:28:49'} -{'device': 'JK21TkVFQLsvKUG0', 'segment_ts': '1MVW', 'time_q': '2023-02-06 15:20:46'} -{'device': 'cjo9bUnDXUQHlnCa', 'segment_ts': 'PT3c', 'time_q': '2023-01-25 10:30:12'} -{'device': '2a0x5y7y1dd0WSHe', 'segment_ts': 'hrIh', 'time_q': '2023-02-06 13:52:24'} -{'device': 'k0TSwJgBwwVCZLlv', 'segment_ts': 'vRB7', 'time_q': '2023-02-01 22:19:25'} -{'device': 'j5LFK6TZErFfQ1VZ', 'segment_ts': 'tTr2', 'time_q': '2023-01-30 20:49:48'} -{'device': 'JGYeTG9SlkPY6wFa', 'segment_ts': 'Z9Pu', 'time_q': '2023-02-06 20:31:31'} -{'device': 'Ymh6ayRgp1H60AFc', 'segment_ts': 'Bwf8', 'time_q': '2023-02-06 00:29:22'} -{'device': 'Td0EpJOxCHYWUbrR', 'segment_ts': 'v25d', 'time_q': '2023-01-28 05:25:00'} -{'device': 'mS6Iy8Qa2mmoqYQu', 'segment_ts': '9q3k', 'time_q': '2023-01-30 00:18:20'} -{'device': 'fc6guCFHdG8VzC3p', 'segment_ts': 'QjWp', 'time_q': '2023-02-08 11:44:52'} -{'device': 'ItkmsLuHy98vfLUQ', 'segment_ts': 'JoNb', 'time_q': '2023-02-04 16:39:25'} -{'device': 'zEKKWQ45k04JGcCV', 'segment_ts': 'Tpwm', 'time_q': '2023-02-05 19:39:55'} -{'device': 'JPsTfxDVBAtWfW8r', 'segment_ts': '6sUs', 'time_q': '2023-01-29 20:29:27'} -{'device': 'yIeSaXpU3rkCbxUr', 'segment_ts': 'yZ7s', 'time_q': '2023-02-18 01:23:16'} -{'device': 'ekMZsH5A0zvUgkOr', 'segment_ts': 'KSlf', 'time_q': '2023-01-30 07:30:21'} -{'device': '4c1W84L1nNUfHkZV', 'segment_ts': 'vvM0', 'time_q': '2023-02-01 17:47:15'} -{'device': 'RBN3tJA4BYV71FBJ', 'segment_ts': 'eQg8', 'time_q': '2023-02-21 13:02:27'} -{'device': '9DS3pYWFruFTvJZS', 'segment_ts': 'rRuc', 'time_q': '2023-01-25 06:18:46'} -{'device': 'Oge628HPmV0XJxLE', 'segment_ts': 'XDA1', 'time_q': '2023-02-20 09:45:25'} -{'device': 'qHZcNOXzeJFDOsbi', 'segment_ts': 'X3q3', 'time_q': '2023-02-03 12:14:04'} -{'device': 'Z4TwRqUaxlM4nWox', 'segment_ts': 'G8xC', 'time_q': '2023-01-25 21:21:40'} -{'device': 'cN2qypm0e5KXaM4f', 'segment_ts': 'DPPO', 'time_q': '2023-02-20 05:51:54'} -{'device': 'G1umjEcxzSJpDeVf', 'segment_ts': 'ivk4', 'time_q': '2023-02-04 14:30:20'} -{'device': 'Y4YHyYf0eIA0dQT2', 'segment_ts': 'w9bo', 'time_q': '2023-01-30 04:27:41'} -{'device': 'sybCGj8bJcuWVkMH', 'segment_ts': 'n7r6', 'time_q': '2023-02-10 08:55:38'} -{'device': 'G9qtN4oPylocAhLo', 'segment_ts': 'ygKY', 'time_q': '2023-02-11 15:58:38'} -{'device': 'dqbOQmU1nCEM1tsb', 'segment_ts': 'snFa', 'time_q': '2023-01-26 05:18:19'} -{'device': 'PvXh8nHFVEKnfyw5', 'segment_ts': 'es0t', 'time_q': '2023-02-03 04:33:00'} -{'device': '9evTiKiWziX3THWt', 'segment_ts': 'mrI5', 'time_q': '2023-02-09 12:39:43'} -{'device': '8vWA3DPg0sCubmHt', 'segment_ts': 'VyRj', 'time_q': '2023-01-30 17:09:05'} -{'device': 'JBh1VD2mJSzS93VN', 'segment_ts': 'A2E6', 'time_q': '2023-01-23 22:53:01'} -{'device': 'AVTuFDv6MFEeDkkg', 'segment_ts': 'FQxa', 'time_q': '2023-02-16 15:15:49'} -{'device': 'jEnRgZZOuDplk7CK', 'segment_ts': 'McUm', 'time_q': '2023-02-13 16:58:42'} -{'device': 'w9eyvBTQMP0TlFPg', 'segment_ts': '061H', 'time_q': '2023-02-14 07:07:22'} -{'device': 'HxBZsLSXkTOwIj7U', 'segment_ts': 'Yny1', 'time_q': '2023-02-16 18:30:09'} -{'device': 'OOp6NIdrHxTZgb1a', 'segment_ts': '3gVe', 'time_q': '2023-01-27 03:18:26'} -{'device': 'XQeEiBgEVGX3xRyy', 'segment_ts': 'iK88', 'time_q': '2023-01-31 16:16:42'} -{'device': '7XXbvAMS3NbVIuSb', 'segment_ts': 'ikv3', 'time_q': '2023-01-27 17:58:57'} -{'device': 'Z77uepdP4uNOMFEm', 'segment_ts': 'aNrW', 'time_q': '2023-01-25 19:27:06'} -{'device': 'x7Ru0sut9cCNKErD', 'segment_ts': 'fSS4', 'time_q': '2023-02-16 13:17:10'} -{'device': 'tfF7ARhQav7Vs1sW', 'segment_ts': 'yFy4', 'time_q': '2023-02-18 08:18:49'} -{'device': 'IezPnCOr2g6m2iq3', 'segment_ts': 'YOF3', 'time_q': '2023-02-09 18:31:03'} -{'device': 'VOA5cvCyvFZ7bnzk', 'segment_ts': 'Viph', 'time_q': '2023-01-24 18:36:49'} -{'device': 'p0On4aZWYDoxDWHI', 'segment_ts': 'FTU0', 'time_q': '2023-02-07 02:13:54'} -{'device': 'FSpOgwOaHynPz0zH', 'segment_ts': 'WwVR', 'time_q': '2023-02-03 21:12:09'} -{'device': 'PVSlKZkOPl5hFKDc', 'segment_ts': 'AZRz', 'time_q': '2023-02-09 05:04:38'} -{'device': 'hXJ4b5HgGPtZxZDG', 'segment_ts': 'eNW4', 'time_q': '2023-02-06 20:20:25'} -{'device': 'iyM6IB8Kyie9rXTk', 'segment_ts': 'WbDH', 'time_q': '2023-01-25 02:38:32'} -{'device': 'ZHDFyhQFjT36sYEX', 'segment_ts': 'RrPC', 'time_q': '2023-01-24 13:30:11'} -{'device': 'FAmkVIvvqiBv3J2v', 'segment_ts': 'BLc4', 'time_q': '2023-02-05 01:57:00'} -{'device': 'uUlaL9nusB5K9wah', 'segment_ts': '6NJ4', 'time_q': '2023-02-04 05:09:45'} -{'device': 'uq7LIwGy8Cs8VS8S', 'segment_ts': 'Ksz1', 'time_q': '2023-01-31 14:34:35'} -{'device': 'Sljf8wt6YxMJlQ9B', 'segment_ts': '3LOG', 'time_q': '2023-02-08 01:47:29'} -{'device': '8tnuglv62PYooaqm', 'segment_ts': 'vXhF', 'time_q': '2023-02-02 02:09:46'} -{'device': 'B00CXFzWHZ7T6vw5', 'segment_ts': '993f', 'time_q': '2023-01-28 01:31:50'} -{'device': 'FsKiFfXLePj7r3xn', 'segment_ts': 'uR2f', 'time_q': '2023-02-16 05:49:48'} -{'device': 'P2Pssg8wIUhJfJge', 'segment_ts': 'SACW', 'time_q': '2023-01-23 19:12:29'} -{'device': 'F83dj5jXAaNW08tN', 'segment_ts': 'NI7t', 'time_q': '2023-02-03 06:42:10'} -{'device': 'tz2BamJyhAvZSmKL', 'segment_ts': 'WFNu', 'time_q': '2023-02-05 21:04:43'} -{'device': 'od15jkcehIGTLhka', 'segment_ts': 'pY0b', 'time_q': '2023-02-11 09:23:34'} -{'device': 'o4J3jbqm2n2VMxAV', 'segment_ts': '4G0L', 'time_q': '2023-01-24 13:22:35'} -{'device': 'uyBW48TE0tyYNDtB', 'segment_ts': 'RhKk', 'time_q': '2023-01-25 13:14:11'} -{'device': 'ANOQhJ0WObAmU99M', 'segment_ts': 'NyJ1', 'time_q': '2023-02-10 22:25:45'} -{'device': 'vDQzeXOS56cwm5Ha', 'segment_ts': '3G8I', 'time_q': '2023-02-11 19:06:38'} -{'device': 'YqyX9Xk6Kw1DXOad', 'segment_ts': 'Qk0n', 'time_q': '2023-02-10 21:58:02'} -{'device': '9neVj3Sy1VeKQxzF', 'segment_ts': 'j592', 'time_q': '2023-01-28 02:02:51'} -{'device': 'I1750EUbKfHl2yT0', 'segment_ts': 'Zztj', 'time_q': '2023-02-06 23:04:30'} -{'device': 'cdId56I5EVFYECEn', 'segment_ts': 'Trhf', 'time_q': '2023-02-05 18:35:57'} -{'device': 'mnuDFIng7Ffw33N7', 'segment_ts': '7dQn', 'time_q': '2023-01-31 14:53:16'} -{'device': 'x6GzEx5oAis07KM2', 'segment_ts': 'XcKm', 'time_q': '2023-01-23 06:33:10'} -{'device': 'k27SrvBpgJ43XRj8', 'segment_ts': 'zKI7', 'time_q': '2023-02-14 21:54:58'} -{'device': 'lhj3vbaLb1F2K4lH', 'segment_ts': 'EQLa', 'time_q': '2023-02-18 03:17:10'} -{'device': 'TRlW7EbIXGs2Bpy8', 'segment_ts': 'iOm2', 'time_q': '2023-02-04 11:54:32'} -{'device': 'klEC6kMcr7NP6sNg', 'segment_ts': 'bRK7', 'time_q': '2023-02-19 00:37:12'} -{'device': 'TEemRDFbCJdjyNsQ', 'segment_ts': 'D16L', 'time_q': '2023-02-17 11:29:57'} -{'device': 'jptd7AJiFEAk4tSw', 'segment_ts': 'IAOL', 'time_q': '2023-01-31 08:48:08'} -{'device': 'L45DvspGDQF2fOyi', 'segment_ts': 'sALF', 'time_q': '2023-02-05 03:25:00'} -{'device': 'R2NkbrNMNldSBJUG', 'segment_ts': '609r', 'time_q': '2023-02-02 09:49:49'} -{'device': '8ZqVW4kT45fPZy5a', 'segment_ts': 'l00H', 'time_q': '2023-02-11 13:05:33'} -{'device': 'jHbO5YUIUxVvdzJA', 'segment_ts': '8dJN', 'time_q': '2023-02-17 02:03:26'} -{'device': 'ktIY7e9VGCKMjL7P', 'segment_ts': 'Mxrj', 'time_q': '2023-02-20 23:12:59'} -{'device': 'uuOZC7aaCy9sUiT5', 'segment_ts': 'EUD1', 'time_q': '2023-02-01 23:31:46'} -{'device': '3RtgCZcjgxxCmg5T', 'segment_ts': '5JQi', 'time_q': '2023-01-23 12:12:05'} -{'device': 'YnI4T3vi1Lxyt5wa', 'segment_ts': 'rXIU', 'time_q': '2023-02-12 23:58:24'} -{'device': 'ev9IcNjTEZekqALB', 'segment_ts': 'GAxJ', 'time_q': '2023-01-24 12:16:20'} -{'device': 'vp5qi7TRC8KQWk7d', 'segment_ts': 'TJ6P', 'time_q': '2023-02-07 17:28:23'} -{'device': 'bSoAX2L1JHmPXODo', 'segment_ts': 'fasF', 'time_q': '2023-02-06 15:59:59'} -{'device': '3XauXuq6ws73yQla', 'segment_ts': 'SsnB', 'time_q': '2023-01-26 16:11:14'} -{'device': 'TyYnqrqHKPeqZ82q', 'segment_ts': 'Wzwh', 'time_q': '2023-02-16 20:02:19'} -{'device': 'O8hOzOOgxw2iQiHx', 'segment_ts': 'Yj0D', 'time_q': '2023-02-14 00:19:14'} -{'device': 'u4Pxs4Cyj00rtA60', 'segment_ts': '1vCq', 'time_q': '2023-01-31 16:09:15'} -{'device': 'jo8xd3EdVQRVKZG0', 'segment_ts': 'ltBU', 'time_q': '2023-01-31 07:14:48'} -{'device': 'W06dd1sDTy0apH3Y', 'segment_ts': 'ygKY', 'time_q': '2023-02-09 21:21:10'} -{'device': 'Sh7dZSxHMgFlutjC', 'segment_ts': 'MbqF', 'time_q': '2023-02-11 23:49:25'} -{'device': 'o5CLa2P1xlbs3BYb', 'segment_ts': 'v0BW', 'time_q': '2023-02-01 11:00:06'} -{'device': '4wegF6Zxl955mSik', 'segment_ts': '281a', 'time_q': '2023-01-29 11:24:20'} -{'device': 'bx31bZeyk6v3ZATS', 'segment_ts': 'wT1j', 'time_q': '2023-02-03 22:21:54'} -{'device': 'kfBaygAngrkZHyhR', 'segment_ts': 'KFml', 'time_q': '2023-02-11 02:35:20'} -{'device': 'IENwCndPscOSssCf', 'segment_ts': 'tGZF', 'time_q': '2023-02-05 15:36:28'} -{'device': 'JU1KqBsrNFR4vOqo', 'segment_ts': 'U1XA', 'time_q': '2023-01-31 18:39:08'} -{'device': '8qqyRLZQv8IAoQgP', 'segment_ts': '1WZE', 'time_q': '2023-02-03 06:48:42'} -{'device': 'rBCNpDGLi9AodGdX', 'segment_ts': '3J3I', 'time_q': '2023-02-17 06:28:51'} -{'device': 'x1VmoWLCRvrSmkEh', 'segment_ts': 'lqnk', 'time_q': '2023-01-24 18:01:20'} -{'device': 'iL3Swp4VajwaNf3O', 'segment_ts': 'gtdx', 'time_q': '2023-01-23 11:19:34'} -{'device': 'g7iVs7QsRXDubZvV', 'segment_ts': 'Oyvx', 'time_q': '2023-01-29 11:33:47'} -{'device': '4iCnj0wRJoMFKwx4', 'segment_ts': 'XU65', 'time_q': '2023-01-26 13:56:11'} -{'device': 'WWBQzqhhgRMwMVYH', 'segment_ts': 'MwvV', 'time_q': '2023-01-26 21:09:06'} -{'device': 'ZjwDtsLC8jMJIsQ6', 'segment_ts': 'NhT6', 'time_q': '2023-02-10 05:06:17'} -{'device': 'YkI6KyY3TgyviASv', 'segment_ts': 'CZdM', 'time_q': '2023-02-04 03:49:11'} -{'device': 'kemopcvgaWvww445', 'segment_ts': '9EjB', 'time_q': '2023-01-25 18:58:42'} -{'device': 'Nfuv9QZRaTdtZ1j1', 'segment_ts': 'nyoq', 'time_q': '2023-02-08 15:39:28'} -{'device': 'IDNpRkPQqBYG8rIL', 'segment_ts': 'nrGc', 'time_q': '2023-02-19 09:20:51'} -{'device': 'woj5K9AYcOPi8wbH', 'segment_ts': 'q9o4', 'time_q': '2023-01-30 00:47:23'} -{'device': 'A3Q592GiKPtRBifW', 'segment_ts': '479s', 'time_q': '2023-02-03 13:21:56'} -{'device': 't0tSKqKvPJsSLKek', 'segment_ts': 'PzBM', 'time_q': '2023-02-09 19:16:28'} -{'device': 'xvu9RL3gVCt6YIqd', 'segment_ts': 'OE7b', 'time_q': '2023-02-10 13:47:41'} -{'device': 'WMwNdlIc5a3s0o4d', 'segment_ts': 'WFNu', 'time_q': '2023-02-10 20:06:12'} -{'device': 'kkvalugYPF7CeARB', 'segment_ts': 'xOmt', 'time_q': '2023-01-27 20:04:35'} -{'device': 'O2Pxt9eTcZ8DXb7Z', 'segment_ts': '3sv4', 'time_q': '2023-02-10 23:46:23'} -{'device': 'n67yYlHFaWmgsEIn', 'segment_ts': 'MbqF', 'time_q': '2023-01-30 20:38:08'} -{'device': '4vHRlcIj4fA6nyem', 'segment_ts': 'vroI', 'time_q': '2023-02-14 16:20:29'} -{'device': 'TMbltseSls9UOzxq', 'segment_ts': 'v5gi', 'time_q': '2023-02-20 09:03:38'} -{'device': 'Jj6eg1CNcUpAtY4b', 'segment_ts': 'xd2m', 'time_q': '2023-02-16 06:33:10'} -{'device': 'ipuU6YIHXJVKYBb5', 'segment_ts': 'LwNw', 'time_q': '2023-01-28 00:19:52'} -{'device': 't66AJJAWxgfr54og', 'segment_ts': 'q74O', 'time_q': '2023-01-30 06:53:54'} -{'device': '72tpVrLi7idQ711R', 'segment_ts': 'TuWP', 'time_q': '2023-02-19 22:09:35'} -{'device': 'Hvq7OAQ6pXafCIaP', 'segment_ts': 'OdFT', 'time_q': '2023-02-17 20:46:42'} -{'device': 'pme4dFY8cxRdM0iD', 'segment_ts': '7exz', 'time_q': '2023-02-16 00:45:20'} -{'device': 'urddNEFtldtGx3WR', 'segment_ts': 'eZiz', 'time_q': '2023-02-10 06:35:13'} -{'device': 'uZvYbARAUvBgqqiC', 'segment_ts': 'AwpA', 'time_q': '2023-02-05 08:54:00'} -{'device': 'qynlelnsdXES0L3o', 'segment_ts': 'qe17', 'time_q': '2023-02-04 03:25:06'} -{'device': 'VvuWbHbURvKgsqUF', 'segment_ts': '7Sxg', 'time_q': '2023-02-14 10:47:24'} -{'device': 'TlnhHfBk12CQiQ0K', 'segment_ts': 'QVU0', 'time_q': '2023-02-11 01:01:08'} -{'device': 'pPUdA3tMUudFfuu8', 'segment_ts': 'Ubci', 'time_q': '2023-02-20 07:54:52'} -{'device': 'zgZbzsxqUD8EYtTt', 'segment_ts': 'Oyvx', 'time_q': '2023-02-16 23:14:46'} -{'device': 'mJ4o0n4HIVfcYlOo', 'segment_ts': '1Xba', 'time_q': '2023-02-09 23:33:48'} -{'device': 'aqh9qEavdsDiY54R', 'segment_ts': 'O951', 'time_q': '2023-01-23 21:18:26'} -{'device': 'baPCRewYf20VOMzV', 'segment_ts': '7QZg', 'time_q': '2023-01-30 03:32:06'} -{'device': 'JGqBYR14WAkGBwFM', 'segment_ts': '3ES4', 'time_q': '2023-02-06 06:31:37'} -{'device': 'k5Jpfk62razi9ZgC', 'segment_ts': 'jefZ', 'time_q': '2023-02-21 06:43:37'} -{'device': 'J4R3L5UMS3DjpooS', 'segment_ts': 'J8fh', 'time_q': '2023-02-12 03:19:39'} -{'device': 'mKaEAZkAqmHVYbYx', 'segment_ts': 'NZxR', 'time_q': '2023-02-20 12:07:35'} -{'device': 'mnFNrh7YGDVK0LF0', 'segment_ts': '1yCJ', 'time_q': '2023-01-29 11:05:00'} -{'device': '15x8g2YTRH5wxg8D', 'segment_ts': 'hOB3', 'time_q': '2023-02-15 09:54:07'} -{'device': 'MDhSc461Le45qIOD', 'segment_ts': 'Oqnw', 'time_q': '2023-02-13 12:55:24'} -{'device': '2wtPLtzsT0nw72Ki', 'segment_ts': 'iC7M', 'time_q': '2023-02-13 08:02:54'} -{'device': 'EJN36J2EWxJO6d3A', 'segment_ts': 'aCKQ', 'time_q': '2023-01-26 18:11:35'} -{'device': '4Cibygd0vko8tblj', 'segment_ts': 'xF4p', 'time_q': '2023-01-27 09:25:50'} -{'device': '1EryLdrZkHz9TBM2', 'segment_ts': 'M97m', 'time_q': '2023-02-06 08:09:39'} -{'device': 'NUp95xg6x9K07uJw', 'segment_ts': 'DuKl', 'time_q': '2023-02-15 14:42:37'} -{'device': 'QTvOJ3fHPnsnFksM', 'segment_ts': 'PNnd', 'time_q': '2023-01-23 22:31:31'} \ No newline at end of file From 26747362e13f1f612aa29b83a9bf96c6b84c9f92 Mon Sep 17 00:00:00 2001 From: jacob Date: Wed, 22 Feb 2023 08:12:21 -0600 Subject: [PATCH 09/19] clean up time quantum test --- idk/kafka/cmd_test.go | 32 +++++++------------------------- 1 file changed, 7 insertions(+), 25 deletions(-) diff --git a/idk/kafka/cmd_test.go b/idk/kafka/cmd_test.go index 397a2e6de..2cba2a680 100644 --- a/idk/kafka/cmd_test.go +++ b/idk/kafka/cmd_test.go @@ -643,11 +643,11 @@ func TestCmdSchemaChange(t *testing.T) { func TestTimeQuantums(t *testing.T) { t.Parallel() /* - at a high level, a test here represents - - an avro schema - - a set of records to ingest to kafka - - an ingest configuration - - query to run to confirm the data was ingest properly + at a high level, a test here represents + - an avro schema + - a set of records to ingest to kafka + - an ingest configuration + - query to run to confirm the data was ingest properly */ tests := []struct { name string @@ -687,11 +687,7 @@ func TestTimeQuantums(t *testing.T) { }, } - fmt.Printf("created tests") - for _, test := range tests { - - fmt.Printf("starting test") // define some vars now := time.Now().UnixNano() index := fmt.Sprintf("%s_%d", test.index, now) @@ -716,18 +712,6 @@ func TestTimeQuantums(t *testing.T) { data = make(map[string]interface{}) } - /* - records, err := ioutil.ReadFile(test.pathToRecords) - if err != nil { - t.Errorf("issue reading records file") - } - err = json.Unmarshal(records, &data) - if err != nil { - t.Errorf("unmarshal json: %s", err) - } - */ - fmt.Printf("finished reading records") - // configure the consumer consumer, err := NewMain() if err != nil { @@ -736,8 +720,8 @@ func TestTimeQuantums(t *testing.T) { configureTestFlags(consumer) consumer.Index = index consumer.Topics = []string{topic} - //consumer.KafkaBootstrapServers = []string{test.kafkaHost} - //consumer.SchemaRegistryURL = test.registryURL + consumer.KafkaBootstrapServers = []string{test.kafkaHost} + consumer.SchemaRegistryURL = test.registryURL switch test.idType { case "id": consumer.IDField = test.keyField @@ -751,8 +735,6 @@ func TestTimeQuantums(t *testing.T) { } consumer.MaxMsgs = uint64(len(records)) - fmt.Println("finished configuring the consumer") - // load schema registry, create produce, topic and run consumer data licodec := liDecodeTestSchema(t, test.pathToAvroSchema) schemaID := postSchema(t, test.pathToAvroSchema, fmt.Sprintf("%s_id", topic), consumer.SchemaRegistryURL, nil) From df7e813f01ccea53465105b2f18fde271fd387d1 Mon Sep 17 00:00:00 2001 From: Jacob Brinlee <66123601+jrbrinlee1@users.noreply.github.com> Date: Wed, 22 Feb 2023 11:30:14 -0600 Subject: [PATCH 10/19] SUP-302 (#2243) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * add ability to use Θ in all field names & with PQL --- idk/header.go | 2 +- idk/header_test.go | 5 + pilosa.go | 4 +- pilosa_test.go | 26 + pql/parser_test.go | 14 + pql/pql.peg | 10 +- pql/pql.peg.go | 2107 ++++++++++++++++++++++---------------------- pql/pqlpeg_test.go | 11 +- 8 files changed, 1138 insertions(+), 1041 deletions(-) diff --git a/idk/header.go b/idk/header.go index 74581effa..cb7a2166d 100644 --- a/idk/header.go +++ b/idk/header.go @@ -32,7 +32,7 @@ const ( var ( ErrNoFieldSpec = errors.New("no field spec in this header") - ErrInvalidFieldName = errors.New("field name must match [a-z][a-z0-9_-]{0,229}") + ErrInvalidFieldName = errors.New("field name must match [a-z][a-z0-9Θ_-]{0,229}") ErrParsingEpoch = "parsing epoch for " ErrDecodingConfig = "decoding config for field " ) diff --git a/idk/header_test.go b/idk/header_test.go index 053b66f57..44e1df537 100644 --- a/idk/header_test.go +++ b/idk/header_test.go @@ -351,6 +351,11 @@ func TestHeaderToField(t *testing.T) { input: "a__LookupText", exp: LookupTextField{NameVal: "a", DestNameVal: "a"}, }, + { + name: "theta", + input: "fldΘnameΘ__String", + exp: StringField{NameVal: "fldΘnameΘ", DestNameVal: "fldΘnameΘ"}, + }, } for _, test := range tests { diff --git a/pilosa.go b/pilosa.go index dbfe64377..a95417216 100644 --- a/pilosa.go +++ b/pilosa.go @@ -47,7 +47,7 @@ var ( ErrInvalidView = errors.New("invalid view") ErrInvalidCacheType = errors.New("invalid cache type") - ErrName = errors.New("invalid index or field name, must match [a-z][a-z0-9_-]* and contain at most 230 characters") + ErrName = errors.New("invalid index or field name, must match [a-z][a-z0-9Θ_-]* and contain at most 230 characters") // ErrFragmentNotFound is returned when a fragment does not exist. ErrFragmentNotFound = errors.New("fragment not found") @@ -135,7 +135,7 @@ func newPreconditionFailedError(err error) PreconditionFailedError { } // Regular expression to validate index and field names. -var nameRegexp = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,229}$`) +var nameRegexp = regexp.MustCompile(`^[a-z][a-z0-9Θ_-]{0,229}$`) // TimeFormat is the go-style time format used to parse string dates. const TimeFormat = "2006-01-02T15:04" diff --git a/pilosa_test.go b/pilosa_test.go index 1b21e2e29..1d49181d6 100644 --- a/pilosa_test.go +++ b/pilosa_test.go @@ -3,6 +3,7 @@ package pilosa_test import ( + "fmt" "strings" "testing" @@ -43,3 +44,28 @@ func TestAddressWithDefaults(t *testing.T) { } } } + +func TestValidateName(t *testing.T) { + tests := []struct { + name string + err error + }{ + {name: "a_name", err: nil}, + {name: "a-name", err: nil}, + {name: "a-name-10", err: nil}, + {name: "A-name", err: fmt.Errorf("'A-name': %s", pilosa.ErrName)}, + {name: "-a-name", err: fmt.Errorf("'-a-name': %s", pilosa.ErrName)}, + {name: "8th_name", err: fmt.Errorf("'8th_name': %s", pilosa.ErrName)}, + {name: "Θ_name", err: fmt.Errorf("'Θ_name': %s", pilosa.ErrName)}, + {name: "a_NaMe", err: fmt.Errorf("'a_NaMe': %s", pilosa.ErrName)}, + {name: "indexΘname", err: nil}, + } + for _, test := range tests { + err := pilosa.ValidateName(test.name) + if !(err == nil && test.err == nil) { + if err.Error() != test.err.Error() { + t.Errorf("expected error: %v, but got: %v", test.err, err) + } + } + } +} diff --git a/pql/parser_test.go b/pql/parser_test.go index 9296f9935..a7c1815d0 100644 --- a/pql/parser_test.go +++ b/pql/parser_test.go @@ -307,6 +307,20 @@ func TestParser_Parse(t *testing.T) { } } }) + + t.Run("ParseTheta", func(t *testing.T) { + q, err := pql.ParseString(`Row(fldΘname=fldΘval)`) + if err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(q.Calls[0], + &pql.Call{ + Name: "Row", + Args: map[string]interface{}{"fldΘname": "fldΘval"}, + }, + ) { + t.Fatalf("unexpected call: %#v", q.Calls[0]) + } + }) } func TestUnquote(t *testing.T) { diff --git a/pql/pql.peg b/pql/pql.peg index d76322853..eedbdfdd6 100644 --- a/pql/pql.peg +++ b/pql/pql.peg @@ -20,7 +20,7 @@ Call <- "Set" {p.startCall("Set")} open col comma args (comma time)? close {p.e / "Sum" {p.startCall("Sum")} open posfield (comma allargs)? close {p.endCall()} / "Range" {p.startCall("Range")} open field eq value comma 'from='? {p.addField("from")} timefmt {p.addVal(text)} comma 'to='? sp {p.addField("to")} timefmt {p.addVal(text)} close {p.endCall()} / < IDENT > { p.startCall(text) } open allargs comma? close { p.endCall() } -ivyExpr <- ( '(' / '_' / '/' / '[' / ']' /'.' / '=' / '&' / '<'/ '>' /',' / ')' / '^' / '!' / '|' / [*+] / '-' / '?' / [[A-Z]] / [0-9] / '#' / [ \t\n] )* +ivyExpr <- ( '(' / '_' / '/' / '[' / ']' /'.' / '=' / '&' / '<'/ '>' /',' / ')' / '^' / '!' / '|' / [*+] / '-' / '?' / [[A-Z]] / [0-9] / '#' / [ \t\n] / 'Θ')* ivyprogram <- '"'? '"' { p.addPosStr("_ivy", text) } ivyprogram2 <- '"'? '"' { p.addPosStr("_ivyReduce", text) } @@ -55,16 +55,16 @@ item <- 'null' &(comma / close) { p.addVal(nil) } / timestampfmt { p.addTimestampVal(text) } / < decimal > { p.addNumVal(text) } / < IDENT > { p.startCall(text) } open allargs comma? close { p.addVal(p.endCall()) } - / < ([[A-Z]] / [0-9] / '-' / '_' / ':')+ > { p.addVal(text) } + / < ([[A-Z]] / [0-9] / '-' / '_' / ':' / 'Θ')+ > { p.addVal(text) } / < '"' doublequotedstring '"' > { p.addVal(text) } / < '\'' singlequotedstring '\'' > { p.addVal(text) } doublequotedstring <- ( '\\"' / '\\\\' / '\\n' / '\\t' / [^"\\] )* singlequotedstring <- ( '\\\'' / '\\\\' / '\\n' / '\\t' / [^'\\] )* -variable <- ( [[A-Z]] / '_' ) ( [[A-Z]] / [0-9] / '_' / '-' )* +variable <- ( [[A-Z]] / '_' ) ( [[A-Z]] / [0-9] / '_' / '-' / 'Θ')* -fieldExpr <- ( [[A-Z]] / '_' / '$' ) ( [[A-Z]] / [0-9] / '_' / '-' )* +fieldExpr <- ( [[A-Z]] / '_' / '$' ) ( [[A-Z]] / [0-9] / '_' / '-' / 'Θ')* field <- { p.addField(text) } reserved <- '_row' / '_col' / '_start' / '_end' / '_timestamp' / '_field' posfield <- 'field='? { p.addPosStr("_field", text) } @@ -79,7 +79,7 @@ eq <- sp '=' sp comma <- sp ',' sp lbrack <- '[' sp rbrack <- sp ']' sp -IDENT <- [[A-Z]] ([[A-Z]] / [0-9])* +IDENT <- [[A-Z]] ([[A-Z]] / [0-9] / 'Θ')* digits <- [0-9]+ signedDigits <- '-'? digits decimal <- signedDigits ('.' digits?)? diff --git a/pql/pql.peg.go b/pql/pql.peg.go index ba8057489..b5abe7044 100644 --- a/pql/pql.peg.go +++ b/pql/pql.peg.go @@ -1,6 +1,7 @@ package pql // Code generated by peg -inline pql.peg DO NOT EDIT. +// run go install github.com/pointlander/peg@latest import ( "fmt" @@ -2232,7 +2233,7 @@ func (p *PQL) Init(options ...func(*PQL) error) error { position, tokenIndex = position5, tokenIndex5 return false }, - /* 2 ivyExpr <- <('(' / '_' / '/' / '[' / ']' / '.' / '=' / '&' / '<' / '>' / ',' / ')' / '^' / '!' / '|' / ('*' / '+') / '-' / '?' / ([a-z] / [A-Z]) / [0-9] / '#' / (' ' / '\t' / '\n'))*> */ + /* 2 ivyExpr <- <('(' / '_' / '/' / '[' / ']' / '.' / '=' / '&' / '<' / '>' / ',' / ')' / '^' / '!' / '|' / ('*' / '+') / '-' / '?' / ([a-z] / [A-Z]) / [0-9] / '#' / (' ' / '\t' / '\n') / 'Θ')*> */ func() bool { { position218 := position @@ -2411,27 +2412,34 @@ func (p *PQL) Init(options ...func(*PQL) error) error { l246: position, tokenIndex = position221, tokenIndex221 { - position247, tokenIndex247 := position, tokenIndex + position248, tokenIndex248 := position, tokenIndex if buffer[position] != rune(' ') { - goto l248 - } - position++ - goto l247 - l248: - position, tokenIndex = position247, tokenIndex247 - if buffer[position] != rune('\t') { goto l249 } position++ - goto l247 + goto l248 l249: - position, tokenIndex = position247, tokenIndex247 + position, tokenIndex = position248, tokenIndex248 + if buffer[position] != rune('\t') { + goto l250 + } + position++ + goto l248 + l250: + position, tokenIndex = position248, tokenIndex248 if buffer[position] != rune('\n') { - goto l220 + goto l247 } position++ } + l248: + goto l221 l247: + position, tokenIndex = position221, tokenIndex221 + if buffer[position] != rune('Θ') { + goto l220 + } + position++ } l221: goto l219 @@ -2448,267 +2456,267 @@ func (p *PQL) Init(options ...func(*PQL) error) error { nil, /* 5 allargs <- <((Call (comma Call)* (comma args)?) / args / sp)> */ func() bool { - position252, tokenIndex252 := position, tokenIndex + position253, tokenIndex253 := position, tokenIndex { - position253 := position + position254 := position { - position254, tokenIndex254 := position, tokenIndex + position255, tokenIndex255 := position, tokenIndex if !_rules[ruleCall]() { - goto l255 - } - l256: - { - position257, tokenIndex257 := position, tokenIndex - if !_rules[rulecomma]() { - goto l257 - } - if !_rules[ruleCall]() { - goto l257 - } goto l256 - l257: - position, tokenIndex = position257, tokenIndex257 } + l257: { position258, tokenIndex258 := position, tokenIndex if !_rules[rulecomma]() { goto l258 } - if !_rules[ruleargs]() { + if !_rules[ruleCall]() { goto l258 } - goto l259 + goto l257 l258: position, tokenIndex = position258, tokenIndex258 } - l259: - goto l254 - l255: - position, tokenIndex = position254, tokenIndex254 - if !_rules[ruleargs]() { + { + position259, tokenIndex259 := position, tokenIndex + if !_rules[rulecomma]() { + goto l259 + } + if !_rules[ruleargs]() { + goto l259 + } goto l260 + l259: + position, tokenIndex = position259, tokenIndex259 } - goto l254 l260: - position, tokenIndex = position254, tokenIndex254 + goto l255 + l256: + position, tokenIndex = position255, tokenIndex255 + if !_rules[ruleargs]() { + goto l261 + } + goto l255 + l261: + position, tokenIndex = position255, tokenIndex255 if !_rules[rulesp]() { - goto l252 + goto l253 } } - l254: - add(ruleallargs, position253) + l255: + add(ruleallargs, position254) } return true - l252: - position, tokenIndex = position252, tokenIndex252 + l253: + position, tokenIndex = position253, tokenIndex253 return false }, /* 6 args <- <(arg (comma args)? sp)> */ func() bool { - position261, tokenIndex261 := position, tokenIndex + position262, tokenIndex262 := position, tokenIndex { - position262 := position + position263 := position if !_rules[rulearg]() { - goto l261 + goto l262 } { - position263, tokenIndex263 := position, tokenIndex + position264, tokenIndex264 := position, tokenIndex if !_rules[rulecomma]() { - goto l263 + goto l264 } if !_rules[ruleargs]() { - goto l263 + goto l264 } - goto l264 - l263: - position, tokenIndex = position263, tokenIndex263 + goto l265 + l264: + position, tokenIndex = position264, tokenIndex264 } - l264: + l265: if !_rules[rulesp]() { - goto l261 + goto l262 } - add(ruleargs, position262) + add(ruleargs, position263) } return true - l261: - position, tokenIndex = position261, tokenIndex261 + l262: + position, tokenIndex = position262, tokenIndex262 return false }, /* 7 arg <- <((field eq value) / (field sp COND sp value) / conditional)> */ func() bool { - position265, tokenIndex265 := position, tokenIndex + position266, tokenIndex266 := position, tokenIndex { - position266 := position + position267 := position { - position267, tokenIndex267 := position, tokenIndex + position268, tokenIndex268 := position, tokenIndex if !_rules[rulefield]() { - goto l268 + goto l269 } if !_rules[ruleeq]() { - goto l268 + goto l269 } if !_rules[rulevalue]() { - goto l268 - } - goto l267 - l268: - position, tokenIndex = position267, tokenIndex267 - if !_rules[rulefield]() { goto l269 } + goto l268 + l269: + position, tokenIndex = position268, tokenIndex268 + if !_rules[rulefield]() { + goto l270 + } if !_rules[rulesp]() { - goto l269 + goto l270 } { - position270 := position + position271 := position { - position271, tokenIndex271 := position, tokenIndex + position272, tokenIndex272 := position, tokenIndex if buffer[position] != rune('>') { - goto l272 + goto l273 } position++ if buffer[position] != rune('<') { - goto l272 + goto l273 } position++ { add(ruleAction34, position) } - goto l271 - l272: - position, tokenIndex = position271, tokenIndex271 + goto l272 + l273: + position, tokenIndex = position272, tokenIndex272 if buffer[position] != rune('<') { - goto l274 + goto l275 } position++ if buffer[position] != rune('=') { - goto l274 + goto l275 } position++ { add(ruleAction35, position) } - goto l271 - l274: - position, tokenIndex = position271, tokenIndex271 + goto l272 + l275: + position, tokenIndex = position272, tokenIndex272 if buffer[position] != rune('>') { - goto l276 + goto l277 } position++ if buffer[position] != rune('=') { - goto l276 + goto l277 } position++ { add(ruleAction36, position) } - goto l271 - l276: - position, tokenIndex = position271, tokenIndex271 + goto l272 + l277: + position, tokenIndex = position272, tokenIndex272 if buffer[position] != rune('=') { - goto l278 + goto l279 } position++ if buffer[position] != rune('=') { - goto l278 + goto l279 } position++ { add(ruleAction37, position) } - goto l271 - l278: - position, tokenIndex = position271, tokenIndex271 + goto l272 + l279: + position, tokenIndex = position272, tokenIndex272 if buffer[position] != rune('!') { - goto l280 + goto l281 } position++ if buffer[position] != rune('=') { - goto l280 + goto l281 } position++ { add(ruleAction38, position) } - goto l271 - l280: - position, tokenIndex = position271, tokenIndex271 + goto l272 + l281: + position, tokenIndex = position272, tokenIndex272 if buffer[position] != rune('<') { - goto l282 + goto l283 } position++ { add(ruleAction39, position) } - goto l271 - l282: - position, tokenIndex = position271, tokenIndex271 + goto l272 + l283: + position, tokenIndex = position272, tokenIndex272 if buffer[position] != rune('>') { - goto l269 + goto l270 } position++ { add(ruleAction40, position) } } - l271: - add(ruleCOND, position270) + l272: + add(ruleCOND, position271) } if !_rules[rulesp]() { - goto l269 + goto l270 } if !_rules[rulevalue]() { - goto l269 + goto l270 } - goto l267 - l269: - position, tokenIndex = position267, tokenIndex267 + goto l268 + l270: + position, tokenIndex = position268, tokenIndex268 { - position285 := position + position286 := position { add(ruleAction41, position) } if !_rules[rulecondintOrTime]() { - goto l265 + goto l266 } if !_rules[rulecondLT]() { - goto l265 + goto l266 } { - position287 := position + position288 := position { - position288 := position + position289 := position if !_rules[rulefieldExpr]() { - goto l265 + goto l266 } - add(rulePegText, position288) + add(rulePegText, position289) } if !_rules[rulesp]() { - goto l265 + goto l266 } { add(ruleAction46, position) } - add(rulecondfield, position287) + add(rulecondfield, position288) } if !_rules[rulecondLT]() { - goto l265 + goto l266 } if !_rules[rulecondintOrTime]() { - goto l265 + goto l266 } { add(ruleAction42, position) } - add(ruleconditional, position285) + add(ruleconditional, position286) } } - l267: - add(rulearg, position266) + l268: + add(rulearg, position267) } return true - l265: - position, tokenIndex = position265, tokenIndex265 + l266: + position, tokenIndex = position266, tokenIndex266 return false }, /* 8 COND <- <(('>' '<' Action34) / ('<' '=' Action35) / ('>' '=' Action36) / ('=' '=' Action37) / ('!' '=' Action38) / ('<' Action39) / ('>' Action40))> */ @@ -2717,55 +2725,55 @@ func (p *PQL) Init(options ...func(*PQL) error) error { nil, /* 10 condintOrTime <- <(condint / timefmtS)> */ func() bool { - position293, tokenIndex293 := position, tokenIndex + position294, tokenIndex294 := position, tokenIndex { - position294 := position + position295 := position { - position295, tokenIndex295 := position, tokenIndex + position296, tokenIndex296 := position, tokenIndex { - position297 := position + position298 := position { - position298 := position + position299 := position if !_rules[ruledecimal]() { - goto l296 + goto l297 } - add(rulePegText, position298) + add(rulePegText, position299) } if !_rules[rulesp]() { - goto l296 + goto l297 } { add(ruleAction44, position) } - add(rulecondint, position297) + add(rulecondint, position298) } - goto l295 - l296: - position, tokenIndex = position295, tokenIndex295 + goto l296 + l297: + position, tokenIndex = position296, tokenIndex296 { - position300 := position + position301 := position { - position301 := position + position302 := position if !_rules[ruletimestampfmt]() { - goto l293 + goto l294 } - add(rulePegText, position301) + add(rulePegText, position302) } if !_rules[rulesp]() { - goto l293 + goto l294 } { add(ruleAction43, position) } - add(ruletimefmtS, position300) + add(ruletimefmtS, position301) } } - l295: - add(rulecondintOrTime, position294) + l296: + add(rulecondintOrTime, position295) } return true - l293: - position, tokenIndex = position293, tokenIndex293 + l294: + position, tokenIndex = position294, tokenIndex294 return false }, /* 11 timefmtS <- <( sp Action43)> */ @@ -2774,1489 +2782,1524 @@ func (p *PQL) Init(options ...func(*PQL) error) error { nil, /* 13 condLT <- <(<(('<' '=') / '<')> sp Action45)> */ func() bool { - position305, tokenIndex305 := position, tokenIndex + position306, tokenIndex306 := position, tokenIndex { - position306 := position + position307 := position { - position307 := position + position308 := position { - position308, tokenIndex308 := position, tokenIndex + position309, tokenIndex309 := position, tokenIndex if buffer[position] != rune('<') { - goto l309 + goto l310 } position++ if buffer[position] != rune('=') { - goto l309 + goto l310 } position++ - goto l308 - l309: - position, tokenIndex = position308, tokenIndex308 + goto l309 + l310: + position, tokenIndex = position309, tokenIndex309 if buffer[position] != rune('<') { - goto l305 + goto l306 } position++ } - l308: - add(rulePegText, position307) + l309: + add(rulePegText, position308) } if !_rules[rulesp]() { - goto l305 + goto l306 } { add(ruleAction45, position) } - add(rulecondLT, position306) + add(rulecondLT, position307) } return true - l305: - position, tokenIndex = position305, tokenIndex305 + l306: + position, tokenIndex = position306, tokenIndex306 return false }, /* 14 condfield <- <( sp Action46)> */ nil, /* 15 value <- <(item / (lbrack Action47 items rbrack Action48))> */ func() bool { - position312, tokenIndex312 := position, tokenIndex + position313, tokenIndex313 := position, tokenIndex { - position313 := position + position314 := position { - position314, tokenIndex314 := position, tokenIndex + position315, tokenIndex315 := position, tokenIndex if !_rules[ruleitem]() { - goto l315 + goto l316 } - goto l314 - l315: - position, tokenIndex = position314, tokenIndex314 + goto l315 + l316: + position, tokenIndex = position315, tokenIndex315 { - position316 := position + position317 := position if buffer[position] != rune('[') { - goto l312 + goto l313 } position++ if !_rules[rulesp]() { - goto l312 + goto l313 } - add(rulelbrack, position316) + add(rulelbrack, position317) } { add(ruleAction47, position) } if !_rules[ruleitems]() { - goto l312 + goto l313 } { - position318 := position + position319 := position if !_rules[rulesp]() { - goto l312 + goto l313 } if buffer[position] != rune(']') { - goto l312 + goto l313 } position++ if !_rules[rulesp]() { - goto l312 + goto l313 } - add(rulerbrack, position318) + add(rulerbrack, position319) } { add(ruleAction48, position) } } - l314: - add(rulevalue, position313) + l315: + add(rulevalue, position314) } return true - l312: - position, tokenIndex = position312, tokenIndex312 + l313: + position, tokenIndex = position313, tokenIndex313 return false }, /* 16 items <- <(item (comma items)?)> */ func() bool { - position320, tokenIndex320 := position, tokenIndex + position321, tokenIndex321 := position, tokenIndex { - position321 := position + position322 := position if !_rules[ruleitem]() { - goto l320 + goto l321 } { - position322, tokenIndex322 := position, tokenIndex + position323, tokenIndex323 := position, tokenIndex if !_rules[rulecomma]() { - goto l322 + goto l323 } if !_rules[ruleitems]() { - goto l322 + goto l323 } - goto l323 - l322: - position, tokenIndex = position322, tokenIndex322 + goto l324 + l323: + position, tokenIndex = position323, tokenIndex323 } - l323: - add(ruleitems, position321) + l324: + add(ruleitems, position322) } return true - l320: - position, tokenIndex = position320, tokenIndex320 + l321: + position, tokenIndex = position321, tokenIndex321 return false }, - /* 17 item <- <(('n' 'u' 'l' 'l' &(comma / close) Action49) / ('t' 'r' 'u' 'e' &(comma / close) Action50) / ('f' 'a' 'l' 's' 'e' &(comma / close) Action51) / ('$' Action52) / (timefmt Action53) / (timestampfmt Action54) / ( Action55) / ( Action56 open allargs comma? close Action57) / (<([a-z] / [A-Z] / [0-9] / '-' / '_' / ':')+> Action58) / (<('"' doublequotedstring '"')> Action59) / (<('\'' singlequotedstring '\'')> Action60))> */ + /* 17 item <- <(('n' 'u' 'l' 'l' &(comma / close) Action49) / ('t' 'r' 'u' 'e' &(comma / close) Action50) / ('f' 'a' 'l' 's' 'e' &(comma / close) Action51) / ('$' Action52) / (timefmt Action53) / (timestampfmt Action54) / ( Action55) / ( Action56 open allargs comma? close Action57) / (<([a-z] / [A-Z] / [0-9] / '-' / '_' / ':' / 'Θ')+> Action58) / (<('"' doublequotedstring '"')> Action59) / (<('\'' singlequotedstring '\'')> Action60))> */ func() bool { - position324, tokenIndex324 := position, tokenIndex + position325, tokenIndex325 := position, tokenIndex { - position325 := position + position326 := position { - position326, tokenIndex326 := position, tokenIndex + position327, tokenIndex327 := position, tokenIndex if buffer[position] != rune('n') { - goto l327 + goto l328 } position++ if buffer[position] != rune('u') { - goto l327 + goto l328 } position++ if buffer[position] != rune('l') { - goto l327 + goto l328 } position++ if buffer[position] != rune('l') { - goto l327 + goto l328 } position++ { - position328, tokenIndex328 := position, tokenIndex + position329, tokenIndex329 := position, tokenIndex { - position329, tokenIndex329 := position, tokenIndex + position330, tokenIndex330 := position, tokenIndex if !_rules[rulecomma]() { - goto l330 + goto l331 } - goto l329 - l330: - position, tokenIndex = position329, tokenIndex329 + goto l330 + l331: + position, tokenIndex = position330, tokenIndex330 if !_rules[ruleclose]() { - goto l327 + goto l328 } } - l329: - position, tokenIndex = position328, tokenIndex328 + l330: + position, tokenIndex = position329, tokenIndex329 } { add(ruleAction49, position) } - goto l326 - l327: - position, tokenIndex = position326, tokenIndex326 + goto l327 + l328: + position, tokenIndex = position327, tokenIndex327 if buffer[position] != rune('t') { - goto l332 + goto l333 } position++ if buffer[position] != rune('r') { - goto l332 + goto l333 } position++ if buffer[position] != rune('u') { - goto l332 + goto l333 } position++ if buffer[position] != rune('e') { - goto l332 + goto l333 } position++ { - position333, tokenIndex333 := position, tokenIndex + position334, tokenIndex334 := position, tokenIndex { - position334, tokenIndex334 := position, tokenIndex + position335, tokenIndex335 := position, tokenIndex if !_rules[rulecomma]() { - goto l335 + goto l336 } - goto l334 - l335: - position, tokenIndex = position334, tokenIndex334 + goto l335 + l336: + position, tokenIndex = position335, tokenIndex335 if !_rules[ruleclose]() { - goto l332 + goto l333 } } - l334: - position, tokenIndex = position333, tokenIndex333 + l335: + position, tokenIndex = position334, tokenIndex334 } { add(ruleAction50, position) } - goto l326 - l332: - position, tokenIndex = position326, tokenIndex326 + goto l327 + l333: + position, tokenIndex = position327, tokenIndex327 if buffer[position] != rune('f') { - goto l337 + goto l338 } position++ if buffer[position] != rune('a') { - goto l337 + goto l338 } position++ if buffer[position] != rune('l') { - goto l337 + goto l338 } position++ if buffer[position] != rune('s') { - goto l337 + goto l338 } position++ if buffer[position] != rune('e') { - goto l337 + goto l338 } position++ { - position338, tokenIndex338 := position, tokenIndex + position339, tokenIndex339 := position, tokenIndex { - position339, tokenIndex339 := position, tokenIndex + position340, tokenIndex340 := position, tokenIndex if !_rules[rulecomma]() { - goto l340 + goto l341 } - goto l339 - l340: - position, tokenIndex = position339, tokenIndex339 + goto l340 + l341: + position, tokenIndex = position340, tokenIndex340 if !_rules[ruleclose]() { - goto l337 + goto l338 } } - l339: - position, tokenIndex = position338, tokenIndex338 + l340: + position, tokenIndex = position339, tokenIndex339 } { add(ruleAction51, position) } - goto l326 - l337: - position, tokenIndex = position326, tokenIndex326 + goto l327 + l338: + position, tokenIndex = position327, tokenIndex327 if buffer[position] != rune('$') { - goto l342 + goto l343 } position++ { - position343 := position + position344 := position { - position344 := position + position345 := position { - position345, tokenIndex345 := position, tokenIndex + position346, tokenIndex346 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l346 - } - position++ - goto l345 - l346: - position, tokenIndex = position345, tokenIndex345 - if c := buffer[position]; c < rune('A') || c > rune('Z') { goto l347 } position++ - goto l345 + goto l346 l347: - position, tokenIndex = position345, tokenIndex345 + position, tokenIndex = position346, tokenIndex346 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l348 + } + position++ + goto l346 + l348: + position, tokenIndex = position346, tokenIndex346 if buffer[position] != rune('_') { - goto l342 + goto l343 } position++ } - l345: - l348: + l346: + l349: { - position349, tokenIndex349 := position, tokenIndex + position350, tokenIndex350 := position, tokenIndex { - position350, tokenIndex350 := position, tokenIndex + position351, tokenIndex351 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l351 - } - position++ - goto l350 - l351: - position, tokenIndex = position350, tokenIndex350 - if c := buffer[position]; c < rune('A') || c > rune('Z') { goto l352 } position++ - goto l350 + goto l351 l352: - position, tokenIndex = position350, tokenIndex350 - if c := buffer[position]; c < rune('0') || c > rune('9') { + position, tokenIndex = position351, tokenIndex351 + if c := buffer[position]; c < rune('A') || c > rune('Z') { goto l353 } position++ - goto l350 + goto l351 l353: - position, tokenIndex = position350, tokenIndex350 - if buffer[position] != rune('_') { + position, tokenIndex = position351, tokenIndex351 + if c := buffer[position]; c < rune('0') || c > rune('9') { goto l354 } position++ - goto l350 + goto l351 l354: - position, tokenIndex = position350, tokenIndex350 + position, tokenIndex = position351, tokenIndex351 + if buffer[position] != rune('_') { + goto l355 + } + position++ + goto l351 + l355: + position, tokenIndex = position351, tokenIndex351 if buffer[position] != rune('-') { - goto l349 + goto l356 + } + position++ + goto l351 + l356: + position, tokenIndex = position351, tokenIndex351 + if buffer[position] != rune('Θ') { + goto l350 } position++ } + l351: + goto l349 l350: - goto l348 - l349: - position, tokenIndex = position349, tokenIndex349 + position, tokenIndex = position350, tokenIndex350 } - add(rulevariable, position344) + add(rulevariable, position345) } - add(rulePegText, position343) + add(rulePegText, position344) } { add(ruleAction52, position) } - goto l326 - l342: - position, tokenIndex = position326, tokenIndex326 + goto l327 + l343: + position, tokenIndex = position327, tokenIndex327 if !_rules[ruletimefmt]() { - goto l356 + goto l358 } { add(ruleAction53, position) } - goto l326 - l356: - position, tokenIndex = position326, tokenIndex326 + goto l327 + l358: + position, tokenIndex = position327, tokenIndex327 if !_rules[ruletimestampfmt]() { - goto l358 + goto l360 } { add(ruleAction54, position) } - goto l326 - l358: - position, tokenIndex = position326, tokenIndex326 + goto l327 + l360: + position, tokenIndex = position327, tokenIndex327 { - position361 := position + position363 := position if !_rules[ruledecimal]() { - goto l360 + goto l362 } - add(rulePegText, position361) + add(rulePegText, position363) } { add(ruleAction55, position) } - goto l326 - l360: - position, tokenIndex = position326, tokenIndex326 + goto l327 + l362: + position, tokenIndex = position327, tokenIndex327 { - position364 := position + position366 := position if !_rules[ruleIDENT]() { - goto l363 + goto l365 } - add(rulePegText, position364) + add(rulePegText, position366) } { add(ruleAction56, position) } if !_rules[ruleopen]() { - goto l363 + goto l365 } if !_rules[ruleallargs]() { - goto l363 + goto l365 } { - position366, tokenIndex366 := position, tokenIndex + position368, tokenIndex368 := position, tokenIndex if !_rules[rulecomma]() { - goto l366 + goto l368 } - goto l367 - l366: - position, tokenIndex = position366, tokenIndex366 + goto l369 + l368: + position, tokenIndex = position368, tokenIndex368 } - l367: + l369: if !_rules[ruleclose]() { - goto l363 + goto l365 } { add(ruleAction57, position) } - goto l326 - l363: - position, tokenIndex = position326, tokenIndex326 + goto l327 + l365: + position, tokenIndex = position327, tokenIndex327 { - position370 := position + position372 := position { - position373, tokenIndex373 := position, tokenIndex + position375, tokenIndex375 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l374 - } - position++ - goto l373 - l374: - position, tokenIndex = position373, tokenIndex373 - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l375 - } - position++ - goto l373 - l375: - position, tokenIndex = position373, tokenIndex373 - if c := buffer[position]; c < rune('0') || c > rune('9') { goto l376 } position++ - goto l373 + goto l375 l376: - position, tokenIndex = position373, tokenIndex373 - if buffer[position] != rune('-') { + position, tokenIndex = position375, tokenIndex375 + if c := buffer[position]; c < rune('A') || c > rune('Z') { goto l377 } position++ - goto l373 + goto l375 l377: - position, tokenIndex = position373, tokenIndex373 - if buffer[position] != rune('_') { + position, tokenIndex = position375, tokenIndex375 + if c := buffer[position]; c < rune('0') || c > rune('9') { goto l378 } position++ - goto l373 + goto l375 l378: - position, tokenIndex = position373, tokenIndex373 + position, tokenIndex = position375, tokenIndex375 + if buffer[position] != rune('-') { + goto l379 + } + position++ + goto l375 + l379: + position, tokenIndex = position375, tokenIndex375 + if buffer[position] != rune('_') { + goto l380 + } + position++ + goto l375 + l380: + position, tokenIndex = position375, tokenIndex375 if buffer[position] != rune(':') { - goto l369 + goto l381 + } + position++ + goto l375 + l381: + position, tokenIndex = position375, tokenIndex375 + if buffer[position] != rune('Θ') { + goto l371 } position++ } + l375: l373: - l371: { - position372, tokenIndex372 := position, tokenIndex + position374, tokenIndex374 := position, tokenIndex { - position379, tokenIndex379 := position, tokenIndex + position382, tokenIndex382 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l380 - } - position++ - goto l379 - l380: - position, tokenIndex = position379, tokenIndex379 - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l381 - } - position++ - goto l379 - l381: - position, tokenIndex = position379, tokenIndex379 - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l382 - } - position++ - goto l379 - l382: - position, tokenIndex = position379, tokenIndex379 - if buffer[position] != rune('-') { goto l383 } position++ - goto l379 + goto l382 l383: - position, tokenIndex = position379, tokenIndex379 - if buffer[position] != rune('_') { + position, tokenIndex = position382, tokenIndex382 + if c := buffer[position]; c < rune('A') || c > rune('Z') { goto l384 } position++ - goto l379 + goto l382 l384: - position, tokenIndex = position379, tokenIndex379 + position, tokenIndex = position382, tokenIndex382 + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l385 + } + position++ + goto l382 + l385: + position, tokenIndex = position382, tokenIndex382 + if buffer[position] != rune('-') { + goto l386 + } + position++ + goto l382 + l386: + position, tokenIndex = position382, tokenIndex382 + if buffer[position] != rune('_') { + goto l387 + } + position++ + goto l382 + l387: + position, tokenIndex = position382, tokenIndex382 if buffer[position] != rune(':') { - goto l372 + goto l388 + } + position++ + goto l382 + l388: + position, tokenIndex = position382, tokenIndex382 + if buffer[position] != rune('Θ') { + goto l374 } position++ } - l379: - goto l371 - l372: - position, tokenIndex = position372, tokenIndex372 + l382: + goto l373 + l374: + position, tokenIndex = position374, tokenIndex374 } - add(rulePegText, position370) + add(rulePegText, position372) } { add(ruleAction58, position) } - goto l326 - l369: - position, tokenIndex = position326, tokenIndex326 + goto l327 + l371: + position, tokenIndex = position327, tokenIndex327 { - position387 := position + position391 := position if buffer[position] != rune('"') { - goto l386 + goto l390 } position++ if !_rules[ruledoublequotedstring]() { - goto l386 + goto l390 } if buffer[position] != rune('"') { - goto l386 + goto l390 } position++ - add(rulePegText, position387) + add(rulePegText, position391) } { add(ruleAction59, position) } - goto l326 - l386: - position, tokenIndex = position326, tokenIndex326 + goto l327 + l390: + position, tokenIndex = position327, tokenIndex327 { - position389 := position + position393 := position if buffer[position] != rune('\'') { - goto l324 + goto l325 } position++ if !_rules[rulesinglequotedstring]() { - goto l324 + goto l325 } if buffer[position] != rune('\'') { - goto l324 + goto l325 } position++ - add(rulePegText, position389) + add(rulePegText, position393) } { add(ruleAction60, position) } } - l326: - add(ruleitem, position325) + l327: + add(ruleitem, position326) } return true - l324: - position, tokenIndex = position324, tokenIndex324 + l325: + position, tokenIndex = position325, tokenIndex325 return false }, /* 18 doublequotedstring <- <(('\\' '"') / ('\\' '\\') / ('\\' 'n') / ('\\' 't') / (!('"' / '\\') .))*> */ func() bool { { - position392 := position - l393: + position396 := position + l397: { - position394, tokenIndex394 := position, tokenIndex + position398, tokenIndex398 := position, tokenIndex { - position395, tokenIndex395 := position, tokenIndex + position399, tokenIndex399 := position, tokenIndex if buffer[position] != rune('\\') { - goto l396 + goto l400 } position++ if buffer[position] != rune('"') { - goto l396 + goto l400 } position++ - goto l395 - l396: - position, tokenIndex = position395, tokenIndex395 + goto l399 + l400: + position, tokenIndex = position399, tokenIndex399 if buffer[position] != rune('\\') { - goto l397 + goto l401 } position++ if buffer[position] != rune('\\') { - goto l397 + goto l401 } position++ - goto l395 - l397: - position, tokenIndex = position395, tokenIndex395 + goto l399 + l401: + position, tokenIndex = position399, tokenIndex399 if buffer[position] != rune('\\') { - goto l398 + goto l402 } position++ if buffer[position] != rune('n') { - goto l398 + goto l402 } position++ - goto l395 - l398: - position, tokenIndex = position395, tokenIndex395 + goto l399 + l402: + position, tokenIndex = position399, tokenIndex399 if buffer[position] != rune('\\') { - goto l399 + goto l403 } position++ if buffer[position] != rune('t') { - goto l399 + goto l403 } position++ - goto l395 - l399: - position, tokenIndex = position395, tokenIndex395 + goto l399 + l403: + position, tokenIndex = position399, tokenIndex399 { - position400, tokenIndex400 := position, tokenIndex + position404, tokenIndex404 := position, tokenIndex { - position401, tokenIndex401 := position, tokenIndex + position405, tokenIndex405 := position, tokenIndex if buffer[position] != rune('"') { - goto l402 + goto l406 } position++ - goto l401 - l402: - position, tokenIndex = position401, tokenIndex401 + goto l405 + l406: + position, tokenIndex = position405, tokenIndex405 if buffer[position] != rune('\\') { - goto l400 + goto l404 } position++ } - l401: - goto l394 - l400: - position, tokenIndex = position400, tokenIndex400 + l405: + goto l398 + l404: + position, tokenIndex = position404, tokenIndex404 } if !matchDot() { - goto l394 + goto l398 } } - l395: - goto l393 - l394: - position, tokenIndex = position394, tokenIndex394 + l399: + goto l397 + l398: + position, tokenIndex = position398, tokenIndex398 } - add(ruledoublequotedstring, position392) + add(ruledoublequotedstring, position396) } return true }, /* 19 singlequotedstring <- <(('\\' '\'') / ('\\' '\\') / ('\\' 'n') / ('\\' 't') / (!('\'' / '\\') .))*> */ func() bool { { - position404 := position - l405: + position408 := position + l409: { - position406, tokenIndex406 := position, tokenIndex + position410, tokenIndex410 := position, tokenIndex { - position407, tokenIndex407 := position, tokenIndex + position411, tokenIndex411 := position, tokenIndex if buffer[position] != rune('\\') { - goto l408 + goto l412 } position++ if buffer[position] != rune('\'') { - goto l408 + goto l412 } position++ - goto l407 - l408: - position, tokenIndex = position407, tokenIndex407 + goto l411 + l412: + position, tokenIndex = position411, tokenIndex411 if buffer[position] != rune('\\') { - goto l409 + goto l413 } position++ if buffer[position] != rune('\\') { - goto l409 + goto l413 } position++ - goto l407 - l409: - position, tokenIndex = position407, tokenIndex407 + goto l411 + l413: + position, tokenIndex = position411, tokenIndex411 if buffer[position] != rune('\\') { - goto l410 + goto l414 } position++ if buffer[position] != rune('n') { - goto l410 + goto l414 } position++ - goto l407 - l410: - position, tokenIndex = position407, tokenIndex407 + goto l411 + l414: + position, tokenIndex = position411, tokenIndex411 if buffer[position] != rune('\\') { - goto l411 + goto l415 } position++ if buffer[position] != rune('t') { - goto l411 + goto l415 } position++ - goto l407 - l411: - position, tokenIndex = position407, tokenIndex407 + goto l411 + l415: + position, tokenIndex = position411, tokenIndex411 { - position412, tokenIndex412 := position, tokenIndex + position416, tokenIndex416 := position, tokenIndex { - position413, tokenIndex413 := position, tokenIndex + position417, tokenIndex417 := position, tokenIndex if buffer[position] != rune('\'') { - goto l414 + goto l418 } position++ - goto l413 - l414: - position, tokenIndex = position413, tokenIndex413 + goto l417 + l418: + position, tokenIndex = position417, tokenIndex417 if buffer[position] != rune('\\') { - goto l412 + goto l416 } position++ } - l413: - goto l406 - l412: - position, tokenIndex = position412, tokenIndex412 + l417: + goto l410 + l416: + position, tokenIndex = position416, tokenIndex416 } if !matchDot() { - goto l406 + goto l410 } } - l407: - goto l405 - l406: - position, tokenIndex = position406, tokenIndex406 + l411: + goto l409 + l410: + position, tokenIndex = position410, tokenIndex410 } - add(rulesinglequotedstring, position404) + add(rulesinglequotedstring, position408) } return true }, - /* 20 variable <- <(([a-z] / [A-Z] / '_') ([a-z] / [A-Z] / [0-9] / '_' / '-')*)> */ + /* 20 variable <- <(([a-z] / [A-Z] / '_') ([a-z] / [A-Z] / [0-9] / '_' / '-' / 'Θ')*)> */ nil, - /* 21 fieldExpr <- <(([a-z] / [A-Z] / '_' / '$') ([a-z] / [A-Z] / [0-9] / '_' / '-')*)> */ + /* 21 fieldExpr <- <(([a-z] / [A-Z] / '_' / '$') ([a-z] / [A-Z] / [0-9] / '_' / '-' / 'Θ')*)> */ func() bool { - position416, tokenIndex416 := position, tokenIndex + position420, tokenIndex420 := position, tokenIndex { - position417 := position + position421 := position { - position418, tokenIndex418 := position, tokenIndex + position422, tokenIndex422 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l419 + goto l423 } position++ - goto l418 - l419: - position, tokenIndex = position418, tokenIndex418 + goto l422 + l423: + position, tokenIndex = position422, tokenIndex422 if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l424 + } + position++ + goto l422 + l424: + position, tokenIndex = position422, tokenIndex422 + if buffer[position] != rune('_') { + goto l425 + } + position++ + goto l422 + l425: + position, tokenIndex = position422, tokenIndex422 + if buffer[position] != rune('$') { goto l420 } position++ - goto l418 - l420: - position, tokenIndex = position418, tokenIndex418 - if buffer[position] != rune('_') { - goto l421 - } - position++ - goto l418 - l421: - position, tokenIndex = position418, tokenIndex418 - if buffer[position] != rune('$') { - goto l416 - } - position++ } - l418: l422: + l426: { - position423, tokenIndex423 := position, tokenIndex + position427, tokenIndex427 := position, tokenIndex { - position424, tokenIndex424 := position, tokenIndex + position428, tokenIndex428 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l425 + goto l429 } position++ - goto l424 - l425: - position, tokenIndex = position424, tokenIndex424 + goto l428 + l429: + position, tokenIndex = position428, tokenIndex428 if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l426 + goto l430 } position++ - goto l424 - l426: - position, tokenIndex = position424, tokenIndex424 + goto l428 + l430: + position, tokenIndex = position428, tokenIndex428 if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l431 + } + position++ + goto l428 + l431: + position, tokenIndex = position428, tokenIndex428 + if buffer[position] != rune('_') { + goto l432 + } + position++ + goto l428 + l432: + position, tokenIndex = position428, tokenIndex428 + if buffer[position] != rune('-') { + goto l433 + } + position++ + goto l428 + l433: + position, tokenIndex = position428, tokenIndex428 + if buffer[position] != rune('Θ') { goto l427 } position++ - goto l424 - l427: - position, tokenIndex = position424, tokenIndex424 - if buffer[position] != rune('_') { - goto l428 - } - position++ - goto l424 - l428: - position, tokenIndex = position424, tokenIndex424 - if buffer[position] != rune('-') { - goto l423 - } - position++ } - l424: - goto l422 - l423: - position, tokenIndex = position423, tokenIndex423 + l428: + goto l426 + l427: + position, tokenIndex = position427, tokenIndex427 } - add(rulefieldExpr, position417) + add(rulefieldExpr, position421) } return true - l416: - position, tokenIndex = position416, tokenIndex416 + l420: + position, tokenIndex = position420, tokenIndex420 return false }, /* 22 field <- <(<(fieldExpr / reserved)> Action61)> */ func() bool { - position429, tokenIndex429 := position, tokenIndex + position434, tokenIndex434 := position, tokenIndex { - position430 := position + position435 := position { - position431 := position + position436 := position { - position432, tokenIndex432 := position, tokenIndex + position437, tokenIndex437 := position, tokenIndex if !_rules[rulefieldExpr]() { - goto l433 + goto l438 } - goto l432 - l433: - position, tokenIndex = position432, tokenIndex432 + goto l437 + l438: + position, tokenIndex = position437, tokenIndex437 { - position434 := position + position439 := position { - position435, tokenIndex435 := position, tokenIndex + position440, tokenIndex440 := position, tokenIndex if buffer[position] != rune('_') { - goto l436 + goto l441 } position++ if buffer[position] != rune('r') { - goto l436 + goto l441 } position++ if buffer[position] != rune('o') { - goto l436 + goto l441 } position++ if buffer[position] != rune('w') { - goto l436 + goto l441 } position++ - goto l435 - l436: - position, tokenIndex = position435, tokenIndex435 + goto l440 + l441: + position, tokenIndex = position440, tokenIndex440 if buffer[position] != rune('_') { - goto l437 + goto l442 } position++ if buffer[position] != rune('c') { - goto l437 + goto l442 } position++ if buffer[position] != rune('o') { - goto l437 + goto l442 } position++ if buffer[position] != rune('l') { - goto l437 + goto l442 } position++ - goto l435 - l437: - position, tokenIndex = position435, tokenIndex435 + goto l440 + l442: + position, tokenIndex = position440, tokenIndex440 if buffer[position] != rune('_') { - goto l438 + goto l443 } position++ if buffer[position] != rune('s') { - goto l438 + goto l443 } position++ if buffer[position] != rune('t') { - goto l438 + goto l443 } position++ if buffer[position] != rune('a') { - goto l438 + goto l443 } position++ if buffer[position] != rune('r') { - goto l438 + goto l443 } position++ if buffer[position] != rune('t') { - goto l438 + goto l443 } position++ - goto l435 - l438: - position, tokenIndex = position435, tokenIndex435 + goto l440 + l443: + position, tokenIndex = position440, tokenIndex440 if buffer[position] != rune('_') { - goto l439 + goto l444 } position++ if buffer[position] != rune('e') { - goto l439 + goto l444 } position++ if buffer[position] != rune('n') { - goto l439 + goto l444 } position++ if buffer[position] != rune('d') { - goto l439 + goto l444 } position++ - goto l435 - l439: - position, tokenIndex = position435, tokenIndex435 + goto l440 + l444: + position, tokenIndex = position440, tokenIndex440 if buffer[position] != rune('_') { - goto l440 + goto l445 } position++ if buffer[position] != rune('t') { - goto l440 + goto l445 } position++ if buffer[position] != rune('i') { - goto l440 + goto l445 } position++ if buffer[position] != rune('m') { - goto l440 + goto l445 } position++ if buffer[position] != rune('e') { - goto l440 + goto l445 } position++ if buffer[position] != rune('s') { - goto l440 + goto l445 } position++ if buffer[position] != rune('t') { - goto l440 + goto l445 } position++ if buffer[position] != rune('a') { - goto l440 + goto l445 } position++ if buffer[position] != rune('m') { - goto l440 + goto l445 } position++ if buffer[position] != rune('p') { - goto l440 + goto l445 } position++ - goto l435 - l440: - position, tokenIndex = position435, tokenIndex435 + goto l440 + l445: + position, tokenIndex = position440, tokenIndex440 if buffer[position] != rune('_') { - goto l429 + goto l434 } position++ if buffer[position] != rune('f') { - goto l429 + goto l434 } position++ if buffer[position] != rune('i') { - goto l429 + goto l434 } position++ if buffer[position] != rune('e') { - goto l429 + goto l434 } position++ if buffer[position] != rune('l') { - goto l429 + goto l434 } position++ if buffer[position] != rune('d') { - goto l429 + goto l434 } position++ } - l435: - add(rulereserved, position434) + l440: + add(rulereserved, position439) } } - l432: - add(rulePegText, position431) + l437: + add(rulePegText, position436) } { add(ruleAction61, position) } - add(rulefield, position430) + add(rulefield, position435) } return true - l429: - position, tokenIndex = position429, tokenIndex429 + l434: + position, tokenIndex = position434, tokenIndex434 return false }, /* 23 reserved <- <(('_' 'r' 'o' 'w') / ('_' 'c' 'o' 'l') / ('_' 's' 't' 'a' 'r' 't') / ('_' 'e' 'n' 'd') / ('_' 't' 'i' 'm' 'e' 's' 't' 'a' 'm' 'p') / ('_' 'f' 'i' 'e' 'l' 'd'))> */ nil, /* 24 posfield <- <(('f' 'i' 'e' 'l' 'd' '=')? Action62)> */ func() bool { - position443, tokenIndex443 := position, tokenIndex + position448, tokenIndex448 := position, tokenIndex { - position444 := position + position449 := position { - position445, tokenIndex445 := position, tokenIndex + position450, tokenIndex450 := position, tokenIndex if buffer[position] != rune('f') { - goto l445 + goto l450 } position++ if buffer[position] != rune('i') { - goto l445 + goto l450 } position++ if buffer[position] != rune('e') { - goto l445 + goto l450 } position++ if buffer[position] != rune('l') { - goto l445 + goto l450 } position++ if buffer[position] != rune('d') { - goto l445 + goto l450 } position++ if buffer[position] != rune('=') { - goto l445 + goto l450 } position++ - goto l446 - l445: - position, tokenIndex = position445, tokenIndex445 + goto l451 + l450: + position, tokenIndex = position450, tokenIndex450 } - l446: + l451: { - position447 := position + position452 := position if !_rules[rulefieldExpr]() { - goto l443 + goto l448 } - add(rulePegText, position447) + add(rulePegText, position452) } { add(ruleAction62, position) } - add(ruleposfield, position444) + add(ruleposfield, position449) } return true - l443: - position, tokenIndex = position443, tokenIndex443 + l448: + position, tokenIndex = position448, tokenIndex448 return false }, /* 25 col <- <(( Action63) / (<('\'' singlequotedstring '\'')> Action64) / (<('"' doublequotedstring '"')> Action65))> */ func() bool { - position449, tokenIndex449 := position, tokenIndex + position454, tokenIndex454 := position, tokenIndex { - position450 := position + position455 := position { - position451, tokenIndex451 := position, tokenIndex + position456, tokenIndex456 := position, tokenIndex { - position453 := position + position458 := position if !_rules[ruledigits]() { - goto l452 + goto l457 } - add(rulePegText, position453) + add(rulePegText, position458) } { add(ruleAction63, position) } - goto l451 - l452: - position, tokenIndex = position451, tokenIndex451 + goto l456 + l457: + position, tokenIndex = position456, tokenIndex456 { - position456 := position + position461 := position if buffer[position] != rune('\'') { - goto l455 + goto l460 } position++ if !_rules[rulesinglequotedstring]() { - goto l455 + goto l460 } if buffer[position] != rune('\'') { - goto l455 + goto l460 } position++ - add(rulePegText, position456) + add(rulePegText, position461) } { add(ruleAction64, position) } - goto l451 - l455: - position, tokenIndex = position451, tokenIndex451 + goto l456 + l460: + position, tokenIndex = position456, tokenIndex456 { - position458 := position + position463 := position if buffer[position] != rune('"') { - goto l449 + goto l454 } position++ if !_rules[ruledoublequotedstring]() { - goto l449 + goto l454 } if buffer[position] != rune('"') { - goto l449 + goto l454 } position++ - add(rulePegText, position458) + add(rulePegText, position463) } { add(ruleAction65, position) } } - l451: - add(rulecol, position450) + l456: + add(rulecol, position455) } return true - l449: - position, tokenIndex = position449, tokenIndex449 + l454: + position, tokenIndex = position454, tokenIndex454 return false }, /* 26 open <- <('(' sp)> */ func() bool { - position460, tokenIndex460 := position, tokenIndex + position465, tokenIndex465 := position, tokenIndex { - position461 := position + position466 := position if buffer[position] != rune('(') { - goto l460 + goto l465 } position++ if !_rules[rulesp]() { - goto l460 + goto l465 } - add(ruleopen, position461) + add(ruleopen, position466) } return true - l460: - position, tokenIndex = position460, tokenIndex460 + l465: + position, tokenIndex = position465, tokenIndex465 return false }, /* 27 close <- <(sp ')' sp)> */ func() bool { - position462, tokenIndex462 := position, tokenIndex + position467, tokenIndex467 := position, tokenIndex { - position463 := position + position468 := position if !_rules[rulesp]() { - goto l462 + goto l467 } if buffer[position] != rune(')') { - goto l462 + goto l467 } position++ if !_rules[rulesp]() { - goto l462 + goto l467 } - add(ruleclose, position463) + add(ruleclose, position468) } return true - l462: - position, tokenIndex = position462, tokenIndex462 + l467: + position, tokenIndex = position467, tokenIndex467 return false }, /* 28 sp <- <(' ' / '\t' / '\n')*> */ func() bool { { - position465 := position - l466: + position470 := position + l471: { - position467, tokenIndex467 := position, tokenIndex + position472, tokenIndex472 := position, tokenIndex { - position468, tokenIndex468 := position, tokenIndex + position473, tokenIndex473 := position, tokenIndex if buffer[position] != rune(' ') { - goto l469 + goto l474 } position++ - goto l468 - l469: - position, tokenIndex = position468, tokenIndex468 + goto l473 + l474: + position, tokenIndex = position473, tokenIndex473 if buffer[position] != rune('\t') { - goto l470 + goto l475 } position++ - goto l468 - l470: - position, tokenIndex = position468, tokenIndex468 + goto l473 + l475: + position, tokenIndex = position473, tokenIndex473 if buffer[position] != rune('\n') { - goto l467 + goto l472 } position++ } - l468: - goto l466 - l467: - position, tokenIndex = position467, tokenIndex467 + l473: + goto l471 + l472: + position, tokenIndex = position472, tokenIndex472 } - add(rulesp, position465) + add(rulesp, position470) } return true }, /* 29 eq <- <(sp '=' sp)> */ func() bool { - position471, tokenIndex471 := position, tokenIndex + position476, tokenIndex476 := position, tokenIndex { - position472 := position + position477 := position if !_rules[rulesp]() { - goto l471 + goto l476 } if buffer[position] != rune('=') { - goto l471 + goto l476 } position++ if !_rules[rulesp]() { - goto l471 + goto l476 } - add(ruleeq, position472) + add(ruleeq, position477) } return true - l471: - position, tokenIndex = position471, tokenIndex471 + l476: + position, tokenIndex = position476, tokenIndex476 return false }, /* 30 comma <- <(sp ',' sp)> */ func() bool { - position473, tokenIndex473 := position, tokenIndex + position478, tokenIndex478 := position, tokenIndex { - position474 := position + position479 := position if !_rules[rulesp]() { - goto l473 + goto l478 } if buffer[position] != rune(',') { - goto l473 + goto l478 } position++ if !_rules[rulesp]() { - goto l473 + goto l478 } - add(rulecomma, position474) + add(rulecomma, position479) } return true - l473: - position, tokenIndex = position473, tokenIndex473 + l478: + position, tokenIndex = position478, tokenIndex478 return false }, /* 31 lbrack <- <('[' sp)> */ nil, /* 32 rbrack <- <(sp ']' sp)> */ nil, - /* 33 IDENT <- <(([a-z] / [A-Z]) ([a-z] / [A-Z] / [0-9])*)> */ + /* 33 IDENT <- <(([a-z] / [A-Z]) ([a-z] / [A-Z] / [0-9] / 'Θ')*)> */ func() bool { - position477, tokenIndex477 := position, tokenIndex + position482, tokenIndex482 := position, tokenIndex { - position478 := position + position483 := position { - position479, tokenIndex479 := position, tokenIndex + position484, tokenIndex484 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l480 + goto l485 } position++ - goto l479 - l480: - position, tokenIndex = position479, tokenIndex479 + goto l484 + l485: + position, tokenIndex = position484, tokenIndex484 if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l477 + goto l482 } position++ } - l479: - l481: + l484: + l486: { - position482, tokenIndex482 := position, tokenIndex + position487, tokenIndex487 := position, tokenIndex { - position483, tokenIndex483 := position, tokenIndex + position488, tokenIndex488 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l484 + goto l489 } position++ - goto l483 - l484: - position, tokenIndex = position483, tokenIndex483 + goto l488 + l489: + position, tokenIndex = position488, tokenIndex488 if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l485 + goto l490 } position++ - goto l483 - l485: - position, tokenIndex = position483, tokenIndex483 + goto l488 + l490: + position, tokenIndex = position488, tokenIndex488 if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l482 + goto l491 + } + position++ + goto l488 + l491: + position, tokenIndex = position488, tokenIndex488 + if buffer[position] != rune('Θ') { + goto l487 } position++ } - l483: - goto l481 - l482: - position, tokenIndex = position482, tokenIndex482 + l488: + goto l486 + l487: + position, tokenIndex = position487, tokenIndex487 } - add(ruleIDENT, position478) + add(ruleIDENT, position483) } return true - l477: - position, tokenIndex = position477, tokenIndex477 + l482: + position, tokenIndex = position482, tokenIndex482 return false }, /* 34 digits <- <[0-9]+> */ func() bool { - position486, tokenIndex486 := position, tokenIndex + position492, tokenIndex492 := position, tokenIndex { - position487 := position + position493 := position if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l486 + goto l492 } position++ - l488: + l494: { - position489, tokenIndex489 := position, tokenIndex + position495, tokenIndex495 := position, tokenIndex if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l489 + goto l495 } position++ - goto l488 - l489: - position, tokenIndex = position489, tokenIndex489 + goto l494 + l495: + position, tokenIndex = position495, tokenIndex495 } - add(ruledigits, position487) + add(ruledigits, position493) } return true - l486: - position, tokenIndex = position486, tokenIndex486 + l492: + position, tokenIndex = position492, tokenIndex492 return false }, /* 35 signedDigits <- <('-'? digits)> */ nil, /* 36 decimal <- <((signedDigits ('.' digits?)?) / ('-'? '.' digits))> */ func() bool { - position491, tokenIndex491 := position, tokenIndex + position497, tokenIndex497 := position, tokenIndex { - position492 := position + position498 := position { - position493, tokenIndex493 := position, tokenIndex + position499, tokenIndex499 := position, tokenIndex { - position495 := position + position501 := position { - position496, tokenIndex496 := position, tokenIndex + position502, tokenIndex502 := position, tokenIndex if buffer[position] != rune('-') { - goto l496 + goto l502 } position++ - goto l497 - l496: - position, tokenIndex = position496, tokenIndex496 + goto l503 + l502: + position, tokenIndex = position502, tokenIndex502 } - l497: + l503: if !_rules[ruledigits]() { - goto l494 + goto l500 } - add(rulesignedDigits, position495) + add(rulesignedDigits, position501) } { - position498, tokenIndex498 := position, tokenIndex + position504, tokenIndex504 := position, tokenIndex if buffer[position] != rune('.') { - goto l498 + goto l504 } position++ { - position500, tokenIndex500 := position, tokenIndex + position506, tokenIndex506 := position, tokenIndex if !_rules[ruledigits]() { - goto l500 + goto l506 } - goto l501 - l500: - position, tokenIndex = position500, tokenIndex500 + goto l507 + l506: + position, tokenIndex = position506, tokenIndex506 } - l501: - goto l499 - l498: - position, tokenIndex = position498, tokenIndex498 + l507: + goto l505 + l504: + position, tokenIndex = position504, tokenIndex504 } - l499: - goto l493 - l494: - position, tokenIndex = position493, tokenIndex493 + l505: + goto l499 + l500: + position, tokenIndex = position499, tokenIndex499 { - position502, tokenIndex502 := position, tokenIndex + position508, tokenIndex508 := position, tokenIndex if buffer[position] != rune('-') { - goto l502 + goto l508 } position++ - goto l503 - l502: - position, tokenIndex = position502, tokenIndex502 + goto l509 + l508: + position, tokenIndex = position508, tokenIndex508 } - l503: + l509: if buffer[position] != rune('.') { - goto l491 + goto l497 } position++ if !_rules[ruledigits]() { - goto l491 + goto l497 } } - l493: - add(ruledecimal, position492) + l499: + add(ruledecimal, position498) } return true - l491: - position, tokenIndex = position491, tokenIndex491 + l497: + position, tokenIndex = position497, tokenIndex497 return false }, /* 37 tz <- <('Z' / ('-' [0-9] [0-9] ':' [0-9] [0-9]) / ('+' [0-9] [0-9] ':' [0-9] [0-9]))> */ func() bool { - position504, tokenIndex504 := position, tokenIndex + position510, tokenIndex510 := position, tokenIndex { - position505 := position + position511 := position { - position506, tokenIndex506 := position, tokenIndex + position512, tokenIndex512 := position, tokenIndex if buffer[position] != rune('Z') { - goto l507 + goto l513 } position++ - goto l506 - l507: - position, tokenIndex = position506, tokenIndex506 + goto l512 + l513: + position, tokenIndex = position512, tokenIndex512 if buffer[position] != rune('-') { - goto l508 + goto l514 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l508 + goto l514 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l508 + goto l514 } position++ if buffer[position] != rune(':') { - goto l508 + goto l514 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l508 + goto l514 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l508 + goto l514 } position++ - goto l506 - l508: - position, tokenIndex = position506, tokenIndex506 + goto l512 + l514: + position, tokenIndex = position512, tokenIndex512 if buffer[position] != rune('+') { - goto l504 + goto l510 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l504 + goto l510 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l504 + goto l510 } position++ if buffer[position] != rune(':') { - goto l504 + goto l510 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l504 + goto l510 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l504 + goto l510 } position++ } - l506: - add(ruletz, position505) + l512: + add(ruletz, position511) } return true - l504: - position, tokenIndex = position504, tokenIndex504 + l510: + position, tokenIndex = position510, tokenIndex510 return false }, /* 38 iso8601 <- <([0-9] [0-9] [0-9] [0-9] '-' ('0' / '1') [0-9] '-' [0-3] [0-9] 'T' [0-9] [0-9] ':' [0-9] [0-9] ':' [0-9] [0-9] )> */ @@ -4265,151 +4308,31 @@ func (p *PQL) Init(options ...func(*PQL) error) error { nil, /* 40 timestampbasicfmt <- <(iso8601nano / iso8601)> */ func() bool { - position511, tokenIndex511 := position, tokenIndex + position517, tokenIndex517 := position, tokenIndex { - position512 := position + position518 := position { - position513, tokenIndex513 := position, tokenIndex - { - position515 := position - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l514 - } - position++ - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l514 - } - position++ - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l514 - } - position++ - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l514 - } - position++ - if buffer[position] != rune('-') { - goto l514 - } - position++ - { - position516, tokenIndex516 := position, tokenIndex - if buffer[position] != rune('0') { - goto l517 - } - position++ - goto l516 - l517: - position, tokenIndex = position516, tokenIndex516 - if buffer[position] != rune('1') { - goto l514 - } - position++ - } - l516: - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l514 - } - position++ - if buffer[position] != rune('-') { - goto l514 - } - position++ - if c := buffer[position]; c < rune('0') || c > rune('3') { - goto l514 - } - position++ - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l514 - } - position++ - if buffer[position] != rune('T') { - goto l514 - } - position++ - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l514 - } - position++ - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l514 - } - position++ - if buffer[position] != rune(':') { - goto l514 - } - position++ - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l514 - } - position++ - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l514 - } - position++ - if buffer[position] != rune(':') { - goto l514 - } - position++ - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l514 - } - position++ - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l514 - } - position++ - if buffer[position] != rune('.') { - goto l514 - } - position++ - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l514 - } - position++ - l518: - { - position519, tokenIndex519 := position, tokenIndex - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l519 - } - position++ - goto l518 - l519: - position, tokenIndex = position519, tokenIndex519 - } - { - position520 := position - if !_rules[ruletz]() { - goto l514 - } - add(rulePegText, position520) - } - add(ruleiso8601nano, position515) - } - goto l513 - l514: - position, tokenIndex = position513, tokenIndex513 + position519, tokenIndex519 := position, tokenIndex { position521 := position if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l511 + goto l520 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l511 + goto l520 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l511 + goto l520 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l511 + goto l520 } position++ if buffer[position] != rune('-') { - goto l511 + goto l520 } position++ { @@ -4422,284 +4345,404 @@ func (p *PQL) Init(options ...func(*PQL) error) error { l523: position, tokenIndex = position522, tokenIndex522 if buffer[position] != rune('1') { - goto l511 + goto l520 } position++ } l522: if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l511 + goto l520 } position++ if buffer[position] != rune('-') { - goto l511 + goto l520 } position++ if c := buffer[position]; c < rune('0') || c > rune('3') { - goto l511 + goto l520 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l511 + goto l520 } position++ if buffer[position] != rune('T') { - goto l511 + goto l520 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l511 + goto l520 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l511 + goto l520 } position++ if buffer[position] != rune(':') { - goto l511 + goto l520 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l511 + goto l520 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l511 + goto l520 } position++ if buffer[position] != rune(':') { - goto l511 + goto l520 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l511 + goto l520 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l511 + goto l520 + } + position++ + if buffer[position] != rune('.') { + goto l520 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l520 + } + position++ + l524: + { + position525, tokenIndex525 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l525 + } + position++ + goto l524 + l525: + position, tokenIndex = position525, tokenIndex525 + } + { + position526 := position + if !_rules[ruletz]() { + goto l520 + } + add(rulePegText, position526) + } + add(ruleiso8601nano, position521) + } + goto l519 + l520: + position, tokenIndex = position519, tokenIndex519 + { + position527 := position + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l517 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l517 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l517 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l517 + } + position++ + if buffer[position] != rune('-') { + goto l517 } position++ { - position524 := position - if !_rules[ruletz]() { - goto l511 + position528, tokenIndex528 := position, tokenIndex + if buffer[position] != rune('0') { + goto l529 } - add(rulePegText, position524) + position++ + goto l528 + l529: + position, tokenIndex = position528, tokenIndex528 + if buffer[position] != rune('1') { + goto l517 + } + position++ } - add(ruleiso8601, position521) + l528: + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l517 + } + position++ + if buffer[position] != rune('-') { + goto l517 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('3') { + goto l517 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l517 + } + position++ + if buffer[position] != rune('T') { + goto l517 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l517 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l517 + } + position++ + if buffer[position] != rune(':') { + goto l517 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l517 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l517 + } + position++ + if buffer[position] != rune(':') { + goto l517 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l517 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l517 + } + position++ + { + position530 := position + if !_rules[ruletz]() { + goto l517 + } + add(rulePegText, position530) + } + add(ruleiso8601, position527) } } - l513: - add(ruletimestampbasicfmt, position512) + l519: + add(ruletimestampbasicfmt, position518) } return true - l511: - position, tokenIndex = position511, tokenIndex511 + l517: + position, tokenIndex = position517, tokenIndex517 return false }, /* 41 timestampfmt <- <(('"' '"') / ('\'' '\'') / )> */ func() bool { - position525, tokenIndex525 := position, tokenIndex + position531, tokenIndex531 := position, tokenIndex { - position526 := position + position532 := position { - position527, tokenIndex527 := position, tokenIndex + position533, tokenIndex533 := position, tokenIndex if buffer[position] != rune('"') { - goto l528 + goto l534 } position++ { - position529 := position + position535 := position if !_rules[ruletimestampbasicfmt]() { - goto l528 + goto l534 } - add(rulePegText, position529) + add(rulePegText, position535) } if buffer[position] != rune('"') { - goto l528 + goto l534 } position++ - goto l527 - l528: - position, tokenIndex = position527, tokenIndex527 + goto l533 + l534: + position, tokenIndex = position533, tokenIndex533 if buffer[position] != rune('\'') { - goto l530 + goto l536 } position++ { - position531 := position + position537 := position if !_rules[ruletimestampbasicfmt]() { - goto l530 + goto l536 } - add(rulePegText, position531) + add(rulePegText, position537) } if buffer[position] != rune('\'') { - goto l530 + goto l536 } position++ - goto l527 - l530: - position, tokenIndex = position527, tokenIndex527 + goto l533 + l536: + position, tokenIndex = position533, tokenIndex533 { - position532 := position + position538 := position if !_rules[ruletimestampbasicfmt]() { - goto l525 + goto l531 } - add(rulePegText, position532) + add(rulePegText, position538) } } - l527: - add(ruletimestampfmt, position526) + l533: + add(ruletimestampfmt, position532) } return true - l525: - position, tokenIndex = position525, tokenIndex525 + l531: + position, tokenIndex = position531, tokenIndex531 return false }, /* 42 timebasicfmt <- <([0-9] [0-9] [0-9] [0-9] '-' ('0' / '1') [0-9] '-' [0-3] [0-9] 'T' [0-9] [0-9] ':' [0-9] [0-9])> */ func() bool { - position533, tokenIndex533 := position, tokenIndex + position539, tokenIndex539 := position, tokenIndex { - position534 := position + position540 := position if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l533 + goto l539 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l533 + goto l539 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l533 + goto l539 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l533 + goto l539 } position++ if buffer[position] != rune('-') { - goto l533 + goto l539 } position++ { - position535, tokenIndex535 := position, tokenIndex + position541, tokenIndex541 := position, tokenIndex if buffer[position] != rune('0') { - goto l536 + goto l542 } position++ - goto l535 - l536: - position, tokenIndex = position535, tokenIndex535 + goto l541 + l542: + position, tokenIndex = position541, tokenIndex541 if buffer[position] != rune('1') { - goto l533 + goto l539 } position++ } - l535: + l541: if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l533 + goto l539 } position++ if buffer[position] != rune('-') { - goto l533 + goto l539 } position++ if c := buffer[position]; c < rune('0') || c > rune('3') { - goto l533 + goto l539 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l533 + goto l539 } position++ if buffer[position] != rune('T') { - goto l533 + goto l539 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l533 + goto l539 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l533 + goto l539 } position++ if buffer[position] != rune(':') { - goto l533 + goto l539 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l533 + goto l539 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l533 + goto l539 } position++ - add(ruletimebasicfmt, position534) + add(ruletimebasicfmt, position540) } return true - l533: - position, tokenIndex = position533, tokenIndex533 + l539: + position, tokenIndex = position539, tokenIndex539 return false }, /* 43 timefmt <- <(('"' '"') / ('\'' '\'') / )> */ func() bool { - position537, tokenIndex537 := position, tokenIndex + position543, tokenIndex543 := position, tokenIndex { - position538 := position + position544 := position { - position539, tokenIndex539 := position, tokenIndex + position545, tokenIndex545 := position, tokenIndex if buffer[position] != rune('"') { - goto l540 + goto l546 } position++ { - position541 := position + position547 := position if !_rules[ruletimebasicfmt]() { - goto l540 + goto l546 } - add(rulePegText, position541) + add(rulePegText, position547) } if buffer[position] != rune('"') { - goto l540 + goto l546 } position++ - goto l539 - l540: - position, tokenIndex = position539, tokenIndex539 + goto l545 + l546: + position, tokenIndex = position545, tokenIndex545 if buffer[position] != rune('\'') { - goto l542 + goto l548 } position++ { - position543 := position + position549 := position if !_rules[ruletimebasicfmt]() { - goto l542 + goto l548 } - add(rulePegText, position543) + add(rulePegText, position549) } if buffer[position] != rune('\'') { - goto l542 + goto l548 } position++ - goto l539 - l542: - position, tokenIndex = position539, tokenIndex539 + goto l545 + l548: + position, tokenIndex = position545, tokenIndex545 { - position544 := position + position550 := position if !_rules[ruletimebasicfmt]() { - goto l537 + goto l543 } - add(rulePegText, position544) + add(rulePegText, position550) } } - l539: - add(ruletimefmt, position538) + l545: + add(ruletimefmt, position544) } return true - l537: - position, tokenIndex = position537, tokenIndex537 + l543: + position, tokenIndex = position543, tokenIndex543 return false }, /* 44 time <- <( Action66)> */ diff --git a/pql/pqlpeg_test.go b/pql/pqlpeg_test.go index 93fecba9f..83fb24c27 100644 --- a/pql/pqlpeg_test.go +++ b/pql/pqlpeg_test.go @@ -353,7 +353,7 @@ func TestPQLDeepEquality(t *testing.T) { }, }}, { - name: "RowWithUnicode", + name: "RowValWithUnicode", call: `Row(unicode="Æ漢д ☮♬ ♞🜻💣")`, exp: &Call{ Name: "Row", @@ -361,6 +361,15 @@ func TestPQLDeepEquality(t *testing.T) { "unicode": `Æ漢д ☮♬ ♞🜻💣`, }, }}, + { + name: "RowWithUnicode", + call: `Row(uniΘcode="Æ漢д ☮♬ ♞🜻💣")`, + exp: &Call{ + Name: "Row", + Args: map[string]interface{}{ + "uniΘcode": `Æ漢д ☮♬ ♞🜻💣`, + }, + }}, { name: "RowsWithUnicode", call: `Rows(job, previous="💣")`, From 8724eb09b0710b4d819a3cfdc1cf1c1dd3e4bf94 Mon Sep 17 00:00:00 2001 From: Seebs Date: Tue, 21 Feb 2023 11:38:09 -0600 Subject: [PATCH 11/19] improve ramdisk config in Makefile, use it for everything in tests So, we were special-casing creating a ramdisk, and setting a special environment variable for it, for boltdb translate files, to improve performance. But actually, etcd and test cluster data and so on all go in $TMPDIR, and if you move all of those also into a ram disk, you get way better performance. But 2GB may not be enough for that. So! We unify on $TMPDIR, we bump the default size to 4GB, we make the size configurable, and we stop using the name RAMDISK. This should improve performance on MacOS significantly for `make test` and things like it, and also simplifies our lives by not having a special case for the boltdb translate files. Also change the environment variable names used in our CI config. (I don't see where we mount the ramdisks, but I think that's happening in our setup.) --- .gitlab/.gitlab-ci.yml | 4 ++-- Makefile | 16 ++++++++++++---- translate.go | 2 +- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index e1ac9fa06..57ea4d522 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -216,7 +216,7 @@ run go tests race: script: - echo "Running featurebase race tests..." - PKG_LIST=$(go list ./... | grep -Ev 'internal/clustertests|simulacraData|batch|idk|v3/dax/test/dax' | paste -s -d, -) - - RAMDISK=/mnt/ramdisk go test -race -v -timeout=10m ${PKG_LIST//,/ } + - TMPDIR=/mnt/ramdisk go test -race -v -timeout=10m ${PKG_LIST//,/ } tags: - docker @@ -232,7 +232,7 @@ run go tests: script: - echo "Running featurebase unit tests..." - PKG_LIST=$(go list ./... | grep -Ev 'internal/clustertests|simulacraData|batch|idk|v3/dax/test/dax' | paste -s -d, -) - - RAMDISK=/mnt/ramdisk go test -tags=shardwidth22 -timeout=10m -coverprofile=coverage.out -covermode=atomic -coverpkg=${PKG_LIST} ${PKG_LIST//,/ } + - TMPDIR=/mnt/ramdisk go test -tags=shardwidth22 -timeout=10m -coverprofile=coverage.out -covermode=atomic -coverpkg=${PKG_LIST} ${PKG_LIST//,/ } artifacts: paths: - coverage.out diff --git a/Makefile b/Makefile index a3ff35bea..d482b2f4e 100644 --- a/Makefile +++ b/Makefile @@ -24,6 +24,9 @@ BUILD_TAGS += TEST_TAGS = roaringparanoia TEST_TIMEOUT=10m RACE_TEST_TIMEOUT=10m +# size in GB to use for ramdisk, ?= so you can override it with env +# 4GB is not enough for `make test`, 8GB usually is. +RAMDISK_SIZE ?= 8 export GO111MODULE=on export GOPRIVATE=github.com/molecula @@ -77,13 +80,18 @@ testvsub: echo; echo "999 done testing subpkg $$pkg"; \ done -# make a 2GB RAMDisk. Speed up tests by running them with RAMDISK=/mnt/ramdisk +# make a $(RAMDISK_SIZE)GB RAMDisk. Speed up tests by running +# them with TMPDIR=/mnt/ramdisk. ramdisk-linux: - mount -o size=2G -t tmpfs none /mnt/ramdisk + mount -o size=$(RAMDISK__SIZE)G -t tmpfs none /mnt/ramdisk -# make a 2GB RAMDisk. Speed up tests by running them with RAMDISK=/Volumes/RAMDisk +# make a $(RAMDISK_SIZE)GB RAMDisk. Speed up tests by running +# them with TMPDIR=/Volumes/RAMDisk. This is more important on +# OS X than it is on Linux, because there's performance issues +# with fsync on OS X that can make the SSD slow down to moving-platters +# drive speeds. Oops. ramdisk-osx: - diskutil erasevolume HFS+ 'RAMDisk' `hdiutil attach -nobrowse -nomount ram://4194304` + diskutil erasevolume HFS+ 'RAMDisk' $$(hdiutil attach -nobrowse -nomount ram://$$(expr 2097152 \* $(RAMDISK_SIZE))) detach-ramdisk-osx: hdiutil detach /Volumes/RAMDisk diff --git a/translate.go b/translate.go index 852420abc..2573a14f4 100644 --- a/translate.go +++ b/translate.go @@ -289,7 +289,7 @@ func OpenInMemTranslateStore(rawurl, index, field string, partitionID, partition fname = fname[:10] } - tf, err := os.CreateTemp(os.Getenv("RAMDISK"), fmt.Sprintf("bolt-i%s-f%s-%d-%d-", iname, fname, partitionID, partitionN)) + tf, err := os.CreateTemp("", fmt.Sprintf("bolt-i%s-f%s-%d-%d-", iname, fname, partitionID, partitionN)) if err != nil { return nil, errors.Wrap(err, "making temp file for boltdb key translation") } From d1dbcabb3dd8948bf6103d5e343281357035dcee Mon Sep 17 00:00:00 2001 From: Seebs Date: Wed, 22 Feb 2023 13:32:05 -0600 Subject: [PATCH 12/19] fix cleanup logic for ramdisk usage we want to be sure we delete the files we created during the run even if the test run panics, but not files other runs may have created also in /mnt/ramdisk. --- .gitlab/.gitlab-ci.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 57ea4d522..c5e3f1cfb 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -216,7 +216,11 @@ run go tests race: script: - echo "Running featurebase race tests..." - PKG_LIST=$(go list ./... | grep -Ev 'internal/clustertests|simulacraData|batch|idk|v3/dax/test/dax' | paste -s -d, -) - - TMPDIR=/mnt/ramdisk go test -race -v -timeout=10m ${PKG_LIST//,/ } + - export TMPDIR=/mnt/ramdisk/test-$CI_JOB_ID + - mkdir -p $TMPDIR + - go test -race -v -timeout=10m ${PKG_LIST//,/ } + after_script: + - rm -rf /mnt/ramdisk/test-$CI_JOB_ID tags: - docker @@ -232,7 +236,11 @@ run go tests: script: - echo "Running featurebase unit tests..." - PKG_LIST=$(go list ./... | grep -Ev 'internal/clustertests|simulacraData|batch|idk|v3/dax/test/dax' | paste -s -d, -) - - TMPDIR=/mnt/ramdisk go test -tags=shardwidth22 -timeout=10m -coverprofile=coverage.out -covermode=atomic -coverpkg=${PKG_LIST} ${PKG_LIST//,/ } + - export TMPDIR=/mnt/ramdisk/test-$CI_JOB_ID + - mkdir -p $TMPDIR + - go test -tags=shardwidth22 -timeout=10m -coverprofile=coverage.out -covermode=atomic -coverpkg=${PKG_LIST} ${PKG_LIST//,/ } + after_script: + - rm -rf /mnt/ramdisk/test-$CI_JOB_ID artifacts: paths: - coverage.out From a633b72f3de4011d141b5aa211a8cc937c38a12b Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Wed, 22 Feb 2023 15:33:11 -0600 Subject: [PATCH 13/19] Separate the featurebase and fbsql make build targets (#2272) * Separate the featurebase and fbsql make build targets Using the same build target was problematic because they shared the same flags. Since the `-o` output flag was used, the fbsql binary was overwriting the featurebase binary. * make sure the make package target builds fbsql --- .gitlab/.gitlab-ci.yml | 4 ++++ Makefile | 6 +++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index c5e3f1cfb..f0bac6ce6 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -161,6 +161,10 @@ build featurebase: - GOOS="linux" GOARCH="arm64" make build FLAGS="-o featurebase_linux_arm64" - GOOS="darwin" GOARCH="amd64" make build FLAGS="-o featurebase_darwin_amd64" - GOOS="darwin" GOARCH="arm64" make build FLAGS="-o featurebase_darwin_arm64" + - GOOS="linux" GOARCH="amd64" make build-fbsql FLAGS="-o fbsql_linux_amd64" + - GOOS="linux" GOARCH="arm64" make build-fbsql FLAGS="-o fbsql_linux_arm64" + - GOOS="darwin" GOARCH="amd64" make build-fbsql FLAGS="-o fbsql_darwin_amd64" + - GOOS="darwin" GOARCH="arm64" make build-fbsql FLAGS="-o fbsql_darwin_arm64" artifacts: paths: - featurebase_linux_amd64 diff --git a/Makefile b/Makefile index d482b2f4e..23e9d0ba9 100644 --- a/Makefile +++ b/Makefile @@ -116,14 +116,18 @@ cover: cover-viz: cover $(GO) tool cover -html=build/coverage.out -# Compile Pilosa +# Build featurebase build: $(GO) build -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/featurebase + +# Build fbsql +build-fbsql: $(GO) build -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/fbsql package: GOOS=$(GOOS) GOARCH=$(GOARCH) $(MAKE) build + GOOS=$(GOOS) GOARCH=$(GOARCH) $(MAKE) build-fbsql GOARCH=$(GOARCH) VERSION=$(VERSION) nfpm package --packager deb --target featurebase.$(VERSION).$(GOARCH).deb GOARCH=$(GOARCH) VERSION=$(VERSION) nfpm package --packager rpm --target featurebase.$(VERSION).$(GOARCH).rpm From 528ebc93dbcacd35093b70f3764333bbb88e2258 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Wed, 22 Feb 2023 15:52:58 -0600 Subject: [PATCH 14/19] CLI variables (#2263) * Add meta-commands \set and \unset (for variables) * WIP: first pass at variable replacement * Use a mapReplacer instead of having Command implement replacer * remove circular reference with variables * remove the `replacer` interface; just have it be a struct * use a lexer for variable replacement --- cli/cli.go | 9 +++- cli/meta.go | 70 +++++++++++++++++++++++++-- cli/replacer.go | 109 +++++++++++++++++++++++++++++++++++++++++++ cli/replacer_test.go | 109 +++++++++++++++++++++++++++++++++++++++++++ cli/splitter.go | 19 ++++++-- cli/splitter_test.go | 2 +- go.mod | 1 + go.sum | 2 + 8 files changed, 311 insertions(+), 10 deletions(-) create mode 100644 cli/replacer.go create mode 100644 cli/replacer_test.go diff --git a/cli/cli.go b/cli/cli.go index 7a3ee1606..cc8f2e0a3 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -79,6 +79,9 @@ type Command struct { // i.e. it will quit after the command is complete. Files []string `json:"files"` + // variables holds the variables created with the \set meta-command. + variables map[string]string + // nonInteractiveMode is set to true when fbsql is running in // non-ineracative mode. And example of this is when the user has provided a // `-c` flag in the command line. @@ -89,6 +92,8 @@ type Command struct { } func NewCommand(logdest logger.Logger) *Command { + variables := make(map[string]string) + return &Command{ Config: &Config{ Host: defaultHost, @@ -107,8 +112,8 @@ func NewCommand(logdest logger.Logger) *Command { HistoryPath: "", }, - splitter: newSplitter(), buffer: newBuffer(), + splitter: newSplitter(newReplacer(variables)), workingDir: newWorkingDir(), Stdin: Stdin, @@ -118,6 +123,8 @@ func NewCommand(logdest logger.Logger) *Command { output: Stdout, writeOptions: defaultWriteOptions(), + variables: variables, + quit: make(chan struct{}), } } diff --git a/cli/meta.go b/cli/meta.go index b249691fd..142f9b78a 100644 --- a/cli/meta.go +++ b/cli/meta.go @@ -7,6 +7,7 @@ import ( "os/exec" "os/signal" "path/filepath" + "sort" "strconv" "strings" "syscall" @@ -54,6 +55,7 @@ var _ metaCommand = (*metaReset)(nil) var _ metaCommand = (*metaSet)(nil) var _ metaCommand = (*metaTiming)(nil) var _ metaCommand = (*metaTuplesOnly)(nil) +var _ metaCommand = (*metaUnset)(nil) var _ metaCommand = (*metaWarn)(nil) var _ metaCommand = (*metaWatch)(nil) var _ metaCommand = (*metaWrite)(nil) @@ -333,6 +335,10 @@ Operating System \cd [DIR] change the current working directory \timing [on|off] toggle timing of commands \! [COMMAND] execute command in shell or start interactive shell + +Variables + \set [NAME [VALUE]] set internal variable, or list all if no parameters + \unset NAME unset (delete) internal variable ` cmd.Printf("%s\n", helpText) @@ -367,7 +373,7 @@ func executeFile(cmd *Command, fileName string) (action, error) { } defer file.Close() - splitter := newSplitter() + splitter := newSplitter(newReplacer(cmd.variables)) buffer := newBuffer() // Read the file by line, pushing the lines into a new line splitter, then @@ -647,7 +653,29 @@ func newMetaSet(args []string) *metaSet { } func (m *metaSet) execute(cmd *Command) (action, error) { - // TODO: set the variable (or clear it, etc) + switch len(m.args) { + case 0: + // Sort the variables before printing them. + keys := make([]string, 0, len(cmd.variables)) + for k := range cmd.variables { + keys = append(keys, k) + } + sort.Strings(keys) + + // Print out the variables. + for _, k := range keys { + cmd.Printf("%s = '%s'\n", k, cmd.variables[k]) + } + cmd.writeOptions.timing = !cmd.writeOptions.timing + default: + // The first arg is the key, the remaining args are concatenated together to form the values + // For example: + // \set one two three + // will result in `one = 'twothree'` + k := m.args[0] + v := strings.Join(m.args[1:], "") + cmd.variables[k] = v + } return actionNone, nil } @@ -729,6 +757,35 @@ func (m *metaTuplesOnly) execute(cmd *Command) (action, error) { return actionNone, nil } +// //////////////////////////////////////////////////////////////////////////// +// unset +// //////////////////////////////////////////////////////////////////////////// +type metaUnset struct { + args []string +} + +func newMetaUnset(args []string) *metaUnset { + return &metaUnset{ + args: args, + } +} + +func (m *metaUnset) execute(cmd *Command) (action, error) { + switch len(m.args) { + case 0: + cmd.Printf("\\unset: missing required argument\n") + return actionNone, nil + default: + if len(m.args) > 1 { + for _, s := range m.args[1:] { + cmd.Printf("\\unset: extra argument \"%s\" ignored\n", s) + } + } + delete(cmd.variables, m.args[0]) + } + return actionNone, nil +} + // //////////////////////////////////////////////////////////////////////////// // warn // //////////////////////////////////////////////////////////////////////////// @@ -854,7 +911,7 @@ func (m *metaWrite) execute(cmd *Command) (action, error) { // `cmd 'arg1' arg2 'arg three'` // // It returns the metaCommand which maps to `cmd`. -func splitMetaCommand(in string) (metaCommand, error) { +func splitMetaCommand(in string, replacer *replacer) (metaCommand, error) { parts := strings.SplitN(in, ` `, 2) key := strings.TrimRightFunc(parts[0], unicode.IsSpace) @@ -877,6 +934,11 @@ func splitMetaCommand(in string) (metaCommand, error) { } } + // Do variable replacement. + for i := range args { + args[i] = replacer.replace(args[i]) + } + switch key { case "!": return newMetaBang(args), nil @@ -916,6 +978,8 @@ func splitMetaCommand(in string) (metaCommand, error) { return newMetaTuplesOnly(args), nil case "timing": return newMetaTiming(args), nil + case "unset": + return newMetaUnset(args), nil case "warn": return newMetaWarn(args), nil case "watch": diff --git a/cli/replacer.go b/cli/replacer.go new file mode 100644 index 000000000..b22330a8a --- /dev/null +++ b/cli/replacer.go @@ -0,0 +1,109 @@ +package cli + +import ( + "strings" + + "github.com/benhoyt/goawk/lexer" +) + +// replacer can replace parts of a string based on some rules and the provided +// map[string]string. For example, the Command can replace strings with values +// in its `variables` map. +type replacer struct { + m map[string]string +} + +func newReplacer(m map[string]string) *replacer { + return &replacer{ + m: m, + } +} + +// replace replaces all instances of the string pattern `:key` with the value at +// m[key]. For example we want something like this: +// +// GIVEN: `start :one,:'two', :"three" ::four ::` +// +// with map +// +// map[string]string{ +// "one": "repl1", +// "three": "repl3", +// } +// +// WANT: `start repl1,:'two', "repl3" ::four ::` +func (r *replacer) replace(s string) string { + // If no variables have been added to the map, there's no need to parse the + // string for variable replacement. + if len(r.m) == 0 { + return s + } + + line := []byte(s) + lex := lexer.NewLexer(line) + + // finger contains the index into line at the start of non-variable text + // that we want to include, as-is in the output. + var finger int + + // sb builds the string which will be the final output. + var sb strings.Builder + for { + pos, tok, _ := lex.Scan() + + switch tok { + case lexer.COLON: + // last is the last normal character position before the colon. + last := pos.Column - 1 + + // Get the next byte to see if the colon value is quoted, and if so, + // whether its has single or double quotes. + b := lex.PeekByte() + + // padding is the amount of padding we have to consider around the + // variable name. If the variable is not quoted, it doesn't require + // any padding. But if it has quotes, it needs 2 characters of + // paddings to accomodate the quotes. + padding := 0 + + // quote holds the character to use to quote the final, replaced + // output value. Because the lexer doesn't tell us how a certain + // `string` token was quoted, we need to keep track of that here so + // we can put them back. + quote := "" + switch b { + case byte('\''): // single quote + quote = `'` + padding = 2 + case byte('"'): // double quote + quote = `"` + padding = 2 + } + + pos, tok, key := lex.Scan() + switch tok { + case lexer.NAME, lexer.STRING: + // Write the normal text up to the variable replacement + // position. + sb.Write(line[finger:last]) + + if v, ok := r.m[key]; ok { + // Write replaced variable with the quotes it had. + sb.WriteString(quote + v + quote) + } else { + // Since the variable was not found in the map, just write + // back what was already there. + sb.WriteString(":" + quote + key + quote) + } + + // Reset finger to point to the next position after the + // variable. + finger = pos.Column + len(key) + padding - 1 + } + case lexer.EOF: + // Write the remainder of the string and return. + sb.Write(line[finger:]) + return sb.String() + } + } +} diff --git a/cli/replacer_test.go b/cli/replacer_test.go new file mode 100644 index 000000000..4cf1069ce --- /dev/null +++ b/cli/replacer_test.go @@ -0,0 +1,109 @@ +package cli + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestReplacer(t *testing.T) { + t.Run("general replace function", func(t *testing.T) { + + m := map[string]string{ + "v1": "newVone", + "v2": "newVtwo", + } + + tests := []struct { + s string + m map[string]string + exp string + }{ + { + // no variables present + s: "foo", + m: m, + exp: "foo", + }, + { + // variable prefix, but not in map + s: ":foo", + m: m, + exp: ":foo", + }, + { + // variable name match, but missing prefix + s: "v1", + m: m, + exp: "v1", + }, + { + // variable name match + s: ":v1", + m: m, + exp: "newVone", + }, + { + // two variables, the same, no space + s: ":v1:v1", + m: m, + exp: "newVonenewVone", + }, + { + // two variables, different, no space + s: ":v1:v2", + m: m, + exp: "newVonenewVtwo", + }, + { + // two variables, different, spaces + s: ":v1 :v2", + m: m, + exp: "newVone newVtwo", + }, + { + // one variable, one non-variable, no space + s: ":v1:foo", + m: m, + exp: "newVone:foo", + }, + { + // one non-variable, one variable, no space + s: "foo:v1", + m: m, + exp: "foonewVone", + }, + { + // two variables, different, comma + s: ":v1, :v2", + m: m, + exp: "newVone, newVtwo", + }, + { + // single quotes + s: ":'v1'", + m: m, + exp: "'newVone'", + }, + { + // double quotes + s: `:"v2"`, + m: m, + exp: `"newVtwo"`, + }, + { + // more quotes + s: `start :v1,:'two', :"v2" ::four :: `, + m: m, + exp: `start newVone,:'two', "newVtwo" ::four :: `, + }, + } + for i, test := range tests { + t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) { + replacer := newReplacer(test.m) + assert.Equal(t, test.exp, replacer.replace(test.s)) + }) + } + }) +} diff --git a/cli/splitter.go b/cli/splitter.go index ca62eb257..bb3c5f2d1 100644 --- a/cli/splitter.go +++ b/cli/splitter.go @@ -10,10 +10,14 @@ import ( // metaCommands. It may not be necessary to have this be a separate struct since // it contains no members and just has the one `split()` method, but here we // are. -type splitter struct{} +type splitter struct { + replacer *replacer +} -func newSplitter() *splitter { - return &splitter{} +func newSplitter(r *replacer) *splitter { + return &splitter{ + replacer: r, + } } // split splits the given line into queryParts and metaCommands. @@ -65,6 +69,11 @@ func (s *splitter) splitQueryParts(line string) ([]queryPart, error) { // Look for a termination character; parts := strings.Split(line, terminationChar) + // Do variable replacement. + for i := range parts { + parts[i] = s.replacer.replace(parts[i]) + } + if len(parts) == 1 { part0 := strings.TrimSpace(parts[0]) return []queryPart{ @@ -95,7 +104,7 @@ func (s *splitter) splitQueryParts(line string) ([]queryPart, error) { func (s *splitter) splitMetaCommands(in string) ([]metaCommand, error) { parts := strings.Split(in, `\`) if len(parts) == 1 { - mc, err := splitMetaCommand(parts[0]) + mc, err := splitMetaCommand(parts[0], s.replacer) if err != nil { return nil, errors.Wrapf(err, "splitting meta command: %s", parts[0]) } @@ -108,7 +117,7 @@ func (s *splitter) splitMetaCommands(in string) ([]metaCommand, error) { if part == "" { continue } - mc, err := splitMetaCommand(part) + mc, err := splitMetaCommand(part, s.replacer) if err != nil { return nil, errors.Wrapf(err, "splitting meta command: %s", part) } diff --git a/cli/splitter_test.go b/cli/splitter_test.go index 54b0fb3eb..ae22251f5 100644 --- a/cli/splitter_test.go +++ b/cli/splitter_test.go @@ -8,7 +8,7 @@ import ( ) func TestSplitter(t *testing.T) { - s := newSplitter() + s := newSplitter(newReplacer(nil)) t.Run("Split", func(t *testing.T) { tests := []struct { line string diff --git a/go.mod b/go.mod index f92aa51ab..d4bd82bed 100644 --- a/go.mod +++ b/go.mod @@ -83,6 +83,7 @@ require ( github.com/PaesslerAG/gval v1.0.0 github.com/PaesslerAG/jsonpath v0.1.1 github.com/apache/arrow/go/v10 v10.0.0-20221021053532-2f627c213fc3 + github.com/benhoyt/goawk v1.21.0 github.com/gomem/gomem v0.1.0 github.com/google/uuid v1.3.0 github.com/jaffee/commandeer v0.6.0 diff --git a/go.sum b/go.sum index fdbd41d61..ba7e26a9f 100644 --- a/go.sum +++ b/go.sum @@ -139,6 +139,8 @@ github.com/beevik/ntp v0.3.0 h1:xzVrPrE4ziasFXgBVBZJDP0Wg/KpMwk2KHJ4Ba8GrDw= github.com/beevik/ntp v0.3.0/go.mod h1:hIHWr+l3+/clUnF44zdK+CWW7fO8dR5cIylAQ76NRpg= github.com/benbjohnson/immutable v0.4.0 h1:CTqXbEerYso8YzVPxmWxh2gnoRQbbB9X1quUC8+vGZA= github.com/benbjohnson/immutable v0.4.0/go.mod h1:iAr8OjJGLnLmVUr9MZ/rz4PWUy6Ouc2JLYuMArmvAJM= +github.com/benhoyt/goawk v1.21.0 h1:GASuhJXHMFZ/2TJBPh+2Ah3kclVGNvGjt+uh3ajMdLk= +github.com/benhoyt/goawk v1.21.0/go.mod h1:UG1Ld6CjkkHhoyQmErQGSTwmavsTqFnCDYsLSJbovqU= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= From c294bc70dc5f2622c622bac60ef515bd956c4982 Mon Sep 17 00:00:00 2001 From: tgruben Date: Thu, 23 Feb 2023 18:35:52 -0600 Subject: [PATCH 15/19] limit memory for backuptar/restoretar (#2270) --- buffer/filebuffer.go | 110 +++++++++++++++++++++++++++++++ cmd/backup_tar.go | 1 + cmd/restore_tar.go | 2 + ctl/backup_tar.go | 150 ++++++++++++++----------------------------- ctl/restore_tar.go | 89 +++++++++++++++++-------- 5 files changed, 222 insertions(+), 130 deletions(-) create mode 100644 buffer/filebuffer.go diff --git a/buffer/filebuffer.go b/buffer/filebuffer.go new file mode 100644 index 000000000..3f0647465 --- /dev/null +++ b/buffer/filebuffer.go @@ -0,0 +1,110 @@ +package buffer + +import ( + "bytes" + "io" + "io/ioutil" + "os" + "sync" +) + +// NewFileBuffer returns a file buffer which will use an in-memory buffer, until `max` bytes have been written, at which point it will write the contents of memory to a file, and continue writing future data to the file. +// The file will be written to `temp` directory. The buffer fulfills the io.Reader and io.Writer interface +func NewFileBuffer(max int, temp string) *FileBuffer { + return &FileBuffer{max: max, tempDir: temp} +} + +type FileBuffer struct { + max int + buf bytes.Buffer + file *os.File + tempDir string + reading bool + files []*os.File + mu sync.Mutex +} + +func (fb *FileBuffer) Write(p []byte) (n int, err error) { + if fb.reading { + panic("cannot write after read") + } + if fb.file != nil { + return fb.file.Write(p) + } + n, err = fb.buf.Write(p) + if err != nil { + return + } + if fb.buf.Len() > fb.max { + fb.file, err = ioutil.TempFile(fb.tempDir, "filebuffer-") + if err != nil { + return + } + _, err = io.Copy(fb.file, &fb.buf) + fb.buf.Reset() + } + return +} + +func (fb *FileBuffer) Len() (int64, error) { + if fb.file == nil { + return int64(fb.buf.Len()), nil + } + fi, err := fb.file.Stat() + if err != nil { + return 0, err + } + + return fi.Size(), nil +} + +func (fb *FileBuffer) Read(p []byte) (n int, err error) { + if fb.file != nil { + if !fb.reading { + fb.reading = true + _, err = fb.file.Seek(0, 0) + if err != nil { + return + } + } + return fb.file.Read(p) + } + fb.reading = true + return fb.buf.Read(p) +} + +func (fb *FileBuffer) Close() error { + if fb.file != nil { + name := fb.file.Name() + if err := fb.file.Close(); err != nil { + return err + } + for _, f := range fb.files { + f.Close() + } + fb.files = fb.files[:0] + fb.file = nil + return os.Remove(name) + } + return nil +} + +func (fb *FileBuffer) Reset() error { + fb.mu.Lock() + defer fb.mu.Unlock() + fb.reading = false + fb.buf.Reset() + return fb.Close() +} + +func (fb *FileBuffer) NewReader() (io.Reader, error) { + fb.mu.Lock() + defer fb.mu.Unlock() + fb.reading = true + if fb.file == nil { + return bytes.NewReader(fb.buf.Bytes()), nil + } + f, err := os.OpenFile(fb.file.Name(), os.O_RDONLY, 0) + fb.files = append(fb.files, f) + return f, err +} diff --git a/cmd/backup_tar.go b/cmd/backup_tar.go index 6c9612d77..641f4ee51 100644 --- a/cmd/backup_tar.go +++ b/cmd/backup_tar.go @@ -28,6 +28,7 @@ Backs up a FeatureBase server to a local, tar-formatted snapshot file. ctl.SetTLSConfig(flags, "", &cmd.TLS.CertificatePath, &cmd.TLS.CertificateKeyPath, &cmd.TLS.CACertPath, &cmd.TLS.SkipVerify, &cmd.TLS.EnableClientVerification) flags.StringVar(&cmd.AuthToken, "auth-token", "", "Authentication token") flags.StringVar(&cmd.HeaderTimeoutStr, "header-timeout", cmd.HeaderTimeoutStr, "Length of time to wait for initial HTTP response before giving up.") + flags.StringVar(&cmd.TempDir, "temp-dir", cmd.TempDir, "Location of temporary spillover files. The default is the system's default (usually /tmp)") return ccmd } diff --git a/cmd/restore_tar.go b/cmd/restore_tar.go index d626a488c..9a1cdc3fc 100644 --- a/cmd/restore_tar.go +++ b/cmd/restore_tar.go @@ -23,6 +23,8 @@ The Restore command will take a tar-formatted backup archive and restore it to a flags.DurationVar(&cmd.RetryPeriod, "retry-period", cmd.RetryPeriod, "Length of time after HTTP request failure to continue retrying request.") flags.StringVar(&cmd.Pprof, "pprof", cmd.Pprof, "host:port to listen for profiling requests at /debug/pprof and /debug/fgprof.") flags.StringVar(&cmd.AuthToken, "auth-token", "", "Authentication token") + flags.StringVar(&cmd.TempDir, "temp-dir", cmd.TempDir, "Location of temporary spillover files. The default is the system's default (usually /tmp)") + ctl.SetTLSConfig( flags, "", &cmd.TLS.CertificatePath, diff --git a/ctl/backup_tar.go b/ctl/backup_tar.go index 0b0780b0d..e8f65d90d 100644 --- a/ctl/backup_tar.go +++ b/ctl/backup_tar.go @@ -3,7 +3,6 @@ package ctl import ( "archive/tar" - "bytes" "context" "crypto/tls" "encoding/json" @@ -17,6 +16,7 @@ import ( pilosa "github.com/featurebasedb/featurebase/v3" "github.com/featurebasedb/featurebase/v3/authn" + "github.com/featurebasedb/featurebase/v3/buffer" "github.com/featurebasedb/featurebase/v3/disco" "github.com/featurebasedb/featurebase/v3/encoding/proto" "github.com/featurebasedb/featurebase/v3/logger" @@ -37,6 +37,9 @@ type BackupTarCommand struct { // nolint: maligned // Path to write the backup to. OutputPath string + // TempDir location of scratch files + TempDir string + // Amount of time after first failed request to continue retrying. RetryPeriod time.Duration `json:"retry-period"` @@ -164,17 +167,16 @@ func (cmd *BackupTarCommand) Run(ctx context.Context) (err error) { tw := tar.NewWriter(w) defer tw.Close() - buf := new(bytes.Buffer) // Backup schema. if err := cmd.backupTarSchema(ctx, tw, schema); err != nil { return fmt.Errorf("cannot back up schema: %w", err) - } else if err := cmd.backupTarIDAllocData(ctx, tw, buf); err != nil { + } else if err := cmd.backupTarIDAllocData(ctx, tw); err != nil { return fmt.Errorf("cannot back up id alloc data: %w", err) } // Backup data for each index. for _, ii := range schema.Indexes { - if err := cmd.backupTarIndex(ctx, tw, ii, buf); err != nil { + if err := cmd.backupTarIndex(ctx, tw, ii); err != nil { return err } } @@ -220,8 +222,7 @@ func (cmd *BackupTarCommand) backupTarSchema(ctx context.Context, tw *tar.Writer return nil } -func (cmd *BackupTarCommand) backupTarIDAllocData(ctx context.Context, tw *tar.Writer, buf *bytes.Buffer) error { - buf.Reset() +func (cmd *BackupTarCommand) backupTarIDAllocData(ctx context.Context, tw *tar.Writer) error { logger := cmd.Logger() logger.Printf("backing up id alloc data") @@ -231,28 +232,11 @@ func (cmd *BackupTarCommand) backupTarIDAllocData(ctx context.Context, tw *tar.W } defer rc.Close() - // Read to buffer to determine size. - if _, err := buf.ReadFrom(rc); err != nil { - return fmt.Errorf("copying id alloc data to memory: %w", err) - } - - // Build header & copy data to archive. - if err = tw.WriteHeader(&tar.Header{ - Name: "idalloc", - Mode: 0o666, - Size: int64(buf.Len()), - ModTime: time.Now(), - }); err != nil { - return err - } else if _, err := io.Copy(tw, buf); err != nil { - return fmt.Errorf("copying id alloc data to archive: %w", err) - } - - return nil + return writeToTar(tw, "idalloc", rc, cmd.TempDir) } // backupTarIndex backs up all shards for a given index. -func (cmd *BackupTarCommand) backupTarIndex(ctx context.Context, tw *tar.Writer, ii *pilosa.IndexInfo, buf *bytes.Buffer) error { +func (cmd *BackupTarCommand) backupTarIndex(ctx context.Context, tw *tar.Writer, ii *pilosa.IndexInfo) error { logger := cmd.Logger() logger.Printf("backing up index: %q", ii.Name) @@ -263,14 +247,14 @@ func (cmd *BackupTarCommand) backupTarIndex(ctx context.Context, tw *tar.Writer, // Back up all bitmap data for the index. for _, shard := range shards { - if err := cmd.backupTarShard(ctx, tw, ii.Name, shard, buf); err != nil { + if err := cmd.backupTarShard(ctx, tw, ii.Name, shard); err != nil { return fmt.Errorf("cannot backup shard %d on index %q: %w", shard, ii.Name, err) } } if ii.Options.Keys { // Back up translation data after bitmap data so we ensurean translate all data. - if err := cmd.backupTarIndexTranslateData(ctx, tw, ii.Name, buf); err != nil { + if err := cmd.backupTarIndexTranslateData(ctx, tw, ii.Name); err != nil { return err } } @@ -280,7 +264,7 @@ func (cmd *BackupTarCommand) backupTarIndex(ctx context.Context, tw *tar.Writer, if !fi.Options.Keys { continue } - if err := cmd.backupTarFieldTranslateData(ctx, tw, ii.Name, fi.Name, buf); err != nil { + if err := cmd.backupTarFieldTranslateData(ctx, tw, ii.Name, fi.Name); err != nil { return fmt.Errorf("cannot backup field translation data for field %q on index %q: %w", fi.Name, ii.Name, err) } } @@ -289,7 +273,7 @@ func (cmd *BackupTarCommand) backupTarIndex(ctx context.Context, tw *tar.Writer, } // backupTarShard backs up a single shard from a single index. -func (cmd *BackupTarCommand) backupTarShard(ctx context.Context, tw *tar.Writer, indexName string, shard uint64, buf *bytes.Buffer) (err error) { +func (cmd *BackupTarCommand) backupTarShard(ctx context.Context, tw *tar.Writer, indexName string, shard uint64) (err error) { nodes, err := cmd.client.FragmentNodes(ctx, indexName, shard) if err != nil { return fmt.Errorf("cannot determine fragment nodes: %w", err) @@ -298,7 +282,7 @@ func (cmd *BackupTarCommand) backupTarShard(ctx context.Context, tw *tar.Writer, } for _, node := range nodes { - if e := cmd.backupTarShardNode(ctx, tw, indexName, shard, node, buf); e == nil { + if e := cmd.backupTarShardNode(ctx, tw, indexName, shard, node); e == nil { break } else if err == nil { err = e // save first error, try next node @@ -306,7 +290,7 @@ func (cmd *BackupTarCommand) backupTarShard(ctx context.Context, tw *tar.Writer, } for _, node := range nodes { - if e := cmd.backupTarShardDataframe(ctx, tw, indexName, shard, node, buf); e == nil { + if e := cmd.backupTarShardDataframe(ctx, tw, indexName, shard, node); e == nil { break } else if err == nil { err = e // save first error, try next node @@ -316,8 +300,7 @@ func (cmd *BackupTarCommand) backupTarShard(ctx context.Context, tw *tar.Writer, } // backupTarShardNode backs up a single shard from a single index on a specific node. -func (cmd *BackupTarCommand) backupTarShardNode(ctx context.Context, tw *tar.Writer, indexName string, shard uint64, node *disco.Node, buf *bytes.Buffer) error { - buf.Reset() +func (cmd *BackupTarCommand) backupTarShardNode(ctx context.Context, tw *tar.Writer, indexName string, shard uint64, node *disco.Node) error { logger := cmd.Logger() logger.Printf("backing up shard: index=%q id=%d", indexName, shard) @@ -332,30 +315,10 @@ func (cmd *BackupTarCommand) backupTarShardNode(ctx context.Context, tw *tar.Wri return fmt.Errorf("fetching shard reader: %w", err) } defer rc.Close() - - // Read to buffer to determine size. - // TODO: Provide size via the reader itself. - if _, err := buf.ReadFrom(rc); err != nil { - return fmt.Errorf("copying shard data to memory: %w", err) - } - - // Build header & copy data to archive. - if err = tw.WriteHeader(&tar.Header{ - Name: filename, - Mode: 0o666, - Size: int64(buf.Len()), - ModTime: time.Now(), - }); err != nil { - return err - } else if _, err := io.Copy(tw, buf); err != nil { - return fmt.Errorf("copying shard data to archive: %w", err) - } - - return nil + return writeToTar(tw, filename, rc, cmd.TempDir) } -func (cmd *BackupTarCommand) backupTarShardDataframe(ctx context.Context, tw *tar.Writer, indexName string, shard uint64, node *disco.Node, buf *bytes.Buffer) error { - buf.Reset() +func (cmd *BackupTarCommand) backupTarShardDataframe(ctx context.Context, tw *tar.Writer, indexName string, shard uint64, node *disco.Node) error { logger := cmd.Logger() logger.Printf("backing up dataframe shard: index=%q shard=%d", indexName, shard) @@ -375,38 +338,21 @@ func (cmd *BackupTarCommand) backupTarShardDataframe(ctx context.Context, tw *ta } filename := filepath.Join("indexes", indexName, "dataframe", fmt.Sprintf("%04d", shard)) - logger.Printf("writing %v", filename) - if _, err := buf.ReadFrom(resp.Body); err != nil { - return fmt.Errorf("copying shard data to memory: %w", err) - } - - // Build header & copy data to archive. - if err = tw.WriteHeader(&tar.Header{ - Name: filename, - Mode: 0o666, - Size: int64(buf.Len()), - ModTime: time.Now(), - }); err != nil { - return err - } else if _, err := io.Copy(tw, buf); err != nil { - return fmt.Errorf("copying shard data to archive: %w", err) - } - return nil + return writeToTar(tw, filename, resp.Body, cmd.TempDir) } -func (cmd *BackupTarCommand) backupTarIndexTranslateData(ctx context.Context, tw *tar.Writer, name string, buf *bytes.Buffer) error { +func (cmd *BackupTarCommand) backupTarIndexTranslateData(ctx context.Context, tw *tar.Writer, name string) error { // TODO: Fetch holder partition count. partitionN := disco.DefaultPartitionN for partitionID := 0; partitionID < partitionN; partitionID++ { - if err := cmd.backupTarIndexPartitionTranslateData(ctx, tw, name, partitionID, buf); err != nil { + if err := cmd.backupTarIndexPartitionTranslateData(ctx, tw, name, partitionID); err != nil { return fmt.Errorf("cannot backup index translation data for partition %d on %q: %w", partitionID, name, err) } } return nil } -func (cmd *BackupTarCommand) backupTarIndexPartitionTranslateData(ctx context.Context, tw *tar.Writer, name string, partitionID int, buf *bytes.Buffer) error { - buf.Reset() +func (cmd *BackupTarCommand) backupTarIndexPartitionTranslateData(ctx context.Context, tw *tar.Writer, name string, partitionID int) error { logger := cmd.Logger() logger.Printf("backing up index translation data: %s/%d", name, partitionID) @@ -418,28 +364,10 @@ func (cmd *BackupTarCommand) backupTarIndexPartitionTranslateData(ctx context.Co } defer rc.Close() - // Read to buffer to determine size. - if _, err := buf.ReadFrom(rc); err != nil { - return fmt.Errorf("copying translate data to memory: %w", err) - } - - // Build header & copy data to archive. - if err = tw.WriteHeader(&tar.Header{ - Name: path.Join("indexes", name, "translate", fmt.Sprintf("%04d", partitionID)), - Mode: 0o666, - Size: int64(buf.Len()), - ModTime: time.Now(), - }); err != nil { - return err - } else if _, err := io.Copy(tw, buf); err != nil { - return fmt.Errorf("copying translate data to archive: %w", err) - } - - return nil + return writeToTar(tw, path.Join("indexes", name, "translate", fmt.Sprintf("%04d", partitionID)), rc, cmd.TempDir) } -func (cmd *BackupTarCommand) backupTarFieldTranslateData(ctx context.Context, tw *tar.Writer, indexName, fieldName string, buf *bytes.Buffer) error { - buf.Reset() +func (cmd *BackupTarCommand) backupTarFieldTranslateData(ctx context.Context, tw *tar.Writer, indexName, fieldName string) error { logger := cmd.Logger() logger.Printf("backing up field translation data: %s/%s", indexName, fieldName) @@ -450,17 +378,37 @@ func (cmd *BackupTarCommand) backupTarFieldTranslateData(ctx context.Context, tw return fmt.Errorf("fetching translate data reader: %w", err) } defer rc.Close() + return writeToTar(tw, path.Join("indexes", indexName, "fields", fieldName, "translate"), rc, cmd.TempDir) +} +func (cmd *BackupTarCommand) TLSHost() string { return cmd.Host } + +func (cmd *BackupTarCommand) TLSConfiguration() server.TLSConfig { return cmd.TLS } + +func writeToTar(tw *tar.Writer, entryName string, rc io.Reader, tmpDir string) error { + spillFile, err := os.CreateTemp("", "spill") + if err != nil { + return fmt.Errorf("creating temp file : %w", err) + } + defer func() { + spillFile.Close() + os.Remove(spillFile.Name()) + }() + mb512 := 2 << 29 + buf := buffer.NewFileBuffer(mb512, tmpDir) + defer buf.Close() + + n, err := io.Copy(buf, rc) // Read to buffer to determine size. - if _, err := buf.ReadFrom(rc); err != nil { + if err != nil { return fmt.Errorf("copying translate data to memory: %w", err) } // Build header & copy data to archive. if err = tw.WriteHeader(&tar.Header{ - Name: path.Join("indexes", indexName, "fields", fieldName, "translate"), + Name: entryName, Mode: 0o666, - Size: int64(buf.Len()), + Size: n, ModTime: time.Now(), }); err != nil { return err @@ -469,7 +417,3 @@ func (cmd *BackupTarCommand) backupTarFieldTranslateData(ctx context.Context, tw } return nil } - -func (cmd *BackupTarCommand) TLSHost() string { return cmd.Host } - -func (cmd *BackupTarCommand) TLSConfiguration() server.TLSConfig { return cmd.TLS } diff --git a/ctl/restore_tar.go b/ctl/restore_tar.go index 80bf9c5e7..be1052f99 100644 --- a/ctl/restore_tar.go +++ b/ctl/restore_tar.go @@ -3,12 +3,13 @@ package ctl import ( "archive/tar" - "bytes" "compress/gzip" "context" "crypto/tls" "fmt" "io" + "io/ioutil" + "net/http" gohttp "net/http" "os" "strconv" @@ -17,6 +18,7 @@ import ( pilosa "github.com/featurebasedb/featurebase/v3" "github.com/featurebasedb/featurebase/v3/authn" + "github.com/featurebasedb/featurebase/v3/buffer" "github.com/featurebasedb/featurebase/v3/disco" "github.com/featurebasedb/featurebase/v3/logger" "github.com/featurebasedb/featurebase/v3/server" @@ -48,6 +50,9 @@ type RestoreTarCommand struct { TLS server.TLSConfig AuthToken string + + // TempDir location of scratch files + TempDir string } // Logger returns the command's associated Logger to maintain CommandWithTLSSupport interface compatibility @@ -134,7 +139,10 @@ func (cmd *RestoreTarCommand) Run(ctx context.Context) (err error) { return errors.New("no primary") } c := &gohttp.Client{} - buf := new(bytes.Buffer) + // buf := new(bytes.Buffer) + mb512 := 2 << 29 + buf := buffer.NewFileBuffer(mb512, cmd.TempDir) + defer buf.Reset() for { buf.Reset() header, err := tarReader.Next() @@ -177,22 +185,21 @@ func (cmd *RestoreTarCommand) Run(ctx context.Context) (err error) { } else if len(fragmentNodes) == 0 { return fmt.Errorf("no fragmentNodes available") } - _, err = io.Copy(buf, tarReader) if err != nil { return errors.Wrap(err, "copying") } - g, _ := errgroup.WithContext(ctx) for _, node := range fragmentNodes { node := node + rd, err := buf.NewReader() + if err != nil { + return err + } g.Go(func() error { - client := &gohttp.Client{} - rd := bytes.NewReader(buf.Bytes()) logger.Printf("shard %v %v", shard, indexName) url := node.URI.Path(fmt.Sprintf("/internal/restore/%v/%v", indexName, shard)) - _, err = client.Post(url, "application/octet-stream", rd) - return err + return Post(ctx, url, "application/octet-stream", rd, nil) }) } if err := g.Wait(); err != nil { @@ -218,13 +225,14 @@ func (cmd *RestoreTarCommand) Run(ctx context.Context) (err error) { g, _ := errgroup.WithContext(ctx) for _, node := range fragmentNodes { node := node + rd, err := buf.NewReader() + if err != nil { + return err + } g.Go(func() error { - client := &gohttp.Client{} - rd := bytes.NewReader(buf.Bytes()) logger.Printf("dataframe shard %v %v", shard, indexName) url := node.URI.Path(fmt.Sprintf("/internal/dataframe/restore/%v/%v", indexName, shard)) - _, err = client.Post(url, "application/octet-stream", rd) - return err + return Post(ctx, url, "application/octet-stream", rd, nil) }) } if err := g.Wait(); err != nil { @@ -249,13 +257,13 @@ func (cmd *RestoreTarCommand) Run(ctx context.Context) (err error) { g, _ := errgroup.WithContext(ctx) for _, node := range partitionNodes { node := node + rd, err := buf.NewReader() + if err != nil { + return err + } g.Go(func() error { - // rd := bytes.NewReader(shardBytes) - rd := func() (io.Reader, error) { - return bytes.NewReader(buf.Bytes()), nil - } - - return cmd.client.ImportIndexKeys(ctx, &node.URI, indexName, partitionID, false, rd) + url := node.URI.Path(fmt.Sprintf("/internal/translate/index/%s/%d", indexName, partitionID)) + return Post(ctx, url, "application/octet-stream", rd, nil) }) } if err := g.Wait(); err != nil { @@ -269,23 +277,20 @@ func (cmd *RestoreTarCommand) Run(ctx context.Context) (err error) { switch action := record[4]; action { case "translate": logger.Printf("field keys %v %v", indexName, fieldName) - // needs to go to all nodes - _, err = io.Copy(buf, tarReader) if err != nil { return errors.Wrap(err, "copying") } - g, _ := errgroup.WithContext(ctx) for _, node := range nodes { node := node + rd, err := buf.NewReader() + if err != nil { + return err + } g.Go(func() error { - // rd := bytes.NewReader(shardBytes) - rd := func() (io.Reader, error) { - return bytes.NewReader(buf.Bytes()), nil - } - - return cmd.client.ImportFieldKeys(ctx, &node.URI, indexName, fieldName, false, rd) + url := node.URI.Path(fmt.Sprintf("/internal/translate/field/%s/%s", indexName, fieldName)) + return Post(ctx, url, "application/octet-stream", rd, nil) }) } if err := g.Wait(); err != nil { @@ -312,3 +317,33 @@ func (cmd *RestoreTarCommand) Run(ctx context.Context) (err error) { func (cmd *RestoreTarCommand) TLSHost() string { return cmd.Host } func (cmd *RestoreTarCommand) TLSConfiguration() server.TLSConfig { return cmd.TLS } + +func Post(ctx context.Context, url, contentType string, rd io.Reader, query map[string]string) error { + client := &gohttp.Client{} + req, err := http.NewRequest(http.MethodPost, url, rd) + if err != nil { + return err + } + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("Content-Type", contentType) + pilosa.AddAuthToken(ctx, &req.Header) + + // appending to existing query args + + q := req.URL.Query() + for k, v := range query { + q.Add(k, v) + } + + // assign encoded query string to http request + req.URL.RawQuery = q.Encode() + + resp, err := client.Do(req) + if err != nil { + fmt.Println("Errored when sending request to the server") + return err + } + defer resp.Body.Close() + _, err = ioutil.ReadAll(resp.Body) // drain the response + return err +} From 186da6b302fbe3f95cf6c2b12b6ab072f12e8639 Mon Sep 17 00:00:00 2001 From: Garrison Davis Date: Wed, 22 Feb 2023 15:58:47 -0700 Subject: [PATCH 16/19] Refactor build-fbsql and upload to S3 in CI --- .gitlab/.gitlab-ci.yml | 38 ++++++++++++++++++-------------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index f0bac6ce6..75311906a 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -157,20 +157,17 @@ build featurebase: - go install github.com/rakyll/statik@v0.1.7 - $GOPATH/bin/statik -src=lattice - export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) - - GOOS="linux" GOARCH="amd64" make build FLAGS="-o featurebase_linux_amd64" - - GOOS="linux" GOARCH="arm64" make build FLAGS="-o featurebase_linux_arm64" - - GOOS="darwin" GOARCH="amd64" make build FLAGS="-o featurebase_darwin_amd64" - - GOOS="darwin" GOARCH="arm64" make build FLAGS="-o featurebase_darwin_arm64" - - GOOS="linux" GOARCH="amd64" make build-fbsql FLAGS="-o fbsql_linux_amd64" - - GOOS="linux" GOARCH="arm64" make build-fbsql FLAGS="-o fbsql_linux_arm64" - - GOOS="darwin" GOARCH="amd64" make build-fbsql FLAGS="-o fbsql_darwin_amd64" - - GOOS="darwin" GOARCH="arm64" make build-fbsql FLAGS="-o fbsql_darwin_arm64" + - | + for goos in "darwin" "linux"; do + for goarch in "amd64" "arm64"; do + GOOS="${goos}" GOARCH="${goarch}" make build FLAGS="-o featurebase_${goos}_${goarch}" + GOOS="${goos}" GOARCH="${goarch}" make build-fbsql FLAGS="-o fbsql_${goos}_${goarch}" + done + done artifacts: paths: - - featurebase_linux_amd64 - - featurebase_linux_arm64 - - featurebase_darwin_amd64 - - featurebase_darwin_arm64 + - featurebase_* + - fbsql_* needs: - job: build lattice @@ -636,14 +633,15 @@ s3 dump: - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY - aws configure set region "us-east-2" - aws configure set aws_profile $PROFILE - - aws s3 cp featurebase_linux_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/featurebase_linux_amd64 - - aws s3 cp featurebase_linux_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_linux_amd64 - - aws s3 cp featurebase_linux_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/featurebase_linux_arm64 - - aws s3 cp featurebase_linux_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_linux_arm64 - - aws s3 cp featurebase_darwin_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/featurebase_darwin_amd64 - - aws s3 cp featurebase_darwin_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_darwin_amd64 - - aws s3 cp featurebase_darwin_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/featurebase_darwin_arm64 - - aws s3 cp featurebase_darwin_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_darwin_arm64 + - | + for goos in "darwin" "linux"; do + for goarch in "amd64" "arm64"; do + for binary in "featurebase" "fbsql"; do + aws s3 cp ${binary}_${goos}_${goarch} s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/${binary}_${goos}_${goarch} + aws s3 cp ${binary}_${goos}_${goarch} s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/${binary}_${goos}_${goarch} + done + done + done needs: - job: build featurebase From dbea3056381d7771a05baad931e535ac141c9bc2 Mon Sep 17 00:00:00 2001 From: tgruben Date: Fri, 24 Feb 2023 12:30:57 -0600 Subject: [PATCH 17/19] add a batch size to limit upload payloads (#2275) --- cmd/dataframe-csv-loader.go | 1 + ctl/dataframe-csv-loader.go | 29 ++++++++++++++++++++++++----- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/cmd/dataframe-csv-loader.go b/cmd/dataframe-csv-loader.go index a0f677ca5..fd0679b1d 100644 --- a/cmd/dataframe-csv-loader.go +++ b/cmd/dataframe-csv-loader.go @@ -24,6 +24,7 @@ func newDataframeCsvLoaderCommand(logdest logger.Logger) *cobra.Command { flags.StringVar(&cmd.AuthToken, "auth-token", "", "Authentication token") flags.StringVar(&cmd.Index, "index", "", "Destination Index. ") flags.IntVar(&cmd.MaxCapacity, "buffer", 0, "Maximum size of of the line buffer defaults to go bufio default ") + flags.IntVar(&cmd.BatchSize, "batch-size", 1048576, "Maximum number of records to send in a single batch ") ctl.SetTLSConfig( flags, "", &cmd.TLS.CertificatePath, diff --git a/ctl/dataframe-csv-loader.go b/ctl/dataframe-csv-loader.go index 70cc45e3e..2a5f893dc 100644 --- a/ctl/dataframe-csv-loader.go +++ b/ctl/dataframe-csv-loader.go @@ -50,6 +50,9 @@ type DataframeCsvLoaderCommand struct { // max line length of csv file MaxCapacity int + // Batch Size + BatchSize int + // Host:port on which to listen for pprof. Pprof string `json:"pprof"` @@ -234,7 +237,9 @@ func (cmd *DataframeCsvLoaderCommand) Run(ctx context.Context) (err error) { fileScanner.Scan() // skip the header cmd.Logger().Infof("Build the dataframe input package in memory") id := uint64(0) + recordCounter := 0 for fileScanner.Scan() { + recordCounter++ records := strings.Split(fileScanner.Text(), ",") if cmd.needTranslation { id = lookup[records[0]] @@ -278,13 +283,23 @@ func (cmd *DataframeCsvLoaderCommand) Run(ctx context.Context) (err error) { } } } + if recordCounter > cmd.BatchSize { + cmd.Logger().Infof("sending package to featurebase") + err = sharder.Store(arrowSchema, cmd.client) + if err != nil { + return err + } + sharder.Reset() + recordCounter = 0 + } } - cmd.Logger().Infof("sending package to featurebase") - err = sharder.Store(arrowSchema, cmd.client) - if err != nil { - return err + if recordCounter > 0 { + err = sharder.Store(arrowSchema, cmd.client) + if err != nil { + return err + } } - return err + return nil } type pair struct { @@ -378,6 +393,10 @@ type Sharder struct { log logger.Logger } +func (s *Sharder) Reset() { + s.shards = make(map[uint64]*ShardDiff) +} + func (s *Sharder) GetShard(shard uint64) (*ShardDiff, error) { f, ok := s.shards[shard] if ok { From 6777e3dc07ef98c2e216cf044e0a9cbd1b2ad289 Mon Sep 17 00:00:00 2001 From: Vengata Krishnan <122897205+vkrishnanfb@users.noreply.github.com> Date: Fri, 24 Feb 2023 14:57:17 -0500 Subject: [PATCH 18/19] FB-1968 timestamp data type related fixes and enhancements (#2256) * Removed support for EPOCH column constraint from TIMESTAMP SQL data type. * Implicit conversion of integers to timestamp will treat the integer value as seconds since unix epoch. * Add new ToTimeStamp(num, timeunit) SQL scalar function to help convert integer values to timestamp. --- dax/table.go | 3 - dax/table_test.go | 4 +- sql3/parser/ast_test.go | 6 +- sql3/parser/parser.go | 9 --- sql3/planner/expression.go | 4 +- sql3/planner/expressionanalyzercall.go | 2 + sql3/planner/expressiontypes.go | 6 +- sql3/planner/inbuiltfunctionsdate.go | 75 +++++++++++++++++++++++ sql3/planner/opbulkinsert.go | 9 ++- sql3/planner/opinsert.go | 21 ++++++- sql3/sql_complex_test.go | 12 ++-- sql3/test/defs/defs_date_functions.go | 64 +++++++++++++++++++ sql3/test/defs/defs_inserts.go | 11 +++- sql3/test/defs/defs_timestamp_literals.go | 37 ++++++++++- 14 files changed, 226 insertions(+), 37 deletions(-) diff --git a/dax/table.go b/dax/table.go index 91f065300..c77ffdbed 100644 --- a/dax/table.go +++ b/dax/table.go @@ -755,9 +755,6 @@ func (f *Field) constraints() string { case BaseTypeTimestamp: if f.Options.TimeUnit != "" { sql += fmt.Sprintf(" TIMEUNIT '%s'", f.Options.TimeUnit) - if !f.Options.Epoch.IsZero() { - sql += fmt.Sprintf(" EPOCH '%s'", f.Options.Epoch.Format(time.RFC3339)) // time.RFC3339 - } } } diff --git a/dax/table_test.go b/dax/table_test.go index 7947a8735..a4280137c 100644 --- a/dax/table_test.go +++ b/dax/table_test.go @@ -6,7 +6,6 @@ import ( "sort" "strings" "testing" - "time" "github.com/featurebasedb/featurebase/v3/dax" "github.com/featurebasedb/featurebase/v3/pql" @@ -193,12 +192,11 @@ func TestTable(t *testing.T) { Type: "timestamp", Options: dax.FieldOptions{ TimeUnit: "s", - Epoch: time.Date(2009, 11, 10, 23, 34, 56, 0, time.UTC), }, }, }, }, - expSQL: "CREATE TABLE all_field_types_with_options (_id string, an_id id CACHETYPE ranked SIZE 500, a_string string CACHETYPE ranked SIZE 500, an_id_set idset CACHETYPE ranked SIZE 500, a_string_set stringset CACHETYPE ranked SIZE 500, an_int int MIN -100 MAX 200, a_decimal decimal, a_timestamp timestamp TIMEUNIT 's' EPOCH '2009-11-10T23:34:56Z') KEYPARTITIONS 0", + expSQL: "CREATE TABLE all_field_types_with_options (_id string, an_id id CACHETYPE ranked SIZE 500, a_string string CACHETYPE ranked SIZE 500, an_id_set idset CACHETYPE ranked SIZE 500, a_string_set stringset CACHETYPE ranked SIZE 500, an_int int MIN -100 MAX 200, a_decimal decimal, a_timestamp timestamp TIMEUNIT 's') KEYPARTITIONS 0", }, } for i, test := range tests { diff --git a/sql3/parser/ast_test.go b/sql3/parser/ast_test.go index eb25286fe..dcddcfaa3 100644 --- a/sql3/parser/ast_test.go +++ b/sql3/parser/ast_test.go @@ -240,9 +240,7 @@ func TestCreateTableStatement_String(t *testing.T) { Type: &parser.Type{Name: &parser.Ident{Name: "TIMESTAMP"}}, Constraints: []parser.Constraint{ &parser.TimeUnitConstraint{ - Expr: &parser.StringLit{Value: "s"}, - Epoch: pos(0), - EpochExpr: &parser.StringLit{Value: "2021-01-01T00:00:00Z"}, + Expr: &parser.StringLit{Value: "s"}, }, }, }, @@ -255,7 +253,7 @@ func TestCreateTableStatement_String(t *testing.T) { `intcol INTEGER MIN 100 MAX 1000, `+ `stringcol STRING CACHETYPE RANKED SIZE 10000, `+ `stringsetcol STRINGSET CACHETYPE RANKED SIZE 10000, `+ - `timestampcol TIMESTAMP TIMEUNIT 's' EPOCH '2021-01-01T00:00:00Z'`+ + `timestampcol TIMESTAMP TIMEUNIT 's'`+ `)`) } diff --git a/sql3/parser/parser.go b/sql3/parser/parser.go index 8b290c324..32ad9c593 100644 --- a/sql3/parser/parser.go +++ b/sql3/parser/parser.go @@ -860,15 +860,6 @@ func (p *Parser) parseTimeUnitConstraint(constraintPos Pos, name *Ident) (_ *Tim } else { return &cons, p.errorExpected(p.pos, p.tok, "literal") } - if p.peek() == EPOCH { - cons.Epoch, _, _ = p.scan() - - if isLiteralToken(p.peek()) { - cons.EpochExpr = p.mustParseLiteral() - } else { - return &cons, p.errorExpected(p.pos, p.tok, "literal") - } - } return &cons, nil } diff --git a/sql3/planner/expression.go b/sql3/planner/expression.go index c014fe140..33937e6e0 100644 --- a/sql3/planner/expression.go +++ b/sql3/planner/expression.go @@ -41,7 +41,6 @@ func coerceValue(sourceType parser.ExprDataType, targetType parser.ExprDataType, return nil, sql3.NewErrInternalf("unexpected value type '%T'", value) } return pql.NewDecimal(val*int64(math.Pow(10, float64(t.Scale))), t.Scale), nil - case *parser.DataTypeTimestamp: val, ok := value.(int64) if !ok { @@ -65,7 +64,6 @@ func coerceValue(sourceType parser.ExprDataType, targetType parser.ExprDataType, return nil, sql3.NewErrInternalf("unexpected value type '%T'", value) } return pql.NewDecimal(int64(val)*int64(math.Pow(10, float64(t.Scale))), t.Scale), nil - case *parser.DataTypeTimestamp: val, ok := value.(int64) if !ok { @@ -1574,6 +1572,8 @@ func (n *callPlanExpression) Evaluate(currentRow []interface{}) (interface{}, er return n.EvaluateFormat(currentRow) case "CHARINDEX": return n.EvaluateCharIndex(currentRow) + case "TOTIMESTAMP": + return n.EvaluateToTimestamp(currentRow) case "STR": return n.EvaluateStr(currentRow) default: diff --git a/sql3/planner/expressionanalyzercall.go b/sql3/planner/expressionanalyzercall.go index 85deb038f..698b994c8 100644 --- a/sql3/planner/expressionanalyzercall.go +++ b/sql3/planner/expressionanalyzercall.go @@ -254,6 +254,8 @@ func (p *ExecutionPlanner) analyzeCallExpression(ctx context.Context, call *pars return p.analyseFunctionFormat(call, scope) case "CHARINDEX": return p.analyseFunctionCharIndex(call, scope) + case "TOTIMESTAMP": + return p.analyzeFunctionToTimestamp(call, scope) case "STR": return p.analyseFunctionStr(call, scope) default: diff --git a/sql3/planner/expressiontypes.go b/sql3/planner/expressiontypes.go index 242bf287d..569dc1f0e 100644 --- a/sql3/planner/expressiontypes.go +++ b/sql3/planner/expressiontypes.go @@ -346,12 +346,12 @@ func typesAreAssignmentCompatible(targetType parser.ExprDataType, sourceType par switch sourceType.(type) { case *parser.DataTypeTimestamp: return true - case *parser.DataTypeInt: - //could be a int convertable to a date - return true case *parser.DataTypeString: //could be a string parseable as a date return true + case *parser.DataTypeInt: + //integers coerced to timestamp will be treated as time represented in number of seconds since unix epoch + return true default: return false } diff --git a/sql3/planner/inbuiltfunctionsdate.go b/sql3/planner/inbuiltfunctionsdate.go index 4149893f8..49a531c52 100644 --- a/sql3/planner/inbuiltfunctionsdate.go +++ b/sql3/planner/inbuiltfunctionsdate.go @@ -4,6 +4,7 @@ import ( "strings" "time" + featurebase "github.com/featurebasedb/featurebase/v3" "github.com/featurebasedb/featurebase/v3/sql3" "github.com/featurebasedb/featurebase/v3/sql3/parser" ) @@ -43,6 +44,33 @@ func (p *ExecutionPlanner) analyzeFunctionDatePart(call *parser.Call, scope pars return call, nil } +func (p *ExecutionPlanner) analyzeFunctionToTimestamp(call *parser.Call, scope parser.Statement) (parser.Expr, error) { + //param1 is the number to be converted to timestamp. This param is required. + //param2 is the time unit of the numeric value in param 1. This param is optional. + //ToTimestamp can be invoked with just param1. + if len(call.Args) != 1 && len(call.Args) != 2 { + return nil, sql3.NewErrCallParameterCountMismatch(call.Rparen.Line, call.Rparen.Column, call.Name.Name, 2, len(call.Args)) + } + + //param1 is a integer of type int64 + param1Type := parser.NewDataTypeInt() + if !typesAreAssignmentCompatible(param1Type, call.Args[0].DataType()) { + return nil, sql3.NewErrParameterTypeMistmatch(call.Args[0].Pos().Line, call.Args[0].Pos().Column, call.Args[0].DataType().TypeDescription(), param1Type.TypeDescription()) + } + + //param2 is a string and it should be one of 's', 'ms', 'us', 'ns'. + //param2 is optional, will be defaulted to 's' if not supplied. + if len(call.Args) == 2 { + param2Type := parser.NewDataTypeString() + if !typesAreAssignmentCompatible(param2Type, call.Args[1].DataType()) { + return nil, sql3.NewErrParameterTypeMistmatch(call.Args[1].Pos().Line, call.Args[1].Pos().Column, call.Args[1].DataType().TypeDescription(), param2Type.TypeDescription()) + } + } + //ToTimestamp returns a timestamp calculated from param1 using time unit passed in param 2 + call.ResultDataType = parser.NewDataTypeTimestamp() + return call, nil +} + func (n *callPlanExpression) EvaluateDatepart(currentRow []interface{}) (interface{}, error) { intervalEval, err := n.args[0].Evaluate(currentRow) if err != nil { @@ -121,3 +149,50 @@ func (n *callPlanExpression) EvaluateDatepart(currentRow []interface{}) (interfa } } + +func (n *callPlanExpression) EvaluateToTimestamp(currentRow []interface{}) (interface{}, error) { + //retrieve param1, the number to be converted to timestamp + param1, err := n.args[0].Evaluate(currentRow) + if err != nil { + return nil, err + } else if param1 == nil { + //if the param1 is null silently return null timestamp value + return nil, nil + } + coercedParam1, err := coerceValue(n.args[0].Type(), parser.NewDataTypeInt(), param1, parser.Pos{Line: 0, Column: 0}) + if err != nil { + //raise error if param 1 is not an integer. Should we return nil instead of raising error here? see note at return. + return nil, err + } + num, ok := coercedParam1.(int64) + if !ok { + //raise error if param 1 is not an integer. Should we return nil instead of raising error here? see note at return. + return nil, sql3.NewErrInternalf("unable to convert value") + } + + //retrieve param2, time unit for param1, if not supplied default to seconds 's'. + var unit string = featurebase.TimeUnitSeconds + if len(n.args) == 2 { + param2, err := n.args[1].Evaluate(currentRow) + if err != nil { + //raise error if unable to retieve the argument for param2 + return nil, err + } + coercedParam2, err := coerceValue(n.args[1].Type(), parser.NewDataTypeString(), param2, parser.Pos{Line: 0, Column: 0}) + if err != nil { + //raise error if param2 is not a string + return nil, err + } + unit, ok = coercedParam2.(string) + if !ok { + //raise error if param2 is not a string + return nil, sql3.NewErrInternalf("unable to convert value") + } + if !featurebase.IsValidTimeUnit(unit) { + //raise error is param2 is not a valid time unit + return nil, sql3.NewErrCallParameterValueInvalid(0, 0, unit, "timeunit") + } + } + //should we throw error or return nil if the conversion fails? what is the desired behaviour when ToTimestamp errors for one bad record in a batch of thousands? + return featurebase.ValToTimestamp(unit, num) +} diff --git a/sql3/planner/opbulkinsert.go b/sql3/planner/opbulkinsert.go index 1f3ad4dcb..223aa66ac 100644 --- a/sql3/planner/opbulkinsert.go +++ b/sql3/planner/opbulkinsert.go @@ -320,7 +320,8 @@ func (i *bulkInsertSourceCSVRowIter) Next(ctx context.Context) (types.Row, error return nil, sql3.NewErrTypeConversionOnMap(0, 0, evalValue, mapColumn.colType.TypeDescription()) } } else { - result[idx] = time.UnixMilli(intVal).UTC() + //implicit conversion of int to timestamp will treat int as seconds since unix epoch + result[idx] = time.Unix(intVal, 0).UTC() } case *parser.DataTypeString: @@ -651,7 +652,8 @@ func (i *bulkInsertSourceNDJsonRowIter) Next(ctx context.Context) (types.Row, er case float64: // if v is a whole number then make it an int if v == float64(int64(v)) { - result[idx] = time.UnixMilli(int64(v)).UTC() + //implicit conversion of int to timestamp will treat int as seconds since unix epoch + result[idx] = time.Unix(int64(v), 0).UTC() } else { return nil, sql3.NewErrTypeConversionOnMap(0, 0, v, mapColumn.colType.TypeDescription()) } @@ -1153,7 +1155,8 @@ func (i *bulkInsertSourceParquetRowIter) Next(ctx context.Context) (types.Row, e case *parser.DataTypeTimestamp: if intVal, ok := evalValue.(int64); ok { - result[idx] = time.UnixMilli(intVal).UTC() + //implicit conversion of int to timestamp will treat int as seconds since unix epoch + result[idx] = time.Unix(intVal, 0).UTC() } else if stringVal, ok := evalValue.(string); ok { if tm, err := time.ParseInLocation(time.RFC3339Nano, stringVal, time.UTC); err == nil { result[idx] = tm diff --git a/sql3/planner/opinsert.go b/sql3/planner/opinsert.go index e2914c61a..87dfee2d4 100644 --- a/sql3/planner/opinsert.go +++ b/sql3/planner/opinsert.go @@ -390,7 +390,26 @@ func (i *insertRowIter) Next(ctx context.Context) (types.Row, error) { return nil, errors.Wrapf(err, "converting timestamp to int64: %s", v) } row.Values[posVals[idx]] = i64 - + //integers passed as input for Timestamp fields will be treated as time represented in number of seconds since epoch defined for the field + //for timestamp fields created using SQL epoch will be defaulted to unix epoch + case int64: + // Convert the input seconds to target timeunit defined for the Timestamp field + // Add Base, which is the base epoch for Timestamp fields, to the input before saving + unit := fbbatch.TimeUnit(opts.TimeUnit) + i64 := opts.Base + switch unit { + case fbbatch.TimeUnitSeconds: + i64 = i64 + v + case fbbatch.TimeUnitMilliseconds: + i64 = i64 + (v * 1000) + case fbbatch.TimeUnitMicroseconds, fbbatch.TimeUnitUSeconds: + i64 = i64 + (v * 1000000) + case fbbatch.TimeUnitNanoseconds: + i64 = i64 + (v * 1000000000) + default: + return nil, errors.Wrapf(err, "unknown time unit: %s", unit) + } + row.Values[posVals[idx]] = i64 // nil is to support `null` values. case nil: row.Values[posVals[idx]] = eval diff --git a/sql3/sql_complex_test.go b/sql3/sql_complex_test.go index 081adfcc6..a25314807 100644 --- a/sql3/sql_complex_test.go +++ b/sql3/sql_complex_test.go @@ -460,13 +460,13 @@ func TestPlanner_CoverCreateTable(t *testing.T) { { name: "timestampcol", typ: "timestamp", - constraints: "timeunit 'ms' epoch '2021-01-01T00:00:00Z'", + constraints: "timeunit 'ms'", expOptions: pilosa.FieldOptions{ - Base: 1609459200000, + Base: 0, Type: "timestamp", TimeUnit: "ms", - Min: pql.NewDecimal(-63745055999000, 0), - Max: pql.NewDecimal(251792841599000, 0), + Min: pql.NewDecimal(-62135596799000, 0), + Max: pql.NewDecimal(253402300799000, 0), }, }, { @@ -713,7 +713,7 @@ func TestPlanner_CreateTable(t *testing.T) { _id id, intcol int min 0 max 10000, boolcol bool, - timestampcol timestamp timeunit 'ms' epoch '2010-01-01T00:00:00Z', + timestampcol timestamp timeunit 'ms', decimalcol decimal(2), stringcol string cachetype ranked size 1000, stringsetcol stringset cachetype lru size 1000, @@ -2957,7 +2957,7 @@ func TestPlanner_BulkInsertParquet(t *testing.T) { now := time.Now() simpleParquetMaker(t, tmpfile, 1, []tb{ {Name: "id", Type: arrow.PrimitiveTypes.Int64, Value: []int64{1}}, - {Name: "unixtime", Type: arrow.PrimitiveTypes.Int64, Value: []int64{now.UnixMilli()}}, + {Name: "unixtime", Type: arrow.PrimitiveTypes.Int64, Value: []int64{now.Unix()}}, {Name: "stringtime", Type: arrow.BinaryTypes.String, Value: []string{now.Format(time.RFC3339)}}, }) diff --git a/sql3/test/defs/defs_date_functions.go b/sql3/test/defs/defs_date_functions.go index 74383589f..f5cb037a9 100644 --- a/sql3/test/defs/defs_date_functions.go +++ b/sql3/test/defs/defs_date_functions.go @@ -1,5 +1,7 @@ package defs +import "time" + // datepart tests var datePartTests = TableTest{ @@ -34,6 +36,30 @@ var datePartTests = TableTest{ ), ExpErr: "invalid value '1' for parameter 'interval'", }, + { + SQLs: sqls( + "select totimestamp()", + ), + ExpErr: "count of formal parameters (2) does not match count of actual parameters (0)", + }, + { + SQLs: sqls( + "select totimestamp('a')", + ), + ExpErr: "an expression of type 'string' cannot be passed to a parameter of type 'int'", + }, + { + SQLs: sqls( + "select totimestamp(1, 2)", + ), + ExpErr: "an expression of type 'int' cannot be passed to a parameter of type 'string'", + }, + { + SQLs: sqls( + "select totimestamp(1, 'x')", + ), + ExpErr: "invalid value 'x' for parameter 'timeunit'", + }, { SQLs: sqls( "select _id, datepart('yy', ts) from dateparttests", @@ -177,5 +203,43 @@ var datePartTests = TableTest{ ), Compare: CompareExactUnordered, }, + { + //test datepart(timestamp, part) for implicit conversion of integer value passed as argument to timestamp param + SQLs: sqls( + "select datepart('yy', 0) as \"yy\", datepart('m', 0) as \"m\", datepart('d', 0) as \"d\"", + ), + ExpHdrs: hdrs( + hdr("yy", fldTypeInt), + hdr("m", fldTypeInt), + hdr("d", fldTypeInt), + ), + ExpRows: rows( + row(int64(1970), int64(1), int64(1)), + ), + Compare: CompareExactUnordered, + }, + { + //test ToTimestamp(num, timeunit) for all possible time unit values + SQLs: sqls( + "select totimestamp(1000) as \"default\", totimestamp(1000, 's') as \"s\", totimestamp(1000000, 'ms') as \"ms\", totimestamp(1000000000, 'us') as \"us\", totimestamp(1000000000, 'µs') as \"µs\", totimestamp(1000000000000, 'ns') as \"ns\"", + ), + ExpHdrs: hdrs( + hdr("default", fldTypeTimestamp), + hdr("s", fldTypeTimestamp), + hdr("ms", fldTypeTimestamp), + hdr("us", fldTypeTimestamp), + hdr("µs", fldTypeTimestamp), + hdr("ns", fldTypeTimestamp), + ), + ExpRows: rows( + row(time.Unix(1000, 0).UTC(), + time.Unix(1000, 0).UTC(), + time.UnixMilli(1000000).UTC(), + time.UnixMicro(1000000000).UTC(), + time.UnixMicro(1000000000).UTC(), + time.Unix(0, 1000000000000).UTC()), + ), + Compare: CompareExactUnordered, + }, }, } diff --git a/sql3/test/defs/defs_inserts.go b/sql3/test/defs/defs_inserts.go index 65cb18ad8..135e276f3 100644 --- a/sql3/test/defs/defs_inserts.go +++ b/sql3/test/defs/defs_inserts.go @@ -159,7 +159,7 @@ var insertTimestampTest = TableTest{ SQLTests: []SQLTest{ { SQLs: sqls( - "CREATE TABLE insertTimestampTest (_id id, time timestamp timeunit 'ms' epoch '2022-01-01T00:00:00Z', ids idset, strings stringset);", + "CREATE TABLE insertTimestampTest (_id id, time timestamp timeunit 'ms', ids idset, strings stringset);", ), ExpHdrs: hdrs(), ExpRows: rows(), @@ -173,6 +173,14 @@ var insertTimestampTest = TableTest{ ExpRows: rows(), Compare: CompareExactUnordered, }, + { + SQLs: sqls( + "INSERT INTO insertTimestampTest(_id, time, ids, strings) VALUES (2, 1672531200, [6 , 1, 9], ['red', 'blue', 'green']);", + ), + ExpHdrs: hdrs(), + ExpRows: rows(), + Compare: CompareExactUnordered, + }, { SQLs: sqls( "select time from insertTimestampTest;", @@ -182,6 +190,7 @@ var insertTimestampTest = TableTest{ ), ExpRows: rows( row(timestampFromString("2023-01-01T00:00:00Z")), + row(timestampFromString("2023-01-01T00:00:00Z")), ), Compare: CompareExactUnordered, SortStringKeys: true, diff --git a/sql3/test/defs/defs_timestamp_literals.go b/sql3/test/defs/defs_timestamp_literals.go index b22ee806a..b4cd6a0c0 100644 --- a/sql3/test/defs/defs_timestamp_literals.go +++ b/sql3/test/defs/defs_timestamp_literals.go @@ -18,7 +18,7 @@ var timestampLiterals = TableTest{ { // InsertWithCurrentTimestamp SQLs: sqls( - "insert into testtimestampliterals (_id, a, b, d, ts, event, ievent) values (4, 40, 400, 10.12, current_timestamp, ['A', 'B', 'C'], [1, 2, 3])", + "insert into testtimestampliterals (_id, a, b, d, ts, event, ievent) values (1, 40, 400, 10.12, current_timestamp, ['A', 'B', 'C'], [1, 2, 3])", ), ExpHdrs: hdrs(), ExpRows: rows(), @@ -27,11 +27,44 @@ var timestampLiterals = TableTest{ { // InsertWithCurrentDate SQLs: sqls( - "insert into testtimestampliterals (_id, a, b, d, ts, event, ievent) values (4, 40, 400, 10.12, current_date, ['A', 'B', 'C'], [1, 2, 3])", + "insert into testtimestampliterals (_id, a, b, d, ts, event, ievent) values (2, 40, 400, 10.12, current_date, ['A', 'B', 'C'], [1, 2, 3])", ), ExpHdrs: hdrs(), ExpRows: rows(), Compare: CompareExactUnordered, }, + { + // Insert literal 0 into a timestamp, it should be stored as 1970-01-01 00:00:00 +0000 UTC (unix epoch base value) + SQLs: sqls( + "insert into testtimestampliterals (_id, a, b, d, ts, event, ievent) values (3, 40, 400, 10.12, 0, ['A', 'B', 'C'], [1, 2, 3])", + ), + ExpHdrs: hdrs(), + ExpRows: rows(), + Compare: CompareExactUnordered, + }, + { + // Insert literal -86400 into a timestamp, it should be stored as 1969-12-31 00:00:00 +0000 UTC (unix epoch base value) + SQLs: sqls( + "insert into testtimestampliterals (_id, a, b, d, ts, event, ievent) values (4, 40, 400, 10.12, -86400, ['A', 'B', 'C'], [1, 2, 3])", + ), + ExpHdrs: hdrs(), + ExpRows: rows(), + Compare: CompareExactUnordered, + }, + { + //compare test is done only for integer test cases (_id in (3,4), because only for these cases we have a determinate year(1970, 1969) to look for. + SQLs: sqls( + "select _id, datepart('yy', ts) as \"yy\" from testtimestampliterals where _id in (3,4)", + ), + ExpHdrs: hdrs( + hdr("_id", fldTypeID), + hdr("yy", fldTypeInt), + ), + ExpRows: rows( + row(int64(3), int64(1970)), + row(int64(4), int64(1969)), + ), + Compare: CompareExactUnordered, + }, }, } From 933767ec07a00c116dddbf24c8bd98ba1509e9f4 Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Mon, 27 Feb 2023 10:41:53 -0600 Subject: [PATCH 19/19] move closing of profiling stuff to Close method so it runs for duration of process (#2277) --- server/server.go | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/server/server.go b/server/server.go index 827d6edcd..89e9985d5 100644 --- a/server/server.go +++ b/server/server.go @@ -73,6 +73,8 @@ type Command struct { // done will be closed when Command.Close() is called done chan struct{} + traceCloser io.Closer + logOutput io.Writer queryLogOutput io.Writer logger loggerLogger @@ -834,7 +836,6 @@ func (m *Command) setupProfilingAndTracing() error { if err != nil { return errors.Wrap(err, "starting datadog") } - defer profiler.Stop() } if m.Config.Tracing.SamplerType != "off" { @@ -852,12 +853,10 @@ func (m *Command) setupProfilingAndTracing() error { if err != nil { return errors.Wrap(err, "initializing jaeger tracer") } - defer closer.Close() + m.traceCloser = closer tracing.GlobalTracer = opentracing.NewTracer(tracer, m.Logger()) - } else if m.Config.DataDog.EnableTracing { // Give preference to legacy support of jaeger t := opentracer.New(tracer.WithServiceName(m.Config.DataDog.Service)) - defer tracer.Stop() tracing.GlobalTracer = opentracing.NewTracer(t, m.Logger()) } return nil @@ -883,6 +882,14 @@ func (m *Command) Close() error { err := eg.Wait() _ = testhook.Closed(pilosa.NewAuditor(), m, nil) + if m.Config.DataDog.Enable { + defer profiler.Stop() + } + if m.traceCloser != nil { + defer m.traceCloser.Close() + } else if m.Config.DataDog.EnableTracing { + defer tracer.Stop() + } close(m.done) return errors.Wrap(err, "closing everything")