diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index e1ac9fa06..75311906a 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -157,16 +157,17 @@ build featurebase: - go install github.com/rakyll/statik@v0.1.7 - $GOPATH/bin/statik -src=lattice - export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) - - GOOS="linux" GOARCH="amd64" make build FLAGS="-o featurebase_linux_amd64" - - GOOS="linux" GOARCH="arm64" make build FLAGS="-o featurebase_linux_arm64" - - GOOS="darwin" GOARCH="amd64" make build FLAGS="-o featurebase_darwin_amd64" - - GOOS="darwin" GOARCH="arm64" make build FLAGS="-o featurebase_darwin_arm64" + - | + for goos in "darwin" "linux"; do + for goarch in "amd64" "arm64"; do + GOOS="${goos}" GOARCH="${goarch}" make build FLAGS="-o featurebase_${goos}_${goarch}" + GOOS="${goos}" GOARCH="${goarch}" make build-fbsql FLAGS="-o fbsql_${goos}_${goarch}" + done + done artifacts: paths: - - featurebase_linux_amd64 - - featurebase_linux_arm64 - - featurebase_darwin_amd64 - - featurebase_darwin_arm64 + - featurebase_* + - fbsql_* needs: - job: build lattice @@ -216,7 +217,11 @@ run go tests race: script: - echo "Running featurebase race tests..." - PKG_LIST=$(go list ./... | grep -Ev 'internal/clustertests|simulacraData|batch|idk|v3/dax/test/dax' | paste -s -d, -) - - RAMDISK=/mnt/ramdisk go test -race -v -timeout=10m ${PKG_LIST//,/ } + - export TMPDIR=/mnt/ramdisk/test-$CI_JOB_ID + - mkdir -p $TMPDIR + - go test -race -v -timeout=10m ${PKG_LIST//,/ } + after_script: + - rm -rf /mnt/ramdisk/test-$CI_JOB_ID tags: - docker @@ -232,7 +237,11 @@ run go tests: script: - echo "Running featurebase unit tests..." - PKG_LIST=$(go list ./... | grep -Ev 'internal/clustertests|simulacraData|batch|idk|v3/dax/test/dax' | paste -s -d, -) - - RAMDISK=/mnt/ramdisk go test -tags=shardwidth22 -timeout=10m -coverprofile=coverage.out -covermode=atomic -coverpkg=${PKG_LIST} ${PKG_LIST//,/ } + - export TMPDIR=/mnt/ramdisk/test-$CI_JOB_ID + - mkdir -p $TMPDIR + - go test -tags=shardwidth22 -timeout=10m -coverprofile=coverage.out -covermode=atomic -coverpkg=${PKG_LIST} ${PKG_LIST//,/ } + after_script: + - rm -rf /mnt/ramdisk/test-$CI_JOB_ID artifacts: paths: - coverage.out @@ -624,14 +633,15 @@ s3 dump: - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY - aws configure set region "us-east-2" - aws configure set aws_profile $PROFILE - - aws s3 cp featurebase_linux_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/featurebase_linux_amd64 - - aws s3 cp featurebase_linux_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_linux_amd64 - - aws s3 cp featurebase_linux_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/featurebase_linux_arm64 - - aws s3 cp featurebase_linux_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_linux_arm64 - - aws s3 cp featurebase_darwin_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/featurebase_darwin_amd64 - - aws s3 cp featurebase_darwin_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_darwin_amd64 - - aws s3 cp featurebase_darwin_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/featurebase_darwin_arm64 - - aws s3 cp featurebase_darwin_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_darwin_arm64 + - | + for goos in "darwin" "linux"; do + for goarch in "amd64" "arm64"; do + for binary in "featurebase" "fbsql"; do + aws s3 cp ${binary}_${goos}_${goarch} s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/${binary}_${goos}_${goarch} + aws s3 cp ${binary}_${goos}_${goarch} s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/${binary}_${goos}_${goarch} + done + done + done needs: - job: build featurebase diff --git a/Makefile b/Makefile index 2e952d166..23e9d0ba9 100644 --- a/Makefile +++ b/Makefile @@ -24,6 +24,9 @@ BUILD_TAGS += TEST_TAGS = roaringparanoia TEST_TIMEOUT=10m RACE_TEST_TIMEOUT=10m +# size in GB to use for ramdisk, ?= so you can override it with env +# 4GB is not enough for `make test`, 8GB usually is. +RAMDISK_SIZE ?= 8 export GO111MODULE=on export GOPRIVATE=github.com/molecula @@ -77,13 +80,18 @@ testvsub: echo; echo "999 done testing subpkg $$pkg"; \ done -# make a 2GB RAMDisk. Speed up tests by running them with RAMDISK=/mnt/ramdisk +# make a $(RAMDISK_SIZE)GB RAMDisk. Speed up tests by running +# them with TMPDIR=/mnt/ramdisk. ramdisk-linux: - mount -o size=2G -t tmpfs none /mnt/ramdisk + mount -o size=$(RAMDISK__SIZE)G -t tmpfs none /mnt/ramdisk -# make a 2GB RAMDisk. Speed up tests by running them with RAMDISK=/Volumes/RAMDisk +# make a $(RAMDISK_SIZE)GB RAMDisk. Speed up tests by running +# them with TMPDIR=/Volumes/RAMDisk. This is more important on +# OS X than it is on Linux, because there's performance issues +# with fsync on OS X that can make the SSD slow down to moving-platters +# drive speeds. Oops. ramdisk-osx: - diskutil erasevolume HFS+ 'RAMDisk' `hdiutil attach -nobrowse -nomount ram://4194304` + diskutil erasevolume HFS+ 'RAMDisk' $$(hdiutil attach -nobrowse -nomount ram://$$(expr 2097152 \* $(RAMDISK_SIZE))) detach-ramdisk-osx: hdiutil detach /Volumes/RAMDisk @@ -108,12 +116,18 @@ cover: cover-viz: cover $(GO) tool cover -html=build/coverage.out -# Compile Pilosa +# Build featurebase build: $(GO) build -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/featurebase +# Build fbsql +build-fbsql: + $(GO) build -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/fbsql + + package: - GOOS=$(GOOS) GOARCH=$(GOARCH) FLAGS="-o featurebase" $(MAKE) build + GOOS=$(GOOS) GOARCH=$(GOARCH) $(MAKE) build + GOOS=$(GOOS) GOARCH=$(GOARCH) $(MAKE) build-fbsql GOARCH=$(GOARCH) VERSION=$(VERSION) nfpm package --packager deb --target featurebase.$(VERSION).$(GOARCH).deb GOARCH=$(GOARCH) VERSION=$(VERSION) nfpm package --packager rpm --target featurebase.$(VERSION).$(GOARCH).rpm diff --git a/buffer/filebuffer.go b/buffer/filebuffer.go new file mode 100644 index 000000000..3f0647465 --- /dev/null +++ b/buffer/filebuffer.go @@ -0,0 +1,110 @@ +package buffer + +import ( + "bytes" + "io" + "io/ioutil" + "os" + "sync" +) + +// NewFileBuffer returns a file buffer which will use an in-memory buffer, until `max` bytes have been written, at which point it will write the contents of memory to a file, and continue writing future data to the file. +// The file will be written to `temp` directory. The buffer fulfills the io.Reader and io.Writer interface +func NewFileBuffer(max int, temp string) *FileBuffer { + return &FileBuffer{max: max, tempDir: temp} +} + +type FileBuffer struct { + max int + buf bytes.Buffer + file *os.File + tempDir string + reading bool + files []*os.File + mu sync.Mutex +} + +func (fb *FileBuffer) Write(p []byte) (n int, err error) { + if fb.reading { + panic("cannot write after read") + } + if fb.file != nil { + return fb.file.Write(p) + } + n, err = fb.buf.Write(p) + if err != nil { + return + } + if fb.buf.Len() > fb.max { + fb.file, err = ioutil.TempFile(fb.tempDir, "filebuffer-") + if err != nil { + return + } + _, err = io.Copy(fb.file, &fb.buf) + fb.buf.Reset() + } + return +} + +func (fb *FileBuffer) Len() (int64, error) { + if fb.file == nil { + return int64(fb.buf.Len()), nil + } + fi, err := fb.file.Stat() + if err != nil { + return 0, err + } + + return fi.Size(), nil +} + +func (fb *FileBuffer) Read(p []byte) (n int, err error) { + if fb.file != nil { + if !fb.reading { + fb.reading = true + _, err = fb.file.Seek(0, 0) + if err != nil { + return + } + } + return fb.file.Read(p) + } + fb.reading = true + return fb.buf.Read(p) +} + +func (fb *FileBuffer) Close() error { + if fb.file != nil { + name := fb.file.Name() + if err := fb.file.Close(); err != nil { + return err + } + for _, f := range fb.files { + f.Close() + } + fb.files = fb.files[:0] + fb.file = nil + return os.Remove(name) + } + return nil +} + +func (fb *FileBuffer) Reset() error { + fb.mu.Lock() + defer fb.mu.Unlock() + fb.reading = false + fb.buf.Reset() + return fb.Close() +} + +func (fb *FileBuffer) NewReader() (io.Reader, error) { + fb.mu.Lock() + defer fb.mu.Unlock() + fb.reading = true + if fb.file == nil { + return bytes.NewReader(fb.buf.Bytes()), nil + } + f, err := os.OpenFile(fb.file.Name(), os.O_RDONLY, 0) + fb.files = append(fb.files, f) + return f, err +} diff --git a/cli/cli.go b/cli/cli.go index 7a3ee1606..cc8f2e0a3 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -79,6 +79,9 @@ type Command struct { // i.e. it will quit after the command is complete. Files []string `json:"files"` + // variables holds the variables created with the \set meta-command. + variables map[string]string + // nonInteractiveMode is set to true when fbsql is running in // non-ineracative mode. And example of this is when the user has provided a // `-c` flag in the command line. @@ -89,6 +92,8 @@ type Command struct { } func NewCommand(logdest logger.Logger) *Command { + variables := make(map[string]string) + return &Command{ Config: &Config{ Host: defaultHost, @@ -107,8 +112,8 @@ func NewCommand(logdest logger.Logger) *Command { HistoryPath: "", }, - splitter: newSplitter(), buffer: newBuffer(), + splitter: newSplitter(newReplacer(variables)), workingDir: newWorkingDir(), Stdin: Stdin, @@ -118,6 +123,8 @@ func NewCommand(logdest logger.Logger) *Command { output: Stdout, writeOptions: defaultWriteOptions(), + variables: variables, + quit: make(chan struct{}), } } diff --git a/cli/meta.go b/cli/meta.go index b249691fd..142f9b78a 100644 --- a/cli/meta.go +++ b/cli/meta.go @@ -7,6 +7,7 @@ import ( "os/exec" "os/signal" "path/filepath" + "sort" "strconv" "strings" "syscall" @@ -54,6 +55,7 @@ var _ metaCommand = (*metaReset)(nil) var _ metaCommand = (*metaSet)(nil) var _ metaCommand = (*metaTiming)(nil) var _ metaCommand = (*metaTuplesOnly)(nil) +var _ metaCommand = (*metaUnset)(nil) var _ metaCommand = (*metaWarn)(nil) var _ metaCommand = (*metaWatch)(nil) var _ metaCommand = (*metaWrite)(nil) @@ -333,6 +335,10 @@ Operating System \cd [DIR] change the current working directory \timing [on|off] toggle timing of commands \! [COMMAND] execute command in shell or start interactive shell + +Variables + \set [NAME [VALUE]] set internal variable, or list all if no parameters + \unset NAME unset (delete) internal variable ` cmd.Printf("%s\n", helpText) @@ -367,7 +373,7 @@ func executeFile(cmd *Command, fileName string) (action, error) { } defer file.Close() - splitter := newSplitter() + splitter := newSplitter(newReplacer(cmd.variables)) buffer := newBuffer() // Read the file by line, pushing the lines into a new line splitter, then @@ -647,7 +653,29 @@ func newMetaSet(args []string) *metaSet { } func (m *metaSet) execute(cmd *Command) (action, error) { - // TODO: set the variable (or clear it, etc) + switch len(m.args) { + case 0: + // Sort the variables before printing them. + keys := make([]string, 0, len(cmd.variables)) + for k := range cmd.variables { + keys = append(keys, k) + } + sort.Strings(keys) + + // Print out the variables. + for _, k := range keys { + cmd.Printf("%s = '%s'\n", k, cmd.variables[k]) + } + cmd.writeOptions.timing = !cmd.writeOptions.timing + default: + // The first arg is the key, the remaining args are concatenated together to form the values + // For example: + // \set one two three + // will result in `one = 'twothree'` + k := m.args[0] + v := strings.Join(m.args[1:], "") + cmd.variables[k] = v + } return actionNone, nil } @@ -729,6 +757,35 @@ func (m *metaTuplesOnly) execute(cmd *Command) (action, error) { return actionNone, nil } +// //////////////////////////////////////////////////////////////////////////// +// unset +// //////////////////////////////////////////////////////////////////////////// +type metaUnset struct { + args []string +} + +func newMetaUnset(args []string) *metaUnset { + return &metaUnset{ + args: args, + } +} + +func (m *metaUnset) execute(cmd *Command) (action, error) { + switch len(m.args) { + case 0: + cmd.Printf("\\unset: missing required argument\n") + return actionNone, nil + default: + if len(m.args) > 1 { + for _, s := range m.args[1:] { + cmd.Printf("\\unset: extra argument \"%s\" ignored\n", s) + } + } + delete(cmd.variables, m.args[0]) + } + return actionNone, nil +} + // //////////////////////////////////////////////////////////////////////////// // warn // //////////////////////////////////////////////////////////////////////////// @@ -854,7 +911,7 @@ func (m *metaWrite) execute(cmd *Command) (action, error) { // `cmd 'arg1' arg2 'arg three'` // // It returns the metaCommand which maps to `cmd`. -func splitMetaCommand(in string) (metaCommand, error) { +func splitMetaCommand(in string, replacer *replacer) (metaCommand, error) { parts := strings.SplitN(in, ` `, 2) key := strings.TrimRightFunc(parts[0], unicode.IsSpace) @@ -877,6 +934,11 @@ func splitMetaCommand(in string) (metaCommand, error) { } } + // Do variable replacement. + for i := range args { + args[i] = replacer.replace(args[i]) + } + switch key { case "!": return newMetaBang(args), nil @@ -916,6 +978,8 @@ func splitMetaCommand(in string) (metaCommand, error) { return newMetaTuplesOnly(args), nil case "timing": return newMetaTiming(args), nil + case "unset": + return newMetaUnset(args), nil case "warn": return newMetaWarn(args), nil case "watch": diff --git a/cli/replacer.go b/cli/replacer.go new file mode 100644 index 000000000..b22330a8a --- /dev/null +++ b/cli/replacer.go @@ -0,0 +1,109 @@ +package cli + +import ( + "strings" + + "github.com/benhoyt/goawk/lexer" +) + +// replacer can replace parts of a string based on some rules and the provided +// map[string]string. For example, the Command can replace strings with values +// in its `variables` map. +type replacer struct { + m map[string]string +} + +func newReplacer(m map[string]string) *replacer { + return &replacer{ + m: m, + } +} + +// replace replaces all instances of the string pattern `:key` with the value at +// m[key]. For example we want something like this: +// +// GIVEN: `start :one,:'two', :"three" ::four ::` +// +// with map +// +// map[string]string{ +// "one": "repl1", +// "three": "repl3", +// } +// +// WANT: `start repl1,:'two', "repl3" ::four ::` +func (r *replacer) replace(s string) string { + // If no variables have been added to the map, there's no need to parse the + // string for variable replacement. + if len(r.m) == 0 { + return s + } + + line := []byte(s) + lex := lexer.NewLexer(line) + + // finger contains the index into line at the start of non-variable text + // that we want to include, as-is in the output. + var finger int + + // sb builds the string which will be the final output. + var sb strings.Builder + for { + pos, tok, _ := lex.Scan() + + switch tok { + case lexer.COLON: + // last is the last normal character position before the colon. + last := pos.Column - 1 + + // Get the next byte to see if the colon value is quoted, and if so, + // whether its has single or double quotes. + b := lex.PeekByte() + + // padding is the amount of padding we have to consider around the + // variable name. If the variable is not quoted, it doesn't require + // any padding. But if it has quotes, it needs 2 characters of + // paddings to accomodate the quotes. + padding := 0 + + // quote holds the character to use to quote the final, replaced + // output value. Because the lexer doesn't tell us how a certain + // `string` token was quoted, we need to keep track of that here so + // we can put them back. + quote := "" + switch b { + case byte('\''): // single quote + quote = `'` + padding = 2 + case byte('"'): // double quote + quote = `"` + padding = 2 + } + + pos, tok, key := lex.Scan() + switch tok { + case lexer.NAME, lexer.STRING: + // Write the normal text up to the variable replacement + // position. + sb.Write(line[finger:last]) + + if v, ok := r.m[key]; ok { + // Write replaced variable with the quotes it had. + sb.WriteString(quote + v + quote) + } else { + // Since the variable was not found in the map, just write + // back what was already there. + sb.WriteString(":" + quote + key + quote) + } + + // Reset finger to point to the next position after the + // variable. + finger = pos.Column + len(key) + padding - 1 + } + case lexer.EOF: + // Write the remainder of the string and return. + sb.Write(line[finger:]) + return sb.String() + } + } +} diff --git a/cli/replacer_test.go b/cli/replacer_test.go new file mode 100644 index 000000000..4cf1069ce --- /dev/null +++ b/cli/replacer_test.go @@ -0,0 +1,109 @@ +package cli + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestReplacer(t *testing.T) { + t.Run("general replace function", func(t *testing.T) { + + m := map[string]string{ + "v1": "newVone", + "v2": "newVtwo", + } + + tests := []struct { + s string + m map[string]string + exp string + }{ + { + // no variables present + s: "foo", + m: m, + exp: "foo", + }, + { + // variable prefix, but not in map + s: ":foo", + m: m, + exp: ":foo", + }, + { + // variable name match, but missing prefix + s: "v1", + m: m, + exp: "v1", + }, + { + // variable name match + s: ":v1", + m: m, + exp: "newVone", + }, + { + // two variables, the same, no space + s: ":v1:v1", + m: m, + exp: "newVonenewVone", + }, + { + // two variables, different, no space + s: ":v1:v2", + m: m, + exp: "newVonenewVtwo", + }, + { + // two variables, different, spaces + s: ":v1 :v2", + m: m, + exp: "newVone newVtwo", + }, + { + // one variable, one non-variable, no space + s: ":v1:foo", + m: m, + exp: "newVone:foo", + }, + { + // one non-variable, one variable, no space + s: "foo:v1", + m: m, + exp: "foonewVone", + }, + { + // two variables, different, comma + s: ":v1, :v2", + m: m, + exp: "newVone, newVtwo", + }, + { + // single quotes + s: ":'v1'", + m: m, + exp: "'newVone'", + }, + { + // double quotes + s: `:"v2"`, + m: m, + exp: `"newVtwo"`, + }, + { + // more quotes + s: `start :v1,:'two', :"v2" ::four :: `, + m: m, + exp: `start newVone,:'two', "newVtwo" ::four :: `, + }, + } + for i, test := range tests { + t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) { + replacer := newReplacer(test.m) + assert.Equal(t, test.exp, replacer.replace(test.s)) + }) + } + }) +} diff --git a/cli/splitter.go b/cli/splitter.go index ca62eb257..bb3c5f2d1 100644 --- a/cli/splitter.go +++ b/cli/splitter.go @@ -10,10 +10,14 @@ import ( // metaCommands. It may not be necessary to have this be a separate struct since // it contains no members and just has the one `split()` method, but here we // are. -type splitter struct{} +type splitter struct { + replacer *replacer +} -func newSplitter() *splitter { - return &splitter{} +func newSplitter(r *replacer) *splitter { + return &splitter{ + replacer: r, + } } // split splits the given line into queryParts and metaCommands. @@ -65,6 +69,11 @@ func (s *splitter) splitQueryParts(line string) ([]queryPart, error) { // Look for a termination character; parts := strings.Split(line, terminationChar) + // Do variable replacement. + for i := range parts { + parts[i] = s.replacer.replace(parts[i]) + } + if len(parts) == 1 { part0 := strings.TrimSpace(parts[0]) return []queryPart{ @@ -95,7 +104,7 @@ func (s *splitter) splitQueryParts(line string) ([]queryPart, error) { func (s *splitter) splitMetaCommands(in string) ([]metaCommand, error) { parts := strings.Split(in, `\`) if len(parts) == 1 { - mc, err := splitMetaCommand(parts[0]) + mc, err := splitMetaCommand(parts[0], s.replacer) if err != nil { return nil, errors.Wrapf(err, "splitting meta command: %s", parts[0]) } @@ -108,7 +117,7 @@ func (s *splitter) splitMetaCommands(in string) ([]metaCommand, error) { if part == "" { continue } - mc, err := splitMetaCommand(part) + mc, err := splitMetaCommand(part, s.replacer) if err != nil { return nil, errors.Wrapf(err, "splitting meta command: %s", part) } diff --git a/cli/splitter_test.go b/cli/splitter_test.go index 54b0fb3eb..ae22251f5 100644 --- a/cli/splitter_test.go +++ b/cli/splitter_test.go @@ -8,7 +8,7 @@ import ( ) func TestSplitter(t *testing.T) { - s := newSplitter() + s := newSplitter(newReplacer(nil)) t.Run("Split", func(t *testing.T) { tests := []struct { line string diff --git a/cmd/backup_tar.go b/cmd/backup_tar.go index 6c9612d77..641f4ee51 100644 --- a/cmd/backup_tar.go +++ b/cmd/backup_tar.go @@ -28,6 +28,7 @@ Backs up a FeatureBase server to a local, tar-formatted snapshot file. ctl.SetTLSConfig(flags, "", &cmd.TLS.CertificatePath, &cmd.TLS.CertificateKeyPath, &cmd.TLS.CACertPath, &cmd.TLS.SkipVerify, &cmd.TLS.EnableClientVerification) flags.StringVar(&cmd.AuthToken, "auth-token", "", "Authentication token") flags.StringVar(&cmd.HeaderTimeoutStr, "header-timeout", cmd.HeaderTimeoutStr, "Length of time to wait for initial HTTP response before giving up.") + flags.StringVar(&cmd.TempDir, "temp-dir", cmd.TempDir, "Location of temporary spillover files. The default is the system's default (usually /tmp)") return ccmd } diff --git a/cmd/dataframe-csv-loader.go b/cmd/dataframe-csv-loader.go index a0f677ca5..fd0679b1d 100644 --- a/cmd/dataframe-csv-loader.go +++ b/cmd/dataframe-csv-loader.go @@ -24,6 +24,7 @@ func newDataframeCsvLoaderCommand(logdest logger.Logger) *cobra.Command { flags.StringVar(&cmd.AuthToken, "auth-token", "", "Authentication token") flags.StringVar(&cmd.Index, "index", "", "Destination Index. ") flags.IntVar(&cmd.MaxCapacity, "buffer", 0, "Maximum size of of the line buffer defaults to go bufio default ") + flags.IntVar(&cmd.BatchSize, "batch-size", 1048576, "Maximum number of records to send in a single batch ") ctl.SetTLSConfig( flags, "", &cmd.TLS.CertificatePath, diff --git a/cmd/restore_tar.go b/cmd/restore_tar.go index d626a488c..9a1cdc3fc 100644 --- a/cmd/restore_tar.go +++ b/cmd/restore_tar.go @@ -23,6 +23,8 @@ The Restore command will take a tar-formatted backup archive and restore it to a flags.DurationVar(&cmd.RetryPeriod, "retry-period", cmd.RetryPeriod, "Length of time after HTTP request failure to continue retrying request.") flags.StringVar(&cmd.Pprof, "pprof", cmd.Pprof, "host:port to listen for profiling requests at /debug/pprof and /debug/fgprof.") flags.StringVar(&cmd.AuthToken, "auth-token", "", "Authentication token") + flags.StringVar(&cmd.TempDir, "temp-dir", cmd.TempDir, "Location of temporary spillover files. The default is the system's default (usually /tmp)") + ctl.SetTLSConfig( flags, "", &cmd.TLS.CertificatePath, diff --git a/ctl/backup_tar.go b/ctl/backup_tar.go index 0b0780b0d..e8f65d90d 100644 --- a/ctl/backup_tar.go +++ b/ctl/backup_tar.go @@ -3,7 +3,6 @@ package ctl import ( "archive/tar" - "bytes" "context" "crypto/tls" "encoding/json" @@ -17,6 +16,7 @@ import ( pilosa "github.com/featurebasedb/featurebase/v3" "github.com/featurebasedb/featurebase/v3/authn" + "github.com/featurebasedb/featurebase/v3/buffer" "github.com/featurebasedb/featurebase/v3/disco" "github.com/featurebasedb/featurebase/v3/encoding/proto" "github.com/featurebasedb/featurebase/v3/logger" @@ -37,6 +37,9 @@ type BackupTarCommand struct { // nolint: maligned // Path to write the backup to. OutputPath string + // TempDir location of scratch files + TempDir string + // Amount of time after first failed request to continue retrying. RetryPeriod time.Duration `json:"retry-period"` @@ -164,17 +167,16 @@ func (cmd *BackupTarCommand) Run(ctx context.Context) (err error) { tw := tar.NewWriter(w) defer tw.Close() - buf := new(bytes.Buffer) // Backup schema. if err := cmd.backupTarSchema(ctx, tw, schema); err != nil { return fmt.Errorf("cannot back up schema: %w", err) - } else if err := cmd.backupTarIDAllocData(ctx, tw, buf); err != nil { + } else if err := cmd.backupTarIDAllocData(ctx, tw); err != nil { return fmt.Errorf("cannot back up id alloc data: %w", err) } // Backup data for each index. for _, ii := range schema.Indexes { - if err := cmd.backupTarIndex(ctx, tw, ii, buf); err != nil { + if err := cmd.backupTarIndex(ctx, tw, ii); err != nil { return err } } @@ -220,8 +222,7 @@ func (cmd *BackupTarCommand) backupTarSchema(ctx context.Context, tw *tar.Writer return nil } -func (cmd *BackupTarCommand) backupTarIDAllocData(ctx context.Context, tw *tar.Writer, buf *bytes.Buffer) error { - buf.Reset() +func (cmd *BackupTarCommand) backupTarIDAllocData(ctx context.Context, tw *tar.Writer) error { logger := cmd.Logger() logger.Printf("backing up id alloc data") @@ -231,28 +232,11 @@ func (cmd *BackupTarCommand) backupTarIDAllocData(ctx context.Context, tw *tar.W } defer rc.Close() - // Read to buffer to determine size. - if _, err := buf.ReadFrom(rc); err != nil { - return fmt.Errorf("copying id alloc data to memory: %w", err) - } - - // Build header & copy data to archive. - if err = tw.WriteHeader(&tar.Header{ - Name: "idalloc", - Mode: 0o666, - Size: int64(buf.Len()), - ModTime: time.Now(), - }); err != nil { - return err - } else if _, err := io.Copy(tw, buf); err != nil { - return fmt.Errorf("copying id alloc data to archive: %w", err) - } - - return nil + return writeToTar(tw, "idalloc", rc, cmd.TempDir) } // backupTarIndex backs up all shards for a given index. -func (cmd *BackupTarCommand) backupTarIndex(ctx context.Context, tw *tar.Writer, ii *pilosa.IndexInfo, buf *bytes.Buffer) error { +func (cmd *BackupTarCommand) backupTarIndex(ctx context.Context, tw *tar.Writer, ii *pilosa.IndexInfo) error { logger := cmd.Logger() logger.Printf("backing up index: %q", ii.Name) @@ -263,14 +247,14 @@ func (cmd *BackupTarCommand) backupTarIndex(ctx context.Context, tw *tar.Writer, // Back up all bitmap data for the index. for _, shard := range shards { - if err := cmd.backupTarShard(ctx, tw, ii.Name, shard, buf); err != nil { + if err := cmd.backupTarShard(ctx, tw, ii.Name, shard); err != nil { return fmt.Errorf("cannot backup shard %d on index %q: %w", shard, ii.Name, err) } } if ii.Options.Keys { // Back up translation data after bitmap data so we ensurean translate all data. - if err := cmd.backupTarIndexTranslateData(ctx, tw, ii.Name, buf); err != nil { + if err := cmd.backupTarIndexTranslateData(ctx, tw, ii.Name); err != nil { return err } } @@ -280,7 +264,7 @@ func (cmd *BackupTarCommand) backupTarIndex(ctx context.Context, tw *tar.Writer, if !fi.Options.Keys { continue } - if err := cmd.backupTarFieldTranslateData(ctx, tw, ii.Name, fi.Name, buf); err != nil { + if err := cmd.backupTarFieldTranslateData(ctx, tw, ii.Name, fi.Name); err != nil { return fmt.Errorf("cannot backup field translation data for field %q on index %q: %w", fi.Name, ii.Name, err) } } @@ -289,7 +273,7 @@ func (cmd *BackupTarCommand) backupTarIndex(ctx context.Context, tw *tar.Writer, } // backupTarShard backs up a single shard from a single index. -func (cmd *BackupTarCommand) backupTarShard(ctx context.Context, tw *tar.Writer, indexName string, shard uint64, buf *bytes.Buffer) (err error) { +func (cmd *BackupTarCommand) backupTarShard(ctx context.Context, tw *tar.Writer, indexName string, shard uint64) (err error) { nodes, err := cmd.client.FragmentNodes(ctx, indexName, shard) if err != nil { return fmt.Errorf("cannot determine fragment nodes: %w", err) @@ -298,7 +282,7 @@ func (cmd *BackupTarCommand) backupTarShard(ctx context.Context, tw *tar.Writer, } for _, node := range nodes { - if e := cmd.backupTarShardNode(ctx, tw, indexName, shard, node, buf); e == nil { + if e := cmd.backupTarShardNode(ctx, tw, indexName, shard, node); e == nil { break } else if err == nil { err = e // save first error, try next node @@ -306,7 +290,7 @@ func (cmd *BackupTarCommand) backupTarShard(ctx context.Context, tw *tar.Writer, } for _, node := range nodes { - if e := cmd.backupTarShardDataframe(ctx, tw, indexName, shard, node, buf); e == nil { + if e := cmd.backupTarShardDataframe(ctx, tw, indexName, shard, node); e == nil { break } else if err == nil { err = e // save first error, try next node @@ -316,8 +300,7 @@ func (cmd *BackupTarCommand) backupTarShard(ctx context.Context, tw *tar.Writer, } // backupTarShardNode backs up a single shard from a single index on a specific node. -func (cmd *BackupTarCommand) backupTarShardNode(ctx context.Context, tw *tar.Writer, indexName string, shard uint64, node *disco.Node, buf *bytes.Buffer) error { - buf.Reset() +func (cmd *BackupTarCommand) backupTarShardNode(ctx context.Context, tw *tar.Writer, indexName string, shard uint64, node *disco.Node) error { logger := cmd.Logger() logger.Printf("backing up shard: index=%q id=%d", indexName, shard) @@ -332,30 +315,10 @@ func (cmd *BackupTarCommand) backupTarShardNode(ctx context.Context, tw *tar.Wri return fmt.Errorf("fetching shard reader: %w", err) } defer rc.Close() - - // Read to buffer to determine size. - // TODO: Provide size via the reader itself. - if _, err := buf.ReadFrom(rc); err != nil { - return fmt.Errorf("copying shard data to memory: %w", err) - } - - // Build header & copy data to archive. - if err = tw.WriteHeader(&tar.Header{ - Name: filename, - Mode: 0o666, - Size: int64(buf.Len()), - ModTime: time.Now(), - }); err != nil { - return err - } else if _, err := io.Copy(tw, buf); err != nil { - return fmt.Errorf("copying shard data to archive: %w", err) - } - - return nil + return writeToTar(tw, filename, rc, cmd.TempDir) } -func (cmd *BackupTarCommand) backupTarShardDataframe(ctx context.Context, tw *tar.Writer, indexName string, shard uint64, node *disco.Node, buf *bytes.Buffer) error { - buf.Reset() +func (cmd *BackupTarCommand) backupTarShardDataframe(ctx context.Context, tw *tar.Writer, indexName string, shard uint64, node *disco.Node) error { logger := cmd.Logger() logger.Printf("backing up dataframe shard: index=%q shard=%d", indexName, shard) @@ -375,38 +338,21 @@ func (cmd *BackupTarCommand) backupTarShardDataframe(ctx context.Context, tw *ta } filename := filepath.Join("indexes", indexName, "dataframe", fmt.Sprintf("%04d", shard)) - logger.Printf("writing %v", filename) - if _, err := buf.ReadFrom(resp.Body); err != nil { - return fmt.Errorf("copying shard data to memory: %w", err) - } - - // Build header & copy data to archive. - if err = tw.WriteHeader(&tar.Header{ - Name: filename, - Mode: 0o666, - Size: int64(buf.Len()), - ModTime: time.Now(), - }); err != nil { - return err - } else if _, err := io.Copy(tw, buf); err != nil { - return fmt.Errorf("copying shard data to archive: %w", err) - } - return nil + return writeToTar(tw, filename, resp.Body, cmd.TempDir) } -func (cmd *BackupTarCommand) backupTarIndexTranslateData(ctx context.Context, tw *tar.Writer, name string, buf *bytes.Buffer) error { +func (cmd *BackupTarCommand) backupTarIndexTranslateData(ctx context.Context, tw *tar.Writer, name string) error { // TODO: Fetch holder partition count. partitionN := disco.DefaultPartitionN for partitionID := 0; partitionID < partitionN; partitionID++ { - if err := cmd.backupTarIndexPartitionTranslateData(ctx, tw, name, partitionID, buf); err != nil { + if err := cmd.backupTarIndexPartitionTranslateData(ctx, tw, name, partitionID); err != nil { return fmt.Errorf("cannot backup index translation data for partition %d on %q: %w", partitionID, name, err) } } return nil } -func (cmd *BackupTarCommand) backupTarIndexPartitionTranslateData(ctx context.Context, tw *tar.Writer, name string, partitionID int, buf *bytes.Buffer) error { - buf.Reset() +func (cmd *BackupTarCommand) backupTarIndexPartitionTranslateData(ctx context.Context, tw *tar.Writer, name string, partitionID int) error { logger := cmd.Logger() logger.Printf("backing up index translation data: %s/%d", name, partitionID) @@ -418,28 +364,10 @@ func (cmd *BackupTarCommand) backupTarIndexPartitionTranslateData(ctx context.Co } defer rc.Close() - // Read to buffer to determine size. - if _, err := buf.ReadFrom(rc); err != nil { - return fmt.Errorf("copying translate data to memory: %w", err) - } - - // Build header & copy data to archive. - if err = tw.WriteHeader(&tar.Header{ - Name: path.Join("indexes", name, "translate", fmt.Sprintf("%04d", partitionID)), - Mode: 0o666, - Size: int64(buf.Len()), - ModTime: time.Now(), - }); err != nil { - return err - } else if _, err := io.Copy(tw, buf); err != nil { - return fmt.Errorf("copying translate data to archive: %w", err) - } - - return nil + return writeToTar(tw, path.Join("indexes", name, "translate", fmt.Sprintf("%04d", partitionID)), rc, cmd.TempDir) } -func (cmd *BackupTarCommand) backupTarFieldTranslateData(ctx context.Context, tw *tar.Writer, indexName, fieldName string, buf *bytes.Buffer) error { - buf.Reset() +func (cmd *BackupTarCommand) backupTarFieldTranslateData(ctx context.Context, tw *tar.Writer, indexName, fieldName string) error { logger := cmd.Logger() logger.Printf("backing up field translation data: %s/%s", indexName, fieldName) @@ -450,17 +378,37 @@ func (cmd *BackupTarCommand) backupTarFieldTranslateData(ctx context.Context, tw return fmt.Errorf("fetching translate data reader: %w", err) } defer rc.Close() + return writeToTar(tw, path.Join("indexes", indexName, "fields", fieldName, "translate"), rc, cmd.TempDir) +} +func (cmd *BackupTarCommand) TLSHost() string { return cmd.Host } + +func (cmd *BackupTarCommand) TLSConfiguration() server.TLSConfig { return cmd.TLS } + +func writeToTar(tw *tar.Writer, entryName string, rc io.Reader, tmpDir string) error { + spillFile, err := os.CreateTemp("", "spill") + if err != nil { + return fmt.Errorf("creating temp file : %w", err) + } + defer func() { + spillFile.Close() + os.Remove(spillFile.Name()) + }() + mb512 := 2 << 29 + buf := buffer.NewFileBuffer(mb512, tmpDir) + defer buf.Close() + + n, err := io.Copy(buf, rc) // Read to buffer to determine size. - if _, err := buf.ReadFrom(rc); err != nil { + if err != nil { return fmt.Errorf("copying translate data to memory: %w", err) } // Build header & copy data to archive. if err = tw.WriteHeader(&tar.Header{ - Name: path.Join("indexes", indexName, "fields", fieldName, "translate"), + Name: entryName, Mode: 0o666, - Size: int64(buf.Len()), + Size: n, ModTime: time.Now(), }); err != nil { return err @@ -469,7 +417,3 @@ func (cmd *BackupTarCommand) backupTarFieldTranslateData(ctx context.Context, tw } return nil } - -func (cmd *BackupTarCommand) TLSHost() string { return cmd.Host } - -func (cmd *BackupTarCommand) TLSConfiguration() server.TLSConfig { return cmd.TLS } diff --git a/ctl/dataframe-csv-loader.go b/ctl/dataframe-csv-loader.go index 70cc45e3e..2a5f893dc 100644 --- a/ctl/dataframe-csv-loader.go +++ b/ctl/dataframe-csv-loader.go @@ -50,6 +50,9 @@ type DataframeCsvLoaderCommand struct { // max line length of csv file MaxCapacity int + // Batch Size + BatchSize int + // Host:port on which to listen for pprof. Pprof string `json:"pprof"` @@ -234,7 +237,9 @@ func (cmd *DataframeCsvLoaderCommand) Run(ctx context.Context) (err error) { fileScanner.Scan() // skip the header cmd.Logger().Infof("Build the dataframe input package in memory") id := uint64(0) + recordCounter := 0 for fileScanner.Scan() { + recordCounter++ records := strings.Split(fileScanner.Text(), ",") if cmd.needTranslation { id = lookup[records[0]] @@ -278,13 +283,23 @@ func (cmd *DataframeCsvLoaderCommand) Run(ctx context.Context) (err error) { } } } + if recordCounter > cmd.BatchSize { + cmd.Logger().Infof("sending package to featurebase") + err = sharder.Store(arrowSchema, cmd.client) + if err != nil { + return err + } + sharder.Reset() + recordCounter = 0 + } } - cmd.Logger().Infof("sending package to featurebase") - err = sharder.Store(arrowSchema, cmd.client) - if err != nil { - return err + if recordCounter > 0 { + err = sharder.Store(arrowSchema, cmd.client) + if err != nil { + return err + } } - return err + return nil } type pair struct { @@ -378,6 +393,10 @@ type Sharder struct { log logger.Logger } +func (s *Sharder) Reset() { + s.shards = make(map[uint64]*ShardDiff) +} + func (s *Sharder) GetShard(shard uint64) (*ShardDiff, error) { f, ok := s.shards[shard] if ok { diff --git a/ctl/restore_tar.go b/ctl/restore_tar.go index 80bf9c5e7..be1052f99 100644 --- a/ctl/restore_tar.go +++ b/ctl/restore_tar.go @@ -3,12 +3,13 @@ package ctl import ( "archive/tar" - "bytes" "compress/gzip" "context" "crypto/tls" "fmt" "io" + "io/ioutil" + "net/http" gohttp "net/http" "os" "strconv" @@ -17,6 +18,7 @@ import ( pilosa "github.com/featurebasedb/featurebase/v3" "github.com/featurebasedb/featurebase/v3/authn" + "github.com/featurebasedb/featurebase/v3/buffer" "github.com/featurebasedb/featurebase/v3/disco" "github.com/featurebasedb/featurebase/v3/logger" "github.com/featurebasedb/featurebase/v3/server" @@ -48,6 +50,9 @@ type RestoreTarCommand struct { TLS server.TLSConfig AuthToken string + + // TempDir location of scratch files + TempDir string } // Logger returns the command's associated Logger to maintain CommandWithTLSSupport interface compatibility @@ -134,7 +139,10 @@ func (cmd *RestoreTarCommand) Run(ctx context.Context) (err error) { return errors.New("no primary") } c := &gohttp.Client{} - buf := new(bytes.Buffer) + // buf := new(bytes.Buffer) + mb512 := 2 << 29 + buf := buffer.NewFileBuffer(mb512, cmd.TempDir) + defer buf.Reset() for { buf.Reset() header, err := tarReader.Next() @@ -177,22 +185,21 @@ func (cmd *RestoreTarCommand) Run(ctx context.Context) (err error) { } else if len(fragmentNodes) == 0 { return fmt.Errorf("no fragmentNodes available") } - _, err = io.Copy(buf, tarReader) if err != nil { return errors.Wrap(err, "copying") } - g, _ := errgroup.WithContext(ctx) for _, node := range fragmentNodes { node := node + rd, err := buf.NewReader() + if err != nil { + return err + } g.Go(func() error { - client := &gohttp.Client{} - rd := bytes.NewReader(buf.Bytes()) logger.Printf("shard %v %v", shard, indexName) url := node.URI.Path(fmt.Sprintf("/internal/restore/%v/%v", indexName, shard)) - _, err = client.Post(url, "application/octet-stream", rd) - return err + return Post(ctx, url, "application/octet-stream", rd, nil) }) } if err := g.Wait(); err != nil { @@ -218,13 +225,14 @@ func (cmd *RestoreTarCommand) Run(ctx context.Context) (err error) { g, _ := errgroup.WithContext(ctx) for _, node := range fragmentNodes { node := node + rd, err := buf.NewReader() + if err != nil { + return err + } g.Go(func() error { - client := &gohttp.Client{} - rd := bytes.NewReader(buf.Bytes()) logger.Printf("dataframe shard %v %v", shard, indexName) url := node.URI.Path(fmt.Sprintf("/internal/dataframe/restore/%v/%v", indexName, shard)) - _, err = client.Post(url, "application/octet-stream", rd) - return err + return Post(ctx, url, "application/octet-stream", rd, nil) }) } if err := g.Wait(); err != nil { @@ -249,13 +257,13 @@ func (cmd *RestoreTarCommand) Run(ctx context.Context) (err error) { g, _ := errgroup.WithContext(ctx) for _, node := range partitionNodes { node := node + rd, err := buf.NewReader() + if err != nil { + return err + } g.Go(func() error { - // rd := bytes.NewReader(shardBytes) - rd := func() (io.Reader, error) { - return bytes.NewReader(buf.Bytes()), nil - } - - return cmd.client.ImportIndexKeys(ctx, &node.URI, indexName, partitionID, false, rd) + url := node.URI.Path(fmt.Sprintf("/internal/translate/index/%s/%d", indexName, partitionID)) + return Post(ctx, url, "application/octet-stream", rd, nil) }) } if err := g.Wait(); err != nil { @@ -269,23 +277,20 @@ func (cmd *RestoreTarCommand) Run(ctx context.Context) (err error) { switch action := record[4]; action { case "translate": logger.Printf("field keys %v %v", indexName, fieldName) - // needs to go to all nodes - _, err = io.Copy(buf, tarReader) if err != nil { return errors.Wrap(err, "copying") } - g, _ := errgroup.WithContext(ctx) for _, node := range nodes { node := node + rd, err := buf.NewReader() + if err != nil { + return err + } g.Go(func() error { - // rd := bytes.NewReader(shardBytes) - rd := func() (io.Reader, error) { - return bytes.NewReader(buf.Bytes()), nil - } - - return cmd.client.ImportFieldKeys(ctx, &node.URI, indexName, fieldName, false, rd) + url := node.URI.Path(fmt.Sprintf("/internal/translate/field/%s/%s", indexName, fieldName)) + return Post(ctx, url, "application/octet-stream", rd, nil) }) } if err := g.Wait(); err != nil { @@ -312,3 +317,33 @@ func (cmd *RestoreTarCommand) Run(ctx context.Context) (err error) { func (cmd *RestoreTarCommand) TLSHost() string { return cmd.Host } func (cmd *RestoreTarCommand) TLSConfiguration() server.TLSConfig { return cmd.TLS } + +func Post(ctx context.Context, url, contentType string, rd io.Reader, query map[string]string) error { + client := &gohttp.Client{} + req, err := http.NewRequest(http.MethodPost, url, rd) + if err != nil { + return err + } + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("Content-Type", contentType) + pilosa.AddAuthToken(ctx, &req.Header) + + // appending to existing query args + + q := req.URL.Query() + for k, v := range query { + q.Add(k, v) + } + + // assign encoded query string to http request + req.URL.RawQuery = q.Encode() + + resp, err := client.Do(req) + if err != nil { + fmt.Println("Errored when sending request to the server") + return err + } + defer resp.Body.Close() + _, err = ioutil.ReadAll(resp.Body) // drain the response + return err +} diff --git a/ctl/server.go b/ctl/server.go index 03261ffc2..0c03dca24 100644 --- a/ctl/server.go +++ b/ctl/server.go @@ -49,6 +49,8 @@ func serverFlagSet(srv *server.Config, prefix string) *pflag.FlagSet { flags.DurationVar((*time.Duration)(&srv.LongQueryTime), pre("long-query-time"), time.Duration(srv.LongQueryTime), "Duration that will trigger log and stat messages for slow queries. Zero to disable.") flags.IntVar(&srv.QueryHistoryLength, pre("query-history-length"), srv.QueryHistoryLength, "Number of queries to remember in history.") flags.Int64Var(&srv.MaxQueryMemory, pre("max-query-memory"), srv.MaxQueryMemory, "Maximum memory allowed per Extract() or SELECT query.") + flags.StringVar(&srv.VerChkAddress, pre("verchk-address"), srv.VerChkAddress, "Address to contact to check for latest version.") + flags.StringVar(&srv.UUIDFile, pre("uuid-file"), srv.UUIDFile, "File to store UUID used in checking latest version. If this is a relative path, the file will be stored in the server's data directory.") // TLS SetTLSConfig(flags, pre(""), &srv.TLS.CertificatePath, &srv.TLS.CertificateKeyPath, &srv.TLS.CACertPath, &srv.TLS.SkipVerify, &srv.TLS.EnableClientVerification) diff --git a/ctl/server_test.go b/ctl/server_test.go index 495c9b03c..1bc4cff30 100644 --- a/ctl/server_test.go +++ b/ctl/server_test.go @@ -20,4 +20,10 @@ func TestBuildServerFlags(t *testing.T) { if cm.Flags().Lookup("log-path").Name == "" { t.Fatal("log-path flag is required") } + if cm.Flags().Lookup("verchk-address").Name == "" { + t.Fatal("verchk-address flag is required") + } + if cm.Flags().Lookup("uuid-file").Name == "" { + t.Fatal("uuid-file flag is required") + } } diff --git a/dax/table.go b/dax/table.go index 91f065300..c77ffdbed 100644 --- a/dax/table.go +++ b/dax/table.go @@ -755,9 +755,6 @@ func (f *Field) constraints() string { case BaseTypeTimestamp: if f.Options.TimeUnit != "" { sql += fmt.Sprintf(" TIMEUNIT '%s'", f.Options.TimeUnit) - if !f.Options.Epoch.IsZero() { - sql += fmt.Sprintf(" EPOCH '%s'", f.Options.Epoch.Format(time.RFC3339)) // time.RFC3339 - } } } diff --git a/dax/table_test.go b/dax/table_test.go index 7947a8735..a4280137c 100644 --- a/dax/table_test.go +++ b/dax/table_test.go @@ -6,7 +6,6 @@ import ( "sort" "strings" "testing" - "time" "github.com/featurebasedb/featurebase/v3/dax" "github.com/featurebasedb/featurebase/v3/pql" @@ -193,12 +192,11 @@ func TestTable(t *testing.T) { Type: "timestamp", Options: dax.FieldOptions{ TimeUnit: "s", - Epoch: time.Date(2009, 11, 10, 23, 34, 56, 0, time.UTC), }, }, }, }, - expSQL: "CREATE TABLE all_field_types_with_options (_id string, an_id id CACHETYPE ranked SIZE 500, a_string string CACHETYPE ranked SIZE 500, an_id_set idset CACHETYPE ranked SIZE 500, a_string_set stringset CACHETYPE ranked SIZE 500, an_int int MIN -100 MAX 200, a_decimal decimal, a_timestamp timestamp TIMEUNIT 's' EPOCH '2009-11-10T23:34:56Z') KEYPARTITIONS 0", + expSQL: "CREATE TABLE all_field_types_with_options (_id string, an_id id CACHETYPE ranked SIZE 500, a_string string CACHETYPE ranked SIZE 500, an_id_set idset CACHETYPE ranked SIZE 500, a_string_set stringset CACHETYPE ranked SIZE 500, an_int int MIN -100 MAX 200, a_decimal decimal, a_timestamp timestamp TIMEUNIT 's') KEYPARTITIONS 0", }, } for i, test := range tests { diff --git a/go.mod b/go.mod index f92aa51ab..d4bd82bed 100644 --- a/go.mod +++ b/go.mod @@ -83,6 +83,7 @@ require ( github.com/PaesslerAG/gval v1.0.0 github.com/PaesslerAG/jsonpath v0.1.1 github.com/apache/arrow/go/v10 v10.0.0-20221021053532-2f627c213fc3 + github.com/benhoyt/goawk v1.21.0 github.com/gomem/gomem v0.1.0 github.com/google/uuid v1.3.0 github.com/jaffee/commandeer v0.6.0 diff --git a/go.sum b/go.sum index fdbd41d61..ba7e26a9f 100644 --- a/go.sum +++ b/go.sum @@ -139,6 +139,8 @@ github.com/beevik/ntp v0.3.0 h1:xzVrPrE4ziasFXgBVBZJDP0Wg/KpMwk2KHJ4Ba8GrDw= github.com/beevik/ntp v0.3.0/go.mod h1:hIHWr+l3+/clUnF44zdK+CWW7fO8dR5cIylAQ76NRpg= github.com/benbjohnson/immutable v0.4.0 h1:CTqXbEerYso8YzVPxmWxh2gnoRQbbB9X1quUC8+vGZA= github.com/benbjohnson/immutable v0.4.0/go.mod h1:iAr8OjJGLnLmVUr9MZ/rz4PWUy6Ouc2JLYuMArmvAJM= +github.com/benhoyt/goawk v1.21.0 h1:GASuhJXHMFZ/2TJBPh+2Ah3kclVGNvGjt+uh3ajMdLk= +github.com/benhoyt/goawk v1.21.0/go.mod h1:UG1Ld6CjkkHhoyQmErQGSTwmavsTqFnCDYsLSJbovqU= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= diff --git a/idk/header.go b/idk/header.go index 74581effa..cb7a2166d 100644 --- a/idk/header.go +++ b/idk/header.go @@ -32,7 +32,7 @@ const ( var ( ErrNoFieldSpec = errors.New("no field spec in this header") - ErrInvalidFieldName = errors.New("field name must match [a-z][a-z0-9_-]{0,229}") + ErrInvalidFieldName = errors.New("field name must match [a-z][a-z0-9Θ_-]{0,229}") ErrParsingEpoch = "parsing epoch for " ErrDecodingConfig = "decoding config for field " ) diff --git a/idk/header_test.go b/idk/header_test.go index 053b66f57..44e1df537 100644 --- a/idk/header_test.go +++ b/idk/header_test.go @@ -351,6 +351,11 @@ func TestHeaderToField(t *testing.T) { input: "a__LookupText", exp: LookupTextField{NameVal: "a", DestNameVal: "a"}, }, + { + name: "theta", + input: "fldΘnameΘ__String", + exp: StringField{NameVal: "fldΘnameΘ", DestNameVal: "fldΘnameΘ"}, + }, } for _, test := range tests { diff --git a/idk/kafka/cmd_test.go b/idk/kafka/cmd_test.go index f3cd26477..2cba2a680 100644 --- a/idk/kafka/cmd_test.go +++ b/idk/kafka/cmd_test.go @@ -4,6 +4,8 @@ package kafka import ( + "bufio" + "encoding/json" "fmt" "io" "math/rand" @@ -638,6 +640,132 @@ func TestCmdSchemaChange(t *testing.T) { } } +func TestTimeQuantums(t *testing.T) { + t.Parallel() + /* + at a high level, a test here represents + - an avro schema + - a set of records to ingest to kafka + - an ingest configuration + - query to run to confirm the data was ingest properly + */ + tests := []struct { + name string + pathToAvroSchema string + pathToRecords string + idType string // must be "generated", "id", or "string" + keyField string // field used for "id" or "string" record keys + pilosaHosts string + kafkaHost string + registryURL string + topic string + index string + queries []string + expectedResults []string + }{ + { // confirm time quantums are being ingested + name: "time quantums exist", + pathToAvroSchema: "timeQuantum.json", + pathToRecords: "./testdata/records/timeQuantum.json", + idType: "string", + keyField: "device", + pilosaHosts: pilosaHost, + registryURL: registryHost, + kafkaHost: kafkaHost, + topic: "timequantums", + index: "timequantums", + queries: []string{ + "Row(segment_ts='7R83')", + "Row(segment_ts='7R83', from=\"2023-02-17T00:00\", to=\"2023-02-18T00:00\")", + "Row(segment_ts='7R83', from=\"2023-02-16T00:00\", to=\"2023-02-17T00:00\")", + }, + expectedResults: []string{ + "{\"results\":[{\"columns\":[],\"keys\":[\"0QKtSTqJYXMZWvVe\"]}]}\n", + "{\"results\":[{\"columns\":[],\"keys\":[\"0QKtSTqJYXMZWvVe\"]}]}\n", + "{\"results\":[{\"columns\":[]}]}\n", + }, + }, + } + + for _, test := range tests { + // define some vars + now := time.Now().UnixNano() + index := fmt.Sprintf("%s_%d", test.index, now) + topic := fmt.Sprintf("%s_%d", test.topic, now) + + // read in records + var records []map[string]interface{} + var data map[string]interface{} + recordsFile, err := os.Open(test.pathToRecords) + if err != nil { + t.Errorf("opening records file") + } + defer recordsFile.Close() + + s := bufio.NewScanner(recordsFile) + for s.Scan() { + err := json.Unmarshal(s.Bytes(), &data) + if err != nil { + t.Errorf("unmarshal json: %s", err) + } + records = append(records, data) + data = make(map[string]interface{}) + } + + // configure the consumer + consumer, err := NewMain() + if err != nil { + t.Fatalf("creating main %v", err) + } + configureTestFlags(consumer) + consumer.Index = index + consumer.Topics = []string{topic} + consumer.KafkaBootstrapServers = []string{test.kafkaHost} + consumer.SchemaRegistryURL = test.registryURL + switch test.idType { + case "id": + consumer.IDField = test.keyField + case "string": + consumer.PrimaryKeyFields = []string{test.keyField} + case "generate": + consumer.AutoGenerate = true + consumer.ExternalGenerate = true + default: + t.Errorf("incorrect idType supplied") + } + consumer.MaxMsgs = uint64(len(records)) + + // load schema registry, create produce, topic and run consumer data + licodec := liDecodeTestSchema(t, test.pathToAvroSchema) + schemaID := postSchema(t, test.pathToAvroSchema, fmt.Sprintf("%s_id", topic), consumer.SchemaRegistryURL, nil) + p, err := confluent.NewProducer(&confluent.ConfigMap{ + "bootstrap.servers": kafkaHost, + }) + if err != nil { + t.Fatalf("Failed to create producer: %s", err) + } + defer p.Close() + tCreateTopic(t, topic, p) + tPutRecordsKafka(t, p, topic, schemaID, licodec, "akey", records...) + err = consumer.Run() + if err != nil { + t.Fatalf("running consumer: %v", err) + } + + // now run queries and confirm the data is as expected + client := consumer.PilosaClient() + for i, q := range test.queries { + status, body, err := client.HTTPRequest("POST", fmt.Sprintf("/index/%s/query", index), []byte(q), nil) + if err != nil { + t.Fatalf("querying featurebase: status: %d, response: %s, error: %s", status, body, err) + } + if string(body[:]) != test.expectedResults[i] { + t.Fatalf("running query: %s... expected %s but got %s", q, test.expectedResults[i], body) + } + } + } +} + type sortableCRI []pilosaclient.CountResultItem func (s sortableCRI) Len() int { return len(s) } diff --git a/idk/kafka/source.go b/idk/kafka/source.go index bcc519d20..003c8d76f 100644 --- a/idk/kafka/source.go +++ b/idk/kafka/source.go @@ -648,7 +648,8 @@ func avroToPDKField(aField *avro.SchemaField) (idk.Field, error) { return nil, errors.Errorf("required property for RecordTimeField: layout, err:%v", err) } return idk.RecordTimeField{ - Layout: layout, + NameVal: aField.Name, + Layout: layout, }, nil } diff --git a/idk/kafka/source_test.go b/idk/kafka/source_test.go index 0738bc753..e695cba11 100644 --- a/idk/kafka/source_test.go +++ b/idk/kafka/source_test.go @@ -226,6 +226,19 @@ func TestAvroToPDKField(t *testing.T) { expField: idk.IDField{NameVal: "int-ttl"}, expErr: "nil", }, + { + name: "recordTime", + schemaField: &avro.SchemaField{ + Name: "record-time", + Type: &avro.BytesSchema{}, + Properties: map[string]interface{}{ + "layout": "2006-01-02 15:04:05", + "fieldType": "recordTime", + }, + }, + expField: idk.RecordTimeField{NameVal: "record-time", Layout: "2006-01-02 15:04:05"}, + expErr: "nil", + }, } for _, test := range tests { diff --git a/idk/kafka/testdata/records/timeQuantum.json b/idk/kafka/testdata/records/timeQuantum.json new file mode 100644 index 000000000..7404128f8 --- /dev/null +++ b/idk/kafka/testdata/records/timeQuantum.json @@ -0,0 +1,500 @@ +{"device": "XbJ7vwASddz1xBQ4", "segment_ts": "fub2", "time_q": "2023-02-13 18:11:08"} +{"device": "g3AwJJUDwrN1LYzY", "segment_ts": "Qn3I", "time_q": "2023-02-13 19:39:01"} +{"device": "DIxrncsISTdUJdsH", "segment_ts": "o8Ig", "time_q": "2023-01-27 09:41:54"} +{"device": "yR1RUTVnjwUvjWLS", "segment_ts": "BUnG", "time_q": "2023-01-25 06:55:04"} +{"device": "dpeJtjO0mVsN3wiZ", "segment_ts": "Udwh", "time_q": "2023-01-25 14:34:42"} +{"device": "hOJrFi9xmOnM5J2B", "segment_ts": "9jED", "time_q": "2023-01-27 17:05:19"} +{"device": "xPLZ3qjA029BWoD8", "segment_ts": "PBbR", "time_q": "2023-02-17 08:47:05"} +{"device": "sdnPNO80Trlnkfpi", "segment_ts": "ACVo", "time_q": "2023-02-21 09:01:47"} +{"device": "fcMcP2T8hIBzYzrB", "segment_ts": "HppX", "time_q": "2023-02-18 00:59:36"} +{"device": "xiRO1wD8bdg6tsy2", "segment_ts": "iSQp", "time_q": "2023-01-28 06:15:59"} +{"device": "nlsTgLlvTBhQGewH", "segment_ts": "CZHg", "time_q": "2023-02-20 18:53:14"} +{"device": "bpLtS0dLbzC1t0aM", "segment_ts": "oanZ", "time_q": "2023-02-11 08:25:43"} +{"device": "RcYtSPPTAQyOE80E", "segment_ts": "57He", "time_q": "2023-02-14 06:40:17"} +{"device": "BLqS1Js4hqpAGKbs", "segment_ts": "8Esc", "time_q": "2023-02-13 01:59:32"} +{"device": "NDJ13gRlrfjSpJul", "segment_ts": "fBcm", "time_q": "2023-01-24 07:05:02"} +{"device": "CUS2BbP8fJrBuJRB", "segment_ts": "kUK0", "time_q": "2023-02-05 22:33:45"} +{"device": "lwC1KW3eNMX7kqwS", "segment_ts": "8s12", "time_q": "2023-02-15 19:55:08"} +{"device": "jtEPRgME5UKtmv2O", "segment_ts": "Ew1L", "time_q": "2023-02-10 10:44:56"} +{"device": "zJJQ58p1vZEVr8pV", "segment_ts": "cb9G", "time_q": "2023-01-31 14:49:08"} +{"device": "zGLCqlcb9WEVYeTD", "segment_ts": "JatF", "time_q": "2023-02-19 17:37:42"} +{"device": "PgyMZVp5lPVUAzju", "segment_ts": "H4w7", "time_q": "2023-01-25 01:13:51"} +{"device": "KS2ftxZpJaobrFcH", "segment_ts": "lzde", "time_q": "2023-02-08 07:29:00"} +{"device": "pvfQ90NOCNxkZ9qp", "segment_ts": "meI7", "time_q": "2023-02-01 09:29:32"} +{"device": "et0tVgDi4gQLBbCZ", "segment_ts": "8roi", "time_q": "2023-02-17 13:59:05"} +{"device": "KVpmUGw3YFQWFKUJ", "segment_ts": "reQ2", "time_q": "2023-02-07 05:12:56"} +{"device": "7e62NVroqaZ5KCCG", "segment_ts": "mRqe", "time_q": "2023-01-25 18:55:33"} +{"device": "2jhbxRtroaywIHz0", "segment_ts": "vGvz", "time_q": "2023-01-31 06:31:33"} +{"device": "i8A3oSUCLIRVI3z6", "segment_ts": "bRK7", "time_q": "2023-01-22 15:45:16"} +{"device": "w9m2zmvBebNOYm7M", "segment_ts": "EUTl", "time_q": "2023-02-12 17:45:44"} +{"device": "6goOWv6GmSB5SImL", "segment_ts": "IcRw", "time_q": "2023-02-02 19:44:49"} +{"device": "9TdRfZY8fyUv0MDA", "segment_ts": "jxeF", "time_q": "2023-02-15 12:46:23"} +{"device": "XFT7MaPT04gy9giN", "segment_ts": "NDT5", "time_q": "2023-01-27 02:02:09"} +{"device": "dYf2esJeNHts76qt", "segment_ts": "nD5r", "time_q": "2023-02-20 19:18:20"} +{"device": "XO3Mw07kTWxq4S6A", "segment_ts": "wu98", "time_q": "2023-02-19 09:11:04"} +{"device": "PR20MB3DrKZyxBYN", "segment_ts": "VxO2", "time_q": "2023-02-14 02:48:28"} +{"device": "8HoHoySkF3ONDVMT", "segment_ts": "RZW0", "time_q": "2023-02-17 09:47:36"} +{"device": "D9EMY6KMiUupVo86", "segment_ts": "uKvn", "time_q": "2023-02-13 20:34:37"} +{"device": "6vw27rBMy4z08u0H", "segment_ts": "IhoS", "time_q": "2023-02-07 21:19:31"} +{"device": "8B3tax9WJNOuceOO", "segment_ts": "2sCg", "time_q": "2023-02-13 06:09:00"} +{"device": "q1kmQxWdgK1fI2Aq", "segment_ts": "tpUd", "time_q": "2023-01-22 22:14:51"} +{"device": "VPe3DuOwKmbOf9hj", "segment_ts": "Q1EB", "time_q": "2023-02-07 01:49:40"} +{"device": "bcn6NL9OVcDI9NOE", "segment_ts": "pmBn", "time_q": "2023-02-06 19:24:58"} +{"device": "xveWOtVYpFmdKoE3", "segment_ts": "3pUE", "time_q": "2023-01-29 05:45:06"} +{"device": "O9p8Ij6SiXU7w9Bo", "segment_ts": "wDD7", "time_q": "2023-02-07 02:09:17"} +{"device": "JU1KI8rx8qLXtY9Y", "segment_ts": "NBSG", "time_q": "2023-02-15 13:20:17"} +{"device": "zvc3NHLWCKRbwL4w", "segment_ts": "wAqp", "time_q": "2023-02-06 19:57:45"} +{"device": "uIKOal9xGdwgmxQZ", "segment_ts": "3x0X", "time_q": "2023-02-19 00:21:42"} +{"device": "0Rn9nBxCnr1abKBu", "segment_ts": "K3q7", "time_q": "2023-02-07 03:02:30"} +{"device": "DcPFdJzT9aCgD7hZ", "segment_ts": "1WZE", "time_q": "2023-02-04 22:53:14"} +{"device": "I33U4ld8p1ZTEo2z", "segment_ts": "h327", "time_q": "2023-02-15 09:02:02"} +{"device": "HzV2F1u15P6MfTq4", "segment_ts": "Oi4K", "time_q": "2023-02-18 03:07:18"} +{"device": "Gs8bQMX2wl9AjM5I", "segment_ts": "CT5V", "time_q": "2023-02-16 01:45:58"} +{"device": "aPv5uNdC2x9Zrs31", "segment_ts": "ijS2", "time_q": "2023-01-29 20:43:50"} +{"device": "gmBMopHkBCbcGRaS", "segment_ts": "nD5r", "time_q": "2023-02-11 05:05:58"} +{"device": "g3Ns2caZMQVlOSPr", "segment_ts": "9vzm", "time_q": "2023-01-27 00:22:33"} +{"device": "JWY6hPLwnhy76KGZ", "segment_ts": "2IS2", "time_q": "2023-02-11 21:47:10"} +{"device": "0QKtSTqJYXMZWvVe", "segment_ts": "7R83", "time_q": "2023-02-17 05:40:55"} +{"device": "Ximq2ebfKR1eqySD", "segment_ts": "Jpup", "time_q": "2023-01-23 17:34:53"} +{"device": "YWGLIrHDKwe6UUM1", "segment_ts": "Smbg", "time_q": "2023-02-11 22:29:38"} +{"device": "i8Hy1Hhy6vbQ7Gnj", "segment_ts": "Xjn9", "time_q": "2023-02-07 05:52:25"} +{"device": "trQjczSWWWqD3xip", "segment_ts": "zSyE", "time_q": "2023-02-14 03:51:05"} +{"device": "cRAvzL88YU74zOVR", "segment_ts": "FAk3", "time_q": "2023-02-02 06:26:40"} +{"device": "H7RUb4moxDU5RY8L", "segment_ts": "6sUs", "time_q": "2023-02-06 15:29:02"} +{"device": "g6uZFROq6hA9VwXU", "segment_ts": "9v3C", "time_q": "2023-01-30 05:29:57"} +{"device": "yIGidxsJbbyaTMJu", "segment_ts": "5TbD", "time_q": "2023-01-31 20:18:06"} +{"device": "DC9B8r6fLUrQEKWd", "segment_ts": "VDz6", "time_q": "2023-02-09 22:19:07"} +{"device": "rymp2l5MJvVdAEiU", "segment_ts": "fyte", "time_q": "2023-02-02 11:53:13"} +{"device": "Ua1gF72JZ4WAc8Bd", "segment_ts": "v5sp", "time_q": "2023-01-24 02:06:04"} +{"device": "9qSO5sST0XGp5jMS", "segment_ts": "7dBw", "time_q": "2023-02-10 09:35:42"} +{"device": "p0HAiSpiSNAaCrZB", "segment_ts": "a1iQ", "time_q": "2023-01-31 15:45:28"} +{"device": "wkYtecM7PdltlJ6U", "segment_ts": "tCye", "time_q": "2023-01-29 14:50:59"} +{"device": "efTNUHAxzw90hiOb", "segment_ts": "9Wf1", "time_q": "2023-02-18 02:29:02"} +{"device": "fQI3EBaehVoTHSuQ", "segment_ts": "zRBT", "time_q": "2023-02-11 19:06:37"} +{"device": "udeapVgb384YhUvm", "segment_ts": "oL67", "time_q": "2023-02-05 10:27:36"} +{"device": "eqbom6mcHS60rV4o", "segment_ts": "GNdZ", "time_q": "2023-02-11 08:44:18"} +{"device": "1wWPwlCsbs9ZBcZC", "segment_ts": "zaKC", "time_q": "2023-02-08 00:49:32"} +{"device": "ecY2Gsp5o5y8RDEy", "segment_ts": "QWzq", "time_q": "2023-01-29 02:43:04"} +{"device": "DxqHMz1dx4Bqsobg", "segment_ts": "2RQv", "time_q": "2023-02-18 21:50:04"} +{"device": "XU1b7NOcHSuGzCUv", "segment_ts": "Yuoy", "time_q": "2023-02-13 21:54:55"} +{"device": "lO2wwnU1eJPm8GXQ", "segment_ts": "AP3b", "time_q": "2023-01-28 14:02:06"} +{"device": "tLnTsvtbSVqjrYe9", "segment_ts": "fFTT", "time_q": "2023-01-24 23:23:30"} +{"device": "BD3RZkRDscI24QIW", "segment_ts": "3Bry", "time_q": "2023-02-14 17:16:33"} +{"device": "EJTuSaV3BN56Wj2O", "segment_ts": "W4JQ", "time_q": "2023-02-01 18:56:00"} +{"device": "TfinqcstHOBc9vgU", "segment_ts": "VybU", "time_q": "2023-01-30 13:41:42"} +{"device": "oLCaL36jSe58AJzI", "segment_ts": "28pM", "time_q": "2023-02-03 07:47:20"} +{"device": "I85aiLVze3vm3TkQ", "segment_ts": "y8y1", "time_q": "2023-02-19 19:12:25"} +{"device": "J1Z6A5uaqDbcJCfB", "segment_ts": "Wzwh", "time_q": "2023-02-10 08:51:06"} +{"device": "DUJWSG7TcQrG1SiU", "segment_ts": "qKgC", "time_q": "2023-02-18 22:00:14"} +{"device": "oq4StgCLUoilIYHQ", "segment_ts": "MNLg", "time_q": "2023-02-05 10:11:01"} +{"device": "P44lydZpIGjbmQOy", "segment_ts": "uHCR", "time_q": "2023-01-22 21:38:00"} +{"device": "8lV1uEL9zdwqOuwT", "segment_ts": "8wB6", "time_q": "2023-02-05 21:52:08"} +{"device": "uVGi6rqE7NKl7D0W", "segment_ts": "kge3", "time_q": "2023-02-12 14:45:28"} +{"device": "fBDDRmw5ifz9Ibk1", "segment_ts": "e8wv", "time_q": "2023-02-12 01:36:43"} +{"device": "q8q2AXRsgPrtU7MF", "segment_ts": "lznG", "time_q": "2023-01-31 09:08:30"} +{"device": "rKVwah03kvEN1Xf8", "segment_ts": "CIf7", "time_q": "2023-02-17 15:32:01"} +{"device": "5gqkkJu0HnW8e7Jd", "segment_ts": "11VE", "time_q": "2023-02-21 12:50:18"} +{"device": "67aeTL1Lhk7zPniw", "segment_ts": "7WnQ", "time_q": "2023-02-15 09:54:03"} +{"device": "JSqkbIUNs9XsDhyf", "segment_ts": "Vef3", "time_q": "2023-01-28 05:40:03"} +{"device": "O6pY363lkXtNFnPW", "segment_ts": "JPG3", "time_q": "2023-02-09 07:18:08"} +{"device": "tpldKtJXhNOJs5is", "segment_ts": "x40J", "time_q": "2023-02-19 03:15:47"} +{"device": "zJFXIMKFyyMjvHzJ", "segment_ts": "ddlX", "time_q": "2023-02-12 02:18:47"} +{"device": "o0W2DatNPhPHeUCG", "segment_ts": "3Lzx", "time_q": "2023-02-01 03:05:37"} +{"device": "0qNRr7JlwEJo0UZs", "segment_ts": "x1R8", "time_q": "2023-02-04 16:06:55"} +{"device": "ybv2vBmnJIMXDQ7C", "segment_ts": "DyAY", "time_q": "2023-02-04 14:33:24"} +{"device": "7bkYnhDEWrdlKPzi", "segment_ts": "CHYf", "time_q": "2023-02-18 13:52:19"} +{"device": "gS5FFLJPIgSpPn2p", "segment_ts": "73ea", "time_q": "2023-02-20 13:38:30"} +{"device": "Iz5J7QbGpOaw9Mc3", "segment_ts": "UD8U", "time_q": "2023-02-19 21:56:03"} +{"device": "Q4zAccg3hJyiWPQV", "segment_ts": "WbDH", "time_q": "2023-02-19 21:57:53"} +{"device": "vwowDnNJnQ3Noych", "segment_ts": "YyvE", "time_q": "2023-02-07 21:35:53"} +{"device": "V4Yg3xtwNAdXT7AC", "segment_ts": "OQ1f", "time_q": "2023-01-27 03:41:03"} +{"device": "kBcU57h5YJlps0M7", "segment_ts": "X9xH", "time_q": "2023-02-16 21:18:05"} +{"device": "mTOckeiB1Myp9PzA", "segment_ts": "wDD7", "time_q": "2023-02-16 06:44:48"} +{"device": "jFqBXPPMyJJaM4VG", "segment_ts": "csTs", "time_q": "2023-02-06 02:52:21"} +{"device": "vKNH7K6ozve7702k", "segment_ts": "SJ25", "time_q": "2023-02-08 01:27:29"} +{"device": "oyT2TtgfCba2Hdml", "segment_ts": "Tvqb", "time_q": "2023-02-21 03:13:30"} +{"device": "rsOlS9C8eEy347rZ", "segment_ts": "hr7p", "time_q": "2023-02-19 02:30:46"} +{"device": "90GjHMWLi1dcpp0y", "segment_ts": "W4JQ", "time_q": "2023-01-25 02:29:48"} +{"device": "fvudPtjiPgFEJ7bD", "segment_ts": "lwno", "time_q": "2023-01-24 05:44:13"} +{"device": "VBWAgZvfVhht5RLf", "segment_ts": "Ftw2", "time_q": "2023-01-27 13:08:57"} +{"device": "sZK0cJqcYBnZXwyj", "segment_ts": "9G1V", "time_q": "2023-01-27 21:13:09"} +{"device": "Nw2Qbh1azig2aS8i", "segment_ts": "VPUK", "time_q": "2023-02-20 06:55:48"} +{"device": "YrmDmrxR6yVVsmBK", "segment_ts": "kEmM", "time_q": "2023-02-17 04:13:52"} +{"device": "FJg8emti6uDPkZWb", "segment_ts": "cu9S", "time_q": "2023-01-31 15:15:17"} +{"device": "D2KVrcXONfBL0DOq", "segment_ts": "E7q4", "time_q": "2023-02-07 05:21:14"} +{"device": "zpjzEqsoIDSfgkrw", "segment_ts": "kuSy", "time_q": "2023-02-08 21:39:35"} +{"device": "TyBhOHhPHg3mABfs", "segment_ts": "FOLC", "time_q": "2023-02-05 01:50:35"} +{"device": "SSsebFMnz2vz0lKT", "segment_ts": "qXUp", "time_q": "2023-02-17 15:12:43"} +{"device": "oeMORvnHFqNvuqA9", "segment_ts": "D16L", "time_q": "2023-02-17 12:56:55"} +{"device": "pZGE5Q8ax0QnrKlK", "segment_ts": "6O2l", "time_q": "2023-02-17 06:43:06"} +{"device": "xXUDPhByePkaTvpe", "segment_ts": "UqWC", "time_q": "2023-02-05 01:26:33"} +{"device": "8zOXbhQQFeXPuMgW", "segment_ts": "v5s5", "time_q": "2023-01-28 21:54:59"} +{"device": "Kvd0ii9qinTBgkEK", "segment_ts": "YWRA", "time_q": "2023-02-15 04:02:15"} +{"device": "JoC3FaiJFH7nWQbY", "segment_ts": "RV1o", "time_q": "2023-01-23 11:40:04"} +{"device": "JZtyjzapfAhKTdEL", "segment_ts": "6Hyt", "time_q": "2023-02-12 23:31:41"} +{"device": "lsE2LnvCTkfUv3Hb", "segment_ts": "mvNJ", "time_q": "2023-02-20 04:17:14"} +{"device": "ZaG5u6BctGV7Phak", "segment_ts": "xS0C", "time_q": "2023-02-12 21:40:07"} +{"device": "VqkvcCFemWmmdLIN", "segment_ts": "1vCq", "time_q": "2023-02-04 13:43:54"} +{"device": "MnWYYlFI787w7ENs", "segment_ts": "YHny", "time_q": "2023-02-02 04:12:38"} +{"device": "5dFd1cLHS8fjZaOz", "segment_ts": "0JSK", "time_q": "2023-02-14 05:41:43"} +{"device": "G7r2BebALuPczt3b", "segment_ts": "nowH", "time_q": "2023-02-19 08:55:48"} +{"device": "7V6SBGSNrMZUvrw8", "segment_ts": "KKuH", "time_q": "2023-02-07 03:00:14"} +{"device": "YWmt4ZWofrE68Eg4", "segment_ts": "9ub1", "time_q": "2023-01-24 10:32:07"} +{"device": "RgsF3Paw1CFHmXrz", "segment_ts": "GKFG", "time_q": "2023-02-05 14:23:12"} +{"device": "uRUd8DnEptcCg051", "segment_ts": "w6bQ", "time_q": "2023-02-12 02:55:44"} +{"device": "gpJn2OhNJ17yFKfj", "segment_ts": "Ms6E", "time_q": "2023-02-11 12:28:14"} +{"device": "OS8p3UgcZwCtitJ5", "segment_ts": "lUE6", "time_q": "2023-02-12 12:43:29"} +{"device": "AaBf2goUzSIOLwKU", "segment_ts": "jRhp", "time_q": "2023-01-28 02:43:57"} +{"device": "Gi7o9G6zuxIje6TA", "segment_ts": "TPjM", "time_q": "2023-01-22 21:42:14"} +{"device": "Zgp8HrnLjtoUBstx", "segment_ts": "669s", "time_q": "2023-01-23 07:40:22"} +{"device": "yZ39wYgza1sjjH6a", "segment_ts": "80La", "time_q": "2023-01-25 23:58:17"} +{"device": "gKSePVA4eUpPeQeB", "segment_ts": "uPAl", "time_q": "2023-02-06 04:22:11"} +{"device": "ZvPnZt45ZdDJf2wV", "segment_ts": "eQg8", "time_q": "2023-01-29 16:01:24"} +{"device": "RGaReXrDxjLvIN1h", "segment_ts": "wV8N", "time_q": "2023-01-27 01:38:59"} +{"device": "DBG2BbMzd2AiJ1ra", "segment_ts": "or1s", "time_q": "2023-02-14 19:21:12"} +{"device": "LaRpvrLjCem5I1iG", "segment_ts": "MKyg", "time_q": "2023-01-22 20:08:37"} +{"device": "6VPMKOpB2Gax57AM", "segment_ts": "jWnU", "time_q": "2023-02-02 08:27:23"} +{"device": "WWmDiCgEhcFaRoxd", "segment_ts": "IcRw", "time_q": "2023-01-28 20:36:01"} +{"device": "g8tKyCuvwcmnOleP", "segment_ts": "l57C", "time_q": "2023-02-21 12:15:51"} +{"device": "mGmZoqlgLWwRHhWd", "segment_ts": "MJWQ", "time_q": "2023-01-26 00:54:26"} +{"device": "9vYzND4LhGIpLRPs", "segment_ts": "X8An", "time_q": "2023-02-17 17:12:20"} +{"device": "TqAci7xzsz9HKRwX", "segment_ts": "HQvb", "time_q": "2023-02-07 19:26:19"} +{"device": "jXeyesYOxpErf0Wr", "segment_ts": "9Pah", "time_q": "2023-01-29 07:43:44"} +{"device": "PKpsammUcBhHIj4d", "segment_ts": "SJ96", "time_q": "2023-02-01 11:22:01"} +{"device": "d43Y87nTdkx4nw0N", "segment_ts": "qt7V", "time_q": "2023-02-13 07:35:55"} +{"device": "3wSdpuLVAqb2DqwQ", "segment_ts": "1uul", "time_q": "2023-02-20 17:20:48"} +{"device": "b86HMaTJskEFSegv", "segment_ts": "e0pl", "time_q": "2023-02-20 04:23:59"} +{"device": "q7XDusNiRySmmaRj", "segment_ts": "7dBw", "time_q": "2023-01-29 01:47:01"} +{"device": "LrWDgsZruGK3akSe", "segment_ts": "0vLc", "time_q": "2023-02-20 13:19:00"} +{"device": "JJOo2fsdKEG6QCRu", "segment_ts": "LN5W", "time_q": "2023-02-12 01:36:02"} +{"device": "iXc2nZTT1U0iWf2W", "segment_ts": "3ftI", "time_q": "2023-02-21 10:12:08"} +{"device": "YJ4TjVpH6ZQhMvrH", "segment_ts": "IVww", "time_q": "2023-01-24 12:14:46"} +{"device": "4cBW0bSimlW8ygs0", "segment_ts": "tGGE", "time_q": "2023-02-13 11:54:22"} +{"device": "Ye2BuE77YVbChRNA", "segment_ts": "SRVM", "time_q": "2023-01-25 16:55:03"} +{"device": "BD4sQkvrmTWcfgdd", "segment_ts": "uqv1", "time_q": "2023-02-17 14:21:23"} +{"device": "kB9K8PKJpQnkJTcd", "segment_ts": "WCit", "time_q": "2023-02-20 20:54:19"} +{"device": "uSAbl2Nyu6nK3ddO", "segment_ts": "0cq0", "time_q": "2023-01-28 17:04:49"} +{"device": "uJEVeGx0jSYrxfsx", "segment_ts": "GL95", "time_q": "2023-02-19 21:43:48"} +{"device": "wowb2b399vUqVxWH", "segment_ts": "YcDM", "time_q": "2023-02-03 10:03:34"} +{"device": "Yt42EPAk8TRsKsha", "segment_ts": "9vzm", "time_q": "2023-02-06 16:06:01"} +{"device": "nuQgvAgPYUF1ML4s", "segment_ts": "SOUQ", "time_q": "2023-02-02 10:23:33"} +{"device": "LEuEmNLEgjoCu3p1", "segment_ts": "Mhpw", "time_q": "2023-01-31 19:35:49"} +{"device": "4H1WoLSdS2qs5jOM", "segment_ts": "II21", "time_q": "2023-02-10 14:24:09"} +{"device": "dN5oTZXijbMTz8OP", "segment_ts": "NOvo", "time_q": "2023-01-27 11:27:12"} +{"device": "i2ITUxoWBCEdW8Bs", "segment_ts": "FqD3", "time_q": "2023-01-30 06:33:32"} +{"device": "NyZi9ibptktAZTjI", "segment_ts": "C24z", "time_q": "2023-02-13 18:48:37"} +{"device": "5MMCow4pOwRmSIlc", "segment_ts": "NuYD", "time_q": "2023-02-03 15:50:35"} +{"device": "lK52wLIAUCcXP4AH", "segment_ts": "geC5", "time_q": "2023-02-14 07:33:08"} +{"device": "kdEACkrL59EG8vBX", "segment_ts": "44WL", "time_q": "2023-02-15 16:46:30"} +{"device": "BZhrV8EAlPBft7XX", "segment_ts": "w04k", "time_q": "2023-02-16 03:47:55"} +{"device": "8mKdbrORBbygBkPb", "segment_ts": "3x0X", "time_q": "2023-01-27 04:58:59"} +{"device": "C3ue47cXeXukj7rU", "segment_ts": "7AXG", "time_q": "2023-02-14 15:59:38"} +{"device": "PFZtQxNxLhvhEuil", "segment_ts": "mCKg", "time_q": "2023-02-12 16:29:55"} +{"device": "Qfa7tEckTSssilkU", "segment_ts": "3Epz", "time_q": "2023-01-30 04:49:42"} +{"device": "OZGXqiDoUPo7codg", "segment_ts": "k234", "time_q": "2023-02-21 02:14:45"} +{"device": "VcXbfEsURAyiOlXz", "segment_ts": "XXHG", "time_q": "2023-02-06 21:09:36"} +{"device": "d9jY3TNnU9v39DOE", "segment_ts": "JBAB", "time_q": "2023-02-13 05:44:52"} +{"device": "bY1aO1ktexSWKAjB", "segment_ts": "OUWG", "time_q": "2023-02-19 16:34:05"} +{"device": "4J0KbbaKXbT17iAy", "segment_ts": "THLZ", "time_q": "2023-02-19 16:01:05"} +{"device": "TdfnSWLdZuoHDmT4", "segment_ts": "9bQC", "time_q": "2023-02-13 07:47:03"} +{"device": "vKvuSLFrmEElACFv", "segment_ts": "WgTc", "time_q": "2023-02-01 05:41:50"} +{"device": "EgUuFOflBQ4zUJz2", "segment_ts": "dfJl", "time_q": "2023-02-11 07:30:24"} +{"device": "3cWfkAf3Ma8T8KkO", "segment_ts": "PDJS", "time_q": "2023-02-05 03:09:15"} +{"device": "xoCum1PFgRMFFAtd", "segment_ts": "bdMN", "time_q": "2023-02-10 21:10:02"} +{"device": "G10WOxXP7vDSrqG2", "segment_ts": "9vHF", "time_q": "2023-01-25 10:44:21"} +{"device": "GyDiESn1pQ7C2eOq", "segment_ts": "X5WH", "time_q": "2023-02-10 05:32:32"} +{"device": "Ihum52kryw40U2Pq", "segment_ts": "YQGo", "time_q": "2023-01-29 03:27:14"} +{"device": "9DeVCz49iKTNFPyr", "segment_ts": "2rzd", "time_q": "2023-01-24 10:45:49"} +{"device": "7mW6uwngb7RD2lEp", "segment_ts": "5jBK", "time_q": "2023-02-11 22:13:48"} +{"device": "xuW4yuuFgbdd4a6p", "segment_ts": "lkBB", "time_q": "2023-01-31 18:54:23"} +{"device": "5kzp9NXHYWNPgYoh", "segment_ts": "fioj", "time_q": "2023-01-26 03:08:34"} +{"device": "gmWHoGjwiWF82jHM", "segment_ts": "kiO6", "time_q": "2023-01-22 18:57:36"} +{"device": "zFyepAAJnlCoeqEU", "segment_ts": "tJnv", "time_q": "2023-02-01 00:26:55"} +{"device": "EglsUmauQATFBwu6", "segment_ts": "fGsr", "time_q": "2023-01-30 09:02:11"} +{"device": "qUd53Kl4mymteZxS", "segment_ts": "TdgJ", "time_q": "2023-01-27 05:11:33"} +{"device": "F5zBVO0bkWwMjrQ3", "segment_ts": "yEsz", "time_q": "2023-02-17 00:44:53"} +{"device": "HDgL5w5o1AyW8KNq", "segment_ts": "WMDV", "time_q": "2023-02-20 04:25:58"} +{"device": "b080loiW7g8hJYZN", "segment_ts": "s8vj", "time_q": "2023-01-26 23:15:47"} +{"device": "PLB1THGiEIKtVnuq", "segment_ts": "eqHY", "time_q": "2023-02-17 07:42:59"} +{"device": "Y90nqduLp4v38lVD", "segment_ts": "hZVY", "time_q": "2023-02-15 14:26:35"} +{"device": "fh35hFi6oIEaA5Df", "segment_ts": "TJ6P", "time_q": "2023-01-28 02:34:05"} +{"device": "RBLsLZ4eIt3nrLPH", "segment_ts": "7WHK", "time_q": "2023-01-29 20:05:29"} +{"device": "9Db2XAc0xDm9Mbmx", "segment_ts": "ls7g", "time_q": "2023-02-18 08:15:15"} +{"device": "M1Ih4aY0dYWKWegA", "segment_ts": "sgfr", "time_q": "2023-02-04 00:43:41"} +{"device": "0YtmJi8QdEGnWaHz", "segment_ts": "5HQX", "time_q": "2023-01-25 06:09:08"} +{"device": "LjuzkroRYOCdQSz5", "segment_ts": "dzph", "time_q": "2023-01-24 00:27:56"} +{"device": "K0y5JORnnCPqA3PL", "segment_ts": "6jsI", "time_q": "2023-02-06 10:18:08"} +{"device": "75xJYP1muHt9LVvt", "segment_ts": "2PcN", "time_q": "2023-01-25 23:49:07"} +{"device": "aa15VzCz3SZrbBAl", "segment_ts": "FfVa", "time_q": "2023-02-07 04:37:48"} +{"device": "65lf7rvFyN74HbTL", "segment_ts": "q9o4", "time_q": "2023-02-10 16:57:52"} +{"device": "BkATNfc6INXfprd4", "segment_ts": "QW7U", "time_q": "2023-02-18 04:08:48"} +{"device": "HSBPAi87vKtuhabY", "segment_ts": "Ft0J", "time_q": "2023-01-29 04:20:36"} +{"device": "Rb23K4sxSmDFCiK2", "segment_ts": "CZHg", "time_q": "2023-02-04 22:53:44"} +{"device": "VzfrrwW2GG8Au619", "segment_ts": "qmhZ", "time_q": "2023-01-29 07:18:18"} +{"device": "UW5oYzVVoP9VUNvi", "segment_ts": "FD5Q", "time_q": "2023-02-13 03:55:02"} +{"device": "7ZqKm1mzPqTjKAK8", "segment_ts": "yAxf", "time_q": "2023-02-03 08:04:47"} +{"device": "RKs79WJe0pmc30Ji", "segment_ts": "iQ7P", "time_q": "2023-02-14 20:17:09"} +{"device": "fawEL3nd7h4bsCSA", "segment_ts": "5JQi", "time_q": "2023-02-09 09:47:37"} +{"device": "1cK5y0fv2oasCsEg", "segment_ts": "FZfm", "time_q": "2023-01-24 23:15:26"} +{"device": "fZlV1wQfgN0slT78", "segment_ts": "QjWp", "time_q": "2023-01-23 18:30:49"} +{"device": "ReYnQoZKQD7FhiPY", "segment_ts": "uTjn", "time_q": "2023-01-25 01:24:16"} +{"device": "NPuU31MUQXT5txEw", "segment_ts": "clSh", "time_q": "2023-02-04 23:56:25"} +{"device": "uu9ToklvVI6lAa9s", "segment_ts": "9v3C", "time_q": "2023-01-23 19:15:43"} +{"device": "iVXaaRjEBBkz0hz3", "segment_ts": "aV1U", "time_q": "2023-02-05 20:11:12"} +{"device": "xM0S07A9cCsd9Yfn", "segment_ts": "MeDE", "time_q": "2023-01-24 04:13:01"} +{"device": "nQHMNXhbyFZ6UoDn", "segment_ts": "K4Ze", "time_q": "2023-02-10 16:22:30"} +{"device": "fXaNQOwz30KmlHrP", "segment_ts": "36ap", "time_q": "2023-02-14 12:08:13"} +{"device": "dw2ya1E1U6tjsd7T", "segment_ts": "Jb8U", "time_q": "2023-02-14 03:53:01"} +{"device": "GZ0h5bYPi6E6ohn4", "segment_ts": "44WL", "time_q": "2023-01-29 13:46:23"} +{"device": "wNZt7rqhrv3YtKuz", "segment_ts": "zaKC", "time_q": "2023-01-25 03:03:32"} +{"device": "RpROmDVcSKPVt0C8", "segment_ts": "lvES", "time_q": "2023-02-17 21:12:47"} +{"device": "jbZJ7XFo3s3aLnjJ", "segment_ts": "cvD6", "time_q": "2023-02-17 14:55:37"} +{"device": "4Qw6ei7d9Ux7cTDd", "segment_ts": "gwio", "time_q": "2023-02-01 08:26:39"} +{"device": "JLYhTM9PRqiptyZD", "segment_ts": "QrPy", "time_q": "2023-01-26 09:58:00"} +{"device": "6FO0vI0i8EqKQhd6", "segment_ts": "hH9B", "time_q": "2023-01-31 23:43:46"} +{"device": "iGeoTrx9INiZGRHX", "segment_ts": "X2bk", "time_q": "2023-01-30 16:07:24"} +{"device": "5pRaiBzTt31JZkve", "segment_ts": "8Auu", "time_q": "2023-02-05 09:10:17"} +{"device": "OvWvtocpkdNLqrny", "segment_ts": "Inti", "time_q": "2023-02-12 21:28:35"} +{"device": "bYwqcLrv6vNbGiov", "segment_ts": "ysSv", "time_q": "2023-02-14 22:07:17"} +{"device": "o9lqa5RcsnYdxzPx", "segment_ts": "R8bW", "time_q": "2023-01-31 19:56:44"} +{"device": "WmUqr1hnmvlRw6YO", "segment_ts": "5viW", "time_q": "2023-01-28 20:23:04"} +{"device": "JhzC3ZigvlLK1seo", "segment_ts": "J5sY", "time_q": "2023-02-12 20:59:49"} +{"device": "evKPQwjRV6cBQvjy", "segment_ts": "dDZY", "time_q": "2023-01-31 04:46:11"} +{"device": "sLMrZiBFi2lonYwg", "segment_ts": "unUy", "time_q": "2023-02-15 08:51:34"} +{"device": "o4lT6snRzZnQa1HO", "segment_ts": "Z7Ft", "time_q": "2023-02-03 21:15:32"} +{"device": "hqZWBbZog5N1kD9V", "segment_ts": "WBtx", "time_q": "2023-02-19 18:01:06"} +{"device": "xXzLPLm0OsnDJwUz", "segment_ts": "K1Yr", "time_q": "2023-01-28 23:13:12"} +{"device": "KXSUj8l7UVFARe7n", "segment_ts": "BvbW", "time_q": "2023-01-29 21:44:04"} +{"device": "UvZA7BW1fWlt0GbE", "segment_ts": "ajjJ", "time_q": "2023-01-24 03:31:50"} +{"device": "cQGXvK61LboETbwo", "segment_ts": "Ey3g", "time_q": "2023-01-23 21:46:37"} +{"device": "CQ101gI3Mx4eORJw", "segment_ts": "Jb8U", "time_q": "2023-01-28 03:33:22"} +{"device": "ed2KkpZOCurKwUW1", "segment_ts": "ZWmw", "time_q": "2023-02-15 06:18:12"} +{"device": "cmsyEAqBqINzHOdc", "segment_ts": "MXoi", "time_q": "2023-02-10 07:11:49"} +{"device": "HrEkRR54fFMG0LDI", "segment_ts": "siac", "time_q": "2023-02-07 23:36:13"} +{"device": "wxGldUWtgrjazC1G", "segment_ts": "tvS0", "time_q": "2023-02-04 05:46:44"} +{"device": "RxPCG6v5f4sqllhf", "segment_ts": "Ysq9", "time_q": "2023-02-12 12:54:14"} +{"device": "r7xQr423rEko5eka", "segment_ts": "XPPE", "time_q": "2023-02-07 03:44:00"} +{"device": "wxuOz2ANcqk4RoaO", "segment_ts": "lDol", "time_q": "2023-02-16 15:04:43"} +{"device": "PlSBAhofMJFpB2uV", "segment_ts": "RFaV", "time_q": "2023-02-11 22:02:11"} +{"device": "hZ1tbXscywB9ibyA", "segment_ts": "xnZO", "time_q": "2023-02-20 15:51:50"} +{"device": "KbFXsiA8Q0SqfSF2", "segment_ts": "glBE", "time_q": "2023-02-18 23:04:11"} +{"device": "MRPK2r2JhCir2u2l", "segment_ts": "Med8", "time_q": "2023-01-24 09:27:31"} +{"device": "cEFmg5V2xJHPyqp5", "segment_ts": "Kd02", "time_q": "2023-02-10 09:41:59"} +{"device": "1lwjNCCV5WEJzXT7", "segment_ts": "5CQq", "time_q": "2023-01-27 01:07:21"} +{"device": "VvOulNjY9xQCblv7", "segment_ts": "C8F7", "time_q": "2023-01-27 21:04:34"} +{"device": "dF1H0PGvmSbpRyiI", "segment_ts": "eD44", "time_q": "2023-01-31 13:38:45"} +{"device": "NkDgwYVx0XveN5Dt", "segment_ts": "WxCz", "time_q": "2023-02-21 07:16:46"} +{"device": "LBsuQUNvWKpyd89V", "segment_ts": "V9Fm", "time_q": "2023-02-10 12:25:41"} +{"device": "n1KoKQk8oZ8lUPtV", "segment_ts": "J7YR", "time_q": "2023-02-16 05:30:48"} +{"device": "YDqHeGZquVwEjqQc", "segment_ts": "2GHM", "time_q": "2023-02-15 00:35:09"} +{"device": "RgQGcEnSZcPxClfB", "segment_ts": "hvML", "time_q": "2023-01-23 08:49:45"} +{"device": "jSvUQlxGgJWX0n7M", "segment_ts": "HMSz", "time_q": "2023-02-12 06:13:19"} +{"device": "K7lIlpxjv71aoW9n", "segment_ts": "rkkj", "time_q": "2023-02-02 15:11:54"} +{"device": "tlcCO6MF2RYl0g96", "segment_ts": "10GK", "time_q": "2023-02-17 18:09:56"} +{"device": "p4o8sJpApQwJOiox", "segment_ts": "XAUS", "time_q": "2023-01-29 10:34:46"} +{"device": "525KZ78kEncXH9VI", "segment_ts": "ugYO", "time_q": "2023-02-03 20:14:10"} +{"device": "7arB9WqfPZVvkzAs", "segment_ts": "Jydo", "time_q": "2023-02-09 13:03:19"} +{"device": "GsqKaz0OuOy0p8D5", "segment_ts": "s3Xs", "time_q": "2023-02-01 17:04:58"} +{"device": "NJRTnXytfyhbSltj", "segment_ts": "pa4b", "time_q": "2023-01-22 23:47:33"} +{"device": "V6BcYDFUslTaeKz2", "segment_ts": "c3ip", "time_q": "2023-01-24 12:24:16"} +{"device": "Lali4GnxbDXiICfd", "segment_ts": "5HQX", "time_q": "2023-02-08 07:21:08"} +{"device": "4G67CFMQFfw5nKhH", "segment_ts": "HfZ3", "time_q": "2023-01-27 13:08:26"} +{"device": "VOU1PEANNqFpksPD", "segment_ts": "UCFz", "time_q": "2023-02-02 17:55:58"} +{"device": "NQkRwi0QBjL9OUfe", "segment_ts": "peAr", "time_q": "2023-01-31 09:44:07"} +{"device": "C1Nzu0HNqUqhiv4v", "segment_ts": "ido8", "time_q": "2023-02-20 11:23:58"} +{"device": "jc76NMsTRXg6yFT8", "segment_ts": "IlVL", "time_q": "2023-02-04 08:44:26"} +{"device": "mkPyfZXT9VoyITp1", "segment_ts": "AElH", "time_q": "2023-02-01 22:49:56"} +{"device": "aODXvK25bBgLB89w", "segment_ts": "UyMT", "time_q": "2023-02-06 18:46:10"} +{"device": "bIRJnUdmTWC4bQjs", "segment_ts": "q6P2", "time_q": "2023-01-25 10:01:19"} +{"device": "NwthNY2x1PiG4NXu", "segment_ts": "ziKj", "time_q": "2023-01-26 00:26:01"} +{"device": "Y0Jwgj0CtaaVPGLU", "segment_ts": "uRmO", "time_q": "2023-02-13 01:29:07"} +{"device": "P59OfLIdsNz1u2bf", "segment_ts": "hm6H", "time_q": "2023-01-25 21:43:28"} +{"device": "dHwliEj94uGOjy4z", "segment_ts": "bgF8", "time_q": "2023-02-16 14:27:07"} +{"device": "Tp85tKVvwtOGxCsN", "segment_ts": "z871", "time_q": "2023-02-03 03:24:04"} +{"device": "Nu5VHjVpGjotVjHh", "segment_ts": "u8t8", "time_q": "2023-02-14 04:32:01"} +{"device": "rWcINdZOiu9BHzkL", "segment_ts": "CVvt", "time_q": "2023-02-07 13:15:38"} +{"device": "yGILVrj3iinOAS4U", "segment_ts": "0fQb", "time_q": "2023-02-01 08:06:25"} +{"device": "CkVe5GZWBIuoCDh9", "segment_ts": "0JA6", "time_q": "2023-02-14 22:41:17"} +{"device": "gGG9uQjtROdibp0O", "segment_ts": "mwXi", "time_q": "2023-02-18 08:50:46"} +{"device": "GljeFNPy2bmdxmEz", "segment_ts": "7Jts", "time_q": "2023-02-06 12:45:43"} +{"device": "3xlhVhGTJmURk3MX", "segment_ts": "NZxR", "time_q": "2023-02-02 14:51:14"} +{"device": "47ujFWpbmPFKdX1g", "segment_ts": "KNqJ", "time_q": "2023-01-25 06:27:30"} +{"device": "gNDkMVt00vJT84jK", "segment_ts": "WX3F", "time_q": "2023-02-09 12:57:48"} +{"device": "LCaDtdl6EqAmPlGU", "segment_ts": "AVAC", "time_q": "2023-02-14 13:38:47"} +{"device": "RMS3UOMf4mt5Nlfx", "segment_ts": "dDQ0", "time_q": "2023-02-15 20:39:12"} +{"device": "mCCyHPx2EyQkjDI6", "segment_ts": "mvNJ", "time_q": "2023-02-13 15:16:22"} +{"device": "TWoLnaxEPgoNH3Fk", "segment_ts": "bwdO", "time_q": "2023-02-11 09:10:07"} +{"device": "2U0w1oZWqt42ge69", "segment_ts": "OQPD", "time_q": "2023-02-08 05:38:25"} +{"device": "Wr1pqW7ogHm1l57C", "segment_ts": "yEsz", "time_q": "2023-02-19 15:25:54"} +{"device": "X2x7ETjtNN7XU6wE", "segment_ts": "R4fq", "time_q": "2023-02-09 09:30:45"} +{"device": "JJsmrj8eGnKEpkgT", "segment_ts": "jHFs", "time_q": "2023-01-28 19:05:13"} +{"device": "jQOGpX4X5gohJjgf", "segment_ts": "NDT5", "time_q": "2023-01-28 07:12:42"} +{"device": "CktBzqEuXCvd6ThV", "segment_ts": "ItFE", "time_q": "2023-01-24 21:16:41"} +{"device": "o9JPJauPfRraIlLu", "segment_ts": "76j0", "time_q": "2023-01-23 20:11:55"} +{"device": "QdHyZk5P5vaBYTak", "segment_ts": "mfIG", "time_q": "2023-02-08 05:08:49"} +{"device": "u4QrAiPlcTouwbT5", "segment_ts": "YZHZ", "time_q": "2023-02-14 11:46:14"} +{"device": "65dIzY2Z9TJWgbhv", "segment_ts": "EqdZ", "time_q": "2023-01-28 12:55:23"} +{"device": "kT4R6IGOTSqzw7uB", "segment_ts": "SGHa", "time_q": "2023-02-11 03:18:04"} +{"device": "66KWZA8L7G89WDZI", "segment_ts": "wTC5", "time_q": "2023-02-21 14:23:50"} +{"device": "LRxsGXs41oqHtRqM", "segment_ts": "u7El", "time_q": "2023-02-09 14:03:09"} +{"device": "hgyESiq7XGtiQCql", "segment_ts": "hxmr", "time_q": "2023-01-26 12:29:40"} +{"device": "0qbdQklTFNy7ISCH", "segment_ts": "R9B9", "time_q": "2023-01-29 15:23:30"} +{"device": "xYAjqmLYwZPiLGsd", "segment_ts": "lIlA", "time_q": "2023-02-17 07:43:40"} +{"device": "BzgAUk3wTrBt2JWM", "segment_ts": "sfqh", "time_q": "2023-02-16 04:38:46"} +{"device": "UrSOAFfBipxV2iV0", "segment_ts": "Il3r", "time_q": "2023-02-13 10:27:30"} +{"device": "ALr9hJYdlrw6oNcJ", "segment_ts": "Keh8", "time_q": "2023-01-23 20:17:27"} +{"device": "GVDWDnyS3E5WQkFK", "segment_ts": "uW7S", "time_q": "2023-02-11 17:04:10"} +{"device": "2OKxK3JW7y96EHEa", "segment_ts": "KFml", "time_q": "2023-01-28 00:05:53"} +{"device": "FPkKren0hQTn7grY", "segment_ts": "eShQ", "time_q": "2023-02-04 04:45:29"} +{"device": "FWyRyqknGMhkVpo3", "segment_ts": "ZdoH", "time_q": "2023-02-15 23:28:49"} +{"device": "JK21TkVFQLsvKUG0", "segment_ts": "1MVW", "time_q": "2023-02-06 15:20:46"} +{"device": "cjo9bUnDXUQHlnCa", "segment_ts": "PT3c", "time_q": "2023-01-25 10:30:12"} +{"device": "2a0x5y7y1dd0WSHe", "segment_ts": "hrIh", "time_q": "2023-02-06 13:52:24"} +{"device": "k0TSwJgBwwVCZLlv", "segment_ts": "vRB7", "time_q": "2023-02-01 22:19:25"} +{"device": "j5LFK6TZErFfQ1VZ", "segment_ts": "tTr2", "time_q": "2023-01-30 20:49:48"} +{"device": "JGYeTG9SlkPY6wFa", "segment_ts": "Z9Pu", "time_q": "2023-02-06 20:31:31"} +{"device": "Ymh6ayRgp1H60AFc", "segment_ts": "Bwf8", "time_q": "2023-02-06 00:29:22"} +{"device": "Td0EpJOxCHYWUbrR", "segment_ts": "v25d", "time_q": "2023-01-28 05:25:00"} +{"device": "mS6Iy8Qa2mmoqYQu", "segment_ts": "9q3k", "time_q": "2023-01-30 00:18:20"} +{"device": "fc6guCFHdG8VzC3p", "segment_ts": "QjWp", "time_q": "2023-02-08 11:44:52"} +{"device": "ItkmsLuHy98vfLUQ", "segment_ts": "JoNb", "time_q": "2023-02-04 16:39:25"} +{"device": "zEKKWQ45k04JGcCV", "segment_ts": "Tpwm", "time_q": "2023-02-05 19:39:55"} +{"device": "JPsTfxDVBAtWfW8r", "segment_ts": "6sUs", "time_q": "2023-01-29 20:29:27"} +{"device": "yIeSaXpU3rkCbxUr", "segment_ts": "yZ7s", "time_q": "2023-02-18 01:23:16"} +{"device": "ekMZsH5A0zvUgkOr", "segment_ts": "KSlf", "time_q": "2023-01-30 07:30:21"} +{"device": "4c1W84L1nNUfHkZV", "segment_ts": "vvM0", "time_q": "2023-02-01 17:47:15"} +{"device": "RBN3tJA4BYV71FBJ", "segment_ts": "eQg8", "time_q": "2023-02-21 13:02:27"} +{"device": "9DS3pYWFruFTvJZS", "segment_ts": "rRuc", "time_q": "2023-01-25 06:18:46"} +{"device": "Oge628HPmV0XJxLE", "segment_ts": "XDA1", "time_q": "2023-02-20 09:45:25"} +{"device": "qHZcNOXzeJFDOsbi", "segment_ts": "X3q3", "time_q": "2023-02-03 12:14:04"} +{"device": "Z4TwRqUaxlM4nWox", "segment_ts": "G8xC", "time_q": "2023-01-25 21:21:40"} +{"device": "cN2qypm0e5KXaM4f", "segment_ts": "DPPO", "time_q": "2023-02-20 05:51:54"} +{"device": "G1umjEcxzSJpDeVf", "segment_ts": "ivk4", "time_q": "2023-02-04 14:30:20"} +{"device": "Y4YHyYf0eIA0dQT2", "segment_ts": "w9bo", "time_q": "2023-01-30 04:27:41"} +{"device": "sybCGj8bJcuWVkMH", "segment_ts": "n7r6", "time_q": "2023-02-10 08:55:38"} +{"device": "G9qtN4oPylocAhLo", "segment_ts": "ygKY", "time_q": "2023-02-11 15:58:38"} +{"device": "dqbOQmU1nCEM1tsb", "segment_ts": "snFa", "time_q": "2023-01-26 05:18:19"} +{"device": "PvXh8nHFVEKnfyw5", "segment_ts": "es0t", "time_q": "2023-02-03 04:33:00"} +{"device": "9evTiKiWziX3THWt", "segment_ts": "mrI5", "time_q": "2023-02-09 12:39:43"} +{"device": "8vWA3DPg0sCubmHt", "segment_ts": "VyRj", "time_q": "2023-01-30 17:09:05"} +{"device": "JBh1VD2mJSzS93VN", "segment_ts": "A2E6", "time_q": "2023-01-23 22:53:01"} +{"device": "AVTuFDv6MFEeDkkg", "segment_ts": "FQxa", "time_q": "2023-02-16 15:15:49"} +{"device": "jEnRgZZOuDplk7CK", "segment_ts": "McUm", "time_q": "2023-02-13 16:58:42"} +{"device": "w9eyvBTQMP0TlFPg", "segment_ts": "061H", "time_q": "2023-02-14 07:07:22"} +{"device": "HxBZsLSXkTOwIj7U", "segment_ts": "Yny1", "time_q": "2023-02-16 18:30:09"} +{"device": "OOp6NIdrHxTZgb1a", "segment_ts": "3gVe", "time_q": "2023-01-27 03:18:26"} +{"device": "XQeEiBgEVGX3xRyy", "segment_ts": "iK88", "time_q": "2023-01-31 16:16:42"} +{"device": "7XXbvAMS3NbVIuSb", "segment_ts": "ikv3", "time_q": "2023-01-27 17:58:57"} +{"device": "Z77uepdP4uNOMFEm", "segment_ts": "aNrW", "time_q": "2023-01-25 19:27:06"} +{"device": "x7Ru0sut9cCNKErD", "segment_ts": "fSS4", "time_q": "2023-02-16 13:17:10"} +{"device": "tfF7ARhQav7Vs1sW", "segment_ts": "yFy4", "time_q": "2023-02-18 08:18:49"} +{"device": "IezPnCOr2g6m2iq3", "segment_ts": "YOF3", "time_q": "2023-02-09 18:31:03"} +{"device": "VOA5cvCyvFZ7bnzk", "segment_ts": "Viph", "time_q": "2023-01-24 18:36:49"} +{"device": "p0On4aZWYDoxDWHI", "segment_ts": "FTU0", "time_q": "2023-02-07 02:13:54"} +{"device": "FSpOgwOaHynPz0zH", "segment_ts": "WwVR", "time_q": "2023-02-03 21:12:09"} +{"device": "PVSlKZkOPl5hFKDc", "segment_ts": "AZRz", "time_q": "2023-02-09 05:04:38"} +{"device": "hXJ4b5HgGPtZxZDG", "segment_ts": "eNW4", "time_q": "2023-02-06 20:20:25"} +{"device": "iyM6IB8Kyie9rXTk", "segment_ts": "WbDH", "time_q": "2023-01-25 02:38:32"} +{"device": "ZHDFyhQFjT36sYEX", "segment_ts": "RrPC", "time_q": "2023-01-24 13:30:11"} +{"device": "FAmkVIvvqiBv3J2v", "segment_ts": "BLc4", "time_q": "2023-02-05 01:57:00"} +{"device": "uUlaL9nusB5K9wah", "segment_ts": "6NJ4", "time_q": "2023-02-04 05:09:45"} +{"device": "uq7LIwGy8Cs8VS8S", "segment_ts": "Ksz1", "time_q": "2023-01-31 14:34:35"} +{"device": "Sljf8wt6YxMJlQ9B", "segment_ts": "3LOG", "time_q": "2023-02-08 01:47:29"} +{"device": "8tnuglv62PYooaqm", "segment_ts": "vXhF", "time_q": "2023-02-02 02:09:46"} +{"device": "B00CXFzWHZ7T6vw5", "segment_ts": "993f", "time_q": "2023-01-28 01:31:50"} +{"device": "FsKiFfXLePj7r3xn", "segment_ts": "uR2f", "time_q": "2023-02-16 05:49:48"} +{"device": "P2Pssg8wIUhJfJge", "segment_ts": "SACW", "time_q": "2023-01-23 19:12:29"} +{"device": "F83dj5jXAaNW08tN", "segment_ts": "NI7t", "time_q": "2023-02-03 06:42:10"} +{"device": "tz2BamJyhAvZSmKL", "segment_ts": "WFNu", "time_q": "2023-02-05 21:04:43"} +{"device": "od15jkcehIGTLhka", "segment_ts": "pY0b", "time_q": "2023-02-11 09:23:34"} +{"device": "o4J3jbqm2n2VMxAV", "segment_ts": "4G0L", "time_q": "2023-01-24 13:22:35"} +{"device": "uyBW48TE0tyYNDtB", "segment_ts": "RhKk", "time_q": "2023-01-25 13:14:11"} +{"device": "ANOQhJ0WObAmU99M", "segment_ts": "NyJ1", "time_q": "2023-02-10 22:25:45"} +{"device": "vDQzeXOS56cwm5Ha", "segment_ts": "3G8I", "time_q": "2023-02-11 19:06:38"} +{"device": "YqyX9Xk6Kw1DXOad", "segment_ts": "Qk0n", "time_q": "2023-02-10 21:58:02"} +{"device": "9neVj3Sy1VeKQxzF", "segment_ts": "j592", "time_q": "2023-01-28 02:02:51"} +{"device": "I1750EUbKfHl2yT0", "segment_ts": "Zztj", "time_q": "2023-02-06 23:04:30"} +{"device": "cdId56I5EVFYECEn", "segment_ts": "Trhf", "time_q": "2023-02-05 18:35:57"} +{"device": "mnuDFIng7Ffw33N7", "segment_ts": "7dQn", "time_q": "2023-01-31 14:53:16"} +{"device": "x6GzEx5oAis07KM2", "segment_ts": "XcKm", "time_q": "2023-01-23 06:33:10"} +{"device": "k27SrvBpgJ43XRj8", "segment_ts": "zKI7", "time_q": "2023-02-14 21:54:58"} +{"device": "lhj3vbaLb1F2K4lH", "segment_ts": "EQLa", "time_q": "2023-02-18 03:17:10"} +{"device": "TRlW7EbIXGs2Bpy8", "segment_ts": "iOm2", "time_q": "2023-02-04 11:54:32"} +{"device": "klEC6kMcr7NP6sNg", "segment_ts": "bRK7", "time_q": "2023-02-19 00:37:12"} +{"device": "TEemRDFbCJdjyNsQ", "segment_ts": "D16L", "time_q": "2023-02-17 11:29:57"} +{"device": "jptd7AJiFEAk4tSw", "segment_ts": "IAOL", "time_q": "2023-01-31 08:48:08"} +{"device": "L45DvspGDQF2fOyi", "segment_ts": "sALF", "time_q": "2023-02-05 03:25:00"} +{"device": "R2NkbrNMNldSBJUG", "segment_ts": "609r", "time_q": "2023-02-02 09:49:49"} +{"device": "8ZqVW4kT45fPZy5a", "segment_ts": "l00H", "time_q": "2023-02-11 13:05:33"} +{"device": "jHbO5YUIUxVvdzJA", "segment_ts": "8dJN", "time_q": "2023-02-17 02:03:26"} +{"device": "ktIY7e9VGCKMjL7P", "segment_ts": "Mxrj", "time_q": "2023-02-20 23:12:59"} +{"device": "uuOZC7aaCy9sUiT5", "segment_ts": "EUD1", "time_q": "2023-02-01 23:31:46"} +{"device": "3RtgCZcjgxxCmg5T", "segment_ts": "5JQi", "time_q": "2023-01-23 12:12:05"} +{"device": "YnI4T3vi1Lxyt5wa", "segment_ts": "rXIU", "time_q": "2023-02-12 23:58:24"} +{"device": "ev9IcNjTEZekqALB", "segment_ts": "GAxJ", "time_q": "2023-01-24 12:16:20"} +{"device": "vp5qi7TRC8KQWk7d", "segment_ts": "TJ6P", "time_q": "2023-02-07 17:28:23"} +{"device": "bSoAX2L1JHmPXODo", "segment_ts": "fasF", "time_q": "2023-02-06 15:59:59"} +{"device": "3XauXuq6ws73yQla", "segment_ts": "SsnB", "time_q": "2023-01-26 16:11:14"} +{"device": "TyYnqrqHKPeqZ82q", "segment_ts": "Wzwh", "time_q": "2023-02-16 20:02:19"} +{"device": "O8hOzOOgxw2iQiHx", "segment_ts": "Yj0D", "time_q": "2023-02-14 00:19:14"} +{"device": "u4Pxs4Cyj00rtA60", "segment_ts": "1vCq", "time_q": "2023-01-31 16:09:15"} +{"device": "jo8xd3EdVQRVKZG0", "segment_ts": "ltBU", "time_q": "2023-01-31 07:14:48"} +{"device": "W06dd1sDTy0apH3Y", "segment_ts": "ygKY", "time_q": "2023-02-09 21:21:10"} +{"device": "Sh7dZSxHMgFlutjC", "segment_ts": "MbqF", "time_q": "2023-02-11 23:49:25"} +{"device": "o5CLa2P1xlbs3BYb", "segment_ts": "v0BW", "time_q": "2023-02-01 11:00:06"} +{"device": "4wegF6Zxl955mSik", "segment_ts": "281a", "time_q": "2023-01-29 11:24:20"} +{"device": "bx31bZeyk6v3ZATS", "segment_ts": "wT1j", "time_q": "2023-02-03 22:21:54"} +{"device": "kfBaygAngrkZHyhR", "segment_ts": "KFml", "time_q": "2023-02-11 02:35:20"} +{"device": "IENwCndPscOSssCf", "segment_ts": "tGZF", "time_q": "2023-02-05 15:36:28"} +{"device": "JU1KqBsrNFR4vOqo", "segment_ts": "U1XA", "time_q": "2023-01-31 18:39:08"} +{"device": "8qqyRLZQv8IAoQgP", "segment_ts": "1WZE", "time_q": "2023-02-03 06:48:42"} +{"device": "rBCNpDGLi9AodGdX", "segment_ts": "3J3I", "time_q": "2023-02-17 06:28:51"} +{"device": "x1VmoWLCRvrSmkEh", "segment_ts": "lqnk", "time_q": "2023-01-24 18:01:20"} +{"device": "iL3Swp4VajwaNf3O", "segment_ts": "gtdx", "time_q": "2023-01-23 11:19:34"} +{"device": "g7iVs7QsRXDubZvV", "segment_ts": "Oyvx", "time_q": "2023-01-29 11:33:47"} +{"device": "4iCnj0wRJoMFKwx4", "segment_ts": "XU65", "time_q": "2023-01-26 13:56:11"} +{"device": "WWBQzqhhgRMwMVYH", "segment_ts": "MwvV", "time_q": "2023-01-26 21:09:06"} +{"device": "ZjwDtsLC8jMJIsQ6", "segment_ts": "NhT6", "time_q": "2023-02-10 05:06:17"} +{"device": "YkI6KyY3TgyviASv", "segment_ts": "CZdM", "time_q": "2023-02-04 03:49:11"} +{"device": "kemopcvgaWvww445", "segment_ts": "9EjB", "time_q": "2023-01-25 18:58:42"} +{"device": "Nfuv9QZRaTdtZ1j1", "segment_ts": "nyoq", "time_q": "2023-02-08 15:39:28"} +{"device": "IDNpRkPQqBYG8rIL", "segment_ts": "nrGc", "time_q": "2023-02-19 09:20:51"} +{"device": "woj5K9AYcOPi8wbH", "segment_ts": "q9o4", "time_q": "2023-01-30 00:47:23"} +{"device": "A3Q592GiKPtRBifW", "segment_ts": "479s", "time_q": "2023-02-03 13:21:56"} +{"device": "t0tSKqKvPJsSLKek", "segment_ts": "PzBM", "time_q": "2023-02-09 19:16:28"} +{"device": "xvu9RL3gVCt6YIqd", "segment_ts": "OE7b", "time_q": "2023-02-10 13:47:41"} +{"device": "WMwNdlIc5a3s0o4d", "segment_ts": "WFNu", "time_q": "2023-02-10 20:06:12"} +{"device": "kkvalugYPF7CeARB", "segment_ts": "xOmt", "time_q": "2023-01-27 20:04:35"} +{"device": "O2Pxt9eTcZ8DXb7Z", "segment_ts": "3sv4", "time_q": "2023-02-10 23:46:23"} +{"device": "n67yYlHFaWmgsEIn", "segment_ts": "MbqF", "time_q": "2023-01-30 20:38:08"} +{"device": "4vHRlcIj4fA6nyem", "segment_ts": "vroI", "time_q": "2023-02-14 16:20:29"} +{"device": "TMbltseSls9UOzxq", "segment_ts": "v5gi", "time_q": "2023-02-20 09:03:38"} +{"device": "Jj6eg1CNcUpAtY4b", "segment_ts": "xd2m", "time_q": "2023-02-16 06:33:10"} +{"device": "ipuU6YIHXJVKYBb5", "segment_ts": "LwNw", "time_q": "2023-01-28 00:19:52"} +{"device": "t66AJJAWxgfr54og", "segment_ts": "q74O", "time_q": "2023-01-30 06:53:54"} +{"device": "72tpVrLi7idQ711R", "segment_ts": "TuWP", "time_q": "2023-02-19 22:09:35"} +{"device": "Hvq7OAQ6pXafCIaP", "segment_ts": "OdFT", "time_q": "2023-02-17 20:46:42"} +{"device": "pme4dFY8cxRdM0iD", "segment_ts": "7exz", "time_q": "2023-02-16 00:45:20"} +{"device": "urddNEFtldtGx3WR", "segment_ts": "eZiz", "time_q": "2023-02-10 06:35:13"} +{"device": "uZvYbARAUvBgqqiC", "segment_ts": "AwpA", "time_q": "2023-02-05 08:54:00"} +{"device": "qynlelnsdXES0L3o", "segment_ts": "qe17", "time_q": "2023-02-04 03:25:06"} +{"device": "VvuWbHbURvKgsqUF", "segment_ts": "7Sxg", "time_q": "2023-02-14 10:47:24"} +{"device": "TlnhHfBk12CQiQ0K", "segment_ts": "QVU0", "time_q": "2023-02-11 01:01:08"} +{"device": "pPUdA3tMUudFfuu8", "segment_ts": "Ubci", "time_q": "2023-02-20 07:54:52"} +{"device": "zgZbzsxqUD8EYtTt", "segment_ts": "Oyvx", "time_q": "2023-02-16 23:14:46"} +{"device": "mJ4o0n4HIVfcYlOo", "segment_ts": "1Xba", "time_q": "2023-02-09 23:33:48"} +{"device": "aqh9qEavdsDiY54R", "segment_ts": "O951", "time_q": "2023-01-23 21:18:26"} +{"device": "baPCRewYf20VOMzV", "segment_ts": "7QZg", "time_q": "2023-01-30 03:32:06"} +{"device": "JGqBYR14WAkGBwFM", "segment_ts": "3ES4", "time_q": "2023-02-06 06:31:37"} +{"device": "k5Jpfk62razi9ZgC", "segment_ts": "jefZ", "time_q": "2023-02-21 06:43:37"} +{"device": "J4R3L5UMS3DjpooS", "segment_ts": "J8fh", "time_q": "2023-02-12 03:19:39"} +{"device": "mKaEAZkAqmHVYbYx", "segment_ts": "NZxR", "time_q": "2023-02-20 12:07:35"} +{"device": "mnFNrh7YGDVK0LF0", "segment_ts": "1yCJ", "time_q": "2023-01-29 11:05:00"} +{"device": "15x8g2YTRH5wxg8D", "segment_ts": "hOB3", "time_q": "2023-02-15 09:54:07"} +{"device": "MDhSc461Le45qIOD", "segment_ts": "Oqnw", "time_q": "2023-02-13 12:55:24"} +{"device": "2wtPLtzsT0nw72Ki", "segment_ts": "iC7M", "time_q": "2023-02-13 08:02:54"} +{"device": "EJN36J2EWxJO6d3A", "segment_ts": "aCKQ", "time_q": "2023-01-26 18:11:35"} +{"device": "4Cibygd0vko8tblj", "segment_ts": "xF4p", "time_q": "2023-01-27 09:25:50"} +{"device": "1EryLdrZkHz9TBM2", "segment_ts": "M97m", "time_q": "2023-02-06 08:09:39"} +{"device": "NUp95xg6x9K07uJw", "segment_ts": "DuKl", "time_q": "2023-02-15 14:42:37"} +{"device": "QTvOJ3fHPnsnFksM", "segment_ts": "PNnd", "time_q": "2023-01-23 22:31:31"} \ No newline at end of file diff --git a/idk/kafka/testdata/schemas/timeQuantum.json b/idk/kafka/testdata/schemas/timeQuantum.json new file mode 100644 index 000000000..a3565666e --- /dev/null +++ b/idk/kafka/testdata/schemas/timeQuantum.json @@ -0,0 +1,26 @@ +{ + "doc": "Records for testing time quantums", + "fields": [ + { + "doc": "A device ID", + "name": "device", + "type": "string" + }, + { + "doc": "The segment the device belongs to which is related to time", + "name": "segment_ts", + "type": "string", + "quantum": "YMD" + }, + { + "doc": "The timestamp associated with segment_ts", + "name": "time_q", + "type": "bytes", + "fieldType": "recordTime", + "layout": "2006-01-02 15:04:05" + } + ], + "name": "tq_record", + "namespace": "com.featurebase.tq", + "type": "record" +} \ No newline at end of file diff --git a/nfpm.yaml b/nfpm.yaml index 2c0c4d254..b4db53b39 100644 --- a/nfpm.yaml +++ b/nfpm.yaml @@ -10,6 +10,8 @@ homepage: "https://molecula.com" contents: - src: ./featurebase dst: /usr/bin/featurebase + - src: ./fbsql + dst: /usr/bin/fbsql - src: ./install/featurebase.conf dst: /etc/featurebase/featurebase.conf type: config|noreplace @@ -22,7 +24,7 @@ contents: - src: ./install/featurebase.debian.service dst: /lib/systemd/system/featurebase.service packager: deb - - dst: /var/log/molecula # We use vendor name on log directory in case other molecula components need it. + - dst: /var/log/molecula # We use vendor name on log directory in case other molecula components need it. type: dir file_info: mode: 0755 diff --git a/pilosa.go b/pilosa.go index dbfe64377..a95417216 100644 --- a/pilosa.go +++ b/pilosa.go @@ -47,7 +47,7 @@ var ( ErrInvalidView = errors.New("invalid view") ErrInvalidCacheType = errors.New("invalid cache type") - ErrName = errors.New("invalid index or field name, must match [a-z][a-z0-9_-]* and contain at most 230 characters") + ErrName = errors.New("invalid index or field name, must match [a-z][a-z0-9Θ_-]* and contain at most 230 characters") // ErrFragmentNotFound is returned when a fragment does not exist. ErrFragmentNotFound = errors.New("fragment not found") @@ -135,7 +135,7 @@ func newPreconditionFailedError(err error) PreconditionFailedError { } // Regular expression to validate index and field names. -var nameRegexp = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,229}$`) +var nameRegexp = regexp.MustCompile(`^[a-z][a-z0-9Θ_-]{0,229}$`) // TimeFormat is the go-style time format used to parse string dates. const TimeFormat = "2006-01-02T15:04" diff --git a/pilosa_test.go b/pilosa_test.go index 1b21e2e29..1d49181d6 100644 --- a/pilosa_test.go +++ b/pilosa_test.go @@ -3,6 +3,7 @@ package pilosa_test import ( + "fmt" "strings" "testing" @@ -43,3 +44,28 @@ func TestAddressWithDefaults(t *testing.T) { } } } + +func TestValidateName(t *testing.T) { + tests := []struct { + name string + err error + }{ + {name: "a_name", err: nil}, + {name: "a-name", err: nil}, + {name: "a-name-10", err: nil}, + {name: "A-name", err: fmt.Errorf("'A-name': %s", pilosa.ErrName)}, + {name: "-a-name", err: fmt.Errorf("'-a-name': %s", pilosa.ErrName)}, + {name: "8th_name", err: fmt.Errorf("'8th_name': %s", pilosa.ErrName)}, + {name: "Θ_name", err: fmt.Errorf("'Θ_name': %s", pilosa.ErrName)}, + {name: "a_NaMe", err: fmt.Errorf("'a_NaMe': %s", pilosa.ErrName)}, + {name: "indexΘname", err: nil}, + } + for _, test := range tests { + err := pilosa.ValidateName(test.name) + if !(err == nil && test.err == nil) { + if err.Error() != test.err.Error() { + t.Errorf("expected error: %v, but got: %v", test.err, err) + } + } + } +} diff --git a/pql/parser_test.go b/pql/parser_test.go index 9296f9935..a7c1815d0 100644 --- a/pql/parser_test.go +++ b/pql/parser_test.go @@ -307,6 +307,20 @@ func TestParser_Parse(t *testing.T) { } } }) + + t.Run("ParseTheta", func(t *testing.T) { + q, err := pql.ParseString(`Row(fldΘname=fldΘval)`) + if err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(q.Calls[0], + &pql.Call{ + Name: "Row", + Args: map[string]interface{}{"fldΘname": "fldΘval"}, + }, + ) { + t.Fatalf("unexpected call: %#v", q.Calls[0]) + } + }) } func TestUnquote(t *testing.T) { diff --git a/pql/pql.peg b/pql/pql.peg index d76322853..eedbdfdd6 100644 --- a/pql/pql.peg +++ b/pql/pql.peg @@ -20,7 +20,7 @@ Call <- "Set" {p.startCall("Set")} open col comma args (comma time)? close {p.e / "Sum" {p.startCall("Sum")} open posfield (comma allargs)? close {p.endCall()} / "Range" {p.startCall("Range")} open field eq value comma 'from='? {p.addField("from")} timefmt {p.addVal(text)} comma 'to='? sp {p.addField("to")} timefmt {p.addVal(text)} close {p.endCall()} / < IDENT > { p.startCall(text) } open allargs comma? close { p.endCall() } -ivyExpr <- ( '(' / '_' / '/' / '[' / ']' /'.' / '=' / '&' / '<'/ '>' /',' / ')' / '^' / '!' / '|' / [*+] / '-' / '?' / [[A-Z]] / [0-9] / '#' / [ \t\n] )* +ivyExpr <- ( '(' / '_' / '/' / '[' / ']' /'.' / '=' / '&' / '<'/ '>' /',' / ')' / '^' / '!' / '|' / [*+] / '-' / '?' / [[A-Z]] / [0-9] / '#' / [ \t\n] / 'Θ')* ivyprogram <- '"'? '"' { p.addPosStr("_ivy", text) } ivyprogram2 <- '"'? '"' { p.addPosStr("_ivyReduce", text) } @@ -55,16 +55,16 @@ item <- 'null' &(comma / close) { p.addVal(nil) } / timestampfmt { p.addTimestampVal(text) } / < decimal > { p.addNumVal(text) } / < IDENT > { p.startCall(text) } open allargs comma? close { p.addVal(p.endCall()) } - / < ([[A-Z]] / [0-9] / '-' / '_' / ':')+ > { p.addVal(text) } + / < ([[A-Z]] / [0-9] / '-' / '_' / ':' / 'Θ')+ > { p.addVal(text) } / < '"' doublequotedstring '"' > { p.addVal(text) } / < '\'' singlequotedstring '\'' > { p.addVal(text) } doublequotedstring <- ( '\\"' / '\\\\' / '\\n' / '\\t' / [^"\\] )* singlequotedstring <- ( '\\\'' / '\\\\' / '\\n' / '\\t' / [^'\\] )* -variable <- ( [[A-Z]] / '_' ) ( [[A-Z]] / [0-9] / '_' / '-' )* +variable <- ( [[A-Z]] / '_' ) ( [[A-Z]] / [0-9] / '_' / '-' / 'Θ')* -fieldExpr <- ( [[A-Z]] / '_' / '$' ) ( [[A-Z]] / [0-9] / '_' / '-' )* +fieldExpr <- ( [[A-Z]] / '_' / '$' ) ( [[A-Z]] / [0-9] / '_' / '-' / 'Θ')* field <- { p.addField(text) } reserved <- '_row' / '_col' / '_start' / '_end' / '_timestamp' / '_field' posfield <- 'field='? { p.addPosStr("_field", text) } @@ -79,7 +79,7 @@ eq <- sp '=' sp comma <- sp ',' sp lbrack <- '[' sp rbrack <- sp ']' sp -IDENT <- [[A-Z]] ([[A-Z]] / [0-9])* +IDENT <- [[A-Z]] ([[A-Z]] / [0-9] / 'Θ')* digits <- [0-9]+ signedDigits <- '-'? digits decimal <- signedDigits ('.' digits?)? diff --git a/pql/pql.peg.go b/pql/pql.peg.go index ba8057489..b5abe7044 100644 --- a/pql/pql.peg.go +++ b/pql/pql.peg.go @@ -1,6 +1,7 @@ package pql // Code generated by peg -inline pql.peg DO NOT EDIT. +// run go install github.com/pointlander/peg@latest import ( "fmt" @@ -2232,7 +2233,7 @@ func (p *PQL) Init(options ...func(*PQL) error) error { position, tokenIndex = position5, tokenIndex5 return false }, - /* 2 ivyExpr <- <('(' / '_' / '/' / '[' / ']' / '.' / '=' / '&' / '<' / '>' / ',' / ')' / '^' / '!' / '|' / ('*' / '+') / '-' / '?' / ([a-z] / [A-Z]) / [0-9] / '#' / (' ' / '\t' / '\n'))*> */ + /* 2 ivyExpr <- <('(' / '_' / '/' / '[' / ']' / '.' / '=' / '&' / '<' / '>' / ',' / ')' / '^' / '!' / '|' / ('*' / '+') / '-' / '?' / ([a-z] / [A-Z]) / [0-9] / '#' / (' ' / '\t' / '\n') / 'Θ')*> */ func() bool { { position218 := position @@ -2411,27 +2412,34 @@ func (p *PQL) Init(options ...func(*PQL) error) error { l246: position, tokenIndex = position221, tokenIndex221 { - position247, tokenIndex247 := position, tokenIndex + position248, tokenIndex248 := position, tokenIndex if buffer[position] != rune(' ') { - goto l248 - } - position++ - goto l247 - l248: - position, tokenIndex = position247, tokenIndex247 - if buffer[position] != rune('\t') { goto l249 } position++ - goto l247 + goto l248 l249: - position, tokenIndex = position247, tokenIndex247 + position, tokenIndex = position248, tokenIndex248 + if buffer[position] != rune('\t') { + goto l250 + } + position++ + goto l248 + l250: + position, tokenIndex = position248, tokenIndex248 if buffer[position] != rune('\n') { - goto l220 + goto l247 } position++ } + l248: + goto l221 l247: + position, tokenIndex = position221, tokenIndex221 + if buffer[position] != rune('Θ') { + goto l220 + } + position++ } l221: goto l219 @@ -2448,267 +2456,267 @@ func (p *PQL) Init(options ...func(*PQL) error) error { nil, /* 5 allargs <- <((Call (comma Call)* (comma args)?) / args / sp)> */ func() bool { - position252, tokenIndex252 := position, tokenIndex + position253, tokenIndex253 := position, tokenIndex { - position253 := position + position254 := position { - position254, tokenIndex254 := position, tokenIndex + position255, tokenIndex255 := position, tokenIndex if !_rules[ruleCall]() { - goto l255 - } - l256: - { - position257, tokenIndex257 := position, tokenIndex - if !_rules[rulecomma]() { - goto l257 - } - if !_rules[ruleCall]() { - goto l257 - } goto l256 - l257: - position, tokenIndex = position257, tokenIndex257 } + l257: { position258, tokenIndex258 := position, tokenIndex if !_rules[rulecomma]() { goto l258 } - if !_rules[ruleargs]() { + if !_rules[ruleCall]() { goto l258 } - goto l259 + goto l257 l258: position, tokenIndex = position258, tokenIndex258 } - l259: - goto l254 - l255: - position, tokenIndex = position254, tokenIndex254 - if !_rules[ruleargs]() { + { + position259, tokenIndex259 := position, tokenIndex + if !_rules[rulecomma]() { + goto l259 + } + if !_rules[ruleargs]() { + goto l259 + } goto l260 + l259: + position, tokenIndex = position259, tokenIndex259 } - goto l254 l260: - position, tokenIndex = position254, tokenIndex254 + goto l255 + l256: + position, tokenIndex = position255, tokenIndex255 + if !_rules[ruleargs]() { + goto l261 + } + goto l255 + l261: + position, tokenIndex = position255, tokenIndex255 if !_rules[rulesp]() { - goto l252 + goto l253 } } - l254: - add(ruleallargs, position253) + l255: + add(ruleallargs, position254) } return true - l252: - position, tokenIndex = position252, tokenIndex252 + l253: + position, tokenIndex = position253, tokenIndex253 return false }, /* 6 args <- <(arg (comma args)? sp)> */ func() bool { - position261, tokenIndex261 := position, tokenIndex + position262, tokenIndex262 := position, tokenIndex { - position262 := position + position263 := position if !_rules[rulearg]() { - goto l261 + goto l262 } { - position263, tokenIndex263 := position, tokenIndex + position264, tokenIndex264 := position, tokenIndex if !_rules[rulecomma]() { - goto l263 + goto l264 } if !_rules[ruleargs]() { - goto l263 + goto l264 } - goto l264 - l263: - position, tokenIndex = position263, tokenIndex263 + goto l265 + l264: + position, tokenIndex = position264, tokenIndex264 } - l264: + l265: if !_rules[rulesp]() { - goto l261 + goto l262 } - add(ruleargs, position262) + add(ruleargs, position263) } return true - l261: - position, tokenIndex = position261, tokenIndex261 + l262: + position, tokenIndex = position262, tokenIndex262 return false }, /* 7 arg <- <((field eq value) / (field sp COND sp value) / conditional)> */ func() bool { - position265, tokenIndex265 := position, tokenIndex + position266, tokenIndex266 := position, tokenIndex { - position266 := position + position267 := position { - position267, tokenIndex267 := position, tokenIndex + position268, tokenIndex268 := position, tokenIndex if !_rules[rulefield]() { - goto l268 + goto l269 } if !_rules[ruleeq]() { - goto l268 + goto l269 } if !_rules[rulevalue]() { - goto l268 - } - goto l267 - l268: - position, tokenIndex = position267, tokenIndex267 - if !_rules[rulefield]() { goto l269 } + goto l268 + l269: + position, tokenIndex = position268, tokenIndex268 + if !_rules[rulefield]() { + goto l270 + } if !_rules[rulesp]() { - goto l269 + goto l270 } { - position270 := position + position271 := position { - position271, tokenIndex271 := position, tokenIndex + position272, tokenIndex272 := position, tokenIndex if buffer[position] != rune('>') { - goto l272 + goto l273 } position++ if buffer[position] != rune('<') { - goto l272 + goto l273 } position++ { add(ruleAction34, position) } - goto l271 - l272: - position, tokenIndex = position271, tokenIndex271 + goto l272 + l273: + position, tokenIndex = position272, tokenIndex272 if buffer[position] != rune('<') { - goto l274 + goto l275 } position++ if buffer[position] != rune('=') { - goto l274 + goto l275 } position++ { add(ruleAction35, position) } - goto l271 - l274: - position, tokenIndex = position271, tokenIndex271 + goto l272 + l275: + position, tokenIndex = position272, tokenIndex272 if buffer[position] != rune('>') { - goto l276 + goto l277 } position++ if buffer[position] != rune('=') { - goto l276 + goto l277 } position++ { add(ruleAction36, position) } - goto l271 - l276: - position, tokenIndex = position271, tokenIndex271 + goto l272 + l277: + position, tokenIndex = position272, tokenIndex272 if buffer[position] != rune('=') { - goto l278 + goto l279 } position++ if buffer[position] != rune('=') { - goto l278 + goto l279 } position++ { add(ruleAction37, position) } - goto l271 - l278: - position, tokenIndex = position271, tokenIndex271 + goto l272 + l279: + position, tokenIndex = position272, tokenIndex272 if buffer[position] != rune('!') { - goto l280 + goto l281 } position++ if buffer[position] != rune('=') { - goto l280 + goto l281 } position++ { add(ruleAction38, position) } - goto l271 - l280: - position, tokenIndex = position271, tokenIndex271 + goto l272 + l281: + position, tokenIndex = position272, tokenIndex272 if buffer[position] != rune('<') { - goto l282 + goto l283 } position++ { add(ruleAction39, position) } - goto l271 - l282: - position, tokenIndex = position271, tokenIndex271 + goto l272 + l283: + position, tokenIndex = position272, tokenIndex272 if buffer[position] != rune('>') { - goto l269 + goto l270 } position++ { add(ruleAction40, position) } } - l271: - add(ruleCOND, position270) + l272: + add(ruleCOND, position271) } if !_rules[rulesp]() { - goto l269 + goto l270 } if !_rules[rulevalue]() { - goto l269 + goto l270 } - goto l267 - l269: - position, tokenIndex = position267, tokenIndex267 + goto l268 + l270: + position, tokenIndex = position268, tokenIndex268 { - position285 := position + position286 := position { add(ruleAction41, position) } if !_rules[rulecondintOrTime]() { - goto l265 + goto l266 } if !_rules[rulecondLT]() { - goto l265 + goto l266 } { - position287 := position + position288 := position { - position288 := position + position289 := position if !_rules[rulefieldExpr]() { - goto l265 + goto l266 } - add(rulePegText, position288) + add(rulePegText, position289) } if !_rules[rulesp]() { - goto l265 + goto l266 } { add(ruleAction46, position) } - add(rulecondfield, position287) + add(rulecondfield, position288) } if !_rules[rulecondLT]() { - goto l265 + goto l266 } if !_rules[rulecondintOrTime]() { - goto l265 + goto l266 } { add(ruleAction42, position) } - add(ruleconditional, position285) + add(ruleconditional, position286) } } - l267: - add(rulearg, position266) + l268: + add(rulearg, position267) } return true - l265: - position, tokenIndex = position265, tokenIndex265 + l266: + position, tokenIndex = position266, tokenIndex266 return false }, /* 8 COND <- <(('>' '<' Action34) / ('<' '=' Action35) / ('>' '=' Action36) / ('=' '=' Action37) / ('!' '=' Action38) / ('<' Action39) / ('>' Action40))> */ @@ -2717,55 +2725,55 @@ func (p *PQL) Init(options ...func(*PQL) error) error { nil, /* 10 condintOrTime <- <(condint / timefmtS)> */ func() bool { - position293, tokenIndex293 := position, tokenIndex + position294, tokenIndex294 := position, tokenIndex { - position294 := position + position295 := position { - position295, tokenIndex295 := position, tokenIndex + position296, tokenIndex296 := position, tokenIndex { - position297 := position + position298 := position { - position298 := position + position299 := position if !_rules[ruledecimal]() { - goto l296 + goto l297 } - add(rulePegText, position298) + add(rulePegText, position299) } if !_rules[rulesp]() { - goto l296 + goto l297 } { add(ruleAction44, position) } - add(rulecondint, position297) + add(rulecondint, position298) } - goto l295 - l296: - position, tokenIndex = position295, tokenIndex295 + goto l296 + l297: + position, tokenIndex = position296, tokenIndex296 { - position300 := position + position301 := position { - position301 := position + position302 := position if !_rules[ruletimestampfmt]() { - goto l293 + goto l294 } - add(rulePegText, position301) + add(rulePegText, position302) } if !_rules[rulesp]() { - goto l293 + goto l294 } { add(ruleAction43, position) } - add(ruletimefmtS, position300) + add(ruletimefmtS, position301) } } - l295: - add(rulecondintOrTime, position294) + l296: + add(rulecondintOrTime, position295) } return true - l293: - position, tokenIndex = position293, tokenIndex293 + l294: + position, tokenIndex = position294, tokenIndex294 return false }, /* 11 timefmtS <- <( sp Action43)> */ @@ -2774,1489 +2782,1524 @@ func (p *PQL) Init(options ...func(*PQL) error) error { nil, /* 13 condLT <- <(<(('<' '=') / '<')> sp Action45)> */ func() bool { - position305, tokenIndex305 := position, tokenIndex + position306, tokenIndex306 := position, tokenIndex { - position306 := position + position307 := position { - position307 := position + position308 := position { - position308, tokenIndex308 := position, tokenIndex + position309, tokenIndex309 := position, tokenIndex if buffer[position] != rune('<') { - goto l309 + goto l310 } position++ if buffer[position] != rune('=') { - goto l309 + goto l310 } position++ - goto l308 - l309: - position, tokenIndex = position308, tokenIndex308 + goto l309 + l310: + position, tokenIndex = position309, tokenIndex309 if buffer[position] != rune('<') { - goto l305 + goto l306 } position++ } - l308: - add(rulePegText, position307) + l309: + add(rulePegText, position308) } if !_rules[rulesp]() { - goto l305 + goto l306 } { add(ruleAction45, position) } - add(rulecondLT, position306) + add(rulecondLT, position307) } return true - l305: - position, tokenIndex = position305, tokenIndex305 + l306: + position, tokenIndex = position306, tokenIndex306 return false }, /* 14 condfield <- <( sp Action46)> */ nil, /* 15 value <- <(item / (lbrack Action47 items rbrack Action48))> */ func() bool { - position312, tokenIndex312 := position, tokenIndex + position313, tokenIndex313 := position, tokenIndex { - position313 := position + position314 := position { - position314, tokenIndex314 := position, tokenIndex + position315, tokenIndex315 := position, tokenIndex if !_rules[ruleitem]() { - goto l315 + goto l316 } - goto l314 - l315: - position, tokenIndex = position314, tokenIndex314 + goto l315 + l316: + position, tokenIndex = position315, tokenIndex315 { - position316 := position + position317 := position if buffer[position] != rune('[') { - goto l312 + goto l313 } position++ if !_rules[rulesp]() { - goto l312 + goto l313 } - add(rulelbrack, position316) + add(rulelbrack, position317) } { add(ruleAction47, position) } if !_rules[ruleitems]() { - goto l312 + goto l313 } { - position318 := position + position319 := position if !_rules[rulesp]() { - goto l312 + goto l313 } if buffer[position] != rune(']') { - goto l312 + goto l313 } position++ if !_rules[rulesp]() { - goto l312 + goto l313 } - add(rulerbrack, position318) + add(rulerbrack, position319) } { add(ruleAction48, position) } } - l314: - add(rulevalue, position313) + l315: + add(rulevalue, position314) } return true - l312: - position, tokenIndex = position312, tokenIndex312 + l313: + position, tokenIndex = position313, tokenIndex313 return false }, /* 16 items <- <(item (comma items)?)> */ func() bool { - position320, tokenIndex320 := position, tokenIndex + position321, tokenIndex321 := position, tokenIndex { - position321 := position + position322 := position if !_rules[ruleitem]() { - goto l320 + goto l321 } { - position322, tokenIndex322 := position, tokenIndex + position323, tokenIndex323 := position, tokenIndex if !_rules[rulecomma]() { - goto l322 + goto l323 } if !_rules[ruleitems]() { - goto l322 + goto l323 } - goto l323 - l322: - position, tokenIndex = position322, tokenIndex322 + goto l324 + l323: + position, tokenIndex = position323, tokenIndex323 } - l323: - add(ruleitems, position321) + l324: + add(ruleitems, position322) } return true - l320: - position, tokenIndex = position320, tokenIndex320 + l321: + position, tokenIndex = position321, tokenIndex321 return false }, - /* 17 item <- <(('n' 'u' 'l' 'l' &(comma / close) Action49) / ('t' 'r' 'u' 'e' &(comma / close) Action50) / ('f' 'a' 'l' 's' 'e' &(comma / close) Action51) / ('$' Action52) / (timefmt Action53) / (timestampfmt Action54) / ( Action55) / ( Action56 open allargs comma? close Action57) / (<([a-z] / [A-Z] / [0-9] / '-' / '_' / ':')+> Action58) / (<('"' doublequotedstring '"')> Action59) / (<('\'' singlequotedstring '\'')> Action60))> */ + /* 17 item <- <(('n' 'u' 'l' 'l' &(comma / close) Action49) / ('t' 'r' 'u' 'e' &(comma / close) Action50) / ('f' 'a' 'l' 's' 'e' &(comma / close) Action51) / ('$' Action52) / (timefmt Action53) / (timestampfmt Action54) / ( Action55) / ( Action56 open allargs comma? close Action57) / (<([a-z] / [A-Z] / [0-9] / '-' / '_' / ':' / 'Θ')+> Action58) / (<('"' doublequotedstring '"')> Action59) / (<('\'' singlequotedstring '\'')> Action60))> */ func() bool { - position324, tokenIndex324 := position, tokenIndex + position325, tokenIndex325 := position, tokenIndex { - position325 := position + position326 := position { - position326, tokenIndex326 := position, tokenIndex + position327, tokenIndex327 := position, tokenIndex if buffer[position] != rune('n') { - goto l327 + goto l328 } position++ if buffer[position] != rune('u') { - goto l327 + goto l328 } position++ if buffer[position] != rune('l') { - goto l327 + goto l328 } position++ if buffer[position] != rune('l') { - goto l327 + goto l328 } position++ { - position328, tokenIndex328 := position, tokenIndex + position329, tokenIndex329 := position, tokenIndex { - position329, tokenIndex329 := position, tokenIndex + position330, tokenIndex330 := position, tokenIndex if !_rules[rulecomma]() { - goto l330 + goto l331 } - goto l329 - l330: - position, tokenIndex = position329, tokenIndex329 + goto l330 + l331: + position, tokenIndex = position330, tokenIndex330 if !_rules[ruleclose]() { - goto l327 + goto l328 } } - l329: - position, tokenIndex = position328, tokenIndex328 + l330: + position, tokenIndex = position329, tokenIndex329 } { add(ruleAction49, position) } - goto l326 - l327: - position, tokenIndex = position326, tokenIndex326 + goto l327 + l328: + position, tokenIndex = position327, tokenIndex327 if buffer[position] != rune('t') { - goto l332 + goto l333 } position++ if buffer[position] != rune('r') { - goto l332 + goto l333 } position++ if buffer[position] != rune('u') { - goto l332 + goto l333 } position++ if buffer[position] != rune('e') { - goto l332 + goto l333 } position++ { - position333, tokenIndex333 := position, tokenIndex + position334, tokenIndex334 := position, tokenIndex { - position334, tokenIndex334 := position, tokenIndex + position335, tokenIndex335 := position, tokenIndex if !_rules[rulecomma]() { - goto l335 + goto l336 } - goto l334 - l335: - position, tokenIndex = position334, tokenIndex334 + goto l335 + l336: + position, tokenIndex = position335, tokenIndex335 if !_rules[ruleclose]() { - goto l332 + goto l333 } } - l334: - position, tokenIndex = position333, tokenIndex333 + l335: + position, tokenIndex = position334, tokenIndex334 } { add(ruleAction50, position) } - goto l326 - l332: - position, tokenIndex = position326, tokenIndex326 + goto l327 + l333: + position, tokenIndex = position327, tokenIndex327 if buffer[position] != rune('f') { - goto l337 + goto l338 } position++ if buffer[position] != rune('a') { - goto l337 + goto l338 } position++ if buffer[position] != rune('l') { - goto l337 + goto l338 } position++ if buffer[position] != rune('s') { - goto l337 + goto l338 } position++ if buffer[position] != rune('e') { - goto l337 + goto l338 } position++ { - position338, tokenIndex338 := position, tokenIndex + position339, tokenIndex339 := position, tokenIndex { - position339, tokenIndex339 := position, tokenIndex + position340, tokenIndex340 := position, tokenIndex if !_rules[rulecomma]() { - goto l340 + goto l341 } - goto l339 - l340: - position, tokenIndex = position339, tokenIndex339 + goto l340 + l341: + position, tokenIndex = position340, tokenIndex340 if !_rules[ruleclose]() { - goto l337 + goto l338 } } - l339: - position, tokenIndex = position338, tokenIndex338 + l340: + position, tokenIndex = position339, tokenIndex339 } { add(ruleAction51, position) } - goto l326 - l337: - position, tokenIndex = position326, tokenIndex326 + goto l327 + l338: + position, tokenIndex = position327, tokenIndex327 if buffer[position] != rune('$') { - goto l342 + goto l343 } position++ { - position343 := position + position344 := position { - position344 := position + position345 := position { - position345, tokenIndex345 := position, tokenIndex + position346, tokenIndex346 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l346 - } - position++ - goto l345 - l346: - position, tokenIndex = position345, tokenIndex345 - if c := buffer[position]; c < rune('A') || c > rune('Z') { goto l347 } position++ - goto l345 + goto l346 l347: - position, tokenIndex = position345, tokenIndex345 + position, tokenIndex = position346, tokenIndex346 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l348 + } + position++ + goto l346 + l348: + position, tokenIndex = position346, tokenIndex346 if buffer[position] != rune('_') { - goto l342 + goto l343 } position++ } - l345: - l348: + l346: + l349: { - position349, tokenIndex349 := position, tokenIndex + position350, tokenIndex350 := position, tokenIndex { - position350, tokenIndex350 := position, tokenIndex + position351, tokenIndex351 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l351 - } - position++ - goto l350 - l351: - position, tokenIndex = position350, tokenIndex350 - if c := buffer[position]; c < rune('A') || c > rune('Z') { goto l352 } position++ - goto l350 + goto l351 l352: - position, tokenIndex = position350, tokenIndex350 - if c := buffer[position]; c < rune('0') || c > rune('9') { + position, tokenIndex = position351, tokenIndex351 + if c := buffer[position]; c < rune('A') || c > rune('Z') { goto l353 } position++ - goto l350 + goto l351 l353: - position, tokenIndex = position350, tokenIndex350 - if buffer[position] != rune('_') { + position, tokenIndex = position351, tokenIndex351 + if c := buffer[position]; c < rune('0') || c > rune('9') { goto l354 } position++ - goto l350 + goto l351 l354: - position, tokenIndex = position350, tokenIndex350 + position, tokenIndex = position351, tokenIndex351 + if buffer[position] != rune('_') { + goto l355 + } + position++ + goto l351 + l355: + position, tokenIndex = position351, tokenIndex351 if buffer[position] != rune('-') { - goto l349 + goto l356 + } + position++ + goto l351 + l356: + position, tokenIndex = position351, tokenIndex351 + if buffer[position] != rune('Θ') { + goto l350 } position++ } + l351: + goto l349 l350: - goto l348 - l349: - position, tokenIndex = position349, tokenIndex349 + position, tokenIndex = position350, tokenIndex350 } - add(rulevariable, position344) + add(rulevariable, position345) } - add(rulePegText, position343) + add(rulePegText, position344) } { add(ruleAction52, position) } - goto l326 - l342: - position, tokenIndex = position326, tokenIndex326 + goto l327 + l343: + position, tokenIndex = position327, tokenIndex327 if !_rules[ruletimefmt]() { - goto l356 + goto l358 } { add(ruleAction53, position) } - goto l326 - l356: - position, tokenIndex = position326, tokenIndex326 + goto l327 + l358: + position, tokenIndex = position327, tokenIndex327 if !_rules[ruletimestampfmt]() { - goto l358 + goto l360 } { add(ruleAction54, position) } - goto l326 - l358: - position, tokenIndex = position326, tokenIndex326 + goto l327 + l360: + position, tokenIndex = position327, tokenIndex327 { - position361 := position + position363 := position if !_rules[ruledecimal]() { - goto l360 + goto l362 } - add(rulePegText, position361) + add(rulePegText, position363) } { add(ruleAction55, position) } - goto l326 - l360: - position, tokenIndex = position326, tokenIndex326 + goto l327 + l362: + position, tokenIndex = position327, tokenIndex327 { - position364 := position + position366 := position if !_rules[ruleIDENT]() { - goto l363 + goto l365 } - add(rulePegText, position364) + add(rulePegText, position366) } { add(ruleAction56, position) } if !_rules[ruleopen]() { - goto l363 + goto l365 } if !_rules[ruleallargs]() { - goto l363 + goto l365 } { - position366, tokenIndex366 := position, tokenIndex + position368, tokenIndex368 := position, tokenIndex if !_rules[rulecomma]() { - goto l366 + goto l368 } - goto l367 - l366: - position, tokenIndex = position366, tokenIndex366 + goto l369 + l368: + position, tokenIndex = position368, tokenIndex368 } - l367: + l369: if !_rules[ruleclose]() { - goto l363 + goto l365 } { add(ruleAction57, position) } - goto l326 - l363: - position, tokenIndex = position326, tokenIndex326 + goto l327 + l365: + position, tokenIndex = position327, tokenIndex327 { - position370 := position + position372 := position { - position373, tokenIndex373 := position, tokenIndex + position375, tokenIndex375 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l374 - } - position++ - goto l373 - l374: - position, tokenIndex = position373, tokenIndex373 - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l375 - } - position++ - goto l373 - l375: - position, tokenIndex = position373, tokenIndex373 - if c := buffer[position]; c < rune('0') || c > rune('9') { goto l376 } position++ - goto l373 + goto l375 l376: - position, tokenIndex = position373, tokenIndex373 - if buffer[position] != rune('-') { + position, tokenIndex = position375, tokenIndex375 + if c := buffer[position]; c < rune('A') || c > rune('Z') { goto l377 } position++ - goto l373 + goto l375 l377: - position, tokenIndex = position373, tokenIndex373 - if buffer[position] != rune('_') { + position, tokenIndex = position375, tokenIndex375 + if c := buffer[position]; c < rune('0') || c > rune('9') { goto l378 } position++ - goto l373 + goto l375 l378: - position, tokenIndex = position373, tokenIndex373 + position, tokenIndex = position375, tokenIndex375 + if buffer[position] != rune('-') { + goto l379 + } + position++ + goto l375 + l379: + position, tokenIndex = position375, tokenIndex375 + if buffer[position] != rune('_') { + goto l380 + } + position++ + goto l375 + l380: + position, tokenIndex = position375, tokenIndex375 if buffer[position] != rune(':') { - goto l369 + goto l381 + } + position++ + goto l375 + l381: + position, tokenIndex = position375, tokenIndex375 + if buffer[position] != rune('Θ') { + goto l371 } position++ } + l375: l373: - l371: { - position372, tokenIndex372 := position, tokenIndex + position374, tokenIndex374 := position, tokenIndex { - position379, tokenIndex379 := position, tokenIndex + position382, tokenIndex382 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l380 - } - position++ - goto l379 - l380: - position, tokenIndex = position379, tokenIndex379 - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l381 - } - position++ - goto l379 - l381: - position, tokenIndex = position379, tokenIndex379 - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l382 - } - position++ - goto l379 - l382: - position, tokenIndex = position379, tokenIndex379 - if buffer[position] != rune('-') { goto l383 } position++ - goto l379 + goto l382 l383: - position, tokenIndex = position379, tokenIndex379 - if buffer[position] != rune('_') { + position, tokenIndex = position382, tokenIndex382 + if c := buffer[position]; c < rune('A') || c > rune('Z') { goto l384 } position++ - goto l379 + goto l382 l384: - position, tokenIndex = position379, tokenIndex379 + position, tokenIndex = position382, tokenIndex382 + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l385 + } + position++ + goto l382 + l385: + position, tokenIndex = position382, tokenIndex382 + if buffer[position] != rune('-') { + goto l386 + } + position++ + goto l382 + l386: + position, tokenIndex = position382, tokenIndex382 + if buffer[position] != rune('_') { + goto l387 + } + position++ + goto l382 + l387: + position, tokenIndex = position382, tokenIndex382 if buffer[position] != rune(':') { - goto l372 + goto l388 + } + position++ + goto l382 + l388: + position, tokenIndex = position382, tokenIndex382 + if buffer[position] != rune('Θ') { + goto l374 } position++ } - l379: - goto l371 - l372: - position, tokenIndex = position372, tokenIndex372 + l382: + goto l373 + l374: + position, tokenIndex = position374, tokenIndex374 } - add(rulePegText, position370) + add(rulePegText, position372) } { add(ruleAction58, position) } - goto l326 - l369: - position, tokenIndex = position326, tokenIndex326 + goto l327 + l371: + position, tokenIndex = position327, tokenIndex327 { - position387 := position + position391 := position if buffer[position] != rune('"') { - goto l386 + goto l390 } position++ if !_rules[ruledoublequotedstring]() { - goto l386 + goto l390 } if buffer[position] != rune('"') { - goto l386 + goto l390 } position++ - add(rulePegText, position387) + add(rulePegText, position391) } { add(ruleAction59, position) } - goto l326 - l386: - position, tokenIndex = position326, tokenIndex326 + goto l327 + l390: + position, tokenIndex = position327, tokenIndex327 { - position389 := position + position393 := position if buffer[position] != rune('\'') { - goto l324 + goto l325 } position++ if !_rules[rulesinglequotedstring]() { - goto l324 + goto l325 } if buffer[position] != rune('\'') { - goto l324 + goto l325 } position++ - add(rulePegText, position389) + add(rulePegText, position393) } { add(ruleAction60, position) } } - l326: - add(ruleitem, position325) + l327: + add(ruleitem, position326) } return true - l324: - position, tokenIndex = position324, tokenIndex324 + l325: + position, tokenIndex = position325, tokenIndex325 return false }, /* 18 doublequotedstring <- <(('\\' '"') / ('\\' '\\') / ('\\' 'n') / ('\\' 't') / (!('"' / '\\') .))*> */ func() bool { { - position392 := position - l393: + position396 := position + l397: { - position394, tokenIndex394 := position, tokenIndex + position398, tokenIndex398 := position, tokenIndex { - position395, tokenIndex395 := position, tokenIndex + position399, tokenIndex399 := position, tokenIndex if buffer[position] != rune('\\') { - goto l396 + goto l400 } position++ if buffer[position] != rune('"') { - goto l396 + goto l400 } position++ - goto l395 - l396: - position, tokenIndex = position395, tokenIndex395 + goto l399 + l400: + position, tokenIndex = position399, tokenIndex399 if buffer[position] != rune('\\') { - goto l397 + goto l401 } position++ if buffer[position] != rune('\\') { - goto l397 + goto l401 } position++ - goto l395 - l397: - position, tokenIndex = position395, tokenIndex395 + goto l399 + l401: + position, tokenIndex = position399, tokenIndex399 if buffer[position] != rune('\\') { - goto l398 + goto l402 } position++ if buffer[position] != rune('n') { - goto l398 + goto l402 } position++ - goto l395 - l398: - position, tokenIndex = position395, tokenIndex395 + goto l399 + l402: + position, tokenIndex = position399, tokenIndex399 if buffer[position] != rune('\\') { - goto l399 + goto l403 } position++ if buffer[position] != rune('t') { - goto l399 + goto l403 } position++ - goto l395 - l399: - position, tokenIndex = position395, tokenIndex395 + goto l399 + l403: + position, tokenIndex = position399, tokenIndex399 { - position400, tokenIndex400 := position, tokenIndex + position404, tokenIndex404 := position, tokenIndex { - position401, tokenIndex401 := position, tokenIndex + position405, tokenIndex405 := position, tokenIndex if buffer[position] != rune('"') { - goto l402 + goto l406 } position++ - goto l401 - l402: - position, tokenIndex = position401, tokenIndex401 + goto l405 + l406: + position, tokenIndex = position405, tokenIndex405 if buffer[position] != rune('\\') { - goto l400 + goto l404 } position++ } - l401: - goto l394 - l400: - position, tokenIndex = position400, tokenIndex400 + l405: + goto l398 + l404: + position, tokenIndex = position404, tokenIndex404 } if !matchDot() { - goto l394 + goto l398 } } - l395: - goto l393 - l394: - position, tokenIndex = position394, tokenIndex394 + l399: + goto l397 + l398: + position, tokenIndex = position398, tokenIndex398 } - add(ruledoublequotedstring, position392) + add(ruledoublequotedstring, position396) } return true }, /* 19 singlequotedstring <- <(('\\' '\'') / ('\\' '\\') / ('\\' 'n') / ('\\' 't') / (!('\'' / '\\') .))*> */ func() bool { { - position404 := position - l405: + position408 := position + l409: { - position406, tokenIndex406 := position, tokenIndex + position410, tokenIndex410 := position, tokenIndex { - position407, tokenIndex407 := position, tokenIndex + position411, tokenIndex411 := position, tokenIndex if buffer[position] != rune('\\') { - goto l408 + goto l412 } position++ if buffer[position] != rune('\'') { - goto l408 + goto l412 } position++ - goto l407 - l408: - position, tokenIndex = position407, tokenIndex407 + goto l411 + l412: + position, tokenIndex = position411, tokenIndex411 if buffer[position] != rune('\\') { - goto l409 + goto l413 } position++ if buffer[position] != rune('\\') { - goto l409 + goto l413 } position++ - goto l407 - l409: - position, tokenIndex = position407, tokenIndex407 + goto l411 + l413: + position, tokenIndex = position411, tokenIndex411 if buffer[position] != rune('\\') { - goto l410 + goto l414 } position++ if buffer[position] != rune('n') { - goto l410 + goto l414 } position++ - goto l407 - l410: - position, tokenIndex = position407, tokenIndex407 + goto l411 + l414: + position, tokenIndex = position411, tokenIndex411 if buffer[position] != rune('\\') { - goto l411 + goto l415 } position++ if buffer[position] != rune('t') { - goto l411 + goto l415 } position++ - goto l407 - l411: - position, tokenIndex = position407, tokenIndex407 + goto l411 + l415: + position, tokenIndex = position411, tokenIndex411 { - position412, tokenIndex412 := position, tokenIndex + position416, tokenIndex416 := position, tokenIndex { - position413, tokenIndex413 := position, tokenIndex + position417, tokenIndex417 := position, tokenIndex if buffer[position] != rune('\'') { - goto l414 + goto l418 } position++ - goto l413 - l414: - position, tokenIndex = position413, tokenIndex413 + goto l417 + l418: + position, tokenIndex = position417, tokenIndex417 if buffer[position] != rune('\\') { - goto l412 + goto l416 } position++ } - l413: - goto l406 - l412: - position, tokenIndex = position412, tokenIndex412 + l417: + goto l410 + l416: + position, tokenIndex = position416, tokenIndex416 } if !matchDot() { - goto l406 + goto l410 } } - l407: - goto l405 - l406: - position, tokenIndex = position406, tokenIndex406 + l411: + goto l409 + l410: + position, tokenIndex = position410, tokenIndex410 } - add(rulesinglequotedstring, position404) + add(rulesinglequotedstring, position408) } return true }, - /* 20 variable <- <(([a-z] / [A-Z] / '_') ([a-z] / [A-Z] / [0-9] / '_' / '-')*)> */ + /* 20 variable <- <(([a-z] / [A-Z] / '_') ([a-z] / [A-Z] / [0-9] / '_' / '-' / 'Θ')*)> */ nil, - /* 21 fieldExpr <- <(([a-z] / [A-Z] / '_' / '$') ([a-z] / [A-Z] / [0-9] / '_' / '-')*)> */ + /* 21 fieldExpr <- <(([a-z] / [A-Z] / '_' / '$') ([a-z] / [A-Z] / [0-9] / '_' / '-' / 'Θ')*)> */ func() bool { - position416, tokenIndex416 := position, tokenIndex + position420, tokenIndex420 := position, tokenIndex { - position417 := position + position421 := position { - position418, tokenIndex418 := position, tokenIndex + position422, tokenIndex422 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l419 + goto l423 } position++ - goto l418 - l419: - position, tokenIndex = position418, tokenIndex418 + goto l422 + l423: + position, tokenIndex = position422, tokenIndex422 if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l424 + } + position++ + goto l422 + l424: + position, tokenIndex = position422, tokenIndex422 + if buffer[position] != rune('_') { + goto l425 + } + position++ + goto l422 + l425: + position, tokenIndex = position422, tokenIndex422 + if buffer[position] != rune('$') { goto l420 } position++ - goto l418 - l420: - position, tokenIndex = position418, tokenIndex418 - if buffer[position] != rune('_') { - goto l421 - } - position++ - goto l418 - l421: - position, tokenIndex = position418, tokenIndex418 - if buffer[position] != rune('$') { - goto l416 - } - position++ } - l418: l422: + l426: { - position423, tokenIndex423 := position, tokenIndex + position427, tokenIndex427 := position, tokenIndex { - position424, tokenIndex424 := position, tokenIndex + position428, tokenIndex428 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l425 + goto l429 } position++ - goto l424 - l425: - position, tokenIndex = position424, tokenIndex424 + goto l428 + l429: + position, tokenIndex = position428, tokenIndex428 if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l426 + goto l430 } position++ - goto l424 - l426: - position, tokenIndex = position424, tokenIndex424 + goto l428 + l430: + position, tokenIndex = position428, tokenIndex428 if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l431 + } + position++ + goto l428 + l431: + position, tokenIndex = position428, tokenIndex428 + if buffer[position] != rune('_') { + goto l432 + } + position++ + goto l428 + l432: + position, tokenIndex = position428, tokenIndex428 + if buffer[position] != rune('-') { + goto l433 + } + position++ + goto l428 + l433: + position, tokenIndex = position428, tokenIndex428 + if buffer[position] != rune('Θ') { goto l427 } position++ - goto l424 - l427: - position, tokenIndex = position424, tokenIndex424 - if buffer[position] != rune('_') { - goto l428 - } - position++ - goto l424 - l428: - position, tokenIndex = position424, tokenIndex424 - if buffer[position] != rune('-') { - goto l423 - } - position++ } - l424: - goto l422 - l423: - position, tokenIndex = position423, tokenIndex423 + l428: + goto l426 + l427: + position, tokenIndex = position427, tokenIndex427 } - add(rulefieldExpr, position417) + add(rulefieldExpr, position421) } return true - l416: - position, tokenIndex = position416, tokenIndex416 + l420: + position, tokenIndex = position420, tokenIndex420 return false }, /* 22 field <- <(<(fieldExpr / reserved)> Action61)> */ func() bool { - position429, tokenIndex429 := position, tokenIndex + position434, tokenIndex434 := position, tokenIndex { - position430 := position + position435 := position { - position431 := position + position436 := position { - position432, tokenIndex432 := position, tokenIndex + position437, tokenIndex437 := position, tokenIndex if !_rules[rulefieldExpr]() { - goto l433 + goto l438 } - goto l432 - l433: - position, tokenIndex = position432, tokenIndex432 + goto l437 + l438: + position, tokenIndex = position437, tokenIndex437 { - position434 := position + position439 := position { - position435, tokenIndex435 := position, tokenIndex + position440, tokenIndex440 := position, tokenIndex if buffer[position] != rune('_') { - goto l436 + goto l441 } position++ if buffer[position] != rune('r') { - goto l436 + goto l441 } position++ if buffer[position] != rune('o') { - goto l436 + goto l441 } position++ if buffer[position] != rune('w') { - goto l436 + goto l441 } position++ - goto l435 - l436: - position, tokenIndex = position435, tokenIndex435 + goto l440 + l441: + position, tokenIndex = position440, tokenIndex440 if buffer[position] != rune('_') { - goto l437 + goto l442 } position++ if buffer[position] != rune('c') { - goto l437 + goto l442 } position++ if buffer[position] != rune('o') { - goto l437 + goto l442 } position++ if buffer[position] != rune('l') { - goto l437 + goto l442 } position++ - goto l435 - l437: - position, tokenIndex = position435, tokenIndex435 + goto l440 + l442: + position, tokenIndex = position440, tokenIndex440 if buffer[position] != rune('_') { - goto l438 + goto l443 } position++ if buffer[position] != rune('s') { - goto l438 + goto l443 } position++ if buffer[position] != rune('t') { - goto l438 + goto l443 } position++ if buffer[position] != rune('a') { - goto l438 + goto l443 } position++ if buffer[position] != rune('r') { - goto l438 + goto l443 } position++ if buffer[position] != rune('t') { - goto l438 + goto l443 } position++ - goto l435 - l438: - position, tokenIndex = position435, tokenIndex435 + goto l440 + l443: + position, tokenIndex = position440, tokenIndex440 if buffer[position] != rune('_') { - goto l439 + goto l444 } position++ if buffer[position] != rune('e') { - goto l439 + goto l444 } position++ if buffer[position] != rune('n') { - goto l439 + goto l444 } position++ if buffer[position] != rune('d') { - goto l439 + goto l444 } position++ - goto l435 - l439: - position, tokenIndex = position435, tokenIndex435 + goto l440 + l444: + position, tokenIndex = position440, tokenIndex440 if buffer[position] != rune('_') { - goto l440 + goto l445 } position++ if buffer[position] != rune('t') { - goto l440 + goto l445 } position++ if buffer[position] != rune('i') { - goto l440 + goto l445 } position++ if buffer[position] != rune('m') { - goto l440 + goto l445 } position++ if buffer[position] != rune('e') { - goto l440 + goto l445 } position++ if buffer[position] != rune('s') { - goto l440 + goto l445 } position++ if buffer[position] != rune('t') { - goto l440 + goto l445 } position++ if buffer[position] != rune('a') { - goto l440 + goto l445 } position++ if buffer[position] != rune('m') { - goto l440 + goto l445 } position++ if buffer[position] != rune('p') { - goto l440 + goto l445 } position++ - goto l435 - l440: - position, tokenIndex = position435, tokenIndex435 + goto l440 + l445: + position, tokenIndex = position440, tokenIndex440 if buffer[position] != rune('_') { - goto l429 + goto l434 } position++ if buffer[position] != rune('f') { - goto l429 + goto l434 } position++ if buffer[position] != rune('i') { - goto l429 + goto l434 } position++ if buffer[position] != rune('e') { - goto l429 + goto l434 } position++ if buffer[position] != rune('l') { - goto l429 + goto l434 } position++ if buffer[position] != rune('d') { - goto l429 + goto l434 } position++ } - l435: - add(rulereserved, position434) + l440: + add(rulereserved, position439) } } - l432: - add(rulePegText, position431) + l437: + add(rulePegText, position436) } { add(ruleAction61, position) } - add(rulefield, position430) + add(rulefield, position435) } return true - l429: - position, tokenIndex = position429, tokenIndex429 + l434: + position, tokenIndex = position434, tokenIndex434 return false }, /* 23 reserved <- <(('_' 'r' 'o' 'w') / ('_' 'c' 'o' 'l') / ('_' 's' 't' 'a' 'r' 't') / ('_' 'e' 'n' 'd') / ('_' 't' 'i' 'm' 'e' 's' 't' 'a' 'm' 'p') / ('_' 'f' 'i' 'e' 'l' 'd'))> */ nil, /* 24 posfield <- <(('f' 'i' 'e' 'l' 'd' '=')? Action62)> */ func() bool { - position443, tokenIndex443 := position, tokenIndex + position448, tokenIndex448 := position, tokenIndex { - position444 := position + position449 := position { - position445, tokenIndex445 := position, tokenIndex + position450, tokenIndex450 := position, tokenIndex if buffer[position] != rune('f') { - goto l445 + goto l450 } position++ if buffer[position] != rune('i') { - goto l445 + goto l450 } position++ if buffer[position] != rune('e') { - goto l445 + goto l450 } position++ if buffer[position] != rune('l') { - goto l445 + goto l450 } position++ if buffer[position] != rune('d') { - goto l445 + goto l450 } position++ if buffer[position] != rune('=') { - goto l445 + goto l450 } position++ - goto l446 - l445: - position, tokenIndex = position445, tokenIndex445 + goto l451 + l450: + position, tokenIndex = position450, tokenIndex450 } - l446: + l451: { - position447 := position + position452 := position if !_rules[rulefieldExpr]() { - goto l443 + goto l448 } - add(rulePegText, position447) + add(rulePegText, position452) } { add(ruleAction62, position) } - add(ruleposfield, position444) + add(ruleposfield, position449) } return true - l443: - position, tokenIndex = position443, tokenIndex443 + l448: + position, tokenIndex = position448, tokenIndex448 return false }, /* 25 col <- <(( Action63) / (<('\'' singlequotedstring '\'')> Action64) / (<('"' doublequotedstring '"')> Action65))> */ func() bool { - position449, tokenIndex449 := position, tokenIndex + position454, tokenIndex454 := position, tokenIndex { - position450 := position + position455 := position { - position451, tokenIndex451 := position, tokenIndex + position456, tokenIndex456 := position, tokenIndex { - position453 := position + position458 := position if !_rules[ruledigits]() { - goto l452 + goto l457 } - add(rulePegText, position453) + add(rulePegText, position458) } { add(ruleAction63, position) } - goto l451 - l452: - position, tokenIndex = position451, tokenIndex451 + goto l456 + l457: + position, tokenIndex = position456, tokenIndex456 { - position456 := position + position461 := position if buffer[position] != rune('\'') { - goto l455 + goto l460 } position++ if !_rules[rulesinglequotedstring]() { - goto l455 + goto l460 } if buffer[position] != rune('\'') { - goto l455 + goto l460 } position++ - add(rulePegText, position456) + add(rulePegText, position461) } { add(ruleAction64, position) } - goto l451 - l455: - position, tokenIndex = position451, tokenIndex451 + goto l456 + l460: + position, tokenIndex = position456, tokenIndex456 { - position458 := position + position463 := position if buffer[position] != rune('"') { - goto l449 + goto l454 } position++ if !_rules[ruledoublequotedstring]() { - goto l449 + goto l454 } if buffer[position] != rune('"') { - goto l449 + goto l454 } position++ - add(rulePegText, position458) + add(rulePegText, position463) } { add(ruleAction65, position) } } - l451: - add(rulecol, position450) + l456: + add(rulecol, position455) } return true - l449: - position, tokenIndex = position449, tokenIndex449 + l454: + position, tokenIndex = position454, tokenIndex454 return false }, /* 26 open <- <('(' sp)> */ func() bool { - position460, tokenIndex460 := position, tokenIndex + position465, tokenIndex465 := position, tokenIndex { - position461 := position + position466 := position if buffer[position] != rune('(') { - goto l460 + goto l465 } position++ if !_rules[rulesp]() { - goto l460 + goto l465 } - add(ruleopen, position461) + add(ruleopen, position466) } return true - l460: - position, tokenIndex = position460, tokenIndex460 + l465: + position, tokenIndex = position465, tokenIndex465 return false }, /* 27 close <- <(sp ')' sp)> */ func() bool { - position462, tokenIndex462 := position, tokenIndex + position467, tokenIndex467 := position, tokenIndex { - position463 := position + position468 := position if !_rules[rulesp]() { - goto l462 + goto l467 } if buffer[position] != rune(')') { - goto l462 + goto l467 } position++ if !_rules[rulesp]() { - goto l462 + goto l467 } - add(ruleclose, position463) + add(ruleclose, position468) } return true - l462: - position, tokenIndex = position462, tokenIndex462 + l467: + position, tokenIndex = position467, tokenIndex467 return false }, /* 28 sp <- <(' ' / '\t' / '\n')*> */ func() bool { { - position465 := position - l466: + position470 := position + l471: { - position467, tokenIndex467 := position, tokenIndex + position472, tokenIndex472 := position, tokenIndex { - position468, tokenIndex468 := position, tokenIndex + position473, tokenIndex473 := position, tokenIndex if buffer[position] != rune(' ') { - goto l469 + goto l474 } position++ - goto l468 - l469: - position, tokenIndex = position468, tokenIndex468 + goto l473 + l474: + position, tokenIndex = position473, tokenIndex473 if buffer[position] != rune('\t') { - goto l470 + goto l475 } position++ - goto l468 - l470: - position, tokenIndex = position468, tokenIndex468 + goto l473 + l475: + position, tokenIndex = position473, tokenIndex473 if buffer[position] != rune('\n') { - goto l467 + goto l472 } position++ } - l468: - goto l466 - l467: - position, tokenIndex = position467, tokenIndex467 + l473: + goto l471 + l472: + position, tokenIndex = position472, tokenIndex472 } - add(rulesp, position465) + add(rulesp, position470) } return true }, /* 29 eq <- <(sp '=' sp)> */ func() bool { - position471, tokenIndex471 := position, tokenIndex + position476, tokenIndex476 := position, tokenIndex { - position472 := position + position477 := position if !_rules[rulesp]() { - goto l471 + goto l476 } if buffer[position] != rune('=') { - goto l471 + goto l476 } position++ if !_rules[rulesp]() { - goto l471 + goto l476 } - add(ruleeq, position472) + add(ruleeq, position477) } return true - l471: - position, tokenIndex = position471, tokenIndex471 + l476: + position, tokenIndex = position476, tokenIndex476 return false }, /* 30 comma <- <(sp ',' sp)> */ func() bool { - position473, tokenIndex473 := position, tokenIndex + position478, tokenIndex478 := position, tokenIndex { - position474 := position + position479 := position if !_rules[rulesp]() { - goto l473 + goto l478 } if buffer[position] != rune(',') { - goto l473 + goto l478 } position++ if !_rules[rulesp]() { - goto l473 + goto l478 } - add(rulecomma, position474) + add(rulecomma, position479) } return true - l473: - position, tokenIndex = position473, tokenIndex473 + l478: + position, tokenIndex = position478, tokenIndex478 return false }, /* 31 lbrack <- <('[' sp)> */ nil, /* 32 rbrack <- <(sp ']' sp)> */ nil, - /* 33 IDENT <- <(([a-z] / [A-Z]) ([a-z] / [A-Z] / [0-9])*)> */ + /* 33 IDENT <- <(([a-z] / [A-Z]) ([a-z] / [A-Z] / [0-9] / 'Θ')*)> */ func() bool { - position477, tokenIndex477 := position, tokenIndex + position482, tokenIndex482 := position, tokenIndex { - position478 := position + position483 := position { - position479, tokenIndex479 := position, tokenIndex + position484, tokenIndex484 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l480 + goto l485 } position++ - goto l479 - l480: - position, tokenIndex = position479, tokenIndex479 + goto l484 + l485: + position, tokenIndex = position484, tokenIndex484 if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l477 + goto l482 } position++ } - l479: - l481: + l484: + l486: { - position482, tokenIndex482 := position, tokenIndex + position487, tokenIndex487 := position, tokenIndex { - position483, tokenIndex483 := position, tokenIndex + position488, tokenIndex488 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l484 + goto l489 } position++ - goto l483 - l484: - position, tokenIndex = position483, tokenIndex483 + goto l488 + l489: + position, tokenIndex = position488, tokenIndex488 if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l485 + goto l490 } position++ - goto l483 - l485: - position, tokenIndex = position483, tokenIndex483 + goto l488 + l490: + position, tokenIndex = position488, tokenIndex488 if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l482 + goto l491 + } + position++ + goto l488 + l491: + position, tokenIndex = position488, tokenIndex488 + if buffer[position] != rune('Θ') { + goto l487 } position++ } - l483: - goto l481 - l482: - position, tokenIndex = position482, tokenIndex482 + l488: + goto l486 + l487: + position, tokenIndex = position487, tokenIndex487 } - add(ruleIDENT, position478) + add(ruleIDENT, position483) } return true - l477: - position, tokenIndex = position477, tokenIndex477 + l482: + position, tokenIndex = position482, tokenIndex482 return false }, /* 34 digits <- <[0-9]+> */ func() bool { - position486, tokenIndex486 := position, tokenIndex + position492, tokenIndex492 := position, tokenIndex { - position487 := position + position493 := position if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l486 + goto l492 } position++ - l488: + l494: { - position489, tokenIndex489 := position, tokenIndex + position495, tokenIndex495 := position, tokenIndex if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l489 + goto l495 } position++ - goto l488 - l489: - position, tokenIndex = position489, tokenIndex489 + goto l494 + l495: + position, tokenIndex = position495, tokenIndex495 } - add(ruledigits, position487) + add(ruledigits, position493) } return true - l486: - position, tokenIndex = position486, tokenIndex486 + l492: + position, tokenIndex = position492, tokenIndex492 return false }, /* 35 signedDigits <- <('-'? digits)> */ nil, /* 36 decimal <- <((signedDigits ('.' digits?)?) / ('-'? '.' digits))> */ func() bool { - position491, tokenIndex491 := position, tokenIndex + position497, tokenIndex497 := position, tokenIndex { - position492 := position + position498 := position { - position493, tokenIndex493 := position, tokenIndex + position499, tokenIndex499 := position, tokenIndex { - position495 := position + position501 := position { - position496, tokenIndex496 := position, tokenIndex + position502, tokenIndex502 := position, tokenIndex if buffer[position] != rune('-') { - goto l496 + goto l502 } position++ - goto l497 - l496: - position, tokenIndex = position496, tokenIndex496 + goto l503 + l502: + position, tokenIndex = position502, tokenIndex502 } - l497: + l503: if !_rules[ruledigits]() { - goto l494 + goto l500 } - add(rulesignedDigits, position495) + add(rulesignedDigits, position501) } { - position498, tokenIndex498 := position, tokenIndex + position504, tokenIndex504 := position, tokenIndex if buffer[position] != rune('.') { - goto l498 + goto l504 } position++ { - position500, tokenIndex500 := position, tokenIndex + position506, tokenIndex506 := position, tokenIndex if !_rules[ruledigits]() { - goto l500 + goto l506 } - goto l501 - l500: - position, tokenIndex = position500, tokenIndex500 + goto l507 + l506: + position, tokenIndex = position506, tokenIndex506 } - l501: - goto l499 - l498: - position, tokenIndex = position498, tokenIndex498 + l507: + goto l505 + l504: + position, tokenIndex = position504, tokenIndex504 } - l499: - goto l493 - l494: - position, tokenIndex = position493, tokenIndex493 + l505: + goto l499 + l500: + position, tokenIndex = position499, tokenIndex499 { - position502, tokenIndex502 := position, tokenIndex + position508, tokenIndex508 := position, tokenIndex if buffer[position] != rune('-') { - goto l502 + goto l508 } position++ - goto l503 - l502: - position, tokenIndex = position502, tokenIndex502 + goto l509 + l508: + position, tokenIndex = position508, tokenIndex508 } - l503: + l509: if buffer[position] != rune('.') { - goto l491 + goto l497 } position++ if !_rules[ruledigits]() { - goto l491 + goto l497 } } - l493: - add(ruledecimal, position492) + l499: + add(ruledecimal, position498) } return true - l491: - position, tokenIndex = position491, tokenIndex491 + l497: + position, tokenIndex = position497, tokenIndex497 return false }, /* 37 tz <- <('Z' / ('-' [0-9] [0-9] ':' [0-9] [0-9]) / ('+' [0-9] [0-9] ':' [0-9] [0-9]))> */ func() bool { - position504, tokenIndex504 := position, tokenIndex + position510, tokenIndex510 := position, tokenIndex { - position505 := position + position511 := position { - position506, tokenIndex506 := position, tokenIndex + position512, tokenIndex512 := position, tokenIndex if buffer[position] != rune('Z') { - goto l507 + goto l513 } position++ - goto l506 - l507: - position, tokenIndex = position506, tokenIndex506 + goto l512 + l513: + position, tokenIndex = position512, tokenIndex512 if buffer[position] != rune('-') { - goto l508 + goto l514 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l508 + goto l514 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l508 + goto l514 } position++ if buffer[position] != rune(':') { - goto l508 + goto l514 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l508 + goto l514 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l508 + goto l514 } position++ - goto l506 - l508: - position, tokenIndex = position506, tokenIndex506 + goto l512 + l514: + position, tokenIndex = position512, tokenIndex512 if buffer[position] != rune('+') { - goto l504 + goto l510 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l504 + goto l510 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l504 + goto l510 } position++ if buffer[position] != rune(':') { - goto l504 + goto l510 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l504 + goto l510 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l504 + goto l510 } position++ } - l506: - add(ruletz, position505) + l512: + add(ruletz, position511) } return true - l504: - position, tokenIndex = position504, tokenIndex504 + l510: + position, tokenIndex = position510, tokenIndex510 return false }, /* 38 iso8601 <- <([0-9] [0-9] [0-9] [0-9] '-' ('0' / '1') [0-9] '-' [0-3] [0-9] 'T' [0-9] [0-9] ':' [0-9] [0-9] ':' [0-9] [0-9] )> */ @@ -4265,151 +4308,31 @@ func (p *PQL) Init(options ...func(*PQL) error) error { nil, /* 40 timestampbasicfmt <- <(iso8601nano / iso8601)> */ func() bool { - position511, tokenIndex511 := position, tokenIndex + position517, tokenIndex517 := position, tokenIndex { - position512 := position + position518 := position { - position513, tokenIndex513 := position, tokenIndex - { - position515 := position - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l514 - } - position++ - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l514 - } - position++ - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l514 - } - position++ - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l514 - } - position++ - if buffer[position] != rune('-') { - goto l514 - } - position++ - { - position516, tokenIndex516 := position, tokenIndex - if buffer[position] != rune('0') { - goto l517 - } - position++ - goto l516 - l517: - position, tokenIndex = position516, tokenIndex516 - if buffer[position] != rune('1') { - goto l514 - } - position++ - } - l516: - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l514 - } - position++ - if buffer[position] != rune('-') { - goto l514 - } - position++ - if c := buffer[position]; c < rune('0') || c > rune('3') { - goto l514 - } - position++ - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l514 - } - position++ - if buffer[position] != rune('T') { - goto l514 - } - position++ - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l514 - } - position++ - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l514 - } - position++ - if buffer[position] != rune(':') { - goto l514 - } - position++ - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l514 - } - position++ - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l514 - } - position++ - if buffer[position] != rune(':') { - goto l514 - } - position++ - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l514 - } - position++ - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l514 - } - position++ - if buffer[position] != rune('.') { - goto l514 - } - position++ - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l514 - } - position++ - l518: - { - position519, tokenIndex519 := position, tokenIndex - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l519 - } - position++ - goto l518 - l519: - position, tokenIndex = position519, tokenIndex519 - } - { - position520 := position - if !_rules[ruletz]() { - goto l514 - } - add(rulePegText, position520) - } - add(ruleiso8601nano, position515) - } - goto l513 - l514: - position, tokenIndex = position513, tokenIndex513 + position519, tokenIndex519 := position, tokenIndex { position521 := position if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l511 + goto l520 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l511 + goto l520 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l511 + goto l520 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l511 + goto l520 } position++ if buffer[position] != rune('-') { - goto l511 + goto l520 } position++ { @@ -4422,284 +4345,404 @@ func (p *PQL) Init(options ...func(*PQL) error) error { l523: position, tokenIndex = position522, tokenIndex522 if buffer[position] != rune('1') { - goto l511 + goto l520 } position++ } l522: if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l511 + goto l520 } position++ if buffer[position] != rune('-') { - goto l511 + goto l520 } position++ if c := buffer[position]; c < rune('0') || c > rune('3') { - goto l511 + goto l520 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l511 + goto l520 } position++ if buffer[position] != rune('T') { - goto l511 + goto l520 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l511 + goto l520 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l511 + goto l520 } position++ if buffer[position] != rune(':') { - goto l511 + goto l520 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l511 + goto l520 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l511 + goto l520 } position++ if buffer[position] != rune(':') { - goto l511 + goto l520 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l511 + goto l520 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l511 + goto l520 + } + position++ + if buffer[position] != rune('.') { + goto l520 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l520 + } + position++ + l524: + { + position525, tokenIndex525 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l525 + } + position++ + goto l524 + l525: + position, tokenIndex = position525, tokenIndex525 + } + { + position526 := position + if !_rules[ruletz]() { + goto l520 + } + add(rulePegText, position526) + } + add(ruleiso8601nano, position521) + } + goto l519 + l520: + position, tokenIndex = position519, tokenIndex519 + { + position527 := position + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l517 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l517 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l517 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l517 + } + position++ + if buffer[position] != rune('-') { + goto l517 } position++ { - position524 := position - if !_rules[ruletz]() { - goto l511 + position528, tokenIndex528 := position, tokenIndex + if buffer[position] != rune('0') { + goto l529 } - add(rulePegText, position524) + position++ + goto l528 + l529: + position, tokenIndex = position528, tokenIndex528 + if buffer[position] != rune('1') { + goto l517 + } + position++ } - add(ruleiso8601, position521) + l528: + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l517 + } + position++ + if buffer[position] != rune('-') { + goto l517 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('3') { + goto l517 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l517 + } + position++ + if buffer[position] != rune('T') { + goto l517 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l517 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l517 + } + position++ + if buffer[position] != rune(':') { + goto l517 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l517 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l517 + } + position++ + if buffer[position] != rune(':') { + goto l517 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l517 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l517 + } + position++ + { + position530 := position + if !_rules[ruletz]() { + goto l517 + } + add(rulePegText, position530) + } + add(ruleiso8601, position527) } } - l513: - add(ruletimestampbasicfmt, position512) + l519: + add(ruletimestampbasicfmt, position518) } return true - l511: - position, tokenIndex = position511, tokenIndex511 + l517: + position, tokenIndex = position517, tokenIndex517 return false }, /* 41 timestampfmt <- <(('"' '"') / ('\'' '\'') / )> */ func() bool { - position525, tokenIndex525 := position, tokenIndex + position531, tokenIndex531 := position, tokenIndex { - position526 := position + position532 := position { - position527, tokenIndex527 := position, tokenIndex + position533, tokenIndex533 := position, tokenIndex if buffer[position] != rune('"') { - goto l528 + goto l534 } position++ { - position529 := position + position535 := position if !_rules[ruletimestampbasicfmt]() { - goto l528 + goto l534 } - add(rulePegText, position529) + add(rulePegText, position535) } if buffer[position] != rune('"') { - goto l528 + goto l534 } position++ - goto l527 - l528: - position, tokenIndex = position527, tokenIndex527 + goto l533 + l534: + position, tokenIndex = position533, tokenIndex533 if buffer[position] != rune('\'') { - goto l530 + goto l536 } position++ { - position531 := position + position537 := position if !_rules[ruletimestampbasicfmt]() { - goto l530 + goto l536 } - add(rulePegText, position531) + add(rulePegText, position537) } if buffer[position] != rune('\'') { - goto l530 + goto l536 } position++ - goto l527 - l530: - position, tokenIndex = position527, tokenIndex527 + goto l533 + l536: + position, tokenIndex = position533, tokenIndex533 { - position532 := position + position538 := position if !_rules[ruletimestampbasicfmt]() { - goto l525 + goto l531 } - add(rulePegText, position532) + add(rulePegText, position538) } } - l527: - add(ruletimestampfmt, position526) + l533: + add(ruletimestampfmt, position532) } return true - l525: - position, tokenIndex = position525, tokenIndex525 + l531: + position, tokenIndex = position531, tokenIndex531 return false }, /* 42 timebasicfmt <- <([0-9] [0-9] [0-9] [0-9] '-' ('0' / '1') [0-9] '-' [0-3] [0-9] 'T' [0-9] [0-9] ':' [0-9] [0-9])> */ func() bool { - position533, tokenIndex533 := position, tokenIndex + position539, tokenIndex539 := position, tokenIndex { - position534 := position + position540 := position if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l533 + goto l539 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l533 + goto l539 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l533 + goto l539 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l533 + goto l539 } position++ if buffer[position] != rune('-') { - goto l533 + goto l539 } position++ { - position535, tokenIndex535 := position, tokenIndex + position541, tokenIndex541 := position, tokenIndex if buffer[position] != rune('0') { - goto l536 + goto l542 } position++ - goto l535 - l536: - position, tokenIndex = position535, tokenIndex535 + goto l541 + l542: + position, tokenIndex = position541, tokenIndex541 if buffer[position] != rune('1') { - goto l533 + goto l539 } position++ } - l535: + l541: if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l533 + goto l539 } position++ if buffer[position] != rune('-') { - goto l533 + goto l539 } position++ if c := buffer[position]; c < rune('0') || c > rune('3') { - goto l533 + goto l539 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l533 + goto l539 } position++ if buffer[position] != rune('T') { - goto l533 + goto l539 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l533 + goto l539 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l533 + goto l539 } position++ if buffer[position] != rune(':') { - goto l533 + goto l539 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l533 + goto l539 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l533 + goto l539 } position++ - add(ruletimebasicfmt, position534) + add(ruletimebasicfmt, position540) } return true - l533: - position, tokenIndex = position533, tokenIndex533 + l539: + position, tokenIndex = position539, tokenIndex539 return false }, /* 43 timefmt <- <(('"' '"') / ('\'' '\'') / )> */ func() bool { - position537, tokenIndex537 := position, tokenIndex + position543, tokenIndex543 := position, tokenIndex { - position538 := position + position544 := position { - position539, tokenIndex539 := position, tokenIndex + position545, tokenIndex545 := position, tokenIndex if buffer[position] != rune('"') { - goto l540 + goto l546 } position++ { - position541 := position + position547 := position if !_rules[ruletimebasicfmt]() { - goto l540 + goto l546 } - add(rulePegText, position541) + add(rulePegText, position547) } if buffer[position] != rune('"') { - goto l540 + goto l546 } position++ - goto l539 - l540: - position, tokenIndex = position539, tokenIndex539 + goto l545 + l546: + position, tokenIndex = position545, tokenIndex545 if buffer[position] != rune('\'') { - goto l542 + goto l548 } position++ { - position543 := position + position549 := position if !_rules[ruletimebasicfmt]() { - goto l542 + goto l548 } - add(rulePegText, position543) + add(rulePegText, position549) } if buffer[position] != rune('\'') { - goto l542 + goto l548 } position++ - goto l539 - l542: - position, tokenIndex = position539, tokenIndex539 + goto l545 + l548: + position, tokenIndex = position545, tokenIndex545 { - position544 := position + position550 := position if !_rules[ruletimebasicfmt]() { - goto l537 + goto l543 } - add(rulePegText, position544) + add(rulePegText, position550) } } - l539: - add(ruletimefmt, position538) + l545: + add(ruletimefmt, position544) } return true - l537: - position, tokenIndex = position537, tokenIndex537 + l543: + position, tokenIndex = position543, tokenIndex543 return false }, /* 44 time <- <( Action66)> */ diff --git a/pql/pqlpeg_test.go b/pql/pqlpeg_test.go index 93fecba9f..83fb24c27 100644 --- a/pql/pqlpeg_test.go +++ b/pql/pqlpeg_test.go @@ -353,7 +353,7 @@ func TestPQLDeepEquality(t *testing.T) { }, }}, { - name: "RowWithUnicode", + name: "RowValWithUnicode", call: `Row(unicode="Æ漢д ☮♬ ♞🜻💣")`, exp: &Call{ Name: "Row", @@ -361,6 +361,15 @@ func TestPQLDeepEquality(t *testing.T) { "unicode": `Æ漢д ☮♬ ♞🜻💣`, }, }}, + { + name: "RowWithUnicode", + call: `Row(uniΘcode="Æ漢д ☮♬ ♞🜻💣")`, + exp: &Call{ + Name: "Row", + Args: map[string]interface{}{ + "uniΘcode": `Æ漢д ☮♬ ♞🜻💣`, + }, + }}, { name: "RowsWithUnicode", call: `Rows(job, previous="💣")`, diff --git a/server.go b/server.go index 007cd6add..b04feea7f 100644 --- a/server.go +++ b/server.go @@ -89,6 +89,8 @@ type Server struct { // nolint: maligned defaultClient *InternalClient dataDir string + verChkAddress string + uuidFile string // Threshold for logging long-running queries longQueryTime time.Duration @@ -167,6 +169,24 @@ func OptServerDataDir(dir string) ServerOption { } } +// OptServerVerChkAddress is a functional option on Server +// used to set the address to check for the current version. +func OptServerVerChkAddress(addr string) ServerOption { + return func(s *Server) error { + s.verChkAddress = addr + return nil + } +} + +// OptServerUUIDFile is a functional option on Server +// used to set the file name for storing the checkin UUID. +func OptServerUUIDFile(uf string) ServerOption { + return func(s *Server) error { + s.uuidFile = uf + return nil + } +} + // OptServerViewsRemovalInterval is a functional option on Server // used to set the ttl removal interval. func OptServerViewsRemovalInterval(interval time.Duration) ServerOption { @@ -604,16 +624,21 @@ func (s *Server) Open() error { log.Println(errors.Wrap(err, "logging startup")) } - // Do version check in. This is in a goroutine so that we don't block server startup if the server endpoint is down/having issues. + // Do version check in. This is in a goroutine so that we don't block server + // startup if the server endpoint is down/having issues. go func() { s.logger.Printf("Beginning featurebase version check-in") - vc := VersionChecker{URL: "https://analytics.featurebase.com/v2/featurebase/metrics"} - resp, err := vc.CheckIn() + vc := newVersionChecker(s.cluster.Path, s.verChkAddress, s.uuidFile) + resp, err := vc.checkIn() if err != nil { s.logger.Errorf("doing version checkin. Error was %s", err) return } - s.logger.Printf("Version check-in complete. Latest version is %s", resp.Version) + if resp.Error != "" { + s.logger.Printf("Version check-in failed, endpoint response was %s", resp.Error) + } else { + s.logger.Printf("Version check-in complete. Latest version is %s", resp.Version) + } }() // Start DisCo. diff --git a/server/config.go b/server/config.go index f2d42e8ac..92e86c10d 100644 --- a/server/config.go +++ b/server/config.go @@ -152,6 +152,15 @@ type Config struct { // Limits the total amount of memory to be used by Extract() & SELECT queries. MaxQueryMemory int64 `toml:"max-query-memory"` + // On startup, featurebase server contacts a web server to check the latest version. + // This stores the address for that check + VerChkAddress string `toml:"verchk-address"` + + // When checking version, server sends a UUID so that we can keep track of + // how many unique Featurebase installs are out there. The file is stored in + // the data directory; this stores the filename to use. + UUIDFile string `toml:"uuid-file"` + Cluster struct { ReplicaN int `toml:"replicas"` Name string `toml:"name"` @@ -383,6 +392,8 @@ func NewConfig() *Config { LongQueryTime: toml.Duration(-time.Minute), CheckInInterval: 60 * time.Second, + VerChkAddress: "https://analytics.featurebase.com/v2/featurebase/metrics", + UUIDFile: ".client_id.txt", } // Cluster config. diff --git a/server/server.go b/server/server.go index fb50cb2c8..89e9985d5 100644 --- a/server/server.go +++ b/server/server.go @@ -73,6 +73,8 @@ type Command struct { // done will be closed when Command.Close() is called done chan struct{} + traceCloser io.Closer + logOutput io.Writer queryLogOutput io.Writer logger loggerLogger @@ -596,6 +598,8 @@ func (m *Command) setupServer() error { pilosa.OptServerServerlessStorage(m.serverlessStorage), pilosa.OptServerIsDataframeEnabled(m.Config.Dataframe.Enable), pilosa.OptServerDataframeUseParquet(m.Config.Dataframe.UseParquet), + pilosa.OptServerVerChkAddress(m.Config.VerChkAddress), + pilosa.OptServerUUIDFile(m.Config.UUIDFile), } if m.isComputeNode { @@ -832,7 +836,6 @@ func (m *Command) setupProfilingAndTracing() error { if err != nil { return errors.Wrap(err, "starting datadog") } - defer profiler.Stop() } if m.Config.Tracing.SamplerType != "off" { @@ -850,12 +853,10 @@ func (m *Command) setupProfilingAndTracing() error { if err != nil { return errors.Wrap(err, "initializing jaeger tracer") } - defer closer.Close() + m.traceCloser = closer tracing.GlobalTracer = opentracing.NewTracer(tracer, m.Logger()) - } else if m.Config.DataDog.EnableTracing { // Give preference to legacy support of jaeger t := opentracer.New(tracer.WithServiceName(m.Config.DataDog.Service)) - defer tracer.Stop() tracing.GlobalTracer = opentracing.NewTracer(t, m.Logger()) } return nil @@ -881,6 +882,14 @@ func (m *Command) Close() error { err := eg.Wait() _ = testhook.Closed(pilosa.NewAuditor(), m, nil) + if m.Config.DataDog.Enable { + defer profiler.Stop() + } + if m.traceCloser != nil { + defer m.traceCloser.Close() + } else if m.Config.DataDog.EnableTracing { + defer tracer.Stop() + } close(m.done) return errors.Wrap(err, "closing everything") diff --git a/sql3/parser/ast_test.go b/sql3/parser/ast_test.go index eb25286fe..dcddcfaa3 100644 --- a/sql3/parser/ast_test.go +++ b/sql3/parser/ast_test.go @@ -240,9 +240,7 @@ func TestCreateTableStatement_String(t *testing.T) { Type: &parser.Type{Name: &parser.Ident{Name: "TIMESTAMP"}}, Constraints: []parser.Constraint{ &parser.TimeUnitConstraint{ - Expr: &parser.StringLit{Value: "s"}, - Epoch: pos(0), - EpochExpr: &parser.StringLit{Value: "2021-01-01T00:00:00Z"}, + Expr: &parser.StringLit{Value: "s"}, }, }, }, @@ -255,7 +253,7 @@ func TestCreateTableStatement_String(t *testing.T) { `intcol INTEGER MIN 100 MAX 1000, `+ `stringcol STRING CACHETYPE RANKED SIZE 10000, `+ `stringsetcol STRINGSET CACHETYPE RANKED SIZE 10000, `+ - `timestampcol TIMESTAMP TIMEUNIT 's' EPOCH '2021-01-01T00:00:00Z'`+ + `timestampcol TIMESTAMP TIMEUNIT 's'`+ `)`) } diff --git a/sql3/parser/parser.go b/sql3/parser/parser.go index 8b290c324..32ad9c593 100644 --- a/sql3/parser/parser.go +++ b/sql3/parser/parser.go @@ -860,15 +860,6 @@ func (p *Parser) parseTimeUnitConstraint(constraintPos Pos, name *Ident) (_ *Tim } else { return &cons, p.errorExpected(p.pos, p.tok, "literal") } - if p.peek() == EPOCH { - cons.Epoch, _, _ = p.scan() - - if isLiteralToken(p.peek()) { - cons.EpochExpr = p.mustParseLiteral() - } else { - return &cons, p.errorExpected(p.pos, p.tok, "literal") - } - } return &cons, nil } diff --git a/sql3/planner/compileselect.go b/sql3/planner/compileselect.go index 9a03bda0f..940aff215 100644 --- a/sql3/planner/compileselect.go +++ b/sql3/planner/compileselect.go @@ -11,7 +11,6 @@ import ( "github.com/featurebasedb/featurebase/v3/sql3" "github.com/featurebasedb/featurebase/v3/sql3/parser" "github.com/featurebasedb/featurebase/v3/sql3/planner/types" - "github.com/pkg/errors" ) // compileSelectStatment compiles a parser.SelectStatment AST into a PlanOperator @@ -20,12 +19,12 @@ func (p *ExecutionPlanner) compileSelectStatement(stmt *parser.SelectStatement, aggregates := make([]types.PlanExpression, 0) - // handle projections + // compile select list and generate a list of projections projections := make([]types.PlanExpression, 0) for _, c := range stmt.Columns { planExpr, err := p.compileExpr(c.Expr) if err != nil { - return nil, errors.Wrap(err, "planning select column expression") + return nil, err } if c.Alias != nil { planExpr = newAliasPlanExpression(c.Alias.Name, planExpr) @@ -34,7 +33,7 @@ func (p *ExecutionPlanner) compileSelectStatement(stmt *parser.SelectStatement, aggregates = p.gatherExprAggregates(planExpr, aggregates) } - // group by clause. + // compile group by clause and generate a list of group by expressions groupByExprs := make([]types.PlanExpression, 0) for _, expr := range stmt.GroupByExprs { switch expr := expr.(type) { @@ -44,32 +43,32 @@ func (p *ExecutionPlanner) compileSelectStatement(stmt *parser.SelectStatement, return nil, sql3.NewErrInternalf("unsupported expression type in GROUP BY clause: %T", expr) } } - var err error - // handle the where clause + // compile the where clause where, err := p.compileExpr(stmt.WhereExpr) if err != nil { return nil, err } - // source expression + // compile source expression source, err := p.compileSource(query, stmt.Source) if err != nil { return nil, err } - // if we did have a where, insert the filter op + // if we did have a where, insert the filter op after source if where != nil { aggregates = p.gatherExprAggregates(where, aggregates) source = NewPlanOpFilter(p, where, source) } - // handle the having clause + // compile the having clause having, err := p.compileExpr(stmt.HavingExpr) if err != nil { return nil, err } + // if we have a having, check references if having != nil { // gather aggregates aggregates = p.gatherExprAggregates(having, aggregates) @@ -129,9 +128,49 @@ func (p *ExecutionPlanner) compileSelectStatement(stmt *parser.SelectStatement, } } - // do we have straight projection or a group by? + // compile order by and generate a list of ordering expressions + orderByExprs := make([]*OrderByExpression, 0) + nonReferenceOrderByExpressions := make([]types.PlanExpression, 0) + if len(stmt.OrderingTerms) > 0 { + for _, ot := range stmt.OrderingTerms { + // compile the ordering term + expr, err := p.compileOrderingTermExpr(ot.X, projections, stmt.Source) + if err != nil { + return nil, err + } + + f := &OrderByExpression{ + Expr: expr, + } + f.Order = orderByAsc + if ot.Desc.IsValid() { + f.Order = orderByDesc + } + orderByExprs = append(orderByExprs, f) + } + + // if the expression is just references, we + // can put the sort directly after the source + for _, oe := range orderByExprs { + _, ok := oe.Expr.(*qualifiedRefPlanExpression) + if !ok { + nonReferenceOrderByExpressions = append(nonReferenceOrderByExpressions, oe.Expr) + } + } + + // all the order by expressions are references, so we can put the order by before the + // projection + if len(nonReferenceOrderByExpressions) == 0 { + source = NewPlanOpOrderBy(orderByExprs, source) + } + } + var compiledOp types.PlanOperator + + // do we have straight projection or a group by? if len(aggregates) > 0 { + // we have a group by + //check that any projections that are not aggregates are in the group by list nonAggregateReferences := make([]*qualifiedRefPlanExpression, 0) for _, expr := range projections { @@ -172,37 +211,98 @@ func (p *ExecutionPlanner) compileSelectStatement(stmt *parser.SelectStatement, } compiledOp = NewPlanOpProjection(projections, groupByOp) } else { + // no group by, just a straight projection compiledOp = NewPlanOpProjection(projections, source) } - // handle order by - if len(stmt.OrderingTerms) > 0 { - orderByFields := make([]*OrderByExpression, 0) - for _, ot := range stmt.OrderingTerms { - index, err := p.compileOrderingTermExpr(ot.X) - if err != nil { - return nil, err - } - // get the data type from the projection - projDataType := projections[index].Type() + // handle the case where we have order by expressions and they are not references + // in this case we need to put the order by after the projection + if len(orderByExprs) > 0 && len(nonReferenceOrderByExpressions) > 0 { - // don't let a sort happen on something unsortable right now - switch projDataType.(type) { - case *parser.DataTypeStringSet, *parser.DataTypeIDSet: - return nil, sql3.NewErrExpectedSortableExpression(0, 0, projDataType.TypeDescription()) - } + // if the order by expressions contain a reference not in the projection list, + // we have to create a new projection, add references to current projection, + // and place the new order by in between - f := &OrderByExpression{ - Index: index, - ExprType: projDataType, + // get a list of all the refs for the order by exprs + orderByRefs := make(map[string]*qualifiedRefPlanExpression) + for _, oe := range orderByExprs { + ex, ok := oe.Expr.(*qualifiedRefPlanExpression) + if ok { + orderByRefs[ex.String()] = ex } - f.Order = orderByAsc - if ot.Desc.IsValid() { - f.Order = orderByDesc - } - orderByFields = append(orderByFields, f) } - compiledOp = NewPlanOpOrderBy(orderByFields, compiledOp) + + // get a list of all the projection refs + projRefs := make(map[string]*qualifiedRefPlanExpression) + for _, p := range projections { + InspectExpression(p, func(expr types.PlanExpression) bool { + switch ex := expr.(type) { + case *qualifiedRefPlanExpression: + projRefs[ex.String()] = ex + return false + } + return true + }) + } + + // iterate the order by terms, make a list of the ones not projected + unprojectedRefs := make([]*qualifiedRefPlanExpression, 0) + for kobr, obr := range orderByRefs { + _, found := projRefs[kobr] + if !found { + unprojectedRefs = append(unprojectedRefs, obr) + } + } + + // sigh - ok. If we have unprojected refs, we need to insert a projection + if len(unprojectedRefs) > 0 { + // create the final projection list - this will go before the order by + newProjections := make([]types.PlanExpression, len(projections)) + for i, p := range projections { + switch pe := p.(type) { + case *aliasPlanExpression: + newProjections[i] = newQualifiedRefPlanExpression("", pe.aliasName, i, pe.Type()) + case *qualifiedRefPlanExpression: + newProjections[i] = newQualifiedRefPlanExpression(pe.tableName, pe.columnName, i, pe.Type()) + default: + newProjections[i] = newQualifiedRefPlanExpression("", p.String(), i, p.Type()) + } + } + + // add the unprojected refs to the existing projection op + projectionOp, ok := compiledOp.(*PlanOpProjection) + if !ok { + return nil, sql3.NewErrInternalf("unexpected compiledOp type '%T'", compiledOp) + } + for _, uref := range unprojectedRefs { + projectionOp.Projections = append(projectionOp.Projections, uref) + } + + // add the order by on top of this + // rewrite all the order by expressions that are not qualified refs to be qualified + // refs referring to the expression + for i, oe := range orderByExprs { + _, ok := oe.Expr.(*qualifiedRefPlanExpression) + if !ok { + orderByExprs[i].Expr = newQualifiedRefPlanExpression("", oe.Expr.String(), 0, oe.Expr.Type()) + } + } + compiledOp = NewPlanOpOrderBy(orderByExprs, compiledOp) + + // add the final projection on top of this + compiledOp = NewPlanOpProjection(newProjections, compiledOp) + + } else { + // rewrite all the order by expressions that are not qualified refs to be qualified + // refs referring to the expression + for i, oe := range orderByExprs { + _, ok := oe.Expr.(*qualifiedRefPlanExpression) + if !ok { + orderByExprs[i].Expr = newQualifiedRefPlanExpression("", oe.Expr.String(), 0, oe.Expr.Type()) + } + } + compiledOp = NewPlanOpOrderBy(orderByExprs, compiledOp) + } } // insert the top operator if it exists @@ -583,11 +683,10 @@ func (p *ExecutionPlanner) analyzeSelectStatement(ctx context.Context, stmt *par } for _, term := range stmt.OrderingTerms { - expr, err := p.analyzeOrderingTermExpression(term.X, stmt) + err := p.analyzeOrderingTermExpression(term.X, stmt) if err != nil { return nil, err } - term.X = expr } return stmt, nil diff --git a/sql3/planner/expression.go b/sql3/planner/expression.go index 292e0716d..33937e6e0 100644 --- a/sql3/planner/expression.go +++ b/sql3/planner/expression.go @@ -41,7 +41,6 @@ func coerceValue(sourceType parser.ExprDataType, targetType parser.ExprDataType, return nil, sql3.NewErrInternalf("unexpected value type '%T'", value) } return pql.NewDecimal(val*int64(math.Pow(10, float64(t.Scale))), t.Scale), nil - case *parser.DataTypeTimestamp: val, ok := value.(int64) if !ok { @@ -65,7 +64,6 @@ func coerceValue(sourceType parser.ExprDataType, targetType parser.ExprDataType, return nil, sql3.NewErrInternalf("unexpected value type '%T'", value) } return pql.NewDecimal(int64(val)*int64(math.Pow(10, float64(t.Scale))), t.Scale), nil - case *parser.DataTypeTimestamp: val, ok := value.(int64) if !ok { @@ -1574,6 +1572,8 @@ func (n *callPlanExpression) Evaluate(currentRow []interface{}) (interface{}, er return n.EvaluateFormat(currentRow) case "CHARINDEX": return n.EvaluateCharIndex(currentRow) + case "TOTIMESTAMP": + return n.EvaluateToTimestamp(currentRow) case "STR": return n.EvaluateStr(currentRow) default: @@ -2817,26 +2817,62 @@ func (p *ExecutionPlanner) compileCallExpr(expr *parser.Call) (_ types.PlanExpre } } -func (p *ExecutionPlanner) compileOrderingTermExpr(expr parser.Expr) (index int, err error) { +func (p *ExecutionPlanner) compileOrderingTermExpr(expr parser.Expr, projections []types.PlanExpression, source parser.Source) (types.PlanExpression, error) { if expr == nil { - return 0, nil + return nil, nil } switch thisExpr := expr.(type) { - case *parser.QualifiedRef: - return thisExpr.ColumnIndex, nil + case *parser.Ident: + for _, proj := range projections { + switch p := proj.(type) { + case *qualifiedRefPlanExpression: + if strings.EqualFold(thisExpr.Name, p.columnName) { + if !typeCanBeSortedOn(p.Type()) { + return nil, sql3.NewErrExpectedSortableExpression(0, 0, p.Type().TypeDescription()) + } + return p, nil + } + case *aliasPlanExpression: + if strings.EqualFold(thisExpr.Name, p.aliasName) { + if !typeCanBeSortedOn(p.expr.Type()) { + return nil, sql3.NewErrExpectedSortableExpression(0, 0, p.expr.Type().TypeDescription()) + } + return p.expr, nil + } + + } + } + + // we didn't find in projection list so go look in the source columns + for _, col := range source.PossibleOutputColumns() { + if strings.EqualFold(thisExpr.Name, col.ColumnName) { + orderExpr := newQualifiedRefPlanExpression(col.TableName, col.ColumnName, col.ColumnIndex, col.Datatype) + if !typeCanBeSortedOn(orderExpr.Type()) { + return nil, sql3.NewErrExpectedSortableExpression(0, 0, orderExpr.Type().TypeDescription()) + } + return orderExpr, nil + } + } + + return nil, sql3.NewErrColumnNotFound(thisExpr.NamePos.Line, thisExpr.NamePos.Column, thisExpr.Name) case *parser.IntegerLit: val, err := strconv.ParseInt(thisExpr.Value, 10, 64) if err != nil { - return 0, err + return nil, err } // subtract one because ordering terms are 1 based, not 0 based - return int(val - 1), nil + index := int(val - 1) + // get the expr from the projection + orderExpr := projections[index] + if !typeCanBeSortedOn(orderExpr.Type()) { + return nil, sql3.NewErrExpectedSortableExpression(0, 0, orderExpr.Type().TypeDescription()) + } + return orderExpr, nil default: - return 0, sql3.NewErrInternalf("unexpected ordering expression type: %T", expr) - + return nil, sql3.NewErrInternalf("unexpected ordering expression type: %T", expr) } } diff --git a/sql3/planner/expressionanalyzer.go b/sql3/planner/expressionanalyzer.go index 1214bd05a..78aa5d5b7 100644 --- a/sql3/planner/expressionanalyzer.go +++ b/sql3/planner/expressionanalyzer.go @@ -831,60 +831,54 @@ func (p *ExecutionPlanner) analyzeCaseBlockExpression(ctx context.Context, expr return expr, nil } -func (p *ExecutionPlanner) analyzeOrderingTermExpression(expr parser.Expr, scope parser.Statement) (parser.Expr, error) { +func (p *ExecutionPlanner) analyzeOrderingTermExpression(expr parser.Expr, scope parser.Statement) error { if expr == nil { - return nil, nil + return nil } - // ordering terms need to be either a column name, an alias name or an integer literal representing - // position of column in the select list + // ordering terms can be: + // 1. a *parser.Ident reference to either a column name in the source, or a reference to a a column or alias name in the projection list + // 2. a *parser.IntegerLit representing position of column in the projection list switch thisExpr := expr.(type) { case *parser.Ident: switch sc := scope.(type) { case *parser.SelectStatement: - // go find the first ident in the projection list that matches - columnIndex := 0 - found := false - for idx, proj := range sc.Columns { + // go look for the first ident in the projection list that matches + foundInProjectionList := false + for _, proj := range sc.Columns { // if the expression is a qualified ref, check the name colExpr, ok := proj.Expr.(*parser.QualifiedRef) if ok && strings.EqualFold(thisExpr.Name, colExpr.Column.Name) { - columnIndex = idx - found = true + foundInProjectionList = true break } // try the alias is there is one if proj.Alias != nil && strings.EqualFold(thisExpr.Name, proj.Alias.Name) { - columnIndex = idx - found = true + foundInProjectionList = true break } } - if !found { - return nil, sql3.NewErrColumnNotFound(thisExpr.NamePos.Line, thisExpr.NamePos.Column, thisExpr.Name) - } + if !foundInProjectionList { + // we didn't find in projection list so go look in the source columns + foundInSource := false + for _, col := range sc.Source.PossibleOutputColumns() { + if strings.EqualFold(thisExpr.Name, col.ColumnName) { + foundInSource = true + break + } + } - // turn *parser.Ident into *parser.QualifiedRef - ident := &parser.QualifiedRef{ - Table: &parser.Ident{ - Name: "", - NamePos: parser.Pos{Line: 0, Column: 0}, - }, - Column: &parser.Ident{ - Name: thisExpr.Name, - NamePos: thisExpr.NamePos, - }, - ColumnIndex: columnIndex, - // since this is a ordring term, we don't care about the type - RefDataType: parser.NewDataTypeVoid(), + if !foundInSource { + return sql3.NewErrColumnNotFound(thisExpr.NamePos.Line, thisExpr.NamePos.Column, thisExpr.Name) + } } - return ident, nil + return nil default: - return nil, sql3.NewErrInternalf("unhandled scope type '%T'", sc) + return sql3.NewErrInternalf("unhandled scope type '%T'", sc) } case *parser.IntegerLit: @@ -893,17 +887,17 @@ func (p *ExecutionPlanner) analyzeOrderingTermExpression(expr parser.Expr, scope // check to see if the offset is in the range value, err := strconv.ParseInt(thisExpr.Value, 10, 64) if err != nil { - return nil, sql3.NewErrInternalf("unexpected integer literal value") + return sql3.NewErrInternalf("unexpected integer literal value") } if value < 1 || value > int64(len(sc.Columns)) { - return nil, sql3.NewErrExpectedSortExpressionReference(0, 0) + return sql3.NewErrExpectedSortExpressionReference(0, 0) } default: - return nil, sql3.NewErrInternalf("unhandled scope type '%T'", sc) + return sql3.NewErrInternalf("unhandled scope type '%T'", sc) } + return nil default: - return nil, sql3.NewErrExpectedSortExpressionReference(expr.Pos().Line, expr.Pos().Column) + return sql3.NewErrExpectedSortExpressionReference(expr.Pos().Line, expr.Pos().Column) } - return expr, nil } diff --git a/sql3/planner/expressionanalyzercall.go b/sql3/planner/expressionanalyzercall.go index 85deb038f..698b994c8 100644 --- a/sql3/planner/expressionanalyzercall.go +++ b/sql3/planner/expressionanalyzercall.go @@ -254,6 +254,8 @@ func (p *ExecutionPlanner) analyzeCallExpression(ctx context.Context, call *pars return p.analyseFunctionFormat(call, scope) case "CHARINDEX": return p.analyseFunctionCharIndex(call, scope) + case "TOTIMESTAMP": + return p.analyzeFunctionToTimestamp(call, scope) case "STR": return p.analyseFunctionStr(call, scope) default: diff --git a/sql3/planner/expressiontypes.go b/sql3/planner/expressiontypes.go index 43ff26f3c..569dc1f0e 100644 --- a/sql3/planner/expressiontypes.go +++ b/sql3/planner/expressiontypes.go @@ -346,12 +346,12 @@ func typesAreAssignmentCompatible(targetType parser.ExprDataType, sourceType par switch sourceType.(type) { case *parser.DataTypeTimestamp: return true - case *parser.DataTypeInt: - //could be a int convertable to a date - return true case *parser.DataTypeString: //could be a string parseable as a date return true + case *parser.DataTypeInt: + //integers coerced to timestamp will be treated as time represented in number of seconds since unix epoch + return true default: return false } @@ -466,7 +466,7 @@ func typeIsTimeQuantum(testType parser.ExprDataType) (bool, parser.ExprDataType) switch testType.(type) { case *parser.DataTypeIDSetQuantum: return true, parser.NewDataTypeIDSet() - case *parser.DataTypeStringSet: + case *parser.DataTypeStringSetQuantum: return true, parser.NewDataTypeStringSet() default: return false, nil @@ -542,6 +542,16 @@ func typeIsBSI(testType parser.ExprDataType) bool { } } +// returns true if we can sort on a type +func typeCanBeSortedOn(testType parser.ExprDataType) bool { + switch testType.(type) { + case *parser.DataTypeStringSet, *parser.DataTypeIDSet: + return false + default: + return true + } +} + // returns true if the types can be compared func typesAreComparable(testTypeL parser.ExprDataType, testTypeR parser.ExprDataType) bool { switch testTypeL.(type) { diff --git a/sql3/planner/inbuiltfunctionsdate.go b/sql3/planner/inbuiltfunctionsdate.go index 4149893f8..49a531c52 100644 --- a/sql3/planner/inbuiltfunctionsdate.go +++ b/sql3/planner/inbuiltfunctionsdate.go @@ -4,6 +4,7 @@ import ( "strings" "time" + featurebase "github.com/featurebasedb/featurebase/v3" "github.com/featurebasedb/featurebase/v3/sql3" "github.com/featurebasedb/featurebase/v3/sql3/parser" ) @@ -43,6 +44,33 @@ func (p *ExecutionPlanner) analyzeFunctionDatePart(call *parser.Call, scope pars return call, nil } +func (p *ExecutionPlanner) analyzeFunctionToTimestamp(call *parser.Call, scope parser.Statement) (parser.Expr, error) { + //param1 is the number to be converted to timestamp. This param is required. + //param2 is the time unit of the numeric value in param 1. This param is optional. + //ToTimestamp can be invoked with just param1. + if len(call.Args) != 1 && len(call.Args) != 2 { + return nil, sql3.NewErrCallParameterCountMismatch(call.Rparen.Line, call.Rparen.Column, call.Name.Name, 2, len(call.Args)) + } + + //param1 is a integer of type int64 + param1Type := parser.NewDataTypeInt() + if !typesAreAssignmentCompatible(param1Type, call.Args[0].DataType()) { + return nil, sql3.NewErrParameterTypeMistmatch(call.Args[0].Pos().Line, call.Args[0].Pos().Column, call.Args[0].DataType().TypeDescription(), param1Type.TypeDescription()) + } + + //param2 is a string and it should be one of 's', 'ms', 'us', 'ns'. + //param2 is optional, will be defaulted to 's' if not supplied. + if len(call.Args) == 2 { + param2Type := parser.NewDataTypeString() + if !typesAreAssignmentCompatible(param2Type, call.Args[1].DataType()) { + return nil, sql3.NewErrParameterTypeMistmatch(call.Args[1].Pos().Line, call.Args[1].Pos().Column, call.Args[1].DataType().TypeDescription(), param2Type.TypeDescription()) + } + } + //ToTimestamp returns a timestamp calculated from param1 using time unit passed in param 2 + call.ResultDataType = parser.NewDataTypeTimestamp() + return call, nil +} + func (n *callPlanExpression) EvaluateDatepart(currentRow []interface{}) (interface{}, error) { intervalEval, err := n.args[0].Evaluate(currentRow) if err != nil { @@ -121,3 +149,50 @@ func (n *callPlanExpression) EvaluateDatepart(currentRow []interface{}) (interfa } } + +func (n *callPlanExpression) EvaluateToTimestamp(currentRow []interface{}) (interface{}, error) { + //retrieve param1, the number to be converted to timestamp + param1, err := n.args[0].Evaluate(currentRow) + if err != nil { + return nil, err + } else if param1 == nil { + //if the param1 is null silently return null timestamp value + return nil, nil + } + coercedParam1, err := coerceValue(n.args[0].Type(), parser.NewDataTypeInt(), param1, parser.Pos{Line: 0, Column: 0}) + if err != nil { + //raise error if param 1 is not an integer. Should we return nil instead of raising error here? see note at return. + return nil, err + } + num, ok := coercedParam1.(int64) + if !ok { + //raise error if param 1 is not an integer. Should we return nil instead of raising error here? see note at return. + return nil, sql3.NewErrInternalf("unable to convert value") + } + + //retrieve param2, time unit for param1, if not supplied default to seconds 's'. + var unit string = featurebase.TimeUnitSeconds + if len(n.args) == 2 { + param2, err := n.args[1].Evaluate(currentRow) + if err != nil { + //raise error if unable to retieve the argument for param2 + return nil, err + } + coercedParam2, err := coerceValue(n.args[1].Type(), parser.NewDataTypeString(), param2, parser.Pos{Line: 0, Column: 0}) + if err != nil { + //raise error if param2 is not a string + return nil, err + } + unit, ok = coercedParam2.(string) + if !ok { + //raise error if param2 is not a string + return nil, sql3.NewErrInternalf("unable to convert value") + } + if !featurebase.IsValidTimeUnit(unit) { + //raise error is param2 is not a valid time unit + return nil, sql3.NewErrCallParameterValueInvalid(0, 0, unit, "timeunit") + } + } + //should we throw error or return nil if the conversion fails? what is the desired behaviour when ToTimestamp errors for one bad record in a batch of thousands? + return featurebase.ValToTimestamp(unit, num) +} diff --git a/sql3/planner/inbuiltfunctionstable.go b/sql3/planner/inbuiltfunctionstable.go index 0968e659f..19bc67c93 100644 --- a/sql3/planner/inbuiltfunctionstable.go +++ b/sql3/planner/inbuiltfunctionstable.go @@ -47,7 +47,7 @@ func (p *ExecutionPlanner) analyzeFunctionSubtable(call *parser.Call, scope pars ok, _ := typeIsTimeQuantum(call.Args[0].DataType()) if !ok { // TODO (pok) send back the right error - return nil, sql3.NewErrSetExpressionExpected(call.Args[1].Pos().Line, call.Args[1].Pos().Column) + return nil, sql3.NewErrSetExpressionExpected(call.Args[0].Pos().Line, call.Args[0].Pos().Column) } call.ResultDataType = parser.NewDataTypeSubtable([]*parser.SubtableColumn{ { diff --git a/sql3/planner/opbulkinsert.go b/sql3/planner/opbulkinsert.go index 1f3ad4dcb..223aa66ac 100644 --- a/sql3/planner/opbulkinsert.go +++ b/sql3/planner/opbulkinsert.go @@ -320,7 +320,8 @@ func (i *bulkInsertSourceCSVRowIter) Next(ctx context.Context) (types.Row, error return nil, sql3.NewErrTypeConversionOnMap(0, 0, evalValue, mapColumn.colType.TypeDescription()) } } else { - result[idx] = time.UnixMilli(intVal).UTC() + //implicit conversion of int to timestamp will treat int as seconds since unix epoch + result[idx] = time.Unix(intVal, 0).UTC() } case *parser.DataTypeString: @@ -651,7 +652,8 @@ func (i *bulkInsertSourceNDJsonRowIter) Next(ctx context.Context) (types.Row, er case float64: // if v is a whole number then make it an int if v == float64(int64(v)) { - result[idx] = time.UnixMilli(int64(v)).UTC() + //implicit conversion of int to timestamp will treat int as seconds since unix epoch + result[idx] = time.Unix(int64(v), 0).UTC() } else { return nil, sql3.NewErrTypeConversionOnMap(0, 0, v, mapColumn.colType.TypeDescription()) } @@ -1153,7 +1155,8 @@ func (i *bulkInsertSourceParquetRowIter) Next(ctx context.Context) (types.Row, e case *parser.DataTypeTimestamp: if intVal, ok := evalValue.(int64); ok { - result[idx] = time.UnixMilli(intVal).UTC() + //implicit conversion of int to timestamp will treat int as seconds since unix epoch + result[idx] = time.Unix(intVal, 0).UTC() } else if stringVal, ok := evalValue.(string); ok { if tm, err := time.ParseInLocation(time.RFC3339Nano, stringVal, time.UTC); err == nil { result[idx] = tm diff --git a/sql3/planner/opinsert.go b/sql3/planner/opinsert.go index e2914c61a..87dfee2d4 100644 --- a/sql3/planner/opinsert.go +++ b/sql3/planner/opinsert.go @@ -390,7 +390,26 @@ func (i *insertRowIter) Next(ctx context.Context) (types.Row, error) { return nil, errors.Wrapf(err, "converting timestamp to int64: %s", v) } row.Values[posVals[idx]] = i64 - + //integers passed as input for Timestamp fields will be treated as time represented in number of seconds since epoch defined for the field + //for timestamp fields created using SQL epoch will be defaulted to unix epoch + case int64: + // Convert the input seconds to target timeunit defined for the Timestamp field + // Add Base, which is the base epoch for Timestamp fields, to the input before saving + unit := fbbatch.TimeUnit(opts.TimeUnit) + i64 := opts.Base + switch unit { + case fbbatch.TimeUnitSeconds: + i64 = i64 + v + case fbbatch.TimeUnitMilliseconds: + i64 = i64 + (v * 1000) + case fbbatch.TimeUnitMicroseconds, fbbatch.TimeUnitUSeconds: + i64 = i64 + (v * 1000000) + case fbbatch.TimeUnitNanoseconds: + i64 = i64 + (v * 1000000000) + default: + return nil, errors.Wrapf(err, "unknown time unit: %s", unit) + } + row.Values[posVals[idx]] = i64 // nil is to support `null` values. case nil: row.Values[posVals[idx]] = eval diff --git a/sql3/planner/oporderby.go b/sql3/planner/oporderby.go index b11c431a1..feced2451 100644 --- a/sql3/planner/oporderby.go +++ b/sql3/planner/oporderby.go @@ -32,8 +32,7 @@ const ( // OrderByExpression is the expression on which an order by can be computed type OrderByExpression struct { - Index int - ExprType parser.ExprDataType + Expr types.PlanExpression Order orderByOrder NullOrdering nullOrdering } @@ -79,6 +78,24 @@ func (n *PlanOpOrderBy) WithChildren(children ...types.PlanOperator) (types.Plan return NewPlanOpOrderBy(n.orderByFields, children[0]), nil } +func (n *PlanOpOrderBy) Expressions() []types.PlanExpression { + res := make([]types.PlanExpression, 0) + for _, e := range n.orderByFields { + res = append(res, e.Expr) + } + return res +} + +func (n *PlanOpOrderBy) WithUpdatedExpressions(exprs ...types.PlanExpression) (types.PlanOperator, error) { + if len(exprs) != len(n.orderByFields) { + return nil, sql3.NewErrInternalf("unexpected number of exprs '%d'", len(exprs)) + } + for i, e := range exprs { + n.orderByFields[i].Expr = e + } + return n, nil +} + func (n *PlanOpOrderBy) String() string { return "" } @@ -92,8 +109,7 @@ func (n *PlanOpOrderBy) Plan() map[string]interface{} { ps := make([]interface{}, 0) for _, e := range n.orderByFields { ps = append(ps, &map[string]interface{}{ - "index": e.Index, - "exprType": e.ExprType.TypeDescription(), + "expr": e.Expr.Plan(), "order": e.Order, "nullOrdering": e.NullOrdering, }) @@ -200,8 +216,20 @@ func (s *OrderBySorter) Less(i, j int) bool { a := s.Rows[i] b := s.Rows[j] for _, sf := range s.SortFields { - av := a[sf.Index] - bv := b[sf.Index] + + var sortIndex int + switch se := sf.Expr.(type) { + case *qualifiedRefPlanExpression: + sortIndex = se.columnIndex + case *intLiteralPlanExpression: + sortIndex = int(se.value) + default: + s.LastError = sql3.NewErrInternalf("unexpected sort field expression type '%T'", se) + return false + } + + av := a[sortIndex] + bv := b[sortIndex] if sf.Order == orderByDesc { av, bv = bv, av @@ -215,8 +243,8 @@ func (s *OrderBySorter) Less(i, j int) bool { return sf.NullOrdering != nullOrderingFirst } - switch sf.ExprType.(type) { - case *parser.DataTypeInt, *parser.DataTypeID: + switch t := sf.Expr.Type().(type) { + case *parser.DataTypeInt: avInt, aok := av.(int64) bvInt, bok := bv.(int64) if !(aok && bok) { @@ -228,6 +256,18 @@ func (s *OrderBySorter) Less(i, j int) bool { } return true + case *parser.DataTypeID: + avInt, aok := av.(uint64) + bvInt, bok := bv.(uint64) + if !(aok && bok) { + s.LastError = sql3.NewErrInternalf("unexpected type conversion result") + return false + } + if avInt > bvInt { + return false + } + return true + case *parser.DataTypeBool: avBool, aok := av.(bool) bvBool, bok := bv.(bool) @@ -277,7 +317,7 @@ func (s *OrderBySorter) Less(i, j int) bool { return true default: - s.LastError = sql3.NewErrInternalf("unhandled data type '%T'", sf.ExprType) + s.LastError = sql3.NewErrInternalf("unhandled data type '%T'", t) return false } } diff --git a/sql3/planner/planoptimizer.go b/sql3/planner/planoptimizer.go index e5c1761ef..114c29922 100644 --- a/sql3/planner/planoptimizer.go +++ b/sql3/planner/planoptimizer.go @@ -16,7 +16,6 @@ import ( //TODO(pok) push filter down into join condition if terms reference either side of join //TODO(pok) push order by down as far as possible -//TODO(pok) handle the case of the order by expressions not being in a projection list //TODO(pok) you can't group by _id in PQL, so we need to not use a PQL group by operator here //TODO(pok) move constant folding to in here @@ -1052,7 +1051,7 @@ func fixProjectionReferences(ctx context.Context, a *ExecutionPlanner, n types.P return thisNode, false, nil // everything else that can be a child of projection - case *PlanOpRelAlias, *PlanOpFilter, *PlanOpPQLTableScan, *PlanOpPQLDistinctScan, *PlanOpNestedLoops: + case *PlanOpRelAlias, *PlanOpFilter, *PlanOpPQLTableScan, *PlanOpPQLDistinctScan, *PlanOpNestedLoops, *PlanOpOrderBy: exprs, same, err := fixFieldRefIndexesOnExpressions(ctx, scope, a, childOp.Schema(), thisNode.Projections...) if err != nil { return thisNode, true, err @@ -1073,6 +1072,44 @@ func fixProjectionReferences(ctx context.Context, a *ExecutionPlanner, n types.P func fixFieldRefs(ctx context.Context, a *ExecutionPlanner, n types.PlanOperator, scope *OptimizerScope) (types.PlanOperator, bool, error) { return TransformPlanOp(n, func(node types.PlanOperator) (types.PlanOperator, bool, error) { switch thisNode := node.(type) { + case *PlanOpOrderBy: + switch childOp := thisNode.ChildOp.(type) { + case *PlanOpProjection: + expressions := thisNode.Expressions() + + for _, ex := range expressions { + ref, ok := ex.(*qualifiedRefPlanExpression) + if !ok { + return nil, true, sql3.NewErrInternalf("unexpected expression type '%T'", ex) + } + for i, proj := range childOp.Projections { + if strings.EqualFold(ref.String(), proj.String()) { + ref.columnIndex = i + break + } + } + } + newNode, err := thisNode.WithUpdatedExpressions(expressions...) + if err != nil { + return nil, true, err + } + return newNode, false, nil + + default: + // fix references for the expressions referenced in the order by list + schema := childOp.Schema() + expressions := thisNode.Expressions() + fixed, same, err := fixFieldRefIndexesOnExpressions(ctx, scope, a, schema, expressions...) + if err != nil { + return nil, true, err + } + newNode, err := thisNode.WithUpdatedExpressions(fixed...) + if err != nil { + return nil, true, err + } + return newNode, same, nil + } + case *PlanOpFilter: // fix references for the expressions referenced in the filter predicate expression schema := thisNode.Schema() diff --git a/sql3/sql_complex_test.go b/sql3/sql_complex_test.go index 081adfcc6..a25314807 100644 --- a/sql3/sql_complex_test.go +++ b/sql3/sql_complex_test.go @@ -460,13 +460,13 @@ func TestPlanner_CoverCreateTable(t *testing.T) { { name: "timestampcol", typ: "timestamp", - constraints: "timeunit 'ms' epoch '2021-01-01T00:00:00Z'", + constraints: "timeunit 'ms'", expOptions: pilosa.FieldOptions{ - Base: 1609459200000, + Base: 0, Type: "timestamp", TimeUnit: "ms", - Min: pql.NewDecimal(-63745055999000, 0), - Max: pql.NewDecimal(251792841599000, 0), + Min: pql.NewDecimal(-62135596799000, 0), + Max: pql.NewDecimal(253402300799000, 0), }, }, { @@ -713,7 +713,7 @@ func TestPlanner_CreateTable(t *testing.T) { _id id, intcol int min 0 max 10000, boolcol bool, - timestampcol timestamp timeunit 'ms' epoch '2010-01-01T00:00:00Z', + timestampcol timestamp timeunit 'ms', decimalcol decimal(2), stringcol string cachetype ranked size 1000, stringsetcol stringset cachetype lru size 1000, @@ -2957,7 +2957,7 @@ func TestPlanner_BulkInsertParquet(t *testing.T) { now := time.Now() simpleParquetMaker(t, tmpfile, 1, []tb{ {Name: "id", Type: arrow.PrimitiveTypes.Int64, Value: []int64{1}}, - {Name: "unixtime", Type: arrow.PrimitiveTypes.Int64, Value: []int64{now.UnixMilli()}}, + {Name: "unixtime", Type: arrow.PrimitiveTypes.Int64, Value: []int64{now.Unix()}}, {Name: "stringtime", Type: arrow.BinaryTypes.String, Value: []string{now.Format(time.RFC3339)}}, }) diff --git a/sql3/test/defs/defs_date_functions.go b/sql3/test/defs/defs_date_functions.go index 74383589f..f5cb037a9 100644 --- a/sql3/test/defs/defs_date_functions.go +++ b/sql3/test/defs/defs_date_functions.go @@ -1,5 +1,7 @@ package defs +import "time" + // datepart tests var datePartTests = TableTest{ @@ -34,6 +36,30 @@ var datePartTests = TableTest{ ), ExpErr: "invalid value '1' for parameter 'interval'", }, + { + SQLs: sqls( + "select totimestamp()", + ), + ExpErr: "count of formal parameters (2) does not match count of actual parameters (0)", + }, + { + SQLs: sqls( + "select totimestamp('a')", + ), + ExpErr: "an expression of type 'string' cannot be passed to a parameter of type 'int'", + }, + { + SQLs: sqls( + "select totimestamp(1, 2)", + ), + ExpErr: "an expression of type 'int' cannot be passed to a parameter of type 'string'", + }, + { + SQLs: sqls( + "select totimestamp(1, 'x')", + ), + ExpErr: "invalid value 'x' for parameter 'timeunit'", + }, { SQLs: sqls( "select _id, datepart('yy', ts) from dateparttests", @@ -177,5 +203,43 @@ var datePartTests = TableTest{ ), Compare: CompareExactUnordered, }, + { + //test datepart(timestamp, part) for implicit conversion of integer value passed as argument to timestamp param + SQLs: sqls( + "select datepart('yy', 0) as \"yy\", datepart('m', 0) as \"m\", datepart('d', 0) as \"d\"", + ), + ExpHdrs: hdrs( + hdr("yy", fldTypeInt), + hdr("m", fldTypeInt), + hdr("d", fldTypeInt), + ), + ExpRows: rows( + row(int64(1970), int64(1), int64(1)), + ), + Compare: CompareExactUnordered, + }, + { + //test ToTimestamp(num, timeunit) for all possible time unit values + SQLs: sqls( + "select totimestamp(1000) as \"default\", totimestamp(1000, 's') as \"s\", totimestamp(1000000, 'ms') as \"ms\", totimestamp(1000000000, 'us') as \"us\", totimestamp(1000000000, 'µs') as \"µs\", totimestamp(1000000000000, 'ns') as \"ns\"", + ), + ExpHdrs: hdrs( + hdr("default", fldTypeTimestamp), + hdr("s", fldTypeTimestamp), + hdr("ms", fldTypeTimestamp), + hdr("us", fldTypeTimestamp), + hdr("µs", fldTypeTimestamp), + hdr("ns", fldTypeTimestamp), + ), + ExpRows: rows( + row(time.Unix(1000, 0).UTC(), + time.Unix(1000, 0).UTC(), + time.UnixMilli(1000000).UTC(), + time.UnixMicro(1000000000).UTC(), + time.UnixMicro(1000000000).UTC(), + time.Unix(0, 1000000000000).UTC()), + ), + Compare: CompareExactUnordered, + }, }, } diff --git a/sql3/test/defs/defs_inserts.go b/sql3/test/defs/defs_inserts.go index 65cb18ad8..135e276f3 100644 --- a/sql3/test/defs/defs_inserts.go +++ b/sql3/test/defs/defs_inserts.go @@ -159,7 +159,7 @@ var insertTimestampTest = TableTest{ SQLTests: []SQLTest{ { SQLs: sqls( - "CREATE TABLE insertTimestampTest (_id id, time timestamp timeunit 'ms' epoch '2022-01-01T00:00:00Z', ids idset, strings stringset);", + "CREATE TABLE insertTimestampTest (_id id, time timestamp timeunit 'ms', ids idset, strings stringset);", ), ExpHdrs: hdrs(), ExpRows: rows(), @@ -173,6 +173,14 @@ var insertTimestampTest = TableTest{ ExpRows: rows(), Compare: CompareExactUnordered, }, + { + SQLs: sqls( + "INSERT INTO insertTimestampTest(_id, time, ids, strings) VALUES (2, 1672531200, [6 , 1, 9], ['red', 'blue', 'green']);", + ), + ExpHdrs: hdrs(), + ExpRows: rows(), + Compare: CompareExactUnordered, + }, { SQLs: sqls( "select time from insertTimestampTest;", @@ -182,6 +190,7 @@ var insertTimestampTest = TableTest{ ), ExpRows: rows( row(timestampFromString("2023-01-01T00:00:00Z")), + row(timestampFromString("2023-01-01T00:00:00Z")), ), Compare: CompareExactUnordered, SortStringKeys: true, diff --git a/sql3/test/defs/defs_orderby.go b/sql3/test/defs/defs_orderby.go index 8d2deb7fa..aded96bb8 100644 --- a/sql3/test/defs/defs_orderby.go +++ b/sql3/test/defs/defs_orderby.go @@ -14,10 +14,10 @@ var orderByTests = TableTest{ srcHdr("a_decimal", fldTypeDecimal2), ), srcRows( - srcRow(int64(1), int64(11), []int64{11, 12, 13}, int64(101), "str1", []string{"a1", "b1", "c1"}, float64(123.45)), - srcRow(int64(2), int64(22), []int64{21, 22, 23}, int64(201), "str2", []string{"a2", "b2", "c2"}, float64(234.56)), - srcRow(int64(3), int64(33), []int64{31, 32, 33}, int64(301), "str3", []string{"a3", "b3", "c3"}, float64(345.67)), - srcRow(int64(4), int64(44), []int64{41, 42, 43}, int64(401), "str4", []string{"a4", "b4", "c4"}, float64(456.78)), + srcRow(int64(1), int64(44), []int64{11, 12, 13}, int64(101), "str1", []string{"a1", "b1", "c1"}, float64(123.45)), + srcRow(int64(2), int64(33), []int64{21, 22, 23}, int64(201), "str2", []string{"a2", "b2", "c2"}, float64(234.56)), + srcRow(int64(3), int64(21), []int64{31, 32, 33}, int64(301), "str3", []string{"a3", "b3", "c3"}, float64(345.67)), + srcRow(int64(4), int64(10), []int64{41, 42, 43}, int64(401), "str4", []string{"a4", "b4", "c4"}, float64(456.78)), ), ), SQLTests: []SQLTest{ @@ -35,5 +35,127 @@ var orderByTests = TableTest{ ), ExpErr: "unable to sort a column of type 'idset'", }, + { + SQLs: sqls( + "select an_int from order_by_test order by an_id asc", + ), + ExpHdrs: hdrs( + hdr("an_int", fldTypeInt), + ), + ExpRows: rows( + row(int64(44)), + row(int64(33)), + row(int64(21)), + row(int64(10)), + ), + Compare: CompareExactOrdered, + }, + { + SQLs: sqls( + "select an_int, an_id from order_by_test order by a_decimal asc", + ), + ExpHdrs: hdrs( + hdr("an_int", fldTypeInt), + hdr("an_id", fldTypeID), + ), + ExpRows: rows( + row(int64(44), int64(101)), + row(int64(33), int64(201)), + row(int64(21), int64(301)), + row(int64(10), int64(401)), + ), + Compare: CompareExactOrdered, + }, + { + SQLs: sqls( + "select an_int + 1 as foo, an_id from order_by_test order by foo asc, a_decimal asc", + ), + ExpHdrs: hdrs( + hdr("foo", fldTypeInt), + hdr("an_id", fldTypeID), + ), + ExpRows: rows( + row(int64(11), int64(401)), + row(int64(22), int64(301)), + row(int64(34), int64(201)), + row(int64(45), int64(101)), + ), + Compare: CompareExactOrdered, + }, + { + SQLs: sqls( + "select an_int from order_by_test order by an_int asc", + ), + ExpHdrs: hdrs( + hdr("an_int", fldTypeInt), + ), + ExpRows: rows( + row(int64(10)), + row(int64(21)), + row(int64(33)), + row(int64(44)), + ), + Compare: CompareExactOrdered, + }, + { + SQLs: sqls( + "select an_int as foo from order_by_test order by foo asc", + ), + ExpHdrs: hdrs( + hdr("foo", fldTypeInt), + ), + ExpRows: rows( + row(int64(10)), + row(int64(21)), + row(int64(33)), + row(int64(44)), + ), + Compare: CompareExactOrdered, + }, + { + SQLs: sqls( + "select an_int as foo from order_by_test order by 1 asc", + ), + ExpHdrs: hdrs( + hdr("foo", fldTypeInt), + ), + ExpRows: rows( + row(int64(10)), + row(int64(21)), + row(int64(33)), + row(int64(44)), + ), + Compare: CompareExactOrdered, + }, + { + SQLs: sqls( + "select an_int + 1 from order_by_test order by 1 asc", + ), + ExpHdrs: hdrs( + hdr("", fldTypeInt), + ), + ExpRows: rows( + row(int64(11)), + row(int64(22)), + row(int64(34)), + row(int64(45)), + ), + Compare: CompareExactOrdered, + }, + { + SQLs: sqls( + "select an_int + 1 as bar from order_by_test order by bar desc", + ), + ExpHdrs: hdrs( + hdr("bar", fldTypeInt), + ), + ExpRows: rows( + row(int64(45)), + row(int64(34)), + row(int64(22)), + row(int64(11)), + ), + Compare: CompareExactOrdered, + }, }, } diff --git a/sql3/test/defs/defs_timestamp_literals.go b/sql3/test/defs/defs_timestamp_literals.go index b22ee806a..b4cd6a0c0 100644 --- a/sql3/test/defs/defs_timestamp_literals.go +++ b/sql3/test/defs/defs_timestamp_literals.go @@ -18,7 +18,7 @@ var timestampLiterals = TableTest{ { // InsertWithCurrentTimestamp SQLs: sqls( - "insert into testtimestampliterals (_id, a, b, d, ts, event, ievent) values (4, 40, 400, 10.12, current_timestamp, ['A', 'B', 'C'], [1, 2, 3])", + "insert into testtimestampliterals (_id, a, b, d, ts, event, ievent) values (1, 40, 400, 10.12, current_timestamp, ['A', 'B', 'C'], [1, 2, 3])", ), ExpHdrs: hdrs(), ExpRows: rows(), @@ -27,11 +27,44 @@ var timestampLiterals = TableTest{ { // InsertWithCurrentDate SQLs: sqls( - "insert into testtimestampliterals (_id, a, b, d, ts, event, ievent) values (4, 40, 400, 10.12, current_date, ['A', 'B', 'C'], [1, 2, 3])", + "insert into testtimestampliterals (_id, a, b, d, ts, event, ievent) values (2, 40, 400, 10.12, current_date, ['A', 'B', 'C'], [1, 2, 3])", ), ExpHdrs: hdrs(), ExpRows: rows(), Compare: CompareExactUnordered, }, + { + // Insert literal 0 into a timestamp, it should be stored as 1970-01-01 00:00:00 +0000 UTC (unix epoch base value) + SQLs: sqls( + "insert into testtimestampliterals (_id, a, b, d, ts, event, ievent) values (3, 40, 400, 10.12, 0, ['A', 'B', 'C'], [1, 2, 3])", + ), + ExpHdrs: hdrs(), + ExpRows: rows(), + Compare: CompareExactUnordered, + }, + { + // Insert literal -86400 into a timestamp, it should be stored as 1969-12-31 00:00:00 +0000 UTC (unix epoch base value) + SQLs: sqls( + "insert into testtimestampliterals (_id, a, b, d, ts, event, ievent) values (4, 40, 400, 10.12, -86400, ['A', 'B', 'C'], [1, 2, 3])", + ), + ExpHdrs: hdrs(), + ExpRows: rows(), + Compare: CompareExactUnordered, + }, + { + //compare test is done only for integer test cases (_id in (3,4), because only for these cases we have a determinate year(1970, 1969) to look for. + SQLs: sqls( + "select _id, datepart('yy', ts) as \"yy\" from testtimestampliterals where _id in (3,4)", + ), + ExpHdrs: hdrs( + hdr("_id", fldTypeID), + hdr("yy", fldTypeInt), + ), + ExpRows: rows( + row(int64(3), int64(1970)), + row(int64(4), int64(1969)), + ), + Compare: CompareExactUnordered, + }, }, } diff --git a/translate.go b/translate.go index 852420abc..2573a14f4 100644 --- a/translate.go +++ b/translate.go @@ -289,7 +289,7 @@ func OpenInMemTranslateStore(rawurl, index, field string, partitionID, partition fname = fname[:10] } - tf, err := os.CreateTemp(os.Getenv("RAMDISK"), fmt.Sprintf("bolt-i%s-f%s-%d-%d-", iname, fname, partitionID, partitionN)) + tf, err := os.CreateTemp("", fmt.Sprintf("bolt-i%s-f%s-%d-%d-", iname, fname, partitionID, partitionN)) if err != nil { return nil, errors.Wrap(err, "making temp file for boltdb key translation") } diff --git a/verchk.go b/verchk.go index 0041a9ca1..1dcf93558 100644 --- a/verchk.go +++ b/verchk.go @@ -6,25 +6,29 @@ import ( "io" "net/http" "os" + "path/filepath" "strings" "github.com/google/uuid" ) -type VersionChecker struct { - URL string +type versionChecker struct { + path string + url string + idfile string } -func NewVersionChecker(endpoint string) *VersionChecker { - v := VersionChecker{ - URL: endpoint, +func newVersionChecker(path, endpoint, idfile string) *versionChecker { + v := versionChecker{ + path: path, + url: endpoint, + idfile: idfile, } return &v } -func (v *VersionChecker) CheckIn() (*VerCheckResponse, error) { - - id, err := v.WriteClientUUID() +func (v *versionChecker) checkIn() (*verCheckResponse, error) { + id, err := v.writeClientUUID() if err != nil { return nil, err } @@ -38,79 +42,70 @@ func (v *VersionChecker) CheckIn() (*VerCheckResponse, error) { if err != nil { return nil, err } - wReq := bytes.NewReader(req) + var jsonResp verCheckResponse + r, err := http.Post(v.url, "application/json", wReq) if err != nil { return nil, err } - var json_resp VerCheckResponse - r, err := http.Post(v.URL, "application/json", wReq) - if err != nil { - return nil, err - } data, err := io.ReadAll(r.Body) - if err != nil { return nil, err } - err = json.Unmarshal(data, &json_resp) + err = json.Unmarshal(data, &jsonResp) if err != nil { return nil, err } - return &json_resp, nil - + return &jsonResp, nil } -func (v *VersionChecker) GenerateClientUUID() (string, error) { +func (v *versionChecker) generateClientUUID() (string, error) { clientUUID := uuid.New() cleanedUUID := strings.Replace(clientUUID.String(), "-", "", -1) return cleanedUUID, nil } -func (v *VersionChecker) WriteClientUUID() (string, error) { - filename := ".client_id.txt" - _, err := os.Stat(filename) - if err != nil { - if os.IsNotExist(err) { - fh, err := os.Create(filename) - if err != nil { - return "", err - } - defer fh.Close() - id, err := v.GenerateClientUUID() - if err != nil { - return "", err - } - - _, err = fh.WriteString(id) - if err != nil { - return "", err - } - - return "", err - } else { - return "", err - } +func (v *versionChecker) writeClientUUID() (string, error) { + var filename string + // if v.idfile starts with a path separator then it's an absolute path + // otherwise it's a relative path and the file goes in the data directory + if v.idfile[0] == os.PathSeparator { + filename = v.idfile + } else { + filename = filepath.Join(v.path, v.idfile) } - - fh, err := os.Open(filename) + fh, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE, 0744) if err != nil { return "", err } defer fh.Close() - buf, err := os.ReadFile(filename) + buf, err := os.ReadFile(filename) if err != nil { return "", err } + // this just checks to see if there was anything at all in the file. + // we should probably check to make sure it's a valid UUID + if string(buf) == "" { + id, err := v.generateClientUUID() + if err != nil { + return "", err + } + _, err = fh.WriteString(id) + if err != nil { + return "", err + } + return id, nil + } + return string(buf), nil - } -type VerCheckResponse struct { +type verCheckResponse struct { Version string `json:"latest_version"` + Error string `json:"error"` }