scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-08-28 02:44:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Batch insert via SQL (multiple tuples) (#2243)

Browse source 
* Formatting adjustments made during code review.

While reviewing the BULK INSERT logic (in order to decide how best to
approach "ingest via sql" in the cloud), I made a few formatting and
comment changes. I'm just adding them here as a separate commit so they
don't muddy up my actual work.

* Parser modifications to support mulitple tuples in INSERT INTO

This commit doesn't include all of the changes required in the
planner. Fow now, the planner is simply modified to continue supporting
a single tuple (the first tuple in the list).

* Update the planner to handle multiple INSERT INTO tuples

This is part 1. It's still using the existing logic which builds an
ImportRequest for every record (and every field!).

The next step will involve using a client.Batch to handle the records.

* Introduce client.Importer interface (used by client.Batch)

Instead of the Batch having a pointer to a client, this puts an
interface there instead (which the client implements). It also allows us
to inject a different importer (i.e. other than a featurebase.client)
into the Batch.

* Decouple batch from client

This commit pulls batch-specific code out of the client package and into
a new batch package. It introduces the batch.Importer interface, the
methods of which replace all the calls that batch was previously making
directly to client methods.

Finally, it contains two implementations of the batch.Importer
interface: one is a wrapper around client, and the other is a wrapper
around featurebase.API.

* Use docker (instead of MustRunCluster) for internal batch tests

Because the `batch` package tests are internal, using
test.MustRunCluster() resulted in an import loop (because it eventually
imports `server`, and we can't have that). So this commit replaces the
use of `test.MustRunCluster()` with docker. The setup is basically the
same as that used in the idk docker tests.

Here we also remove all client-side references to `UseIngestAPI`, which
is an experimental (json) ingest api. It's still suppored on the server,
but here we remove the external usage of it.

* cherry-pick fix

* Use batch.Import() for sql3 INSERT INTO statements

* Thread logger into sql3

* fix batch test

* Fix some shadowing complaint by linter

* Address some test issues related to stringsets

* Exclude batch integration tests from CI

* Address PR feedback

- Added description to batch.README
- Consolidated grep commands in .gitlab-ci.yml
- Removed some debugging comments
- Replaces some inadvertantly removed license headers

* Add batch package to gitlab CI

* Updated CI for batch package

Updated CI include path

Update gitlab ci

Update CI

Update CI

Trying new include path for ci

Updated gitlab ci include path

Made idk race job optional for sonarcloud upload

add testdata directory

remove testenv from dockercompose file

use GIT_STRATEGY clone in batch CI

add testdata volume to dockercompose

Co-authored-by: Fletcher Haynes <fletcher.haynes@generalassemb.ly>
This commit is contained in:
Travis Turner 2022-10-29 14:24:33 -05:00 • committed by GitHub
parent e3d137f29c
commit 00ef2380e5
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
47 changed files with 4353 additions and 3256 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.gitignore vendored
View file

@ -24,6 +24,8 @@ builds/
*.tfstate.backup
.vscode
batch/testdata/batch*.out
idk/testdata/idk*.out
idk/testenv/certs/*

20
.gitlab/.gitlab-ci.yml
View file

@ -1,4 +1,5 @@
include:
- local: /.gitlab/batch-ci.yml
- template: Security/SAST.gitlab-ci.yml
- template: Security/License-Scanning.gitlab-ci.yml
- template: Security/Dependency-Scanning.gitlab-ci.yml
@ -197,7 +198,7 @@ run go tests:
retry: 1
script:
- echo "Running featurebase unit tests..."
- go test -v -timeout=10m $(go list ./... | grep -Ev idk)
- go test -v -timeout=10m $(go list ./... | grep -Ev 'batch|idk')
tags:
- aws
@ -211,7 +212,7 @@ run go tests race:
needs: ["smoke build"] # we do block on smoke build though bc it's pretty dumb to test stuff if it doesn't build
script:
- echo "Running featurebase race tests..."
- go test -race -v -timeout=10m $(go list ./... | grep -Ev idk)
- go test -race -v -timeout=10m $(go list ./... | grep -Ev 'batch|idk')
tags:
- aws
@ -223,7 +224,7 @@ run go tests shardwidth22:
- if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
script:
- echo "Running featurebase shardwidth22 tests..."
- go test -timeout=10m -tags=shardwidth22 -v $(go list ./... | grep -Ev idk)
- go test -timeout=10m -tags=shardwidth22 -v $(go list ./... | grep -Ev 'batch|idk')
tags:
- aws
@ -239,8 +240,8 @@ run go tests future:
retry: 1
script:
- echo "Running featurebase unit tests..."
- PKG_LIST=$(go list ./... | grep -Ev 'internal/clustertests|simulacraData' | grep -Ev 'idk' | paste -s -d, -)
- go test -timeout=10m -json -coverprofile=coverage.out -covermode=atomic -coverpkg=${PKG_LIST} $(go list ./... | grep -Ev idk) | tee test-report.out
- PKG_LIST=$(go list ./... | grep -Ev 'internal/clustertests|simulacraData' | grep -Ev 'batch|idk' | paste -s -d, -)
- go test -timeout=10m -json -coverprofile=coverage.out -covermode=atomic -coverpkg=${PKG_LIST} $(go list ./... | grep -Ev 'batch|idk') | tee test-report.out
artifacts:
paths:
- coverage.out
@ -257,8 +258,8 @@ run go tests future plg:
retry: 1
script:
- echo "Running featurebase plg-specific unit tests..."
- PKG_LIST=$(go list ./... | grep -Ev 'internal/clustertests|simulacraData' | grep -Ev 'idk' | paste -s -d, -)
- go test -tags=plg -timeout=10m -coverprofile=coverage-plg.out -covermode=atomic -coverpkg=${PKG_LIST} $(go list ./... | grep -Ev idk) | tee test-report-plg.out
- PKG_LIST=$(go list ./... | grep -Ev 'internal/clustertests|simulacraData' | grep -Ev 'batch|idk' | paste -s -d, -)
- go test -tags=plg -timeout=10m -coverprofile=coverage-plg.out -covermode=atomic -coverpkg=${PKG_LIST} $(go list ./... | grep -Ev 'batch|idk') | tee test-report-plg.out
artifacts:
paths:
- coverage-plg.out
@ -399,6 +400,7 @@ run go tests idk sasl:
needs:
- job: build amd container fb
upload to sonarcloud:
stage: integration
image: sonarsource/sonar-scanner-cli:4.6
@ -407,7 +409,7 @@ upload to sonarcloud:
rules:
- if: '$CI_COMMIT_TAG == null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")'
script:
- sonar-scanner -Dsonar.projectKey=molecula_featurebase -Dsonar.organization=molecula -Dsonar.sources=. -Dsonar.host.url=https://sonarcloud.io -Dsonar.go.coverage.reportPaths=coverage*.out,results/coverage*out,idk/testdata/*coverage.out -Dsonar.go.tests.reportPaths=test-report*.out,idk/testdata/*report.out -Dsonar.javascript.lcov.reportPaths=lattice/coverage/lcov.info
- sonar-scanner -Dsonar.projectKey=molecula_featurebase -Dsonar.organization=molecula -Dsonar.sources=. -Dsonar.host.url=https://sonarcloud.io -Dsonar.go.coverage.reportPaths=coverage*.out,results/coverage*out,idk/testdata/*coverage.out,batch/testdata/*coverage.out -Dsonar.go.tests.reportPaths=test-report*.out,idk/testdata/*report.out,batch/testdata/*report.out -Dsonar.javascript.lcov.reportPaths=lattice/coverage/lcov.info
needs:
- job: run go tests future plg
- job: run go tests future
@ -421,6 +423,8 @@ upload to sonarcloud:
optional: true
- job: run go tests idk 533
optional: true
- job: run go tests batch
optional: true
package for linux amd64:
stage: build

28
.gitlab/batch-ci.yml Normal file
View file

@ -0,0 +1,28 @@
run go tests batch:
extends:
- .setup_ssh
variables:
USERNAME: fb-idk-access
PROJECT: batch_${CI_CONCURRENT_ID}
GIT_STRATEGY: clone
stage: test
retry: 1
script:
- echo "Running test-all"
- cd ./batch/
- echo $PROJECT
- echo $DOCKER_PASSWORD | docker login registry.gitlab.com --username "$USERNAME" --password-stdin
- BRANCH_NAME=${CI_COMMIT_REF_SLUG} make test-all
after_script:
- make save-pilosa-logs
- make shutdown
artifacts:
paths:
- ./batch/testdata/*_coverage.out
- ./batch/testdata/*_report.out
- ./batch/testdata/*_logs.txt
tags:
- shell
- aws
needs:
- job: build amd container fb

6
Makefile
View file

@ -45,9 +45,9 @@ vendor: go.mod
version:
@echo $(VERSION)
# We build a list of packages that omits the IDK packages because the IDK
# packages require fancy environment setup.
GOPACKAGES := $(shell $(GO) list ./... | grep -v "/idk")
# We build a list of packages that omits the IDK and batch packages because
# those packages require fancy environment setup.
GOPACKAGES := $(shell $(GO) list ./... | grep -v "/idk" | grep -v "/batch")
# Run test suite
test:

5
api.go
View file

@ -3391,6 +3391,11 @@ type ComputeAPI interface {
Txf() *TxFactory
}
// QueryAPI is a subset of the API methods which have to do with query.
type QueryAPI interface {
Query(ctx context.Context, req *QueryRequest) (QueryResponse, error)
}
// FeatureBaseSchemaAPI is a wrapper around pilosa.API. It implements the
// SchemaAPI interface with methods which are not a part of pilosa.API.
type FeatureBaseSchemaAPI struct {

11
batch/Dockerfile-test Normal file
View file

@ -0,0 +1,11 @@
ARG GO_VERSION=1.19
FROM golang:${GO_VERSION}
WORKDIR /go/src/github.com/molecula/featurebase/
COPY . .
WORKDIR /go/src/github.com/molecula/featurebase/batch/
CMD ["go","test","-v","-mod=vendor","-tags=odbc,dynamic","./..."]

8
batch/Dockerfile-wait Normal file
View file

@ -0,0 +1,8 @@
FROM ubuntu:18.04
RUN ["apt-get", "update", "-y"]
RUN ["apt-get", "install", "-y", "curl", "netcat"]
ADD wait.sh /wait
ENTRYPOINT ["/wait"]

54
batch/Makefile Normal file
View file

@ -0,0 +1,54 @@
GO ?= go
# We allow setting a custom docker-compose "project". Multiple of the
# same docker-compose environment can exist simultaneously as long as
# they use different projects (the project name is prepended to
# container names and such). This is useful in a CI environment where
# we might be running multiple instances of the tests concurrently.
PROJECT ?= batch
DOCKER_COMPOSE = docker-compose -p $(PROJECT)
BRANCH_NAME ?= ""
.pulled:
$(DOCKER_COMPOSE) pull
touch .pulled
vendor: ../go.mod
$(GO) mod vendor
build-%:
$(DOCKER_COMPOSE) build $*
pull-%:
$(DOCKER_COMPOSE) pull $*
test-all:
$(MAKE) startup
$(MAKE) test-run
$(MAKE) shutdown
start-all: .pulled build-wait
echo "branch name" ${BRANCH_NAME}
BRANCH_NAME=${BRANCH_NAME} $(DOCKER_COMPOSE) up -d featurebase
$(DOCKER_COMPOSE) run -T wait featurebase curl --silent --fail http://featurebase:10101/status
startup: start-all
shutdown:
$(DOCKER_COMPOSE) down -v --remove-orphans
rm -f .pulled
save-%-logs:
$(DOCKER_COMPOSE) logs $* > ./testdata/$(PROJECT)_$*_logs.txt
TCMD ?= ./...
# do "make startup", then e.g. "make test-run-local TCMD='-run=MyFavTest ./kafka'"
test-run-local:
pwd
$(DOCKER_COMPOSE) build batch-test
$(DOCKER_COMPOSE) run -T batch-test go test -mod=vendor -tags=odbc,dynamic $(TCMD)
TPKG ?= ./...
test-run: vendor
$(DOCKER_COMPOSE) build batch-test
$(DOCKER_COMPOSE) run -T batch-test bash -c "set -o pipefail; go test -v -mod=vendor -tags=odbc,dynamic $(TPKG) -covermode=atomic -coverpkg=$(TPKG) -json -coverprofile=/testdata/$(PROJECT)_base_coverage.out | tee /testdata/$(PROJECT)_report.out"

46
batch/README.md Normal file
View file

@ -0,0 +1,46 @@
# batch
The `batch` package provides a standard tool set for batching records in a way
that is most performant for ingesting those records into FeatureBase. The main
implementation is `Batch` (which can be initated with the `NewBatch()`
function). The `NewBatch()` function takes an `Importer` which contains all of
the methods required to interact with FeatureBase; these include methods for
doing string/id translation as well as for importing shards of data.
IDK uses the `batch` package internally. Another example where the `batch`
package is used in the `sql3` package. When an "INSERT INTO" statement is
executed, the SQL engine uses a `Batch` to do key translation and build import
batches prior to doing the final import.
## Integration tests
To run the tests, you will need to install the following dependencies:
1. [Docker](https://docs.docker.com/install/)
2. [Docker Compose](https://docs.docker.com/compose/install/)
In addition to these dependancies, you will need to be added to the molecula [Gitlab](https://registry.gitlab.com/molecula) account.
First start the test environment. This is a docker-compose environment that includes featurebase.
BRANCH_NAME=master make startup
To build and run the integration tests, run:
make test-run-local
Then to shut down the test environment, run:
make shutdown
The previous command is equivalent to running the following:
make startup
sleep 30 # wait for services to come up
make test-run
make shutdown
To run an individual test, you can run the command directly using docker-compose. Note that you must run `docker-compose build batch-test` for docker to run the latest code. Modify the following as needed:
make startup
docker-compose build batch-test
docker-compose run batch-test /usr/local/go/bin/go test -count=1 -mod=vendor -run=TestCmdMainOne .

319
client/batch.go → batch/batch.go
View file

@ -1,16 +1,18 @@
// Copyright 2021 Molecula Corp. All rights reserved.
package client
// Package batch provides tooling to prepare batches of records for ingest.
package batch
import (
"bytes"
"context"
"math/bits"
"sort"
"sync"
"time"
featurebase "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/client/egpool"
"github.com/molecula/featurebase/v3/batch/egpool"
"github.com/molecula/featurebase/v3/logger"
"github.com/molecula/featurebase/v3/pql"
"github.com/molecula/featurebase/v3/roaring"
"github.com/pkg/errors"
)
@ -18,6 +20,7 @@ import (
// Batch defaults.
const (
DefaultKeyTranslateBatchSize = 100000
existenceFieldName = "_exists"
)
// TODO if using column translation, column ids might get way out of
@ -76,9 +79,9 @@ type agedTranslation struct {
// Batch implements RecordBatch.
//
// It supports Values of type string, uint64, int64, or nil. The
// following table describes what Pilosa field each type of value must
// map to. Fields are set up when calling "NewBatch".
// It supports Values of type string, uint64, int64, float64, or nil. The
// following table describes what Pilosa field each type of value must map to.
// Fields are set up when calling "NewBatch".
//
// | type | pilosa field type | options |
// |--------+-------------------+-----------|
@ -91,10 +94,10 @@ type agedTranslation struct {
//
// nil values are ignored.
type Batch struct {
client *Client
index *Index
header []*Field
headerMap map[string]*Field
importer Importer
index *featurebase.IndexInfo
header []*featurebase.FieldInfo
headerMap map[string]*featurebase.FieldInfo
// prevDuration records the time that each doImport() takes. This
// is used to set the timeout for transactions to a reasonable
@ -230,16 +233,25 @@ func OptUseShardTransactionalEndpoint(use bool) BatchOption {
}
}
// NewBatch initializes a new Batch object which will use the given
// Pilosa client, index, set of fields, and will take "size" records
// before returning ErrBatchNowFull. The positions of the Fields in
// 'fields' correspond to the positions of values in the Row's Values
// passed to Batch.Add().
func NewBatch(client *Client, size int, index *Index, fields []*Field, opts ...BatchOption) (*Batch, error) {
if len(fields) == 0 || size == 0 {
return nil, errors.New("can't batch with no fields or batch size")
func OptImporter(i Importer) BatchOption {
return func(b *Batch) error {
b.importer = i
return nil
}
headerMap := make(map[string]*Field, len(fields))
}
// NewBatch initializes a new Batch object which will use the given Importer,
// index, set of fields, and will take "size" records before returning
// ErrBatchNowFull. The positions of the Fields in 'fields' correspond to the
// positions of values in the Row's Values passed to Batch.Add().
func NewBatch(importer Importer, size int, index *featurebase.IndexInfo, fields []*featurebase.FieldInfo, opts ...BatchOption) (*Batch, error) {
if len(fields) == 0 {
return nil, errors.New("can't batch with no fields")
} else if size == 0 {
return nil, errors.New("can't batch with no batch size")
}
headerMap := make(map[string]*featurebase.FieldInfo, len(fields))
rowIDs := make(map[int][]uint64, len(fields))
values := make(map[string][]int64)
boolValues := make(map[string]map[int]bool)
@ -248,35 +260,43 @@ func NewBatch(client *Client, size int, index *Index, fields []*Field, opts ...B
ttSets := make(map[string]map[string][]int)
hasTime := false
for i, field := range fields {
headerMap[field.Name()] = field
opts := field.Opts()
switch typ := opts.Type(); typ {
case FieldTypeDefault, FieldTypeSet, FieldTypeTime:
if opts.Keys() {
headerMap[field.Name] = field
opts := field.Options
// The client package has a FieldTypeDefault, but featurebase does not.
// When this code was moved from the client package to the batch
// package, FieldTypeDefault was no longer available. It probably isn't
// necessary, but to ensure backwards compatiblity, we continue to
// support it here with an unexported variable.
fieldTypeDefault := ""
switch typ := opts.Type; typ {
case fieldTypeDefault, featurebase.FieldTypeSet, featurebase.FieldTypeTime:
if opts.Keys {
tt[i] = make(map[string][]int)
ttSets[field.Name()] = make(map[string][]int)
ttSets[field.Name] = make(map[string][]int)
}
hasTime = typ == FieldTypeTime || hasTime
case FieldTypeInt, FieldTypeDecimal, FieldTypeTimestamp:
hasTime = typ == featurebase.FieldTypeTime || hasTime
case featurebase.FieldTypeInt, featurebase.FieldTypeDecimal, featurebase.FieldTypeTimestamp:
// tt line only needed if int field is string foreign key
tt[i] = make(map[string][]int)
values[field.Name()] = make([]int64, 0, size)
case FieldTypeMutex:
values[field.Name] = make([]int64, 0, size)
case featurebase.FieldTypeMutex:
// similar to set/time fields, but no need to support sets
// of values (hence no ttSets)
if opts.Keys() {
if opts.Keys {
tt[i] = make(map[string][]int)
}
rowIDs[i] = make([]uint64, 0, size)
case FieldTypeBool:
boolValues[field.Name()] = make(map[int]bool)
case featurebase.FieldTypeBool:
boolValues[field.Name] = make(map[int]bool)
default:
return nil, errors.Errorf("field type '%s' is not currently supported through Batch", typ)
}
}
b := &Batch{
client: client,
importer: importer,
header: fields,
headerMap: headerMap,
prevDuration: time.Minute * 11,
@ -375,7 +395,7 @@ func (qt *QuantizedTime) Reset() {
// views builds the list of Pilosa views for this particular time,
// given a quantum.
func (qt *QuantizedTime) views(q TimeQuantum) ([]string, error) {
func (qt *QuantizedTime) views(q featurebase.TimeQuantum) ([]string, error) {
zero := QuantizedTime{}
if *qt == zero {
return nil, nil
@ -495,19 +515,19 @@ func (b *Batch) Add(rec Row) error {
field := b.header[i]
switch val := rec.Values[i].(type) {
case string:
switch field.Opts().Type() {
case FieldTypeInt:
switch field.Options.Type {
case featurebase.FieldTypeInt:
if val == "" {
// copied from the `case nil:` section for ints and decimals
b.values[field.Name()] = append(b.values[field.Name()], 0)
nullIndices, ok := b.nullIndices[field.Name()]
b.values[field.Name] = append(b.values[field.Name], 0)
nullIndices, ok := b.nullIndices[field.Name]
if !ok {
nullIndices = make([]uint64, 0)
}
nullIndices = append(nullIndices, uint64(curPos))
b.nullIndices[field.Name()] = nullIndices
} else if intVal, ok := b.getRowTranslation(field.Name(), val); ok {
b.values[field.Name()] = append(b.values[field.Name()], int64(intVal))
b.nullIndices[field.Name] = nullIndices
} else if intVal, ok := b.getRowTranslation(field.Name, val); ok {
b.values[field.Name] = append(b.values[field.Name], int64(intVal))
} else {
ints, ok := b.toTranslate[i][val]
if !ok {
@ -515,9 +535,9 @@ func (b *Batch) Add(rec Row) error {
}
ints = append(ints, curPos)
b.toTranslate[i][val] = ints
b.values[field.Name()] = append(b.values[field.Name()], 0)
b.values[field.Name] = append(b.values[field.Name], 0)
}
case FieldTypeBool:
case featurebase.FieldTypeBool:
// If we want to support bools as string values, we would do
// that here.
default:
@ -530,7 +550,7 @@ func (b *Batch) Add(rec Row) error {
if val == "" { //
b.rowIDs[i] = append(rowIDs, nilSentinel)
} else if rowID, ok := b.getRowTranslation(field.Name(), val); ok {
} else if rowID, ok := b.getRowTranslation(field.Name, val); ok {
b.rowIDs[i] = append(rowIDs, rowID)
} else {
ints, ok := b.toTranslate[i][val]
@ -549,15 +569,15 @@ func (b *Batch) Add(rec Row) error {
}
b.rowIDs[i] = append(b.rowIDs[i], val)
case int64:
b.values[field.Name()] = append(b.values[field.Name()], val)
b.values[field.Name] = append(b.values[field.Name], val)
case []string:
if len(val) == 0 {
continue
}
rowIDSets, ok := b.rowIDSets[field.Name()]
rowIDSets, ok := b.rowIDSets[field.Name]
if !ok {
rowIDSets = make([][]uint64, len(b.ids)-1, cap(b.ids))
b.rowIDSets[field.Name()] = rowIDSets
b.rowIDSets[field.Name] = rowIDSets
}
for len(rowIDSets) < len(b.ids)-1 {
rowIDSets = append(rowIDSets, nil) // nil extend
@ -568,53 +588,53 @@ func (b *Batch) Add(rec Row) error {
if k == "" {
continue
}
if rowID, ok := b.getRowTranslation(field.Name(), k); ok {
if rowID, ok := b.getRowTranslation(field.Name, k); ok {
rowIDs = append(rowIDs, rowID)
} else {
ttsets, ok := b.toTranslateSets[field.Name()]
ttsets, ok := b.toTranslateSets[field.Name]
if !ok {
ttsets = make(map[string][]int)
b.toTranslateSets[field.Name()] = make(map[string][]int)
b.toTranslateSets[field.Name] = make(map[string][]int)
}
ints, ok := ttsets[k]
if !ok {
ints = make([]int, 0, 1)
}
ints = append(ints, curPos)
b.toTranslateSets[field.Name()][k] = ints
b.toTranslateSets[field.Name][k] = ints
}
}
b.rowIDSets[field.Name()] = append(rowIDSets, rowIDs)
b.rowIDSets[field.Name] = append(rowIDSets, rowIDs)
case []uint64:
if len(val) == 0 {
continue
}
rowIDSets, ok := b.rowIDSets[field.Name()]
rowIDSets, ok := b.rowIDSets[field.Name]
if !ok {
rowIDSets = make([][]uint64, len(b.ids)-1, cap(b.ids))
}
for len(rowIDSets) < len(b.ids)-1 {
rowIDSets = append(rowIDSets, nil) // nil extend
}
b.rowIDSets[field.Name()] = append(rowIDSets, val)
b.rowIDSets[field.Name] = append(rowIDSets, val)
case nil:
switch field.Opts().Type() {
case FieldTypeInt, FieldTypeDecimal, FieldTypeTimestamp:
b.values[field.Name()] = append(b.values[field.Name()], 0)
nullIndices, ok := b.nullIndices[field.Name()]
switch field.Options.Type {
case featurebase.FieldTypeInt, featurebase.FieldTypeDecimal, featurebase.FieldTypeTimestamp:
b.values[field.Name] = append(b.values[field.Name], 0)
nullIndices, ok := b.nullIndices[field.Name]
if !ok {
nullIndices = make([]uint64, 0)
}
nullIndices = append(nullIndices, uint64(curPos))
b.nullIndices[field.Name()] = nullIndices
b.nullIndices[field.Name] = nullIndices
case FieldTypeBool:
boolNulls, ok := b.boolNulls[field.Name()]
case featurebase.FieldTypeBool:
boolNulls, ok := b.boolNulls[field.Name]
if !ok {
boolNulls = make([]uint64, 0)
}
boolNulls = append(boolNulls, uint64(curPos))
b.boolNulls[field.Name()] = boolNulls
b.boolNulls[field.Name] = boolNulls
default:
// only append nil to rowIDs if this field already has
@ -629,7 +649,10 @@ func (b *Batch) Add(rec Row) error {
}
case bool:
b.boolValues[field.Name()][curPos] = val
b.boolValues[field.Name][curPos] = val
case pql.Decimal:
b.values[field.Name] = append(b.values[field.Name], val.ToInt64(field.Options.Scale))
default:
return errors.Errorf("Val %v Type %[1]T is not currently supported. Use string, uint64 (row id), or int64 (integer value)", val)
@ -645,7 +668,7 @@ func (b *Batch) Add(rec Row) error {
case string:
clearRows := b.clearRowIDs[i]
// translate val and add to clearRows
if rowID, ok := b.getRowTranslation(field.Name(), val); ok {
if rowID, ok := b.getRowTranslation(field.Name, val); ok {
clearRows[curPos] = rowID
} else {
_, ok := b.toTranslateClear[i]
@ -662,14 +685,14 @@ func (b *Batch) Add(rec Row) error {
case uint64:
b.clearRowIDs[i][curPos] = val
case nil:
if field.Opts().Type() == FieldTypeMutex {
if field.Options.Type == featurebase.FieldTypeMutex {
for len(b.rowIDs[i]) <= curPos {
b.rowIDs[i] = append(b.rowIDs[i], nilSentinel)
}
b.rowIDs[i][len(b.rowIDs[i])-1] = clearSentinel
}
default:
return errors.Errorf("Clearing a value '%v' Type %[1]T is not currently supported (field '%s')", val, field.Name())
return errors.Errorf("Clearing a value '%v' Type %[1]T is not currently supported (field '%s')", val, field.Name)
}
// nil extend b.rowIDs so we don't run into a horrible bug
// where we skip doing clears because b.rowIDs doesn't have a
@ -693,8 +716,8 @@ func (b *Batch) Add(rec Row) error {
return nil
}
// ErrBatchNowFull, similar to io.EOF, is a marker error to notify the
// user of a batch that it is time to call Import.
// ErrBatchNowFull — similar to io.EOF — is a marker error to notify the user of
// a batch that it is time to call Import.
var ErrBatchNowFull = errors.New("batch is now full - you cannot add any more records (though the one you just added was accepted)")
// ErrBatchAlreadyFull is a real error saying that Batch.Add did not
@ -714,17 +737,19 @@ var ErrBatchNowStale = errors.New("batch is stale and needs to be imported (howe
// continues. split batch mode DOES NOT CURRENTLY SUPPORT MUTEX
// OR INT FIELDS!
func (b *Batch) Import() error {
ctx := context.Background()
start := time.Now()
trns, err := b.client.StartTransaction("", b.prevDuration*10, false, time.Hour)
trns, err := b.importer.StartTransaction(ctx, "", b.prevDuration*10, false, time.Hour)
if err != nil {
return errors.Wrap(err, "starting transaction")
}
defer func() {
trnsl, err := b.client.FinishTransaction(trns.ID)
if err != nil {
b.log.Errorf("error finishing transaction: %v. trns: %+v", err, trnsl)
if trns != nil {
if trnsl, err := b.importer.FinishTransaction(ctx, trns.ID); err != nil {
b.log.Errorf("error finishing transaction: %v. trns: %+v", err, trnsl)
}
}
b.client.Stats.Timing(MetricBatchImportDurationSeconds, time.Since(start), 1.0)
b.importer.StatsTiming(MetricBatchImportDurationSeconds, time.Since(start), 1.0)
}()
size := len(b.ids)
@ -781,21 +806,23 @@ func (b *Batch) Import() error {
// imports the stored data to Pilosa. Otherwise it simply returns
// nil.
func (b *Batch) Flush() error {
ctx := context.Background()
if !b.splitBatchMode {
return nil
}
start := time.Now()
trns, err := b.client.StartTransaction("", b.prevDuration*10, false, time.Hour)
trns, err := b.importer.StartTransaction(ctx, "", b.prevDuration*10, false, time.Hour)
if err != nil {
return errors.Wrap(err, "starting transaction")
}
defer func() {
trnsl, err := b.client.FinishTransaction(trns.ID)
trnsl, err := b.importer.FinishTransaction(ctx, trns.ID)
if err != nil {
b.log.Errorf("error finishing transaction: %v. trns: %+v", err, trnsl)
}
b.client.Stats.Timing(MetricBatchFlushDurationSeconds, time.Since(start), 1.0)
b.importer.StatsTiming(MetricBatchFlushDurationSeconds, time.Since(start), 1.0)
}()
importStart := time.Now()
@ -886,7 +913,7 @@ func (b *Batch) doTranslation() error {
// Look up the associated field.
field := b.header[i]
fieldName := field.Name()
fieldName := field.Name
// Fetch the translation cache.
rowCache := b.rowTranslations[fieldName]
@ -924,8 +951,8 @@ func (b *Batch) doTranslation() error {
}
rowCacheLock.Unlock()
switch ftype := field.Opts().Type(); ftype {
case FieldTypeSet, FieldTypeMutex, FieldTypeTime:
switch ftype := field.Options.Type; ftype {
case featurebase.FieldTypeSet, featurebase.FieldTypeMutex, featurebase.FieldTypeTime:
// Fill out missing IDs in local batch records with translated IDs.
rows := b.rowIDs[i]
for key, idxs := range tt {
@ -952,7 +979,7 @@ func (b *Batch) doTranslation() error {
}
}
case FieldTypeInt:
case featurebase.FieldTypeInt:
// Handle foreign key int fields — fill out b.values instead of b.rows.
vals := b.values[fieldName]
for key, idxs := range tt {
@ -1036,10 +1063,12 @@ func (b *Batch) doTranslation() error {
return eg.Wait()
}
func (b *Batch) createIndexKeys(index *Index, keys ...string) (map[string]uint64, error) {
func (b *Batch) createIndexKeys(index *featurebase.IndexInfo, keys ...string) (map[string]uint64, error) {
ctx := context.Background()
batchSize := b.keyTranslateBatchSize
if batchSize <= 0 || len(keys) <= batchSize {
return b.client.CreateIndexKeys(index, keys...)
return b.importer.CreateIndexKeys(ctx, index, keys...)
}
results := make(map[string]uint64, len(keys))
@ -1049,7 +1078,7 @@ func (b *Batch) createIndexKeys(index *Index, keys ...string) (map[string]uint64
keySlice = keySlice[:batchSize]
}
trans, err := b.client.CreateIndexKeys(index, keySlice...)
trans, err := b.importer.CreateIndexKeys(ctx, index, keySlice...)
if err != nil {
return nil, err
} else if len(trans) != len(keySlice) {
@ -1065,10 +1094,12 @@ func (b *Batch) createIndexKeys(index *Index, keys ...string) (map[string]uint64
return results, nil
}
func (b *Batch) createFieldKeys(field *Field, keys ...string) (map[string]uint64, error) {
func (b *Batch) createFieldKeys(field *featurebase.FieldInfo, keys ...string) (map[string]uint64, error) {
ctx := context.Background()
batchSize := b.keyTranslateBatchSize
if batchSize <= 0 || len(keys) <= batchSize {
return b.client.CreateFieldKeys(field, keys...)
return b.importer.CreateFieldKeys(ctx, b.index.Name, field, keys...)
}
results := make(map[string]uint64, len(keys))
@ -1078,7 +1109,7 @@ func (b *Batch) createFieldKeys(field *Field, keys ...string) (map[string]uint64
keySlice = keySlice[:batchSize]
}
trans, err := b.client.CreateFieldKeys(field, keySlice...)
trans, err := b.importer.CreateFieldKeys(ctx, b.index.Name, field, keySlice...)
if err != nil {
return nil, err
} else if len(trans) != len(keySlice) {
@ -1095,6 +1126,8 @@ func (b *Batch) createFieldKeys(field *Field, keys ...string) (map[string]uint64
}
func (b *Batch) doImportShardTransactional(frags, clearFrags fragments) error {
ctx := context.Background()
start := time.Now()
requests := make(map[uint64]*featurebase.ImportRoaringShardRequest)
getOrCreate := func(requests map[uint64]*featurebase.ImportRoaringShardRequest, shard uint64) *featurebase.ImportRoaringShardRequest {
@ -1149,24 +1182,25 @@ func (b *Batch) doImportShardTransactional(frags, clearFrags fragments) error {
}
}
b.client.Stats.Timing(MetricBatchShardImportBuildRequestsSeconds, time.Since(start), 1.0)
b.importer.StatsTiming(MetricBatchShardImportBuildRequestsSeconds, time.Since(start), 1.0)
start = time.Now()
eg := egpool.Group{PoolSize: 20}
for shard, request := range requests {
shard := shard
request := request
eg.Go(func() error {
return b.client.ImportRoaringShard(b.index.Name(), shard, request)
return b.importer.ImportRoaringShard(ctx, b.index.Name, shard, request)
})
}
err := eg.Wait()
dur := time.Since(start)
b.client.Stats.Timing(MetricBatchShardImportDurationSeconds, dur, 1.0)
b.importer.StatsTiming(MetricBatchShardImportDurationSeconds, dur, 1.0)
b.log.Printf("import shard took: %v\n", dur)
return errors.Wrap(err, "doing shard-transactional imports")
}
func (b *Batch) doImport(frags, clearFrags fragments) error {
ctx := context.Background()
start := time.Now()
eg := egpool.Group{PoolSize: 20}
@ -1188,7 +1222,7 @@ func (b *Batch) doImport(frags, clearFrags fragments) error {
clearViewMap := clearFrags.GetViewMap(shard, field)
if len(clearViewMap) > 0 {
startx := time.Now()
err := b.client.ImportRoaringBitmap(b.index.Field(field), shard, clearViewMap, true)
err := b.importer.ImportRoaringBitmap(ctx, b.index.Name, b.indexField(field), shard, clearViewMap, true)
if err != nil {
return errors.Wrapf(err, "import clearing clearing data for %s", field)
}
@ -1196,7 +1230,7 @@ func (b *Batch) doImport(frags, clearFrags fragments) error {
}
starty := time.Now()
err := b.client.ImportRoaringBitmap(b.index.Field(field), shard, viewMap, false)
err := b.importer.ImportRoaringBitmap(ctx, b.index.Name, b.indexField(field), shard, viewMap, false)
b.log.Debugf("imp-roar %s,shard:%d,views:%d %v", field, shard, len(clearViewMap), time.Since(starty))
return errors.Wrapf(err, "importing data for %s", field)
})
@ -1215,6 +1249,22 @@ func (b *Batch) doImport(frags, clearFrags fragments) error {
return nil
}
// indexField is a helper function which was introduced when we switched the
// index and field types from being client types (e.g client.Index,
// client.Field) to being featurebase types (e.g. featurebase.IndexInfo,
// featurebase.FieldInfo). Unlike client.Index, featurebase.IndexInfo is not
// expected to contain the "_exists" field. So calling Field("_exists") on
// IndexInfo results in a nil field. This method creates an instance of
// FieldInfo for the "_exists" field.
func (b *Batch) indexField(field string) *featurebase.FieldInfo {
if field == existenceFieldName {
return &featurebase.FieldInfo{
Name: existenceFieldName,
}
}
return b.index.Field(field)
}
func anyCause(cause error, errs ...error) error {
if cause == nil {
return nil
@ -1229,11 +1279,7 @@ func anyCause(cause error, errs ...error) error {
}
func (b *Batch) shardWidth() uint64 {
shardWidth := b.index.ShardWidth()
if shardWidth == 0 {
shardWidth = DefaultShardWidth
}
return shardWidth
return featurebase.ShardWidth
}
// this is kind of bad as it means we can never import column id
@ -1251,7 +1297,7 @@ func (b *Batch) makeFragments(frags, clearFrags fragments) (fragments, fragments
emptyClearRows := make(map[int]uint64)
// create _exists fragments if needed
if b.index.Opts().TrackExistence() {
if b.index.Options.TrackExistence {
var curBM *roaring.Bitmap
curShard := ^uint64(0) // impossible sentinel value for shard.
for _, col := range b.ids {
@ -1272,8 +1318,8 @@ func (b *Batch) makeFragments(frags, clearFrags fragments) (fragments, fragments
clearRows = emptyClearRows
}
field := b.header[i]
opts := field.Opts()
if opts.Type() == FieldTypeMutex {
opts := field.Options
if opts.Type == featurebase.FieldTypeMutex {
continue // we handle mutex fields separately — they can't use importRoaring
}
curShard := ^uint64(0) // impossible sentinel value for shard.
@ -1291,8 +1337,8 @@ func (b *Batch) makeFragments(frags, clearFrags fragments) (fragments, fragments
if col/shardWidth != curShard {
curShard = col / shardWidth
curBM = frags.GetOrCreate(curShard, field.Name(), "")
clearBM = clearFrags.GetOrCreate(curShard, field.Name(), "")
curBM = frags.GetOrCreate(curShard, field.Name, "")
clearBM = clearFrags.GetOrCreate(curShard, field.Name, "")
}
if row != nilSentinel {
// TODO this is super ugly, but we want to avoid setting
@ -1300,16 +1346,16 @@ func (b *Batch) makeFragments(frags, clearFrags fragments) (fragments, fragments
// there isn't one. Should probably refactor this whole
// loop to be more general w.r.t. views. Also... tests for
// the NoStandardView case would be great.
if !(opts.Type() == FieldTypeTime && opts.NoStandardView()) {
if !(opts.Type == featurebase.FieldTypeTime && opts.NoStandardView) {
curBM.DirectAdd(row*shardWidth + (col % shardWidth))
}
if opts.Type() == FieldTypeTime {
views, err := b.times[j].views(opts.TimeQuantum())
if opts.Type == featurebase.FieldTypeTime {
views, err := b.times[j].views(opts.TimeQuantum)
if err != nil {
return nil, nil, errors.Wrap(err, "calculating views")
}
for _, view := range views {
tbm := frags.GetOrCreate(curShard, field.Name(), view)
tbm := frags.GetOrCreate(curShard, field.Name, view)
tbm.DirectAdd(row*shardWidth + (col % shardWidth))
}
}
@ -1337,7 +1383,7 @@ func (b *Batch) makeFragments(frags, clearFrags fragments) (fragments, fragments
rowIDSets = rowIDSets[:len(b.ids)]
}
field := b.headerMap[fname]
opts := field.Opts()
opts := field.Options
curShard := ^uint64(0) // impossible sentinel value for shard.
var curBM *roaring.Bitmap
for j := range b.ids {
@ -1354,13 +1400,13 @@ func (b *Batch) makeFragments(frags, clearFrags fragments) (fragments, fragments
// there isn't one. Should probably refactor this whole
// loop to be more general w.r.t. views. Also... tests for
// the NoStandardView case would be great.
if !(opts.Type() == FieldTypeTime && opts.NoStandardView()) {
if !(opts.Type == featurebase.FieldTypeTime && opts.NoStandardView) {
for _, row := range rowIDs {
curBM.DirectAdd(row*shardWidth + (col % shardWidth))
}
}
if opts.Type() == FieldTypeTime {
views, err := b.times[j].views(opts.TimeQuantum())
if opts.Type == featurebase.FieldTypeTime {
views, err := b.times[j].views(opts.TimeQuantum)
if err != nil {
return nil, nil, errors.Wrap(err, "calculating views")
}
@ -1409,8 +1455,8 @@ func (b *Batch) makeSingleValFragments(frags, clearFrags fragments) (fragments,
sort.Stable(sc)
}
field := b.headerMap[fieldName]
base := field.Options().base
if field.Options().Type() == FieldTypeTimestamp {
base := field.Options.Base
if field.Options.Type == featurebase.FieldTypeTimestamp {
base = 0
}
@ -1454,7 +1500,7 @@ func (b *Batch) makeSingleValFragments(frags, clearFrags fragments) (fragments,
// -------------------------
for findex, rowIDs := range b.rowIDs {
field := b.header[findex]
if field.Opts().Type() != FieldTypeMutex {
if field.Options.Type != featurebase.FieldTypeMutex {
continue
}
ids = ids[:0]
@ -1483,8 +1529,8 @@ func (b *Batch) makeSingleValFragments(frags, clearFrags fragments) (fragments,
}
shard := ids[0] / shardWidth
bitmap := frags.GetOrCreate(shard, field.Name(), "standard")
clearBM := clearFrags.GetOrCreate(shard, field.Name(), "standard")
bitmap := frags.GetOrCreate(shard, field.Name, "standard")
clearBM := clearFrags.GetOrCreate(shard, field.Name, "standard")
for i, id := range ids {
if i+1 < len(ids) {
// we only want the last value set for each id
@ -1495,8 +1541,8 @@ func (b *Batch) makeSingleValFragments(frags, clearFrags fragments) (fragments,
row := rowIDs[i]
if shard != id/shardWidth {
shard = id / shardWidth
bitmap = frags.GetOrCreate(shard, field.Name(), "standard")
clearBM = clearFrags.GetOrCreate(shard, field.Name(), "standard")
bitmap = frags.GetOrCreate(shard, field.Name, "standard")
clearBM = clearFrags.GetOrCreate(shard, field.Name, "standard")
}
fragmentColumn := id % shardWidth
clearBM.Add(fragmentColumn) // Will use this to clear columns.
@ -1521,15 +1567,16 @@ func (b *Batch) makeSingleValFragments(frags, clearFrags fragments) (fragments,
// records (for all rows) to clear.
for fieldname, boolNulls := range b.boolNulls {
field := b.headerMap[fieldname]
if field.Opts().Type() != featurebase.FieldTypeBool {
if field.Options.Type != featurebase.FieldTypeBool {
continue
}
for _, pos := range boolNulls {
recID := b.ids[pos]
shard := recID / shardWidth
clearBM := clearFrags.GetOrCreate(shard, field.Name(), "standard")
clearBM := clearFrags.GetOrCreate(shard, field.Name, "standard")
fragmentColumn := recID % shardWidth
clearBM.Add(fragmentColumn)
}
}
@ -1540,7 +1587,7 @@ func (b *Batch) makeSingleValFragments(frags, clearFrags fragments) (fragments,
// the bit in the "true" row.
for fieldname, boolMap := range b.boolValues {
field := b.headerMap[fieldname]
if field.Opts().Type() != featurebase.FieldTypeBool {
if field.Options.Type != featurebase.FieldTypeBool {
continue
}
@ -1548,8 +1595,8 @@ func (b *Batch) makeSingleValFragments(frags, clearFrags fragments) (fragments,
recID := b.ids[pos]
shard := recID / shardWidth
bitmap := frags.GetOrCreate(shard, field.Name(), "standard")
clearBM := clearFrags.GetOrCreate(shard, field.Name(), "standard")
bitmap := frags.GetOrCreate(shard, field.Name, "standard")
clearBM := clearFrags.GetOrCreate(shard, field.Name, "standard")
fragmentColumn := recID % shardWidth
clearBM.Add(fragmentColumn)
@ -1582,10 +1629,9 @@ func (v *valsByIDsSortable) Swap(i, j int) {
// importValueData imports data for int fields.
func (b *Batch) importValueData() error {
shardWidth := b.index.ShardWidth()
if shardWidth == 0 {
shardWidth = DefaultShardWidth
}
ctx := context.Background()
shardWidth := uint64(featurebase.ShardWidth)
eg := egpool.Group{PoolSize: 20}
ids := make([]uint64, len(b.ids))
@ -1630,15 +1676,15 @@ func (b *Batch) importValueData() error {
endIdx := i
shard := curShard
field := b.headerMap[fieldName]
path, data, err := b.client.EncodeImportValues(field, shard, bvalues[startIdx:endIdx], ids[startIdx:endIdx], false)
path, data, err := b.importer.EncodeImportValues(ctx, b.index.Name, field, shard, bvalues[startIdx:endIdx], ids[startIdx:endIdx], false)
if err != nil {
return errors.Wrap(err, "encoding import values")
}
eg.Go(func() error {
start := time.Now()
err := b.client.DoImportValues(b.index.Name(), shard, path, data)
err := b.importer.DoImport(ctx, b.index.Name, field, shard, path, data)
b.log.Debugf("imp-vals %s,shard:%d,data:%d %v", field, shard, len(data), time.Since(start))
return errors.Wrapf(err, "importing values for field = %s", field)
return errors.Wrapf(err, "importing values for field = %s", field.Name)
})
startIdx = i
curShard = recordID / shardWidth
@ -1673,16 +1719,15 @@ func (v *rowsByIDsSortable) Swap(i, j int) {
// TODO this should work for bools as well - just need to support them
// at batch creation time and when calling Add, I think.
func (b *Batch) importMutexData() error {
shardWidth := b.index.ShardWidth()
if shardWidth == 0 {
shardWidth = DefaultShardWidth
}
ctx := context.Background()
shardWidth := uint64(featurebase.ShardWidth)
eg := egpool.Group{PoolSize: 20}
ids := make([]uint64, 0, len(b.ids))
for findex, rowIDs := range b.rowIDs {
field := b.header[findex]
if field.Opts().Type() != FieldTypeMutex {
if field.Options.Type != featurebase.FieldTypeMutex {
continue
}
ids = ids[:0]
@ -1724,15 +1769,15 @@ func (b *Batch) importMutexData() error {
endIdx := i
shard := curShard
field := field
path, data, err := b.client.EncodeImport(field, shard, rowIDs[startIdx:endIdx], ids[startIdx:endIdx], false)
path, data, err := b.importer.EncodeImport(ctx, b.index.Name, field, shard, rowIDs[startIdx:endIdx], ids[startIdx:endIdx], false)
if err != nil {
return errors.Wrap(err, "encoding mutex import")
}
eg.Go(func() error {
start := time.Now()
err := b.client.DoImport(b.index.Name(), shard, path, data)
b.log.Debugf("imp-mux %s,shard:%d,data:%d %v", field.Name(), shard, len(data), time.Since(start))
return errors.Wrapf(err, "importing values for field = %s", field)
err := b.importer.DoImport(ctx, b.index.Name, field, shard, path, data)
b.log.Debugf("imp-mux %s,shard:%d,data:%d %v", field.Name, shard, len(data), time.Since(start))
return errors.Wrapf(err, "importing values for field = %s", field.Name)
})
startIdx = i
curShard = recordID / shardWidth

2359
batch/batch_test.go Normal file
View file

File diff suppressed because it is too large Load diff

145
batch/convert.go Normal file
View file

@ -0,0 +1,145 @@
package batch
import (
"time"
featurebase "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/errors"
)
var (
MinTimestampNano = time.Unix(-1<<32, 0).UTC() // 1833-11-24T17:31:44Z
MaxTimestampNano = time.Unix(1<<32, 0).UTC() // 2106-02-07T06:28:16Z
MinTimestamp = time.Unix(-62135596799, 0).UTC() // 0001-01-01T00:00:01Z
MaxTimestamp = time.Unix(253402300799, 0).UTC() // 9999-12-31T23:59:59Z
ErrTimestampOutOfRange = errors.New("", "value provided for timestamp field is out of range")
)
type TimeUnit string
const (
TimeUnitSeconds = TimeUnit(featurebase.TimeUnitSeconds)
TimeUnitMilliseconds = TimeUnit(featurebase.TimeUnitMilliseconds)
TimeUnitMicroseconds = TimeUnit(featurebase.TimeUnitMicroseconds)
TimeUnitUSeconds = TimeUnit(featurebase.TimeUnitUSeconds)
TimeUnitNanoseconds = TimeUnit(featurebase.TimeUnitNanoseconds)
)
// TimestampToInt64 converts the provided timestamp to an int64 as the number of
// units past the epoch.
func TimestampToInt64(unit TimeUnit, epoch time.Time, ts time.Time) (int64, error) {
var err error
unit, err = validateTimeUnit(unit)
if err != nil {
return 0, errors.Wrap(err, "validating time unit")
}
epoch, err = validateEpoch(epoch)
if err != nil {
return 0, errors.Wrap(err, "validating epoch")
}
// Check if the epoch alone is out-of-range. If so, ingest should halt,
// regardless of state of the timestamp out-of-range CLI option.
if err := validateTimestamp(unit, epoch); err != nil {
return 0, errors.Wrap(err, "validating epoch")
}
epochAsInt64 := timestampToInt(unit, epoch)
// Check if the timestamp is out-of-range.
if err := validateTimestamp(unit, ts); err != nil {
return 0, errors.Wrapf(ErrTimestampOutOfRange, "validating timestamp: %s", ts)
}
tsAsInt64 := timestampToInt(unit, ts)
return tsAsInt64 - epochAsInt64, nil
}
// validateTimeUnit checks if the time unit is supported. If the provided unit
// is blank, validateTimeUnit returns the default TimeUnit.
func validateTimeUnit(unit TimeUnit) (TimeUnit, error) {
switch unit {
case "":
return TimeUnitSeconds, nil
case TimeUnitSeconds,
TimeUnitMilliseconds,
TimeUnitMicroseconds,
TimeUnitUSeconds,
TimeUnitNanoseconds:
return unit, nil
}
return "", errors.Errorf("unsupported time unit: %s", unit)
}
// validateEpoch checks if the epoch is supported. If the provided epoch
// is "zero", validateEpoch returns the default epoch value.
func validateEpoch(epoch time.Time) (time.Time, error) {
if epoch.IsZero() {
return time.Unix(0, 0), nil
}
return epoch, nil
}
// validateTimestamp checks if the timestamp is within the range of what FB accepts.
func validateTimestamp(unit TimeUnit, ts time.Time) error {
// Min and Max timestamps that Featurebase accepts
var minStamp, maxStamp time.Time
switch unit {
case TimeUnitNanoseconds:
minStamp = MinTimestampNano
maxStamp = MaxTimestampNano
default:
minStamp = MinTimestamp
maxStamp = MaxTimestamp
}
if ts.Before(minStamp) || ts.After(maxStamp) {
return errors.Errorf("timestamp value (%v) must be within min: %v and max: %v", ts, minStamp, maxStamp)
}
return nil
}
// timestampToInt takes a time unit and a time.Time and converts it to an
// integer value.
func timestampToInt(unit TimeUnit, ts time.Time) int64 {
switch unit {
case TimeUnitSeconds:
return ts.Unix()
case TimeUnitMilliseconds:
return ts.UnixMilli()
case TimeUnitMicroseconds, TimeUnitUSeconds:
return ts.UnixMicro()
case TimeUnitNanoseconds:
return ts.UnixNano()
}
return 0
}
// intToTimestamp takes a timeunit and an integer value and converts it to
// time.Time.
func intToTimestamp(unit TimeUnit, val int64) (time.Time, error) {
switch unit {
case TimeUnitSeconds:
return time.Unix(val, 0).UTC(), nil
case TimeUnitMilliseconds:
return time.UnixMilli(val).UTC(), nil
case TimeUnitMicroseconds, TimeUnitUSeconds:
return time.UnixMicro(val).UTC(), nil
case TimeUnitNanoseconds:
return time.Unix(0, val).UTC(), nil
default:
return time.Time{}, errors.Errorf("Unknown time unit: '%v'", unit)
}
}
// Int64ToTimestamp converts the provided int64 to a timestamp based on the time unit
// and epoch.
func Int64ToTimestamp(unit TimeUnit, epoch time.Time, val int64) (time.Time, error) {
return intToTimestamp(unit, timestampToInt(unit, epoch)+val)
}

26
batch/docker-compose.yml Normal file
View file

@ -0,0 +1,26 @@
version: '3'
services:
featurebase:
build:
context: ../.
dockerfile: ./Dockerfile
environment:
PILOSA_DATA_DIR: /data
PILOSA_BIND: 0.0.0.0:10101
PILOSA_BIND_GRPC: 0.0.0.0:20101
PILOSA_ADVERTISE: featurebase:10101
volumes:
- ./testdata:/testdata
batch-test:
build:
context: ../.
dockerfile: ./batch/Dockerfile-test
volumes:
- ./testdata:/testdata
wait:
build:
context: .
dockerfile: Dockerfile-wait

0
client/egpool/egpool.go → batch/egpool/egpool.go
View file

2
client/egpool/egpool_test.go → batch/egpool/egpool_test.go
View file

@ -5,7 +5,7 @@ import (
"errors"
"testing"
"github.com/molecula/featurebase/v3/client/egpool"
"github.com/molecula/featurebase/v3/batch/egpool"
)
func TestEGPool(t *testing.T) {

8
batch/error.go Normal file
View file

@ -0,0 +1,8 @@
package batch
import "github.com/pkg/errors"
// Predefined batch-related errors.
var (
ErrPreconditionFailed = errors.New("Precondition failed")
)

211
batch/importer.go Normal file
View file

@ -0,0 +1,211 @@
package batch
import (
"context"
"time"
"github.com/golang/protobuf/proto" //nolint:staticcheck
featurebase "github.com/molecula/featurebase/v3"
featurebaseproto "github.com/molecula/featurebase/v3/encoding/proto"
"github.com/molecula/featurebase/v3/pb"
"github.com/molecula/featurebase/v3/roaring"
"github.com/pkg/errors"
)
type Importer interface {
StartTransaction(ctx context.Context, id string, timeout time.Duration, exclusive bool, requestTimeout time.Duration) (*featurebase.Transaction, error)
FinishTransaction(ctx context.Context, id string) (*featurebase.Transaction, error)
CreateIndexKeys(ctx context.Context, idx *featurebase.IndexInfo, keys ...string) (map[string]uint64, error)
CreateFieldKeys(ctx context.Context, index string, field *featurebase.FieldInfo, keys ...string) (map[string]uint64, error)
ImportRoaringBitmap(ctx context.Context, index string, field *featurebase.FieldInfo, shard uint64, views map[string]*roaring.Bitmap, clear bool) error
ImportRoaringShard(ctx context.Context, index string, shard uint64, request *featurebase.ImportRoaringShardRequest) error
EncodeImportValues(ctx context.Context, index string, field *featurebase.FieldInfo, shard uint64, vals []int64, ids []uint64, clear bool) (path string, data []byte, err error)
EncodeImport(ctx context.Context, index string, field *featurebase.FieldInfo, shard uint64, vals, ids []uint64, clear bool) (path string, data []byte, err error)
DoImport(ctx context.Context, index string, field *featurebase.FieldInfo, shard uint64, path string, data []byte) error
StatsTiming(name string, value time.Duration, rate float64)
}
// Ensure type implements interface.
var _ Importer = &nopImporter{}
// NopImporter is an implementation of the Importer interface that doesn't do
// anything.
var NopImporter Importer = &nopImporter{}
type nopImporter struct{}
func (n *nopImporter) StartTransaction(ctx context.Context, id string, timeout time.Duration, exclusive bool, requestTimeout time.Duration) (*featurebase.Transaction, error) {
return nil, nil
}
func (n *nopImporter) FinishTransaction(ctx context.Context, id string) (*featurebase.Transaction, error) {
return nil, nil
}
func (n *nopImporter) CreateIndexKeys(ctx context.Context, idx *featurebase.IndexInfo, keys ...string) (map[string]uint64, error) {
return nil, nil
}
func (n *nopImporter) CreateFieldKeys(ctx context.Context, index string, field *featurebase.FieldInfo, keys ...string) (map[string]uint64, error) {
return nil, nil
}
func (n *nopImporter) ImportRoaringBitmap(ctx context.Context, index string, field *featurebase.FieldInfo, shard uint64, views map[string]*roaring.Bitmap, clear bool) error {
return nil
}
func (n *nopImporter) ImportRoaringShard(ctx context.Context, index string, shard uint64, request *featurebase.ImportRoaringShardRequest) error {
return nil
}
func (n *nopImporter) EncodeImportValues(ctx context.Context, index string, field *featurebase.FieldInfo, shard uint64, vals []int64, ids []uint64, clear bool) (path string, data []byte, err error) {
return "", nil, nil
}
func (n *nopImporter) EncodeImport(ctx context.Context, index string, field *featurebase.FieldInfo, shard uint64, vals, ids []uint64, clear bool) (path string, data []byte, err error) {
return "", nil, nil
}
func (n *nopImporter) DoImport(ctx context.Context, index string, field *featurebase.FieldInfo, shard uint64, path string, data []byte) error {
return nil
}
func (n *nopImporter) StatsTiming(name string, value time.Duration, rate float64) {}
// Ensure type implements interface.
var _ Importer = &FeaturebaseImporter{}
// FeaturebaseImporter is a wrapper around featurebase.API, making it a
// batch.Importer.
type FeaturebaseImporter struct {
*featurebase.API
}
func (f *FeaturebaseImporter) StartTransaction(ctx context.Context, id string, timeout time.Duration, exclusive bool, requestTimeout time.Duration) (*featurebase.Transaction, error) {
return f.API.StartTransaction(ctx, id, timeout, exclusive, false)
}
func (f *FeaturebaseImporter) FinishTransaction(ctx context.Context, id string) (*featurebase.Transaction, error) {
return f.API.FinishTransaction(ctx, id, false)
}
func (f *FeaturebaseImporter) CreateIndexKeys(ctx context.Context, idx *featurebase.IndexInfo, keys ...string) (map[string]uint64, error) {
return f.API.CreateIndexKeys(ctx, idx.Name, keys...)
}
func (f *FeaturebaseImporter) CreateFieldKeys(ctx context.Context, index string, field *featurebase.FieldInfo, keys ...string) (map[string]uint64, error) {
return f.API.CreateFieldKeys(ctx, index, field.Name, keys...)
}
func (f *FeaturebaseImporter) ImportRoaringBitmap(ctx context.Context, index string, field *featurebase.FieldInfo, shard uint64, views map[string]*roaring.Bitmap, clear bool) error {
vs := make(map[string][]byte)
for k, v := range views {
data := roaring.BitmapsToRoaring([]*roaring.Bitmap{v})
if len(data) > 0 {
vs[k] = data
}
}
req := &featurebase.ImportRoaringRequest{
IndexCreatedAt: 0,
FieldCreatedAt: field.CreatedAt,
Clear: clear,
Views: vs,
}
return f.API.ImportRoaring(ctx, index, field.Name, shard, false, req)
}
// ImportRoaringShard doesn't technically need to be implemented here, because
// the method on f.API has the same signature and already satisfies the
// interface. But we put this here to avoid possible confusion.
func (f *FeaturebaseImporter) ImportRoaringShard(ctx context.Context, index string, shard uint64, request *featurebase.ImportRoaringShardRequest) error {
return f.API.ImportRoaringShard(ctx, index, shard, request)
}
// EncodeImportValues is kind of weird. We're trying to mimic what the client
// does here (because the Importer interface was originally based off of the
// client methods). So we end up generating a protobuf-encode byte slice. And we
// don't really use path.
func (f *FeaturebaseImporter) EncodeImportValues(ctx context.Context, index string, field *featurebase.FieldInfo, shard uint64, vals []int64, ids []uint64, clear bool) (path string, data []byte, err error) {
msg := &pb.ImportValueRequest{
Index: index,
IndexCreatedAt: 0,
Field: field.Name,
FieldCreatedAt: field.CreatedAt,
Shard: shard,
ColumnIDs: ids,
Values: vals,
}
data, err = proto.Marshal(msg)
if err != nil {
return "", nil, errors.Wrap(err, "marshaling ImportValue to protobuf")
}
return "", data, nil
}
func (f *FeaturebaseImporter) EncodeImport(ctx context.Context, index string, field *featurebase.FieldInfo, shard uint64, vals, ids []uint64, clear bool) (path string, data []byte, err error) {
msg := &pb.ImportRequest{
Index: index,
IndexCreatedAt: 0,
Field: field.Name,
FieldCreatedAt: field.CreatedAt,
Shard: shard,
RowIDs: vals,
ColumnIDs: ids,
}
data, err = proto.Marshal(msg)
if err != nil {
return "", nil, errors.Wrap(err, "marshaling Import to protobuf")
}
return "", data, nil
}
func (f *FeaturebaseImporter) DoImport(ctx context.Context, index string, field *featurebase.FieldInfo, shard uint64, path string, data []byte) error {
serializer := featurebaseproto.Serializer{}
// Unmarshal request based on field type.
switch field.Options.Type {
case featurebase.FieldTypeInt, featurebase.FieldTypeDecimal, featurebase.FieldTypeTimestamp:
// Marshal into request object.
req := &featurebase.ImportValueRequest{}
if err := serializer.Unmarshal(data, req); err != nil {
return errors.Wrap(err, "unmarshaling import value request")
}
qcx := f.API.Txf().NewQcx()
defer qcx.Abort()
opts := []featurebase.ImportOption{
featurebase.OptImportOptionsClear(req.Clear),
}
if err := f.API.ImportValue(ctx, qcx, req, opts...); err != nil {
return errors.Wrap(err, "importing import value request")
}
if err := qcx.Finish(); err != nil {
return errors.Wrap(err, "finishing qcx")
}
default:
// Marshal into request object.
req := &featurebase.ImportRequest{}
if err := serializer.Unmarshal(data, req); err != nil {
return errors.Wrap(err, "unmarshaling import request")
}
qcx := f.API.Txf().NewQcx()
defer qcx.Abort()
opts := []featurebase.ImportOption{
featurebase.OptImportOptionsClear(req.Clear),
}
if len(req.RowIDs) > 0 {
opts = append(opts, featurebase.OptImportOptionsIgnoreKeyCheck(true))
}
if err := f.API.Import(ctx, qcx, req, opts...); err != nil {
return errors.Wrap(err, "importing import request")
}
if err := qcx.Finish(); err != nil {
return errors.Wrap(err, "finishing qcx")
}
}
return nil
}
func (f *FeaturebaseImporter) StatsTiming(name string, value time.Duration, rate float64) {}

3
client/metrics.go → batch/metrics.go
View file

@ -1,5 +1,4 @@
// Copyright 2021 Molecula Corp. All rights reserved.
package client
package batch
const (
// MetricBatchImportDurationSeconds records the full time of the

3
batch/testdata/README.md vendored Normal file
View file

@ -0,0 +1,3 @@
# testdata
This directory is used in CI tests. I think.

26
batch/wait.sh Executable file
View file

@ -0,0 +1,26 @@
#!/bin/sh
name=$1
shift
_start_ts=$(date +%s)
elapsed=0
timeout=120
while :
do
$@ > /dev/null
_ret=$?
_end_ts=$(date +%s)
if [ $_ret -eq 0 ]; then
echo "$name is available after $((_end_ts - _start_ts)) seconds."
break
else
echo "Waiting for $name after $((_end_ts - _start_ts)) seconds."
fi
sleep 1s
elapsed=$((elapsed+1))
if [ $elapsed -ge $timeout ]; then
exit 110
fi
done
set -ex

275
client/api.go Normal file
View file

@ -0,0 +1,275 @@
package client
import (
"context"
featurebase "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/errors"
)
var _ featurebase.SchemaAPI = &schemaAPI{}
// schemaAPI is a featurebase client wrapper which implements the
// featurebase.SchemaAPI interface. This was introduced for use in batch tests
// when we decoupled Batch from the client package. In other words, this is only
// used for those tests, it may not be functionally complete, and should not be
// used otherwise without further testing and review of this code.
type schemaAPI struct {
*Client
}
func NewSchemaAPI(c *Client) *schemaAPI {
return &schemaAPI{
Client: c,
}
}
func (s *schemaAPI) CreateIndexAndFields(ctx context.Context, indexName string, options featurebase.IndexOptions, fields []featurebase.CreateFieldObj) error {
schema, err := s.Client.Schema()
if err != nil {
return errors.Wrap(err, "getting schema")
}
// Add the index.
idx := schema.Index(indexName,
OptIndexKeys(options.Keys),
OptIndexTrackExistence(true),
)
if err := s.Client.CreateIndex(idx); err != nil {
return errors.Wrap(err, "creating index")
}
// Now add fields.
for _, f := range fields {
fld, err := s.addFieldToIndex(idx, f.Name, f.Options...)
if err != nil {
return errors.Wrapf(err, "adding field to index")
}
if err := s.Client.CreateField(fld); err != nil {
return errors.Wrapf(err, "creating field")
}
}
return nil
}
func (s *schemaAPI) CreateField(ctx context.Context, indexName string, fieldName string, opts ...featurebase.FieldOption) (*featurebase.Field, error) {
schema, err := s.Client.Schema()
if err != nil {
return nil, errors.Wrap(err, "getting schema")
}
if !schema.HasIndex(indexName) {
return nil, featurebase.ErrIndexNotFound
}
idx := schema.Index(indexName)
fld, err := s.addFieldToIndex(idx, fieldName, opts...)
if err != nil {
return nil, errors.Wrapf(err, "adding field to index")
}
if err := s.Client.CreateField(fld); err != nil {
return nil, errors.Wrapf(err, "creating field")
}
return nil, nil
}
func (s *schemaAPI) addFieldToIndex(idx *Index, fieldName string, opts ...featurebase.FieldOption) (*Field, error) {
ffos := &featurebase.FieldOptions{}
for _, opt := range opts {
opt(ffos)
}
cfos := []FieldOption{}
switch ffos.Type {
case featurebase.FieldTypeBool:
cfos = append(cfos, OptFieldTypeBool())
case featurebase.FieldTypeInt:
cfos = append(cfos, OptFieldTypeInt(ffos.Min.ToInt64(0), ffos.Max.ToInt64(0)))
case featurebase.FieldTypeSet:
cfos = append(cfos,
OptFieldTypeSet(CacheType(ffos.CacheType), int(ffos.CacheSize)),
OptFieldKeys(ffos.Keys),
)
case featurebase.FieldTypeMutex:
cfos = append(cfos,
OptFieldTypeMutex(CacheType(ffos.CacheType), int(ffos.CacheSize)),
OptFieldKeys(ffos.Keys),
)
case featurebase.FieldTypeDecimal:
cfos = append(cfos, OptFieldTypeDecimal(ffos.Scale, ffos.Min, ffos.Max))
case featurebase.FieldTypeTime:
cfos = append(cfos,
OptFieldTypeTime(TimeQuantum(ffos.TimeQuantum), ffos.NoStandardView),
OptFieldKeys(ffos.Keys),
)
case featurebase.FieldTypeTimestamp:
cfos = append(cfos, OptFieldTypeTimestamp(featurebase.DefaultEpoch, ffos.TimeUnit))
default:
return nil, errors.Errorf("unsupported field type: %s", ffos.Type)
}
return idx.Field(fieldName, cfos...), nil
}
func (s *schemaAPI) DeleteField(ctx context.Context, indexName string, fieldName string) error {
schema, err := s.Client.Schema()
if err != nil {
return errors.Wrap(err, "getting schema")
}
if !schema.HasIndex(indexName) {
return featurebase.ErrIndexNotFound
}
idx := schema.Index(indexName)
return s.Client.DeleteField(&Field{
name: fieldName,
index: idx,
})
}
func (s *schemaAPI) DeleteIndex(ctx context.Context, indexName string) error {
return s.Client.DeleteIndexByName(indexName)
}
func (s *schemaAPI) IndexInfo(ctx context.Context, indexName string) (*featurebase.IndexInfo, error) {
schema, err := s.Client.Schema()
if err != nil {
return nil, errors.Wrap(err, "getting schema")
}
if !schema.HasIndex(indexName) {
return nil, featurebase.ErrIndexNotFound
}
idx := schema.Index(indexName)
return FromClientIndex(idx), nil
}
func (s *schemaAPI) Schema(ctx context.Context, withViews bool) ([]*featurebase.IndexInfo, error) {
return nil, errors.New("", "schemaAPI.Schema is not implemented")
}
var _ featurebase.QueryAPI = &queryAPI{}
// queryAPI is a featurebase client wrapper which implements the
// featurebase.QueryAPI interface. This was introduced for use in batch tests
// when we decoupled Batch from the client package. In other words, this is only
// used for those tests, it may not be functionally complete, and should not be
// used otherwise without further testing and review of this code.
type queryAPI struct {
*Client
}
func NewQueryAPI(c *Client) *queryAPI {
return &queryAPI{
Client: c,
}
}
func (q *queryAPI) Query(ctx context.Context, req *featurebase.QueryRequest) (featurebase.QueryResponse, error) {
schema, err := q.Client.Schema()
if err != nil {
return featurebase.QueryResponse{}, errors.Wrap(err, "getting schema")
}
if !schema.HasIndex(req.Index) {
return featurebase.QueryResponse{}, featurebase.ErrIndexNotFound
}
idx := schema.Index(req.Index)
qry := NewPQLBaseQuery(req.Query, idx, nil)
res, err := q.Client.Query(qry)
if err != nil {
return featurebase.QueryResponse{}, errors.Wrap(err, "querying client")
}
var rerr error
if res.ErrorMessage != "" {
rerr = errors.New("", res.ErrorMessage)
}
fbResults := make([]interface{}, 0)
// Row, PairsField, GroupCounts
for _, result := range res.ResultList {
switch result.Type() {
case QueryResultTypeRow:
if len(result.Row().Keys) > 0 {
row := featurebase.NewRow()
row.Keys = result.Row().Keys
fbResults = append(fbResults, row)
} else {
fbResults = append(fbResults, featurebase.NewRow(result.Row().Columns...))
}
case QueryResultTypeUint64:
fbResults = append(fbResults, uint64(result.Count()))
case QueryResultTypeBool:
fbResults = append(fbResults, result.Changed())
case QueryResultTypePairsField:
pairs := []featurebase.Pair{}
for _, ci := range result.CountItems() {
pairs = append(pairs, featurebase.Pair{
ID: ci.ID,
Key: ci.Key,
Count: ci.Count,
})
}
pf := &featurebase.PairsField{
Pairs: pairs,
Field: "",
}
fbResults = append(fbResults, pf)
case QueryResultTypeGroupCounts:
groups := []featurebase.GroupCount{}
for _, grpCnt := range result.GroupCounts() {
fieldRows := []featurebase.FieldRow{}
for _, fr := range grpCnt.Groups {
fieldRows = append(fieldRows, featurebase.FieldRow{
Field: fr.FieldName,
RowID: fr.RowID,
RowKey: fr.RowKey,
Value: fr.Value,
})
}
groups = append(groups, featurebase.GroupCount{
Group: fieldRows,
Count: uint64(grpCnt.Count),
Agg: grpCnt.Agg,
// DecimalAgg: ??,
})
}
gc := featurebase.NewGroupCounts("", groups...)
fbResults = append(fbResults, gc)
case QueryResultTypeValCount:
vc := featurebase.ValCount{
Val: result.Value(),
Count: result.Count(),
}
fbResults = append(fbResults, vc)
default:
return featurebase.QueryResponse{}, errors.Errorf("unsupported query result type: %d", result.Type())
}
}
resp := &featurebase.QueryResponse{
Results: fbResults,
Err: rerr,
}
return *resp, nil
}

1883
client/batch_test.go
View file

File diff suppressed because it is too large Load diff

2
client/client.go
View file

@ -39,8 +39,6 @@ const PQLVersion = "1.0"
// DefaultShardWidth is used if an index doesn't have it defined.
const DefaultShardWidth = pilosa.ShardWidth
const maxHosts = 10
// Client is the HTTP client for Pilosa server.
type Client struct {
cluster *Cluster

219
client/importer.go Normal file
View file

@ -0,0 +1,219 @@
package client
import (
"context"
"time"
featurebase "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/roaring"
"github.com/pkg/errors"
)
//////////////////////////////////////////////////////////////////////////////
//
// The following was introduced upon splitting batch out of the client package
// (and into its own package). During that work, we changed batch from using
// client.Index, and instead using featurebase.IndexInfo. The functions below
// convert client types from/to featurebase types in order to satisfy the batch
// requirements.
//
//////////////////////////////////////////////////////////////////////////////
// FromClientIndex converts a client Index to a featurebase IndexInfo.
func FromClientIndex(ci *Index) *featurebase.IndexInfo {
return &featurebase.IndexInfo{
Name: ci.Name(),
CreatedAt: ci.CreatedAt(),
Options: fromClientIndexOptions(ci.Opts()),
Fields: fromClientFieldsMap(ci.Fields()),
ShardWidth: ci.ShardWidth(),
}
}
// fromClientIndexOptions
func fromClientIndexOptions(cio IndexOptions) featurebase.IndexOptions {
return featurebase.IndexOptions{
Keys: cio.Keys(),
TrackExistence: cio.TrackExistence(),
PartitionN: 0, // TODO(tlt): this shouldn't be 0 once we support it.
}
}
// toClientIndex
func toClientIndex(fi *featurebase.IndexInfo) *Index {
sch := NewSchema()
return sch.Index(fi.Name,
OptIndexKeys(fi.Options.Keys),
OptIndexTrackExistence(fi.Options.TrackExistence),
)
}
// fromClientField
func fromClientField(cf *Field) *featurebase.FieldInfo {
return &featurebase.FieldInfo{
Name: cf.Name(),
CreatedAt: cf.CreatedAt(),
Options: fromClientFieldOptions(cf.Opts()),
// TODO(tlt): do we need Views? Because we can't currently get views
// from client.
// Views: ??
}
}
// fromClientFieldOptions
func fromClientFieldOptions(cfo FieldOptions) featurebase.FieldOptions {
return featurebase.FieldOptions{
Base: cfo.Base(),
BitDepth: 0, // TODO(tlt): set this?
Min: cfo.Min(),
Max: cfo.Max(),
Scale: cfo.Scale(),
Keys: cfo.Keys(),
NoStandardView: cfo.NoStandardView(),
CacheSize: uint32(cfo.CacheSize()),
CacheType: string(cfo.CacheType()),
Type: string(cfo.Type()),
TimeUnit: cfo.TimeUnit(),
TimeQuantum: featurebase.TimeQuantum(cfo.TimeQuantum()),
ForeignIndex: cfo.ForeignIndex(),
TTL: cfo.TTL(),
}
}
// toClientField
func toClientField(index string, ff *featurebase.FieldInfo) (*Field, error) {
sch := NewSchema()
idx := sch.Index(index)
opts := []FieldOption{}
switch ff.Options.Type {
case featurebase.FieldTypeBool:
opts = append(opts,
OptFieldTypeBool(),
)
case featurebase.FieldTypeDecimal:
opts = append(opts,
OptFieldTypeDecimal(ff.Options.Scale, ff.Options.Min, ff.Options.Max),
)
case featurebase.FieldTypeMutex:
opts = append(opts,
OptFieldTypeMutex(CacheType(ff.Options.CacheType), int(ff.Options.CacheSize)),
OptFieldKeys(ff.Options.Keys),
)
case featurebase.FieldTypeSet:
opts = append(opts,
OptFieldTypeSet(CacheType(ff.Options.CacheType), int(ff.Options.CacheSize)),
OptFieldKeys(ff.Options.Keys),
)
case featurebase.FieldTypeInt:
opts = append(opts,
OptFieldTypeInt(ff.Options.Min.ToInt64(0), ff.Options.Max.ToInt64(0)),
)
case featurebase.FieldTypeTimestamp:
epoch, err := featurebase.ValToTimestamp(ff.Options.TimeUnit, ff.Options.Base)
if err != nil {
return nil, errors.Wrapf(err, "calculating epoch: %s, %d", ff.Options.TimeUnit, ff.Options.Base)
}
opts = append(opts,
OptFieldTypeTimestamp(epoch, ff.Options.TimeUnit),
)
}
return idx.Field(ff.Name, opts...), nil
}
// FromClientFields converts a slice of client Fields to a slice of featurebase
// FieldInfo.
func FromClientFields(cf []*Field) []*featurebase.FieldInfo {
ff := make([]*featurebase.FieldInfo, len(cf))
for i := range cf {
ff[i] = fromClientField(cf[i])
}
return ff
}
// fromClientFieldsMap
func fromClientFieldsMap(cf map[string]*Field) []*featurebase.FieldInfo {
ff := make([]*featurebase.FieldInfo, 0, len(cf))
for _, v := range cf {
ff = append(ff, fromClientField(v))
}
return ff
}
// We can't import the batch package into client because it results in an import
// loop. That's probably an indication that this interface implementation should
// be moved somewhere else; for example, into a sub-package of the batch package
// (since it's an implementation of one of batch's interfaces).
// var _ batch.Importer = &importer{}
// importer is a pilosa client which implements the batch.Importer interface.
// This wrapper is necessary because of the call into client.Stats.Timing(), and
// because the client takes client specific types (like Index and Field), but
// the interface takes FeatureBase specific types.
type importer struct {
*Client
}
func NewImporter(c *Client) *importer {
return &importer{
Client: c,
}
}
func (i *importer) StartTransaction(ctx context.Context, id string, timeout time.Duration, exclusive bool, requestTimeout time.Duration) (*featurebase.Transaction, error) {
return i.Client.StartTransaction(id, timeout, exclusive, requestTimeout)
}
func (i *importer) FinishTransaction(ctx context.Context, id string) (*featurebase.Transaction, error) {
return i.Client.FinishTransaction(id)
}
func (i *importer) CreateIndexKeys(ctx context.Context, idx *featurebase.IndexInfo, keys ...string) (map[string]uint64, error) {
return i.Client.CreateIndexKeys(toClientIndex(idx), keys...)
}
func (i *importer) CreateFieldKeys(ctx context.Context, index string, field *featurebase.FieldInfo, keys ...string) (map[string]uint64, error) {
fld, err := toClientField(index, field)
if err != nil {
return nil, errors.Wrap(err, "converting to client field")
}
return i.Client.CreateFieldKeys(fld, keys...)
}
func (i *importer) ImportRoaringBitmap(ctx context.Context, index string, field *featurebase.FieldInfo, shard uint64, views map[string]*roaring.Bitmap, clear bool) error {
fld, err := toClientField(index, field)
if err != nil {
return errors.Wrap(err, "converting to client field")
}
return i.Client.ImportRoaringBitmap(fld, shard, views, clear)
}
func (i *importer) ImportRoaringShard(ctx context.Context, index string, shard uint64, request *featurebase.ImportRoaringShardRequest) error {
return i.Client.ImportRoaringShard(index, shard, request)
}
func (i *importer) EncodeImportValues(ctx context.Context, index string, field *featurebase.FieldInfo, shard uint64, vals []int64, ids []uint64, clear bool) (path string, data []byte, err error) {
fld, err := toClientField(index, field)
if err != nil {
return "", nil, errors.Wrap(err, "converting to client field")
}
return i.Client.EncodeImportValues(fld, shard, vals, ids, clear)
}
func (i *importer) EncodeImport(ctx context.Context, index string, field *featurebase.FieldInfo, shard uint64, vals, ids []uint64, clear bool) (path string, data []byte, err error) {
fld, err := toClientField(index, field)
if err != nil {
return "", nil, errors.Wrap(err, "converting to client field")
}
return i.Client.EncodeImport(fld, shard, vals, ids, clear)
}
func (i *importer) DoImport(ctx context.Context, index string, field *featurebase.FieldInfo, shard uint64, path string, data []byte) error {
return i.Client.DoImport(index, shard, path, data)
}
func (i *importer) StatsTiming(name string, value time.Duration, rate float64) {
i.Client.Stats.Timing(name, value, rate)
}

145
client/ingest_api_batch.go
View file

@ -1,145 +0,0 @@
package client
import (
"time"
"github.com/molecula/featurebase/v3/logger"
"github.com/pkg/errors"
)
// NewIngestAPIBatch creates an alternate implementation of
// RecordBatch which exists to aid in testing the new Ingest API and
// is likely far slower than the Batch.
func NewIngestAPIBatch(client *Client, size int, logger logger.Logger, fields []*Field) *ingestAPIBatch {
if len(fields) == 0 {
return nil
}
return &ingestAPIBatch{
client: client,
log: logger,
fields: fields,
keyed: fields[0].index.Opts().Keys(),
index: fields[0].index.Name(),
batchSize: size,
recordsK: make(map[string]map[string]interface{}),
records: make(map[uint64]map[string]interface{}),
}
}
type ingestAPIBatch struct {
client *Client
log logger.Logger
batchSize int
fields []*Field
keyed bool
index string
// map[recordKey][fieldName]value
recordsK map[string]map[string]interface{}
records map[uint64]map[string]interface{}
}
func (b *ingestAPIBatch) Add(row Row) error {
if len(row.Clears) > 0 {
return errors.New("ingest api batch does not support clears")
}
values := make(map[string]interface{})
for i, val := range row.Values {
field := b.fields[i]
// val can be string, uint64, int64, []string, []uint64, nil
// TODO timestamp field might need special handling
// TODO check that the Row.Clears field is only used for packed bools, and then issue a warning/error (in IDK) if the ingest API mode is used in conjunction w/ packed bools.
if val == nil {
continue
}
zero := QuantizedTime{}
if field.Options().Type() == FieldTypeTime && row.Time != zero {
timeq, err := row.Time.Time()
if err != nil {
return errors.Wrap(err, "parsing row time")
}
values[field.Name()] = map[string]interface{}{"time": timeq.Format(time.RFC3339), "values": val}
} else {
values[field.Name()] = val
}
}
if b.keyed {
switch rowID := row.ID.(type) {
case string:
b.recordsK[rowID] = values
case []byte:
b.recordsK[string(rowID)] = values
default:
return errors.Errorf("unsupported rowID %v of type %[1]T, must be string, or []byte for keyed index", rowID)
}
if len(b.recordsK) >= b.batchSize {
return ErrBatchNowFull
}
} else {
rowID, ok := row.ID.(uint64)
if !ok {
return errors.Errorf("unsupported rowID %v of type %[1]T, must be uint64 for unkeyed index", row.ID)
}
b.records[rowID] = values
if len(b.records) >= b.batchSize {
return ErrBatchNowFull
}
}
return nil
}
func (b *ingestAPIBatch) Import() error {
if b.keyed {
return b.importKeyed()
}
return b.importUnkeyed()
}
func (b *ingestAPIBatch) importKeyed() error {
req := []map[string]interface{}{
{
"action": "set",
"records": b.recordsK,
},
}
bod, err := b.client.IngestData(b.index, req)
if err != nil {
return errors.Wrapf(err, "importKeyed, body: %s", bod)
}
for k := range b.recordsK {
delete(b.recordsK, k)
}
return nil
}
func (b *ingestAPIBatch) importUnkeyed() error {
req := []map[string]interface{}{
{
"action": "set",
"records": b.records,
},
}
bod, err := b.client.IngestData(b.index, req)
if err != nil {
return errors.Wrapf(err, "importKeyed, body: %s", bod)
}
for v := range b.records {
delete(b.records, v)
}
return nil
}
func (b *ingestAPIBatch) Len() int {
if b.keyed {
return len(b.recordsK)
}
return len(b.records)
}
func (b *ingestAPIBatch) Flush() error { return nil }

305
client/ingest_api_batch_test.go
View file

@ -1,305 +0,0 @@
package client
import (
"strings"
"testing"
"time"
"github.com/molecula/featurebase/v3/logger"
"github.com/molecula/featurebase/v3/test"
)
func TestIngestAPIBatchAdd(t *testing.T) {
t.Run("unkeyed", func(t *testing.T) {
batch := NewIngestAPIBatch(nil, 10, logger.NopLogger, []*Field{
{
name: "a",
index: &Index{name: "idxname", options: &IndexOptions{}},
options: &FieldOptions{
fieldType: FieldTypeSet,
},
},
{
name: "b",
index: &Index{name: "idxname", options: &IndexOptions{}},
options: &FieldOptions{
fieldType: FieldTypeSet,
keys: true,
},
},
{
name: "c",
index: &Index{name: "idxname", options: &IndexOptions{}},
options: &FieldOptions{
fieldType: FieldTypeTime,
keys: true,
},
},
})
qt := QuantizedTime{}
qt.Set(time.Date(2007, time.January, 1, 15, 0, 0, 0, time.UTC))
err := batch.Add(Row{
ID: uint64(1),
Values: []interface{}{uint64(2), "bkey", "ckey"},
Time: qt,
})
if err != nil {
t.Fatalf("adding row to batch: %v", err)
}
if batch.records[1]["a"] != uint64(2) {
t.Fatalf("unexpected batch.records: %+v", batch.records)
}
if batch.records[1]["b"] != "bkey" {
t.Fatalf("unexpected batch.records: %+v", batch.records)
}
if batch.records[1]["c"].(map[string]interface{})["time"] != "2007-01-01T15:00:00Z" {
t.Fatalf("unexpected batch.records: %+v", batch.records)
}
if batch.records[1]["c"].(map[string]interface{})["values"] != "ckey" {
t.Fatalf("unexpected batch.records: %+v", batch.records)
}
})
t.Run("keyed", func(t *testing.T) {
batch := NewIngestAPIBatch(nil, 10, logger.NopLogger, []*Field{
{
name: "a",
index: &Index{name: "idxname", options: &IndexOptions{keys: true}},
options: &FieldOptions{
fieldType: FieldTypeSet,
},
},
{
name: "b",
index: &Index{name: "idxname", options: &IndexOptions{keys: true}},
options: &FieldOptions{
fieldType: FieldTypeSet,
keys: true,
},
},
{
name: "c",
index: &Index{name: "idxname", options: &IndexOptions{keys: true}},
options: &FieldOptions{
fieldType: FieldTypeTime,
keys: true,
},
},
})
qt := QuantizedTime{}
qt.Set(time.Date(2007, time.January, 1, 15, 0, 0, 0, time.UTC))
err := batch.Add(Row{
ID: "1",
Values: []interface{}{uint64(2), "bkey", "ckey"},
Time: qt,
})
checkResult := func(batch *ingestAPIBatch, id string, err error) {
if err != nil {
t.Fatalf("adding row to batch: %v", err)
}
if batch.recordsK[id]["a"] != uint64(2) {
t.Fatalf("unexpected batch.records: %+v", batch.recordsK)
}
if batch.recordsK[id]["b"] != "bkey" {
t.Fatalf("unexpected batch.records: %+v", batch.recordsK)
}
if batch.recordsK[id]["c"].(map[string]interface{})["time"] != "2007-01-01T15:00:00Z" {
t.Fatalf("unexpected batch.records: %+v", batch.recordsK)
}
if batch.recordsK[id]["c"].(map[string]interface{})["values"] != "ckey" {
t.Fatalf("unexpected batch.records: %+v", batch.recordsK)
}
}
checkResult(batch, "1", err)
// test wrong type row ID
if err := batch.Add(Row{ID: 64.5}); !strings.Contains(err.Error(), "unsupported rowID") {
t.Fatalf("unexpected error w/ floating point rowID: %v", err)
}
// test that byte slice ID works same as string
err = batch.Add(Row{
ID: []byte("2"),
Values: []interface{}{uint64(2), "bkey", "ckey"},
Time: qt,
})
checkResult(batch, "2", err)
})
}
func TestIngestAPIBatch(t *testing.T) {
c := test.MustRunCluster(t, 3)
defer c.Close()
urls := make([]string, len(c.Nodes))
for i, n := range c.Nodes {
urls[i] = n.URL()
}
// Create a new client for the cluster
cli, err := newClientFromAddresses(urls, &ClientOptions{})
if err != nil {
t.Fatalf("getting new client: %v", err)
}
defer cli.Close()
cli.IngestSchema(map[string]interface{}{
"index-name": "test-1",
"index-action": "create",
"primary-key-type": "uint",
"field-action": "create",
"fields": []map[string]interface{}{
{
"field-name": "astr",
"field-type": "string",
"field-options": map[string]interface{}{},
},
{
"field-name": "bint",
"field-type": "int",
"field-options": map[string]interface{}{},
},
{
"field-name": "cid",
"field-type": "id",
"field-options": map[string]interface{}{},
},
{
"field-name": "dtimestamp",
"field-type": "timestamp",
"field-options": map[string]interface{}{
"unit": "s",
},
},
{
"field-name": "etime",
"field-type": "string",
"field-options": map[string]interface{}{
"time-quantum": "YMD",
},
},
{
"field-name": "fdecimal",
"field-type": "decimal",
"field-options": map[string]interface{}{
"scale": 3,
},
},
{
"field-name": "gbool",
"field-type": "bool",
"field-options": map[string]interface{}{},
},
},
})
schema, err := cli.Schema()
if err != nil {
t.Fatalf("getting schema: %v", err)
}
index := schema.Index("test-1")
defer cli.DeleteIndex(index)
batch := NewIngestAPIBatch(cli, 10, logger.NopLogger, []*Field{
{
name: "astr",
index: &Index{name: "test-1", options: &IndexOptions{}},
options: &FieldOptions{fieldType: FieldTypeSet, keys: true},
},
{
name: "bint",
options: &FieldOptions{fieldType: FieldTypeInt},
},
{
name: "cid",
options: &FieldOptions{fieldType: FieldTypeSet, keys: false},
},
{
name: "dtimestamp",
options: &FieldOptions{fieldType: FieldTypeTimestamp},
},
{
name: "etime",
options: &FieldOptions{fieldType: FieldTypeTime, keys: true, timeQuantum: TimeQuantumYearMonthDay},
},
{
name: "fdecimal",
options: &FieldOptions{fieldType: FieldTypeDecimal, scale: 3},
},
{
name: "gbool",
options: &FieldOptions{fieldType: FieldTypeBool},
},
})
qt0 := &QuantizedTime{}
qt0.Set(time.Date(2010, time.January, 1, 0, 0, 0, 0, time.UTC))
if err := batch.Add(Row{
ID: uint64(7),
Values: []interface{}{"a", -2, 9, 1287367623, "e", 1.2345, true},
Time: *qt0,
}); err != nil {
t.Fatalf("adding row: %v", err)
}
// test nil value case
if err := batch.Add(Row{
ID: uint64(8),
Values: []interface{}{nil, nil, nil, nil, nil, nil, nil},
Time: QuantizedTime{},
}); err != nil {
t.Fatalf("error adding all nil batch which should affect nothing: %v", err)
}
if err := batch.Import(); err != nil {
t.Fatalf("importing row: %v", err)
}
if resp, err := cli.Query(NewPQLBaseQuery("Row(astr=a)", &Index{name: "test-1", options: &IndexOptions{}}, nil)); err != nil {
t.Fatalf("querying: %v", err)
} else if len(resp.Result().Row().Columns) != 1 || resp.Result().Row().Columns[0] != uint64(7) {
t.Fatalf("unexpected Row(asr=a) result: %+v", resp.Result().Row().Columns)
}
if resp, err := cli.Query(NewPQLBaseQuery("Row(bint==-2)", &Index{name: "test-1", options: &IndexOptions{}}, nil)); err != nil {
t.Fatalf("querying: %v", err)
} else if len(resp.Result().Row().Columns) != 1 || resp.Result().Row().Columns[0] != uint64(7) {
t.Fatalf("unexpected Row(bint==-2) result: %+v", resp.Result().Row().Columns)
}
if resp, err := cli.Query(NewPQLBaseQuery("Row(cid=9)", &Index{name: "test-1", options: &IndexOptions{}}, nil)); err != nil {
t.Fatalf("querying: %v", err)
} else if len(resp.Result().Row().Columns) != 1 || resp.Result().Row().Columns[0] != uint64(7) {
t.Fatalf("unexpected Row(cid=9) result: %+v", resp.Result().Row().Columns)
}
if resp, err := cli.Query(NewPQLBaseQuery("Row(dtimestamp=='2010-10-18T02:07:03Z')", &Index{name: "test-1", options: &IndexOptions{}}, nil)); err != nil {
t.Fatalf("querying: %v", err)
} else if len(resp.Result().Row().Columns) != 1 || resp.Result().Row().Columns[0] != uint64(7) {
t.Fatalf("unexpected Row(dtimestamp=='2010-10-18T02:07:03Z') result: %+v", resp.Result().Row().Columns)
}
if resp, err := cli.Query(NewPQLBaseQuery("Row(etime=e, from='2010-01-01', to='2010-01-02')", &Index{name: "test-1", options: &IndexOptions{}}, nil)); err != nil {
t.Fatalf("querying: %v", err)
} else if len(resp.Result().Row().Columns) != 1 || resp.Result().Row().Columns[0] != uint64(7) {
t.Fatalf("unexpected Row(etime=e, from='2010-01-01', to='2010-01-02') result: %+v", resp.Result().Row().Columns)
}
if resp, err := cli.Query(NewPQLBaseQuery("Row(fdecimal==1.234)", &Index{name: "test-1", options: &IndexOptions{}}, nil)); err != nil {
t.Fatalf("querying: %v", err)
} else if len(resp.Result().Row().Columns) != 1 || resp.Result().Row().Columns[0] != uint64(7) {
t.Fatalf("unexpected Row(fdecimal==1.234) result: %+v", resp.Result().Row().Columns)
}
if resp, err := cli.Query(NewPQLBaseQuery("Row(gbool=true)", &Index{name: "test-1", options: &IndexOptions{}}, nil)); err != nil {
t.Fatalf("querying: %v", err)
} else if len(resp.Result().Row().Columns) != 1 || resp.Result().Row().Columns[0] != uint64(7) {
t.Fatalf("unexpected Row(gbool=true) result: %+v", resp.Result().Row().Columns)
}
}

12
client/orm.go
View file

@ -208,10 +208,10 @@ func (q PQLRowQuery) Error() error {
//
// Usage:
//
// repo, err := NewIndex("repository")
// stargazer, err := repo.Field("stargazer")
// query := repo.BatchQuery(
// stargazer.Row(5),
// repo, err := NewIndex("repository")
// stargazer, err := repo.Field("stargazer")
// query := repo.BatchQuery(
// stargazer.Row(5),
// stargazer.Row(15),
// repo.Union(stargazer.Row(20), stargazer.Row(25)))
type PQLBatchQuery struct {
@ -797,6 +797,10 @@ func (fo FieldOptions) TimeUnit() string {
return fo.timeUnit
}
func (fo FieldOptions) Base() int64 {
return fo.base
}
// NoStandardView suppresses creating the standard view for supported field types (currently, time)
func (fo FieldOptions) NoStandardView() bool {
return fo.noStandardView

2
idk/datagen/cmd.go
View file

@ -51,7 +51,6 @@ type Main struct {
Seed int64 `short:"" help:"Seed to use for any random number generation."`
TrackProgress bool `short:"" help:"Periodically print status updates on how many records have been sourced."`
UseIngestAPI bool `help:"Experimental: use new HTTP/JSON ingest API instead of low-level import API. Probably slow, does not support packed bools."`
UseShardTransactionalEndpoint bool `flag:"use-shard-transactional-endpoint" help:"Use experimental transactional endpoint"`
@ -281,7 +280,6 @@ func (m *Main) Preload() error {
m.idkMain.CacheLength = m.Pilosa.CacheLength
m.idkMain.NewSource = m.newSource
m.idkMain.TrackProgress = m.TrackProgress
m.idkMain.UseIngestAPI = m.UseIngestAPI
m.idkMain.AuthToken = m.AuthToken
m.idkMain.UseShardTransactionalEndpoint = m.UseShardTransactionalEndpoint
if len(m.Pilosa.Hosts) > 0 {

75
idk/ingest.go
View file

@ -26,6 +26,7 @@ import (
"github.com/felixge/fgprof"
pilosacore "github.com/molecula/featurebase/v3"
pilosagrpc "github.com/molecula/featurebase/v3/api/client"
pilosabatch "github.com/molecula/featurebase/v3/batch"
pilosaclient "github.com/molecula/featurebase/v3/client"
"github.com/molecula/featurebase/v3/logger"
"github.com/molecula/featurebase/v3/pql"
@ -86,7 +87,6 @@ type Main struct {
OffsetMode bool `short:"" help:"Set offset-mode based Autogenerated IDs, for use with a data-source that is offset-based (must be set alongside auto-generate and external-generate)."`
LookupDBDSN string `flag:"lookup-db-dsn" help:"Connection string for connecting to Lookup database."`
LookupBatchSize int `help:"Number of records to batch before writing them to Lookup database."`
UseIngestAPI bool `help:"Experimental: use new HTTP/JSON ingest API instead of low-level import API. Probably slow. Does not support packed bools."`
AuthToken string `flag:"auth-token" help:"Authentication Token for FeatureBase"`
CommitTimeout time.Duration `help:"Maximum time before canceling commit."`
AllowIntOutOfRange bool `help:"Allow ingest to continue when it encounters out of range integers in IntFields. (default false)"`
@ -322,10 +322,10 @@ func (m *Main) ingest(ctx context.Context, source Source, nexter IDAllocator, so
defer func() {
m.log.Printf("metrics: import=%s\n", time.Duration(atomic.LoadInt64((*int64)(&m.importDuration))))
}()
var batch pilosaclient.RecordBatch
var batch pilosabatch.RecordBatch
var recordizers []Recordizer
var prevRec Record
var row *pilosaclient.Row
var row *pilosabatch.Row
var errorCounter int // keeps track of consecuitive errors across records
var anyRecordSuccessful bool
if m.progress != nil {
@ -554,7 +554,7 @@ initialFetch:
m.stats.Count(MetricIngesterRowsAdded, 1, 1)
}
if err == pilosaclient.ErrBatchNowFull || err == pilosaclient.ErrBatchNowStale {
if err == pilosabatch.ErrBatchNowFull || err == pilosabatch.ErrBatchNowStale {
batchLen := batch.Len()
err = m.importBatch(batch)
if err != nil {
@ -763,7 +763,7 @@ func (m *Main) Setup() (onFinishRun func(), err error) {
if m.AutoGenerate {
shardWidth := m.index.ShardWidth()
if shardWidth == 0 {
shardWidth = pilosaclient.DefaultShardWidth
shardWidth = pilosacore.ShardWidth
}
m.ra = NewLocalRangeAllocator(shardWidth)
}
@ -1060,7 +1060,7 @@ func (m *Main) runDeleter(c int, limitCounter *msgCounter) error {
}
var recordizers []Recordizer
var row *pilosaclient.Row
var row *pilosabatch.Row
rec, err := source.Record()
if err == nil {
err = ErrSchemaChange // always need to fetch the schema the first time
@ -1328,7 +1328,7 @@ func (m *Main) findPrimary() (*url.URL, error) {
}
// importBatch executes batch.Import() and saves its timing.
func (m *Main) importBatch(batch pilosaclient.RecordBatch) error {
func (m *Main) importBatch(batch pilosabatch.RecordBatch) error {
t := time.Now()
err := batch.Import()
elapsed := time.Since(t)
@ -1336,9 +1336,9 @@ func (m *Main) importBatch(batch pilosaclient.RecordBatch) error {
return err
}
type Recordizer func(rawRec []interface{}, rec *pilosaclient.Row) error
type Recordizer func(rawRec []interface{}, rec *pilosabatch.Row) error
func (m *Main) batchFromSchema(schema []Field) ([]Recordizer, pilosaclient.RecordBatch, *pilosaclient.Row, []int, error) {
func (m *Main) batchFromSchema(schema []Field) ([]Recordizer, pilosabatch.RecordBatch, *pilosabatch.Row, []int, error) {
// Before attempting to do anything, check for duplicates in the schema.
{
dedup := make(map[string]struct{})
@ -1355,10 +1355,10 @@ func (m *Main) batchFromSchema(schema []Field) ([]Recordizer, pilosaclient.Recor
}
// From the schema, and the configuration stored on Main, we need
// to create a []pilosaclient.Field and a []Recordizer processing
// to create a []pilosacore.Field and a []Recordizer processing
// functions which take a []interface{} which conforms to the
// schema, and converts it to a record which conforms to the
// []pilosaclient.Field.
// []pilosacore.Field.
//
// The relevant config options on Main are:
// 1. PrimaryKeyFields, IDField, AutoGenerate
@ -1406,7 +1406,7 @@ func (m *Main) batchFromSchema(schema []Field) ([]Recordizer, pilosaclient.Recor
}
}
fieldIndex := fieldIndex
rz = func(rawRec []interface{}, rec *pilosaclient.Row) (err error) {
rz = func(rawRec []interface{}, rec *pilosabatch.Row) (err error) {
id, err := field.PilosafyVal(rawRec[fieldIndex])
if err != nil {
return errors.Wrapf(err, "converting %+v to ID", rawRec[fieldIndex])
@ -1503,7 +1503,7 @@ func (m *Main) batchFromSchema(schema []Field) ([]Recordizer, pilosaclient.Recor
// TODO may need to have more sophisticated recordizer by type at some point
switch idkField.(type) {
case RecordTimeField:
recordizers = append(recordizers, func(rawRec []interface{}, rec *pilosaclient.Row) (err error) {
recordizers = append(recordizers, func(rawRec []interface{}, rec *pilosabatch.Row) (err error) {
tyme, err := idkField.PilosafyVal(rawRec[i])
if err != nil {
@ -1514,7 +1514,7 @@ func (m *Main) batchFromSchema(schema []Field) ([]Recordizer, pilosaclient.Recor
return nil
})
case IntField, DecimalField, TimestampField:
recordizers = append(recordizers, func(rawRec []interface{}, rec *pilosaclient.Row) (err error) {
recordizers = append(recordizers, func(rawRec []interface{}, rec *pilosabatch.Row) (err error) {
switch rawRec[i].(type) {
case DeleteSentinel:
rec.Clears[valIdx] = uint64(0)
@ -1525,7 +1525,7 @@ func (m *Main) batchFromSchema(schema []Field) ([]Recordizer, pilosaclient.Recor
})
case IDField, StringField:
hasMutex := HasMutex(idkField)
recordizers = append(recordizers, func(rawRec []interface{}, rec *pilosaclient.Row) (err error) {
recordizers = append(recordizers, func(rawRec []interface{}, rec *pilosabatch.Row) (err error) {
switch rawRec[i].(type) {
case DeleteSentinel:
if hasMutex { //need to clear the mutex
@ -1540,7 +1540,7 @@ func (m *Main) batchFromSchema(schema []Field) ([]Recordizer, pilosaclient.Recor
return errors.Wrapf(err, "converting field %d:%+v, val:%+v", i, idkField, rawRec[i])
})
case BoolField:
recordizers = append(recordizers, func(rawRec []interface{}, rec *pilosaclient.Row) (err error) {
recordizers = append(recordizers, func(rawRec []interface{}, rec *pilosabatch.Row) (err error) {
switch rawRec[i].(type) {
case DeleteSentinel:
rec.Values[valIdx] = nil
@ -1550,7 +1550,7 @@ func (m *Main) batchFromSchema(schema []Field) ([]Recordizer, pilosaclient.Recor
return errors.Wrapf(err, "converting field %d:%+v, val:%+v", i, idkField, rawRec[i])
})
default:
recordizers = append(recordizers, func(rawRec []interface{}, rec *pilosaclient.Row) (err error) {
recordizers = append(recordizers, func(rawRec []interface{}, rec *pilosabatch.Row) (err error) {
rec.Values[valIdx], err = idkField.PilosafyVal(rawRec[i])
return errors.Wrapf(err, "converting field %d:%+v, val:%+v", i, idkField, rawRec[i])
})
@ -1561,7 +1561,7 @@ func (m *Main) batchFromSchema(schema []Field) ([]Recordizer, pilosaclient.Recor
// now handle this field if it was not already found in pilosa
switch fld := idkField.(type) {
case RecordTimeField:
recordizers = append(recordizers, func(rawRec []interface{}, rec *pilosaclient.Row) (err error) {
recordizers = append(recordizers, func(rawRec []interface{}, rec *pilosabatch.Row) (err error) {
tyme, err := idkField.PilosafyVal(rawRec[i])
if err != nil {
return errors.Wrap(err, "converting recordtimefield")
@ -1595,7 +1595,7 @@ func (m *Main) batchFromSchema(schema []Field) ([]Recordizer, pilosaclient.Recor
fields = append(fields, m.index.Field(fld.DestName(), opts...))
valIdx := len(fields) - 1
recordizers = append(recordizers, func(rawRec []interface{}, rec *pilosaclient.Row) (err error) {
recordizers = append(recordizers, func(rawRec []interface{}, rec *pilosabatch.Row) (err error) {
switch rawRec[i].(type) {
case DeleteSentinel:
if hasMutex { //need to clear the mutex
@ -1614,7 +1614,7 @@ func (m *Main) batchFromSchema(schema []Field) ([]Recordizer, pilosaclient.Recor
if m.PackBools == "" {
fields = append(fields, m.index.Field(fld.DestName(), pilosaclient.OptFieldTypeBool()))
valIdx := len(fields) - 1
recordizers = append(recordizers, func(rawRec []interface{}, rec *pilosaclient.Row) (err error) {
recordizers = append(recordizers, func(rawRec []interface{}, rec *pilosabatch.Row) (err error) {
switch rawRec[i].(type) {
case DeleteSentinel:
rec.Values[valIdx] = nil
@ -1632,7 +1632,7 @@ func (m *Main) batchFromSchema(schema []Field) ([]Recordizer, pilosaclient.Recor
} else {
fields = append(fields, boolField, boolFieldExists)
fieldIdx := len(fields) - 2
recordizers = append(recordizers, func(rawRec []interface{}, rec *pilosaclient.Row) (err error) {
recordizers = append(recordizers, func(rawRec []interface{}, rec *pilosabatch.Row) (err error) {
switch rawRec[i].(type) {
case DeleteSentinel:
rec.Clears[fieldIdx] = idkField.DestName() // clear bools bit for this field name
@ -1672,7 +1672,7 @@ func (m *Main) batchFromSchema(schema []Field) ([]Recordizer, pilosaclient.Recor
}
fields = append(fields, m.index.Field(fld.DestName(), opts...))
valIdx := len(fields) - 1
recordizers = append(recordizers, func(rawRec []interface{}, rec *pilosaclient.Row) (err error) {
recordizers = append(recordizers, func(rawRec []interface{}, rec *pilosabatch.Row) (err error) {
switch rawRec[i].(type) {
case DeleteSentinel:
rec.Clears[valIdx] = uint64(0)
@ -1684,7 +1684,7 @@ func (m *Main) batchFromSchema(schema []Field) ([]Recordizer, pilosaclient.Recor
case DecimalField:
fields = append(fields, m.index.Field(fld.DestName(), pilosaclient.OptFieldTypeDecimal(fld.Scale)))
valIdx := len(fields) - 1
recordizers = append(recordizers, func(rawRec []interface{}, rec *pilosaclient.Row) (err error) {
recordizers = append(recordizers, func(rawRec []interface{}, rec *pilosabatch.Row) (err error) {
switch rawRec[i].(type) {
case DeleteSentinel:
rec.Clears[valIdx] = uint64(0)
@ -1696,7 +1696,7 @@ func (m *Main) batchFromSchema(schema []Field) ([]Recordizer, pilosaclient.Recor
case TimestampField:
fields = append(fields, m.index.Field(fld.DestName(), pilosaclient.OptFieldTypeTimestamp(fld.epoch(), string(fld.granularity()))))
valIdx := len(fields) - 1
recordizers = append(recordizers, func(rawRec []interface{}, rec *pilosaclient.Row) (err error) {
recordizers = append(recordizers, func(rawRec []interface{}, rec *pilosabatch.Row) (err error) {
switch rawRec[i].(type) {
case DeleteSentinel:
rec.Clears[valIdx] = uint64(0)
@ -1708,7 +1708,7 @@ func (m *Main) batchFromSchema(schema []Field) ([]Recordizer, pilosaclient.Recor
case DateIntField:
fields = append(fields, m.index.Field(fld.DestName(), pilosaclient.OptFieldTypeInt()))
valIdx := len(fields) - 1
recordizers = append(recordizers, func(rawRec []interface{}, rec *pilosaclient.Row) (err error) {
recordizers = append(recordizers, func(rawRec []interface{}, rec *pilosabatch.Row) (err error) {
rec.Values[valIdx], err = idkField.PilosafyVal(rawRec[i])
return errors.Wrapf(err, "converting field %d:%+v, val:%+v", i, idkField, rawRec[i])
})
@ -1719,7 +1719,7 @@ func (m *Main) batchFromSchema(schema []Field) ([]Recordizer, pilosaclient.Recor
m.index.Field(name+Exists, pilosaclient.OptFieldTypeSet(pilosaclient.CacheTypeRanked, pilosacore.DefaultCacheSize)),
)
valIdx := len(fields) - 2
recordizers = append(recordizers, func(rawRec []interface{}, rec *pilosaclient.Row) (err error) {
recordizers = append(recordizers, func(rawRec []interface{}, rec *pilosabatch.Row) (err error) {
val, err := idkField.PilosafyVal(rawRec[i])
if val == nil && err == nil {
rec.Values[valIdx] = nil
@ -1769,24 +1769,21 @@ func (m *Main) batchFromSchema(schema []Field) ([]Recordizer, pilosaclient.Recor
if err != nil {
return nil, nil, nil, nil, errors.Wrap(err, "creating batch")
}
row := &pilosaclient.Row{
row := &pilosabatch.Row{
Values: make([]interface{}, len(fields)),
Clears: make(map[int]interface{}),
}
return recordizers, batch, row, lookupWriteIdxs, nil
}
func (m *Main) newBatch(fields []*pilosaclient.Field) (pilosaclient.RecordBatch, error) {
if m.UseIngestAPI {
return pilosaclient.NewIngestAPIBatch(m.client, m.BatchSize, m.log, fields), nil
}
return pilosaclient.NewBatch(m.client, m.BatchSize, m.index, fields,
pilosaclient.OptLogger(m.log),
pilosaclient.OptCacheMaxAge(m.CacheLength),
pilosaclient.OptSplitBatchMode(m.ExpSplitBatchMode),
pilosaclient.OptMaxStaleness(m.BatchMaxStaleness),
pilosaclient.OptKeyTranslateBatchSize(m.KeyTranslateBatchSize),
pilosaclient.OptUseShardTransactionalEndpoint(m.UseShardTransactionalEndpoint),
func (m *Main) newBatch(fields []*pilosaclient.Field) (pilosabatch.RecordBatch, error) {
return pilosabatch.NewBatch(pilosaclient.NewImporter(m.client), m.BatchSize, pilosaclient.FromClientIndex(m.index), pilosaclient.FromClientFields(fields),
pilosabatch.OptLogger(m.log),
pilosabatch.OptCacheMaxAge(m.CacheLength),
pilosabatch.OptSplitBatchMode(m.ExpSplitBatchMode),
pilosabatch.OptMaxStaleness(m.BatchMaxStaleness),
pilosabatch.OptKeyTranslateBatchSize(m.KeyTranslateBatchSize),
pilosabatch.OptUseShardTransactionalEndpoint(m.UseShardTransactionalEndpoint),
)
}
@ -2023,7 +2020,7 @@ func getPrimaryKeyRecordizer(schema []Field, pkFields []string) (recordizer Reco
skipFields[fieldIndices[0]] = struct{}{}
}
}
recordizer = func(rawRec []interface{}, rec *pilosaclient.Row) (err error) {
recordizer = func(rawRec []interface{}, rec *pilosabatch.Row) (err error) {
// first, special case for performance when there is a single
// primary key field and it is a byte slice already.
if len(fieldIndices) == 1 {

15
idk/ingest_test.go
View file

@ -16,6 +16,7 @@ import (
"github.com/golang-jwt/jwt"
"github.com/molecula/featurebase/v3/authn"
"github.com/molecula/featurebase/v3/batch"
pilosaclient "github.com/molecula/featurebase/v3/client"
"github.com/molecula/featurebase/v3/idk/idktest"
"github.com/molecula/featurebase/v3/logger"
@ -713,7 +714,7 @@ func TestGetPrimaryKeyRecordizer(t *testing.T) {
t.Errorf("unmatched skips exp/got\n%+v\n%+v", test.expSkip, skips)
}
row := &pilosaclient.Row{}
row := &batch.Row{}
err = rdz(test.rawRec, row)
if err != nil {
t.Fatalf("unexpected error from recordizer: %v", err)
@ -750,11 +751,11 @@ func TestBatchFromSchema(t *testing.T) {
err string
batchErr string
rdzErrs []string
time pilosaclient.QuantizedTime
time batch.QuantizedTime
lookupWriteIdxs []int
}
getQuantizedTime := func(t time.Time) pilosaclient.QuantizedTime {
qt := pilosaclient.QuantizedTime{}
getQuantizedTime := func(t time.Time) batch.QuantizedTime {
qt := batch.QuantizedTime{}
qt.Set(t)
return qt
}
@ -844,13 +845,13 @@ func TestBatchFromSchema(t *testing.T) {
{
name: "empty",
autogen: true,
err: "can't batch with no fields or batch size",
err: "can't batch with no fields",
},
{
name: "empty-w/ExtGen",
autogen: true,
extgen: true,
err: "can't batch with no fields or batch size",
err: "can't batch with no fields",
},
{
name: "no id field",
@ -1275,7 +1276,7 @@ type testSource struct {
schema []Field
}
func (t *testSource) Close() error {
func (s *testSource) Close() error {
return nil
}

8
server/server.go
View file

@ -30,6 +30,7 @@ import (
pilosa "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/authn"
"github.com/molecula/featurebase/v3/authz"
"github.com/molecula/featurebase/v3/batch"
"github.com/molecula/featurebase/v3/boltdb"
"github.com/molecula/featurebase/v3/encoding/proto"
petcd "github.com/molecula/featurebase/v3/etcd"
@ -438,9 +439,10 @@ func (m *Command) SetupServer() error {
m.Config.Etcd.Id = m.Config.Name // TODO(twg) rethink this
e := petcd.NewEtcd(m.Config.Etcd, m.logger, m.Config.Cluster.ReplicaN, version)
executionPlannerFn := func(e pilosa.Executor, a *pilosa.API, s string) sql3.CompilePlanner {
fapi := &pilosa.FeatureBaseSchemaAPI{API: a}
return planner.NewExecutionPlanner(e, fapi, a, s)
executionPlannerFn := func(e pilosa.Executor, api *pilosa.API, sql string) sql3.CompilePlanner {
fapi := &pilosa.FeatureBaseSchemaAPI{API: api}
fimp := &batch.FeaturebaseImporter{API: api}
return planner.NewExecutionPlanner(e, fapi, api, fimp, m.logger, sql)
}
serverOptions := []pilosa.ServerOption{

27
sql3/parser/ast.go
View file

@ -2018,7 +2018,7 @@ func (l *ExprList) Clone() *ExprList {
return &other
}
/*func cloneExprLists(a []*ExprList) []*ExprList {
func cloneExprLists(a []*ExprList) []*ExprList {
if a == nil {
return nil
}
@ -2027,7 +2027,7 @@ func (l *ExprList) Clone() *ExprList {
other[i] = a[i].Clone()
}
return other
}*/
}
// String returns the string representation of the expression.
func (l *ExprList) String() string {
@ -2775,8 +2775,8 @@ type InsertStatement struct {
Columns []*Ident // optional column list
ColumnsRparen Pos // position of column list right paren
Values Pos // position of VALUES keyword
ValueList *ExprList // list of values
Values Pos // position of VALUES keyword
TupleList []*ExprList // multiple tuples
// Select *SelectStatement // SELECT statement
@ -2796,7 +2796,7 @@ func (s *InsertStatement) Clone() *InsertStatement {
other.Table = s.Table.Clone()
other.Alias = s.Alias.Clone()
other.Columns = cloneIdents(s.Columns)
other.ValueList = s.ValueList.Clone()
other.TupleList = cloneExprLists(s.TupleList)
//other.Select = s.Select.Clone()
//other.UpsertClause = s.UpsertClause.Clone()
return &other
@ -2848,14 +2848,19 @@ func (s *InsertStatement) String() string {
// fmt.Fprintf(&buf, " %s", s.Select.String())
//} else {
buf.WriteString(" VALUES")
buf.WriteString(" (")
for j, expr := range s.ValueList.Exprs {
if j != 0 {
buf.WriteString(", ")
for i, tuple := range s.TupleList {
if i != 0 {
buf.WriteString(",")
}
buf.WriteString(expr.String())
buf.WriteString(" (")
for j, expr := range tuple.Exprs {
if j != 0 {
buf.WriteString(", ")
}
buf.WriteString(expr.String())
}
buf.WriteString(")")
}
buf.WriteString(")")
//}
//if s.UpsertClause != nil {

22
sql3/parser/ast_test.go
View file

@ -592,11 +592,29 @@ func TestInsertStatement_String(t *testing.T) {
{Name: "x"},
{Name: "y"},
},
ValueList: &parser.ExprList{
Exprs: []parser.Expr{&parser.NullLit{}, &parser.NullLit{}},
TupleList: []*parser.ExprList{
{
Exprs: []parser.Expr{&parser.NullLit{}, &parser.NullLit{}},
},
},
}, `INSERT INTO "tbl" ("x", "y") VALUES (NULL, NULL)`)
AssertStatementStringer(t, &parser.InsertStatement{
Table: &parser.Ident{Name: "tbl"},
Columns: []*parser.Ident{
{Name: "x"},
{Name: "y"},
},
TupleList: []*parser.ExprList{
{
Exprs: []parser.Expr{&parser.IntegerLit{Value: "1"}, &parser.IntegerLit{Value: "2"}},
},
{
Exprs: []parser.Expr{&parser.IntegerLit{Value: "3"}, &parser.IntegerLit{Value: "4"}},
},
},
}, `INSERT INTO "tbl" ("x", "y") VALUES (1, 2), (3, 4)`)
// AssertStatementStringer(t, &sql.InsertStatement{
// WithClause: &sql.WithClause{
// CTEs: []*sql.CTE{

16
sql3/parser/parser.go
View file

@ -1584,19 +1584,23 @@ func (p *Parser) parseInsertStatement(withClause *WithClause) (_ *InsertStatemen
switch p.peek() {
case VALUES:
stmt.Values, _, _ = p.scan()
// Parse out the value tuples.
stmt.TupleList = make([]*ExprList, 0)
for {
var list ExprList
var tuple ExprList
if p.peek() != LP {
return &stmt, p.errorExpected(p.pos, p.tok, "left paren")
}
list.Lparen, _, _ = p.scan()
tuple.Lparen, _, _ = p.scan()
for {
expr, err := p.ParseExpr()
if err != nil {
return &stmt, err
}
list.Exprs = append(list.Exprs, expr)
tuple.Exprs = append(tuple.Exprs, expr)
if p.peek() == RP {
break
@ -1605,14 +1609,16 @@ func (p *Parser) parseInsertStatement(withClause *WithClause) (_ *InsertStatemen
}
p.scan()
}
list.Rparen, _, _ = p.scan()
stmt.ValueList = &list
tuple.Rparen, _, _ = p.scan()
stmt.TupleList = append(stmt.TupleList, &tuple)
if p.peek() != COMMA {
break
}
p.scan()
}
//case SELECT:
// if stmt.Select, err = p.parseSelectStatement(false, nil); err != nil {
// return &stmt, err

51
sql3/parser/parser_test.go
View file

@ -2269,15 +2269,50 @@ func TestParser_ParseStatement(t *testing.T) {
},
ColumnsRparen: pos(21),
Values: pos(23),
ValueList: &parser.ExprList{
Lparen: pos(30),
Exprs: []parser.Expr{
&parser.IntegerLit{ValuePos: pos(31), Value: "1"},
&parser.IntegerLit{ValuePos: pos(34), Value: "2"},
TupleList: []*parser.ExprList{
{
Lparen: pos(30),
Exprs: []parser.Expr{
&parser.IntegerLit{ValuePos: pos(31), Value: "1"},
&parser.IntegerLit{ValuePos: pos(34), Value: "2"},
},
Rparen: pos(35),
},
Rparen: pos(35),
},
})
// Ensure we can parse multiple tuple values in an INSERT INTO statement.
AssertParseStatement(t, `INSERT INTO tbl (x, y) VALUES (1, 2), (3, 4)`, &parser.InsertStatement{
Insert: pos(0),
Into: pos(7),
Table: &parser.Ident{NamePos: pos(12), Name: "tbl"},
ColumnsLparen: pos(16),
Columns: []*parser.Ident{
{NamePos: pos(17), Name: "x"},
{NamePos: pos(20), Name: "y"},
},
ColumnsRparen: pos(21),
Values: pos(23),
TupleList: []*parser.ExprList{
{
Lparen: pos(30),
Exprs: []parser.Expr{
&parser.IntegerLit{ValuePos: pos(31), Value: "1"},
&parser.IntegerLit{ValuePos: pos(34), Value: "2"},
},
Rparen: pos(35),
},
{
Lparen: pos(38),
Exprs: []parser.Expr{
&parser.IntegerLit{ValuePos: pos(39), Value: "3"},
&parser.IntegerLit{ValuePos: pos(42), Value: "4"},
},
Rparen: pos(43),
},
},
})
/*AssertParseStatement(t, `REPLACE INTO tbl (x, y) VALUES (1, 2), (3, 4)`, &parser.InsertStatement{
Replace: pos(0),
Into: pos(8),
@ -3604,7 +3639,3 @@ func AssertParseExprError(tb testing.TB, s string, want string) {
func pos(offset int) parser.Pos {
return parser.Pos{Offset: offset, Line: 1, Column: offset + 1}
}
func deepEqual(a, b interface{}) string {
return strings.Join(deep.Equal(a, b), "\n")
}

18
sql3/parser/walk.go
View file

@ -275,15 +275,17 @@ func walk(v Visitor, node Node) (_ Node, err error) {
if err := walkIdentList(v, n.Columns); err != nil {
return node, err
}
//for i := range n.ValueLists {
if list, err := walk(v, n.ValueList); err != nil {
return node, err
} else if list != nil {
n.ValueList = list.(*ExprList)
} else {
n.ValueList = nil
for i, tuple := range n.TupleList {
if list, err := walk(v, tuple); err != nil {
return node, err
} else if list != nil {
n.TupleList[i] = list.(*ExprList)
} else {
n.TupleList[i] = nil
}
}
//}
/*if n.Select != nil {
if sel, err := walk(v, n.Select); err != nil {
return node, err

50
sql3/planner/compileinsert.go
View file

@ -18,7 +18,7 @@ func (p *ExecutionPlanner) compileInsertStatement(stmt *parser.InsertStatement)
tableName := parser.IdentName(stmt.Table)
targetColumns := []*qualifiedRefPlanExpression{}
insertValues := []types.PlanExpression{}
insertValues := [][]types.PlanExpression{}
table, err := p.schemaAPI.IndexInfo(context.Background(), tableName)
if err != nil {
@ -54,12 +54,16 @@ func (p *ExecutionPlanner) compileInsertStatement(stmt *parser.InsertStatement)
}
//add expressions from values list
for _, expr := range stmt.ValueList.Exprs {
e, err := p.compileExpr(expr)
if err != nil {
return nil, err
for _, tuple := range stmt.TupleList {
tupleValues := []types.PlanExpression{}
for _, expr := range tuple.Exprs {
e, err := p.compileExpr(expr)
if err != nil {
return nil, err
}
tupleValues = append(tupleValues, e)
}
insertValues = append(insertValues, e)
insertValues = append(insertValues, tupleValues)
}
return NewPlanOpInsert(p, tableName, targetColumns, insertValues), nil
@ -92,8 +96,10 @@ func (p *ExecutionPlanner) analyzeInsertStatement(stmt *parser.InsertStatement)
}
// Make sure (implicit) insert list and expression list have the same
// number of items.
if len(typeNames) != len(stmt.ValueList.Exprs) {
return sql3.NewErrInsertExprTargetCountMismatch(stmt.ValueList.Lparen.Line, stmt.ValueList.Lparen.Column)
for _, tuple := range stmt.TupleList {
if len(typeNames) != len(tuple.Exprs) {
return sql3.NewErrInsertExprTargetCountMismatch(tuple.Lparen.Line, tuple.Lparen.Column)
}
}
} else {
// Check column list refers to actual columns, and that there are no
@ -153,24 +159,28 @@ func (p *ExecutionPlanner) analyzeInsertStatement(stmt *parser.InsertStatement)
}
// Make sure insert list and expression list have the same number of items.
if len(stmt.Columns) != len(stmt.ValueList.Exprs) {
return sql3.NewErrInsertExprTargetCountMismatch(stmt.ValueList.Lparen.Line, stmt.ValueList.Lparen.Column)
for _, tuple := range stmt.TupleList {
if len(stmt.Columns) != len(tuple.Exprs) {
return sql3.NewErrInsertExprTargetCountMismatch(tuple.Lparen.Line, tuple.Lparen.Column)
}
}
}
// Check each of the expressions.
for i, expr := range stmt.ValueList.Exprs {
e, err := p.analyzeExpression(expr, stmt)
if err != nil {
return err
}
for _, tuple := range stmt.TupleList {
for i, expr := range tuple.Exprs {
e, err := p.analyzeExpression(expr, stmt)
if err != nil {
return err
}
// Type check against same ordinal position in column type list.
if !typesAreAssignmentCompatible(typeNames[i], e.DataType()) {
return sql3.NewErrTypeAssignmentIncompatible(expr.Pos().Line, expr.Pos().Column, e.DataType().TypeName(), typeNames[i].TypeName())
}
// Type check against same ordinal position in column type list.
if !typesAreAssignmentCompatible(typeNames[i], e.DataType()) {
return sql3.NewErrTypeAssignmentIncompatible(expr.Pos().Line, expr.Pos().Column, e.DataType().TypeName(), typeNames[i].TypeName())
}
stmt.ValueList.Exprs[i] = e
tuple.Exprs[i] = e
}
}
return nil

18
sql3/planner/executionplanner.go
View file

@ -5,9 +5,10 @@ package planner
import (
"context"
"encoding/json"
"log"
pilosa "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/batch"
"github.com/molecula/featurebase/v3/logger"
"github.com/molecula/featurebase/v3/sql3"
"github.com/molecula/featurebase/v3/sql3/parser"
"github.com/molecula/featurebase/v3/sql3/planner/types"
@ -26,15 +27,19 @@ type ExecutionPlanner struct {
executor pilosa.Executor
schemaAPI pilosa.SchemaAPI
computeAPI pilosa.ComputeAPI
importer batch.Importer
logger logger.Logger
sql string
scopeStack *scopeStack
}
func NewExecutionPlanner(executor pilosa.Executor, schemaAPI pilosa.SchemaAPI, computeAPI pilosa.ComputeAPI, sql string) *ExecutionPlanner {
func NewExecutionPlanner(executor pilosa.Executor, schemaAPI pilosa.SchemaAPI, computeAPI pilosa.ComputeAPI, importer batch.Importer, logger logger.Logger, sql string) *ExecutionPlanner {
return &ExecutionPlanner{
executor: executor,
schemaAPI: schemaAPI,
computeAPI: computeAPI,
importer: importer,
logger: logger,
sql: sql,
scopeStack: newScopeStack(),
}
@ -80,10 +85,15 @@ func (p *ExecutionPlanner) CompilePlan(ctx context.Context, stmt parser.Statemen
}
// Log the plan. This happens even if an error occurred.
if rootOperator != nil {
switch rootOperator.(type) {
case *PlanOpInsert:
// Don't log the insert plan since it can be very large.
case nil:
// pass
default:
plan := rootOperator.Plan()
a, _ := json.MarshalIndent(plan, "", " ")
log.Println(string(a))
p.logger.Debugf(string(a))
}
return rootOperator, err

53
sql3/planner/expression.go
View file

@ -7,6 +7,7 @@ import (
"fmt"
"math"
"regexp"
"sort"
"strconv"
"strings"
"time"
@ -1912,8 +1913,27 @@ func (n *castPlanExpression) Evaluate(currentRow []interface{}) (interface{}, er
case *parser.DataTypeStringSet:
return nl, nil
case *parser.DataTypeString:
//TODO(pok) come up with a better string representation of string set
return fmt.Sprintf("%v", nl), nil
sort.Strings(nl)
var ret strings.Builder
// open bracket
ret.WriteString("[")
// elements
var afterFirst bool
for i := range nl {
if afterFirst {
ret.WriteString(",")
}
ret.WriteString(`"` + strings.ReplaceAll(nl[i], `"`, `\"`) + `"`)
afterFirst = true
}
// close braket
ret.WriteString("]")
return ret.String(), nil
}
case *parser.DataTypeTimestamp:
@ -2097,17 +2117,12 @@ func (n *exprTupleLiteralPlanExpression) Evaluate(currentRow []interface{}) (int
return nil, err
}
//if it is a string, do a coercion
val, ok := timestampEval.(string)
if ok {
if tm, err := time.ParseInLocation(time.RFC3339Nano, val, time.UTC); err == nil {
timestampEval = tm
} else if tm, err := time.ParseInLocation(time.RFC3339, val, time.UTC); err == nil {
timestampEval = tm
} else if tm, err := time.ParseInLocation("2006-01-02", val, time.UTC); err == nil {
timestampEval = tm
} else {
// if it is a string, do a coercion
if val, ok := timestampEval.(string); ok {
if tm, err := timestampFromString(val); err != nil {
return nil, sql3.NewErrInvalidTypeCoercion(0, 0, val, n.members[0].Type().TypeName())
} else {
timestampEval = tm
}
}
@ -2482,3 +2497,17 @@ func wildCardToRegexp(pattern string) string {
return result.String()
}
// timeFromString attempts to parse the string to a time.Time using a series of
// time formats.
func timestampFromString(s string) (time.Time, error) {
if tm, err := time.ParseInLocation(time.RFC3339Nano, s, time.UTC); err == nil {
return tm, nil
} else if tm, err := time.ParseInLocation(time.RFC3339, s, time.UTC); err == nil {
return tm, nil
} else if tm, err := time.ParseInLocation("2006-01-02", s, time.UTC); err == nil {
return tm, nil
}
return time.Time{}, sql3.NewErrInvalidTypeCoercion(0, 0, s, "time.Time")
}

65
sql3/planner/opbulkinsert.go
View file

@ -138,7 +138,11 @@ type bulkInsertCSVRowIter struct {
isKeyed bool
options *bulkInsertOptions
latch *struct{}
// latch is used to indicate if the CSV has been processed. It will
// be set to a non-nil value upon processing. After that, the file
// should not be processed again.
latch *struct{}
currentBatch []interface{}
lastKeyValue uint64
}
@ -146,39 +150,46 @@ type bulkInsertCSVRowIter struct {
var _ types.RowIterator = (*bulkInsertCSVRowIter)(nil)
func (i *bulkInsertCSVRowIter) Next(ctx context.Context) (types.Row, error) {
if i.latch == nil {
i.latch = &struct{}{}
i.lastKeyValue = 0
// If Next has already been called, return early. We only want to process
// the file once.
if i.latch != nil {
return nil, types.ErrNoMoreRows
}
f, err := os.Open(i.options.fileName)
if err != nil {
// Set latch to indicate that Next() has been called.
i.latch = &struct{}{}
i.lastKeyValue = 0
f, err := os.Open(i.options.fileName)
if err != nil {
return nil, err
}
defer f.Close()
linesRead := 0
csvReader := csv.NewReader(f)
for {
rec, err := csvReader.Read()
if err == io.EOF {
break
} else if err != nil {
return nil, err
}
defer f.Close()
// do something with read line
if err = i.processCSVLine(ctx, rec); err != nil {
return nil, err
}
linesRead += 1
linesRead := 0
csvReader := csv.NewReader(f)
for {
rec, err := csvReader.Read()
if err == io.EOF {
break
}
if err != nil {
return nil, err
}
// do something with read line
err = i.processCSVLine(ctx, rec)
if err != nil {
return nil, err
}
linesRead += 1
// bail if we have a rows limit and we've hit it
if i.options.rowsLimit > 0 && linesRead >= i.options.rowsLimit {
break
}
// bail if we have a rows limit and we've hit it
if i.options.rowsLimit > 0 && linesRead >= i.options.rowsLimit {
break
}
}
return nil, types.ErrNoMoreRows
}

578
sql3/planner/opinsert.go
View file

@ -9,10 +9,10 @@ import (
"time"
pilosa "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/pql"
fbbatch "github.com/molecula/featurebase/v3/batch"
"github.com/molecula/featurebase/v3/sql3"
"github.com/molecula/featurebase/v3/sql3/parser"
"github.com/molecula/featurebase/v3/sql3/planner/types"
"github.com/pkg/errors"
)
// PlanOpInsert plan operator to handle INSERT.
@ -20,11 +20,11 @@ type PlanOpInsert struct {
planner *ExecutionPlanner
tableName string
targetColumns []*qualifiedRefPlanExpression
insertValues []types.PlanExpression
insertValues [][]types.PlanExpression
warnings []string
}
func NewPlanOpInsert(p *ExecutionPlanner, tableName string, targetColumns []*qualifiedRefPlanExpression, insertValues []types.PlanExpression) *PlanOpInsert {
func NewPlanOpInsert(p *ExecutionPlanner, tableName string, targetColumns []*qualifiedRefPlanExpression, insertValues [][]types.PlanExpression) *PlanOpInsert {
return &PlanOpInsert{
planner: p,
tableName: tableName,
@ -48,11 +48,15 @@ func (p *PlanOpInsert) Plan() map[string]interface{} {
ps = append(ps, e.Plan())
}
result["targetColumns"] = ps
ps = make([]interface{}, 0)
for _, e := range p.insertValues {
ps = append(ps, e.Plan())
pps := make([]interface{}, 0)
for _, tuple := range p.insertValues {
ps := make([]interface{}, 0)
for _, e := range tuple {
ps = append(ps, e.Plan())
}
pps = append(pps, ps)
}
result["insertValues"] = ps
result["insertValues"] = pps
return result
}
@ -93,362 +97,236 @@ type insertRowIter struct {
planner *ExecutionPlanner
tableName string
targetColumns []*qualifiedRefPlanExpression
insertValues []types.PlanExpression
insertValues [][]types.PlanExpression
}
var _ types.RowIterator = (*insertRowIter)(nil)
func (i *insertRowIter) Next(ctx context.Context) (types.Row, error) {
qcx := i.planner.computeAPI.Txf().NewQcx()
// posID is the position of the "_id" column in both the targetColumns and
// values lists.
var posID int
colIDs := make([]uint64, 0)
colKeys := make([]string, 0)
// posVals maps the position in the tuple to the position in the row.Values.
// It essentially takes the _id column into account and skips it.
//
// So for example, if we have sql:
// INSERT INTO (a, _id, b, c) VALUES ('aa', 1, 'bb', 'cc');
//
// Then we want row.ID = 1 and row.Values to contain {'aa', 'bb', 'cc'}.
// This means posVals would contain the map []int{0, 1*, 1, 2}, which maps
// VALUES positions (0,2,3) to row.Values (0, 1, 2). Note, the _id position
// (shown as 1* in the example above) isn't used because we handle it
// separately.
posVals := make([]int, len(i.targetColumns))
addColID := func(v interface{}) error {
switch id := v.(type) {
case int64:
colIDs = append(colIDs, uint64(id))
case uint64:
colIDs = append(colIDs, id)
case string:
colKeys = append(colKeys, id)
default:
return sql3.NewErrInternalf("unhandled _id data type '%T'", id)
var foundPosID bool
for j := range i.targetColumns {
if foundPosID {
posVals[j] = j - 1
continue
}
return nil
if strings.EqualFold(i.targetColumns[j].columnName, "_id") {
posID = j
foundPosID = true
}
posVals[j] = j
}
//find the _id column and evaluate
var err error
var columnID interface{}
for idx, iv := range i.insertValues {
targetColumn := i.targetColumns[idx]
if strings.EqualFold(targetColumn.columnName, "_id") {
columnID, err = iv.Evaluate(nil)
if err != nil {
return nil, err
// batchSize is currently set to the size of the entire
// VALUES list. In the future we may want to break this up into smaller
// batches.
batchSize := len(i.insertValues)
// idxInfoBase is the full IndexInfo stored in the schema. The instance of
// IndexInfo used in the import (and created below) will be based on the
// information from idxInfoBase, but the fields may be a limited subset, and
// may be in a different order.
idxInfoBase, err := i.planner.schemaAPI.IndexInfo(ctx, i.tableName)
if err != nil {
return nil, sql3.NewErrTableNotFound(0, 0, i.tableName)
}
// idxInfo is a subset of idxInfoBase, containing only those fields included
// in the INSERT INTO statement (i.e. only i.targetcolumns), and in the
// order specified.
idxInfo := &pilosa.IndexInfo{
Name: idxInfoBase.Name,
CreatedAt: idxInfoBase.CreatedAt,
Options: idxInfoBase.Options,
Fields: make([]*pilosa.FieldInfo, len(i.targetColumns)-1),
ShardWidth: idxInfoBase.ShardWidth,
}
// Set up Fields based on i.targetColumns.
var counter int
for ii, targetColumn := range i.targetColumns {
// Skip the "_id" column.
if ii == posID {
continue
}
idxInfo.Fields[counter] = idxInfoBase.Field(targetColumn.columnName)
counter++
}
batch, err := fbbatch.NewBatch(i.planner.importer, batchSize, idxInfo, idxInfo.Fields,
fbbatch.OptUseShardTransactionalEndpoint(true),
)
if err != nil {
return nil, errors.Wrap(err, "setting up batch")
}
// row is the single instance of batch.Row allocated. It is re-used
// throughout the for loop to minimize memory allocation.
var row fbbatch.Row
// Initialize row.Values to the size of the target columns, but exclude the
// record ID ("_id") since that's stored in row.ID.
row.Values = make([]interface{}, len(i.targetColumns)-1)
for _, tuple := range i.insertValues {
// Evaluate and set the record ID.
if eval, err := tuple[posID].Evaluate(nil); err != nil {
return nil, errors.Wrapf(err, "evaluating record id: %v", tuple[posID])
} else {
// These value types correspond to the types supported in batch.Add().
switch recid := eval.(type) {
case string, uint64, []byte:
row.ID = recid
case int64:
if recid < 0 {
return nil, sql3.NewErrInternalf("_id value cannot be negative: %d", recid)
}
row.ID = uint64(recid)
default:
// If we get to here, it's likey that the id type is unsupported
// and will cause an error in batch.Add(). So there's no need to
// return an error here in this default.
row.ID = eval
}
break
}
// Loop over the values in the tuple and populate the Row values.
for idx, iv := range tuple {
// Skip the record ID because that was already handled above
// (prior to this loop).
if idx == posID {
continue
}
eval, err := iv.Evaluate(nil)
if err != nil {
return nil, errors.Wrapf(err, "evaluating tuple value: %v", iv)
}
// batch.Add does not typically look at field type to determine how
// to handle a particular value in a row. Instead, it uses value
// type. As an example, if the value type is int64, then batch.Add
// assumes that it should be handled as if going into an `int`
// field. Therefore, we need to look at the field type here and make
// sure that the value types being sent through batch.Add align with
// the field type assumptions that batch.Add is making.
switch opts := idxInfo.Fields[posVals[idx]].Options; opts.Type {
// By the time we get here, we assume that the planner has already
// determined the value type such that the Evaluate() method called
// above results in the correct value types. There is one exception:
// sql3 treats all integer values as int64. This means that an ID
// field, which expects a uint64 value, would get treated as an int
// field. In order to avoid that, we cast int64 values to uint64
// when the field type is set or mutex.
case pilosa.FieldTypeSet, pilosa.FieldTypeMutex:
switch v := eval.(type) {
case int64:
if v < 0 {
return nil, sql3.NewErrInternalf("converting negative value to uint64: %d", v)
}
row.Values[posVals[idx]] = uint64(v)
case []int64:
uint64s := make([]uint64, len(v))
for i := range v {
if v[i] < 0 {
return nil, sql3.NewErrInternalf("converting negative slice value to uint64: %d", v[i])
}
uint64s[i] = uint64(v[i])
}
row.Values[posVals[idx]] = uint64s
default:
row.Values[posVals[idx]] = eval
}
case pilosa.FieldTypeTimestamp:
switch v := eval.(type) {
// time.Time is used for date literals generated in the parser.
// For example, if using `current_time`, the type received here
// will be a time.Time.
case time.Time:
// Convert Base, which is the epoch for Timestamp fields, to
// a time.Time value.
unit := fbbatch.TimeUnit(opts.TimeUnit)
epoch, err := fbbatch.Int64ToTimestamp(unit, time.Time{}, opts.Base)
if err != nil {
return nil, errors.Wrapf(err, "converting base to epoch: %d", opts.Base)
}
i64, err := fbbatch.TimestampToInt64(unit, epoch, v)
if err != nil {
return nil, errors.Wrapf(err, "converting timestamp to int64: %s", v)
}
row.Values[posVals[idx]] = i64
// string is the normal case for dates; used when the date is
// provided as a string in the INSERT INTO statement.
case string:
ts, err := timestampFromString(v)
if err != nil {
return nil, errors.Wrapf(err, "parsing timestamp: %s", v)
}
// Convert Base, which is the epoch for Timestamp fields, to
// a time.Time value.
unit := fbbatch.TimeUnit(opts.TimeUnit)
epoch, err := fbbatch.Int64ToTimestamp(unit, time.Time{}, opts.Base)
if err != nil {
return nil, errors.Wrapf(err, "converting base to epoch: %d", opts.Base)
}
i64, err := fbbatch.TimestampToInt64(unit, epoch, ts)
if err != nil {
return nil, errors.Wrapf(err, "converting timestamp to int64: %s", v)
}
row.Values[posVals[idx]] = i64
// nil is to support `null` values.
case nil:
row.Values[posVals[idx]] = eval
default:
return nil, sql3.NewErrInternalf("unsupported timestamp type: %T", eval)
}
default:
row.Values[posVals[idx]] = eval
}
}
if err := batch.Add(row); err != nil {
// Breaking here on ErrBatchNowFull is only valid because we are
// explicity setting the batch size to the number of tuples in the
// INSERT INTO statement. Which means we're only handling a single
// batch. If this evolves to handle multiple batches, this will need
// to instead call batch.Import() and continue looping over tuples.
// We may also need to handle ErrBatchNowStale.
if err == fbbatch.ErrBatchNowFull {
break
}
return nil, errors.Wrap(err, "adding record")
}
}
//eval all the expressions and do the insert
for idx, iv := range i.insertValues {
colIDs = make([]uint64, 0)
colKeys = make([]string, 0)
targetColumn := i.targetColumns[idx]
if strings.EqualFold(targetColumn.columnName, "_id") {
continue
}
eval, err := iv.Evaluate(nil)
if err != nil {
return nil, err
}
//nothing to do if a value is null
if eval == nil {
continue
}
sourceType := iv.Type()
switch targetType := i.targetColumns[idx].dataType.(type) {
case *parser.DataTypeInt:
err = addColID(columnID)
if err != nil {
return nil, err
}
vals := make([]int64, 1)
vals[0] = eval.(int64)
req := &pilosa.ImportValueRequest{
Index: i.tableName,
Field: targetColumn.columnName,
Shard: 0, //TODO: handle non-0 shards
ColumnIDs: colIDs,
ColumnKeys: colKeys,
Values: vals,
}
err = i.planner.computeAPI.ImportValue(ctx, qcx, req)
if err != nil {
return nil, err
}
case *parser.DataTypeBool:
err = addColID(columnID)
if err != nil {
return nil, err
}
val := eval.(bool)
vals := make([]uint64, 1)
if val {
vals[0] = 1
} else {
vals[0] = 0
}
req := &pilosa.ImportRequest{
Index: i.tableName,
Field: targetColumn.columnName,
Shard: 0, //TODO: handle non-0 shards
ColumnIDs: colIDs,
ColumnKeys: colKeys,
RowIDs: vals,
}
err = i.planner.computeAPI.Import(ctx, qcx, req)
if err != nil {
return nil, err
}
case *parser.DataTypeDecimal:
err = addColID(columnID)
if err != nil {
return nil, err
}
vals := make([]float64, 1)
vals[0] = eval.(pql.Decimal).Float64()
req := &pilosa.ImportValueRequest{
Index: i.tableName,
Field: targetColumn.columnName,
Shard: 0, //TODO: handle non-0 shards
ColumnIDs: colIDs,
ColumnKeys: colKeys,
FloatValues: vals,
}
err = i.planner.computeAPI.ImportValue(ctx, qcx, req)
if err != nil {
return nil, err
}
case *parser.DataTypeID:
err = addColID(columnID)
if err != nil {
return nil, err
}
coercedVal, err := coerceValue(sourceType, targetType, eval, parser.Pos{Line: 0, Column: 0})
if err != nil {
return nil, err
}
vals := make([]uint64, 1)
vals[0] = uint64(coercedVal.(int64))
req := &pilosa.ImportRequest{
Index: i.tableName,
Field: targetColumn.columnName,
Shard: 0, //TODO: handle non-0 shards
ColumnIDs: colIDs,
ColumnKeys: colKeys,
RowIDs: vals,
}
err = i.planner.computeAPI.Import(ctx, qcx, req)
if err != nil {
return nil, err
}
case *parser.DataTypeIDSet:
rowIDs := make([]uint64, 0)
rowSet := eval.([]int64)
for k := range rowSet {
err = addColID(columnID)
if err != nil {
return nil, err
}
rowIDs = append(rowIDs, uint64(rowSet[k]))
}
req := &pilosa.ImportRequest{
Index: i.tableName,
Field: targetColumn.columnName,
Shard: 0, //TODO: handle non-0 shards
ColumnIDs: colIDs,
ColumnKeys: colKeys,
RowIDs: rowIDs,
}
err = i.planner.computeAPI.Import(ctx, qcx, req)
if err != nil {
return nil, err
}
case *parser.DataTypeIDSetQuantum:
rowIDs := make([]uint64, 0)
timestamps := make([]int64, 0)
coercedVal, err := coerceValue(sourceType, targetType, eval, parser.Pos{Line: 0, Column: 0})
if err != nil {
return nil, err
}
record := coercedVal.([]interface{})
rowSet := record[1].([]int64)
for k := range rowSet {
err = addColID(columnID)
if err != nil {
return nil, err
}
rowIDs = append(rowIDs, uint64(rowSet[k]))
}
if record[0] == nil {
timestamps = nil
} else {
timestamp, ok := record[0].(time.Time)
if !ok {
return nil, sql3.NewErrInternalf("unexpected type '%T'", record[0])
}
for range rowSet {
timestamps = append(timestamps, timestamp.Unix())
}
}
req := &pilosa.ImportRequest{
Index: i.tableName,
Field: targetColumn.columnName,
Shard: 0, //TODO: handle non-0 shards
ColumnIDs: colIDs,
ColumnKeys: colKeys,
RowIDs: rowIDs,
Timestamps: timestamps,
}
err = i.planner.computeAPI.Import(ctx, qcx, req)
if err != nil {
return nil, err
}
case *parser.DataTypeString:
err = addColID(columnID)
if err != nil {
return nil, err
}
rowKeys := make([]string, 1)
rowKeys[0] = eval.(string)
req := &pilosa.ImportRequest{
Index: i.tableName,
Field: targetColumn.columnName,
Shard: 0, //TODO: handle non-0 shards
ColumnIDs: colIDs,
ColumnKeys: colKeys,
RowKeys: rowKeys,
}
err = i.planner.computeAPI.Import(ctx, qcx, req)
if err != nil {
return nil, err
}
case *parser.DataTypeStringSet:
rowKeys := make([]string, 0)
rowSet := eval.([]string)
for k := range rowSet {
err = addColID(columnID)
if err != nil {
return nil, err
}
rowKeys = append(rowKeys, rowSet[k])
}
req := &pilosa.ImportRequest{
Index: i.tableName,
Field: targetColumn.columnName,
Shard: 0, //TODO: handle non-0 shards
ColumnIDs: colIDs,
ColumnKeys: colKeys,
RowKeys: rowKeys,
}
err = i.planner.computeAPI.Import(ctx, qcx, req)
if err != nil {
return nil, err
}
case *parser.DataTypeStringSetQuantum:
rowKeys := make([]string, 0)
timestamps := make([]int64, 0)
coercedVal, err := coerceValue(sourceType, targetType, eval, parser.Pos{Line: 0, Column: 0})
if err != nil {
return nil, err
}
record := coercedVal.([]interface{})
rowSet := record[1].([]string)
for k := range rowSet {
err = addColID(columnID)
if err != nil {
return nil, err
}
rowKeys = append(rowKeys, rowSet[k])
}
if record[0] == nil {
timestamps = nil
} else {
timestamp, ok := record[0].(time.Time)
if !ok {
return nil, sql3.NewErrInternalf("unexpected type '%T'", record[0])
}
for range rowSet {
timestamps = append(timestamps, timestamp.Unix())
}
}
req := &pilosa.ImportRequest{
Index: i.tableName,
Field: targetColumn.columnName,
Shard: 0, //TODO: handle non-0 shards
ColumnIDs: colIDs,
ColumnKeys: colKeys,
RowKeys: rowKeys,
Timestamps: timestamps,
}
err = i.planner.computeAPI.Import(ctx, qcx, req)
if err != nil {
return nil, err
}
case *parser.DataTypeTimestamp:
err = addColID(columnID)
if err != nil {
return nil, err
}
coercedVal, err := coerceValue(sourceType, targetType, eval, parser.Pos{Line: 0, Column: 0})
if err != nil {
return nil, err
}
vals := make([]time.Time, 1)
vals[0] = coercedVal.(time.Time)
req := &pilosa.ImportValueRequest{
Index: i.tableName,
Field: targetColumn.columnName,
Shard: 0, //TODO: handle non-0 shards
ColumnIDs: colIDs,
ColumnKeys: colKeys,
TimestampValues: vals,
}
err = i.planner.computeAPI.ImportValue(ctx, qcx, req)
if err != nil {
return nil, err
}
default:
return nil, sql3.NewErrInternalf("unhandled data type '%T'", targetType)
}
if err := batch.Import(); err != nil {
return nil, errors.Wrap(err, "importing batch")
}
return nil, types.ErrNoMoreRows

5
sql3/planner/planoptimizer.go
View file

@ -5,7 +5,6 @@ package planner
import (
"context"
"fmt"
"log"
"reflect"
"strings"
@ -676,11 +675,11 @@ func tryToRewriteSubtableJoins(ctx context.Context, a *ExecutionPlanner, n types
// for each of the projection operators, for each of the projections
// transform each of the referenced values with a the first arg
log.Printf("%T", projections)
a.logger.Debugf("%T", projections)
}
// there is a join condition, make sure it is one that is permissible (range queries only?)
log.Printf("%T", tvf)
a.logger.Debugf("%T", tvf)
return nl, true, nil
default:

32
sql3/sql_definitions_test.go
View file

@ -71,7 +71,8 @@ var tableTests []tableTest = []tableTest{
row(int64(3), int64(33), []int64{31, 32, 33}, int64(301), "str3", []string{"a3", "b3", "c3"}, pql.NewDecimal(34567, 2)),
row(int64(4), int64(44), []int64{41, 42, 43}, int64(401), "str4", []string{"a4", "b4", "c4"}, pql.NewDecimal(45678, 2)),
),
compare: compareExactUnordered,
compare: compareExactUnordered,
sortStringKeys: true,
},
{
// Select all with top.
@ -92,7 +93,8 @@ var tableTests []tableTest = []tableTest{
row(int64(1), int64(11), []int64{11, 12, 13}, int64(101), "str1", []string{"a1", "b1", "c1"}, pql.NewDecimal(12345, 2)),
row(int64(2), int64(22), []int64{21, 22, 23}, int64(201), "str2", []string{"a2", "b2", "c2"}, pql.NewDecimal(23456, 2)),
),
compare: compareExactUnordered,
compare: compareExactUnordered,
sortStringKeys: true,
},
{
// Select all with where on each field.
@ -114,7 +116,8 @@ var tableTests []tableTest = []tableTest{
expRows: rows(
row(int64(2), int64(22), []int64{21, 22, 23}, int64(201), "str2", []string{"a2", "b2", "c2"}, pql.NewDecimal(23456, 2)),
),
compare: compareExactOrdered,
compare: compareExactOrdered,
sortStringKeys: true,
},
},
},
@ -158,7 +161,8 @@ var tableTests []tableTest = []tableTest{
row("three", int64(33), []int64{31, 32, 33}, int64(301), "str3", []string{"a3", "b3", "c3"}),
row("four", int64(44), []int64{41, 42, 43}, int64(401), "str4", []string{"a4", "b4", "c4"}),
),
compare: compareExactUnordered,
compare: compareExactUnordered,
sortStringKeys: true,
},
{
// Select all with top.
@ -180,8 +184,9 @@ var tableTests []tableTest = []tableTest{
row("three", int64(33), []int64{31, 32, 33}, int64(301), "str3", []string{"a3", "b3", "c3"}),
row("four", int64(44), []int64{41, 42, 43}, int64(401), "str4", []string{"a4", "b4", "c4"}),
),
compare: compareIncludedIn,
expRowCount: 2,
compare: compareIncludedIn,
sortStringKeys: true,
expRowCount: 2,
},
{
// Select all with where on int field.
@ -202,7 +207,8 @@ var tableTests []tableTest = []tableTest{
expRows: rows(
row("two", int64(22), []int64{21, 22, 23}, int64(201), "str2", []string{"a2", "b2", "c2"}),
),
compare: compareExactUnordered,
compare: compareExactUnordered,
sortStringKeys: true,
},
},
},
@ -346,6 +352,9 @@ var tableTests []tableTest = []tableTest{
joinTestsOrders,
joinTests,
//bool (batch logic)
boolTests,
//time quantums
// Skip for now - timeQuantumInsertTest,
}
@ -384,6 +393,15 @@ var insertTest = tableTest{
expRows: rows(),
compare: compareExactUnordered,
},
{
// Insert multiple tuples
sqls: sqls(
"insert into testinsert (_id, a, b, s, bl, d, event, ievent) values (4, 40, 400, 'foo', false, 10.12, ['A', 'B', 'C'], [1, 2, 3]), (5, 50, 500, 'var', true, 20.24, ['X', 'Y', 'Z'], [4, 5, 6])",
),
expHdrs: hdrs(),
expRows: rows(),
compare: compareExactUnordered,
},
{
// Insert with nulls
sqls: sqls(

88
sql3/sql_defs_bool_test.go Normal file
View file

@ -0,0 +1,88 @@
package sql3_test
// BOOL tests
var boolTests = tableTest{
name: "single-bool-field",
table: tbl(
"singleboolfield",
srcHdrs(
srcHdr("_id", fldTypeID),
srcHdr("a_bool", fldTypeBool),
),
srcRows(),
),
sqlTests: []sqlTest{
{
// Insert, step 1.
name: "insert1",
sqls: sqls(
`insert into singleboolfield (_id, a_bool) values
(1, true),
(2, true),
(3, false),
(4, false),
(5, null),
(6, null)`,
),
expHdrs: hdrs(),
expRows: rows(),
compare: compareExactOrdered,
},
{
// Select all, step 1.
name: "select-all1",
sqls: sqls(
"select * from singleboolfield",
),
expHdrs: hdrs(
hdr("_id", fldTypeID),
hdr("a_bool", fldTypeBool),
),
expRows: rows(
row(int64(1), true),
row(int64(2), true),
row(int64(3), false),
row(int64(4), false),
row(int64(5), nil),
row(int64(6), nil),
),
compare: compareExactOrdered,
},
{
// Insert, step 2. Change bool values to all other combinations.
name: "insert2",
sqls: sqls(
`insert into singleboolfield (_id, a_bool) values
(1, false),
(2, null),
(3, true),
(4, null),
(5, false),
(6, true)`,
),
expHdrs: hdrs(),
expRows: rows(),
compare: compareExactOrdered,
},
{
// Select all, step 2.
name: "select-all2",
sqls: sqls(
"select * from singleboolfield",
),
expHdrs: hdrs(
hdr("_id", fldTypeID),
hdr("a_bool", fldTypeBool),
),
expRows: rows(
row(int64(1), false),
row(int64(2), nil),
row(int64(3), true),
row(int64(4), nil),
row(int64(5), false),
row(int64(6), true),
),
compare: compareExactOrdered,
},
},
}

5
sql3/sql_defs_cast_test.go
View file

@ -785,7 +785,7 @@ var castStringSet = tableTest{
hdr("", fldTypeString),
),
expRows: rows(
row(int64(1), string("[101 102]")),
row(int64(1), string(`["101","102"]`)),
),
compare: compareExactUnordered,
},
@ -800,7 +800,8 @@ var castStringSet = tableTest{
expRows: rows(
row(int64(1), []string{"101", "102"}),
),
compare: compareExactUnordered,
compare: compareExactUnordered,
sortStringKeys: true,
},
{
sqls: sqls(

4
sql3/sql_defs_in_test.go
View file

@ -1,6 +1,6 @@
package sql3_test
//IN tests
// IN tests
var inTests = tableTest{
table: tbl(
"in_all_types",
@ -131,7 +131,7 @@ var inTests = tableTest{
},
}
//NOT IN tests
// NOT IN tests
var notInTests = tableTest{
table: tbl(
"not_in_all_types",

354
sql3/sql_test.go
View file

@ -2,14 +2,13 @@
package sql3_test
import (
"context"
"fmt"
"log"
"sort"
"strings"
"testing"
"time"
pilosa "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/sql3/parser"
planner_types "github.com/molecula/featurebase/v3/sql3/planner/types"
sql_test "github.com/molecula/featurebase/v3/sql3/test"
@ -22,18 +21,17 @@ func TestSQL_Execute(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()
ctx := context.Background()
api := c.GetNode(0).API
svr := c.GetNode(0).Server
for i, test := range tableTests {
tableTestName := fmt.Sprintf("table-%d", i)
if test.name != "" {
tableTestName = test.name
} else if test.table.name != "" {
tableTestName = test.table.name
}
t.Run(tableTestName, func(t *testing.T) {
var err error
// Create a table with all field types.
if test.table.columns != nil {
_, _, err := sql_test.MustQueryRows(t, svr, test.table.createTable())
@ -41,241 +39,9 @@ func TestSQL_Execute(t *testing.T) {
}
if len(test.table.rows) > 0 {
// Populate fields with data.
qcx := api.Txf().NewQcx()
// idIdx is the index position of the _id column. If a source provides the
// _id somewhere other than column 0, then we need to add logic here to find
// its index.
idIdx := 0
for i, col := range test.table.columns {
if col.name == "_id" {
continue
}
colIDs := make([]uint64, 0)
colKeys := make([]string, 0)
addColID := func(v interface{}) {
switch id := v.(type) {
case uint64:
colIDs = append(colIDs, id)
case int64:
colIDs = append(colIDs, uint64(id))
case string:
colKeys = append(colKeys, id)
default:
t.Fatalf("unexpected type for colid '%T'", v)
}
}
switch col.typ.(type) {
case *parser.DataTypeInt:
vals := make([]int64, 0)
for _, row := range test.table.rows {
if row[i] == nil {
continue
}
addColID(row[idIdx])
vals = append(vals, row[i].(int64))
}
if len(vals) == 0 {
continue
}
req := &pilosa.ImportValueRequest{
Index: test.table.name,
Field: col.name,
Shard: 0,
ColumnIDs: colIDs,
ColumnKeys: colKeys,
Values: vals,
}
err = api.ImportValue(ctx, qcx, req)
assert.NoError(t, err)
case *parser.DataTypeBool:
vals := make([]uint64, 0)
for _, row := range test.table.rows {
if row[i] == nil {
continue
}
addColID(row[idIdx])
if row[i].(bool) {
vals = append(vals, 1)
} else {
vals = append(vals, 0)
}
}
if len(vals) == 0 {
continue
}
req := &pilosa.ImportRequest{
Index: test.table.name,
Field: col.name,
Shard: 0,
ColumnIDs: colIDs,
ColumnKeys: colKeys,
RowIDs: vals,
}
err = api.Import(ctx, qcx, req)
assert.NoError(t, err)
case *parser.DataTypeDecimal:
vals := make([]float64, 0)
for _, row := range test.table.rows {
if row[i] == nil {
continue
}
addColID(row[idIdx])
vals = append(vals, row[i].(float64))
}
if len(vals) == 0 {
continue
}
req := &pilosa.ImportValueRequest{
Index: test.table.name,
Field: col.name,
Shard: 0,
ColumnIDs: colIDs,
ColumnKeys: colKeys,
FloatValues: vals,
}
err = api.ImportValue(ctx, qcx, req)
assert.NoError(t, err)
case *parser.DataTypeIDSet:
rowIDs := make([]uint64, 0)
for _, row := range test.table.rows {
if row[i] == nil {
continue
}
rowSet := row[i].([]int64)
for k := range rowSet {
addColID(row[idIdx])
rowIDs = append(rowIDs, uint64(rowSet[k]))
}
}
if len(rowIDs) == 0 {
continue
}
req := &pilosa.ImportRequest{
Index: test.table.name,
Field: col.name,
Shard: 0,
ColumnIDs: colIDs,
ColumnKeys: colKeys,
RowIDs: rowIDs,
}
err = api.Import(ctx, qcx, req)
assert.NoError(t, err)
case *parser.DataTypeID:
rowIDs := make([]uint64, 0)
for _, row := range test.table.rows {
if row[i] == nil {
continue
}
addColID(row[idIdx])
rowIDs = append(rowIDs, uint64(row[i].(int64)))
}
if len(rowIDs) == 0 {
continue
}
req := &pilosa.ImportRequest{
Index: test.table.name,
Field: col.name,
Shard: 0,
ColumnIDs: colIDs,
ColumnKeys: colKeys,
RowIDs: rowIDs,
}
err = api.Import(ctx, qcx, req)
assert.NoError(t, err)
case *parser.DataTypeString:
rowKeys := make([]string, 0)
for _, row := range test.table.rows {
if row[i] == nil {
continue
}
addColID(row[idIdx])
rowKeys = append(rowKeys, row[i].(string))
}
if len(rowKeys) == 0 {
continue
}
req := &pilosa.ImportRequest{
Index: test.table.name,
Field: col.name,
Shard: 0,
ColumnIDs: colIDs,
ColumnKeys: colKeys,
RowKeys: rowKeys,
}
err = api.Import(ctx, qcx, req)
assert.NoError(t, err)
case *parser.DataTypeStringSet:
rowKeys := make([]string, 0)
for _, row := range test.table.rows {
if row[i] == nil {
continue
}
rowSet := row[i].([]string)
for k := range rowSet {
addColID(row[idIdx])
rowKeys = append(rowKeys, rowSet[k])
}
}
if len(rowKeys) == 0 {
continue
}
req := &pilosa.ImportRequest{
Index: test.table.name,
Field: col.name,
Shard: 0,
ColumnIDs: colIDs,
ColumnKeys: colKeys,
RowKeys: rowKeys,
}
err = api.Import(ctx, qcx, req)
assert.NoError(t, err)
case *parser.DataTypeTimestamp:
vals := make([]time.Time, 0)
for _, row := range test.table.rows {
if row[i] == nil {
continue
}
addColID(row[idIdx])
vals = append(vals, row[i].(time.Time))
}
if len(vals) == 0 {
continue
}
req := &pilosa.ImportValueRequest{
Index: test.table.name,
Field: col.name,
Shard: 0,
ColumnIDs: colIDs,
ColumnKeys: colKeys,
TimestampValues: vals,
}
err = api.ImportValue(ctx, qcx, req)
assert.NoError(t, err)
default:
t.Fatalf("column type not supported: %s", col.typ)
}
}
_, _, err := sql_test.MustQueryRows(t, svr, test.table.insertInto(t))
assert.NoError(t, err)
}
for i, sqltest := range test.sqlTests {
@ -315,22 +81,25 @@ func TestSQL_Execute(t *testing.T) {
exp[i] = make([]interface{}, len(headers))
for j := range sqltest.expHdrs {
targetIdx := m[sqltest.expHdrs[j].ColumnName]
if !assert.GreaterOrEqual(t, len(sqltest.expRows[i]), len(headers)) {
t.Fatalf("expected row set has fewer columns than returned headers")
}
assert.GreaterOrEqual(t, len(sqltest.expRows[i]), len(headers),
"expected row set has fewer columns than returned headers")
exp[i][targetIdx] = sqltest.expRows[i][j]
}
}
if sqltest.sortStringKeys {
sortStringKeys(rows)
}
switch sqltest.compare {
case compareExactOrdered:
assert.EqualValues(t, len(sqltest.expRows), len(rows))
assert.Equal(t, len(sqltest.expRows), len(rows))
assert.EqualValues(t, exp, rows)
case compareExactUnordered:
assert.EqualValues(t, len(sqltest.expRows), len(rows))
assert.Equal(t, len(sqltest.expRows), len(rows))
assert.ElementsMatch(t, exp, rows)
case compareIncludedIn:
assert.EqualValues(t, sqltest.expRowCount, len(rows))
assert.Equal(t, sqltest.expRowCount, len(rows))
for _, row := range rows {
assert.Contains(t, exp, row)
}
@ -343,6 +112,23 @@ func TestSQL_Execute(t *testing.T) {
}
}
// sortStringKeys goes through an entire set of rows, and for any []string it
// finds, it orders the elements. This is obviously only useful in tests, and
// only in cases where we expect the elements to match, but we don't care what
// order they're in. It's basically the equivalent of assert.ElementsMatch(),
// but the way we use that on rows doesn't recurse down into the field values
// within each row.
func sortStringKeys(in [][]interface{}) {
for i := range in {
for j := range in[i] {
switch v := in[i][j].(type) {
case []string:
sort.Strings(v)
}
}
}
}
//////////////////////////////////////////////////////////////////////
type fldType parser.ExprDataType
@ -375,13 +161,14 @@ type tableTest struct {
}
type sqlTest struct {
name string
sqls []string
expHdrs []*planner_types.PlannerColumn
expRows [][]interface{}
expErr string
compare compareMethod
expRowCount int
name string
sqls []string
expHdrs []*planner_types.PlannerColumn
expRows [][]interface{}
expErr string
compare compareMethod
sortStringKeys bool
expRowCount int
}
// The following "source" types are helpers for creating a test table.
@ -420,6 +207,63 @@ func srcRow(cells ...interface{}) sourceRow {
type sourceRow []interface{}
type sourceRows []sourceRow
// insertTuples returns the list of tuples (as a single string) to use as the
// VALUES value in an INSERT INTO statement.
func (sr sourceRows) insertTuples(t *testing.T) string {
var afterFirstRow bool
var sb strings.Builder
for _, row := range sr {
if afterFirstRow {
sb.WriteString(",")
}
var afterFirstCell bool
sb.WriteString("(")
for _, cell := range row {
if afterFirstCell {
sb.WriteString(",")
}
switch v := cell.(type) {
case string:
sb.WriteString("'" + v + "'")
case int64:
sb.WriteString(fmt.Sprintf("%d", v))
case float64:
sb.WriteString(fmt.Sprintf("%.2f", v))
case []int64:
strs := make([]string, len(v))
for i := range v {
strs[i] = fmt.Sprintf("%d", v[i])
}
sb.WriteString("[" + strings.Join(strs, ",") + "]")
case []string:
if len(v) == 0 {
sb.WriteString("[]")
} else {
sb.WriteString("['" + strings.Join(v, "','") + "']")
}
case bool:
sb.WriteString(fmt.Sprintf("%v", v))
case nil:
sb.WriteString("null")
case time.Time:
sb.WriteString("'" + v.Format(time.RFC3339) + "'")
default:
t.Fatalf("unsupported cell type: %T", cell)
}
afterFirstCell = true
}
sb.WriteString(")")
afterFirstRow = true
}
return sb.String()
}
type source struct {
name string
columns []sourceColumn
@ -445,6 +289,12 @@ func (s source) createTable() string {
return ct
}
func (s source) insertInto(t *testing.T) string {
ii := "INSERT INTO " + s.name + " VALUES "
ii += sourceRows(s.rows).insertTuples(t)
return ii
}
// hdrs is just a helper function to make the test definition look cleaner.
func hdrs(hdrs ...*planner_types.PlannerColumn) []*planner_types.PlannerColumn {
return hdrs