scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-07 17:15:56 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

merge

Browse source
This commit is contained in:
Todd Gruben 2018-04-06 10:29:30 -05:00
commit 645a64e193
51 changed files with 916 additions and 1108 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.dockerignore
View file

@ -1 +0,0 @@
.*

4
.travis.yml
View file

@ -8,7 +8,7 @@ env:
- secure: "VnBFmFfBOrrf7ONLN9WpAFCcV8SEt5G5VPnnHv97TP7PlJG8LWR6k6O+vRJOvf8V4vDMfKCTDonwWLgbssVf3yygo3C8ZoftY2phehEkWGffCgsd9ML/YBNbGq4LYLSE5HKvBqrZjQaOrVby71BAsP8W7RhC6hqzFQ00M/z8dZVfwaQQFwew2eEcSxLEaaDFS8Wgc3/UuwxDRPBq6u3cCN5RxfB+q70HvGVq4TT+0dqS4eCvz688+Z0GIGYx9olNjh0F2Kc8R2Po0lnUNa0GiHrZ21zeQ1DxIK04QABrWWmjL4h+bx3VHNKPFR4GYSKDf+pj1kfaqbfrAg6rMAJdGejgoS+QyjhgCoN4d3qRp8s+1nrxtp0TvezEdjwyxt4quGHbP5TxWUszssbGhWqf4mx6OeJ8MmdTaJjfu0f3NWJXMycqT6J73WKORk4rHeIqF9CIdxdmcpkwYj8rk0TEMTPTsd7WA8w2HIDsCz/jQnRmEgLUiNnTAofYc/uUi/Wg/T2hllkp+oBDTzxk9NTelkqx8TJ0bDmYYL9JWUi1siFHTHiVYTJgyirSfGNpe61u8OLmT0Hak/D399IfL7qgFLlMXk8q92typfO2xEduq6G+8KygeqiOMSsOY+xcDvZf5xtcEihYd21vjtrxRSqFsup/o8DIxEurQnfXBx1B+WA="
- secure: "U4fpHWDVOG4viqZsiVgUDW7OW1JW60uPOZy0q9pfbs86iHvmZq0PaScsZ+YdlYaN2GETVr7endDf6DCcZs1PWfg0F6VQfkOXcShX8HVS9O58lUZA5tyvbDVql9DQs4PbnkZo+ktz+Z0YaXqq2RdtMDOUz4bgZwspLPMA14if+N6w0tqCFpB7bEtpptTGsdbIQPG1n07yvSeNmK4mvrEEs77tWmhulN5iilpOqhpIvD39bJvtCYVALuJpzLd/OjLTPV9l/fl+hJkMXSj+X5ilO1DHINAcCM648iEX2phXAIWmi0O0Rbg2cI4kV9T5ysOIw8ux+YCm9bZDGTCt+VGBW5Fg+Z5iaXXexyKYCGiHleOJ7kCj9kXxh2u8NiYVNgb19dGJV5/HgQ6pcGWjeVEqr8yY1546zMjpTX+SYGQF+XZe+uggEjeAsk53ueXa0pyZTrlrqSvR7BBtWPx47s/dTg2L19FQYv3XpGMxEXLw92RplExQKi1h7QgihRxFpjGgURHhrt7d9eiNiNqBt3ZsHjmh2AkXZHnaDjlgSnFFWaMqP3UtDBWIuO+2BMbZUJVfP+gpQGBZ4gtpUSmV2JDCHgZgX5OAnLD4usxh+ATQ4rvUXF/tf8nMqEKHlGKd8hxpYSyMX21BoqfSfY4/IA0ejVE9BITqlrvqewqkP1yxe7o="
install:
- make vendor generate-statik
- make install-dep install-statik vendor generate-statik
script:
- make test
# TODO: When we drop support for Go <1.10, we should use `-coverprofile=` on both `go test` and `goveralls` so the test suite doesn't run twice. See https://github.com/pilosa/pilosa/issues/1009
@ -23,7 +23,7 @@ deploy:
skip_cleanup: true
on:
all_branches: true
go: 1.9
go: "1.10"
matrix:
allow_failures:
- go: master

72
CONTRIBUTING.md
View file

@ -66,6 +66,78 @@ If you want to help but you aren't sure where to start, check out our [github la
git remote add upstream git@github.com:pilosa/pilosa.git
```
### Makefile
Pilosa includes a Makefile that automates several tasks:
- Install Pilosa:
```sh
make install
```
- Install build dependencies (dep, statik, and protoc):
```sh
make install-build-deps
```
- Create the vendor directory:
```sh
make vendor
```
- Run the test suite:
```sh
make test
```
- View the coverage report:
```sh
make cover-viz
```
- Clear the `vendor/` and `build/` directories:
```sh
make clean
```
- Create release tarballs:
```sh
make release
```
- Generate static assets for the WebUI:
```sh
make generate-statik
```
- Regenerate protocol buffer files in `internal/`:
```sh
make generate-protoc
```
- Create tagged Docker image:
```sh
make docker
```
- Run tests inside Docker container:
```sh
make docker-test
```
Additional commands are available in the `Makefile`.
### Submitting code changes
- Before starting to work on a task, sync your branch with the upstream:

7
Dockerfile
View file

@ -1,12 +1,9 @@
FROM golang:1.10 as builder
ARG ldflags=''
COPY . /go/src/github.com/pilosa/pilosa
COPY . /go/src/github.com/pilosa/pilosa/
RUN cd /go/src/github.com/pilosa/pilosa \
&& make vendor \
&& CGO_ENABLED=0 go install -tags release -a -ldflags "$ldflags" github.com/pilosa/pilosa/cmd/pilosa
&& CGO_ENABLED=0 make install-dep install-statik install FLAGS="-a"
FROM scratch

175
Makefile
View file

@ -1,123 +1,134 @@
.PHONY: dep docker pilosa release-build prerelease-build release prerelease prerelease-upload install generate generate-statik generate-protoc statik test cover cover-pkg cover-viz clean docker-build docker-test
.PHONY: build check-clean clean cover cover-viz default docker docker-build docker-test generate generate-protoc generate-statik install install-build-deps install-dep install-protoc install-protoc-gen-gofast install-statik prerelease prerelease-build prerelease-upload release release-build require-dep require-protoc require-protoc-gen-gofast require-statik test
DEP := $(shell command -v dep 2>/dev/null)
STATIK := $(shell command -v statik 2>/dev/null)
PROTOC := $(shell command -v protoc 2>/dev/null)
VERSION := $(shell git describe --tags 2> /dev/null || echo unknown)
STATUS := $(shell git status --porcelain)
IDENTIFIER := $(VERSION)-$(GOOS)-$(GOARCH)
CLONE_URL=github.com/pilosa/pilosa
PKGS := $(shell cd $(GOPATH)/src/$(CLONE_URL); go list ./... | grep -v vendor)
BUILD_TIME=`date -u +%FT%T%z`
VERSION := $(shell git describe --tags 2> /dev/null || echo unknown)
VERSION_ID := $(VERSION)-$(GOOS)-$(GOARCH)
BRANCH := $(if $(TRAVIS_BRANCH),$(TRAVIS_BRANCH),$(shell git rev-parse --abbrev-ref HEAD))
BRANCH_ID := $(BRANCH)-$(GOOS)-$(GOARCH)
BUILD_TIME := $(shell date -u +%FT%T%z)
LDFLAGS="-X github.com/pilosa/pilosa.Version=$(VERSION) -X github.com/pilosa/pilosa.BuildTime=$(BUILD_TIME)"
DOCKER_GOLANG_IMAGE=golang:latest
GIT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD)
BRANCH := $(if $(TRAVIS_BRANCH),$(TRAVIS_BRANCH),$(GIT_BRANCH))
BRANCH_IDENTIFIER := $(BRANCH)-$(GOOS)-$(GOARCH)
GO_VERSION=latest
default: test pilosa
# Run tests and compile Pilosa
default: test build
# Remove vendor and build directories
clean:
rm -rf vendor build
$(GOPATH)/bin:
mkdir $(GOPATH)/bin
dep: $(GOPATH)/bin
go get -u github.com/golang/dep/cmd/dep
# Set up vendor directory using `dep`
vendor: Gopkg.toml
ifndef DEP
make dep
endif
$(MAKE) require-dep
dep ensure
touch vendor
Gopkg.lock: dep Gopkg.toml
dep ensure
# Run test suite
test: vendor
go test $(PKGS) $(TESTFLAGS)
go test ./... $(TESTFLAGS)
# Run test suite with coverage enabled
cover: vendor
mkdir -p build/coverage
echo "mode: set" > build/coverage/all.out
for pkg in $(PKGS) ; do \
make cover-pkg PKG=$$pkg ; \
done
cover-pkg:
mkdir -p build/coverage
touch build/coverage/$(subst /,-,$(PKG)).out
go test -coverprofile=build/coverage/$(subst /,-,$(PKG)).out $(PKG)
tail -n +2 build/coverage/$(subst /,-,$(PKG)).out >> build/coverage/all.out
mkdir -p build
$(MAKE) test TESTFLAGS="-coverprofile=build/coverage.out"
# Run test suite with coverage enabled and view coverage results in browser
cover-viz: cover
go tool cover -html=build/coverage/all.out
go tool cover -html=build/coverage.out
pilosa: vendor
go build -tags release -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa
# Compile Pilosa
build: vendor
go build -tags release -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa
# Create a single release build under the build directory
release-build: vendor
ifdef DOCKER_BUILD
make docker-build FLAGS="-o build/pilosa-$(IDENTIFIER)/pilosa"
else
make pilosa FLAGS="-o build/pilosa-$(IDENTIFIER)/pilosa"
endif
cp LICENSE README.md build/pilosa-$(IDENTIFIER)
tar -cvz -C build -f build/pilosa-$(IDENTIFIER).tar.gz pilosa-$(IDENTIFIER)/
@echo "Created release build: build/pilosa-$(IDENTIFIER).tar.gz"
$(MAKE) $(if $(DOCKER_BUILD),docker-)build FLAGS="-o build/pilosa-$(VERSION_ID)/pilosa"
cp LICENSE README.md build/pilosa-$(VERSION_ID)
tar -cvz -C build -f build/pilosa-$(VERSION_ID).tar.gz pilosa-$(VERSION_ID)/
@echo Created release build: build/pilosa-$(VERSION_ID).tar.gz
release:
ifeq ($(STATUS),"")
make release-build GOOS=darwin GOARCH=amd64
make release-build GOOS=linux GOARCH=amd64 DOCKER_BUILD=1
make release-build GOOS=linux GOARCH=386 DOCKER_BUILD=1
else
@echo "Will not create release with unclean git status."
endif
# Error out if there are untracked changes in Git
check-clean:
$(if $(shell git status --porcelain),$(error Git status is not clean! Please commit or checkout/reset changes.))
# Create release build tarballs for all supported platforms. Linux compilation happens under Docker.
release: check-clean
$(MAKE) release-build GOOS=darwin GOARCH=amd64
$(MAKE) release-build GOOS=linux GOARCH=amd64 DOCKER_BUILD=1
$(MAKE) release-build GOOS=linux GOARCH=386 DOCKER_BUILD=1
# Create branch-tagged pre-release for client library CI jobs
prerelease-build: vendor
make pilosa FLAGS="-o build/pilosa-$(BRANCH_IDENTIFIER)/pilosa"
cp LICENSE README.md build/pilosa-$(BRANCH_IDENTIFIER)
tar -cvz -C build -f build/pilosa-$(BRANCH_IDENTIFIER).tar.gz pilosa-$(BRANCH_IDENTIFIER)/
@echo "Created pre-release build: build/pilosa-$(BRANCH_IDENTIFIER).tar.gz"
$(MAKE) release-build VERSION_ID=$(BRANCH_ID)
# Create prerelease build for Linux/amd64
prerelease:
make prerelease-build GOOS=linux GOARCH=amd64
$(MAKE) prerelease-build GOOS=linux GOARCH=amd64
# Upload prerelease to S3
prerelease-upload: prerelease
aws s3 cp build/pilosa-$(BRANCH_IDENTIFIER).tar.gz s3://build.pilosa.com/pilosa-$(BRANCH_IDENTIFIER).tar.gz --acl public-read
aws s3 cp build/pilosa-$(BRANCH_ID).tar.gz s3://build.pilosa.com/pilosa-$(BRANCH_ID).tar.gz --acl public-read
# Install Pilosa
install: vendor
go install -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa
go install -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa
.protoc-gen-gofast: vendor
ifndef PROTOC
$(error "protoc is not available. please install protoc from https://github.com/google/protobuf/releases")
endif
go build -o .protoc-gen-gofast ./vendor/github.com/gogo/protobuf/protoc-gen-gofast
cp ./.protoc-gen-gofast $(GOPATH)/bin/protoc-gen-gofast
generate-protoc: .protoc-gen-gofast
# `go generate` protocol buffers
generate-protoc: require-protoc require-protoc-gen-gofast
go generate github.com/pilosa/pilosa/internal
generate-statik: statik
# `go generate` statik assets (WebUI)
generate-statik: require-statik
go generate github.com/pilosa/pilosa/statik
# `go generate` all needed packages
generate: generate-protoc generate-statik
statik:
ifndef STATIK
go get github.com/rakyll/statik
endif
# Create Docker image from Dockerfile
docker:
docker build -t "pilosa:$(VERSION)" --build-arg ldflags=$(LDFLAGS) .
@echo "Created image: pilosa:$(VERSION)"
docker build -t "pilosa:$(VERSION)" .
@echo Created docker image: pilosa:$(VERSION)
# Compile Pilosa inside Docker container
docker-build:
docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) -e GOOS=$(GOOS) -e GOARCH=$(GOARCH) $(DOCKER_GOLANG_IMAGE) go build -tags release -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa
docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) -e GOOS=$(GOOS) -e GOARCH=$(GOARCH) golang:$(GO_VERSION) go build -tags release -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa
# Run Pilosa tests inside Docker container
docker-test:
docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) $(DOCKER_GOLANG_IMAGE) go test $(TESTFLAGS) $(PKGS)
docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) golang:$(GO_VERSION) go test $(TESTFLAGS) ./...
######################
# Build dependencies #
######################
# Verifies that needed build dependency is installed. Errors out if not installed.
define require
$(if $(shell command -v $1 2>/dev/null),
$(info Verified build dependency "$1" is installed.),
$(error Build dependency "$1" not installed. To install, run `make install-$1` or `make install-build-deps`))
endef
require-dep:
$(call require,dep)
require-statik:
$(call require,statik)
require-protoc-gen-gofast:
$(call require,protoc-gen-gofast)
require-protoc:
$(call require,protoc)
install-build-deps: install-dep install-statik install-protoc-gen-gofast install-protoc
install-dep:
go get -u github.com/golang/dep/cmd/dep
install-statik:
go get -u github.com/rakyll/statik
install-protoc-gen-gofast:
go get -u github.com/gogo/protobuf/protoc-gen-gofast
install-protoc:
@echo This tool cannot automatically install protoc. Please download and install protoc from https://google.github.io/proto-lens/installing-protoc.html

96
cluster.go
View file

@ -20,9 +20,7 @@ import (
"errors"
"fmt"
"hash/fnv"
"io"
"io/ioutil"
"log"
"math/rand"
"net/http"
"os"
@ -268,8 +266,7 @@ type Cluster struct {
closing chan struct{}
prefect SecurityManager
// The writer for any logging.
LogOutput io.Writer
Logger Logger
//
RemoteClient *http.Client
@ -289,16 +286,11 @@ func NewCluster() *Cluster {
closing: make(chan struct{}),
joining: make(chan struct{}),
LogOutput: os.Stderr,
prefect: &NopSecurityManager{},
Logger: NopLogger,
prefect: &NopSecurityManager{},
}
}
// logger returns a logger for the cluster.
func (c *Cluster) logger() *log.Logger {
return log.New(c.LogOutput, "", log.LstdFlags)
}
// Coordinator returns the coordinator node.
func (c *Cluster) CoordinatorNode() *Node {
return c.nodeByID(c.Coordinator)
@ -359,7 +351,7 @@ func (c *Cluster) UpdateCoordinator(n *Node) bool {
// AddNode adds a node to the Cluster and updates and saves the
// new topology.
func (c *Cluster) AddNode(node *Node) error {
c.logger().Printf("add node %s to cluster on %s", node, c.Node)
c.Logger.Printf("add node %s to cluster on %s", node, c.Node)
// If the node being added is the coordinator, set it for this node.
if node.IsCoordinator {
@ -437,7 +429,7 @@ func (c *Cluster) setState(state string) {
return
}
c.logger().Printf("change cluster state from %s to %s on %s", c.state, state, c.Node.ID)
c.Logger.Printf("change cluster state from %s to %s on %s", c.state, state, c.Node.ID)
var doCleanup bool
@ -471,7 +463,7 @@ func (c *Cluster) setState(state string) {
// Clean holder.
if err := cleaner.CleanHolder(); err != nil {
c.logger().Printf("holder clean error: err=%s", err)
c.Logger.Printf("holder clean error: err=%s", err)
}
}
}
@ -487,7 +479,7 @@ func (c *Cluster) SetNodeState(state string) error {
State: state,
}
c.logger().Printf("Sending State %s (%s)", state, c.Coordinator)
c.Logger.Printf("Sending State %s (%s)", state, c.Coordinator)
if err := c.sendTo(c.CoordinatorNode(), ns); err != nil {
return fmt.Errorf("sending node state error: err=%s", err)
}
@ -509,7 +501,7 @@ func (c *Cluster) ReceiveNodeState(nodeID string, state string) error {
}
c.Topology.nodeStates[nodeID] = state
c.logger().Printf("received state %s (%s)", state, nodeID)
c.Logger.Printf("received state %s (%s)", state, nodeID)
// Set cluster state to NORMAL.
if c.haveTopologyAgreement() && c.allNodesReady() {
@ -951,9 +943,9 @@ func (c *Cluster) Open() error {
return fmt.Errorf("sending restart NodeJoin: %v", err)
}
c.logger().Printf("wait for joining to complete")
c.Logger.Printf("wait for joining to complete")
<-c.joining
c.logger().Printf("joining has completed")
c.Logger.Printf("joining has completed")
}
return nil
@ -968,7 +960,7 @@ func (c *Cluster) Close() error {
}
func (c *Cluster) markAsJoined() {
c.logger().Printf("mark node as joined (received coordinator update)")
c.Logger.Printf("mark node as joined (received coordinator update)")
if !c.joined {
c.joined = true
close(c.joining)
@ -1001,9 +993,9 @@ func (c *Cluster) allNodesReady() bool {
func (c *Cluster) handleNodeAction(nodeAction nodeAction) error {
j, err := c.generateResizeJob(nodeAction)
if err != nil {
c.logger().Printf("generateResizeJob error: err=%s", err)
c.Logger.Printf("generateResizeJob error: err=%s", err)
if err := c.setStateAndBroadcast(ClusterStateNormal); err != nil {
c.logger().Printf("setStateAndBroadcast error: err=%s", err)
c.Logger.Printf("setStateAndBroadcast error: err=%s", err)
}
return err
}
@ -1017,7 +1009,7 @@ func (c *Cluster) handleNodeAction(nodeAction nodeAction) error {
})
// Wait for the ResizeJob to finish or be aborted.
c.logger().Printf("wait for jobResult")
c.Logger.Printf("wait for jobResult")
jobResult := <-j.result
// Make sure j.Run() didn't return an error.
@ -1025,7 +1017,7 @@ func (c *Cluster) handleNodeAction(nodeAction nodeAction) error {
return err
}
c.logger().Printf("received jobResult: %s", jobResult)
c.Logger.Printf("received jobResult: %s", jobResult)
switch jobResult {
case ResizeJobStateDone:
if err := c.CompleteCurrentJob(ResizeJobStateDone); err != nil {
@ -1048,7 +1040,7 @@ func (c *Cluster) handleNodeAction(nodeAction nodeAction) error {
func (c *Cluster) setStateAndBroadcast(state string) error {
c.SetState(state)
// Broadcast cluster status changes to the cluster.
c.logger().Printf("broadcasting ClusterStatus: %s", state)
c.Logger.Printf("broadcasting ClusterStatus: %s", state)
return c.Broadcaster.SendSync(c.Status())
}
@ -1081,7 +1073,7 @@ func (c *Cluster) listenForJoins() {
case nodeAction := <-c.joiningLeavingNodes:
err := c.handleNodeAction(nodeAction)
if err != nil {
c.logger().Printf("handleNodeAction error: err=%s", err)
c.Logger.Printf("handleNodeAction error: err=%s", err)
continue
}
setNormal = true
@ -1093,7 +1085,7 @@ func (c *Cluster) listenForJoins() {
if setNormal {
// Put the cluster back to state NORMAL and broadcast.
if err := c.setStateAndBroadcast(ClusterStateNormal); err != nil {
c.logger().Printf("setStateAndBroadcast error: err=%s", err)
c.Logger.Printf("setStateAndBroadcast error: err=%s", err)
}
}
@ -1104,7 +1096,7 @@ func (c *Cluster) listenForJoins() {
case nodeAction := <-c.joiningLeavingNodes:
err := c.handleNodeAction(nodeAction)
if err != nil {
c.logger().Printf("handleNodeAction error: err=%s", err)
c.Logger.Printf("handleNodeAction error: err=%s", err)
continue
}
setNormal = true
@ -1117,7 +1109,7 @@ func (c *Cluster) listenForJoins() {
// added/removed. It also saves a reference to the ResizeJob in the `jobs` map
// for future lookup by JobID.
func (c *Cluster) generateResizeJob(nodeAction nodeAction) (*ResizeJob, error) {
c.logger().Printf("generateResizeJob: %v", nodeAction)
c.Logger.Printf("generateResizeJob: %v", nodeAction)
c.mu.Lock()
defer c.mu.Unlock()
@ -1125,7 +1117,7 @@ func (c *Cluster) generateResizeJob(nodeAction nodeAction) (*ResizeJob, error) {
if err != nil {
return nil, err
}
c.logger().Printf("generated ResizeJob: %d", j.ID)
c.Logger.Printf("generated ResizeJob: %d", j.ID)
// Save job in jobs map for future reference.
c.jobs[j.ID] = j
@ -1215,14 +1207,14 @@ func (c *Cluster) CompleteCurrentJob(state string) error {
// FollowResizeInstruction is run by any node that receives a ResizeInstruction.
func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) error {
c.logger().Printf("follow resize instruction on %s", c.Node.ID)
c.Logger.Printf("follow resize instruction on %s", c.Node.ID)
// Make sure the cluster status on this node agrees with the Coordinator
// before attempting a resize.
if err := c.MergeClusterStatus(instr.ClusterStatus); err != nil {
return err
}
c.logger().Printf("MergeClusterStatus done, start goroutine")
c.Logger.Printf("MergeClusterStatus done, start goroutine")
// The actual resizing runs in a goroutine because we don't want to block
// the distribution of other ResizeInstructions to the rest of the cluster.
@ -1242,7 +1234,7 @@ func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) err
if err := func() error {
// Sync the schema received in the resize instruction.
c.logger().Printf("Holder ApplySchema")
c.Logger.Printf("Holder ApplySchema")
if err := c.Holder.ApplySchema(instr.Schema); err != nil {
return err
}
@ -1252,7 +1244,7 @@ func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) err
// Request each source file in ResizeSources.
for _, src := range instr.Sources {
c.logger().Printf("get slice %d for index %s from host %s", src.Slice, src.Index, src.Node.URI)
c.Logger.Printf("get slice %d for index %s from host %s", src.Slice, src.Index, src.Node.URI)
srcURI := decodeURI(src.Node.URI)
@ -1275,7 +1267,7 @@ func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) err
}
// Stream slice from remote node.
c.logger().Printf("retrieve slice %d for index %s from host %s", src.Slice, src.Index, src.Node.URI)
c.Logger.Printf("retrieve slice %d for index %s from host %s", src.Slice, src.Index, src.Node.URI)
rd, err := client.RetrieveSliceFromURI(context.Background(), src.Index, src.Frame, src.View, src.Slice, srcURI)
if err != nil {
// For now it is an acceptable error if the fragment is not found
@ -1309,7 +1301,7 @@ func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) err
}
if err := c.sendTo(DecodeNode(instr.Coordinator), complete); err != nil {
c.logger().Printf("sending resizeInstructionComplete error: err=%s", err)
c.Logger.Printf("sending resizeInstructionComplete error: err=%s", err)
}
}()
return nil
@ -1363,13 +1355,7 @@ type ResizeJob struct {
mu sync.RWMutex
state string
// The writer for any logging.
LogOutput io.Writer
}
// logger returns a logger for the resize job.
func (j *ResizeJob) logger() *log.Logger {
return log.New(j.LogOutput, "", log.LstdFlags)
Logger Logger
}
// NewResizeJob returns a new instance of ResizeJob.
@ -1397,11 +1383,11 @@ func NewResizeJob(existingNodes []*Node, node *Node, action string) *ResizeJob {
}
return &ResizeJob{
ID: rand.Int63(),
IDs: ids,
action: action,
result: make(chan string),
LogOutput: os.Stderr,
ID: rand.Int63(),
IDs: ids,
action: action,
result: make(chan string),
Logger: NopLogger,
}
}
@ -1425,18 +1411,18 @@ func (j *ResizeJob) setState(state string) {
// Run distributes ResizeInstructions.
func (j *ResizeJob) Run() error {
j.logger().Printf("run ResizeJob")
j.Logger.Printf("run ResizeJob")
// Set job state to RUNNING.
j.SetState(ResizeJobStateRunning)
// Job can be considered done in the case where it doesn't require any action.
if !j.nodesArePending() {
j.logger().Printf("ResizeJob contains no pending tasks; mark as done")
j.Logger.Printf("ResizeJob contains no pending tasks; mark as done")
j.result <- ResizeJobStateDone
return nil
}
j.logger().Printf("distribute tasks for ResizeJob")
j.Logger.Printf("distribute tasks for ResizeJob")
err := j.distributeResizeInstructions()
if err != nil {
j.result <- ResizeJobStateAborted
@ -1466,7 +1452,7 @@ func (j *ResizeJob) nodesArePending() bool {
}
func (j *ResizeJob) distributeResizeInstructions() error {
j.logger().Printf("distributeResizeInstructions for job %d", j.ID)
j.Logger.Printf("distributeResizeInstructions for job %d", j.ID)
// Loop through the ResizeInstructions in ResizeJob and send to each host.
for _, instr := range j.Instructions {
// Because the node may not be in the cluster yet, create
@ -1475,7 +1461,7 @@ func (j *ResizeJob) distributeResizeInstructions() error {
ID: instr.Node.ID,
URI: decodeURI(instr.Node.URI),
}
j.logger().Printf("send resize instructions: %v", instr)
j.Logger.Printf("send resize instructions: %v", instr)
if err := j.Broadcaster.SendTo(node, instr); err != nil {
return err
}
@ -1681,7 +1667,7 @@ func (c *Cluster) ReceiveEvent(e *NodeEvent) error {
switch e.Event {
case NodeJoin:
c.logger().Printf("received NodeJoin event: %v", e)
c.Logger.Printf("received NodeJoin event: %v", e)
// Ignore the event if this is not the coordinator.
if !c.IsCoordinator() {
return nil
@ -1701,7 +1687,7 @@ func (c *Cluster) nodeJoin(node *Node) error {
// A host that is not part of the topology can't be added to the STARTING cluster.
if !c.Topology.ContainsID(node.ID) {
err := fmt.Sprintf("host is not in topology: %s", node.ID)
c.logger().Print(err)
c.Logger.Printf("%v", err)
return errors.New(err)
}
@ -1816,7 +1802,7 @@ func (c *Cluster) nodeLeave(node *Node) error {
}
func (c *Cluster) MergeClusterStatus(cs *internal.ClusterStatus) error {
c.logger().Printf("merge cluster status: %v", cs)
c.Logger.Printf("merge cluster status: %v", cs)
// Ignore status updates from self (coordinator).
if c.IsCoordinator() {
return nil

11
cmd/server.go
View file

@ -17,7 +17,6 @@ package cmd
import (
"fmt"
"io"
"log"
"os"
"os/signal"
"runtime/pprof"
@ -43,14 +42,14 @@ func NewServeCmd(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
Long: `pilosa server runs Pilosa.
It will load existing data from the configured
directory, and start listening client connections
directory and start listening for client connections
on the configured port.`,
RunE: func(cmd *cobra.Command, args []string) error {
logOutput, err := server.GetLogWriter(Server.Config.LogPath, stderr)
if err != nil {
return err
// Set up the logger.
if err := Server.SetupLogger(); err != nil {
return fmt.Errorf("error setting up the logger: %v", err)
}
logger := log.New(logOutput, "", log.LstdFlags)
logger := Server.Server.Logger
logger.Printf("Pilosa %s, build time %s\n", pilosa.Version, pilosa.BuildTime)
// Start CPU profiling.

2
config.go
View file

@ -133,6 +133,7 @@ type Config struct {
MaxWritesPerRequest int `toml:"max-writes-per-request"`
LogPath string `toml:"log-path"`
Verbose bool `toml:"verbose"`
// TLS
TLS TLSConfig
@ -178,6 +179,7 @@ func NewConfig() *Config {
Bind: ":" + DefaultPort,
MaxWritesPerRequest: DefaultMaxWritesPerRequest,
// LogPath: "",
// Verbose: false,
TLS: TLSConfig{},
}

1
ctl/server.go
View file

@ -28,6 +28,7 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) {
flags.StringVarP(&srv.Config.Bind, "bind", "b", srv.Config.Bind, "Default URI on which pilosa should listen.")
flags.IntVarP(&srv.Config.MaxWritesPerRequest, "max-writes-per-request", "", srv.Config.MaxWritesPerRequest, "Number of write commands per request.")
flags.StringVar(&srv.Config.LogPath, "log-path", srv.Config.LogPath, "Log path")
flags.BoolVar(&srv.Config.Verbose, "verbose", srv.Config.Verbose, "Enable verbose logging")
// TLS
SetTLSConfig(flags, &srv.Config.TLS.CertificatePath, &srv.Config.TLS.CertificateKeyPath, &srv.Config.TLS.SkipVerify)

21
diagnostics.go
View file

@ -18,9 +18,6 @@ import (
"bytes"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"log"
"net/http"
"strconv"
"strings"
@ -52,7 +49,7 @@ type DiagnosticsCollector struct {
client *http.Client
logOutput io.Writer
Logger Logger
server *Server
}
@ -66,7 +63,7 @@ func NewDiagnosticsCollector(host string) *DiagnosticsCollector {
start: time.Now(),
client: &http.Client{Timeout: 10 * time.Second},
metrics: make(map[string]interface{}),
logOutput: ioutil.Discard,
Logger: NopLogger,
}
}
@ -119,7 +116,7 @@ func (d *DiagnosticsCollector) CheckVersion() error {
d.lastVersion = rsp.Version
if err := d.compareVersion(rsp.Version); err != nil {
d.logger().Printf("%s\n", err.Error())
d.Logger.Printf("%s\n", err.Error())
}
return nil
@ -160,20 +157,10 @@ func (d *DiagnosticsCollector) Set(name string, value interface{}) {
d.metrics[name] = value
}
// SetLogger Set the logger output type.
func (d *DiagnosticsCollector) SetLogger(logger io.Writer) {
d.logOutput = logger
}
// logger returns a logger that writes to LogOutput.
func (d *DiagnosticsCollector) logger() *log.Logger {
return log.New(d.logOutput, "", log.LstdFlags)
}
// logErr logs the error and returns true if an error exists
func (d *DiagnosticsCollector) logErr(err error) bool {
if err != nil {
d.logOutput.Write([]byte(err.Error()))
d.Logger.Printf("%v", err)
return true
}
return false

3
diagnostics_internal_test.go
View file

@ -16,7 +16,6 @@ package pilosa
import (
"encoding/json"
"io/ioutil"
"net/http"
"net/http/httptest"
"reflect"
@ -31,7 +30,6 @@ func TestDiagnosticsClient(t *testing.T) {
// Create a new client.
d := NewDiagnosticsCollector(server.URL)
d.SetLogger(ioutil.Discard)
d.Set("gg", 10)
d.Set("ss", "ss")
@ -146,7 +144,6 @@ func BenchmarkDiagnostics(b *testing.B) {
// Create a new client.
d := NewDiagnosticsCollector(server.URL)
d.SetLogger(ioutil.Discard)
prev := runtime.GOMAXPROCS(4)
defer runtime.GOMAXPROCS(prev)

4
docs/api-reference.md
View file

@ -215,7 +215,7 @@ The request payload is JSON, and it must contain the fields `frames` and `fields
* `cacheType` (string): [ranked](../data-model/#ranked) or [LRU](../data-model/#lru) caching on this frame. Default is `lru`.
* `cacheSize` (int): Number of rows to keep in the cache. Default 50,000.
The `fields` array contains a series of JSON objects describing how to process each field received in the input data. Each `field` object must contain a `name` which maps to the source JSON field name. One field must be defined at the `primaryKey`. The `primarykey` source field name must equal the column label for the `Index`, and its value must be an unsigned integer which maps directly to a columnID in Pilosa.
The `fields` array contains a series of JSON objects describing how to process each field received in the input data. Each `field` object must contain a `name` which maps to the source JSON field name. One field must be defined at the `primaryKey`. The `primarykey` source field's value must be an unsigned integer which maps directly to a columnID in Pilosa.
* `name` (string): Maps the source data field to actions that process the field's corresponding value.
* `actions` (array): List of actions that will process the field's value.
@ -310,7 +310,7 @@ Input definition is deprecated as of v0.9.
Processes the JSON payload using the given input definition.
The request payload is a JSON array of objects containing one field for the primary key that corresponds to the column label, and additional fields that will be handled by corresponding actions in the input definition.
The request payload is a JSON array of objects containing one field for the primary key that corresponds to the column, and additional fields that will be handled by corresponding actions in the input definition.
``` request
curl localhost:10101/index/user/input/stargazer-input \

11
docs/configuration.md
View file

@ -81,6 +81,17 @@ The config file is in the [toml format](https://github.com/toml-lang/toml) and h
log-path = "/path/to/logfile"
```
#### Verbose
* Description: Enable verbose logging.
* Flag: `--verbose`
* Env: `PILOSA_VERBOSE`
* Config:
```toml
verbose = true
```
#### Max Writes Per Request
* Description: Maximum number of mutating commands allowed per request. This includes SetBit, ClearBit, SetRowAttrs, SetColumnAttrs, and SetFieldValue.

11
docs/examples.md
View file

@ -267,12 +267,11 @@ First, follow the instruction in the [getting started](../getting-started/) guid
The option cacheSize should be set as amount of chembl_id to calculate effectively for the whole data set, so we need to calculate amount of chembl_id. We have total 1678393 chembl_id (it will displayed after import_from_sdf.py script running), then the cacheSize should be >= 1678393
```
curl localhost:10101/index/mole \
-X POST \
-d '{"options": {"columnLabel": "position_id"}}'
-X POST
curl localhost:10101/index/mole/frame/fingerprint \
-X POST \
-d '{"options": {"rowLabel": "chembl_id", "inverseEnabled": true, "cacheSize": 2000000, "cacheType": "ranked"}}'
-d '{"options": {"inverseEnabled": true, "cacheSize": 2000000, "cacheType": "ranked"}}'
```
@ -302,7 +301,7 @@ Return chembl_id = 6223. This script uses Pilosa’s Intersection query to get a
* Query all chembl_id that have all "on" positions from the inverse view, return list of chembl_id
```python
bit_maps = ["Bitmap(position_id=%s, frame=%s, inversed=%s)" % (f, frame, True) for f in fp]
bit_maps = ["Bitmap(col=%s, frame=%s, inversed=%s)" % (f, frame, True) for f in fp]
bitmap_string = ', '.join(bit_maps)
intersection = "Intersect(%s)" % bitmap_string
mole_ids = requests.post("http://%s/index/%s/query" % (host, db), data=intersection).json()["results"][0]["bits"]
@ -312,7 +311,7 @@ Return chembl_id = 6223. This script uses Pilosa’s Intersection query to get a
```python
for m in mole_ids:
mol = requests.post("http://%s/index/%s/query" % (host, db), data="Bitmap(chembl_id=%s, frame=%s)" % (m, frame)).json()["results"][0]["bits"]
mol = requests.post("http://%s/index/%s/query" % (host, db), data="Bitmap(row=%s, frame=%s)" % (m, frame)).json()["results"][0]["bits"]
existed_mol = False
if len(mol) == len(fp):
found = m
@ -331,7 +330,7 @@ Return chembl_id = [6223, 269758, 6206, 6228]. This script uses Pilosa’s TopN
* Query Pilosa’s TopN to get list of similarity chembl_id
```python
query_string = 'TopN(Bitmap(chembl_id=6223, frame="fingerprint"), frame="fingerprint", n=2000000, tanimotoThreshold=70)'
query_string = 'TopN(Bitmap(row=6223, frame="fingerprint"), frame="fingerprint", n=2000000, tanimotoThreshold=70)'
topn = requests.post("http://127.0.0.1:10101/index/mol/query" , data=query_string)
```

10
docs/installation.md
View file

@ -134,6 +134,10 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/)
#### Build from Source
<div class="note">
<p>For advanced instructions for building from source, view our <a href="https://github.com/pilosa/pilosa/blob/master/CONTRIBUTING.md">Contributor's Guide.</a></p>
</div>
1. Install the prerequisites:
* [Go](https://golang.org/doc/install). Be sure to set the `$GOPATH` and `$PATH` environment variables as described [here](https://golang.org/doc/code.html#GOPATH).
@ -148,6 +152,7 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/)
3. Build the Pilosa repo (the `make generate-statik` line isn't necessary but builds a nice [webUI](../webui/) into Pilosa):
```
cd $GOPATH/src/github.com/pilosa/pilosa
make install-build-deps
make generate-statik
make install
```
@ -288,6 +293,10 @@ There are three ways to install Pilosa on Linux: download the binary (recommende
#### Build from Source
<div class="note">
<p>For advanced instructions for building from source, view our <a href="https://github.com/pilosa/pilosa/blob/master/CONTRIBUTING.md">Contributor's Guide.</a></p>
</div>
1. Install the prerequisites:
* [Go](https://golang.org/doc/install). Be sure to set the `$GOPATH` and `$PATH` environment variables as described [here](https://golang.org/doc/code.html#GOPATH).
@ -302,6 +311,7 @@ There are three ways to install Pilosa on Linux: download the binary (recommende
3. Build the Pilosa repo (the `make generate-statik` line isn't necessary but builds a nice [webUI](../webui/) into Pilosa):
```
cd $GOPATH/src/github.com/pilosa/pilosa
make install-build-deps
make generate-statik
make install
```

6
docs/query-language.md
View file

@ -21,8 +21,6 @@ This section will provide a detailed reference and examples for the Pilosa Query
There will be one item in the `results` array for each PQL query in the request. The type of each item in the array will depend on the type of query - each query in the reference below lists it's result type.
The default row label is `rowID`, and the default column label is `columnID`. Changing these defaults is deprecated and this feature will be removed in a future release.
#### Conventions
* Angle Brackets `<>` denote required arguments
@ -46,8 +44,6 @@ curl localhost:10101/index/repository/query \
#### Arguments and Types
* `frame` The frame specifies on which Pilosa [frame](../glossary/#frame) the query will operate. Valid frame names are lower case strings; they start with an alphanumeric character, and contain only alphanumeric characters and `_-`. They must be 64 characters or less in length.
* `ROW_LABEL` The default row label is `rowID`, changing the default is deprecated.
* `COL_LABEL` The default column label is `columnID`, changing the default is deprecated.
* `TIMESTAMP` This is a timestamp in quotes with the following format `"YYYY-MM-DDTHH:MM"` (e.g. "2006-01-02T15:04")
* `UINT` An unsigned integer (e.g. 42839)
* `ATTR_NAME` Must be a valid identifier `[A-Za-z][A-Za-z0-9._-]*`
@ -223,7 +219,7 @@ Bitmap(<frame=STRING>, (<ROW_LABEL=UINT> | <COL_LABEL>=UINT))
**Description:**
`Bitmap` retrieves the indices of all the set bits in a row or column based on whether the row label or column label is given in the query. It also retrieves any attributes set on that row or column.
`Bitmap` retrieves the indices of all the set bits in a row or column based on whether the row or column argument is provided in the query. It also retrieves any attributes set on that row or column.
**Result Type:** object with attrs and bits.

52
executor.go
View file

@ -33,6 +33,9 @@ const (
// MinThreshold is the lowest count to use in a Top-N operation when
// looking for additional id/count pairs.
MinThreshold = 1
columnLabel = "col"
rowLabel = "row"
)
// Executor recursively executes calls in a PQL query across all slices.
@ -80,8 +83,6 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic
// MaxSlice can differ between inverse and standard views, so we need
// to send queries to different slices based on orientation.
var inverseSlices []uint64
rowLabel := DefaultRowLabel
columnLabel := DefaultColumnLabel
// If slices aren't specified, then include all of them.
if len(slices) == 0 {
@ -106,9 +107,6 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic
for i := range inverseSlices {
inverseSlices[i] = uint64(i)
}
// Fetch column label from index.
columnLabel = idx.ColumnLabel()
}
}
@ -131,7 +129,6 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic
if f == nil {
return nil, ErrFrameNotFound
}
rowLabel = f.RowLabel()
// If this call is to an inverse frame send to a different list of slices.
if call.IsInverse(rowLabel, columnLabel) {
@ -268,7 +265,6 @@ func (e *Executor) executeBitmapCall(ctx context.Context, index string, c *pql.C
} else {
idx := e.Holder.Index(index)
if idx != nil {
columnLabel := idx.ColumnLabel()
if columnID, ok, err := c.UintArg(columnLabel); ok && err == nil {
attrs, err := idx.ColumnAttrStore().Attrs(columnID)
if err != nil {
@ -280,7 +276,6 @@ func (e *Executor) executeBitmapCall(ctx context.Context, index string, c *pql.C
} else {
frame, _ := c.Args["frame"].(string)
if fr := idx.Frame(frame); fr != nil {
rowLabel := fr.RowLabel()
rowID, _, err := c.UintArg(rowLabel)
if err != nil {
return nil, err
@ -525,7 +520,6 @@ func (e *Executor) executeBitmapSlice(ctx context.Context, index string, c *pql.
if idx == nil {
return nil, ErrIndexNotFound
}
columnLabel := idx.ColumnLabel()
// Fetch frame & row label based on argument.
frame, _ := c.Args["frame"].(string)
@ -536,7 +530,6 @@ func (e *Executor) executeBitmapSlice(ctx context.Context, index string, c *pql.
if f == nil {
return nil, ErrFrameNotFound
}
rowLabel := f.RowLabel()
// Return an error if both the row and column label are specified.
rowID, rowOK, rowErr := c.UintArg(rowLabel)
@ -606,14 +599,12 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C
if idx == nil {
return nil, ErrIndexNotFound
}
columnLabel := idx.ColumnLabel()
// Retrieve base frame.
f := idx.Frame(frame)
if f == nil {
return nil, ErrFrameNotFound
}
rowLabel := f.RowLabel()
// Read row & column id.
columnID, columnOK, err := c.UintArg(columnLabel)
@ -904,10 +895,6 @@ func (e *Executor) executeClearBit(ctx context.Context, index string, c *pql.Cal
return false, ErrFrameNotFound
}
// Retrieve labels.
columnLabel := idx.ColumnLabel()
rowLabel := f.RowLabel()
// Read fields using labels.
rowID, ok, err := c.UintArg(rowLabel)
if err != nil {
@ -998,10 +985,6 @@ func (e *Executor) executeSetBit(ctx context.Context, index string, c *pql.Call,
return false, ErrFrameNotFound
}
// Retrieve labels.
columnLabel := idx.ColumnLabel()
rowLabel := f.RowLabel()
// Read fields using labels.
rowID, ok, err := c.UintArg(rowLabel)
if err != nil {
@ -1093,13 +1076,6 @@ func (e *Executor) executeSetFieldValue(ctx context.Context, index string, c *pq
return errors.New("SetFieldValue() frame required")
}
// Retrieve column label.
idx := e.Holder.Index(index)
if idx == nil {
return ErrIndexNotFound
}
columnLabel := idx.ColumnLabel()
// Retrieve frame.
frame := e.Holder.Frame(index, frameName)
if frame == nil {
@ -1171,7 +1147,6 @@ func (e *Executor) executeSetRowAttrs(ctx context.Context, index string, c *pql.
if frame == nil {
return ErrFrameNotFound
}
rowLabel := frame.RowLabel()
// Parse labels.
rowID, ok, err := c.UintArg(rowLabel)
@ -1232,7 +1207,6 @@ func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, index string, cal
if f == nil {
return nil, ErrFrameNotFound
}
rowLabel := f.RowLabel()
rowID, ok, err := c.UintArg(rowLabel)
if err != nil {
@ -1313,28 +1287,18 @@ func (e *Executor) executeSetColumnAttrs(ctx context.Context, index string, c *p
return ErrIndexNotFound
}
var colName string
id, okID, errID := c.UintArg("id")
if errID != nil || !okID {
// Retrieve columnLabel
columnLabel := idx.columnLabel
col, okCol, errCol := c.UintArg(columnLabel)
if errCol != nil || !okCol {
return fmt.Errorf("reading SetColumnAttrs() id/columnLabel errs: %v/%v found %v/%v", errID, errCol, okID, okCol)
}
id = col
colName = columnLabel
} else {
colName = "id"
col, okCol, errCol := c.UintArg(columnLabel)
if errCol != nil || !okCol {
return fmt.Errorf("reading SetColumnAttrs() col errs: %v found %v", errCol, okCol)
}
// Copy args and remove reserved fields.
attrs := pql.CopyArgs(c.Args)
delete(attrs, colName)
delete(attrs, columnLabel)
delete(attrs, "frame")
// Set attributes.
if err := idx.ColumnAttrStore().SetAttrs(id, attrs); err != nil {
if err := idx.ColumnAttrStore().SetAttrs(col, attrs); err != nil {
return err
}
idx.Stats.Count("SetProfileAttrs", 1, 1.0)

158
executor_test.go
View file

@ -42,9 +42,9 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
// Set bits.
if _, err := e.Execute(context.Background(), "i", test.MustParse(``+
fmt.Sprintf("SetBit(frame=f, rowID=%d, columnID=%d)\n", 10, 3)+
fmt.Sprintf("SetBit(frame=f, rowID=%d, columnID=%d)\n", 10, SliceWidth+1)+
fmt.Sprintf("SetBit(frame=f, rowID=%d, columnID=%d)\n", 20, SliceWidth+1),
fmt.Sprintf("SetBit(frame=f, row=%d, col=%d)\n", 10, 3)+
fmt.Sprintf("SetBit(frame=f, row=%d, col=%d)\n", 10, SliceWidth+1)+
fmt.Sprintf("SetBit(frame=f, row=%d, col=%d)\n", 20, SliceWidth+1),
), nil, nil); err != nil {
t.Fatal(err)
}
@ -52,7 +52,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
t.Fatal(err)
}
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(rowID=10, frame=f)`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{3, SliceWidth + 1}) {
t.Fatalf("unexpected bits: %+v", bits)
@ -61,7 +61,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
}
// Inhibit bits.
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(rowID=10, frame=f)`), nil, &pilosa.ExecOptions{ExcludeBits: true}); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, &pilosa.ExecOptions{ExcludeBits: true}); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{}) {
t.Fatalf("unexpected bits: %+v", bits)
@ -70,7 +70,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
}
// Inhibit attributes.
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(rowID=10, frame=f)`), nil, &pilosa.ExecOptions{ExcludeAttrs: true}); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, &pilosa.ExecOptions{ExcludeAttrs: true}); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{3, SliceWidth + 1}) {
t.Fatalf("unexpected bits: %+v", bits)
@ -91,9 +91,9 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
// Set bits.
if _, err := e.Execute(context.Background(), "i", test.MustParse(``+
fmt.Sprintf("SetBit(frame=f, rowID=%d, columnID=%d)\n", 10, 3)+
fmt.Sprintf("SetBit(frame=f, rowID=%d, columnID=%d)\n", 10, SliceWidth+1)+
fmt.Sprintf("SetBit(frame=f, rowID=%d, columnID=%d)\n", 20, SliceWidth+1),
fmt.Sprintf("SetBit(frame=f, row=%d, col=%d)\n", 10, 3)+
fmt.Sprintf("SetBit(frame=f, row=%d, col=%d)\n", 10, SliceWidth+1)+
fmt.Sprintf("SetBit(frame=f, row=%d, col=%d)\n", 20, SliceWidth+1),
), nil, nil); err != nil {
t.Fatal(err)
}
@ -101,7 +101,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
t.Fatal(err)
}
if res, err := e.Execute(context.Background(), "i", test.MustParse(fmt.Sprintf(`Bitmap(columnID=%d, frame=f)`, SliceWidth+1)), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(fmt.Sprintf(`Bitmap(col=%d, frame=f)`, SliceWidth+1)), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{10, 20}) {
t.Fatalf("unexpected bits: %+v", bits)
@ -122,7 +122,7 @@ func TestExecutor_Execute_Difference(t *testing.T) {
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 4)
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Difference(Bitmap(rowID=10), Bitmap(rowID=11))`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Difference(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{1, 3}) {
t.Fatalf("unexpected bits: %+v", bits)
@ -154,7 +154,7 @@ func TestExecutor_Execute_Intersect(t *testing.T) {
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2)
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Intersect(Bitmap(rowID=10), Bitmap(rowID=11))`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Intersect(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{1, SliceWidth + 2}) {
t.Fatalf("unexpected bits: %+v", bits)
@ -184,7 +184,7 @@ func TestExecutor_Execute_Union(t *testing.T) {
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2)
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Union(Bitmap(rowID=10), Bitmap(rowID=11))`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Union(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{0, 2, SliceWidth + 1, SliceWidth + 2}) {
t.Fatalf("unexpected bits: %+v", bits)
@ -217,7 +217,7 @@ func TestExecutor_Execute_Xor(t *testing.T) {
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2)
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Xor(Bitmap(rowID=10), Bitmap(rowID=11))`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Xor(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{0, 2, SliceWidth + 1}) {
t.Fatalf("unexpected bits: %+v", bits)
@ -233,7 +233,7 @@ func TestExecutor_Execute_Count(t *testing.T) {
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2)
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Bitmap(rowID=10, frame=f))`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Bitmap(row=10, frame=f))`), nil, nil); err != nil {
t.Fatal(err)
} else if res[0] != uint64(3) {
t.Fatalf("unexpected n: %d", res[0])
@ -251,7 +251,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) {
t.Fatalf("unexpected bitmap count: %d", n)
}
if res, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(rowID=11, frame=f, columnID=1)`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=11, frame=f, col=1)`), nil, nil); err != nil {
t.Fatal(err)
} else {
if !res[0].(bool) {
@ -262,7 +262,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) {
if n := f.Row(11).Count(); n != 1 {
t.Fatalf("unexpected bitmap count: %d", n)
}
if res, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(rowID=11, frame=f, columnID=1)`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=11, frame=f, col=1)`), nil, nil); err != nil {
t.Fatal(err)
} else {
if res[0].(bool) {
@ -293,9 +293,9 @@ func TestExecutor_Execute_SetFieldValue(t *testing.T) {
// Set field values.
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(columnID=10, frame=f, field0=25, field1=2)`), nil, nil); err != nil {
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(col=10, frame=f, field0=25, field1=2)`), nil, nil); err != nil {
t.Fatal(err)
} else if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(columnID=100, frame=f, field0=10)`), nil, nil); err != nil {
} else if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(col=100, frame=f, field0=10)`), nil, nil); err != nil {
t.Fatal(err)
}
@ -340,28 +340,28 @@ func TestExecutor_Execute_SetFieldValue(t *testing.T) {
t.Run("ErrFrameRequired", func(t *testing.T) {
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(columnID=10, field0=100)`), nil, nil); err == nil || err.Error() != `SetFieldValue() frame required` {
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(col=10, field0=100)`), nil, nil); err == nil || err.Error() != `SetFieldValue() frame required` {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrColumnFieldRequired", func(t *testing.T) {
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(invalid_column_name=10, frame=f, field0=100)`), nil, nil); err == nil || err.Error() != `SetFieldValue() column field 'columnID' required` {
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(invalid_column_name=10, frame=f, field0=100)`), nil, nil); err == nil || err.Error() != `SetFieldValue() column field 'col' required` {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrColumnFieldValue", func(t *testing.T) {
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(invalid_column_name="bad_column", frame=f, field0=100)`), nil, nil); err == nil || err.Error() != `SetFieldValue() column field 'columnID' required` {
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(invalid_column_name="bad_column", frame=f, field0=100)`), nil, nil); err == nil || err.Error() != `SetFieldValue() column field 'col' required` {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrInvalidFieldValueType", func(t *testing.T) {
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(columnID=10, frame=f, field0="hello")`), nil, nil); err == nil || err.Error() != `invalid field value type` {
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(col=10, frame=f, field0="hello")`), nil, nil); err == nil || err.Error() != `invalid field value type` {
t.Fatalf("unexpected error: %s", err)
}
})
@ -384,16 +384,16 @@ func TestExecutor_Execute_SetRowAttrs(t *testing.T) {
// Set two fields on f/10.
// Also set fields on other bitmaps and frames to test isolation.
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(rowID=10, frame=f, foo="bar")`), nil, nil); err != nil {
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(row=10, frame=f, foo="bar")`), nil, nil); err != nil {
t.Fatal(err)
}
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(rowID=200, frame=f, YYY=1)`), nil, nil); err != nil {
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(row=200, frame=f, YYY=1)`), nil, nil); err != nil {
t.Fatal(err)
}
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(rowID=10, frame=xxx, YYY=1)`), nil, nil); err != nil {
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(row=10, frame=xxx, YYY=1)`), nil, nil); err != nil {
t.Fatal(err)
}
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(rowID=10, frame=f, baz=123, bat=true)`), nil, nil); err != nil {
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(row=10, frame=f, baz=123, bat=true)`), nil, nil); err != nil {
t.Fatal(err)
}
@ -419,15 +419,15 @@ func TestExecutor_Execute_TopN(t *testing.T) {
} else if _, err := idx.CreateFrame("other", pilosa.FrameOptions{InverseEnabled: true}); err != nil {
t.Fatal(err)
} else if _, err := e.Execute(context.Background(), "i", test.MustParse(`
SetBit(frame=f, rowID=0, columnID=0)
SetBit(frame=f, rowID=0, columnID=1)
SetBit(frame=f, rowID=0, columnID=`+strconv.Itoa(SliceWidth)+`)
SetBit(frame=f, rowID=0, columnID=`+strconv.Itoa(SliceWidth+2)+`)
SetBit(frame=f, rowID=0, columnID=`+strconv.Itoa((5*SliceWidth)+100)+`)
SetBit(frame=f, rowID=10, columnID=0)
SetBit(frame=f, rowID=10, columnID=`+strconv.Itoa(SliceWidth)+`)
SetBit(frame=f, rowID=20, columnID=`+strconv.Itoa(SliceWidth)+`)
SetBit(frame=other, rowID=0, columnID=0)
SetBit(frame=f, row=0, col=0)
SetBit(frame=f, row=0, col=1)
SetBit(frame=f, row=0, col=`+strconv.Itoa(SliceWidth)+`)
SetBit(frame=f, row=0, col=`+strconv.Itoa(SliceWidth+2)+`)
SetBit(frame=f, row=0, col=`+strconv.Itoa((5*SliceWidth)+100)+`)
SetBit(frame=f, row=10, col=0)
SetBit(frame=f, row=10, col=`+strconv.Itoa(SliceWidth)+`)
SetBit(frame=f, row=20, col=`+strconv.Itoa(SliceWidth)+`)
SetBit(frame=other, row=0, col=0)
`), nil, nil); err != nil {
t.Fatal(err)
}
@ -543,7 +543,7 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) {
// Execute query.
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(Bitmap(rowID=100, frame=other), frame=f, n=3)`), nil, nil); err != nil {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(Bitmap(row=100, frame=other), frame=f, n=3)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{
{ID: 20, Count: 3},
@ -590,7 +590,7 @@ func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) {
t.Fatal(err)
}
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(Bitmap(rowID=10,frame=f),frame="f", n=1, field="category", filters=[123])`), nil, nil); err != nil {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(Bitmap(row=10,frame=f),frame="f", n=1, field="category", filters=[123])`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{
{ID: 10, Count: 1},
@ -630,15 +630,15 @@ func TestExecutor_Execute_Sum(t *testing.T) {
}
if _, err := e.Execute(context.Background(), "i", test.MustParse(`
SetBit(frame=f, rowID=0, columnID=0)
SetBit(frame=f, rowID=0, columnID=`+strconv.Itoa(SliceWidth+1)+`)
SetBit(frame=f, row=0, col=0)
SetBit(frame=f, row=0, col=`+strconv.Itoa(SliceWidth+1)+`)
SetFieldValue(frame=f, foo=20, bar=2000, columnID=0)
SetFieldValue(frame=f, foo=30, columnID=`+strconv.Itoa(SliceWidth)+`)
SetFieldValue(frame=f, foo=40, columnID=`+strconv.Itoa(SliceWidth+2)+`)
SetFieldValue(frame=f, foo=50, columnID=`+strconv.Itoa((5*SliceWidth)+100)+`)
SetFieldValue(frame=f, foo=60, columnID=`+strconv.Itoa(SliceWidth+1)+`)
SetFieldValue(frame=other, foo=1000, columnID=0)
SetFieldValue(frame=f, foo=20, bar=2000, col=0)
SetFieldValue(frame=f, foo=30, col=`+strconv.Itoa(SliceWidth)+`)
SetFieldValue(frame=f, foo=40, col=`+strconv.Itoa(SliceWidth+2)+`)
SetFieldValue(frame=f, foo=50, col=`+strconv.Itoa((5*SliceWidth)+100)+`)
SetFieldValue(frame=f, foo=60, col=`+strconv.Itoa(SliceWidth+1)+`)
SetFieldValue(frame=other, foo=1000, col=0)
`), nil, nil); err != nil {
t.Fatal(err)
}
@ -652,7 +652,7 @@ func TestExecutor_Execute_Sum(t *testing.T) {
})
t.Run("WithFilter", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Sum(Bitmap(frame=f, rowID=0), frame=f, field=foo)`), nil, nil); err != nil {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Sum(Bitmap(frame=f, row=0), frame=f, field=foo)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result[0], pilosa.SumCount{Sum: 80, Count: 2}) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
@ -679,22 +679,22 @@ func TestExecutor_Execute_Range(t *testing.T) {
// Set bits.
if _, err := e.Execute(context.Background(), "i", test.MustParse(`
SetBit(frame=f, rowID=1, columnID=2, timestamp="1999-12-31T00:00")
SetBit(frame=f, rowID=1, columnID=3, timestamp="2000-01-01T00:00")
SetBit(frame=f, rowID=1, columnID=4, timestamp="2000-01-02T00:00")
SetBit(frame=f, rowID=1, columnID=5, timestamp="2000-02-01T00:00")
SetBit(frame=f, rowID=1, columnID=6, timestamp="2001-01-01T00:00")
SetBit(frame=f, rowID=1, columnID=7, timestamp="2002-01-01T02:00")
SetBit(frame=f, row=1, col=2, timestamp="1999-12-31T00:00")
SetBit(frame=f, row=1, col=3, timestamp="2000-01-01T00:00")
SetBit(frame=f, row=1, col=4, timestamp="2000-01-02T00:00")
SetBit(frame=f, row=1, col=5, timestamp="2000-02-01T00:00")
SetBit(frame=f, row=1, col=6, timestamp="2001-01-01T00:00")
SetBit(frame=f, row=1, col=7, timestamp="2002-01-01T02:00")
SetBit(frame=f, rowID=1, columnID=2, timestamp="1999-12-30T00:00")
SetBit(frame=f, rowID=1, columnID=2, timestamp="2002-02-01T00:00")
SetBit(frame=f, rowID=10, columnID=2, timestamp="2001-01-01T00:00")
SetBit(frame=f, row=1, col=2, timestamp="1999-12-30T00:00")
SetBit(frame=f, row=1, col=2, timestamp="2002-02-01T00:00")
SetBit(frame=f, row=10, col=2, timestamp="2001-01-01T00:00")
`), nil, nil); err != nil {
t.Fatal(err)
}
t.Run("Standard", func(t *testing.T) {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Range(rowID=1, frame=f, start="1999-12-31T00:00", end="2002-01-01T03:00")`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Range(row=1, frame=f, start="1999-12-31T00:00", end="2002-01-01T03:00")`), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{2, 3, 4, 5, 6, 7}) {
t.Fatalf("unexpected bits: %+v", bits)
@ -703,7 +703,7 @@ func TestExecutor_Execute_Range(t *testing.T) {
t.Run("Inverse", func(t *testing.T) {
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Range(columnID=2, frame=f, start="1999-01-01T00:00", end="2003-01-01T00:00")`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Range(col=2, frame=f, start="1999-01-01T00:00", end="2003-01-01T00:00")`), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{1, 10}) {
t.Fatalf("unexpected bits: %+v", bits)
@ -751,17 +751,17 @@ func TestExecutor_Execute_FieldRange(t *testing.T) {
}
if _, err := e.Execute(context.Background(), "i", test.MustParse(`
SetBit(frame=f, rowID=0, columnID=0)
SetBit(frame=f, rowID=0, columnID=`+strconv.Itoa(SliceWidth+1)+`)
SetBit(frame=f, row=0, col=0)
SetBit(frame=f, row=0, col=`+strconv.Itoa(SliceWidth+1)+`)
SetFieldValue(frame=f, foo=20, bar=2000, columnID=50)
SetFieldValue(frame=f, foo=30, columnID=`+strconv.Itoa(SliceWidth)+`)
SetFieldValue(frame=f, foo=10, columnID=`+strconv.Itoa(SliceWidth+2)+`)
SetFieldValue(frame=f, foo=20, columnID=`+strconv.Itoa((5*SliceWidth)+100)+`)
SetFieldValue(frame=f, foo=60, columnID=`+strconv.Itoa(SliceWidth+1)+`)
SetFieldValue(frame=other, foo=1000, columnID=0)
SetFieldValue(frame=edge, foo=100, columnID=0)
SetFieldValue(frame=edge, foo=-100, columnID=1)
SetFieldValue(frame=f, foo=20, bar=2000, col=50)
SetFieldValue(frame=f, foo=30, col=`+strconv.Itoa(SliceWidth)+`)
SetFieldValue(frame=f, foo=10, col=`+strconv.Itoa(SliceWidth+2)+`)
SetFieldValue(frame=f, foo=20, col=`+strconv.Itoa((5*SliceWidth)+100)+`)
SetFieldValue(frame=f, foo=60, col=`+strconv.Itoa(SliceWidth+1)+`)
SetFieldValue(frame=other, foo=1000, col=0)
SetFieldValue(frame=edge, foo=100, col=0)
SetFieldValue(frame=edge, foo=-100, col=1)
`), nil, nil); err != nil {
t.Fatal(err)
}
@ -908,7 +908,7 @@ func TestExecutor_Execute_Remote_Bitmap(t *testing.T) {
s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
if index != "i" {
t.Fatalf("unexpected index: %s", index)
} else if query.String() != `Bitmap(frame="f", rowID=10)` {
} else if query.String() != `Bitmap(frame="f", row=10)` {
t.Fatalf("unexpected query: %s", query.String())
} else if !reflect.DeepEqual(slices, []uint64{1}) {
t.Fatalf("unexpected slices: %+v", slices)
@ -931,7 +931,7 @@ func TestExecutor_Execute_Remote_Bitmap(t *testing.T) {
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(10, (1*SliceWidth)+1)
e := test.NewExecutor(hldr.Holder, c)
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(rowID=10, frame=f)`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{1, 2, 2*SliceWidth + 4}) {
t.Fatalf("unexpected bits: %+v", bits)
@ -966,7 +966,7 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) {
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(10, (2*SliceWidth)+2)
e := test.NewExecutor(hldr.Holder, c)
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Bitmap(rowID=10, frame=f))`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Bitmap(row=10, frame=f))`), nil, nil); err != nil {
t.Fatal(err)
} else if res[0] != uint64(12) {
t.Fatalf("unexpected n: %d", res[0])
@ -994,7 +994,7 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) {
s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
if index != `i` {
t.Fatalf("unexpected index: %s", index)
} else if query.String() != `SetBit(columnID=2, frame="f", rowID=10)` {
} else if query.String() != `SetBit(col=2, frame="f", row=10)` {
t.Fatalf("unexpected query: %s", query.String())
}
remoteCalled = true
@ -1012,7 +1012,7 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) {
}
e := test.NewExecutor(hldr.Holder, c)
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(rowID=10, frame=f, columnID=2)`), nil, nil); err != nil {
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=10, frame=f, col=2)`), nil, nil); err != nil {
t.Fatal(err)
}
@ -1046,7 +1046,7 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) {
s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
if index != `i` {
t.Fatalf("unexpected index: %s", index)
} else if query.String() != `SetBit(columnID=2, frame="f", rowID=10, timestamp="2016-12-11T10:09")` {
} else if query.String() != `SetBit(col=2, frame="f", row=10, timestamp="2016-12-11T10:09")` {
t.Fatalf("unexpected query: %s", query.String())
}
remoteCalled = true
@ -1066,7 +1066,7 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) {
}
e := test.NewExecutor(hldr.Holder, c)
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(rowID=10, frame=f, columnID=2, timestamp="2016-12-11T10:09")`), nil, nil); err != nil {
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=10, frame=f, col=2, timestamp="2016-12-11T10:09")`), nil, nil); err != nil {
t.Fatal(err)
}
@ -1169,11 +1169,11 @@ func TestExectutor_SetColumnAttrs_ExcludeFrame(t *testing.T) {
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
// SetColumnAttrs call should exclude the frame attribute
_, err := e.Execute(context.Background(), "i", test.MustParse("SetBit(frame='f', rowID=1, columnID=10)"), nil, nil)
_, err := e.Execute(context.Background(), "i", test.MustParse("SetBit(frame='f', row=1, col=10)"), nil, nil)
if err != nil {
t.Fatal(err)
}
_, err = e.Execute(context.Background(), "i", test.MustParse("SetColumnAttrs(frame='f', columnID=10, foo='bar')"), nil, nil)
_, err = e.Execute(context.Background(), "i", test.MustParse("SetColumnAttrs(frame='f', col=10, foo='bar')"), nil, nil)
if err != nil {
t.Fatal(err)
}
@ -1186,11 +1186,11 @@ func TestExectutor_SetColumnAttrs_ExcludeFrame(t *testing.T) {
}
// SetColumnAttrs call should not break if frame is not specified
_, err = e.Execute(context.Background(), "i", test.MustParse("SetBit(frame='f', rowID=1, columnID=20)"), nil, nil)
_, err = e.Execute(context.Background(), "i", test.MustParse("SetBit(frame='f', row=1, col=20)"), nil, nil)
if err != nil {
t.Fatal(err)
}
_, err = e.Execute(context.Background(), "i", test.MustParse("SetColumnAttrs(columnID=20, foo='bar')"), nil, nil)
_, err = e.Execute(context.Background(), "i", test.MustParse("SetColumnAttrs(col=20, foo='bar')"), nil, nil)
if err != nil {
t.Fatal(err)
}

29
fragment.go
View file

@ -26,7 +26,6 @@ import (
"hash"
"io"
"io/ioutil"
"log"
"net/http"
"os"
"sort"
@ -103,8 +102,8 @@ type Fragment struct {
// so that they can be mmapped and heap utilization can be kept low.
MaxOpN int
// Writer used for out-of-band log entries.
LogOutput io.Writer
// Logger used for out-of-band log entries.
Logger Logger
// Row attribute storage.
// This is set by the parent frame unless overridden for testing.
@ -124,8 +123,8 @@ func NewFragment(path, index, frame, view string, slice uint64) *Fragment {
CacheType: DefaultCacheType,
CacheSize: DefaultCacheSize,
LogOutput: ioutil.Discard,
MaxOpN: DefaultFragmentMaxOpN,
Logger: NopLogger,
MaxOpN: DefaultFragmentMaxOpN,
stats: NopStatsClient,
}
@ -273,7 +272,7 @@ func (f *Fragment) openCache() error {
// Unmarshal cache data.
var pb internal.Cache
if err := proto.Unmarshal(buf, &pb); err != nil {
f.logger().Printf("error unmarshaling cache data, skipping: path=%s, err=%s", path, err)
f.Logger.Printf("error unmarshaling cache data, skipping: path=%s, err=%s", path, err)
return nil
}
@ -298,13 +297,13 @@ func (f *Fragment) Close() error {
func (f *Fragment) close() error {
// Flush cache if closing gracefully.
if err := f.flushCache(); err != nil {
f.logger().Printf("fragment: error flushing cache on close: err=%s, path=%s", err, f.path)
f.Logger.Printf("fragment: error flushing cache on close: err=%s, path=%s", err, f.path)
return err
}
// Close underlying storage.
if err := f.closeStorage(); err != nil {
f.logger().Printf("fragment: error closing storage: err=%s, path=%s", err, f.path)
f.Logger.Printf("fragment: error closing storage: err=%s, path=%s", err, f.path)
return err
}
@ -343,9 +342,6 @@ func (f *Fragment) closeStorage() error {
return nil
}
// logger returns a logger instance for the fragment.nt.
func (f *Fragment) logger() *log.Logger { return log.New(f.LogOutput, "", log.LstdFlags) }
// Row returns a row by ID.
func (f *Fragment) Row(rowID uint64) *Bitmap {
f.mu.Lock()
@ -1385,18 +1381,17 @@ func (f *Fragment) Snapshot() error {
defer f.mu.Unlock()
return f.snapshot()
}
func track(start time.Time, message string, stats StatsClient, logger *log.Logger) {
func track(start time.Time, message string, stats StatsClient, logger Logger) {
elapsed := time.Since(start)
logger.Printf("%s took %s", message, elapsed)
stats.Histogram("snapshot", elapsed.Seconds(), 1.0)
}
func (f *Fragment) snapshot() error {
logger := f.logger()
logger.Printf("fragment: snapshotting %s/%s/%s/%d", f.index, f.frame, f.view, f.slice)
f.Logger.Printf("fragment: snapshotting %s/%s/%s/%d", f.index, f.frame, f.view, f.slice)
completeMessage := fmt.Sprintf("fragment: snapshot complete %s/%s/%s/%d", f.index, f.frame, f.view, f.slice)
start := time.Now()
defer track(start, completeMessage, f.stats, logger)
defer track(start, completeMessage, f.stats, f.Logger)
// Create a temporary file to snapshot to.
snapshotPath := f.path + SnapshotExt
@ -1841,11 +1836,11 @@ func (s *FragmentSyncer) syncBlock(id int) error {
// Only sync the standard block.
for j := 0; j < len(set.ColumnIDs); j++ {
fmt.Fprintf(&(buffers[count/s.Cluster.MaxWritesPerRequest]), "SetBit(frame=%q, rowID=%d, columnID=%d)\n", f.Frame(), set.RowIDs[j], (f.Slice()*SliceWidth)+set.ColumnIDs[j])
fmt.Fprintf(&(buffers[count/s.Cluster.MaxWritesPerRequest]), "SetBit(frame=%q, row=%d, col=%d)\n", f.Frame(), set.RowIDs[j], (f.Slice()*SliceWidth)+set.ColumnIDs[j])
count++
}
for j := 0; j < len(clear.ColumnIDs); j++ {
fmt.Fprintf(&(buffers[count/s.Cluster.MaxWritesPerRequest]), "ClearBit(frame=%q, rowID=%d, columnID=%d)\n", f.Frame(), clear.RowIDs[j], (f.Slice()*SliceWidth)+clear.ColumnIDs[j])
fmt.Fprintf(&(buffers[count/s.Cluster.MaxWritesPerRequest]), "ClearBit(frame=%q, row=%d, col=%d)\n", f.Frame(), clear.RowIDs[j], (f.Slice()*SliceWidth)+clear.ColumnIDs[j])
count++
}

49
frame.go
View file

@ -17,7 +17,6 @@ package pilosa
import (
"errors"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
@ -32,7 +31,6 @@ import (
// Default frame settings.
const (
DefaultRowLabel = "rowID"
DefaultCacheType = CacheTypeRanked
DefaultInverseEnabled = false
DefaultRangeEnabled = false
@ -57,7 +55,6 @@ type Frame struct {
Stats StatsClient
// Frame options.
rowLabel string
inverseEnabled bool
cacheType string
cacheSize uint32
@ -65,7 +62,7 @@ type Frame struct {
rangeEnabled bool
fields []*Field
LogOutput io.Writer
Logger Logger
}
// NewFrame returns a new instance of frame.
@ -87,7 +84,6 @@ func NewFrame(path, index, name string) (*Frame, error) {
broadcaster: NopBroadcaster,
Stats: NopStatsClient,
rowLabel: DefaultRowLabel,
inverseEnabled: DefaultInverseEnabled,
cacheType: DefaultCacheType,
cacheSize: DefaultCacheSize,
@ -95,7 +91,7 @@ func NewFrame(path, index, name string) (*Frame, error) {
rangeEnabled: DefaultRangeEnabled,
//fields
LogOutput: ioutil.Discard,
Logger: NopLogger,
}, nil
}
@ -139,39 +135,6 @@ func (f *Frame) MaxInverseSlice() uint64 {
return view.MaxSlice()
}
// SetRowLabel sets the row labels. Persists to meta file on update.
func (f *Frame) SetRowLabel(v string) error {
f.mu.Lock()
defer f.mu.Unlock()
// Ignore if no change occurred.
if v == "" || f.rowLabel == v {
return nil
}
// Make sure rowLabel is valid name
err := ValidateLabel(v)
if err != nil {
return err
}
// Persist meta data to disk on change.
f.rowLabel = v
if err := f.saveMeta(); err != nil {
return err
}
return nil
}
// RowLabel returns the row label.
func (f *Frame) RowLabel() string {
f.mu.RLock()
v := f.rowLabel
f.mu.RUnlock()
return v
}
// CacheType returns the caching mode for the frame.
func (f *Frame) CacheType() string {
return f.cacheType
@ -224,7 +187,6 @@ func (f *Frame) Options() FrameOptions {
func (f *Frame) options() FrameOptions {
return FrameOptions{
RowLabel: f.rowLabel,
InverseEnabled: f.inverseEnabled,
RangeEnabled: f.rangeEnabled,
CacheType: f.cacheType,
@ -302,7 +264,6 @@ func (f *Frame) loadMeta() error {
// Read data from meta file.
buf, err := ioutil.ReadFile(filepath.Join(f.path, ".meta"))
if os.IsNotExist(err) {
f.rowLabel = DefaultRowLabel
f.inverseEnabled = DefaultInverseEnabled
f.cacheType = DefaultCacheType
f.cacheSize = DefaultCacheSize
@ -319,7 +280,6 @@ func (f *Frame) loadMeta() error {
}
// Copy metadata fields.
f.rowLabel = pb.RowLabel
f.inverseEnabled = pb.InverseEnabled
f.cacheType = pb.CacheType
if f.cacheType == "" {
@ -624,7 +584,7 @@ func (f *Frame) createViewIfNotExistsBase(name string) (*View, bool, error) {
func (f *Frame) newView(path, name string) *View {
view := NewView(path, f.index, f.name, name, f.cacheSize)
view.cacheType = f.cacheType
view.LogOutput = f.LogOutput
view.Logger = f.Logger
view.RowAttrStore = f.rowAttrStore
view.stats = f.Stats.WithTags(fmt.Sprintf("view:%s", name))
view.broadcaster = f.broadcaster
@ -1030,7 +990,6 @@ func (p frameInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name }
// FrameOptions represents options to set when initializing a frame.
type FrameOptions struct {
RowLabel string `json:"rowLabel,omitempty"`
InverseEnabled bool `json:"inverseEnabled,omitempty"`
RangeEnabled bool `json:"rangeEnabled,omitempty"`
CacheType string `json:"cacheType,omitempty"`
@ -1049,7 +1008,6 @@ func encodeFrameOptions(o *FrameOptions) *internal.FrameMeta {
return nil
}
return &internal.FrameMeta{
RowLabel: o.RowLabel,
InverseEnabled: o.InverseEnabled,
RangeEnabled: o.RangeEnabled,
CacheType: o.CacheType,
@ -1064,7 +1022,6 @@ func decodeFrameOptions(options *internal.FrameMeta) *FrameOptions {
return nil
}
return &FrameOptions{
RowLabel: options.RowLabel,
InverseEnabled: options.InverseEnabled,
RangeEnabled: options.RangeEnabled,
CacheType: options.CacheType,

43
frame_test.go
View file

@ -267,49 +267,6 @@ func TestFrame_NameValidation(t *testing.T) {
}
}
// Ensure that frame RowLable validation is consistent.
func TestFrame_RowLabelValidation(t *testing.T) {
validRowLabels := []string{
"",
"foo",
"hyphen-ated",
"under_score",
"abc123",
"trailing_",
"camelCase",
"UPPERCASE",
}
invalidRowLabels := []string{
"123abc",
"x.y",
"_foo",
"-bar",
"abc def",
"a12345678901234567890123456789012345678901234567890123456789012345",
}
path, err := ioutil.TempDir("", "pilosa-frame-")
if err != nil {
panic(err)
}
f, err := pilosa.NewFrame(path, "i", "f")
if err != nil {
t.Fatalf("unexpected frame error: %s", err)
}
for _, label := range validRowLabels {
if err := f.SetRowLabel(label); err != nil {
t.Fatalf("unexpected row label: %s %s", label, err)
}
}
for _, label := range invalidRowLabels {
if err := f.SetRowLabel(label); err == nil {
t.Fatalf("expected error on row label: %s", label)
}
}
}
// Ensure frame can open and retrieve a view.
func TestFrame_DeleteView(t *testing.T) {
f := test.MustOpenFrame()

144
gossip/gossip.go
View file

@ -16,10 +16,8 @@ package gossip
import (
"fmt"
"io"
"io/ioutil"
"log"
"os"
"strconv"
"strings"
"sync"
@ -51,8 +49,10 @@ type GossipMemberSet struct {
statusHandler pilosa.StatusHandler
config *gossipConfig
// The writer for any logging.
LogOutput io.Writer
Logger pilosa.Logger
logger *log.Logger
transport *Transport
}
// Start implements the BroadcastReceiver interface and sets the BroadcastHandler.
@ -140,11 +140,6 @@ func retry(attempts int, sleep time.Duration, fn func() error) (err error) {
return fmt.Errorf("after %d attempts, last error: %s", attempts, err)
}
// logger returns a logger for the GossipMemberSet.
func (g *GossipMemberSet) logger() *log.Logger {
return log.New(g.LogOutput, "", log.LstdFlags)
}
////////////////////////////////////////////////////////////////
type gossipConfig struct {
@ -152,14 +147,61 @@ type gossipConfig struct {
memberlistConfig *memberlist.Config
}
// NewGossipMemberSetWithTransport returns a new instance of GossipMemberSet given a Transport.
func NewGossipMemberSetWithTransport(name string, cfg *pilosa.Config, transport *Transport, server *pilosa.Server) (*GossipMemberSet, error) {
// GossipMemberSetOption describes a functional option for GossipMemberSet.
type GossipMemberSetOption func(*GossipMemberSet) error
// WithTransport is a functional option for providing a transport to NewGossipMemberSet.
func WithTransport(transport *Transport) func(*GossipMemberSet) error {
return func(g *GossipMemberSet) error {
g.transport = transport
return nil
}
}
// WithLogger is a functional option for providing a logger to NewGossipMemberSet.
func WithLogger(logger *log.Logger) func(*GossipMemberSet) error {
return func(g *GossipMemberSet) error {
g.logger = logger
return nil
}
}
// NewGossipMemberSet returns a new instance of GossipMemberSet based on options.
func NewGossipMemberSet(name string, cfg *pilosa.Config, server *pilosa.Server, options ...GossipMemberSetOption) (*GossipMemberSet, error) {
g := &GossipMemberSet{
LogOutput: server.LogOutput,
Logger: server.Logger,
}
port := transport.Net.GetAutoBindPort()
// options
for _, opt := range options {
if err := opt(g); err != nil {
return nil, err
}
}
if g.transport == nil {
port, err := strconv.Atoi(cfg.Gossip.Port)
if err != nil {
return nil, fmt.Errorf("convert port: %s", err)
}
bindURI, err := pilosa.NewURIFromAddress(cfg.Bind)
if err != nil {
return nil, fmt.Errorf("getting uri from bind address: %s", err)
}
host := bindURI.Host()
// Set up the transport.
transport, err := NewTransport(host, port, g.logger)
if err != nil {
return nil, fmt.Errorf("new tranport: %s", err)
}
g.transport = transport
}
port := g.transport.Net.GetAutoBindPort()
bindURI, err := pilosa.NewURIFromAddress(cfg.Bind)
if err != nil {
@ -177,7 +219,7 @@ func NewGossipMemberSetWithTransport(name string, cfg *pilosa.Config, transport
// memberlist config
conf := memberlist.DefaultWANConfig()
conf.Transport = transport.Net
conf.Transport = g.transport.Net
conf.Name = name
conf.BindAddr = host
conf.BindPort = port
@ -196,6 +238,7 @@ func NewGossipMemberSetWithTransport(name string, cfg *pilosa.Config, transport
conf.Delegate = g
conf.SecretKey = gossipKey
conf.Events = server.Cluster.EventReceiver.(memberlist.EventDelegate)
conf.Logger = g.logger
g.config = &gossipConfig{
memberlistConfig: conf,
@ -207,28 +250,6 @@ func NewGossipMemberSetWithTransport(name string, cfg *pilosa.Config, transport
return g, nil
}
// NewGossipMemberSet returns a new instance of GossipMemberSet given a gossip port.
func NewGossipMemberSet(name string, cfg *pilosa.Config, server *pilosa.Server) (*GossipMemberSet, error) {
port, err := strconv.Atoi(cfg.Gossip.Port)
if err != nil {
return nil, fmt.Errorf("convert port: %s", err)
}
bindURI, err := pilosa.NewURIFromAddress(cfg.Bind)
if err != nil {
return nil, fmt.Errorf("getting uri from bind address: %s", err)
}
host := bindURI.Host()
// Set up the transport.
transport, err := NewTransport(host, port)
if err != nil {
return nil, fmt.Errorf("new tranport: %s", err)
}
return NewGossipMemberSetWithTransport(name, cfg, transport, server)
}
// SendSync implementation of the Broadcaster interface.
func (g *GossipMemberSet) SendSync(pb proto.Message) error {
msg, err := pilosa.MarshalMessage(pb)
@ -276,7 +297,7 @@ func (g *GossipMemberSet) SendAsync(pb proto.Message) error {
func (g *GossipMemberSet) NodeMeta(limit int) []byte {
buf, err := proto.Marshal(pilosa.EncodeNode(g.node))
if err != nil {
g.logger().Printf("marshal message error: %s", err)
g.Logger.Printf("marshal message error: %s", err)
return []byte{}
}
return buf
@ -287,11 +308,11 @@ func (g *GossipMemberSet) NodeMeta(limit int) []byte {
func (g *GossipMemberSet) NotifyMsg(b []byte) {
m, err := pilosa.UnmarshalMessage(b)
if err != nil {
g.logger().Printf("unmarshal message error: %s", err)
g.Logger.Printf("unmarshal message error: %s", err)
return
}
if err := g.handler.ReceiveMessage(m); err != nil {
g.logger().Printf("receive message error: %s", err)
g.Logger.Printf("receive message error: %s", err)
return
}
}
@ -307,14 +328,14 @@ func (g *GossipMemberSet) GetBroadcasts(overhead, limit int) [][]byte {
func (g *GossipMemberSet) LocalState(join bool) []byte {
pb, err := g.statusHandler.LocalStatus()
if err != nil {
g.logger().Printf("error getting local state, err=%s", err)
g.Logger.Printf("error getting local state, err=%s", err)
return []byte{}
}
// Marshal nodestate data to bytes.
buf, err := proto.Marshal(pb)
if err != nil {
g.logger().Printf("error marshalling nodestate data, err=%s", err)
g.Logger.Printf("error marshalling nodestate data, err=%s", err)
return []byte{}
}
return buf
@ -326,12 +347,12 @@ func (g *GossipMemberSet) MergeRemoteState(buf []byte, join bool) {
// Unmarshal nodestate data.
var pb internal.NodeStatus
if err := proto.Unmarshal(buf, &pb); err != nil {
g.logger().Printf("error unmarshalling nodestate data, err=%s", err)
g.Logger.Printf("error unmarshalling nodestate data, err=%s", err)
return
}
err := g.statusHandler.HandleRemoteStatus(&pb)
if err != nil {
g.logger().Printf("merge state error: %s", err)
g.Logger.Printf("merge state error: %s", err)
}
}
@ -344,15 +365,14 @@ type GossipEventReceiver struct {
ch chan memberlist.NodeEvent
eventHandler pilosa.EventHandler
// The writer for any logging.
LogOutput io.Writer
Logger pilosa.Logger
}
// NewGossipEventReceiver returns a new instance of GossipEventReceiver.
func NewGossipEventReceiver(logOutput io.Writer) *GossipEventReceiver {
func NewGossipEventReceiver(logger pilosa.Logger) *GossipEventReceiver {
return &GossipEventReceiver{
ch: make(chan memberlist.NodeEvent, 1),
LogOutput: logOutput,
ch: make(chan memberlist.NodeEvent, 1),
Logger: logger,
}
}
@ -375,11 +395,6 @@ func (g *GossipEventReceiver) Start(h pilosa.EventHandler) error {
return nil
}
// logger returns a logger for the GossipEventReceiver.
func (g *GossipEventReceiver) logger() *log.Logger {
return log.New(g.LogOutput, "", log.LstdFlags)
}
func (g *GossipEventReceiver) listen() {
var nodeEventType pilosa.NodeEventType
for {
@ -407,7 +422,7 @@ func (g *GossipEventReceiver) listen() {
Node: node,
}
if err := g.eventHandler.ReceiveEvent(ne); err != nil {
g.logger().Printf("receive event error: %s", err)
g.Logger.Printf("receive event error: %s", err)
}
}
}
@ -443,12 +458,13 @@ type Transport struct {
// It will dynamically bind to a port if port is 0.
// This is useful for test cases where specifiying a port is not reasonable.
//func NewTransport(host string, port int) (*memberlist.NetTransport, error) {
func NewTransport(host string, port int) (*Transport, error) {
func NewTransport(host string, port int, logger *log.Logger) (*Transport, error) {
// memberlist config
conf := memberlist.DefaultWANConfig()
conf.BindAddr = host
conf.BindPort = port
conf.AdvertisePort = port
conf.Logger = logger
net, err := newTransport(conf)
if err != nil {
@ -469,24 +485,10 @@ func NewTransport(host string, port int) (*Transport, error) {
// newTransport returns a NetTransport based on the memberlist configuration.
// It will dynamically bind to a port if conf.BindPort is 0.
func newTransport(conf *memberlist.Config) (*memberlist.NetTransport, error) {
if conf.LogOutput != nil && conf.Logger != nil {
return nil, fmt.Errorf("Cannot specify both LogOutput and Logger. Please choose a single log configuration setting.")
}
logDest := conf.LogOutput
if logDest == nil {
logDest = os.Stderr
}
logger := conf.Logger
if logger == nil {
logger = log.New(logDest, "", log.LstdFlags)
}
nc := &memberlist.NetTransportConfig{
BindAddrs: []string{conf.BindAddr},
BindPort: conf.BindPort,
Logger: logger,
Logger: conf.Logger,
}
// See comment below for details about the retry in here.
@ -498,7 +500,7 @@ func newTransport(conf *memberlist.Config) (*memberlist.NetTransport, error) {
return nt, nil
}
if strings.Contains(err.Error(), "address already in use") {
logger.Printf("[DEBUG] Got bind error: %v", err)
conf.Logger.Printf("[DEBUG] Got bind error: %v", err)
continue
}
}

121
handler.go
View file

@ -23,12 +23,10 @@ import (
"fmt"
"io"
"io/ioutil"
"log"
"net/http"
"net/url"
// Imported for its side-effect of registering pprof endpoints with the server.
_ "net/http/pprof"
"os"
"runtime/debug"
"strconv"
"strings"
@ -67,8 +65,7 @@ type Handler struct {
Execute(context context.Context, index string, query *pql.Query, slices []uint64, opt *ExecOptions) ([]interface{}, error)
}
// The writer for any logging.
LogOutput io.Writer
Logger Logger
// Keeps the query argument validators for each handler
validators map[string]*queryValidationSpec
@ -98,8 +95,7 @@ func NewHandler() *Handler {
//BroadcastHandler: NopBroadcastHandler, // TODO: implement the nop
//StatusHandler: NopStatusHandler, // TODO: implement the nop
FileSystem: NopFileSystem,
LogOutput: os.Stderr,
Logger: NopLogger,
}
BuildRouters(handler)
handler.populateValidators()
@ -247,7 +243,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusInternalServerError)
stack := debug.Stack()
msg := "PANIC: %s\n%s"
fmt.Fprintf(h.LogOutput, msg, err, stack)
h.Logger.Printf(msg, err, stack)
fmt.Fprintf(w, msg, err, stack)
}
}()
@ -261,7 +257,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
statsTags := make([]string, 0, 3)
if h.Cluster.LongQueryTime > 0 && dif > h.Cluster.LongQueryTime {
h.logger().Printf("%s %s %v", r.Method, r.URL.String(), dif)
h.Logger.Printf("%s %s %v", r.Method, r.URL.String(), dif)
statsTags = append(statsTags, "slow_query")
}
@ -289,7 +285,7 @@ func (h *Handler) handleWebUI(w http.ResponseWriter, r *http.Request) {
filesystem, err := h.FileSystem.New()
if err != nil {
h.writeQueryResponse(w, r, &QueryResponse{Err: err})
h.logger().Println("Pilosa WebUI is not available. Please run `make generate-statik` before building Pilosa with `make install`.")
h.Logger.Printf("Pilosa WebUI is not available. Please run `make generate-statik` before building Pilosa with `make install`.")
return
}
http.FileServer(filesystem).ServeHTTP(w, r)
@ -300,7 +296,7 @@ func (h *Handler) handleGetSchema(w http.ResponseWriter, r *http.Request) {
if err := json.NewEncoder(w).Encode(getSchemaResponse{
Indexes: h.Holder.Schema(),
}); err != nil {
h.logger().Printf("write schema response error: %s", err)
h.Logger.Printf("write schema response error: %s", err)
}
}
@ -308,7 +304,7 @@ func (h *Handler) handleGetSchema(w http.ResponseWriter, r *http.Request) {
func (h *Handler) handleGetStatus(w http.ResponseWriter, r *http.Request) {
pb, err := h.StatusHandler.ClusterStatus()
if err != nil {
h.logger().Printf("cluster status error: %s", err)
h.Logger.Printf("cluster status error: %s", err)
return
}
@ -317,7 +313,7 @@ func (h *Handler) handleGetStatus(w http.ResponseWriter, r *http.Request) {
State: cs.State,
Nodes: DecodeNodes(cs.Nodes),
}); err != nil {
h.logger().Printf("write status response error: %s", err)
h.Logger.Printf("write status response error: %s", err)
}
}
@ -395,7 +391,7 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) {
// Write response back to client.
if err := h.writeQueryResponse(w, r, resp); err != nil {
h.logger().Printf("write query response error: %s", err)
h.Logger.Printf("write query response error: %s", err)
}
}
@ -405,7 +401,7 @@ func (h *Handler) handleGetSlicesMax(w http.ResponseWriter, r *http.Request) {
Standard: h.Holder.MaxSlices(),
Inverse: h.Holder.MaxInverseSlices(),
}); err != nil {
h.logger().Printf("write slices-max response error: %s", err)
h.Logger.Printf("write slices-max response error: %s", err)
}
}
@ -431,7 +427,7 @@ func (h *Handler) handleGetIndex(w http.ResponseWriter, r *http.Request) {
if err := json.NewEncoder(w).Encode(getIndexResponse{
map[string]string{"name": index.Name()},
}); err != nil {
h.logger().Printf("write response error: %s", err)
h.Logger.Printf("write response error: %s", err)
}
}
@ -519,12 +515,12 @@ func (h *Handler) handleDeleteIndex(w http.ResponseWriter, r *http.Request) {
Index: indexName,
})
if err != nil {
h.logger().Printf("problem sending DeleteIndex message: %s", err)
h.Logger.Printf("problem sending DeleteIndex message: %s", err)
}
// Encode response.
if err := json.NewEncoder(w).Encode(deleteIndexResponse{}); err != nil {
h.logger().Printf("response encoding error: %s", err)
h.Logger.Printf("response encoding error: %s", err)
}
h.Holder.Stats.Count("deleteIndex", 1, 1.0)
@ -564,14 +560,14 @@ func (h *Handler) handlePostIndex(w http.ResponseWriter, r *http.Request) {
Meta: req.Options.Encode(),
})
if err != nil {
h.logger().Printf("problem sending CreateIndex message: %s", err)
h.Logger.Printf("problem sending CreateIndex message: %s", err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
// Encode response.
if err := json.NewEncoder(w).Encode(postIndexResponse{}); err != nil {
h.logger().Printf("response encoding error: %s", err)
h.Logger.Printf("response encoding error: %s", err)
}
h.Holder.Stats.Count("createIndex", 1, 1.0)
@ -610,7 +606,7 @@ func (h *Handler) handlePatchIndexTimeQuantum(w http.ResponseWriter, r *http.Req
// Encode response.
if err := json.NewEncoder(w).Encode(patchIndexTimeQuantumResponse{}); err != nil {
h.logger().Printf("response encoding error: %s", err)
h.Logger.Printf("response encoding error: %s", err)
}
}
@ -665,7 +661,7 @@ func (h *Handler) handlePostIndexAttrDiff(w http.ResponseWriter, r *http.Request
if err := json.NewEncoder(w).Encode(postIndexAttrDiffResponse{
Attrs: attrs,
}); err != nil {
h.logger().Printf("response encoding error: %s", err)
h.Logger.Printf("response encoding error: %s", err)
}
}
@ -718,12 +714,12 @@ func (h *Handler) handlePostFrame(w http.ResponseWriter, r *http.Request) {
Meta: req.Options.Encode(),
})
if err != nil {
h.logger().Printf("problem sending CreateFrame message: %s", err)
h.Logger.Printf("problem sending CreateFrame message: %s", err)
}
// Encode response.
if err := json.NewEncoder(w).Encode(postFrameResponse{}); err != nil {
h.logger().Printf("response encoding error: %s", err)
h.Logger.Printf("response encoding error: %s", err)
}
h.Holder.Stats.CountWithCustomTags("createFrame", 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)})
@ -783,7 +779,7 @@ func (h *Handler) handleDeleteFrame(w http.ResponseWriter, r *http.Request) {
index := h.Holder.Index(indexName)
if index == nil {
if err := json.NewEncoder(w).Encode(deleteIndexResponse{}); err != nil {
h.logger().Printf("response encoding error: %s", err)
h.Logger.Printf("response encoding error: %s", err)
}
return
}
@ -801,12 +797,12 @@ func (h *Handler) handleDeleteFrame(w http.ResponseWriter, r *http.Request) {
Frame: frameName,
})
if err != nil {
h.logger().Printf("problem sending DeleteFrame message: %s", err)
h.Logger.Printf("problem sending DeleteFrame message: %s", err)
}
// Encode response.
if err := json.NewEncoder(w).Encode(deleteFrameResponse{}); err != nil {
h.logger().Printf("response encoding error: %s", err)
h.Logger.Printf("response encoding error: %s", err)
}
h.Holder.Stats.CountWithCustomTags("deleteFrame", 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)})
@ -848,7 +844,7 @@ func (h *Handler) handlePatchFrameTimeQuantum(w http.ResponseWriter, r *http.Req
// Encode response.
if err := json.NewEncoder(w).Encode(patchFrameTimeQuantumResponse{}); err != nil {
h.logger().Printf("response encoding error: %s", err)
h.Logger.Printf("response encoding error: %s", err)
}
}
@ -899,12 +895,12 @@ func (h *Handler) handlePostFrameField(w http.ResponseWriter, r *http.Request) {
Field: encodeField(field),
})
if err != nil {
h.logger().Printf("problem sending CreateField message: %s", err)
h.Logger.Printf("problem sending CreateField message: %s", err)
}
// Encode response.
if err := json.NewEncoder(w).Encode(postFrameFieldResponse{}); err != nil {
h.logger().Printf("response encoding error: %s", err)
h.Logger.Printf("response encoding error: %s", err)
}
}
@ -943,12 +939,12 @@ func (h *Handler) handleDeleteFrameField(w http.ResponseWriter, r *http.Request)
Field: fieldName,
})
if err != nil {
h.logger().Printf("problem sending DeleteField message: %s", err)
h.Logger.Printf("problem sending DeleteField message: %s", err)
}
// Encode response.
if err := json.NewEncoder(w).Encode(deleteFrameFieldResponse{}); err != nil {
h.logger().Printf("response encoding error: %s", err)
h.Logger.Printf("response encoding error: %s", err)
}
}
@ -979,7 +975,7 @@ func (h *Handler) handleGetFrameFields(w http.ResponseWriter, r *http.Request) {
// Encode response.
if err := json.NewEncoder(w).Encode(getFrameFieldsResponse{Fields: fields}); err != nil {
h.logger().Printf("response encoding error: %s", err)
h.Logger.Printf("response encoding error: %s", err)
}
}
@ -1012,7 +1008,7 @@ func (h *Handler) handleGetFrameViews(w http.ResponseWriter, r *http.Request) {
// Encode response.
if err := json.NewEncoder(w).Encode(getFrameViewsResponse{Views: names}); err != nil {
h.logger().Printf("response encoding error: %s", err)
h.Logger.Printf("response encoding error: %s", err)
}
}
@ -1046,12 +1042,12 @@ func (h *Handler) handleDeleteView(w http.ResponseWriter, r *http.Request) {
View: viewName,
})
if err != nil {
h.logger().Printf("problem sending DeleteView message: %s", err)
h.Logger.Printf("problem sending DeleteView message: %s", err)
}
// Encode response.
if err := json.NewEncoder(w).Encode(deleteViewResponse{}); err != nil {
h.logger().Printf("response encoding error: %s", err)
h.Logger.Printf("response encoding error: %s", err)
}
}
@ -1107,7 +1103,7 @@ func (h *Handler) handlePostFrameAttrDiff(w http.ResponseWriter, r *http.Request
if err := json.NewEncoder(w).Encode(postFrameAttrDiffResponse{
Attrs: attrs,
}); err != nil {
h.logger().Printf("response encoding error: %s", err)
h.Logger.Printf("response encoding error: %s", err)
}
}
@ -1276,10 +1272,10 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) {
}
// Find the Index.
h.logger().Println("importing:", req.Index, req.Frame, req.Slice)
h.Logger.Printf("importing: %s %s %d", req.Index, req.Frame, req.Slice)
index := h.Holder.Index(req.Index)
if index == nil {
h.logger().Printf("fragment error: index=%s, frame=%s, slice=%d, err=%s", req.Index, req.Frame, req.Slice, ErrIndexNotFound.Error())
h.Logger.Printf("fragment error: index=%s, frame=%s, slice=%d, err=%s", req.Index, req.Frame, req.Slice, ErrIndexNotFound.Error())
http.Error(w, ErrIndexNotFound.Error(), http.StatusNotFound)
return
}
@ -1287,7 +1283,7 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) {
// Retrieve frame.
f := index.Frame(req.Frame)
if f == nil {
h.logger().Printf("frame error: index=%s, frame=%s, slice=%d, err=%s", req.Index, req.Frame, req.Slice, ErrFrameNotFound.Error())
h.Logger.Printf("frame error: index=%s, frame=%s, slice=%d, err=%s", req.Index, req.Frame, req.Slice, ErrFrameNotFound.Error())
http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound)
return
}
@ -1295,7 +1291,7 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) {
// Import into fragment.
err = f.Import(req.RowIDs, req.ColumnIDs, timestamps)
if err != nil {
h.logger().Printf("import error: index=%s, frame=%s, slice=%d, bits=%d, err=%s", req.Index, req.Frame, req.Slice, len(req.ColumnIDs), err)
h.Logger.Printf("import error: index=%s, frame=%s, slice=%d, bits=%d, err=%s", req.Index, req.Frame, req.Slice, len(req.ColumnIDs), err)
return
}
@ -1346,10 +1342,10 @@ func (h *Handler) handlePostImportValue(w http.ResponseWriter, r *http.Request)
}
// Find the Index.
h.logger().Println("importing:", req.Index, req.Frame, req.Slice)
h.Logger.Printf("importing: %s %s %d", req.Index, req.Frame, req.Slice)
index := h.Holder.Index(req.Index)
if index == nil {
h.logger().Printf("fragment error: index=%s, frame=%s, slice=%d, err=%s", req.Index, req.Frame, req.Slice, ErrIndexNotFound.Error())
h.Logger.Printf("fragment error: index=%s, frame=%s, slice=%d, err=%s", req.Index, req.Frame, req.Slice, ErrIndexNotFound.Error())
http.Error(w, ErrIndexNotFound.Error(), http.StatusNotFound)
return
}
@ -1357,7 +1353,7 @@ func (h *Handler) handlePostImportValue(w http.ResponseWriter, r *http.Request)
// Retrieve frame.
f := index.Frame(req.Frame)
if f == nil {
h.logger().Printf("frame error: index=%s, frame=%s, slice=%d, err=%s", req.Index, req.Frame, req.Slice, ErrFrameNotFound.Error())
h.Logger.Printf("frame error: index=%s, frame=%s, slice=%d, err=%s", req.Index, req.Frame, req.Slice, ErrFrameNotFound.Error())
http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound)
return
}
@ -1365,7 +1361,7 @@ func (h *Handler) handlePostImportValue(w http.ResponseWriter, r *http.Request)
// Import into fragment.
err = f.ImportValue(req.Field, req.ColumnIDs, req.Values)
if err != nil {
h.logger().Printf("import error: index=%s, frame=%s, slice=%d, field=%s, bits=%d, err=%s", req.Index, req.Frame, req.Slice, req.Field, len(req.ColumnIDs), err)
h.Logger.Printf("import error: index=%s, frame=%s, slice=%d, field=%s, bits=%d, err=%s", req.Index, req.Frame, req.Slice, req.Field, len(req.ColumnIDs), err)
return
}
@ -1452,7 +1448,7 @@ func (h *Handler) handleGetFragmentNodes(w http.ResponseWriter, r *http.Request)
// Write to response.
if err := json.NewEncoder(w).Encode(nodes); err != nil {
h.logger().Printf("json write error: %s", err)
h.Logger.Printf("json write error: %s", err)
}
}
@ -1475,7 +1471,7 @@ func (h *Handler) handleGetFragmentData(w http.ResponseWriter, r *http.Request)
// Stream fragment to response body.
if _, err := f.WriteTo(w); err != nil {
h.logger().Printf("fragment backup error: %s", err)
h.Logger.Printf("fragment backup error: %s", err)
}
}
@ -1545,7 +1541,7 @@ func (h *Handler) handleGetFragmentBlockData(w http.ResponseWriter, r *http.Requ
// Encode response.
buf, err := proto.Marshal(&resp)
if err != nil {
h.logger().Printf("merge block response encoding error: %s", err)
h.Logger.Printf("merge block response encoding error: %s", err)
return
}
@ -1579,7 +1575,7 @@ func (h *Handler) handleGetFragmentBlocks(w http.ResponseWriter, r *http.Request
if err := json.NewEncoder(w).Encode(getFragmentBlocksResponse{
Blocks: blocks,
}); err != nil {
h.logger().Printf("block response encoding error: %s", err)
h.Logger.Printf("block response encoding error: %s", err)
}
}
@ -1680,7 +1676,7 @@ func (h *Handler) handlePostFrameRestore(w http.ResponseWriter, r *http.Request)
// handleGetHosts handles /hosts requests.
func (h *Handler) handleGetHosts(w http.ResponseWriter, r *http.Request) {
if err := json.NewEncoder(w).Encode(h.Cluster.Nodes); err != nil {
h.logger().Printf("write version response error: %s", err)
h.Logger.Printf("write version response error: %s", err)
}
}
@ -1696,7 +1692,7 @@ func (h *Handler) handleGetVersion(w http.ResponseWriter, r *http.Request) {
}{
Version: version,
}); err != nil {
h.logger().Printf("write version response error: %s", err)
h.Logger.Printf("write version response error: %s", err)
}
}
@ -1716,11 +1712,6 @@ func (h *Handler) handleExpvar(w http.ResponseWriter, r *http.Request) {
fmt.Fprintf(w, "\n}\n")
}
// logger returns a logger for the handler.
func (h *Handler) logger() *log.Logger {
return log.New(h.LogOutput, "", log.LstdFlags)
}
// QueryResult types.
const (
QueryResultTypeNil uint32 = iota
@ -1908,11 +1899,11 @@ func (h *Handler) handlePostInputDefinition(w http.ResponseWriter, r *http.Reque
Definition: def,
})
if err != nil {
h.logger().Printf("problem sending CreateInputDefinition message: %s", err)
h.Logger.Printf("problem sending CreateInputDefinition message: %s", err)
}
if err := json.NewEncoder(w).Encode(defaultInputDefinitionResponse{}); err != nil {
h.logger().Printf("response encoding error: %s", err)
h.Logger.Printf("response encoding error: %s", err)
}
}
@ -1938,7 +1929,7 @@ func (h *Handler) handleGetInputDefinition(w http.ResponseWriter, r *http.Reques
Frames: inputDef.frames,
Fields: inputDef.fields,
}); err != nil {
h.logger().Printf("write status response error: %s", err)
h.Logger.Printf("write status response error: %s", err)
}
}
@ -1967,11 +1958,11 @@ func (h *Handler) handleDeleteInputDefinition(w http.ResponseWriter, r *http.Req
Name: inputDefName,
})
if err != nil {
h.logger().Printf("problem sending DeleteInputDefinition message: %s", err)
h.Logger.Printf("problem sending DeleteInputDefinition message: %s", err)
}
if err := json.NewEncoder(w).Encode(defaultInputDefinitionResponse{}); err != nil {
h.logger().Printf("response encoding error: %s", err)
h.Logger.Printf("response encoding error: %s", err)
}
}
@ -2013,7 +2004,7 @@ func (h *Handler) handlePostInput(w http.ResponseWriter, r *http.Request) {
}
}
if err := json.NewEncoder(w).Encode(defaultInputDefinitionResponse{}); err != nil {
h.logger().Printf("response encoding error: %s", err)
h.Logger.Printf("response encoding error: %s", err)
}
}
@ -2061,7 +2052,7 @@ func (h *Handler) handlePostClusterResizeSetCoordinator(w http.ResponseWriter, r
Old: oldNode,
New: newNode,
}); err != nil {
h.logger().Printf("response encoding error: %s", err)
h.Logger.Printf("response encoding error: %s", err)
}
}
@ -2101,7 +2092,7 @@ func (h *Handler) handlePostClusterResizeRemoveNode(w http.ResponseWriter, r *ht
if err := json.NewEncoder(w).Encode(removeNodeResponse{
Remove: removeNode,
}); err != nil {
h.logger().Printf("response encoding error: %s", err)
h.Logger.Printf("response encoding error: %s", err)
}
}
@ -2134,7 +2125,7 @@ func (h *Handler) handlePostClusterResizeAbort(w http.ResponseWriter, r *http.Re
if err := json.NewEncoder(w).Encode(clusterResizeAbortResponse{
Info: msg,
}); err != nil {
h.logger().Printf("response encoding error: %s", err)
h.Logger.Printf("response encoding error: %s", err)
}
}
@ -2271,7 +2262,7 @@ func (h *Handler) handlePostClusterMessage(w http.ResponseWriter, r *http.Reques
}
if err := json.NewEncoder(w).Encode(defaultClusterMessageResponse{}); err != nil {
h.logger().Printf("response encoding error: %s", err)
h.Logger.Printf("response encoding error: %s", err)
}
}

12
handler_internal_test.go
View file

@ -30,8 +30,7 @@ func TestPostIndexRequestUnmarshalJSON(t *testing.T) {
{json: `{"options": {}}`, expected: postIndexRequest{Options: IndexOptions{}}},
{json: `{"options": 4}`, err: "options is not map[string]interface{}"},
{json: `{"option": {}}`, err: "Unknown key: option:map[]"},
{json: `{"options": {"columnLabel": "test"}}`, expected: postIndexRequest{Options: IndexOptions{ColumnLabel: "test"}}},
{json: `{"options": {"columnLabl": "test"}}`, err: "Unknown key: columnLabl:test"},
{json: `{"options": {"badKey": "test"}}`, err: "Unknown key: badKey:test"},
}
for _, test := range tests {
actual := &postIndexRequest{}
@ -66,11 +65,10 @@ func TestPostFrameRequestUnmarshalJSON(t *testing.T) {
{json: `{"options": {}}`, expected: postFrameRequest{Options: FrameOptions{}}},
{json: `{"options": 4}`, err: "options is not map[string]interface{}"},
{json: `{"option": {}}`, err: "Unknown key: option:map[]"},
{json: `{"options": {"rowLabel": "test"}}`, expected: postFrameRequest{Options: FrameOptions{RowLabel: "test"}}},
{json: `{"options": {"rowLabl": "test"}}`, err: "Unknown key: rowLabl:test"},
{json: `{"options": {"rowLabel": "test", "inverseEnabled": true}}`, expected: postFrameRequest{Options: FrameOptions{RowLabel: "test", InverseEnabled: true}}},
{json: `{"options": {"rowLabel": "test", "inverseEnabled": true, "cacheType": "type"}}`, expected: postFrameRequest{Options: FrameOptions{RowLabel: "test", InverseEnabled: true, CacheType: "type"}}},
{json: `{"options": {"rowLabel": "test", "inverse": true, "cacheType": "type"}}`, err: "Unknown key: inverse:true"},
{json: `{"options": {"badKey": "test"}}`, err: "Unknown key: badKey:test"},
{json: `{"options": {"inverseEnabled": true}}`, expected: postFrameRequest{Options: FrameOptions{InverseEnabled: true}}},
{json: `{"options": {"inverseEnabled": true, "cacheType": "type"}}`, expected: postFrameRequest{Options: FrameOptions{InverseEnabled: true, CacheType: "type"}}},
{json: `{"options": {"inverse": true, "cacheType": "type"}}`, err: "Unknown key: inverse:true"},
}
for _, test := range tests {
actual := &postFrameRequest{}

14
handler_test.go
View file

@ -37,13 +37,13 @@ import (
func TestHandlerPanics(t *testing.T) {
h := test.NewHandler()
buf := &bytes.Buffer{}
h.Handler.LogOutput = buf
bufLogger := test.NewBufferLogger()
h.Handler.Logger = bufLogger
w := httptest.NewRecorder()
// will panic since Handler has no Holder set up
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/index/taxi", nil))
bufbytes, err := ioutil.ReadAll(buf)
bufbytes, err := bufLogger.ReadAll()
if err != nil {
t.Fatalf("reading all logoutput: %v", err)
}
@ -107,7 +107,7 @@ func TestHandler_Schema(t *testing.T) {
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","frames":[{"name":"f0"},{"name":"f1","views":[{"name":"inverse"},{"name":"standard"}]}]},{"name":"i1","frames":[{"name":"f0","views":[{"name":"standard"}]}]}]}`+"\n" {
} else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","frames":[{"name":"f0","options":{"rowLabel":"rowID","cacheType":"ranked","cacheSize":50000}},{"name":"f1","options":{"rowLabel":"rowID","inverseEnabled":true,"cacheType":"ranked","cacheSize":50000},"views":[{"name":"inverse"},{"name":"standard"}]}]},{"name":"i1","frames":[{"name":"f0","options":{"rowLabel":"rowID","cacheType":"ranked","cacheSize":50000},"views":[{"name":"standard"}]}]}]}`+"\n" {
} else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","frames":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000}},{"name":"f1","options":{"inverseEnabled":true,"cacheType":"ranked","cacheSize":50000},"views":[{"name":"inverse"},{"name":"standard"}]}]},{"name":"i1","frames":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000},"views":[{"name":"standard"}]}]}]}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
}
@ -1349,7 +1349,7 @@ func TestHandler_DuplicatePrimaryKey(t *testing.T) {
}
// Ensure throwing error if there's no primary key
hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{ColumnLabel: "id"})
hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{})
unmatchColumnBody := []byte(`
{
"frames":[{
@ -1433,7 +1433,7 @@ func TestHandler_DeleteInputDefinition(t *testing.T) {
// Test input definition is deleted.
index := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{})
frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}}
frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{}}
action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}}
fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}}
def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}}
@ -1470,7 +1470,7 @@ func TestHandler_GetInputDefinition(t *testing.T) {
h.Holder = hldr.Holder
h.Cluster = test.NewCluster(1)
frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}}
frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{}}
action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}}
fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}}
def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}}

37
holder.go
View file

@ -18,9 +18,7 @@ import (
"context"
"errors"
"fmt"
"io"
"io/ioutil"
"log"
"net/http"
"os"
"path"
@ -71,7 +69,7 @@ type Holder struct {
// The interval at which the cached row ids are persisted to disk.
CacheFlushInterval time.Duration
LogOutput io.Writer
Logger Logger
}
// NewHolder returns a new instance of Holder.
@ -89,7 +87,7 @@ func NewHolder() *Holder {
CacheFlushInterval: DefaultCacheFlushInterval,
LogOutput: os.Stderr,
Logger: NopLogger,
}
}
@ -97,7 +95,7 @@ func NewHolder() *Holder {
// without actually loading any data into memory.
// HasData is returned, and h.hasData is set.
func (h *Holder) Peek() bool {
h.logger().Printf("peek at holder path: %s", h.Path)
h.Logger.Printf("peek at holder path: %s", h.Path)
h.hasData = false
// Open path to read all index directories.
@ -127,7 +125,7 @@ func (h *Holder) Peek() bool {
func (h *Holder) Open() error {
h.setFileLimit()
h.logger().Printf("open holder path: %s", h.Path)
h.Logger.Printf("open holder path: %s", h.Path)
if err := os.MkdirAll(h.Path, 0777); err != nil {
return err
}
@ -149,18 +147,18 @@ func (h *Holder) Open() error {
continue
}
h.logger().Printf("opening index: %s", filepath.Base(fi.Name()))
h.Logger.Printf("opening index: %s", filepath.Base(fi.Name()))
index, err := h.newIndex(h.IndexPath(filepath.Base(fi.Name())), filepath.Base(fi.Name()))
if err == ErrName {
h.logger().Printf("ERROR opening index: %s, err=%s", fi.Name(), err)
h.Logger.Printf("ERROR opening index: %s, err=%s", fi.Name(), err)
continue
} else if err != nil {
return err
}
if err := index.Open(); err != nil {
if err == ErrName {
h.logger().Printf("ERROR opening index: %s, err=%s", index.Name(), err)
h.Logger.Printf("ERROR opening index: %s, err=%s", index.Name(), err)
continue
}
return fmt.Errorf("open index: name=%s, err=%s", index.Name(), err)
@ -169,7 +167,7 @@ func (h *Holder) Open() error {
h.indexes[index.Name()] = index
h.mu.Unlock()
}
h.logger().Printf("open holder: complete")
h.Logger.Printf("open holder: complete")
// Periodically flush cache.
h.wg.Add(1)
@ -361,7 +359,6 @@ func (h *Holder) createIndex(name string, opt IndexOptions) (*Index, error) {
}
// Update options.
index.SetColumnLabel(opt.ColumnLabel)
index.SetTimeQuantum(opt.TimeQuantum)
h.indexes[index.Name()] = index
@ -374,7 +371,7 @@ func (h *Holder) newIndex(path, name string) (*Index, error) {
if err != nil {
return nil, err
}
index.LogOutput = h.LogOutput
index.Logger = h.Logger
index.Stats = h.Stats.WithTags(fmt.Sprintf("index:%s", index.Name()))
index.broadcaster = h.Broadcaster
index.NewAttrStore = h.NewAttrStore
@ -464,7 +461,7 @@ func (h *Holder) flushCaches() {
}
if err := fragment.FlushCache(); err != nil {
h.logger().Printf("error flushing cache: err=%s, path=%s", err, fragment.CachePath())
h.Logger.Printf("error flushing cache: err=%s, path=%s", err, fragment.CachePath())
}
}
}
@ -488,7 +485,7 @@ func (h *Holder) setFileLimit() {
newLimit := &syscall.Rlimit{}
if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, oldLimit); err != nil {
h.logger().Printf("ERROR checking open file limit: %s", err)
h.Logger.Printf("ERROR checking open file limit: %s", err)
return
}
// If the soft limit is lower than the FileLimit constant, we will try to change it.
@ -512,32 +509,30 @@ func (h *Holder) setFileLimit() {
}
// Try setting again with lowered Max (hard limit)
if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, newLimit); err != nil {
h.logger().Printf("ERROR setting open file limit: %s", err)
h.Logger.Printf("ERROR setting open file limit: %s", err)
}
// If we weren't trying to change the hard limit, let the user know something is wrong.
} else {
h.logger().Printf("ERROR setting open file limit: %s", err)
h.Logger.Printf("ERROR setting open file limit: %s", err)
}
}
// Check the limit after setting it. OS may not obey Setrlimit call.
if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, oldLimit); err != nil {
h.logger().Printf("ERROR checking open file limit: %s", err)
h.Logger.Printf("ERROR checking open file limit: %s", err)
} else {
if oldLimit.Cur < FileLimit {
h.logger().Printf("WARNING: Tried to set open file limit to %d, but it is %d. You may consider running \"sudo ulimit -n %d\" before starting Pilosa to avoid \"too many open files\" error. See https://www.pilosa.com/docs/administration/#open-file-limits for more information.", FileLimit, oldLimit.Cur, FileLimit)
h.Logger.Printf("WARNING: Tried to set open file limit to %d, but it is %d. You may consider running \"sudo ulimit -n %d\" before starting Pilosa to avoid \"too many open files\" error. See https://www.pilosa.com/docs/administration/#open-file-limits for more information.", FileLimit, oldLimit.Cur, FileLimit)
}
}
}
}
func (h *Holder) logger() *log.Logger { return log.New(h.LogOutput, "", log.LstdFlags) }
func (h *Holder) loadNodeID() (string, error) {
idPath := path.Join(h.Path, "ID")
nodeID := ""
h.logger().Printf("load NodeID: %s", idPath)
h.Logger.Printf("load NodeID: %s", idPath)
if err := os.MkdirAll(h.Path, 0777); err != nil {
return "", err
}

15
holder_test.go
View file

@ -15,6 +15,7 @@
package pilosa_test
import (
"bytes"
"context"
"os"
"path/filepath"
@ -30,6 +31,10 @@ import (
func TestHolder_Open(t *testing.T) {
t.Run("ErrIndexName", func(t *testing.T) {
h := test.MustOpenHolder()
bufLogger := test.NewBufferLogger()
h.Holder.Logger = bufLogger
defer h.Close()
if err := os.Mkdir(h.IndexPath("!"), 0777); err != nil {
@ -39,8 +44,12 @@ func TestHolder_Open(t *testing.T) {
}
if err := h.Reopen(); err != nil {
t.Fatal(err)
} else if logOutput := h.LogOutput.String(); !strings.Contains(logOutput, `ERROR opening index: !`) {
t.Fatalf("expected log error:\n%s", logOutput)
}
if bufbytes, err := bufLogger.ReadAll(); err != nil {
t.Fatal(err)
} else if !bytes.Contains(bufbytes, []byte("ERROR opening index: !")) {
t.Fatalf("expected log error:\n%s", bufbytes)
}
})
@ -68,7 +77,7 @@ func TestHolder_Open(t *testing.T) {
h := test.MustOpenHolder()
defer h.Close()
if _, err := h.CreateIndex("test", pilosa.IndexOptions{}); err != nil {
if _, err := h.CreateIndex("test", pilosa.IndexOptions{TimeQuantum: pilosa.TimeQuantum("YMDH")}); err != nil {
t.Fatal(err)
} else if err := h.Holder.Close(); err != nil {
t.Fatal(err)

62
index.go
View file

@ -17,7 +17,6 @@ package pilosa
import (
"errors"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
@ -31,7 +30,6 @@ import (
// Default index settings.
const (
DefaultColumnLabel = "columnID"
InputDefinitionDir = ".input-definitions"
)
@ -45,9 +43,6 @@ type Index struct {
// This can be overridden by individual frames.
timeQuantum TimeQuantum
// Label used for referring to columns in index.
columnLabel string
// Frames by name.
frames map[string]*Frame
@ -66,7 +61,7 @@ type Index struct {
broadcaster Broadcaster
Stats StatsClient
LogOutput io.Writer
Logger Logger
}
// NewIndex returns a new instance of Index.
@ -88,11 +83,9 @@ func NewIndex(path, name string) (*Index, error) {
NewAttrStore: NewNopAttrStore,
columnAttrStore: NopAttrStore,
columnLabel: DefaultColumnLabel,
broadcaster: NopBroadcaster,
Stats: NopStatsClient,
LogOutput: ioutil.Discard,
Logger: NopLogger,
}, nil
}
@ -105,39 +98,6 @@ func (i *Index) Path() string { return i.path }
// ColumnAttrStore returns the storage for column attributes.
func (i *Index) ColumnAttrStore() AttrStore { return i.columnAttrStore }
// SetColumnLabel sets the column label. Persists to meta file on update.
func (i *Index) SetColumnLabel(v string) error {
i.mu.Lock()
defer i.mu.Unlock()
// Ignore if no change occurred.
if v == "" || i.columnLabel == v {
return nil
}
// Make sure columnLabel is valid name
err := ValidateLabel(v)
if err != nil {
return err
}
// Persist meta data to disk on change.
i.columnLabel = v
if err := i.saveMeta(); err != nil {
return err
}
return nil
}
// ColumnLabel returns the column label.
func (i *Index) ColumnLabel() string {
i.mu.RLock()
v := i.columnLabel
i.mu.RUnlock()
return v
}
// Options returns all options for this index.
func (i *Index) Options() IndexOptions {
i.mu.RLock()
@ -147,7 +107,6 @@ func (i *Index) Options() IndexOptions {
func (i *Index) options() IndexOptions {
return IndexOptions{
ColumnLabel: i.columnLabel,
TimeQuantum: i.timeQuantum,
}
}
@ -217,7 +176,6 @@ func (i *Index) loadMeta() error {
buf, err := ioutil.ReadFile(filepath.Join(i.path, ".meta"))
if os.IsNotExist(err) {
i.timeQuantum = ""
i.columnLabel = DefaultColumnLabel
return nil
} else if err != nil {
return err
@ -229,7 +187,6 @@ func (i *Index) loadMeta() error {
// Copy metadata fields.
i.timeQuantum = TimeQuantum(pb.TimeQuantum)
i.columnLabel = pb.ColumnLabel
return nil
}
@ -239,7 +196,6 @@ func (i *Index) saveMeta() error {
// Marshal metadata.
buf, err := proto.Marshal(&internal.IndexMeta{
TimeQuantum: string(i.timeQuantum),
ColumnLabel: i.columnLabel,
})
if err != nil {
return err
@ -446,11 +402,6 @@ func (i *Index) createFrame(name string, opt FrameOptions) (*Frame, error) {
return nil, ErrInvalidCacheType
}
// Validate that row label does not match column label.
if i.columnLabel == opt.RowLabel || (opt.RowLabel == "" && i.columnLabel == DefaultRowLabel) {
return nil, ErrColumnRowLabelEqual
}
// Validate mutually exclusive options if ranges are enabled.
if opt.RangeEnabled {
if opt.InverseEnabled {
@ -496,10 +447,6 @@ func (i *Index) createFrame(name string, opt FrameOptions) (*Frame, error) {
}
f.cacheType = opt.CacheType
// Set options.
if opt.RowLabel != "" {
f.rowLabel = opt.RowLabel
}
if opt.CacheSize != 0 {
f.cacheSize = opt.CacheSize
}
@ -528,7 +475,7 @@ func (i *Index) newFrame(path, name string) (*Frame, error) {
if err != nil {
return nil, err
}
f.LogOutput = i.LogOutput
f.Logger = i.Logger
f.Stats = i.Stats.WithTags(fmt.Sprintf("frame:%s", name))
f.broadcaster = i.broadcaster
f.rowAttrStore = i.NewAttrStore(filepath.Join(f.path, ".data"))
@ -640,14 +587,12 @@ func encodeIndex(d *Index) *internal.Index {
// IndexOptions represents options to set when initializing an index.
type IndexOptions struct {
ColumnLabel string `json:"columnLabel,omitempty"`
TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"`
}
// Encode converts i into its internal representation.
func (i *IndexOptions) Encode() *internal.IndexMeta {
return &internal.IndexMeta{
ColumnLabel: i.ColumnLabel,
TimeQuantum: string(i.TimeQuantum),
}
}
@ -694,7 +639,6 @@ func (i *Index) createInputDefinition(pb *internal.InputDefinition) (*InputDefin
for _, fr := range pb.Frames {
opt := FrameOptions{
// Deprecating row labels per #810. So, setting the default row label here.
RowLabel: DefaultRowLabel,
InverseEnabled: fr.Meta.InverseEnabled,
CacheType: fr.Meta.CacheType,
CacheSize: fr.Meta.CacheSize,

34
index_test.go
View file

@ -215,32 +215,6 @@ func TestIndex_CreateFrame(t *testing.T) {
}
})
})
// Ensure frame cannot be created with a matching row label.
t.Run("ErrColumnRowLabelEqual", func(t *testing.T) {
t.Run("Explicit", func(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
_, err := index.CreateFrame("f", pilosa.FrameOptions{RowLabel: pilosa.DefaultColumnLabel})
if err != pilosa.ErrColumnRowLabelEqual {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("Default", func(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
if err := index.SetColumnLabel(pilosa.DefaultRowLabel); err != nil {
t.Fatal(err)
}
_, err := index.CreateFrame("f", pilosa.FrameOptions{})
if err != pilosa.ErrColumnRowLabelEqual {
t.Fatalf("unexpected error: %s", err)
}
})
})
}
// Ensure index can delete a frame.
@ -303,7 +277,7 @@ func TestIndex_CreateInputDefinition(t *testing.T) {
defer index.Close()
// Create Input Definition.
frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}}
frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{}}
action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}}
field := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}}
def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field}}
@ -330,7 +304,7 @@ func TestIndex_CreateExistingInputDefinition(t *testing.T) {
}
// Create Input Definition.
frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}}
frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{}}
action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}}
fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}}
def = internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}}
@ -350,7 +324,7 @@ func TestIndex_DeleteInputDefinition(t *testing.T) {
defer index.Close()
// Create Input Definition.
frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}}
frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{}}
action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}}
fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}}
def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}}
@ -381,7 +355,7 @@ func TestIndex_CreateFrameWhenOpenInputDefinition(t *testing.T) {
defer index.Close()
// Create Input Definition.
frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}}
frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{}}
action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}}
fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}}
def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}}

2
input_definition.go
View file

@ -281,7 +281,7 @@ func (i *InputDefinitionInfo) Validate() error {
}
}
// Validate columnLabel and duplicate primaryKey.
// Validate duplicate primaryKey.
for _, field := range i.Fields {
if field.Name == "" {
return ErrInputDefinitionNameRequired

8
input_definition_test.go
View file

@ -30,7 +30,7 @@ func TestInputDefinition_Open(t *testing.T) {
defer index.Close()
// Create Input Definition.
frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}}
frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{}}
action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}}
fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}}
def := internal.InputDefinition{Name: "^", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}}
@ -115,14 +115,14 @@ func TestActionValidation(t *testing.T) {
t.Fatalf("Expect error: %s, actual err: %s", pilosa.ErrInputDefinitionAttrsRequired, err)
}
frame := pilosa.InputFrame{Name: "f", Options: pilosa.FrameOptions{RowLabel: "row"}}
frame := pilosa.InputFrame{Name: "f", Options: pilosa.FrameOptions{}}
info = pilosa.InputDefinitionInfo{Frames: []pilosa.InputFrame{frame}, Fields: []pilosa.InputDefinitionField{field}}
err = info.Validate()
if !strings.Contains(err.Error(), "rowID required for single-row-boolean") {
t.Fatalf("Expected rowID required for single-row-boolean error, actual error: %s", err)
}
frame = pilosa.InputFrame{Name: "^", Options: pilosa.FrameOptions{RowLabel: "row"}}
frame = pilosa.InputFrame{Name: "^", Options: pilosa.FrameOptions{}}
action = pilosa.Action{Frame: "f", ValueDestination: pilosa.InputSingleRowBool, RowID: &rowID}
field = pilosa.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []pilosa.Action{action}}
info = pilosa.InputDefinitionInfo{Frames: []pilosa.InputFrame{frame}, Fields: []pilosa.InputDefinitionField{field}}
@ -131,7 +131,7 @@ func TestActionValidation(t *testing.T) {
t.Fatalf("Expect error: %s, actual err: %s", pilosa.ErrName, err)
}
frame = pilosa.InputFrame{Name: "f", Options: pilosa.FrameOptions{RowLabel: "row"}}
frame = pilosa.InputFrame{Name: "f", Options: pilosa.FrameOptions{}}
action = pilosa.Action{ValueDestination: pilosa.InputSingleRowBool, RowID: &rowID}
field = pilosa.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []pilosa.Action{action}}
info = pilosa.InputDefinitionInfo{Frames: []pilosa.InputFrame{frame}, Fields: []pilosa.InputDefinitionField{field}}

262
internal/private.pb.go
View file

@ -68,7 +68,6 @@ var _ = math.Inf
const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package
type IndexMeta struct {
ColumnLabel string `protobuf:"bytes,1,opt,name=ColumnLabel,proto3" json:"ColumnLabel,omitempty"`
TimeQuantum string `protobuf:"bytes,2,opt,name=TimeQuantum,proto3" json:"TimeQuantum,omitempty"`
}
@ -77,13 +76,6 @@ func (m *IndexMeta) String() string { return proto.CompactTextString(
func (*IndexMeta) ProtoMessage() {}
func (*IndexMeta) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{0} }
func (m *IndexMeta) GetColumnLabel() string {
if m != nil {
return m.ColumnLabel
}
return ""
}
func (m *IndexMeta) GetTimeQuantum() string {
if m != nil {
return m.TimeQuantum
@ -92,7 +84,6 @@ func (m *IndexMeta) GetTimeQuantum() string {
}
type FrameMeta struct {
RowLabel string `protobuf:"bytes,1,opt,name=RowLabel,proto3" json:"RowLabel,omitempty"`
InverseEnabled bool `protobuf:"varint,2,opt,name=InverseEnabled,proto3" json:"InverseEnabled,omitempty"`
CacheType string `protobuf:"bytes,3,opt,name=CacheType,proto3" json:"CacheType,omitempty"`
CacheSize uint32 `protobuf:"varint,4,opt,name=CacheSize,proto3" json:"CacheSize,omitempty"`
@ -106,13 +97,6 @@ func (m *FrameMeta) String() string { return proto.CompactTextString(
func (*FrameMeta) ProtoMessage() {}
func (*FrameMeta) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{1} }
func (m *FrameMeta) GetRowLabel() string {
if m != nil {
return m.RowLabel
}
return ""
}
func (m *FrameMeta) GetInverseEnabled() bool {
if m != nil {
return m.InverseEnabled
@ -1248,12 +1232,6 @@ func (m *IndexMeta) MarshalTo(dAtA []byte) (int, error) {
_ = i
var l int
_ = l
if len(m.ColumnLabel) > 0 {
dAtA[i] = 0xa
i++
i = encodeVarintPrivate(dAtA, i, uint64(len(m.ColumnLabel)))
i += copy(dAtA[i:], m.ColumnLabel)
}
if len(m.TimeQuantum) > 0 {
dAtA[i] = 0x12
i++
@ -1278,12 +1256,6 @@ func (m *FrameMeta) MarshalTo(dAtA []byte) (int, error) {
_ = i
var l int
_ = l
if len(m.RowLabel) > 0 {
dAtA[i] = 0xa
i++
i = encodeVarintPrivate(dAtA, i, uint64(len(m.RowLabel)))
i += copy(dAtA[i:], m.RowLabel)
}
if m.InverseEnabled {
dAtA[i] = 0x10
i++
@ -2774,10 +2746,6 @@ func encodeVarintPrivate(dAtA []byte, offset int, v uint64) int {
func (m *IndexMeta) Size() (n int) {
var l int
_ = l
l = len(m.ColumnLabel)
if l > 0 {
n += 1 + l + sovPrivate(uint64(l))
}
l = len(m.TimeQuantum)
if l > 0 {
n += 1 + l + sovPrivate(uint64(l))
@ -2788,10 +2756,6 @@ func (m *IndexMeta) Size() (n int) {
func (m *FrameMeta) Size() (n int) {
var l int
_ = l
l = len(m.RowLabel)
if l > 0 {
n += 1 + l + sovPrivate(uint64(l))
}
if m.InverseEnabled {
n += 2
}
@ -3475,35 +3439,6 @@ func (m *IndexMeta) Unmarshal(dAtA []byte) error {
return fmt.Errorf("proto: IndexMeta: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field ColumnLabel", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
stringLen |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
intStringLen := int(stringLen)
if intStringLen < 0 {
return ErrInvalidLengthPrivate
}
postIndex := iNdEx + intStringLen
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.ColumnLabel = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
case 2:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field TimeQuantum", wireType)
@ -3583,35 +3518,6 @@ func (m *FrameMeta) Unmarshal(dAtA []byte) error {
return fmt.Errorf("proto: FrameMeta: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field RowLabel", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
stringLen |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
intStringLen := int(stringLen)
if intStringLen < 0 {
return ErrInvalidLengthPrivate
}
postIndex := iNdEx + intStringLen
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.RowLabel = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
case 2:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field InverseEnabled", wireType)
@ -8667,89 +8573,87 @@ var (
func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) }
var fileDescriptorPrivate = []byte{
// 1334 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0x5d, 0x6f, 0x1b, 0x45,
0x17, 0x7e, 0xd7, 0x6b, 0x3b, 0xf6, 0x71, 0x9c, 0x38, 0xd3, 0x34, 0xaf, 0x13, 0x45, 0xae, 0x19,
0x15, 0x1a, 0x2a, 0x11, 0x95, 0x54, 0x42, 0x34, 0x50, 0xa9, 0xc4, 0x76, 0xd5, 0x85, 0x26, 0x94,
0x71, 0x12, 0x24, 0x24, 0x90, 0x26, 0xf6, 0x90, 0xae, 0xb2, 0xde, 0x35, 0xbb, 0xe3, 0x24, 0xee,
0x05, 0x97, 0x08, 0x09, 0x71, 0x8f, 0xb8, 0xe5, 0xcf, 0x70, 0xc9, 0x4f, 0x40, 0xe1, 0x47, 0x20,
0x71, 0x03, 0x9a, 0xaf, 0xdd, 0xf5, 0x57, 0xd2, 0x04, 0xee, 0xf6, 0x3c, 0x73, 0xce, 0x99, 0x67,
0xce, 0xd7, 0xcc, 0x42, 0xb9, 0x1f, 0xba, 0xa7, 0x94, 0xb3, 0xcd, 0x7e, 0x18, 0xf0, 0x00, 0x15,
0x5c, 0x9f, 0xb3, 0xd0, 0xa7, 0x1e, 0xfe, 0x14, 0x8a, 0x8e, 0xdf, 0x65, 0xe7, 0xbb, 0x8c, 0x53,
0x54, 0x87, 0x52, 0x23, 0xf0, 0x06, 0x3d, 0xff, 0x39, 0x3d, 0x62, 0x5e, 0xd5, 0xaa, 0x5b, 0x1b,
0x45, 0x92, 0x86, 0x84, 0xc6, 0xbe, 0xdb, 0x63, 0x9f, 0x0d, 0xa8, 0xcf, 0x07, 0xbd, 0x6a, 0x46,
0x69, 0xa4, 0x20, 0xfc, 0x97, 0x05, 0xc5, 0xa7, 0x21, 0xed, 0x31, 0xe9, 0x71, 0x0d, 0x0a, 0x24,
0x38, 0x4b, 0xbb, 0x8b, 0x65, 0xf4, 0x16, 0x2c, 0x38, 0xfe, 0x29, 0x0b, 0x23, 0xd6, 0xf2, 0xe9,
0x91, 0xc7, 0xba, 0xd2, 0x5d, 0x81, 0x8c, 0xa1, 0x68, 0x1d, 0x8a, 0x0d, 0xda, 0x79, 0xc9, 0xf6,
0x87, 0x7d, 0x56, 0xb5, 0xa5, 0x93, 0x04, 0x88, 0x57, 0xdb, 0xee, 0x2b, 0x56, 0xcd, 0xd6, 0xad,
0x8d, 0x32, 0x49, 0x80, 0x71, 0xbe, 0xb9, 0x09, 0xbe, 0x08, 0xc3, 0x3c, 0xa1, 0xfe, 0x71, 0xcc,
0x21, 0x2f, 0x39, 0x8c, 0x60, 0xe8, 0x1e, 0xe4, 0x9f, 0xba, 0xcc, 0xeb, 0x46, 0xd5, 0xb9, 0xba,
0xbd, 0x51, 0xda, 0x5a, 0xdc, 0x34, 0xf1, 0xdb, 0x94, 0x38, 0xd1, 0xcb, 0x18, 0xc3, 0x82, 0xd3,
0xeb, 0x07, 0x21, 0x27, 0x2c, 0xea, 0x07, 0x7e, 0xc4, 0x50, 0x05, 0xec, 0x56, 0x18, 0xea, 0xb3,
0x8b, 0x4f, 0xfc, 0x2d, 0x54, 0x76, 0xbc, 0xa0, 0x73, 0xd2, 0xa4, 0x9c, 0x12, 0xf6, 0xcd, 0x80,
0x45, 0x1c, 0x2d, 0x43, 0x4e, 0x66, 0x41, 0xeb, 0x29, 0x41, 0xa0, 0x32, 0x92, 0x3a, 0xcc, 0x4a,
0x10, 0xa8, 0xb4, 0x97, 0xa1, 0xc8, 0x12, 0x25, 0x08, 0xb4, 0xed, 0xb9, 0x1d, 0x15, 0x82, 0x2c,
0x51, 0x02, 0x42, 0x90, 0x3d, 0x74, 0xd9, 0x99, 0x3e, 0xb7, 0xfc, 0xc6, 0x0e, 0x2c, 0xa5, 0xf6,
0xd7, 0x34, 0x57, 0x20, 0x4f, 0x82, 0x33, 0xa7, 0x19, 0x55, 0xad, 0xba, 0xbd, 0x91, 0x25, 0x5a,
0x92, 0xd1, 0x95, 0xe9, 0x17, 0x4b, 0x19, 0xb9, 0x94, 0x00, 0x78, 0x15, 0x72, 0x32, 0xd4, 0xe2,
0x94, 0x89, 0xad, 0xf8, 0xc4, 0x7f, 0x5b, 0x50, 0xdc, 0xa5, 0xe7, 0x92, 0x46, 0x84, 0x1e, 0x43,
0xa1, 0xcd, 0xa9, 0xdf, 0xa5, 0x61, 0x57, 0x2a, 0x95, 0xb6, 0xde, 0x48, 0x42, 0x18, 0xab, 0x6d,
0x1a, 0x9d, 0x96, 0xcf, 0xc3, 0x21, 0x89, 0x4d, 0xd0, 0x36, 0xcc, 0xe9, 0x9a, 0x90, 0x1c, 0x4a,
0x5b, 0xf5, 0x69, 0xd6, 0x71, 0xd9, 0x08, 0x63, 0x63, 0xb0, 0xf6, 0x01, 0x94, 0x47, 0xdc, 0x0a,
0xae, 0x27, 0x6c, 0x68, 0x32, 0x72, 0xc2, 0x86, 0x22, 0x76, 0xa7, 0xd4, 0x1b, 0xa8, 0x38, 0x67,
0x89, 0x12, 0xb6, 0x33, 0xef, 0x5b, 0x6b, 0xdb, 0x30, 0x9f, 0xf6, 0x7a, 0x1d, 0x5b, 0xfc, 0x15,
0xa0, 0x46, 0xc8, 0x28, 0x67, 0x92, 0xde, 0x2e, 0x8b, 0x22, 0x7a, 0xcc, 0x66, 0x67, 0x5a, 0x65,
0x2f, 0x93, 0xce, 0xde, 0x3a, 0x14, 0x9d, 0xc8, 0x1c, 0xdc, 0x96, 0x75, 0x99, 0x00, 0xf8, 0x3e,
0xa0, 0x26, 0xf3, 0x18, 0x67, 0xba, 0x7f, 0x2f, 0xf1, 0x8f, 0xdb, 0x86, 0xcb, 0xd5, 0xba, 0xe8,
0x1e, 0x64, 0x45, 0xeb, 0x4a, 0x2a, 0xa5, 0xad, 0x5b, 0x49, 0xa4, 0xe3, 0x39, 0x41, 0xa4, 0x02,
0x76, 0x8d, 0x53, 0xdd, 0xee, 0x57, 0x1c, 0x70, 0x4a, 0x29, 0x9b, 0xad, 0xec, 0xf1, 0xad, 0xe2,
0x01, 0xa2, 0xb7, 0x7a, 0x62, 0xce, 0x7a, 0xd3, 0xad, 0xf0, 0x71, 0x4c, 0x56, 0x74, 0xea, 0x4d,
0xc8, 0xbe, 0x09, 0x39, 0x69, 0xab, 0xd9, 0x4e, 0xcc, 0x00, 0xb5, 0x8a, 0x0f, 0x63, 0xaa, 0x37,
0xdd, 0x68, 0x39, 0xbd, 0x51, 0xd1, 0xf8, 0xfd, 0x42, 0xeb, 0x8a, 0x9e, 0xde, 0x13, 0x36, 0xca,
0x93, 0xfc, 0x9e, 0x9d, 0xb3, 0xb1, 0x40, 0x0a, 0xdf, 0x62, 0x08, 0x44, 0x55, 0xbb, 0x6e, 0x0b,
0xdf, 0x52, 0xc0, 0x0f, 0x21, 0xdf, 0xee, 0xbc, 0x64, 0x3d, 0x8a, 0xde, 0x16, 0x9d, 0xd6, 0x65,
0xe7, 0x2c, 0xd2, 0x7d, 0xba, 0x38, 0x96, 0x7f, 0x62, 0xd6, 0xf1, 0x0f, 0x96, 0x3e, 0xd3, 0x0c,
0x46, 0x79, 0xb9, 0x77, 0x54, 0xcd, 0x4e, 0x8c, 0x4c, 0x81, 0x13, 0xbd, 0x8c, 0x5a, 0x50, 0x71,
0xfc, 0xfe, 0x80, 0x37, 0xd9, 0xd7, 0xae, 0xef, 0x72, 0x37, 0xf0, 0xa3, 0x6a, 0x5e, 0x9a, 0xac,
0xa6, 0xb7, 0x1e, 0xd1, 0x20, 0x13, 0x26, 0xf8, 0x3b, 0x0b, 0x16, 0xc7, 0xc0, 0x2b, 0x78, 0x65,
0x2e, 0xe7, 0xf5, 0x5e, 0x3c, 0xf3, 0x6d, 0xa9, 0x58, 0x9b, 0xc9, 0x66, 0xf4, 0x0a, 0xf8, 0xc5,
0x82, 0xe5, 0x69, 0x0a, 0x53, 0xd9, 0xd4, 0x00, 0x5e, 0x84, 0x6e, 0x8f, 0x86, 0xc3, 0x4f, 0xd8,
0x50, 0x5f, 0x7f, 0x29, 0x04, 0x7d, 0x0e, 0x2b, 0x63, 0xbe, 0x3e, 0xea, 0xa8, 0x10, 0x29, 0x52,
0x77, 0x66, 0x92, 0x52, 0x7a, 0x64, 0x86, 0x39, 0xfe, 0xd3, 0x82, 0xdb, 0x53, 0x97, 0x92, 0x9a,
0xb4, 0xd2, 0x35, 0x79, 0x1f, 0x2a, 0x87, 0x62, 0xb2, 0x35, 0x59, 0xc4, 0x5d, 0x9f, 0x0a, 0x4d,
0x5d, 0xb4, 0x13, 0x38, 0x72, 0xa0, 0x20, 0xb1, 0x5d, 0xda, 0xd7, 0x34, 0xdf, 0xb9, 0x82, 0xe6,
0xa6, 0xd1, 0xd7, 0x83, 0xdf, 0x88, 0x82, 0x8c, 0xbc, 0x88, 0xcc, 0xad, 0x26, 0x05, 0x31, 0xd2,
0x47, 0x0c, 0xae, 0x35, 0x96, 0x03, 0x58, 0x37, 0xa3, 0x70, 0x84, 0xc9, 0xe5, 0x9d, 0xfa, 0x08,
0x20, 0x51, 0xd5, 0x13, 0xe0, 0x92, 0xfa, 0x4c, 0x29, 0xe3, 0x67, 0xb0, 0x6e, 0xe6, 0xf4, 0x35,
0x36, 0x34, 0xd5, 0x92, 0x49, 0xaa, 0x05, 0xb7, 0xc0, 0x3e, 0x20, 0x8e, 0xb8, 0xab, 0x65, 0xb7,
0x9a, 0x14, 0x69, 0x49, 0x98, 0x3c, 0x0b, 0x22, 0x6e, 0x4c, 0xc4, 0xb7, 0xc0, 0x5e, 0x04, 0x21,
0x97, 0x8c, 0xcb, 0x44, 0x7e, 0xe3, 0x2f, 0x21, 0xbb, 0x17, 0x74, 0x19, 0x5a, 0x80, 0x8c, 0xd3,
0xd4, 0x3e, 0x32, 0x4e, 0x13, 0xdd, 0x91, 0xee, 0xf5, 0x0c, 0x29, 0x27, 0x87, 0x3b, 0x20, 0x0e,
0x91, 0x1b, 0xdf, 0x85, 0xb2, 0x13, 0x35, 0x82, 0x20, 0xec, 0x8a, 0x54, 0x07, 0xa1, 0xbe, 0x93,
0x46, 0x41, 0xfc, 0x04, 0x2a, 0xc2, 0x7d, 0x9b, 0x53, 0x1e, 0x4f, 0xea, 0x15, 0xc8, 0x0b, 0x2c,
0xde, 0x4e, 0x4b, 0xf2, 0xde, 0x13, 0x7a, 0x66, 0x00, 0x4a, 0x01, 0x3f, 0x57, 0x1e, 0x5a, 0xa7,
0xcc, 0xe7, 0xa9, 0x28, 0x49, 0x59, 0x3a, 0x28, 0x13, 0x25, 0x20, 0xac, 0x8e, 0xa2, 0x39, 0x2f,
0x24, 0x9c, 0x05, 0x4a, 0xe4, 0x1a, 0xfe, 0xd1, 0x02, 0x30, 0x84, 0x06, 0x51, 0x6c, 0x62, 0xcd,
0x36, 0x41, 0xef, 0xa6, 0xde, 0x2e, 0x93, 0x33, 0x35, 0x5e, 0x22, 0xa9, 0x17, 0xce, 0x86, 0x19,
0xa1, 0xba, 0x38, 0x2a, 0x89, 0xbe, 0xc2, 0x75, 0x9a, 0xc4, 0xb5, 0x59, 0x6e, 0x78, 0x83, 0x88,
0xb3, 0x50, 0x33, 0x12, 0x6f, 0x2c, 0x05, 0xc4, 0xf1, 0x49, 0x80, 0xe9, 0x21, 0x42, 0x77, 0x21,
0x27, 0x98, 0x9a, 0x39, 0x30, 0x7e, 0x0c, 0xb5, 0x88, 0xdb, 0xfa, 0x26, 0x99, 0x3a, 0x7b, 0x10,
0x64, 0xe5, 0x8b, 0x5a, 0x97, 0x8b, 0x7c, 0x4c, 0x57, 0xc0, 0xde, 0x75, 0x55, 0x7d, 0xdb, 0x44,
0x7c, 0x4a, 0x84, 0x9e, 0xcb, 0xfe, 0x13, 0x08, 0x15, 0x6f, 0x89, 0x25, 0xd5, 0x40, 0xe2, 0xee,
0xb8, 0xc9, 0xfd, 0x66, 0x1e, 0xa5, 0x76, 0xea, 0x51, 0xda, 0x86, 0x25, 0xd5, 0x24, 0xff, 0xa5,
0xd3, 0x9f, 0x33, 0xb0, 0x44, 0x58, 0xe4, 0xbe, 0x62, 0x8e, 0x1f, 0xf1, 0x70, 0x10, 0x0f, 0xb8,
0x8f, 0x83, 0x23, 0x1d, 0x6a, 0x9b, 0x28, 0xe1, 0x75, 0x2a, 0x09, 0x3d, 0x10, 0xbf, 0x47, 0xa3,
0xd5, 0x3f, 0xa9, 0x9a, 0x56, 0x41, 0x0f, 0x60, 0xae, 0x1d, 0x0c, 0xc2, 0x4e, 0x7c, 0x0d, 0xae,
0x24, 0xda, 0x8a, 0x99, 0x5a, 0x26, 0x46, 0x2d, 0x55, 0x47, 0xb9, 0xcb, 0xeb, 0x08, 0x3d, 0x1e,
0xab, 0x23, 0xf9, 0xe7, 0x52, 0xda, 0xfa, 0x7f, 0x62, 0x30, 0xb2, 0x4c, 0x46, 0xb5, 0xf1, 0xf7,
0x16, 0xcc, 0xa7, 0x29, 0xbc, 0x56, 0x63, 0xc4, 0x19, 0xc9, 0x4c, 0xcd, 0x88, 0x3d, 0x2d, 0x23,
0xd9, 0x24, 0x23, 0xc9, 0x3b, 0x37, 0x97, 0x7a, 0xe7, 0xe2, 0x13, 0x58, 0x9d, 0x48, 0x53, 0x23,
0xe8, 0xf5, 0x45, 0x3d, 0xfc, 0x8b, 0x74, 0x89, 0x91, 0x11, 0x86, 0x3a, 0x51, 0x45, 0xa2, 0x04,
0xfc, 0x08, 0x6e, 0xb7, 0x19, 0x4f, 0x25, 0xc9, 0x54, 0x5b, 0x1d, 0xec, 0x3d, 0x76, 0x36, 0xe3,
0xf8, 0x62, 0x09, 0x7f, 0x08, 0xd5, 0x83, 0x7e, 0x97, 0x72, 0x76, 0x23, 0xeb, 0x1d, 0x28, 0xec,
0x07, 0xfd, 0xc0, 0x0b, 0x8e, 0x87, 0x57, 0xb4, 0x7c, 0x15, 0xe6, 0xd4, 0x7c, 0x54, 0x8f, 0x94,
0x22, 0x31, 0x22, 0xbe, 0x25, 0x0a, 0xba, 0x43, 0xbd, 0xce, 0xc0, 0x13, 0x34, 0xc4, 0xbf, 0x57,
0xb4, 0x53, 0xf9, 0xf5, 0xa2, 0x66, 0xfd, 0x76, 0x51, 0xb3, 0x7e, 0xbf, 0xa8, 0x59, 0x3f, 0xfd,
0x51, 0xfb, 0xdf, 0x51, 0x5e, 0xfe, 0xe5, 0x3f, 0xfc, 0x27, 0x00, 0x00, 0xff, 0xff, 0x66, 0x19,
0x3d, 0xd2, 0xf6, 0x0f, 0x00, 0x00,
// 1308 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0x4f, 0x6f, 0x1b, 0x45,
0x14, 0x67, 0xbd, 0xb6, 0x13, 0x3f, 0xd7, 0xa9, 0x33, 0x6d, 0x83, 0x5b, 0x45, 0xae, 0x19, 0x15,
0x1a, 0x2a, 0x35, 0x2a, 0xa9, 0x84, 0x68, 0xa1, 0x52, 0x69, 0xec, 0xaa, 0x0b, 0xa4, 0x2a, 0xe3,
0xb6, 0x48, 0x48, 0x20, 0x4d, 0xed, 0x21, 0x5d, 0x65, 0xbd, 0x6b, 0x76, 0xc7, 0x49, 0xdc, 0x03,
0x47, 0x84, 0x84, 0xb8, 0x23, 0xae, 0x7c, 0x19, 0x8e, 0x7c, 0x02, 0x84, 0xc2, 0x87, 0xe0, 0x08,
0x9a, 0x37, 0x33, 0xbb, 0xeb, 0x7f, 0x49, 0x13, 0xb8, 0xed, 0xfb, 0xff, 0x9b, 0xf7, 0x6f, 0x66,
0xa1, 0x36, 0x8c, 0xfd, 0x7d, 0x2e, 0xc5, 0xe6, 0x30, 0x8e, 0x64, 0x44, 0x96, 0xfd, 0x50, 0x8a,
0x38, 0xe4, 0x01, 0xbd, 0x09, 0x15, 0x2f, 0xec, 0x8b, 0xc3, 0x1d, 0x21, 0x39, 0x69, 0x41, 0xf5,
0xa9, 0x3f, 0x10, 0x9f, 0x8f, 0x78, 0x28, 0x47, 0x83, 0x46, 0xa1, 0xe5, 0x6c, 0x54, 0x58, 0x9e,
0x45, 0xff, 0x70, 0xa0, 0xf2, 0x30, 0xe6, 0x03, 0x81, 0xfa, 0xef, 0xc0, 0x8a, 0x17, 0xee, 0x8b,
0x38, 0x11, 0x9d, 0x90, 0xbf, 0x08, 0x44, 0x1f, 0x4d, 0x96, 0xd9, 0x14, 0x97, 0xac, 0x43, 0x65,
0x9b, 0xf7, 0x5e, 0x8a, 0xa7, 0xe3, 0xa1, 0x68, 0xb8, 0xe8, 0x35, 0x63, 0xa4, 0xd2, 0xae, 0xff,
0x4a, 0x34, 0x8a, 0x2d, 0x67, 0xa3, 0xc6, 0x32, 0xc6, 0x34, 0xa6, 0xd2, 0x0c, 0x26, 0x42, 0xe1,
0x1c, 0xe3, 0xe1, 0x6e, 0x8a, 0xa1, 0x8c, 0x18, 0x26, 0x78, 0xe4, 0x3a, 0x94, 0x1f, 0xfa, 0x22,
0xe8, 0x27, 0x8d, 0xa5, 0x96, 0xbb, 0x51, 0xdd, 0x3a, 0xbf, 0x69, 0x33, 0xb0, 0x89, 0x7c, 0x66,
0xc4, 0x94, 0xc2, 0x8a, 0x37, 0x18, 0x46, 0xb1, 0x64, 0x22, 0x19, 0x46, 0x61, 0x22, 0x48, 0x1d,
0xdc, 0x4e, 0x1c, 0x37, 0x1c, 0x0c, 0xac, 0x3e, 0xe9, 0x77, 0x50, 0x7f, 0x10, 0x44, 0xbd, 0xbd,
0x36, 0x97, 0x9c, 0x89, 0x6f, 0x47, 0x22, 0x91, 0xe4, 0x22, 0x94, 0x30, 0x8f, 0x46, 0x4f, 0x13,
0x8a, 0x8b, 0xd9, 0x32, 0xa9, 0xd4, 0x84, 0xe2, 0xa2, 0x3d, 0xa6, 0xa2, 0xc8, 0x34, 0xa1, 0xb8,
0xdd, 0xc0, 0xef, 0xe9, 0x14, 0x14, 0x99, 0x26, 0x08, 0x81, 0xe2, 0x73, 0x5f, 0x1c, 0x98, 0x73,
0xe3, 0x37, 0xf5, 0x60, 0x35, 0x17, 0xdf, 0xc0, 0x5c, 0x83, 0x32, 0x8b, 0x0e, 0xbc, 0x76, 0xd2,
0x70, 0x5a, 0xee, 0x46, 0x91, 0x19, 0x0a, 0xb3, 0x1b, 0x05, 0xa3, 0x41, 0xa8, 0x44, 0x05, 0x14,
0x65, 0x0c, 0x7a, 0x19, 0x4a, 0x98, 0x6a, 0x75, 0xca, 0xcc, 0x56, 0x7d, 0xd2, 0x7f, 0x1c, 0xa8,
0xec, 0xf0, 0x43, 0x84, 0x91, 0x90, 0x7b, 0xb0, 0xdc, 0x95, 0x3c, 0xec, 0xf3, 0xb8, 0x8f, 0x4a,
0xd5, 0xad, 0xb7, 0xb2, 0x14, 0xa6, 0x6a, 0x9b, 0x56, 0xa7, 0x13, 0xca, 0x78, 0xcc, 0x52, 0x13,
0x72, 0x17, 0x96, 0x4c, 0x4f, 0x20, 0x86, 0xea, 0x56, 0x6b, 0x9e, 0x75, 0xda, 0x36, 0xca, 0xd8,
0x1a, 0x5c, 0xf9, 0x10, 0x6a, 0x13, 0x6e, 0x15, 0xd6, 0x3d, 0x31, 0xb6, 0x15, 0xd9, 0x13, 0x63,
0x95, 0xbb, 0x7d, 0x1e, 0x8c, 0x74, 0x9e, 0x8b, 0x4c, 0x13, 0x77, 0x0b, 0x1f, 0x38, 0x57, 0xee,
0xc2, 0xb9, 0xbc, 0xd7, 0xd3, 0xd8, 0xd2, 0xaf, 0x81, 0x6c, 0xc7, 0x82, 0x4b, 0x81, 0xf0, 0x76,
0x44, 0x92, 0xf0, 0x5d, 0xb1, 0xb8, 0xd2, 0xba, 0x7a, 0x85, 0x7c, 0xf5, 0xd6, 0xa1, 0xe2, 0x25,
0xf6, 0xe0, 0x2e, 0xf6, 0x65, 0xc6, 0xa0, 0x37, 0x80, 0xb4, 0x45, 0x20, 0xa4, 0x30, 0x13, 0x78,
0x8c, 0x7f, 0xda, 0xb5, 0x58, 0x4e, 0xd6, 0x25, 0xd7, 0xa1, 0xa8, 0xc6, 0x13, 0xa1, 0x54, 0xb7,
0x2e, 0x64, 0x99, 0x4e, 0x27, 0x9d, 0xa1, 0x02, 0xf5, 0xad, 0x53, 0x33, 0xd2, 0x27, 0x1c, 0x70,
0x4e, 0x2b, 0xdb, 0x50, 0xee, 0x74, 0xa8, 0x74, 0x49, 0x98, 0x50, 0xf7, 0xed, 0x59, 0xcf, 0x1a,
0x8a, 0xee, 0xa6, 0x60, 0xd5, 0xa4, 0x9e, 0x05, 0xec, 0xdb, 0x50, 0x42, 0x5b, 0x83, 0x76, 0x66,
0x07, 0x68, 0x29, 0x7d, 0x9e, 0x42, 0x3d, 0x6b, 0xa0, 0x8b, 0xf9, 0x40, 0x15, 0xeb, 0xf7, 0x4b,
0xa3, 0xab, 0x66, 0xfa, 0xb1, 0xb2, 0xd1, 0x9e, 0xf0, 0x7b, 0x71, 0xcd, 0xa6, 0x12, 0xa9, 0x7c,
0xab, 0x25, 0x90, 0x34, 0xdc, 0x96, 0xab, 0x7c, 0x23, 0x41, 0x6f, 0x43, 0xb9, 0xdb, 0x7b, 0x29,
0x06, 0x9c, 0xbc, 0xab, 0x26, 0xad, 0x2f, 0x0e, 0x45, 0x62, 0xe6, 0xf4, 0xfc, 0x54, 0xfd, 0x99,
0x95, 0xd3, 0x1f, 0x1d, 0x73, 0xa6, 0x05, 0x88, 0xca, 0x18, 0x3b, 0x69, 0x14, 0x67, 0x56, 0xa6,
0xe2, 0x33, 0x23, 0x26, 0x1d, 0xa8, 0x7b, 0xe1, 0x70, 0x24, 0xdb, 0xe2, 0x1b, 0x3f, 0xf4, 0xa5,
0x1f, 0x85, 0x49, 0xa3, 0x8c, 0x26, 0x97, 0xf3, 0xa1, 0x27, 0x34, 0xd8, 0x8c, 0x09, 0xfd, 0xde,
0x81, 0xf3, 0x53, 0xcc, 0x13, 0x70, 0x15, 0x8e, 0xc7, 0xf5, 0x7e, 0xba, 0xf3, 0x5d, 0x54, 0x6c,
0x2e, 0x44, 0x33, 0x79, 0x05, 0xfc, 0xea, 0xc0, 0xc5, 0x79, 0x0a, 0x73, 0xd1, 0x34, 0x01, 0x9e,
0xc4, 0xfe, 0x80, 0xc7, 0xe3, 0x4f, 0xc5, 0xd8, 0x5c, 0x7f, 0x39, 0x0e, 0xf9, 0x02, 0xd6, 0xa6,
0x7c, 0x7d, 0xdc, 0xd3, 0x29, 0xd2, 0xa0, 0xae, 0x2e, 0x04, 0xa5, 0xf5, 0xd8, 0x02, 0x73, 0xfa,
0xb7, 0x03, 0x97, 0xe6, 0x8a, 0xb2, 0x9e, 0x74, 0xf2, 0x3d, 0x79, 0x03, 0xea, 0xcf, 0xd5, 0x66,
0x6b, 0x8b, 0x44, 0xfa, 0x21, 0x57, 0x9a, 0xa6, 0x69, 0x67, 0xf8, 0xc4, 0x83, 0x65, 0xe4, 0xed,
0xf0, 0xa1, 0x81, 0x79, 0xf3, 0x04, 0x98, 0x9b, 0x56, 0xdf, 0x2c, 0x7e, 0x4b, 0x2a, 0x30, 0x78,
0x11, 0xd9, 0x5b, 0x0d, 0x09, 0xb5, 0xd2, 0x27, 0x0c, 0x4e, 0xb5, 0x96, 0x23, 0x58, 0xb7, 0xab,
0x70, 0x02, 0xc9, 0xf1, 0x93, 0x7a, 0x07, 0x20, 0x53, 0x35, 0x1b, 0xe0, 0x98, 0xfe, 0xcc, 0x29,
0xd3, 0x47, 0xb0, 0x6e, 0xf7, 0xf4, 0x29, 0x02, 0xda, 0x6e, 0x29, 0x64, 0xdd, 0x42, 0x3b, 0xe0,
0x3e, 0x63, 0x9e, 0xba, 0xab, 0x71, 0x5a, 0x6d, 0x89, 0x0c, 0xa5, 0x4c, 0x1e, 0x45, 0x89, 0xb4,
0x26, 0xea, 0x5b, 0xf1, 0x9e, 0x44, 0xb1, 0x44, 0xc4, 0x35, 0x86, 0xdf, 0xf4, 0x2b, 0x28, 0x3e,
0x8e, 0xfa, 0x82, 0xac, 0x40, 0xc1, 0x6b, 0x1b, 0x1f, 0x05, 0xaf, 0x4d, 0xae, 0xa2, 0x7b, 0xb3,
0x43, 0x6a, 0xd9, 0xe1, 0x9e, 0x31, 0x8f, 0x61, 0xe0, 0x6b, 0x50, 0xf3, 0x92, 0xed, 0x28, 0x8a,
0xfb, 0xaa, 0xd4, 0x51, 0x6c, 0xee, 0xa4, 0x49, 0x26, 0xbd, 0x0f, 0x75, 0xe5, 0xbe, 0x2b, 0xb9,
0x4c, 0x37, 0xf5, 0x1a, 0x94, 0x15, 0x2f, 0x0d, 0x67, 0x28, 0xbc, 0xf7, 0x94, 0x9e, 0x5d, 0x80,
0x48, 0xd0, 0xcf, 0xb4, 0x87, 0xce, 0xbe, 0x08, 0x65, 0x2e, 0x4b, 0x48, 0xa3, 0x83, 0x1a, 0xd3,
0x04, 0xa1, 0xfa, 0x28, 0x06, 0xf3, 0x4a, 0x86, 0x59, 0x71, 0x19, 0xca, 0xe8, 0x4f, 0x0e, 0x80,
0x05, 0x34, 0x4a, 0x52, 0x13, 0x67, 0xb1, 0x09, 0x79, 0x2f, 0xf7, 0x76, 0x99, 0xdd, 0xa9, 0xa9,
0x88, 0xe5, 0x5e, 0x38, 0x1b, 0x76, 0x85, 0x9a, 0xe6, 0xa8, 0x67, 0xfa, 0x9a, 0x6f, 0xca, 0xa4,
0xae, 0xcd, 0xda, 0x76, 0x30, 0x4a, 0xa4, 0x88, 0x0d, 0x22, 0xf5, 0xc6, 0xd2, 0x8c, 0x34, 0x3f,
0x19, 0x63, 0x7e, 0x8a, 0xc8, 0x35, 0x28, 0x29, 0xa4, 0x76, 0x0f, 0x4c, 0x1f, 0x43, 0x0b, 0x69,
0xd7, 0xdc, 0x24, 0x73, 0x77, 0x0f, 0x81, 0x22, 0xbe, 0xa8, 0x4d, 0xbb, 0xe0, 0x63, 0xba, 0x0e,
0xee, 0x8e, 0xaf, 0xfb, 0xdb, 0x65, 0xea, 0x13, 0x39, 0xfc, 0x10, 0xe7, 0x4f, 0x71, 0xb8, 0x7a,
0x4b, 0xac, 0xea, 0x01, 0x52, 0x77, 0xc7, 0x59, 0xee, 0x37, 0xfb, 0x28, 0x75, 0x73, 0x8f, 0xd2,
0x2e, 0xac, 0xea, 0x21, 0xf9, 0x3f, 0x9d, 0xfe, 0x52, 0x80, 0x55, 0x26, 0x12, 0xff, 0x95, 0xf0,
0xc2, 0x44, 0xc6, 0xa3, 0x74, 0xc1, 0x7d, 0x12, 0xbd, 0x30, 0xa9, 0x76, 0x99, 0x26, 0x5e, 0xa7,
0x93, 0xc8, 0x2d, 0xa8, 0x4e, 0x77, 0xff, 0xac, 0x6a, 0x5e, 0x85, 0xdc, 0x82, 0xa5, 0x6e, 0x34,
0x8a, 0x7b, 0xe9, 0x35, 0xb8, 0x96, 0x69, 0x6b, 0x64, 0x5a, 0xcc, 0xac, 0x5a, 0xae, 0x8f, 0x4a,
0xc7, 0xf7, 0x11, 0xb9, 0x37, 0xd5, 0x47, 0xf8, 0xe7, 0x52, 0xdd, 0x7a, 0x33, 0x33, 0x98, 0x10,
0xb3, 0x49, 0x6d, 0xfa, 0x83, 0x03, 0xe7, 0xf2, 0x10, 0x5e, 0x6b, 0x30, 0xd2, 0x8a, 0x14, 0xe6,
0x56, 0xc4, 0x9d, 0x57, 0x91, 0x62, 0x56, 0x91, 0xec, 0x9d, 0x5b, 0xca, 0xbd, 0x73, 0xe9, 0x1e,
0x5c, 0x9e, 0x29, 0xd3, 0x76, 0x34, 0x18, 0xaa, 0x7e, 0xf8, 0x0f, 0xe5, 0x52, 0x2b, 0x23, 0x8e,
0x4d, 0xa1, 0x2a, 0x4c, 0x13, 0xf4, 0x0e, 0x5c, 0xea, 0x0a, 0x99, 0x2b, 0x92, 0xed, 0xb6, 0x16,
0xb8, 0x8f, 0xc5, 0xc1, 0x82, 0xe3, 0x2b, 0x11, 0xfd, 0x08, 0x1a, 0xcf, 0x86, 0x7d, 0x2e, 0xc5,
0x99, 0xac, 0x1f, 0xc0, 0xf2, 0xd3, 0x68, 0x18, 0x05, 0xd1, 0xee, 0xf8, 0x84, 0x91, 0x6f, 0xc0,
0x92, 0xde, 0x8f, 0xfa, 0x91, 0x52, 0x61, 0x96, 0xa4, 0x17, 0x54, 0x43, 0xf7, 0x78, 0xd0, 0x1b,
0x05, 0x0a, 0x86, 0xfa, 0xf7, 0x4a, 0x1e, 0xd4, 0x7f, 0x3b, 0x6a, 0x3a, 0xbf, 0x1f, 0x35, 0x9d,
0x3f, 0x8f, 0x9a, 0xce, 0xcf, 0x7f, 0x35, 0xdf, 0x78, 0x51, 0xc6, 0xff, 0xf4, 0xdb, 0xff, 0x06,
0x00, 0x00, 0xff, 0xff, 0xda, 0x68, 0xc4, 0x54, 0xb8, 0x0f, 0x00, 0x00,
}

2
internal/private.proto
View file

@ -3,12 +3,10 @@ syntax = "proto3";
package internal;
message IndexMeta {
string ColumnLabel = 1;
string TimeQuantum = 2;
}
message FrameMeta {
string RowLabel = 1;
bool InverseEnabled = 2;
string CacheType = 3;
uint32 CacheSize = 4;

88
logger.go Normal file
View file

@ -0,0 +1,88 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
import (
"io"
"log"
)
// Ensure nopLogger implements interface.
var _ Logger = &nopLogger{}
// Logger represents an interface for a shared logger.
type Logger interface {
Printf(format string, v ...interface{})
Debugf(format string, v ...interface{})
}
func init() {
NopLogger = &nopLogger{}
}
// NopLogger represents a Logger that doesn't do anything.
var NopLogger Logger
type nopLogger struct{}
// Printf is a no-op implementation of the Logger Printf method.
func (n *nopLogger) Printf(format string, v ...interface{}) {}
// Debugf is a no-op implementation of the Logger Debugf method.
func (n *nopLogger) Debugf(format string, v ...interface{}) {}
// StandardLogger is a basic implementation of pilosa.Logger based on log.Logger.
type StandardLogger struct {
logger *log.Logger
}
func NewStandardLogger(w io.Writer) *StandardLogger {
return &StandardLogger{
logger: log.New(w, "", log.LstdFlags),
}
}
func (s *StandardLogger) Printf(format string, v ...interface{}) {
s.logger.Printf(format, v...)
}
func (s *StandardLogger) Debugf(format string, v ...interface{}) {}
func (s *StandardLogger) Logger() *log.Logger {
return s.logger
}
// VerboseLogger is an implementation of pilosa.Logger which includes debug messages.
type VerboseLogger struct {
logger *log.Logger
}
func NewVerboseLogger(w io.Writer) *VerboseLogger {
return &VerboseLogger{
logger: log.New(w, "", log.LstdFlags),
}
}
func (vb *VerboseLogger) Printf(format string, v ...interface{}) {
vb.logger.Printf(format, v...)
}
func (vb *VerboseLogger) Debugf(format string, v ...interface{}) {
vb.logger.Printf(format, v...)
}
func (vb *VerboseLogger) Logger() *log.Logger {
return vb.logger
}

13
pilosa.go
View file

@ -36,12 +36,10 @@ var (
ErrFrameExists = errors.New("frame already exists")
ErrFrameNotFound = errors.New("frame not found")
ErrFrameInverseDisabled = errors.New("frame inverse disabled")
ErrColumnRowLabelEqual = errors.New("column and row labels cannot be equal")
ErrInputDefinitionExists = errors.New("input-definition already exists")
ErrInputDefinitionHasPrimaryKey = errors.New("input-definition must contain one PrimaryKey")
ErrInputDefinitionDupePrimaryKey = errors.New("input-definition can only contain one PrimaryKey")
ErrInputDefinitionColumnLabel = errors.New("PrimaryKey field name does not match columnLabel")
ErrInputDefinitionNameRequired = errors.New("input-definition name required")
ErrInputDefinitionAttrsRequired = errors.New("frames and fields are required")
ErrInputDefinitionValueMap = errors.New("valueMap required for map")
@ -79,9 +77,6 @@ var (
// Regular expression to validate index and frame names.
var nameRegexp = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,63}$`)
// Regular expression to validate row and column labels.
var labelRegexp = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9_-]{0,63}$`)
// ColumnAttrSet represents a set of attributes for a vertical column in an index.
// Can have a set of attributes attached to it.
type ColumnAttrSet struct {
@ -143,14 +138,6 @@ func ValidateName(name string) error {
return nil
}
// ValidateLabel ensures that the label is a valid format.
func ValidateLabel(label string) error {
if labelRegexp.Match([]byte(label)) == false {
return ErrLabel
}
return nil
}
// StringInSlice checks for substring a in the slice.
func StringInSlice(a string, list []string) bool {
for _, b := range list {

24
pilosa_test.go
View file

@ -46,30 +46,6 @@ func TestValidateNameInvalid(t *testing.T) {
}
}
func TestValidateLabel(t *testing.T) {
labels := []string{
"a", "ab", "ab1", "d_e", "A", "Bc", "B1", "aB", "b-c",
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
}
for _, label := range labels {
if pilosa.ValidateLabel(label) != nil {
t.Fatalf("Should be valid label: %s", label)
}
}
}
func TestValidateLabelInvalid(t *testing.T) {
labels := []string{
"", "1", "_", "-", "'", "^", "/", "\\", "*", "a:b", "valid?no", "yüce",
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1",
}
for _, label := range labels {
if pilosa.ValidateLabel(label) == nil {
t.Fatalf("Should be invalid label: %s", label)
}
}
}
func TestStringInSlice(t *testing.T) {
list := []string{"localhost:10101", "localhost:10102", "localhost:10103"}
substr := "localhost:10101"

50
server.go
View file

@ -19,8 +19,6 @@ import (
"crypto/tls"
"errors"
"fmt"
"io"
"log"
"net"
"net/http"
"os"
@ -87,8 +85,7 @@ type Server struct {
// Misc options.
MaxWritesPerRequest int
LogOutput io.Writer
logger *log.Logger
Logger Logger
defaultClient InternalClient
}
@ -115,9 +112,8 @@ func NewServer() *Server {
MetricInterval: 0,
DiagnosticInterval: 0,
LogOutput: os.Stderr,
Logger: NopLogger,
}
s.logger = log.New(s.LogOutput, "", log.LstdFlags)
s.Handler.Holder = s.Holder
s.diagnostics.server = s
@ -126,7 +122,7 @@ func NewServer() *Server {
// Open opens and initializes the server.
func (s *Server) Open() error {
s.Logger().Printf("open server")
s.Logger.Printf("open server")
// s.ln can be configured prior to Open() via s.OpenListener().
if s.ln == nil {
if err := s.OpenListener(); err != nil {
@ -151,7 +147,6 @@ func (s *Server) Open() error {
// Peek at the holder to determine if there is data on disk.
// Don't actually load the data until after the Cluster
// management starts.
s.Holder.LogOutput = s.LogOutput
s.Holder.Peek()
// Create default HTTP client
@ -175,7 +170,6 @@ func (s *Server) Open() error {
s.Handler.Node = node
s.Handler.Cluster = s.Cluster
s.Handler.Executor = e
s.Handler.LogOutput = s.LogOutput
s.Cluster.prefect = s.Handler
@ -186,7 +180,7 @@ func (s *Server) Open() error {
go func() {
err := http.Serve(s.ln, s.Handler)
if err != nil {
s.Logger().Printf("HTTP handler terminated with error: %s\n", err)
s.Logger.Printf("HTTP handler terminated with error: %s\n", err)
}
}()
@ -226,7 +220,7 @@ func (s *Server) Open() error {
// OpenListener opens a listener for the Server.
func (s *Server) OpenListener() error {
s.Logger().Printf("open server listener: %s", s.URI)
s.Logger.Printf("open server listener: %s", s.URI)
if s.ln != nil {
return fmt.Errorf("a listener already exists for server: %s", s.URI)
}
@ -288,7 +282,7 @@ func (s *Server) LoadNodeID() string {
}
nodeID, err := s.Holder.loadNodeID()
if err != nil {
s.Logger().Printf("loading NodeID: %v", err)
s.Logger.Printf("loading NodeID: %v", err)
return s.NodeID
}
return nodeID
@ -321,14 +315,11 @@ func GetHTTPClient(t *tls.Config) *http.Client {
return &http.Client{Transport: transport}
}
// Logger returns a logger that writes to LogOutput
func (s *Server) Logger() *log.Logger { return s.logger }
func (s *Server) monitorAntiEntropy() {
ticker := time.NewTicker(s.AntiEntropyInterval)
defer ticker.Stop()
s.Logger().Printf("holder sync monitor initializing (%s interval)", s.AntiEntropyInterval)
s.Logger.Printf("holder sync monitor initializing (%s interval)", s.AntiEntropyInterval)
for {
// Wait for tick or a close.
@ -339,7 +330,7 @@ func (s *Server) monitorAntiEntropy() {
s.Holder.Stats.Count("AntiEntropy", 1, 1.0)
}
t := time.Now()
s.Logger().Printf("holder sync beginning")
s.Logger.Printf("holder sync beginning")
// Initialize syncer with local holder and remote client.
var syncer HolderSyncer
@ -352,12 +343,12 @@ func (s *Server) monitorAntiEntropy() {
// Sync holders.
if err := syncer.SyncHolder(); err != nil {
s.Logger().Printf("holder sync error: err=%s", err)
s.Logger.Printf("holder sync error: err=%s", err)
continue
}
// Record successful sync in log.
s.Logger().Printf("holder sync complete")
s.Logger.Printf("holder sync complete")
dif := time.Since(t)
s.Holder.Stats.Histogram("AntiEntropyDuration", float64(dif), 1.0)
}
@ -378,7 +369,6 @@ func (s *Server) ReceiveMessage(pb proto.Message) error {
}
case *internal.CreateIndexMessage:
opt := IndexOptions{
ColumnLabel: obj.Meta.ColumnLabel,
TimeQuantum: TimeQuantum(obj.Meta.TimeQuantum),
}
_, err := s.Holder.CreateIndex(obj.Index, opt)
@ -482,7 +472,7 @@ func (s *Server) ReceiveMessage(pb proto.Message) error {
func (s *Server) SendSync(pb proto.Message) error {
var eg errgroup.Group
for _, node := range s.Cluster.Nodes {
s.Logger().Printf("SendSync to: %s", node.URI)
s.Logger.Printf("SendSync to: %s", node.URI)
// Don't forward the message to ourselves.
if s.URI == node.URI {
continue
@ -504,7 +494,7 @@ func (s *Server) SendAsync(pb proto.Message) error {
// SendTo represents an implementation of Broadcaster.
func (s *Server) SendTo(to *Node, pb proto.Message) error {
s.Logger().Printf("SendTo: %s", to.URI)
s.Logger.Printf("SendTo: %s", to.URI)
ctx := context.WithValue(context.Background(), "uri", &to.URI)
return s.defaultClient.SendMessage(ctx, pb)
}
@ -554,7 +544,7 @@ func (s *Server) HandleRemoteStatus(pb proto.Message) error {
err := s.mergeRemoteStatus(pb.(*internal.NodeStatus))
if err != nil {
s.Logger().Printf("merge remote status: %s", err)
s.Logger.Printf("merge remote status: %s", err)
}
}()
@ -579,7 +569,7 @@ func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error {
// if we don't know about an index locally, log an error because
// indexes should be created and synced prior to slice creation
if localIndex == nil {
s.Logger().Printf("Local Index not found: %s", index)
s.Logger.Printf("Local Index not found: %s", index)
continue
}
if newMax > oldmaxslices[index] {
@ -595,7 +585,7 @@ func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error {
// if we don't know about an index locally, log an error because
// indexes should be created and synced prior to slice creation
if localIndex == nil {
s.Logger().Printf("Local Index not found: %s", index)
s.Logger.Printf("Local Index not found: %s", index)
continue
}
if newMaxInverse > oldMaxInverseSlices[index] {
@ -611,13 +601,13 @@ func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error {
func (s *Server) monitorDiagnostics() {
// Do not send more than once a minute
if s.DiagnosticInterval < time.Minute {
s.Logger().Printf("diagnostics disabled")
s.Logger.Printf("diagnostics disabled")
return
} else {
s.Logger().Printf("Pilosa is currently configured to send small diagnostics reports to our team every %v. More information here: https://www.pilosa.com/docs/latest/administration/#diagnostics", s.DiagnosticInterval)
s.Logger.Printf("Pilosa is currently configured to send small diagnostics reports to our team every %v. More information here: https://www.pilosa.com/docs/latest/administration/#diagnostics", s.DiagnosticInterval)
}
s.diagnostics.SetLogger(s.LogOutput)
s.diagnostics.Logger = s.Logger
s.diagnostics.SetVersion(Version)
s.diagnostics.Set("Host", s.URI.host)
s.diagnostics.Set("Cluster", strings.Join(s.Cluster.NodeIDs(), ","))
@ -639,7 +629,7 @@ func (s *Server) monitorDiagnostics() {
s.diagnostics.CheckVersion()
err = s.diagnostics.Flush()
if err != nil {
s.Logger().Printf("Diagnostics error: %s", err)
s.Logger.Printf("Diagnostics error: %s", err)
}
}
@ -670,7 +660,7 @@ func (s *Server) monitorRuntime() {
defer s.GCNotifier.Close()
s.Logger().Printf("runtime stats initializing (%s interval)", s.MetricInterval)
s.Logger.Printf("runtime stats initializing (%s interval)", s.MetricInterval)
for {
// Wait for tick or a close.

20
server/cluster_test.go
View file

@ -54,7 +54,7 @@ func TestMain_SendReceiveMessage(t *testing.T) {
m0.Server.Cluster.Coordinator = m0.Server.NodeID
m0.Server.Cluster.Topology = &pilosa.Topology{NodeIDs: []string{m0.Server.NodeID, m1.Server.NodeID}}
m0.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver(m0.Server.LogOutput)
m0.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver(m0.Server.Logger)
gossipMemberSet0, err := gossip.NewGossipMemberSet(m0.Server.URI.HostPort(), m0.Config, m0.Server)
if err != nil {
t.Fatal(err)
@ -81,7 +81,7 @@ func TestMain_SendReceiveMessage(t *testing.T) {
m1.Config.Gossip.Seeds = []string{gossipMemberSet0.GetBindAddr()}
m1.Server.Cluster.Coordinator = m0.Server.NodeID
m1.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver(m1.Server.LogOutput)
m1.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver(m1.Server.Logger)
gossipMemberSet1, err := gossip.NewGossipMemberSet(m1.Server.URI.HostPort(), m1.Config, m1.Server)
if err != nil {
t.Fatal(err)
@ -153,8 +153,8 @@ func TestMain_SendReceiveMessage(t *testing.T) {
// Write data on first node.
if _, err := m0.Query("i", "", `
SetBit(rowID=1, frame="f", columnID=1)
SetBit(rowID=1, frame="f", columnID=2400000)
SetBit(row=1, frame="f", col=1)
SetBit(row=1, frame="f", col=2400000)
`); err != nil {
t.Fatal(err)
}
@ -187,7 +187,7 @@ func TestMain_SendReceiveMessage(t *testing.T) {
"cacheType": "ranked",
"timeQuantum": "YMD"
}}],
"fields": [{"name": "columnID",
"fields": [{"name": "col",
"primaryKey": true
}]}
`); err != nil {
@ -351,8 +351,8 @@ func TestClusterResize_AddNode(t *testing.T) {
// Write data on first node.
if _, err := m0.Query("i", "", `
SetBit(rowID=1, frame="f", columnID=1)
SetBit(rowID=1, frame="f", columnID=1300000)
SetBit(row=1, frame="f", col=1)
SetBit(row=1, frame="f", col=1300000)
`); err != nil {
t.Fatal(err)
}
@ -406,8 +406,8 @@ func TestClusterResize_AddNode(t *testing.T) {
// Write data on first node. Note that no data is placed on slice 1.
if _, err := m0.Query("i", "", `
SetBit(rowID=1, frame="f", columnID=1)
SetBit(rowID=1, frame="f", columnID=2400000)
SetBit(row=1, frame="f", col=1)
SetBit(row=1, frame="f", col=2400000)
`); err != nil {
t.Fatal(err)
}
@ -560,7 +560,7 @@ func TestClusterResize_RemoveNode(t *testing.T) {
// TODO: Deterministic node IDs would ensure consistent results
setBits := ""
for i := 0; i < 20; i++ {
setBits += fmt.Sprintf("SetBit(rowID=1, frame=\"f\", columnID=%d) ", i*pilosa.SliceWidth)
setBits += fmt.Sprintf("SetBit(row=1, frame=\"f\", col=%d) ", i*pilosa.SliceWidth)
}
if _, err := m0.Query("i", "", setBits); err != nil {

69
server/server.go
View file

@ -23,6 +23,7 @@ import (
"errors"
"fmt"
"io"
"log"
"math/rand"
"os"
"path/filepath"
@ -71,6 +72,10 @@ type Command struct {
Started chan struct{}
// Done will be closed when Command.Close() is called
Done chan struct{}
// Passed to the Gossip implementation.
logOutput io.Writer
logger *log.Logger
}
// NewCommand returns a new instance of Main.
@ -115,7 +120,31 @@ func (m *Command) Run(args ...string) (err error) {
return fmt.Errorf("server.Open: %v", err)
}
m.Server.Logger().Printf("Listening as %s\n", m.Server.URI)
m.Server.Logger.Printf("Listening as %s\n", m.Server.URI)
return nil
}
// SetupLogger sets up the logger based on the configuration.
func (m *Command) SetupLogger() error {
var err error
if m.Config.LogPath == "" {
m.logOutput = m.Stderr
} else {
m.logOutput, err = os.OpenFile(m.Config.LogPath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600)
if err != nil {
return err
}
}
if m.Config.Verbose {
vbl := pilosa.NewVerboseLogger(m.logOutput)
m.logger = vbl.Logger()
m.Server.Logger = vbl
} else {
sl := pilosa.NewStandardLogger(m.logOutput)
m.logger = sl.Logger()
m.Server.Logger = sl
}
return nil
}
@ -126,6 +155,10 @@ func (m *Command) SetupServer() error {
return err
}
m.Server.Handler.Logger = m.Server.Logger
m.Server.Holder.Logger = m.Server.Logger
m.Server.Holder.Stats.SetLogger(m.Server.Logger)
uri, err := pilosa.AddressWithDefaults(m.Config.Bind)
if err != nil {
@ -136,15 +169,10 @@ func (m *Command) SetupServer() error {
cluster := pilosa.NewCluster()
cluster.ReplicaN = m.Config.Cluster.ReplicaN
cluster.Holder = m.Server.Holder
cluster.Logger = m.Server.Logger
m.Server.Cluster = cluster
// Setup logging output.
m.Server.LogOutput, err = GetLogWriter(m.Config.LogPath, m.Stderr)
if err != nil {
return err
}
// Configure data directory (for Cluster .topology)
m.Server.Cluster.Path = m.Config.DataDir
@ -152,7 +180,7 @@ func (m *Command) SetupServer() error {
m.Server.Holder.NewAttrStore = boltdb.NewAttrStore
// Configure holder.
m.Server.Logger().Printf("Using data from: %s\n", m.Config.DataDir)
m.Server.Logger.Printf("Using data from: %s\n", m.Config.DataDir)
m.Server.Holder.Path = m.Config.DataDir
m.Server.MetricInterval = time.Duration(m.Config.Metric.PollInterval)
if m.Config.Metric.Diagnostics {
@ -165,8 +193,6 @@ func (m *Command) SetupServer() error {
return err
}
m.Server.Holder.Stats.SetLogger(m.Server.LogOutput)
// Copy configuration flags.
m.Server.MaxWritesPerRequest = m.Config.MaxWritesPerRequest
@ -246,7 +272,7 @@ func (m *Command) SetupNetworking() error {
if m.GossipTransport != nil {
transport = m.GossipTransport
} else {
transport, err = gossip.NewTransport(gossipHost, gossipPort)
transport, err = gossip.NewTransport(gossipHost, gossipPort, m.logger)
if err != nil {
return err
}
@ -257,8 +283,8 @@ func (m *Command) SetupNetworking() error {
m.Server.Cluster.Coordinator = m.Server.NodeID
}
m.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver(m.Server.LogOutput)
gossipMemberSet, err := gossip.NewGossipMemberSetWithTransport(m.Server.NodeID, m.Config, transport, m.Server)
m.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver(m.Server.Logger)
gossipMemberSet, err := gossip.NewGossipMemberSet(m.Server.NodeID, m.Config, m.Server, gossip.WithLogger(m.logger), gossip.WithTransport(transport))
if err != nil {
return err
}
@ -269,26 +295,11 @@ func (m *Command) SetupNetworking() error {
return nil
}
// GetLogWriter opens a file for logging, or a default io.Writer (such as stderr) for an empty path.
func GetLogWriter(path string, defaultWriter io.Writer) (io.Writer, error) {
// This is split out so it can be used in NewServeCmd as well as SetupServer
if path == "" {
return defaultWriter, nil
} else {
logFile, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600)
if err != nil {
return nil, err
}
return logFile, nil
}
}
// Close shuts down the server.
func (m *Command) Close() error {
var logErr error
serveErr := m.Server.Close()
logOutput := m.Server.LogOutput
if closer, ok := logOutput.(io.Closer); ok {
if closer, ok := m.logOutput.(io.Closer); ok {
logErr = closer.Close()
}
close(m.Done)

96
server/server_test.go
View file

@ -56,7 +56,7 @@ func TestMain_Set_Quick(t *testing.T) {
if err := client.CreateFrame(context.Background(), "i", cmd.Frame, pilosa.FrameOptions{}); err != nil && err != pilosa.ErrFrameExists {
t.Fatal(err)
}
if _, err := m.Query("i", "", fmt.Sprintf(`SetBit(rowID=%d, frame=%q, columnID=%d)`, cmd.ID, cmd.Frame, cmd.ColumnID)); err != nil {
if _, err := m.Query("i", "", fmt.Sprintf(`SetBit(row=%d, frame=%q, col=%d)`, cmd.ID, cmd.Frame, cmd.ColumnID)); err != nil {
t.Fatal(err)
}
}
@ -72,7 +72,7 @@ func TestMain_Set_Quick(t *testing.T) {
},
},
}) + "\n"
if res, err := m.Query("i", "", fmt.Sprintf(`Bitmap(rowID=%d, frame=%q)`, id, frame)); err != nil {
if res, err := m.Query("i", "", fmt.Sprintf(`Bitmap(row=%d, frame=%q)`, id, frame)); err != nil {
t.Fatal(err)
} else if res != exp {
t.Fatalf("unexpected result:\n\ngot=%s\n\nexp=%s\n\n", res, exp)
@ -95,7 +95,7 @@ func TestMain_Set_Quick(t *testing.T) {
},
},
}) + "\n"
if res, err := m.Query("i", "", fmt.Sprintf(`Bitmap(rowID=%d, frame=%q)`, id, frame)); err != nil {
if res, err := m.Query("i", "", fmt.Sprintf(`Bitmap(row=%d, frame=%q)`, id, frame)); err != nil {
t.Fatal(err)
} else if res != exp {
t.Fatalf("unexpected result (reopen):\n\ngot=%s\n\nexp=%s\n\n", res, exp)
@ -131,36 +131,36 @@ func TestMain_SetRowAttrs(t *testing.T) {
}
// Set bits on different rows in different frames.
if _, err := m.Query("i", "", `SetBit(rowID=1, frame="x", columnID=100)`); err != nil {
if _, err := m.Query("i", "", `SetBit(row=1, frame="x", col=100)`); err != nil {
t.Fatal(err)
} else if _, err := m.Query("i", "", `SetBit(rowID=2, frame="x", columnID=100)`); err != nil {
} else if _, err := m.Query("i", "", `SetBit(row=2, frame="x", col=100)`); err != nil {
t.Fatal(err)
} else if _, err := m.Query("i", "", `SetBit(rowID=2, frame="z", columnID=100)`); err != nil {
} else if _, err := m.Query("i", "", `SetBit(row=2, frame="z", col=100)`); err != nil {
t.Fatal(err)
} else if _, err := m.Query("i", "", `SetBit(rowID=3, frame="neg", columnID=100)`); err != nil {
} else if _, err := m.Query("i", "", `SetBit(row=3, frame="neg", col=100)`); err != nil {
t.Fatal(err)
}
// Set row attributes.
if _, err := m.Query("i", "", `SetRowAttrs(rowID=1, frame="x", x=100)`); err != nil {
if _, err := m.Query("i", "", `SetRowAttrs(row=1, frame="x", x=100)`); err != nil {
t.Fatal(err)
} else if _, err := m.Query("i", "", `SetRowAttrs(rowID=2, frame="x", x=-200)`); err != nil {
} else if _, err := m.Query("i", "", `SetRowAttrs(row=2, frame="x", x=-200)`); err != nil {
t.Fatal(err)
} else if _, err := m.Query("i", "", `SetRowAttrs(rowID=2, frame="z", x=300)`); err != nil {
} else if _, err := m.Query("i", "", `SetRowAttrs(row=2, frame="z", x=300)`); err != nil {
t.Fatal(err)
} else if _, err := m.Query("i", "", `SetRowAttrs(rowID=3, frame="neg", x=-0.44)`); err != nil {
} else if _, err := m.Query("i", "", `SetRowAttrs(row=3, frame="neg", x=-0.44)`); err != nil {
t.Fatal(err)
}
// Query row x/1.
if res, err := m.Query("i", "", `Bitmap(rowID=1, frame="x")`); err != nil {
if res, err := m.Query("i", "", `Bitmap(row=1, frame="x")`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{"x":100},"bits":[100]}]}`+"\n" {
t.Fatalf("unexpected result: %s", res)
}
// Query row x/2.
if res, err := m.Query("i", "", `Bitmap(rowID=2, frame="x")`); err != nil {
if res, err := m.Query("i", "", `Bitmap(row=2, frame="x")`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{"x":-200},"bits":[100]}]}`+"\n" {
t.Fatalf("unexpected result: %s", res)
@ -171,19 +171,19 @@ func TestMain_SetRowAttrs(t *testing.T) {
}
// Query rows after reopening.
if res, err := m.Query("i", "columnAttrs=true", `Bitmap(rowID=1, frame="x")`); err != nil {
if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=1, frame="x")`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{"x":100},"bits":[100]}]}`+"\n" {
t.Fatalf("unexpected result(reopen): %s", res)
}
if res, err := m.Query("i", "columnAttrs=true", `Bitmap(rowID=3, frame="neg")`); err != nil {
if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=3, frame="neg")`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{"x":-0.44},"bits":[100]}]}`+"\n" {
t.Fatalf("unexpected result(reopen): %s", res)
}
// Query row x/2.
if res, err := m.Query("i", "", `Bitmap(rowID=2, frame="x")`); err != nil {
if res, err := m.Query("i", "", `Bitmap(row=2, frame="x")`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{"x":-200},"bits":[100]}]}`+"\n" {
t.Fatalf("unexpected result: %s", res)
@ -204,19 +204,19 @@ func TestMain_SetColumnAttrs(t *testing.T) {
}
// Set bits on row.
if _, err := m.Query("i", "", `SetBit(rowID=1, frame="x", columnID=100)`); err != nil {
if _, err := m.Query("i", "", `SetBit(row=1, frame="x", col=100)`); err != nil {
t.Fatal(err)
} else if _, err := m.Query("i", "", `SetBit(rowID=1, frame="x", columnID=101)`); err != nil {
} else if _, err := m.Query("i", "", `SetBit(row=1, frame="x", col=101)`); err != nil {
t.Fatal(err)
}
// Set column attributes.
if _, err := m.Query("i", "", `SetColumnAttrs(id=100, foo="bar")`); err != nil {
if _, err := m.Query("i", "", `SetColumnAttrs(col=100, foo="bar")`); err != nil {
t.Fatal(err)
}
// Query row.
if res, err := m.Query("i", "columnAttrs=true", `Bitmap(rowID=1, frame="x")`); err != nil {
if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=1, frame="x")`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{},"bits":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" {
t.Fatalf("unexpected result: %s", res)
@ -227,47 +227,13 @@ func TestMain_SetColumnAttrs(t *testing.T) {
}
// Query row after reopening.
if res, err := m.Query("i", "columnAttrs=true", `Bitmap(rowID=1, frame="x")`); err != nil {
if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=1, frame="x")`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{},"bits":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" {
t.Fatalf("unexpected result(reopen): %s", res)
}
}
// Ensure program can set column attributes with columnLabel option.
func TestMain_SetColumnAttrsWithColumnOption(t *testing.T) {
m := test.MustRunMain()
defer m.Close()
// Create frames.
client := m.Client()
if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{ColumnLabel: "col"}); err != nil && err != pilosa.ErrIndexExists {
t.Fatal(err)
} else if err := client.CreateFrame(context.Background(), "i", "x", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
}
// Set bits on row.
if _, err := m.Query("i", "", `SetBit(rowID=1, frame="x", col=100)`); err != nil {
t.Fatal(err)
} else if _, err := m.Query("i", "", `SetBit(rowID=1, frame="x", col=101)`); err != nil {
t.Fatal(err)
}
// Set column attributes.
if _, err := m.Query("i", "", `SetColumnAttrs(col=100, foo="bar")`); err != nil {
t.Fatal(err)
}
// Query row.
if res, err := m.Query("i", "columnAttrs=true", `Bitmap(rowID=1, frame="x")`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{},"bits":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" {
t.Fatalf("unexpected result: %s", res)
}
}
// Ensure program can set bits on one cluster and then restore to a second cluster.
func TestMain_FrameRestore(t *testing.T) {
mains1 := test.MustRunMainWithCluster(t, 2)
@ -285,19 +251,19 @@ func TestMain_FrameRestore(t *testing.T) {
// Write data on first cluster.
if _, err := m10.Query("i", "", `
SetBit(rowID=1, frame="f", columnID=100)
SetBit(rowID=1, frame="f", columnID=1000)
SetBit(rowID=1, frame="f", columnID=100000)
SetBit(rowID=1, frame="f", columnID=200000)
SetBit(rowID=1, frame="f", columnID=400000)
SetBit(rowID=1, frame="f", columnID=600000)
SetBit(rowID=1, frame="f", columnID=800000)
SetBit(row=1, frame="f", col=100)
SetBit(row=1, frame="f", col=1000)
SetBit(row=1, frame="f", col=100000)
SetBit(row=1, frame="f", col=200000)
SetBit(row=1, frame="f", col=400000)
SetBit(row=1, frame="f", col=600000)
SetBit(row=1, frame="f", col=800000)
`); err != nil {
t.Fatal("setting bits:", err)
}
// Query row on first cluster.
if res, err := m10.Query("i", "", `Bitmap(rowID=1, frame="f")`); err != nil {
if res, err := m10.Query("i", "", `Bitmap(row=1, frame="f")`); err != nil {
t.Fatal("bitmap query:", err)
} else if res != `{"results":[{"attrs":{},"bits":[100,1000,100000,200000,400000,600000,800000]}]}`+"\n" {
t.Fatalf("unexpected result: %s", res)
@ -335,7 +301,7 @@ func TestMain_FrameRestore(t *testing.T) {
}
// Query row on second cluster.
if res, err := m20.Query("i", "", `Bitmap(rowID=1, frame="f")`); err != nil {
if res, err := m20.Query("i", "", `Bitmap(row=1, frame="f")`); err != nil {
t.Fatal("another bitmap query:", err)
} else if res != `{"results":[{"attrs":{},"bits":[100,1000,100000,200000,400000,600000,800000]}]}`+"\n" {
t.Fatalf("2unexpected result: %s", res)
@ -401,7 +367,7 @@ func TestMain_RecalculateHashes(t *testing.T) {
data := []string{}
for rowID := 1; rowID < 10; rowID++ {
for columnID := 1; columnID < 100; columnID++ {
data = append(data, fmt.Sprintf(`SetBit(rowID=%d, frame="f", columnID=%d)`, rowID, columnID))
data = append(data, fmt.Sprintf(`SetBit(row=%d, frame="f", col=%d)`, rowID, columnID))
}
}
if _, err := cluster[0].Query("i", "", strings.Join(data, "")); err != nil {

9
stats.go
View file

@ -16,7 +16,6 @@ package pilosa
import (
"expvar"
"io"
"sort"
"strings"
"sync"
@ -57,7 +56,7 @@ type StatsClient interface {
Timing(name string, value time.Duration, rate float64)
// SetLogger Set the logger output type
SetLogger(logger io.Writer)
SetLogger(logger Logger)
// Starts the service
Open()
@ -79,7 +78,7 @@ func (c *nopStatsClient) Gauge(name string, value float64, rate float64)
func (c *nopStatsClient) Histogram(name string, value float64, rate float64) {}
func (c *nopStatsClient) Set(name string, value string, rate float64) {}
func (c *nopStatsClient) Timing(name string, value time.Duration, rate float64) {}
func (c *nopStatsClient) SetLogger(logger io.Writer) {}
func (c *nopStatsClient) SetLogger(logger Logger) {}
func (c *nopStatsClient) Open() {}
func (c *nopStatsClient) Close() error { return nil }
@ -154,7 +153,7 @@ func (c *ExpvarStatsClient) Timing(name string, value time.Duration, rate float6
}
// SetLogger has no logger.
func (c *ExpvarStatsClient) SetLogger(logger io.Writer) {
func (c *ExpvarStatsClient) SetLogger(logger Logger) {
}
// Open no-op.
@ -226,7 +225,7 @@ func (a MultiStatsClient) Timing(name string, value time.Duration, rate float64)
}
// SetLogger Sets the StatsD logger output type.
func (a MultiStatsClient) SetLogger(logger io.Writer) {
func (a MultiStatsClient) SetLogger(logger Logger) {
for _, c := range a {
c.SetLogger(logger)
}

11
stats_test.go
View file

@ -16,8 +16,6 @@ package pilosa_test
import (
"context"
"io"
"io/ioutil"
"net/http"
"strings"
"testing"
@ -38,7 +36,6 @@ func TestMultiStatClient_Expvar(t *testing.T) {
ms[0] = c
hldr.Stats = ms
hldr.Stats.SetLogger(ioutil.Discard)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth)
@ -143,7 +140,7 @@ func TestStatsCount_Bitmap(t *testing.T) {
return
},
}
if _, err := e.Execute(context.Background(), "d", test.MustParse(`Bitmap(frame=f, rowID=0)`), nil, nil); err != nil {
if _, err := e.Execute(context.Background(), "d", test.MustParse(`Bitmap(frame=f, row=0)`), nil, nil); err != nil {
t.Fatal(err)
}
if !called {
@ -174,7 +171,7 @@ func TestStatsCount_SetBitmapAttrs(t *testing.T) {
return
},
}
if _, err := e.Execute(context.Background(), "d", test.MustParse(`SetRowAttrs(rowID=10, frame=f, foo="bar")`), nil, nil); err != nil {
if _, err := e.Execute(context.Background(), "d", test.MustParse(`SetRowAttrs(row=10, frame=f, foo="bar")`), nil, nil); err != nil {
t.Fatal(err)
}
if !called {
@ -206,7 +203,7 @@ func TestStatsCount_SetProfileAttrs(t *testing.T) {
return
},
}
if _, err := e.Execute(context.Background(), "d", test.MustParse(`SetColumnAttrs(id=10, frame=f, foo="bar")`), nil, nil); err != nil {
if _, err := e.Execute(context.Background(), "d", test.MustParse(`SetColumnAttrs(col=10, frame=f, foo="bar")`), nil, nil); err != nil {
t.Fatal(err)
}
if !called {
@ -357,6 +354,6 @@ func (c *MockStats) Gauge(name string, value float64, rate float64) {}
func (c *MockStats) Histogram(name string, value float64, rate float64) {}
func (c *MockStats) Set(name string, value string, rate float64) {}
func (c *MockStats) Timing(name string, value time.Duration, rate float64) {}
func (c *MockStats) SetLogger(logger io.Writer) {}
func (c *MockStats) SetLogger(logger pilosa.Logger) {}
func (c *MockStats) Open() {}
func (c *MockStats) Close() error { return nil }

42
statsd/statsd.go
View file

@ -15,9 +15,6 @@
package statsd
import (
"io"
"io/ioutil"
"log"
"time"
"github.com/DataDog/datadog-go/statsd"
@ -40,9 +37,9 @@ var _ pilosa.StatsClient = &StatsClient{}
// StatsClient represents a StatsD implementation of pilosa.StatsClient.
type StatsClient struct {
client *statsd.Client
tags []string
logOutput io.Writer
client *statsd.Client
tags []string
logger pilosa.Logger
}
// NewStatsClient returns a new instance of StatsClient.
@ -53,8 +50,8 @@ func NewStatsClient(host string) (*StatsClient, error) {
}
return &StatsClient{
client: c,
logOutput: ioutil.Discard,
client: c,
logger: pilosa.NopLogger,
}, nil
}
@ -74,16 +71,16 @@ func (c *StatsClient) Tags() []string {
// WithTags returns a new client with additional tags appended.
func (c *StatsClient) WithTags(tags ...string) pilosa.StatsClient {
return &StatsClient{
client: c.client,
tags: pilosa.UnionStringSlice(c.tags, tags),
logOutput: c.logOutput,
client: c.client,
tags: pilosa.UnionStringSlice(c.tags, tags),
logger: c.logger,
}
}
// Count tracks the number of times something occurs per second.
func (c *StatsClient) Count(name string, value int64, rate float64) {
if err := c.client.Count(Prefix+name, value, c.tags, rate); err != nil {
c.logger().Printf("statsd.StatsClient.Count error: %s", err)
c.logger.Printf("statsd.StatsClient.Count error: %s", err)
}
}
@ -91,44 +88,39 @@ func (c *StatsClient) Count(name string, value int64, rate float64) {
func (c *StatsClient) CountWithCustomTags(name string, value int64, rate float64, t []string) {
tags := append(c.tags, t...)
if err := c.client.Count(Prefix+name, value, tags, rate); err != nil {
c.logger().Printf("statsd.StatsClient.Count error: %s", err)
c.logger.Printf("statsd.StatsClient.Count error: %s", err)
}
}
// Gauge sets the value of a metric.
func (c *StatsClient) Gauge(name string, value float64, rate float64) {
if err := c.client.Gauge(Prefix+name, value, c.tags, rate); err != nil {
c.logger().Printf("statsd.StatsClient.Gauge error: %s", err)
c.logger.Printf("statsd.StatsClient.Gauge error: %s", err)
}
}
// Histogram tracks statistical distribution of a metric.
func (c *StatsClient) Histogram(name string, value float64, rate float64) {
if err := c.client.Histogram(Prefix+name, value, c.tags, rate); err != nil {
c.logger().Printf("statsd.StatsClient.Histogram error: %s", err)
c.logger.Printf("statsd.StatsClient.Histogram error: %s", err)
}
}
// Set tracks number of unique elements.
func (c *StatsClient) Set(name string, value string, rate float64) {
if err := c.client.Set(Prefix+name, value, c.tags, rate); err != nil {
c.logger().Printf("statsd.StatsClient.Set error: %s", err)
c.logger.Printf("statsd.StatsClient.Set error: %s", err)
}
}
// Timing tracks timing information for a metric.
func (c *StatsClient) Timing(name string, value time.Duration, rate float64) {
if err := c.client.Timing(Prefix+name, value, c.tags, rate); err != nil {
c.logger().Printf("statsd.StatsClient.Timing error: %s", err)
c.logger.Printf("statsd.StatsClient.Timing error: %s", err)
}
}
// SetLogger has no logger
func (c *StatsClient) SetLogger(logger io.Writer) {
c.logOutput = logger
}
// logger returns a logger that writes to LogOutput
func (c *StatsClient) logger() *log.Logger {
return log.New(c.logOutput, "", log.LstdFlags)
// SetLogger sets the logger for client.
func (c *StatsClient) SetLogger(logger pilosa.Logger) {
c.logger = logger
}

2
statsd/statsd_test.go
View file

@ -15,7 +15,6 @@
package statsd_test
import (
"io/ioutil"
"reflect"
"testing"
"time"
@ -31,7 +30,6 @@ func TestStatsClient_WithTags(t *testing.T) {
t.Fatal(err)
}
defer c.Close()
c.SetLogger(ioutil.Discard)
// Create a new client with additional tags.
c1 := c.WithTags("foo", "bar")

1
test/handler.go
View file

@ -41,7 +41,6 @@ func NewHandler() *Handler {
Handler: pilosa.NewHandler(),
}
h.Handler.Executor = &h.Executor
h.Handler.LogOutput = ioutil.Discard
// Handler test messages can no-op.
h.Broadcaster = pilosa.NopBroadcaster

7
test/holder.go
View file

@ -15,7 +15,6 @@
package test
import (
"bytes"
"io/ioutil"
"os"
@ -26,7 +25,6 @@ import (
// Holder is a test wrapper for pilosa.Holder.
type Holder struct {
*pilosa.Holder
LogOutput bytes.Buffer
}
// NewHolder returns a new instance of Holder with a temporary path.
@ -38,7 +36,6 @@ func NewHolder() *Holder {
h := &Holder{Holder: pilosa.NewHolder()}
h.Path = path
h.Holder.LogOutput = &h.LogOutput
h.Holder.NewAttrStore = boltdb.NewAttrStore
return h
@ -62,10 +59,10 @@ func (h *Holder) Close() error {
// Reopen instantiates and opens a new holder.
// Note that the holder must be Closed first.
func (h *Holder) Reopen() error {
path, logOutput := h.Path, h.Holder.LogOutput
path, logger := h.Path, h.Holder.Logger
h.Holder = pilosa.NewHolder()
h.Holder.Path = path
h.Holder.LogOutput = logOutput
h.Holder.Logger = logger
h.Holder.NewAttrStore = boltdb.NewAttrStore
if err := h.Holder.Open(); err != nil {
return err

48
test/logger.go Normal file
View file

@ -0,0 +1,48 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package test
import (
"bytes"
"fmt"
"io/ioutil"
)
// BufferLogger represents a test Logger that holds log messages
// in a buffer for review.
type BufferLogger struct {
buf *bytes.Buffer
}
// NewBufferLogger returns a new instance of BufferLogger.
func NewBufferLogger() *BufferLogger {
return &BufferLogger{
buf: &bytes.Buffer{},
}
}
func (b *BufferLogger) Printf(format string, v ...interface{}) {
s := fmt.Sprintf(format, v...)
_, err := b.buf.WriteString(s)
if err != nil {
panic(err)
}
}
func (b *BufferLogger) Debugf(format string, v ...interface{}) {}
func (b *BufferLogger) ReadAll() ([]byte, error) {
return ioutil.ReadAll(b.buf)
}

2
test/pilosa.go
View file

@ -181,7 +181,7 @@ func (m *Main) RunWithTransport(host string, bindPort int, joinSeeds []string) (
}
// Open gossip transport to use in SetupServer.
transport, err := gossip.NewTransport(host, bindPort)
transport, err := gossip.NewTransport(host, bindPort, nil)
if err != nil {
return seed, err
}

16
view.go
View file

@ -16,9 +16,6 @@ package pilosa
import (
"fmt"
"io"
"io/ioutil"
"log"
"os"
"path/filepath"
"strconv"
@ -64,7 +61,7 @@ type View struct {
stats StatsClient
RowAttrStore AttrStore
LogOutput io.Writer
Logger Logger
}
// NewView returns a new instance of View.
@ -81,7 +78,7 @@ func NewView(path, index, frame, name string, cacheSize uint32) *View {
broadcaster: NopBroadcaster,
stats: NopStatsClient,
LogOutput: ioutil.Discard,
Logger: NopLogger,
}
}
@ -126,9 +123,6 @@ func (v *View) Open() error {
return nil
}
// logger returns a logger instance for the view.
func (v *View) logger() *log.Logger { return log.New(v.LogOutput, "", log.LstdFlags) }
// openFragments opens and initializes the fragments inside the view.
func (v *View) openFragments() error {
file, err := os.Open(filepath.Join(v.path, "fragments"))
@ -275,7 +269,7 @@ func (v *View) newFragment(path string, slice uint64) *Fragment {
frag := NewFragment(path, v.index, v.frame, v.name, slice)
frag.CacheType = v.cacheType
frag.CacheSize = v.cacheSize
frag.LogOutput = v.LogOutput
frag.Logger = v.Logger
frag.stats = v.stats.WithTags(fmt.Sprintf("slice:%d", slice))
return frag
}
@ -288,7 +282,7 @@ func (v *View) DeleteFragment(slice uint64) error {
return ErrFragmentNotFound
}
v.logger().Printf("delete fragment: (%s/%s/%s) %d", v.index, v.frame, v.name, slice)
v.Logger.Printf("delete fragment: (%s/%s/%s) %d", v.index, v.frame, v.name, slice)
// Close data files before deletion.
if err := fragment.Close(); err != nil {
@ -302,7 +296,7 @@ func (v *View) DeleteFragment(slice uint64) error {
// Delete fragment cache file.
if err := os.Remove(fragment.CachePath()); err != nil {
v.logger().Printf("no cache file to delete for slice %d", slice)
v.Logger.Printf("no cache file to delete for slice %d", slice)
}
delete(v.fragments, slice)

43
webui/assets/main.js
View file

@ -254,12 +254,12 @@ function populate_version() {
xhr.onload = function() {
var version = JSON.parse(xhr.responseText)['version']
var version_major_minor = /(v\d+\.\d+)/.exec(version)[0]
var version_major_minor = /v?(\d+\.\d+).*/.exec(version)[1]
var doc_link = document.getElementById('nav-documentation')
doc_link.onclick = function() {
window.open('https://www.pilosa.com/docs/' + version_major_minor + '/introduction/')
window.open('https://www.pilosa.com/docs/v' + version_major_minor + '/introduction/')
}
node.innerHTML = version
node.innerHTML = "Pilosa v" + version
}
xhr.send(null)
}
@ -295,12 +295,19 @@ function set_active_pane_by_name(name) {
function update_cluster_status() {
var xhr = new XMLHttpRequest();
xhr.open('GET', '/status')
status_node = document.getElementById('status')
xhr.onload = function() {
var status = JSON.parse(xhr.responseText)
render_status(status)
}
xhr.send(null)
var xhrSchema = new XMLHttpRequest();
xhrSchema.open('GET', '/schema')
xhrSchema.onload = function() {
var schema = JSON.parse(xhrSchema.responseText)
render_schema(schema)
}
xhrSchema.send(null)
}
function render_status(status) {
@ -310,7 +317,7 @@ function render_status(status) {
nodes_div.removeChild(nodes_div.firstChild);
}
var nodes = status["status"]["Nodes"]
var nodes = status["nodes"]
table = document.createElement("table")
tbody = document.createElement("tbody")
table.appendChild(tbody)
@ -320,31 +327,35 @@ function render_status(status) {
var header = document.createElement('tr')
markup = `<th>Host</th>
<th>State</th>`
<th>ID</th>
<th>Coordinator</th>`
header.innerHTML = markup
tbody.appendChild(header)
for(var n=0; n<nodes.length; n++) {
var row = document.createElement("tr")
markup = `<td>${nodes[n]["Host"]}</td>
<td>${nodes[n]["State"]}</td>`
markup = `<td>${nodes[n]["uri"]["host"]}:${nodes[n]["uri"]["port"]}</td>
<td>${nodes[n]["id"]}</td>
<td>${nodes[n]["isCoordinator"]}</td>`
row.innerHTML = markup
tbody.appendChild(row)
}
nodes_div.appendChild(table)
}
function render_schema(schema) {
// render index tables
var indexes_div = document.getElementById("status-indexes")
while (indexes_div.firstChild) {
indexes_div.removeChild(indexes_div.firstChild);
}
var indexes = nodes[0]["Indexes"] // TODO currently comes from only node 0
var indexes = schema["indexes"] // TODO currently comes from only node 0
for(var n=0; n<indexes.length; n++) {
table = document.createElement("table")
tbody = document.createElement("tbody")
table.appendChild(tbody)
var caption = document.createElement("caption")
caption.innerHTML = indexes[n]["Name"]
caption.innerHTML = indexes[n]["name"]
table.appendChild(caption)
var header = document.createElement('tr')
@ -354,13 +365,13 @@ function render_status(status) {
header.innerHTML = markup
tbody.appendChild(header)
var frames = indexes[n]["Frames"]
var frames = indexes[n]["frames"]
if(frames) {
for(var m=0; m<frames.length; m++) {
var row = document.createElement("tr")
row.innerHTML = `<td>${frames[m]["Name"]}</td>
<td>${frames[m]["Meta"]["CacheType"]}</td>
<td>${frames[m]["Meta"]["CacheSize"]}</td>`
row.innerHTML = `<td>${frames[m]["name"]}</td>
<td>${frames[m]["options"]["cacheType"]}</td>
<td>${frames[m]["options"]["cacheSize"]}</td>`
tbody.appendChild(row)
}
}
@ -472,7 +483,7 @@ class Autocompleter {
}
init_dynamic_keywords() {
// hit /schema, parse indexes, frames, rowlabels, columnlabels, add to list
// hit /schema, parse indexes, frames, add to list
}
add_keyword() {
@ -583,4 +594,4 @@ function parse_options(option_str) {
}
}
return options;
}
}

4
webui/index.html
View file

@ -55,9 +55,9 @@
<br />
<h5>Special commands</h5>
<div class="code">
:create index test [columnLabel=column]<br />
:create index test<br />
:use test<br />
:create frame foo [rowLabel=row]<br />
:create frame foo<br />
:delete index test<br />
:delete frame foo
</div>