scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-08-28 10:54:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'master' into fb1000

Browse source
This commit is contained in:
souhailanoor 2021-12-10 14:09:39 -06:00 • committed by GitHub
commit e5052a864b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
409 changed files with 1740 additions and 6044 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
.circleci/config.yml
View file

@ -144,16 +144,6 @@ jobs:
- run:
command: make test-external-lookup EXTERNAL_LOOKUP_DSN=postgresql://postgres:password@localhost/circle_test?sslmode=disable
no_output_timeout: 30m
test-backup-restore:
executor:
name: golang
steps:
- checkout-plus
- skip-if-root-unchanged
- setup_remote_docker
- run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin
- run: make backuptests-build
- run: make backuptests
cluster-tests:
executor:
name: golang
@ -273,10 +263,6 @@ workflows:
context: molecula
requires:
- setup
- test-backup-restore:
context: molecula
requires:
- setup
- cluster-tests:
context: molecula
requires:
@ -292,8 +278,6 @@ workflows:
filters:
tags:
only: /^v.*/
branches:
only: master
- publish_release:
context: molecula
requires:

6
.gitlab/.gitlab-ci.yml
View file

@ -74,7 +74,8 @@ run go tests:
extends: .go-cache
script:
- echo "Running featurebase unit tests..."
- ./cover-everything.sh
- PKG_LIST=$(go list ./... | grep -v internal/clustertests | paste -s -d, -)
- go test -coverprofile=coverage.out -covermode=atomic -coverpkg=${PKG_LIST} ./...
artifacts:
paths:
- coverage.out
@ -85,7 +86,8 @@ run go tests future:
extends: .go-cache
script:
- echo "Running featurebase unit tests..."
- ./cover-everything.sh
- PKG_LIST=$(go list ./... | grep -v internal/clustertests | paste -s -d, -)
- go test -coverprofile=coverage.out -covermode=atomic -coverpkg=${PKG_LIST} ./...
artifacts:
paths:
- coverage.out

37
Dockerfile.pilosa
View file

@ -1,37 +0,0 @@
ARG GO_VERSION=latest
######################
### Pilosa builder ###
######################
FROM golang:${GO_VERSION} as pilosa-builder
ARG MAKE_FLAGS
WORKDIR /pilosa
COPY . ./
RUN make build FLAGS="-o build/featurebase" ${MAKE_FLAGS}
#####################
### Pilosa runner ###
#####################
FROM alpine:3.13.2 as runner
LABEL maintainer "dev@molecula.com"
RUN apk add --no-cache curl jq
COPY --from=pilosa-builder /pilosa/build/featurebase /
COPY NOTICE /NOTICE
EXPOSE 10101
VOLUME /data
ENV PILOSA_DATA_DIR /data
ENV PILOSA_BIND 0.0.0.0:10101
ENV PILOSA_BIND_GRPC 0.0.0.0:20101
ENTRYPOINT ["/featurebase"]
CMD ["server"]

23
Dockerfile.runner
View file

@ -1,23 +0,0 @@
ARG GO_VERSION=latest
######################
### Pilosa builder ###
######################
FROM golang:${GO_VERSION} as pilosa-builder
ARG MAKE_FLAGS
WORKDIR /pilosa
COPY . ./
RUN make build FLAGS="-o build/featurebase" ${MAKE_FLAGS}
FROM moleculacorp/idk as idk
LABEL maintainer "dev@molecula.com"
RUN apt-get update -y
RUN apt-get install -y bash curl jq
COPY --from=pilosa-builder /pilosa/build/featurebase /
COPY testBackupRestore.sh /
CMD ["bash","/testBackupRestore.sh"]

9
Makefile
View file

@ -155,15 +155,6 @@ clustertests: vendor
clustertests-build: vendor
docker-compose -f $(DOCKER_COMPOSE) down -v
docker-compose -f $(DOCKER_COMPOSE) up --exit-code-from=client1 --build
# Test Cluster backup and restore
backuptests-build: vendor
docker-compose -f docker-compose-3.yml down
docker-compose -f docker-compose-3.yml build
backuptests: vendor
docker-compose -f docker-compose-3.yml down -v
docker-compose -f docker-compose-3.yml up --exit-code-from=client1 --abort-on-container-exit
# Install Pilosa
install:

64
api.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
//go:generate stringer -type=apiMethod
package pilosa
@ -847,6 +834,15 @@ func (api *API) FragmentData(ctx context.Context, indexName, fieldName, viewName
return f, nil
}
type RedirectError struct {
HostPort string
error string
}
func (r RedirectError) Error() string {
return r.error
}
// TranslateData returns all translation data in the specified partition.
func (api *API) TranslateData(ctx context.Context, indexName string, partition int) (io.WriterTo, error) {
span, _ := tracing.StartSpanFromContext(ctx, "API.TranslateData")
@ -862,6 +858,15 @@ func (api *API) TranslateData(ctx context.Context, indexName string, partition i
return nil, newNotFoundError(ErrIndexNotFound, indexName)
}
snap := topology.NewClusterSnapshot(api.cluster.noder, api.cluster.Hasher, api.cluster.ReplicaN)
nodes := snap.PartitionNodes(partition)
if nodes[0].ID != api.server.NodeID() {
return nil, RedirectError{
HostPort: nodes[0].URI.HostPort(),
error: fmt.Sprintf("can't translate data, this node(%s) does not partition %d", api.server.uri, partition),
}
}
// Retrieve translatestore from holder.
store := idx.TranslateStore(partition)
if store == nil {
@ -991,7 +996,10 @@ func (api *API) Usage(ctx context.Context, remote bool) (map[string]NodeUsage, e
return resp, nil
}
if api.usageCache.lastCalcDuration < usageCacheMinDuration {
api.usageCache.muAssign.Lock()
lastCalc := api.usageCache.lastCalcDuration
api.usageCache.muAssign.Unlock()
if lastCalc < usageCacheMinDuration {
err := api.ResetUsageCache()
if err != nil {
api.server.logger.Infof("could not reset usageCache: %s", err)
@ -2343,7 +2351,10 @@ func (api *API) GetTranslateEntryReader(ctx context.Context, offsets TranslateOf
if field == nil {
return nil, newNotFoundError(ErrFieldNotFound, fieldName)
}
store := field.TranslateStore()
if store == nil {
return nil, ErrTranslateStoreNotFound
}
r, err := field.TranslateStore().EntryReader(ctx, uint64(offset))
if err != nil {
return nil, errors.Wrap(err, "field translate reader")
@ -2643,7 +2654,13 @@ func (api *API) RestoreIDAlloc(r io.Reader) error {
// rd is a boltdb file.
func (api *API) TranslateIndexDB(ctx context.Context, indexName string, partitionID int, rd io.Reader) error {
idx := api.holder.Index(indexName)
if idx == nil {
return fmt.Errorf("index %q not found", indexName)
}
store := idx.TranslateStore(partitionID)
if store == nil {
return fmt.Errorf("index %q has no translate store", indexName)
}
_, err := store.ReadFrom(rd)
return err
}
@ -2651,8 +2668,23 @@ func (api *API) TranslateIndexDB(ctx context.Context, indexName string, partitio
// TranslateFieldDB is an internal function to load the field keys database
func (api *API) TranslateFieldDB(ctx context.Context, indexName, fieldName string, rd io.Reader) error {
idx := api.holder.Index(indexName)
if idx == nil {
return fmt.Errorf("index %q not found", indexName)
}
field := idx.Field(fieldName)
if field == nil {
// Older versions used to accidentally provide an empty translation
// data file for a nonexistent field called "_keys". To make migration
// easier, we politely ignore that.
if fieldName == "_keys" {
return nil
}
return fmt.Errorf("field %q/%q not found", indexName, fieldName)
}
store := field.TranslateStore()
if store == nil {
return fmt.Errorf("field %q/%q has no translate store", indexName, fieldName)
}
_, err := store.ReadFrom(rd)
return err
}

15
api/client/grpc.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
package client
import (

87
api_test.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
package pilosa_test
import (
@ -1356,3 +1343,75 @@ func createFieldForTest(index string, field string, coord *test.Command, t *test
t.Fatalf("creating field: %v", err)
}
}
func TestVariousApiTranslateCalls(t *testing.T) {
for i := 1; i < 8; i += 3 {
m := test.MustRunCluster(t, i)
defer m.Close()
node := m.GetNode(0)
api := node.API
// this should never actually get used because we're testing for errors here
r := strings.NewReader("")
// test index
idx, err := api.Holder().CreateIndex("index", pilosa.IndexOptions{})
if err != nil {
t.Fatalf("%v: could not create test index", err)
}
_, err = idx.CreateFieldIfNotExistsWithOptions("field", &pilosa.FieldOptions{Keys: false})
t.Run("translateIndexDbOnNilIndex",
func(t *testing.T) {
err := api.TranslateIndexDB(context.Background(), "nonExistentIndex", 0, r)
expected := fmt.Errorf("index %q not found", "nonExistentIndex")
if !reflect.DeepEqual(err, expected) {
t.Fatalf("expected '%#v', got '%#v'", expected, err)
}
})
t.Run("translateIndexDbOnNilTranslateStore",
func(t *testing.T) {
err := api.TranslateIndexDB(context.Background(), "index", 0, r)
expected := fmt.Errorf("index %q has no translate store", "index")
if !reflect.DeepEqual(err, expected) {
t.Fatalf("expected '%#v', got '%#v'", expected, err)
}
})
t.Run("translateFieldDbOnNilIndex",
func(t *testing.T) {
err := api.TranslateFieldDB(context.Background(), "nonExistentIndex", "field", r)
expected := fmt.Errorf("index %q not found", "nonExistentIndex")
if !reflect.DeepEqual(err, expected) {
t.Fatalf("expected '%#v', got '%#v'", expected, err)
}
})
t.Run("translateFieldDbOnNilField",
func(t *testing.T) {
err := api.TranslateFieldDB(context.Background(), "index", "nonExistentField", r)
expected := fmt.Errorf("field %q/%q not found", "index", "nonExistentField")
if !reflect.DeepEqual(err, expected) {
t.Fatalf("expected '%#v', got '%#v'", expected, err)
}
})
t.Run("translateFieldDbNilField_keys",
func(t *testing.T) {
err := api.TranslateFieldDB(context.Background(), "index", "_keys", r)
if err != nil {
t.Fatalf("expected 'nil', got '%#v'", err)
}
})
/*
TODO: this test will break, bc currently all fields create translate
stores, which is a bug, but one that we will eventually fix. when we do, this
test might come in handy t.Run("translateFieldDbOnNilTranslateStore",
func(t *testing.T) {
err := api.TranslateFieldDB(context.Background(), "index", "field", r)
expected := fmt.Errorf("field %q/%q has no translate store", "index", "field")
if !reflect.DeepEqual(err, expected) {
t.Fatalf("expected '%#v', got '%#v'", expected, err)
}
})
*/
}
}

15
audit.go
View file

@ -1,17 +1,4 @@
// Copyright 2020 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
package pilosa
import (

15
audit_internal_test.go
View file

@ -1,17 +1,4 @@
// Copyright 2020 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
package pilosa
import (

15
audit_test.go
View file

@ -1,17 +1,4 @@
// Copyright 2020 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
package pilosa_test
import (

15
auth/auth.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
package auth
import (

15
boltdb/translate.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
package boltdb
import (

14
boltdb/translate_test.go
View file

@ -1,16 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
package boltdb_test
import (

15
broadcast.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
package pilosa
import (

15
bsi.go
View file

@ -1,17 +1,4 @@
// Copyright 2020 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
package pilosa
import (

15
bsi_test.go
View file

@ -1,17 +1,4 @@
// Copyright 2020 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
package pilosa
import (

15
cache.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
package pilosa
import (

15
cache_test.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
package pilosa_test
import (

15
catcher.go
View file

@ -1,17 +1,4 @@
// Copyright 2020 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
package pilosa
import (

15
client.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
package pilosa
import (

19
client/batch.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
package client
import (
@ -342,6 +329,10 @@ func (qt *QuantizedTime) SetHour(hour string) {
copy(qt.ymdh[8:10], hour)
}
func (qt *QuantizedTime) Time() (time.Time, error) {
return time.Parse("2006010215", string(qt.ymdh[:]))
}
// Reset sets the time to the zero value which generates no time views.
func (qt *QuantizedTime) Reset() {
for i := range qt.ymdh {

18
client/batch_test.go
View file

@ -1,18 +1,6 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//+build integration
// Copyright 2021 Molecula Corp. All rights reserved.
//go:build integration
// +build integration
package client

26
client/client.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
// package ctl contains all pilosa subcommands other than 'server'. These are
// generally administration, testing, and debugging tools.
package client
@ -782,7 +769,7 @@ func (c *Client) readSchema() ([]SchemaIndex, error) {
func (c *Client) IngestSchema(reqBody map[string]interface{}) (body []byte, err error) {
data, err := json.Marshal(reqBody)
if err != nil {
return data, errors.Wrap(err, " error building Schema body to Ingest")
return data, errors.Wrap(err, "error building Schema body to Ingest")
}
return c.IngestRequest("/internal/schema", data)
}
@ -790,19 +777,19 @@ func (c *Client) IngestSchema(reqBody map[string]interface{}) (body []byte, err
func (c *Client) IngestData(index string, reqBody []map[string]interface{}) (body []byte, err error) {
data, err := json.Marshal(reqBody)
if err != nil {
return data, errors.Wrap(err, " error building request body to Ingest")
return data, errors.Wrap(err, "error building request body to Ingest")
}
return c.IngestRequest("/internal/ingest/"+index, data)
}
func (c *Client) IngestRequest(Uri string, data []byte) (body []byte, err error) {
func (c *Client) IngestRequest(uri string, data []byte) (body []byte, err error) {
var header = make(map[string]string)
header["Content-Type"] = "application/json"
header["Accept"] = "application/json"
header["User-Agent"] = "pilosa/" + pilosa.Version
_, body, err = c.HTTPRequest("POST", Uri, data, header)
status, body, err := c.HTTPRequest("POST", uri, data, header)
if err != nil {
return nil, errors.Wrap(err, "requesting "+Uri)
return nil, errors.Wrapf(err, "requesting %s status: %d", uri, status)
}
return body, err
}
@ -821,7 +808,6 @@ func (c *Client) shardsMax() (map[string]uint64, error) {
}
// HTTPRequest sends an HTTP request to the Pilosa server (used by idk)
// nolint: deadcode
func (c *Client) HTTPRequest(method string, path string, data []byte, headers map[string]string) (status int, body []byte, err error) {
span := c.tracer.StartSpan("Client.HTTPRequest")

15
client/client_it_test.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
package client
import (

15
client/client_test.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
// package ctl contains all pilosa subcommands other than 'server'. These are
// generally administration, testing, and debugging tools.

15
client/cluster.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
// package ctl contains all pilosa subcommands other than 'server'. These are
// generally administration, testing, and debugging tools.

15
client/cluster_test.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
// package ctl contains all pilosa subcommands other than 'server'. These are
// generally administration, testing, and debugging tools.

15
client/csv/csv.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
package csv
import (

18
client/csv/csv_it_test.go
View file

@ -1,18 +1,6 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//+build integration
// Copyright 2021 Molecula Corp. All rights reserved.
//go:build integration
// +build integration
package csv_test

15
client/csv/csv_test.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
package csv_test
import (

15
client/doc.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
// package ctl contains all pilosa subcommands other than 'server'. These are
// generally administration, testing, and debugging tools.

15
client/egpool/egpool.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
package egpool
import (

15
client/egpool/egpool_test.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
package egpool_test
import (

15
client/error.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
package client
import "github.com/pkg/errors"

145
client/ingest_api_batch.go Normal file
View file

@ -0,0 +1,145 @@
package client
import (
"time"
"github.com/molecula/featurebase/v2/logger"
"github.com/pkg/errors"
)
// NewIngestAPIBatch creates an alternate implementation of
// RecordBatch which exists to aid in testing the new Ingest API and
// is likely far slower than the Batch.
func NewIngestAPIBatch(client *Client, size int, logger logger.Logger, fields []*Field) *ingestAPIBatch {
if len(fields) == 0 {
return nil
}
return &ingestAPIBatch{
client: client,
log: logger,
fields: fields,
keyed: fields[0].index.Opts().Keys(),
index: fields[0].index.Name(),
batchSize: size,
recordsK: make(map[string]map[string]interface{}),
records: make(map[uint64]map[string]interface{}),
}
}
type ingestAPIBatch struct {
client *Client
log logger.Logger
batchSize int
fields []*Field
keyed bool
index string
// map[recordKey][fieldName]value
recordsK map[string]map[string]interface{}
records map[uint64]map[string]interface{}
}
func (b *ingestAPIBatch) Add(row Row) error {
if len(row.Clears) > 0 {
return errors.New("ingest api batch does not support clears")
}
values := make(map[string]interface{})
for i, val := range row.Values {
field := b.fields[i]
// val can be string, uint64, int64, []string, []uint64, nil
// TODO timestamp field might need special handling
// TODO check that the Row.Clears field is only used for packed bools, and then issue a warning/error (in IDK) if the ingest API mode is used in conjunction w/ packed bools.
if val == nil {
continue
}
zero := QuantizedTime{}
if field.Options().Type() == FieldTypeTime && row.Time != zero {
timeq, err := row.Time.Time()
if err != nil {
return errors.Wrap(err, "parsing row time")
}
values[field.Name()] = map[string]interface{}{"time": timeq.Format(time.RFC3339), "values": val}
} else {
values[field.Name()] = val
}
}
if b.keyed {
switch rowID := row.ID.(type) {
case string:
b.recordsK[rowID] = values
case []byte:
b.recordsK[string(rowID)] = values
default:
return errors.Errorf("unsupported rowID %v of type %[1]T, must be string, or []byte for keyed index", rowID)
}
if len(b.recordsK) >= b.batchSize {
return ErrBatchNowFull
}
} else {
rowID, ok := row.ID.(uint64)
if !ok {
return errors.Errorf("unsupported rowID %v of type %[1]T, must be uint64 for unkeyed index", row.ID)
}
b.records[rowID] = values
if len(b.records) >= b.batchSize {
return ErrBatchNowFull
}
}
return nil
}
func (b *ingestAPIBatch) Import() error {
if b.keyed {
return b.importKeyed()
}
return b.importUnkeyed()
}
func (b *ingestAPIBatch) importKeyed() error {
req := []map[string]interface{}{
{
"action": "set",
"records": b.recordsK,
},
}
bod, err := b.client.IngestData(b.index, req)
if err != nil {
return errors.Wrapf(err, "importKeyed, body: %s", bod)
}
for k := range b.recordsK {
delete(b.recordsK, k)
}
return nil
}
func (b *ingestAPIBatch) importUnkeyed() error {
req := []map[string]interface{}{
{
"action": "set",
"records": b.records,
},
}
bod, err := b.client.IngestData(b.index, req)
if err != nil {
return errors.Wrapf(err, "importKeyed, body: %s", bod)
}
for v := range b.records {
delete(b.records, v)
}
return nil
}
func (b *ingestAPIBatch) Len() int {
if b.keyed {
return len(b.recordsK)
}
return len(b.records)
}
func (b *ingestAPIBatch) Flush() error { return nil }

305
client/ingest_api_batch_test.go Normal file
View file

@ -0,0 +1,305 @@
package client
import (
"strings"
"testing"
"time"
"github.com/molecula/featurebase/v2/logger"
"github.com/molecula/featurebase/v2/test"
)
func TestIngestAPIBatchAdd(t *testing.T) {
t.Run("unkeyed", func(t *testing.T) {
batch := NewIngestAPIBatch(nil, 10, logger.NopLogger, []*Field{
{
name: "a",
index: &Index{name: "idxname", options: &IndexOptions{}},
options: &FieldOptions{
fieldType: FieldTypeSet,
},
},
{
name: "b",
index: &Index{name: "idxname", options: &IndexOptions{}},
options: &FieldOptions{
fieldType: FieldTypeSet,
keys: true,
},
},
{
name: "c",
index: &Index{name: "idxname", options: &IndexOptions{}},
options: &FieldOptions{
fieldType: FieldTypeTime,
keys: true,
},
},
})
qt := QuantizedTime{}
qt.Set(time.Date(2007, time.January, 1, 15, 0, 0, 0, time.UTC))
err := batch.Add(Row{
ID: uint64(1),
Values: []interface{}{uint64(2), "bkey", "ckey"},
Time: qt,
})
if err != nil {
t.Fatalf("adding row to batch: %v", err)
}
if batch.records[1]["a"] != uint64(2) {
t.Fatalf("unexpected batch.records: %+v", batch.records)
}
if batch.records[1]["b"] != "bkey" {
t.Fatalf("unexpected batch.records: %+v", batch.records)
}
if batch.records[1]["c"].(map[string]interface{})["time"] != "2007-01-01T15:00:00Z" {
t.Fatalf("unexpected batch.records: %+v", batch.records)
}
if batch.records[1]["c"].(map[string]interface{})["values"] != "ckey" {
t.Fatalf("unexpected batch.records: %+v", batch.records)
}
})
t.Run("keyed", func(t *testing.T) {
batch := NewIngestAPIBatch(nil, 10, logger.NopLogger, []*Field{
{
name: "a",
index: &Index{name: "idxname", options: &IndexOptions{keys: true}},
options: &FieldOptions{
fieldType: FieldTypeSet,
},
},
{
name: "b",
index: &Index{name: "idxname", options: &IndexOptions{keys: true}},
options: &FieldOptions{
fieldType: FieldTypeSet,
keys: true,
},
},
{
name: "c",
index: &Index{name: "idxname", options: &IndexOptions{keys: true}},
options: &FieldOptions{
fieldType: FieldTypeTime,
keys: true,
},
},
})
qt := QuantizedTime{}
qt.Set(time.Date(2007, time.January, 1, 15, 0, 0, 0, time.UTC))
err := batch.Add(Row{
ID: "1",
Values: []interface{}{uint64(2), "bkey", "ckey"},
Time: qt,
})
checkResult := func(batch *ingestAPIBatch, id string, err error) {
if err != nil {
t.Fatalf("adding row to batch: %v", err)
}
if batch.recordsK[id]["a"] != uint64(2) {
t.Fatalf("unexpected batch.records: %+v", batch.recordsK)
}
if batch.recordsK[id]["b"] != "bkey" {
t.Fatalf("unexpected batch.records: %+v", batch.recordsK)
}
if batch.recordsK[id]["c"].(map[string]interface{})["time"] != "2007-01-01T15:00:00Z" {
t.Fatalf("unexpected batch.records: %+v", batch.recordsK)
}
if batch.recordsK[id]["c"].(map[string]interface{})["values"] != "ckey" {
t.Fatalf("unexpected batch.records: %+v", batch.recordsK)
}
}
checkResult(batch, "1", err)
// test wrong type row ID
if err := batch.Add(Row{ID: 64.5}); !strings.Contains(err.Error(), "unsupported rowID") {
t.Fatalf("unexpected error w/ floating point rowID: %v", err)
}
// test that byte slice ID works same as string
err = batch.Add(Row{
ID: []byte("2"),
Values: []interface{}{uint64(2), "bkey", "ckey"},
Time: qt,
})
checkResult(batch, "2", err)
})
}
func TestIngestAPIBatch(t *testing.T) {
c := test.MustRunCluster(t, 3)
defer c.Close()
urls := make([]string, len(c.Nodes))
for i, n := range c.Nodes {
urls[i] = n.URL()
}
// Create a new client for the cluster
cli, err := newClientFromAddresses(urls, &ClientOptions{})
if err != nil {
t.Fatalf("getting new client: %v", err)
}
defer cli.Close()
cli.IngestSchema(map[string]interface{}{
"index-name": "test-1",
"index-action": "create",
"primary-key-type": "uint",
"field-action": "create",
"fields": []map[string]interface{}{
{
"field-name": "astr",
"field-type": "string",
"field-options": map[string]interface{}{},
},
{
"field-name": "bint",
"field-type": "int",
"field-options": map[string]interface{}{},
},
{
"field-name": "cid",
"field-type": "id",
"field-options": map[string]interface{}{},
},
{
"field-name": "dtimestamp",
"field-type": "timestamp",
"field-options": map[string]interface{}{
"unit": "s",
},
},
{
"field-name": "etime",
"field-type": "string",
"field-options": map[string]interface{}{
"time-quantum": "YMD",
},
},
{
"field-name": "fdecimal",
"field-type": "decimal",
"field-options": map[string]interface{}{
"scale": 3,
},
},
{
"field-name": "gbool",
"field-type": "bool",
"field-options": map[string]interface{}{},
},
},
})
schema, err := cli.Schema()
if err != nil {
t.Fatalf("getting schema: %v", err)
}
index := schema.Index("test-1")
defer cli.DeleteIndex(index)
batch := NewIngestAPIBatch(cli, 10, logger.NopLogger, []*Field{
{
name: "astr",
index: &Index{name: "test-1", options: &IndexOptions{}},
options: &FieldOptions{fieldType: FieldTypeSet, keys: true},
},
{
name: "bint",
options: &FieldOptions{fieldType: FieldTypeInt},
},
{
name: "cid",
options: &FieldOptions{fieldType: FieldTypeSet, keys: false},
},
{
name: "dtimestamp",
options: &FieldOptions{fieldType: FieldTypeTimestamp},
},
{
name: "etime",
options: &FieldOptions{fieldType: FieldTypeTime, keys: true, timeQuantum: TimeQuantumYearMonthDay},
},
{
name: "fdecimal",
options: &FieldOptions{fieldType: FieldTypeDecimal, scale: 3},
},
{
name: "gbool",
options: &FieldOptions{fieldType: FieldTypeBool},
},
})
qt0 := &QuantizedTime{}
qt0.Set(time.Date(2010, time.January, 1, 0, 0, 0, 0, time.UTC))
if err := batch.Add(Row{
ID: uint64(7),
Values: []interface{}{"a", -2, 9, 1287367623, "e", 1.2345, true},
Time: *qt0,
}); err != nil {
t.Fatalf("adding row: %v", err)
}
// test nil value case
if err := batch.Add(Row{
ID: uint64(8),
Values: []interface{}{nil, nil, nil, nil, nil, nil, nil},
Time: QuantizedTime{},
}); err != nil {
t.Fatalf("error adding all nil batch which should affect nothing: %v", err)
}
if err := batch.Import(); err != nil {
t.Fatalf("importing row: %v", err)
}
if resp, err := cli.Query(NewPQLBaseQuery("Row(astr=a)", &Index{name: "test-1", options: &IndexOptions{}}, nil)); err != nil {
t.Fatalf("querying: %v", err)
} else if len(resp.Result().Row().Columns) != 1 || resp.Result().Row().Columns[0] != uint64(7) {
t.Fatalf("unexpected Row(asr=a) result: %+v", resp.Result().Row().Columns)
}
if resp, err := cli.Query(NewPQLBaseQuery("Row(bint==-2)", &Index{name: "test-1", options: &IndexOptions{}}, nil)); err != nil {
t.Fatalf("querying: %v", err)
} else if len(resp.Result().Row().Columns) != 1 || resp.Result().Row().Columns[0] != uint64(7) {
t.Fatalf("unexpected Row(asr=a) result: %+v", resp.Result().Row().Columns)
}
if resp, err := cli.Query(NewPQLBaseQuery("Row(cid=9)", &Index{name: "test-1", options: &IndexOptions{}}, nil)); err != nil {
t.Fatalf("querying: %v", err)
} else if len(resp.Result().Row().Columns) != 1 || resp.Result().Row().Columns[0] != uint64(7) {
t.Fatalf("unexpected Row(asr=a) result: %+v", resp.Result().Row().Columns)
}
if resp, err := cli.Query(NewPQLBaseQuery("Row(dtimestamp=='2010-10-18T02:07:03Z')", &Index{name: "test-1", options: &IndexOptions{}}, nil)); err != nil {
t.Fatalf("querying: %v", err)
} else if len(resp.Result().Row().Columns) != 1 || resp.Result().Row().Columns[0] != uint64(7) {
t.Fatalf("unexpected Row(asr=a) result: %+v", resp.Result().Row().Columns)
}
if resp, err := cli.Query(NewPQLBaseQuery("Row(etime=e, from='2010-01-01', to='2010-01-02')", &Index{name: "test-1", options: &IndexOptions{}}, nil)); err != nil {
t.Fatalf("querying: %v", err)
} else if len(resp.Result().Row().Columns) != 1 || resp.Result().Row().Columns[0] != uint64(7) {
t.Fatalf("unexpected Row(asr=a) result: %+v", resp.Result().Row().Columns)
}
if resp, err := cli.Query(NewPQLBaseQuery("Row(fdecimal==1.234)", &Index{name: "test-1", options: &IndexOptions{}}, nil)); err != nil {
t.Fatalf("querying: %v", err)
} else if len(resp.Result().Row().Columns) != 1 || resp.Result().Row().Columns[0] != uint64(7) {
t.Fatalf("unexpected Row(asr=a) result: %+v", resp.Result().Row().Columns)
}
if resp, err := cli.Query(NewPQLBaseQuery("Row(gbool=true)", &Index{name: "test-1", options: &IndexOptions{}}, nil)); err != nil {
t.Fatalf("querying: %v", err)
} else if len(resp.Result().Row().Columns) != 1 || resp.Result().Row().Columns[0] != uint64(7) {
t.Fatalf("unexpected Row(asr=a) result: %+v", resp.Result().Row().Columns)
}
}

15
client/logimport.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
package client
import (

15
client/logimport_test.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
package client
import (

15
client/metrics.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
package client
const (

15
client/orm.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
// package ctl contains all pilosa subcommands other than 'server'. These are
// generally administration, testing, and debugging tools.

15
client/orm_test.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
// package ctl contains all pilosa subcommands other than 'server'. These are
// generally administration, testing, and debugging tools.

15
client/record.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
// package ctl contains all pilosa subcommands other than 'server'. These are
// generally administration, testing, and debugging tools.

15
client/record_test.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
// package ctl contains all pilosa subcommands other than 'server'. These are
// generally administration, testing, and debugging tools.

15
client/response.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
// package ctl contains all pilosa subcommands other than 'server'. These are
// generally administration, testing, and debugging tools.

15
client/response_test.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
// package ctl contains all pilosa subcommands other than 'server'. These are
// generally administration, testing, and debugging tools.

15
client/shardnodes.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
// package ctl contains all pilosa subcommands other than 'server'. These are
// generally administration, testing, and debugging tools.

15
client/tracer.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
// package ctl contains all pilosa subcommands other than 'server'. These are
// generally administration, testing, and debugging tools.

15
client/validate.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
// package ctl contains all pilosa subcommands other than 'server'. These are
// generally administration, testing, and debugging tools.

15
client/validate_test.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
// package ctl contains all pilosa subcommands other than 'server'. These are
// generally administration, testing, and debugging tools.

15
client/version.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
// package ctl contains all pilosa subcommands other than 'server'. These are
// generally administration, testing, and debugging tools.

23
cluster.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
package pilosa
import (
@ -1480,6 +1467,10 @@ func (c *cluster) matchField(ctx context.Context, field *Field, like string) ([]
if c.Node.ID == primary.ID {
// The local copy is the authoritative copy.
plan := planLike(like)
store := field.TranslateStore()
if store == nil {
return nil, ErrTranslateStoreNotFound
}
return field.TranslateStore().Match(func(key []byte) bool {
return matchLike(key, plan...)
})
@ -1521,6 +1512,10 @@ func (c *cluster) translateFieldListIDs(field *Field, ids []uint64) (keys []stri
}
if c.Node.ID == primary.ID {
store := field.TranslateStore()
if store == nil {
return nil, ErrTranslateStoreNotFound
}
keys, err = field.TranslateStore().TranslateIDs(ids)
} else {
keys, err = c.InternalClient.TranslateIDsNode(context.Background(), &primary.URI, field.Index(), field.Name(), ids)

15
cluster_internal_test.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
package pilosa
import (

15
cmd.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
package pilosa
import (

15
cmd/backup.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
package cmd
import (

15
cmd/badloader/badloader.go
View file

@ -1,17 +1,4 @@
// Copyright 2020 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
package main
import (

15
cmd/check.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
package cmd
import (

15
cmd/check_test.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
package cmd_test
import (

15
cmd/chksum.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
package cmd
import (

15
cmd/config.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
package cmd
import (

15
cmd/convert.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
package cmd
import (

15
cmd/doc.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
/*
Package cmd contains all the pilosa subcommand definitions (1 per file).

15
cmd/export.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
package cmd
import (

15
cmd/export_test.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
package cmd_test
import (

15
cmd/featurebase-parse-sql/main.go
View file

@ -1,17 +1,4 @@
// Copyright 2021 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
package main
import (

15
cmd/featurebase/main.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
/*
This is the entrypoint for the Pilosa binary.
*/

15
cmd/generate_config.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
package cmd
import (

15
cmd/import.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
package cmd
import (

15
cmd/import_test.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
package cmd_test
import (

15
cmd/inspect_test.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
package cmd_test
import (

15
cmd/pilosa-bench/main.go
View file

@ -1,17 +1,4 @@
// Copyright 2020 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
package main
import (

15
cmd/random-query/main.go
View file

@ -1,17 +1,4 @@
// Copyright 2020 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
package main
import (

15
cmd/random-query/main_test.go
View file

@ -1,17 +1,4 @@
// Copyright 2020 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
package main
import (

15
cmd/rbf.go
View file

@ -1,17 +1,4 @@
// Copyright 2020 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
package cmd
import (

15
cmd/restore.go
View file

@ -1,17 +1,4 @@
// Copyright 2020 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
package cmd
import (

16
cmd/roaring-migrate/ctim_darwin.go
View file

@ -1,17 +1,5 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
//go:build darwin
// +build darwin
package main

15
cmd/roaring-migrate/ctim_linux.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
//go:build linux
// +build linux

23
cmd/roaring-migrate/main.go
View file

@ -1,16 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
package main
import (
@ -243,9 +231,8 @@ func copyFile(src, dest string) error {
}
func Migrate(dataDir, backupPath string) error {
if strings.HasSuffix(dataDir, "/") {
dataDir = dataDir[:len(dataDir)-1]
}
dataDir = strings.TrimSuffix(dataDir, "/")
err := os.MkdirAll(backupPath, 0777)
if err != nil {
return err
@ -332,8 +319,7 @@ func Migrate(dataDir, backupPath string) error {
return err
}
parts := strings.Split(filename, "/")
//destFile := fmt.Sprintf("%v/indexes/%v/translate/%v", backupPath, parts[1], parts[3])
destFile := filepath.Join(backupPath, "indexes", parts[1], "translaste", parts[3])
destFile := filepath.Join(backupPath, "indexes", parts[1], "translate", parts[3])
err = writeIfBigger(destFile, content)
if err != nil {
return err
@ -349,7 +335,6 @@ func Migrate(dataDir, backupPath string) error {
return err
}
parts := strings.Split(filename, "/")
//destFile := fmt.Sprintf("%v/indexes/%v/fields/%v/translate", backupPath, parts[1], parts[2])
destFile := filepath.Join(backupPath, "indexes", parts[1], "fields", parts[2], "translate")
err = writeIfBigger(destFile, content)
if err != nil {

15
cmd/root.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
package cmd
import (

15
cmd/root_test.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
package cmd_test
import (

15
cmd/server.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
package cmd
import (

15
cmd/server_test.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
package cmd_test
import (

15
cmd/slurp/slurp.go
View file

@ -1,17 +1,4 @@
// Copyright 2020 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
package main
import (

15
const_amd64.go
View file

@ -1,17 +1,4 @@
// Copyright 2020 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
//go:build amd64
// +build amd64

16
const_other.go
View file

@ -1,17 +1,5 @@
// Copyright 2020 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
//go:build !amd64
// +build !amd64
package pilosa

9
cover-everything.sh
View file

@ -1,9 +0,0 @@
#!/usr/bin/env bash
# actually get test coverage for every single package and subpackage
# very slow but oh well what are you gonna do, not test things?
# note it skips the roaring migrate
echo "mode: atomic" > coverage.out
for pkg in $(go list all | grep featurebase ); do
go test -coverprofile=pkgcoverage.out -covermode=atomic $pkg;
tail -n +2 pkgcoverage.out >> coverage.out; done

35
ctl/backup.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
package ctl
import (
@ -31,7 +18,7 @@ import (
"golang.org/x/sync/errgroup"
)
// BackupCommand represents a command for backing up a Pilosa node.
// BackupCommand represents a command for backing up a FeatureBase node.
type BackupCommand struct { // nolint: maligned
tlsConfig *tls.Config
@ -196,12 +183,17 @@ func (cmd *BackupCommand) backupIDAllocData(ctx context.Context) error {
func (cmd *BackupCommand) backupIndexTranslation(ctx context.Context, ii *pilosa.IndexInfo) error {
logger := cmd.Logger()
logger.Printf("backing up index translation: %q", ii.Name)
if err := cmd.backupIndexTranslateData(ctx, ii.Name); err != nil {
return err
if ii.Options.Keys {
if err := cmd.backupIndexTranslateData(ctx, ii.Name); err != nil {
return err
}
}
// Back up field translation data.
for _, fi := range ii.Fields {
if !fi.Options.Keys {
continue
}
if err := cmd.backupFieldTranslateData(ctx, ii.Name, fi.Name); err != nil {
return fmt.Errorf("cannot backup field translation data for field %q on index %q: %w", fi.Name, ii.Name, err)
}
@ -299,7 +291,6 @@ func (cmd *BackupCommand) backupShardNode(ctx context.Context, indexName string,
func (cmd *BackupCommand) backupIndexTranslateData(ctx context.Context, name string) error {
partitionN := topology.DefaultPartitionN
// Back up all bitmap data for the index.
ch := make(chan int, partitionN)
for partitionID := 0; partitionID < partitionN; partitionID++ {
ch <- partitionID
@ -331,9 +322,7 @@ func (cmd *BackupCommand) backupIndexPartitionTranslateData(ctx context.Context,
logger.Printf("backing up index translation data: %s/%d", name, partitionID)
rc, err := cmd.client.IndexTranslateDataReader(ctx, name, partitionID)
if err == pilosa.ErrTranslateStoreNotFound {
return nil
} else if err != nil {
if err != nil {
return fmt.Errorf("fetching translate data reader: %w", err)
}
defer rc.Close()
@ -362,9 +351,7 @@ func (cmd *BackupCommand) backupFieldTranslateData(ctx context.Context, indexNam
logger.Printf("backing up field translation data: %s/%s", indexName, fieldName)
rc, err := cmd.client.FieldTranslateDataReader(ctx, indexName, fieldName)
if err == pilosa.ErrTranslateStoreNotFound {
return nil
} else if err != nil {
if err != nil {
return fmt.Errorf("fetching translate data reader: %w", err)
}
defer rc.Close()

15
ctl/check.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
package ctl
import (

15
ctl/check_test.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
package ctl
import (

34
ctl/chksum.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
package ctl
import (
@ -21,7 +8,7 @@ import (
"io"
"github.com/cespare/xxhash"
"github.com/molecula/featurebase/v2"
pilosa "github.com/molecula/featurebase/v2"
"github.com/molecula/featurebase/v2/server"
)
@ -74,14 +61,19 @@ func (cmd *ChkSumCommand) Run(ctx context.Context) (err error) {
h := xxhash.New()
for _, ii := range schema.Indexes {
qa := &pilosa.QueryRequest{Index: ii.Name, Query: "Count(All())"}
qa := &pilosa.QueryRequest{Index: ii.Name, Query: "All()"}
rs, err := client.Query(ctx, ii.Name, qa)
if err != nil {
return err
}
all := rs.Results[0].(uint64)
as := fmt.Sprintf("all=%v", all)
_, _ = h.Write([]byte(as))
all := rs.Results[0].(*pilosa.Row)
if len(all.Keys) > 0 {
allString := fmt.Sprintf("%v", all.Keys)
_, _ = h.Write([]byte(allString))
} else {
_, _ = h.Write(all.Roaring())
}
for _, field := range ii.Fields {
switch field.Options.Type {
@ -105,7 +97,7 @@ func (cmd *ChkSumCommand) Run(ctx context.Context) (err error) {
}
for _, item := range res.Results {
rowids := item.(*pilosa.RowIdentifiers)
//either rowids or keys
// either rowids or keys
for _, row := range rowids.Keys {
countPql := fmt.Sprintf(`Count(Row(%v="%v"))`, field.Name, row)
qr := &pilosa.QueryRequest{Index: ii.Name, Query: countPql}
@ -134,7 +126,7 @@ func (cmd *ChkSumCommand) Run(ctx context.Context) (err error) {
}
}
fmt.Printf("hash:%x\n", h.Sum(nil))
fmt.Fprintf(cmd.Stdout, "hash:%x\n", h.Sum(nil))
}
return nil

15
ctl/common.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
package ctl
import (

15
ctl/config.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
package ctl
import (

15
ctl/config_test.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
package ctl
import (

15
ctl/doc.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
// package ctl contains all pilosa subcommands other than 'server'. These are
// generally administration, testing, and debugging tools.
package ctl

15
ctl/export.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
package ctl
import (

15
ctl/export_test.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
package ctl
import (

15
ctl/generate_config.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
package ctl
import (

15
ctl/generate_config_test.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
package ctl
import (

15
ctl/import.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
package ctl
import (

15
ctl/import_test.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2021 Molecula Corp. All rights reserved.
package ctl
import (

Some files were not shown because too many files have changed in this diff Show more