diff --git a/Makefile b/Makefile
index 847b78eff..511a5900d 100644
--- a/Makefile
+++ b/Makefile
@@ -2,19 +2,18 @@
CLONE_URL=github.com/pilosa/pilosa
VERSION := $(shell git describe --tags 2> /dev/null || echo unknown)
-VERSION_ID = $(if $(ENTERPRISE_ENABLED),enterprise-)$(VERSION)-$(GOOS)-$(GOARCH)
+VARIANT = Molecula
+VERSION_ID = $(VERSION)-$(GOOS)-$(GOARCH)
BRANCH := $(if $(TRAVIS_BRANCH),$(TRAVIS_BRANCH),$(if $(CIRCLE_BRANCH),$(CIRCLE_BRANCH),$(shell git rev-parse --abbrev-ref HEAD)))
BRANCH_ID := $(BRANCH)-$(GOOS)-$(GOARCH)
BUILD_TIME := $(shell date -u +%FT%T%z)
SHARD_WIDTH = 20
-LDFLAGS="-X github.com/pilosa/pilosa/v2.Version=$(VERSION) -X github.com/pilosa/pilosa/v2.BuildTime=$(BUILD_TIME) -X github.com/pilosa/pilosa/v2.Enterprise=$(if $(ENTERPRISE_ENABLED),1)"
+COMMIT := $(shell git describe --exact-match >/dev/null 2>&1 || git rev-parse --short HEAD)
+LDFLAGS="-X github.com/pilosa/pilosa/v2.Version=$(VERSION) -X github.com/pilosa/pilosa/v2.BuildTime=$(BUILD_TIME) -X github.com/pilosa/pilosa/v2.Variant=$(VARIANT) -X github.com/pilosa/pilosa/v2.Commit=$(COMMIT)"
GO_VERSION=latest
-ENTERPRISE ?= 0
-ENTERPRISE_ENABLED = $(subst 0,,$(ENTERPRISE))
RELEASE ?= 0
RELEASE_ENABLED = $(subst 0,,$(RELEASE))
NOCHECKPTR=$(shell go version | grep -q 'go1.1[4,5,6,7]' && echo \"-gcflags=all=-d=checkptr=0\" )
-BUILD_TAGS += $(if $(ENTERPRISE_ENABLED),enterprise)
BUILD_TAGS += $(if $(RELEASE_ENABLED),release)
BUILD_TAGS += shardwidth$(SHARD_WIDTH)
BUILD_TAGS += $(foreach p,$(PLUGINS),plugin$(p))
@@ -66,8 +65,7 @@ build:
# Create a single release build under the build directory
release-build:
$(MAKE) $(if $(DOCKER_BUILD),docker-)build FLAGS="-o build/pilosa-$(VERSION_ID)/pilosa" RELEASE=1
- cp NOTICE README.md build/pilosa-$(VERSION_ID)
- $(if $(ENTERPRISE_ENABLED),cp enterprise/COPYING build/pilosa-$(VERSION_ID),cp LICENSE build/pilosa-$(VERSION_ID))
+ cp NOTICE README.md LICENSE build/pilosa-$(VERSION_ID)
tar -cvz -C build -f build/pilosa-$(VERSION_ID).tar.gz pilosa-$(VERSION_ID)/
@echo Created release build: build/pilosa-$(VERSION_ID).tar.gz
@@ -80,11 +78,8 @@ endif
# Create release build tarballs for all supported platforms. Linux compilation happens under Docker.
release: check-clean
$(MAKE) release-build GOOS=darwin GOARCH=amd64
- $(MAKE) release-build GOOS=darwin GOARCH=amd64 ENTERPRISE=1
$(MAKE) release-build GOOS=linux GOARCH=amd64
- $(MAKE) release-build GOOS=linux GOARCH=amd64 ENTERPRISE=1
$(MAKE) release-build GOOS=linux GOARCH=386
- $(MAKE) release-build GOOS=linux GOARCH=386 ENTERPRISE=1
# try (e.g.) internal/clustertests/docker-compose-replication2.yml
@@ -146,11 +141,6 @@ docker-tag-push: vendor
docker push $(DOCKER_TARGET)
@echo Pushed docker image: $(DOCKER_TARGET)
-# Create Docker image from Dockerfile (enterprise)
-docker-enterprise: vendor
- docker build --build-arg MAKE_FLAGS="ENTERPRISE=1" -t "pilosa-enterprise:$(VERSION)" .
- @echo Created docker image: pilosa-enterprise:$(VERSION)
-
# Compile Pilosa inside Docker container
docker-build:
docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) -e GOOS=$(GOOS) -e GOARCH=$(GOARCH) golang:$(GO_VERSION) go build -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa
diff --git a/NOTICE b/NOTICE
index 63c95e3c3..594272a27 100644
--- a/NOTICE
+++ b/NOTICE
@@ -14,27 +14,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-Enterprise Edition software license
-===================================
-
-Files contained under the directory `enterprise` are subject to the following
-license notice (Full license included in the file `COPYING`):
-
- Copyright (C) 2018 Pilosa Corp. All rights reserved.
-
- Pilosa Enterprise Edition is free software: you can redistribute it and/or modify
- it under the terms of the GNU Affero General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- Pilosa Enterprise Edition is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU Affero General Public License for more details.
-
- You should have received a copy of the GNU Affero General Public License
- along with Pilosa Enterprise Edition. If not, see .
-
Third-party software licenses
=============================
diff --git a/README.md b/README.md
index 8ccae2b91..1619c2b57 100644
--- a/README.md
+++ b/README.md
@@ -63,13 +63,12 @@ There are supported libraries for the following languages:
- [Java](https://www.pilosa.com/docs/client-libraries/#java)
- [Python](https://www.pilosa.com/docs/client-libraries/#python)
-## Licenses
+## License
-The core Pilosa code base and all default builds (referred to as Pilosa Community Edition) are licensed completely under the Apache License, Version 2.0.
-If you build Pilosa with the `enterprise` build tag (Pilosa Enterprise Edition), then that build will include features licensed under the GNU Affero General
-Public License (AGPL). Enterprise code is located entirely in the [github.com/pilosa/pilosa/enterprise](https://github.com/pilosa/pilosa/tree/master/enterprise)
-directory. See [github.com/pilosa/pilosa/NOTICE](https://github.com/pilosa/pilosa/blob/master/NOTICE) and
-[github.com/pilosa/pilosa/LICENSE](https://github.com/pilosa/pilosa/blob/master/LICENSE) for more information about Pilosa licenses.
+Pilosa is licensed under the Apache License, Version 2.0.
+
+A copy of the license is located in [github.com/pilosa/pilosa/LICENSE](https://github.com/pilosa/pilosa/blob/master/LICENSE).
+More details about licensing are found in [github.com/pilosa/pilosa/NOTICE](https://github.com/pilosa/pilosa/blob/master/NOTICE).
## Get Support
diff --git a/api.go b/api.go
index 8b6f3249d..23c36cb0b 100644
--- a/api.go
+++ b/api.go
@@ -1515,6 +1515,10 @@ func (api *API) Info() serverInfo {
}
}
+func (api *API) Inspect(ctx context.Context, req *InspectRequest) (*HolderInfo, error) {
+ return api.holder.Inspect(ctx, req)
+}
+
// GetTranslateEntryReader provides an entry reader for key translation logs starting at offset.
func (api *API) GetTranslateEntryReader(ctx context.Context, offsets TranslateOffsetMap) (_ TranslateEntryReader, err error) {
span, ctx := tracing.StartSpanFromContext(ctx, "API.GetTranslateEntryReader")
diff --git a/cluster_internal_test.go b/cluster_internal_test.go
index b9e714326..059053423 100644
--- a/cluster_internal_test.go
+++ b/cluster_internal_test.go
@@ -96,7 +96,7 @@ func newIndexWithTempPath(name string) *Index {
if err != nil {
panic(err)
}
- index, err := NewIndex(path, name, DefaultPartitionN)
+ index, err := NewIndex(NewHolder(DefaultPartitionN), path, name)
if err != nil {
panic(err)
}
diff --git a/cmd/inspect.go b/cmd/inspect.go
index 4a2e852f6..86452905d 100644
--- a/cmd/inspect.go
+++ b/cmd/inspect.go
@@ -45,5 +45,12 @@ Inspects a data file and provides stats.
return inspector.Run(context.Background())
},
}
+ flags := inspectCmd.Flags()
+ flags.BoolVarP(&inspector.Quiet, "quiet", "q", false, "don't list details of containers")
+ flags.IntVarP(&inspector.Max, "max", "n", 0, "list at most max items (0 = unlimited)")
+ flags.StringVarP(&inspector.InspectOpts.Indexes, "index", "i", "", "filter indexes")
+ flags.StringVarP(&inspector.InspectOpts.Views, "view", "v", "", "filter views")
+ flags.StringVarP(&inspector.InspectOpts.Fields, "field", "f", "", "filter fields")
+ flags.StringVarP(&inspector.InspectOpts.Shards, "shard", "s", "", "filter shards")
return inspectCmd
}
diff --git a/cmd/root.go b/cmd/root.go
index 228d3ed29..e3c25ef50 100644
--- a/cmd/root.go
+++ b/cmd/root.go
@@ -26,10 +26,6 @@ import (
)
func NewRootCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
- productName := "Pilosa " + pilosa.Version
- if pilosa.EnterpriseEnabled {
- productName = "Pilosa Enterprise " + pilosa.Version
- }
rc := &cobra.Command{
Use: "pilosa",
// TODO: These short/long descriptions could use some updating.
@@ -41,8 +37,7 @@ tools for administering Pilosa, importing/exporting data,
backing up, and more. Complete documentation is available
at https://www.pilosa.com/docs/.
-` + productName + `
-Build Time: ` + pilosa.BuildTime + "\n",
+` + pilosa.VersionInfo() + "\n",
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
v := viper.New()
err := setAllConfig(v, cmd.Flags(), "PILOSA")
diff --git a/ctl/inspect.go b/ctl/inspect.go
index 48924b89d..eb24a0597 100644
--- a/ctl/inspect.go
+++ b/ctl/inspect.go
@@ -16,15 +16,23 @@ package ctl
import (
"context"
+ "encoding/binary"
"fmt"
+ "hash/fnv"
"io"
+ "io/ioutil"
"os"
+ "path/filepath"
+ "sort"
+ "strconv"
+ "strings"
"syscall"
"text/tabwriter"
"time"
- "unsafe"
+ "github.com/gogo/protobuf/proto"
"github.com/pilosa/pilosa/v2"
+ "github.com/pilosa/pilosa/v2/internal"
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pkg/errors"
)
@@ -33,6 +41,12 @@ import (
type InspectCommand struct {
// Path to data file
Path string
+ // don't list details of objects
+ Quiet bool
+ // list only this many objects
+ Max int
+ // Filters:
+ InspectOpts pilosa.InspectRequest
// Standard input/output
*pilosa.CmdIO
@@ -45,8 +59,119 @@ func NewInspectCommand(stdin io.Reader, stdout, stderr io.Writer) *InspectComman
}
}
+type pointerContext struct {
+ from, to uintptr
+}
+
+func (p *pointerContext) pretty(c roaring.ContainerInfo) string {
+ var pointer string
+ if c.Mapped {
+ if c.Pointer >= p.from && c.Pointer < p.to {
+ pointer = fmt.Sprintf("@+0x%x", c.Pointer-p.from)
+ } else {
+ pointer = fmt.Sprintf("!0x%x!", c.Pointer)
+ }
+ } else {
+ pointer = fmt.Sprintf("0x%x", c.Pointer)
+ }
+ return fmt.Sprintf("%s \t%d \t%d \t%s ", c.Type, c.N, c.Alloc, pointer)
+}
+
+func (cmd *InspectCommand) PrintOps(info roaring.BitmapInfo) {
+ fmt.Fprintln(cmd.Stdout, " Ops:")
+ tw := tabwriter.NewWriter(cmd.Stdout, 0, 8, 0, '\t', 0)
+ fmt.Fprintf(tw, " \t%s\t%s\t%s\t\n", "TYPE", "OpN", "SIZE")
+ printed := 0
+ for _, op := range info.OpDetails {
+ fmt.Fprintf(tw, "\t%s\t%d\t%d\t\n", op.Type, op.OpN, op.Size)
+ printed++
+ if cmd.Max != 0 && printed >= cmd.Max {
+ break
+ }
+ }
+ tw.Flush()
+}
+
+func (cmd *InspectCommand) PrintContainers(info roaring.BitmapInfo, pC pointerContext) {
+ fmt.Fprintln(cmd.Stdout, " Containers:")
+ tw := tabwriter.NewWriter(cmd.Stdout, 0, 8, 0, '\t', 0)
+ fmt.Fprintf(tw, " \t\tRoaring\t\t\t\tOps\t\t\t\tFlags\t\n")
+ fmt.Fprintf(tw, "\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t\n", "KEY", "TYPE", "N", "ALLOC", "OFFSET", "TYPE", "N", "ALLOC", "OFFSET", "FLAGS")
+ c1s := info.Containers
+ c2s := info.OpContainers
+ l1 := len(c1s)
+ l2 := len(c2s)
+ i1 := 0
+ i2 := 0
+ var c1, c2 roaring.ContainerInfo
+ c1.Key = ^uint64(0)
+ c2.Key = ^uint64(0)
+ c1e := false
+ c2e := false
+ if i1 < l1 {
+ c1 = c1s[i1]
+ i1++
+ c1e = true
+ }
+ if i2 < l2 {
+ c2 = c2s[i2]
+ i2++
+ c2e = true
+ }
+ printed := 0
+ for c1e || c2e {
+ c1used := false
+ c2used := false
+ var key uint64
+ c1fmt := "-\t\t\t"
+ c2fmt := "-\t\t\t"
+ // If c2 exists, we'll always prefer its flags,
+ // if it doesn't, this gets overwritten.
+ flags := c2.Flags
+ if !c2e || (c1e && c1.Key < c2.Key) {
+ c1fmt = pC.pretty(c1)
+ key = c1.Key
+ c1used = true
+ flags = c1.Flags
+ } else if !c1e || (c2e && c2.Key < c1.Key) {
+ c2fmt = pC.pretty(c2)
+ key = c2.Key
+ c2used = true
+ } else {
+ // c1e and c2e both set, and neither key is < the other.
+ c1fmt = pC.pretty(c1)
+ c2fmt = pC.pretty(c2)
+ key = c1.Key
+ c1used = true
+ c2used = true
+ }
+ if c1used {
+ if i1 < l1 {
+ c1 = c1s[i1]
+ i1++
+ } else {
+ c1e = false
+ }
+ }
+ if c2used {
+ if i2 < l2 {
+ c2 = c2s[i2]
+ i2++
+ } else {
+ c2e = false
+ }
+ }
+ fmt.Fprintf(tw, "\t%d\t%s\t%s\t%s\t\n", key, c1fmt, c2fmt, flags)
+ printed++
+ if cmd.Max > 0 && printed >= cmd.Max {
+ break
+ }
+ }
+ tw.Flush()
+}
+
// Run executes the inspect command.
-func (cmd *InspectCommand) Run(_ context.Context) error {
+func (cmd *InspectCommand) Run(ctx context.Context) error {
// Open file handle.
f, err := os.Open(cmd.Path)
if err != nil {
@@ -58,7 +183,173 @@ func (cmd *InspectCommand) Run(_ context.Context) error {
if err != nil {
return errors.Wrap(err, "statting file")
}
+ if fi.IsDir() {
+ total := 0
+ infos, err := f.Readdir(0)
+ if err != nil {
+ return err
+ }
+ if len(infos) == 0 {
+ return errors.New("directory contains no files")
+ }
+ names := make([]string, len(infos))
+ nameToInfo := make(map[string]os.FileInfo, len(infos))
+ // find numeric-only names; we'll operate on
+ // either those, or the whole holder if we find
+ // a .topology file.
+ n := 0
+ for _, fi := range infos {
+ name := fi.Name()
+ if name == ".topology" {
+ return cmd.InspectHolder(ctx, cmd.Path)
+ }
+ if _, err := strconv.Atoi(name); err == nil {
+ names[n] = name
+ nameToInfo[name] = fi
+ n++
+ }
+ }
+ if n == 0 {
+ return fmt.Errorf("directory contains no fragments (looking for numeric names)")
+ }
+ names = names[:n]
+ fmt.Fprintf(cmd.Stdout, "%s contains %d fragments:\n", cmd.Path, n)
+ for _, name := range names {
+ f2, err := os.Open(filepath.Join(cmd.Path, name))
+ if err != nil {
+ return fmt.Errorf("opening %q: %v", name, err)
+ }
+ fmt.Fprintf(cmd.Stdout, "%s/%s:\n", cmd.Path, name)
+ err = cmd.InspectFile(f2, nameToInfo[name])
+ total++
+ f2.Close()
+ if err != nil {
+ return fmt.Errorf("inspecting %q: %v", name, err)
+ }
+ }
+ return nil
+ }
+ return cmd.InspectFile(f, fi)
+}
+
+// loadTopology is copied almost exactly from pilosa/cluster.go.
+func loadTopology(path string) (topology internal.Topology, myID string, err error) {
+ buf, err := ioutil.ReadFile(filepath.Join(path, ".topology"))
+ if os.IsNotExist(err) {
+ return topology, myID, err
+ } else if err != nil {
+ return topology, myID, errors.Wrap(err, "reading file")
+ }
+ if err := proto.Unmarshal(buf, &topology); err != nil {
+ return topology, myID, errors.Wrap(err, "unmarshalling")
+ }
+ sort.Slice(topology.NodeIDs,
+ func(i, j int) bool {
+ return topology.NodeIDs[i] < topology.NodeIDs[j]
+ })
+ buf, err = ioutil.ReadFile(filepath.Join(path, ".id"))
+ if os.IsNotExist(err) {
+ return topology, myID, err
+ } else if err != nil {
+ return topology, myID, nil
+ }
+ myID = strings.TrimSpace(string(buf))
+ return topology, myID, nil
+}
+
+var partitions = make(map[string]map[uint64]int)
+
+func findPartition(index string, shard uint64, partitionN int) (partition int) {
+ var shardMap map[uint64]int
+ var ok bool
+ if shardMap, ok = partitions[index]; !ok {
+ shardMap = make(map[uint64]int)
+ partitions[index] = shardMap
+ }
+ if partition, ok = shardMap[shard]; !ok {
+ var buf [8]byte
+ binary.BigEndian.PutUint64(buf[:], shard)
+
+ // Hash the bytes and mod by partition count.
+ h := fnv.New64a()
+ _, _ = h.Write([]byte(index))
+ _, _ = h.Write(buf[:])
+ partition = int(h.Sum64() % uint64(partitionN))
+ shardMap[shard] = partition
+ }
+ return partition
+}
+
+func findPartitionPath(path string, partitionN int) (int, error) {
+ parts := strings.Split(path, "/")
+ shard, err := strconv.ParseUint(parts[len(parts)-1], 10, 64)
+ if err != nil {
+ return 0, err
+ }
+ return findPartition(parts[0], shard, partitionN), nil
+}
+
+func (cmd *InspectCommand) InspectHolder(ctx context.Context, path string) error {
+ holder := pilosa.NewHolder(pilosa.DefaultPartitionN)
+ holder.Path = path
+ holder.Opts.Inspect = true
+ holder.Opts.ReadOnly = true
+ err := holder.Open()
+ if err != nil {
+ return fmt.Errorf("%s: holder open: %v", path, err)
+ }
+ holderInfo, err := holder.Inspect(ctx, &cmd.InspectOpts)
+ if err != nil {
+ return fmt.Errorf("%s: inspect: %v", path, err)
+ }
+ myPartition := 0
+ topology, myID, err := loadTopology(path)
+ if err == nil {
+ fmt.Fprintf(cmd.Stdout, "Cluster ID: %q\n", topology.ClusterID)
+ if len(topology.NodeIDs) > 1 {
+ fmt.Fprintf(cmd.Stdout, "Cluster of %d nodes, this node %q\n", len(topology.NodeIDs), myID)
+ } else {
+ fmt.Fprintf(cmd.Stdout, "Cluster has only one node: %q\n", myID)
+ }
+ found := false
+ for i := range topology.NodeIDs {
+ if topology.NodeIDs[i] == myID {
+ found = true
+ myPartition = i
+ break
+ }
+ }
+ if !found {
+ fmt.Fprintf(cmd.Stdout, "Warning: node ID %q not found in topology (%q)\n", myID, topology.NodeIDs)
+ }
+ } else {
+ fmt.Fprintf(cmd.Stdout, "warning: reading topology failed: %v\n", err)
+ }
+ for _, name := range holderInfo.FragmentNames {
+ partition, err := findPartitionPath(name, len(topology.NodeIDs))
+ if err != nil {
+ fmt.Fprintf(cmd.Stdout, "%s: [can't find partition: %v]\n", name, err)
+ } else {
+ if partition == myPartition {
+ fmt.Fprintf(cmd.Stdout, "%s:\n", name)
+ } else {
+ fmt.Fprintf(cmd.Stdout, "%s: [primary node %q]\n", name, topology.NodeIDs[partition])
+ }
+ }
+ details := holderInfo.FragmentInfo[name]
+ cmd.DisplayInfo(details.BitmapInfo)
+ if details.BlockChecksums != nil {
+ fmt.Fprintf(cmd.Stdout, " Checksums [%d total]:\n", len(details.BlockChecksums))
+ for _, block := range details.BlockChecksums {
+ fmt.Fprintf(cmd.Stdout, " %8d: %x\n", block.ID, block.Checksum)
+ }
+ }
+ }
+ return nil
+}
+
+func (cmd *InspectCommand) InspectFile(f *os.File, fi os.FileInfo) error {
// Memory map the file.
data, err := syscall.Mmap(int(f.Fd()), 0, int(fi.Size()), syscall.PROT_READ, syscall.MAP_SHARED)
if err != nil {
@@ -72,39 +363,37 @@ func (cmd *InspectCommand) Run(_ context.Context) error {
}()
// Attach the mmap file to the bitmap.
t := time.Now()
- fmt.Fprintf(cmd.Stderr, "unmarshalling bitmap...")
- bm := roaring.NewBitmap()
- if err := bm.UnmarshalBinary(data); err != nil {
- return errors.Wrap(err, "unmarshalling")
+ fmt.Fprintf(cmd.Stderr, "inspecting bitmap...")
+ var info roaring.BitmapInfo
+ _, _, err = roaring.InspectBinary(data, true, &info)
+ fmt.Fprintf(cmd.Stderr, " (%s)\n", time.Since(t))
+ cmd.DisplayInfo(info)
+ if err != nil {
+ return errors.Wrap(err, "inspecting")
}
- fmt.Fprintf(cmd.Stderr, " (%s)\n", time.Since(t))
+ return nil
+}
- // Retrieve stats.
- t = time.Now()
- fmt.Fprintf(cmd.Stderr, "calculating stats...")
- info := bm.Info()
- fmt.Fprintf(cmd.Stderr, " (%s)\n", time.Since(t))
+func (cmd *InspectCommand) DisplayInfo(info roaring.BitmapInfo) {
+ pC := pointerContext{
+ from: info.From,
+ to: info.To,
+ }
// Print top-level info.
- fmt.Fprintf(cmd.Stdout, "== Bitmap Info ==\n")
- fmt.Fprintf(cmd.Stdout, "Containers: %d\n", len(info.Containers))
- fmt.Fprintf(cmd.Stdout, "Operations: %d\n", info.OpN)
+ fmt.Fprintf(cmd.Stdout, " Bitmap Info:\n")
+ fmt.Fprintf(cmd.Stdout, " Bits: %d\n", info.BitCount)
+ fmt.Fprintf(cmd.Stdout, " Containers: %d (%d roaring)\n", info.ContainerCount, len(info.Containers))
+ fmt.Fprintf(cmd.Stdout, " Operations: %d (%d bits)\n", info.Ops, info.OpN)
fmt.Fprintln(cmd.Stdout, "")
// Print info for each container.
- fmt.Fprintln(cmd.Stdout, "== Containers ==")
- tw := tabwriter.NewWriter(cmd.Stdout, 0, 8, 0, '\t', 0)
- fmt.Fprintf(tw, "%s\t%s\t% 8s \t% 8s\t%s\n", "KEY", "TYPE", "N", "ALLOC", "OFFSET")
- for _, ci := range info.Containers {
- fmt.Fprintf(tw, "%d\t%s\t% 8d \t% 8d \t0x%08x\n",
- ci.Key,
- ci.Type,
- ci.N,
- ci.Alloc,
- uintptr(ci.Pointer)-uintptr(unsafe.Pointer(&data[0])),
- )
+ if !cmd.Quiet {
+ if info.ContainerCount > 0 {
+ cmd.PrintContainers(info, pC)
+ }
+ if info.Ops > 0 {
+ cmd.PrintOps(info)
+ }
}
- tw.Flush()
-
- return nil
}
diff --git a/ctl/inspect_test.go b/ctl/inspect_test.go
index 1ff852712..2698dcc03 100644
--- a/ctl/inspect_test.go
+++ b/ctl/inspect_test.go
@@ -41,7 +41,7 @@ func TestInspectCommand_Run(t *testing.T) {
file.Close()
cm.Path = file.Name()
err = cm.Run(context.Background())
- expectedError := "unmarshalling: "
+ expectedError := "inspecting: "
if !strings.Contains(err.Error(), expectedError) {
t.Fatalf("expected error '%s', got '%v'", expectedError, err)
}
@@ -52,7 +52,7 @@ func TestInspectCommand_Run(t *testing.T) {
if err != nil {
t.Fatalf("copying data: %v", err)
}
- if !strings.Contains(buf.String(), "unmarshalling bitmap...") {
+ if !strings.Contains(buf.String(), "inspecting bitmap...") {
t.Fatalf("Inspect doesn't work: %s", err)
}
diff --git a/enterprise/COPYING b/enterprise/COPYING
deleted file mode 100644
index be3f7b28e..000000000
--- a/enterprise/COPYING
+++ /dev/null
@@ -1,661 +0,0 @@
- GNU AFFERO GENERAL PUBLIC LICENSE
- Version 3, 19 November 2007
-
- Copyright (C) 2007 Free Software Foundation, Inc.
- Everyone is permitted to copy and distribute verbatim copies
- of this license document, but changing it is not allowed.
-
- Preamble
-
- The GNU Affero General Public License is a free, copyleft license for
-software and other kinds of works, specifically designed to ensure
-cooperation with the community in the case of network server software.
-
- The licenses for most software and other practical works are designed
-to take away your freedom to share and change the works. By contrast,
-our General Public Licenses are intended to guarantee your freedom to
-share and change all versions of a program--to make sure it remains free
-software for all its users.
-
- When we speak of free software, we are referring to freedom, not
-price. Our General Public Licenses are designed to make sure that you
-have the freedom to distribute copies of free software (and charge for
-them if you wish), that you receive source code or can get it if you
-want it, that you can change the software or use pieces of it in new
-free programs, and that you know you can do these things.
-
- Developers that use our General Public Licenses protect your rights
-with two steps: (1) assert copyright on the software, and (2) offer
-you this License which gives you legal permission to copy, distribute
-and/or modify the software.
-
- A secondary benefit of defending all users' freedom is that
-improvements made in alternate versions of the program, if they
-receive widespread use, become available for other developers to
-incorporate. Many developers of free software are heartened and
-encouraged by the resulting cooperation. However, in the case of
-software used on network servers, this result may fail to come about.
-The GNU General Public License permits making a modified version and
-letting the public access it on a server without ever releasing its
-source code to the public.
-
- The GNU Affero General Public License is designed specifically to
-ensure that, in such cases, the modified source code becomes available
-to the community. It requires the operator of a network server to
-provide the source code of the modified version running there to the
-users of that server. Therefore, public use of a modified version, on
-a publicly accessible server, gives the public access to the source
-code of the modified version.
-
- An older license, called the Affero General Public License and
-published by Affero, was designed to accomplish similar goals. This is
-a different license, not a version of the Affero GPL, but Affero has
-released a new version of the Affero GPL which permits relicensing under
-this license.
-
- The precise terms and conditions for copying, distribution and
-modification follow.
-
- TERMS AND CONDITIONS
-
- 0. Definitions.
-
- "This License" refers to version 3 of the GNU Affero General Public License.
-
- "Copyright" also means copyright-like laws that apply to other kinds of
-works, such as semiconductor masks.
-
- "The Program" refers to any copyrightable work licensed under this
-License. Each licensee is addressed as "you". "Licensees" and
-"recipients" may be individuals or organizations.
-
- To "modify" a work means to copy from or adapt all or part of the work
-in a fashion requiring copyright permission, other than the making of an
-exact copy. The resulting work is called a "modified version" of the
-earlier work or a work "based on" the earlier work.
-
- A "covered work" means either the unmodified Program or a work based
-on the Program.
-
- To "propagate" a work means to do anything with it that, without
-permission, would make you directly or secondarily liable for
-infringement under applicable copyright law, except executing it on a
-computer or modifying a private copy. Propagation includes copying,
-distribution (with or without modification), making available to the
-public, and in some countries other activities as well.
-
- To "convey" a work means any kind of propagation that enables other
-parties to make or receive copies. Mere interaction with a user through
-a computer network, with no transfer of a copy, is not conveying.
-
- An interactive user interface displays "Appropriate Legal Notices"
-to the extent that it includes a convenient and prominently visible
-feature that (1) displays an appropriate copyright notice, and (2)
-tells the user that there is no warranty for the work (except to the
-extent that warranties are provided), that licensees may convey the
-work under this License, and how to view a copy of this License. If
-the interface presents a list of user commands or options, such as a
-menu, a prominent item in the list meets this criterion.
-
- 1. Source Code.
-
- The "source code" for a work means the preferred form of the work
-for making modifications to it. "Object code" means any non-source
-form of a work.
-
- A "Standard Interface" means an interface that either is an official
-standard defined by a recognized standards body, or, in the case of
-interfaces specified for a particular programming language, one that
-is widely used among developers working in that language.
-
- The "System Libraries" of an executable work include anything, other
-than the work as a whole, that (a) is included in the normal form of
-packaging a Major Component, but which is not part of that Major
-Component, and (b) serves only to enable use of the work with that
-Major Component, or to implement a Standard Interface for which an
-implementation is available to the public in source code form. A
-"Major Component", in this context, means a major essential component
-(kernel, window system, and so on) of the specific operating system
-(if any) on which the executable work runs, or a compiler used to
-produce the work, or an object code interpreter used to run it.
-
- The "Corresponding Source" for a work in object code form means all
-the source code needed to generate, install, and (for an executable
-work) run the object code and to modify the work, including scripts to
-control those activities. However, it does not include the work's
-System Libraries, or general-purpose tools or generally available free
-programs which are used unmodified in performing those activities but
-which are not part of the work. For example, Corresponding Source
-includes interface definition files associated with source files for
-the work, and the source code for shared libraries and dynamically
-linked subprograms that the work is specifically designed to require,
-such as by intimate data communication or control flow between those
-subprograms and other parts of the work.
-
- The Corresponding Source need not include anything that users
-can regenerate automatically from other parts of the Corresponding
-Source.
-
- The Corresponding Source for a work in source code form is that
-same work.
-
- 2. Basic Permissions.
-
- All rights granted under this License are granted for the term of
-copyright on the Program, and are irrevocable provided the stated
-conditions are met. This License explicitly affirms your unlimited
-permission to run the unmodified Program. The output from running a
-covered work is covered by this License only if the output, given its
-content, constitutes a covered work. This License acknowledges your
-rights of fair use or other equivalent, as provided by copyright law.
-
- You may make, run and propagate covered works that you do not
-convey, without conditions so long as your license otherwise remains
-in force. You may convey covered works to others for the sole purpose
-of having them make modifications exclusively for you, or provide you
-with facilities for running those works, provided that you comply with
-the terms of this License in conveying all material for which you do
-not control copyright. Those thus making or running the covered works
-for you must do so exclusively on your behalf, under your direction
-and control, on terms that prohibit them from making any copies of
-your copyrighted material outside their relationship with you.
-
- Conveying under any other circumstances is permitted solely under
-the conditions stated below. Sublicensing is not allowed; section 10
-makes it unnecessary.
-
- 3. Protecting Users' Legal Rights From Anti-Circumvention Law.
-
- No covered work shall be deemed part of an effective technological
-measure under any applicable law fulfilling obligations under article
-11 of the WIPO copyright treaty adopted on 20 December 1996, or
-similar laws prohibiting or restricting circumvention of such
-measures.
-
- When you convey a covered work, you waive any legal power to forbid
-circumvention of technological measures to the extent such circumvention
-is effected by exercising rights under this License with respect to
-the covered work, and you disclaim any intention to limit operation or
-modification of the work as a means of enforcing, against the work's
-users, your or third parties' legal rights to forbid circumvention of
-technological measures.
-
- 4. Conveying Verbatim Copies.
-
- You may convey verbatim copies of the Program's source code as you
-receive it, in any medium, provided that you conspicuously and
-appropriately publish on each copy an appropriate copyright notice;
-keep intact all notices stating that this License and any
-non-permissive terms added in accord with section 7 apply to the code;
-keep intact all notices of the absence of any warranty; and give all
-recipients a copy of this License along with the Program.
-
- You may charge any price or no price for each copy that you convey,
-and you may offer support or warranty protection for a fee.
-
- 5. Conveying Modified Source Versions.
-
- You may convey a work based on the Program, or the modifications to
-produce it from the Program, in the form of source code under the
-terms of section 4, provided that you also meet all of these conditions:
-
- a) The work must carry prominent notices stating that you modified
- it, and giving a relevant date.
-
- b) The work must carry prominent notices stating that it is
- released under this License and any conditions added under section
- 7. This requirement modifies the requirement in section 4 to
- "keep intact all notices".
-
- c) You must license the entire work, as a whole, under this
- License to anyone who comes into possession of a copy. This
- License will therefore apply, along with any applicable section 7
- additional terms, to the whole of the work, and all its parts,
- regardless of how they are packaged. This License gives no
- permission to license the work in any other way, but it does not
- invalidate such permission if you have separately received it.
-
- d) If the work has interactive user interfaces, each must display
- Appropriate Legal Notices; however, if the Program has interactive
- interfaces that do not display Appropriate Legal Notices, your
- work need not make them do so.
-
- A compilation of a covered work with other separate and independent
-works, which are not by their nature extensions of the covered work,
-and which are not combined with it such as to form a larger program,
-in or on a volume of a storage or distribution medium, is called an
-"aggregate" if the compilation and its resulting copyright are not
-used to limit the access or legal rights of the compilation's users
-beyond what the individual works permit. Inclusion of a covered work
-in an aggregate does not cause this License to apply to the other
-parts of the aggregate.
-
- 6. Conveying Non-Source Forms.
-
- You may convey a covered work in object code form under the terms
-of sections 4 and 5, provided that you also convey the
-machine-readable Corresponding Source under the terms of this License,
-in one of these ways:
-
- a) Convey the object code in, or embodied in, a physical product
- (including a physical distribution medium), accompanied by the
- Corresponding Source fixed on a durable physical medium
- customarily used for software interchange.
-
- b) Convey the object code in, or embodied in, a physical product
- (including a physical distribution medium), accompanied by a
- written offer, valid for at least three years and valid for as
- long as you offer spare parts or customer support for that product
- model, to give anyone who possesses the object code either (1) a
- copy of the Corresponding Source for all the software in the
- product that is covered by this License, on a durable physical
- medium customarily used for software interchange, for a price no
- more than your reasonable cost of physically performing this
- conveying of source, or (2) access to copy the
- Corresponding Source from a network server at no charge.
-
- c) Convey individual copies of the object code with a copy of the
- written offer to provide the Corresponding Source. This
- alternative is allowed only occasionally and noncommercially, and
- only if you received the object code with such an offer, in accord
- with subsection 6b.
-
- d) Convey the object code by offering access from a designated
- place (gratis or for a charge), and offer equivalent access to the
- Corresponding Source in the same way through the same place at no
- further charge. You need not require recipients to copy the
- Corresponding Source along with the object code. If the place to
- copy the object code is a network server, the Corresponding Source
- may be on a different server (operated by you or a third party)
- that supports equivalent copying facilities, provided you maintain
- clear directions next to the object code saying where to find the
- Corresponding Source. Regardless of what server hosts the
- Corresponding Source, you remain obligated to ensure that it is
- available for as long as needed to satisfy these requirements.
-
- e) Convey the object code using peer-to-peer transmission, provided
- you inform other peers where the object code and Corresponding
- Source of the work are being offered to the general public at no
- charge under subsection 6d.
-
- A separable portion of the object code, whose source code is excluded
-from the Corresponding Source as a System Library, need not be
-included in conveying the object code work.
-
- A "User Product" is either (1) a "consumer product", which means any
-tangible personal property which is normally used for personal, family,
-or household purposes, or (2) anything designed or sold for incorporation
-into a dwelling. In determining whether a product is a consumer product,
-doubtful cases shall be resolved in favor of coverage. For a particular
-product received by a particular user, "normally used" refers to a
-typical or common use of that class of product, regardless of the status
-of the particular user or of the way in which the particular user
-actually uses, or expects or is expected to use, the product. A product
-is a consumer product regardless of whether the product has substantial
-commercial, industrial or non-consumer uses, unless such uses represent
-the only significant mode of use of the product.
-
- "Installation Information" for a User Product means any methods,
-procedures, authorization keys, or other information required to install
-and execute modified versions of a covered work in that User Product from
-a modified version of its Corresponding Source. The information must
-suffice to ensure that the continued functioning of the modified object
-code is in no case prevented or interfered with solely because
-modification has been made.
-
- If you convey an object code work under this section in, or with, or
-specifically for use in, a User Product, and the conveying occurs as
-part of a transaction in which the right of possession and use of the
-User Product is transferred to the recipient in perpetuity or for a
-fixed term (regardless of how the transaction is characterized), the
-Corresponding Source conveyed under this section must be accompanied
-by the Installation Information. But this requirement does not apply
-if neither you nor any third party retains the ability to install
-modified object code on the User Product (for example, the work has
-been installed in ROM).
-
- The requirement to provide Installation Information does not include a
-requirement to continue to provide support service, warranty, or updates
-for a work that has been modified or installed by the recipient, or for
-the User Product in which it has been modified or installed. Access to a
-network may be denied when the modification itself materially and
-adversely affects the operation of the network or violates the rules and
-protocols for communication across the network.
-
- Corresponding Source conveyed, and Installation Information provided,
-in accord with this section must be in a format that is publicly
-documented (and with an implementation available to the public in
-source code form), and must require no special password or key for
-unpacking, reading or copying.
-
- 7. Additional Terms.
-
- "Additional permissions" are terms that supplement the terms of this
-License by making exceptions from one or more of its conditions.
-Additional permissions that are applicable to the entire Program shall
-be treated as though they were included in this License, to the extent
-that they are valid under applicable law. If additional permissions
-apply only to part of the Program, that part may be used separately
-under those permissions, but the entire Program remains governed by
-this License without regard to the additional permissions.
-
- When you convey a copy of a covered work, you may at your option
-remove any additional permissions from that copy, or from any part of
-it. (Additional permissions may be written to require their own
-removal in certain cases when you modify the work.) You may place
-additional permissions on material, added by you to a covered work,
-for which you have or can give appropriate copyright permission.
-
- Notwithstanding any other provision of this License, for material you
-add to a covered work, you may (if authorized by the copyright holders of
-that material) supplement the terms of this License with terms:
-
- a) Disclaiming warranty or limiting liability differently from the
- terms of sections 15 and 16 of this License; or
-
- b) Requiring preservation of specified reasonable legal notices or
- author attributions in that material or in the Appropriate Legal
- Notices displayed by works containing it; or
-
- c) Prohibiting misrepresentation of the origin of that material, or
- requiring that modified versions of such material be marked in
- reasonable ways as different from the original version; or
-
- d) Limiting the use for publicity purposes of names of licensors or
- authors of the material; or
-
- e) Declining to grant rights under trademark law for use of some
- trade names, trademarks, or service marks; or
-
- f) Requiring indemnification of licensors and authors of that
- material by anyone who conveys the material (or modified versions of
- it) with contractual assumptions of liability to the recipient, for
- any liability that these contractual assumptions directly impose on
- those licensors and authors.
-
- All other non-permissive additional terms are considered "further
-restrictions" within the meaning of section 10. If the Program as you
-received it, or any part of it, contains a notice stating that it is
-governed by this License along with a term that is a further
-restriction, you may remove that term. If a license document contains
-a further restriction but permits relicensing or conveying under this
-License, you may add to a covered work material governed by the terms
-of that license document, provided that the further restriction does
-not survive such relicensing or conveying.
-
- If you add terms to a covered work in accord with this section, you
-must place, in the relevant source files, a statement of the
-additional terms that apply to those files, or a notice indicating
-where to find the applicable terms.
-
- Additional terms, permissive or non-permissive, may be stated in the
-form of a separately written license, or stated as exceptions;
-the above requirements apply either way.
-
- 8. Termination.
-
- You may not propagate or modify a covered work except as expressly
-provided under this License. Any attempt otherwise to propagate or
-modify it is void, and will automatically terminate your rights under
-this License (including any patent licenses granted under the third
-paragraph of section 11).
-
- However, if you cease all violation of this License, then your
-license from a particular copyright holder is reinstated (a)
-provisionally, unless and until the copyright holder explicitly and
-finally terminates your license, and (b) permanently, if the copyright
-holder fails to notify you of the violation by some reasonable means
-prior to 60 days after the cessation.
-
- Moreover, your license from a particular copyright holder is
-reinstated permanently if the copyright holder notifies you of the
-violation by some reasonable means, this is the first time you have
-received notice of violation of this License (for any work) from that
-copyright holder, and you cure the violation prior to 30 days after
-your receipt of the notice.
-
- Termination of your rights under this section does not terminate the
-licenses of parties who have received copies or rights from you under
-this License. If your rights have been terminated and not permanently
-reinstated, you do not qualify to receive new licenses for the same
-material under section 10.
-
- 9. Acceptance Not Required for Having Copies.
-
- You are not required to accept this License in order to receive or
-run a copy of the Program. Ancillary propagation of a covered work
-occurring solely as a consequence of using peer-to-peer transmission
-to receive a copy likewise does not require acceptance. However,
-nothing other than this License grants you permission to propagate or
-modify any covered work. These actions infringe copyright if you do
-not accept this License. Therefore, by modifying or propagating a
-covered work, you indicate your acceptance of this License to do so.
-
- 10. Automatic Licensing of Downstream Recipients.
-
- Each time you convey a covered work, the recipient automatically
-receives a license from the original licensors, to run, modify and
-propagate that work, subject to this License. You are not responsible
-for enforcing compliance by third parties with this License.
-
- An "entity transaction" is a transaction transferring control of an
-organization, or substantially all assets of one, or subdividing an
-organization, or merging organizations. If propagation of a covered
-work results from an entity transaction, each party to that
-transaction who receives a copy of the work also receives whatever
-licenses to the work the party's predecessor in interest had or could
-give under the previous paragraph, plus a right to possession of the
-Corresponding Source of the work from the predecessor in interest, if
-the predecessor has it or can get it with reasonable efforts.
-
- You may not impose any further restrictions on the exercise of the
-rights granted or affirmed under this License. For example, you may
-not impose a license fee, royalty, or other charge for exercise of
-rights granted under this License, and you may not initiate litigation
-(including a cross-claim or counterclaim in a lawsuit) alleging that
-any patent claim is infringed by making, using, selling, offering for
-sale, or importing the Program or any portion of it.
-
- 11. Patents.
-
- A "contributor" is a copyright holder who authorizes use under this
-License of the Program or a work on which the Program is based. The
-work thus licensed is called the contributor's "contributor version".
-
- A contributor's "essential patent claims" are all patent claims
-owned or controlled by the contributor, whether already acquired or
-hereafter acquired, that would be infringed by some manner, permitted
-by this License, of making, using, or selling its contributor version,
-but do not include claims that would be infringed only as a
-consequence of further modification of the contributor version. For
-purposes of this definition, "control" includes the right to grant
-patent sublicenses in a manner consistent with the requirements of
-this License.
-
- Each contributor grants you a non-exclusive, worldwide, royalty-free
-patent license under the contributor's essential patent claims, to
-make, use, sell, offer for sale, import and otherwise run, modify and
-propagate the contents of its contributor version.
-
- In the following three paragraphs, a "patent license" is any express
-agreement or commitment, however denominated, not to enforce a patent
-(such as an express permission to practice a patent or covenant not to
-sue for patent infringement). To "grant" such a patent license to a
-party means to make such an agreement or commitment not to enforce a
-patent against the party.
-
- If you convey a covered work, knowingly relying on a patent license,
-and the Corresponding Source of the work is not available for anyone
-to copy, free of charge and under the terms of this License, through a
-publicly available network server or other readily accessible means,
-then you must either (1) cause the Corresponding Source to be so
-available, or (2) arrange to deprive yourself of the benefit of the
-patent license for this particular work, or (3) arrange, in a manner
-consistent with the requirements of this License, to extend the patent
-license to downstream recipients. "Knowingly relying" means you have
-actual knowledge that, but for the patent license, your conveying the
-covered work in a country, or your recipient's use of the covered work
-in a country, would infringe one or more identifiable patents in that
-country that you have reason to believe are valid.
-
- If, pursuant to or in connection with a single transaction or
-arrangement, you convey, or propagate by procuring conveyance of, a
-covered work, and grant a patent license to some of the parties
-receiving the covered work authorizing them to use, propagate, modify
-or convey a specific copy of the covered work, then the patent license
-you grant is automatically extended to all recipients of the covered
-work and works based on it.
-
- A patent license is "discriminatory" if it does not include within
-the scope of its coverage, prohibits the exercise of, or is
-conditioned on the non-exercise of one or more of the rights that are
-specifically granted under this License. You may not convey a covered
-work if you are a party to an arrangement with a third party that is
-in the business of distributing software, under which you make payment
-to the third party based on the extent of your activity of conveying
-the work, and under which the third party grants, to any of the
-parties who would receive the covered work from you, a discriminatory
-patent license (a) in connection with copies of the covered work
-conveyed by you (or copies made from those copies), or (b) primarily
-for and in connection with specific products or compilations that
-contain the covered work, unless you entered into that arrangement,
-or that patent license was granted, prior to 28 March 2007.
-
- Nothing in this License shall be construed as excluding or limiting
-any implied license or other defenses to infringement that may
-otherwise be available to you under applicable patent law.
-
- 12. No Surrender of Others' Freedom.
-
- If conditions are imposed on you (whether by court order, agreement or
-otherwise) that contradict the conditions of this License, they do not
-excuse you from the conditions of this License. If you cannot convey a
-covered work so as to satisfy simultaneously your obligations under this
-License and any other pertinent obligations, then as a consequence you may
-not convey it at all. For example, if you agree to terms that obligate you
-to collect a royalty for further conveying from those to whom you convey
-the Program, the only way you could satisfy both those terms and this
-License would be to refrain entirely from conveying the Program.
-
- 13. Remote Network Interaction; Use with the GNU General Public License.
-
- Notwithstanding any other provision of this License, if you modify the
-Program, your modified version must prominently offer all users
-interacting with it remotely through a computer network (if your version
-supports such interaction) an opportunity to receive the Corresponding
-Source of your version by providing access to the Corresponding Source
-from a network server at no charge, through some standard or customary
-means of facilitating copying of software. This Corresponding Source
-shall include the Corresponding Source for any work covered by version 3
-of the GNU General Public License that is incorporated pursuant to the
-following paragraph.
-
- Notwithstanding any other provision of this License, you have
-permission to link or combine any covered work with a work licensed
-under version 3 of the GNU General Public License into a single
-combined work, and to convey the resulting work. The terms of this
-License will continue to apply to the part which is the covered work,
-but the work with which it is combined will remain governed by version
-3 of the GNU General Public License.
-
- 14. Revised Versions of this License.
-
- The Free Software Foundation may publish revised and/or new versions of
-the GNU Affero General Public License from time to time. Such new versions
-will be similar in spirit to the present version, but may differ in detail to
-address new problems or concerns.
-
- Each version is given a distinguishing version number. If the
-Program specifies that a certain numbered version of the GNU Affero General
-Public License "or any later version" applies to it, you have the
-option of following the terms and conditions either of that numbered
-version or of any later version published by the Free Software
-Foundation. If the Program does not specify a version number of the
-GNU Affero General Public License, you may choose any version ever published
-by the Free Software Foundation.
-
- If the Program specifies that a proxy can decide which future
-versions of the GNU Affero General Public License can be used, that proxy's
-public statement of acceptance of a version permanently authorizes you
-to choose that version for the Program.
-
- Later license versions may give you additional or different
-permissions. However, no additional obligations are imposed on any
-author or copyright holder as a result of your choosing to follow a
-later version.
-
- 15. Disclaimer of Warranty.
-
- THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
-APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
-HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
-OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
-THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
-IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
-ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
-
- 16. Limitation of Liability.
-
- IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
-WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
-THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
-GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
-USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
-DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
-PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
-EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
-SUCH DAMAGES.
-
- 17. Interpretation of Sections 15 and 16.
-
- If the disclaimer of warranty and limitation of liability provided
-above cannot be given local legal effect according to their terms,
-reviewing courts shall apply local law that most closely approximates
-an absolute waiver of all civil liability in connection with the
-Program, unless a warranty or assumption of liability accompanies a
-copy of the Program in return for a fee.
-
- END OF TERMS AND CONDITIONS
-
- How to Apply These Terms to Your New Programs
-
- If you develop a new program, and you want it to be of the greatest
-possible use to the public, the best way to achieve this is to make it
-free software which everyone can redistribute and change under these terms.
-
- To do so, attach the following notices to the program. It is safest
-to attach them to the start of each source file to most effectively
-state the exclusion of warranty; and each file should have at least
-the "copyright" line and a pointer to where the full notice is found.
-
-
- Copyright (C)
-
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU Affero General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU Affero General Public License for more details.
-
- You should have received a copy of the GNU Affero General Public License
- along with this program. If not, see .
-
-Also add information on how to contact you by electronic and paper mail.
-
- If your software can interact with users remotely through a computer
-network, you should also make sure that it provides a way for users to
-get its source. For example, if your program is a web application, its
-interface could display a "Source" link that leads users to an archive
-of the code. There are many ways you could offer source, and different
-solutions will be better for different programs; see section 13 for the
-specific requirements.
-
- You should also get your employer (if you work as a programmer) or school,
-if any, to sign a "copyright disclaimer" for the program, if necessary.
-For more information on this, and how to apply and follow the GNU AGPL, see
-.
diff --git a/enterprise/enterprise.go b/enterprise/enterprise.go
deleted file mode 100644
index f3a897551..000000000
--- a/enterprise/enterprise.go
+++ /dev/null
@@ -1,23 +0,0 @@
-// Copyright (c) 2018 Pilosa Corp. All rights reserved.
-//
-// This file is part of Pilosa Enterprise Edition.
-//
-// Pilosa Enterprise Edition is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// Pilosa Enterprise Edition is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with Pilosa Enterprise Edition. If not, see .
-
-// Package enterprise is now deprecated, and the functionality under
-// enterprise/b has been copied into roaring/. It existed to inject enterprise
-// implementations of various Pilosa features when Pilosa was built with
-// "ENTERPRISE=1 make install". These features were dual-licensed separately
-// from Pilosa community edition under the AGPL and Pilosa's commercial license.
-package enterprise
diff --git a/executor.go b/executor.go
index 6891072ee..25f5c074e 100644
--- a/executor.go
+++ b/executor.go
@@ -381,6 +381,16 @@ func (e *executor) handlePreCalls(ctx context.Context, index string, c *pql.Call
return nil
}
+// dumpPrecomputedCalls throws away precomputed call data. this is used so we
+// can drop any large data associated with a call once we've processed
+// the call.
+func (e *executor) dumpPrecomputedCalls(ctx context.Context, c *pql.Call) {
+ for _, call := range c.Children {
+ e.dumpPrecomputedCalls(ctx, call)
+ }
+ c.Precomputed = nil
+}
+
// handlePreCallChildren handles any pre-calls in the children of a given call.
func (e *executor) handlePreCallChildren(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) error {
for i := range c.Children {
@@ -433,7 +443,7 @@ func (e *executor) execute(ctx context.Context, index string, q *pql.Query, shar
// Execute each call serially.
results := make([]interface{}, 0, len(q.Calls))
- for _, call := range q.Calls {
+ for i, call := range q.Calls {
if err := validateQueryContext(ctx); err != nil {
return nil, err
}
@@ -463,6 +473,11 @@ func (e *executor) execute(ctx context.Context, index string, q *pql.Query, shar
return nil, err
}
results = append(results, v)
+ // Some Calls can have significant data associated with them
+ // that gets generated during processing, such as Precomputed
+ // values. Dumping the precomputed data, if any, lets the GC
+ // free the memory before we get there.
+ e.dumpPrecomputedCalls(ctx, q.Calls[i])
}
return results, nil
}
@@ -684,7 +699,12 @@ func (e *executor) executeIncludesColumnCall(ctx context.Context, index string,
func (e *executor) executeFieldValueCall(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (ValCount, error) {
fieldName, ok := c.Args["field"].(string)
if !ok || fieldName == "" {
- return ValCount{}, errors.New("FieldValue(): field required")
+ return ValCount{}, ErrFieldRequired
+ }
+
+ colKey, ok := c.Args["column"]
+ if !ok || colKey == "" {
+ return ValCount{}, ErrColumnRequired
}
// Fetch index.
@@ -700,8 +720,8 @@ func (e *executor) executeFieldValueCall(ctx context.Context, index string, c *p
}
var colID uint64
- if colKey, ok := c.Args["column"].(string); ok && idx.Keys() {
- id, err := e.Cluster.translateIndexKey(ctx, index, colKey)
+ if key, ok := colKey.(string); ok && idx.Keys() {
+ id, err := e.Cluster.translateIndexKey(ctx, index, key)
if err != nil {
return ValCount{}, errors.Wrap(err, "getting column id")
}
@@ -709,7 +729,6 @@ func (e *executor) executeFieldValueCall(ctx context.Context, index string, c *p
} else {
id, ok, err := c.UintArg("column")
if !ok || err != nil {
- // TODO: this error is getting swallowed somewhere (via curl)
return ValCount{}, errors.Wrap(err, "getting column argument")
}
colID = id
@@ -4026,20 +4045,20 @@ func (e *executor) translateCall(ctx context.Context, indexName string, c *pql.C
// are only two possible values. Instead, they are handled
// directly.
if field.Type() == FieldTypeBool {
- // TODO: This code block doesn't make sense for a `Rows()`
- // queries on a `bool` field. Need to review this better,
- // include it in tests, and probably back-port it to Pilosa.
- if c.Name != "Rows" {
- boolVal, err := callArgBool(c, rowKey)
- if err != nil {
- return errors.Wrap(err, "getting bool key")
- }
- rowID := falseRowID
- if boolVal {
- rowID = trueRowID
- }
- c.Args[rowKey] = rowID
+ if c.Name == "Rows" {
+ // TranslateInfo for Rows returns "previous" as rowKey,
+ // so for bool fields we would get "missing bool argument" error
+ return nil
}
+ boolVal, err := callArgBool(c, rowKey)
+ if err != nil {
+ return errors.Wrapf(err, "getting bool key (%+v)", rowKey)
+ }
+ rowID := falseRowID
+ if boolVal {
+ rowID = trueRowID
+ }
+ c.Args[rowKey] = rowID
} else if field.Keys() {
foreignIndexName := field.ForeignIndex()
if c.Args[rowKey] != nil && isCondition(c.Args[rowKey]) {
@@ -4467,25 +4486,37 @@ func (s SignedRow) ToTable() (*pb.TableResponse, error) {
// ToRows implements the ToRowser interface.
func (s SignedRow) ToRows(callback func(*pb.RowResponse) error) error {
- // TODO: address the overflow issue with values outside the int64 range
+
ci := []*pb.ColumnInfo{{Name: s.Field(), Datatype: "int64"}}
negs := s.Neg.Columns()
for i := len(negs) - 1; i >= 0; i-- {
+ val, err := toNegInt64(negs[i])
+ if err != nil {
+ return errors.Wrap(err, "converting uint64 to int64 (negative)")
+ }
+
if err := callback(&pb.RowResponse{
Headers: ci,
Columns: []*pb.ColumnResponse{
- &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Int64Val{Int64Val: -1 * int64(negs[i])}},
- }}); err != nil {
+ &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Int64Val{Int64Val: val}},
+ },
+ }); err != nil {
return errors.Wrap(err, "calling callback")
}
ci = nil
}
for _, id := range s.Pos.Columns() {
+ val, err := toInt64(id)
+ if err != nil {
+ return errors.Wrap(err, "converting uint64 to int64 (positive)")
+ }
+
if err := callback(&pb.RowResponse{
Headers: ci,
Columns: []*pb.ColumnResponse{
- &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Int64Val{Int64Val: int64(id)}},
- }}); err != nil {
+ &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Int64Val{Int64Val: val}},
+ },
+ }); err != nil {
return errors.Wrap(err, "calling callback")
}
ci = nil
@@ -4493,6 +4524,31 @@ func (s SignedRow) ToRows(callback func(*pb.RowResponse) error) error {
return nil
}
+func toNegInt64(n uint64) (int64, error) {
+ const absMinInt64 = uint64(1 << 63)
+
+ if n > absMinInt64 {
+ return 0, errors.Errorf("value %d overflows int64", n)
+ }
+
+ if n == absMinInt64 {
+ return int64(-1 << 63), nil
+ }
+
+ // n < 1 << 63
+ return -int64(n), nil
+}
+
+func toInt64(n uint64) (int64, error) {
+ const maxInt64 = uint64(1<<63) - 1
+
+ if n > maxInt64 {
+ return 0, errors.Errorf("value %d overflows int64", n)
+ }
+
+ return int64(n), nil
+}
+
func (sr *SignedRow) union(other SignedRow) SignedRow {
ret := SignedRow{&Row{}, &Row{}, ""}
diff --git a/executor_internal_test.go b/executor_internal_test.go
index 86ff6cdde..909f48f7a 100644
--- a/executor_internal_test.go
+++ b/executor_internal_test.go
@@ -133,6 +133,62 @@ func TestExecutor_TranslateGroupByCall(t *testing.T) {
}
}
+func TestExecutor_TranslateRowsOnBool(t *testing.T) {
+ holder := NewHolder(DefaultPartitionN)
+ defer holder.Close()
+
+ e := &executor{
+ Holder: holder,
+ Cluster: NewTestCluster(1),
+ }
+ e.Holder.Path, _ = ioutil.TempDir(*TempDir, "")
+ err := e.Holder.Open()
+ if err != nil {
+ t.Fatalf("opening holder: %v", err)
+ }
+
+ idx, err := e.Holder.CreateIndex("i", IndexOptions{})
+ if err != nil {
+ t.Fatalf("creating index: %v", err)
+ }
+
+ fb, errb := idx.CreateField("b", OptFieldTypeBool())
+ _, errbk := idx.CreateField("bk", OptFieldTypeBool(), OptFieldKeys())
+ if errb != nil || errbk != nil {
+ t.Fatalf("creating fields %v, %v", errb, errbk)
+ }
+
+ _, err1 := fb.SetBit(1, 1, nil)
+ _, err2 := fb.SetBit(2, 2, nil)
+ _, err3 := fb.SetBit(3, 3, nil)
+ if err1 != nil || err2 != nil || err3 != nil {
+ t.Fatalf("setting bit %v, %v, %v", err1, err2, err3)
+ }
+
+ tests := []struct {
+ pql string
+ }{
+ {pql: "Rows(b)"},
+ {pql: "GroupBy(Rows(b))"},
+ {pql: "Set(4, b=true)"},
+ }
+
+ for _, test := range tests {
+ t.Run(test.pql, func(t *testing.T) {
+ query, err := pql.ParseString(test.pql)
+ if err != nil {
+ t.Fatalf("parsing query: %v", err)
+ }
+
+ c := query.Calls[0]
+ err = e.translateCall(context.Background(), "i", c, make(map[string]map[string]uint64))
+ if err != nil {
+ t.Fatalf("translating call: %v", err)
+ }
+ })
+ }
+}
+
func isInt(a interface{}) bool {
switch a.(type) {
case int, int64, uint, uint64:
@@ -439,3 +495,71 @@ func TestValCountComparisons(t *testing.T) {
})
}
}
+
+func TestToNegInt64(t *testing.T) {
+ tests := []struct {
+ u64 uint64
+ i64 int64
+ overflow bool
+ }{
+ {
+ u64: uint64(1 << 63),
+ i64: int64(-1 << 63),
+ },
+ {
+ u64: uint64(1<<63) - 1,
+ i64: int64(-1<<63) + 1,
+ },
+ {
+ u64: uint64(1<<63) + 1,
+ overflow: true,
+ },
+ }
+
+ for _, tc := range tests {
+ val, err := toNegInt64(tc.u64)
+ if err != nil && !tc.overflow {
+ t.Fatalf("error: %+v, expected: %+v", err, tc)
+ }
+
+ if val != tc.i64 {
+ t.Fatalf("Expected: %+v, Got: %+v", tc.i64, val)
+ }
+ }
+}
+
+func TestToInt64(t *testing.T) {
+ tests := []struct {
+ u64 uint64
+ i64 int64
+ overflow bool
+ }{
+ {
+ u64: uint64(1<<63) - 1,
+ i64: 1<<63 - 1,
+ },
+ {
+ u64: uint64(0),
+ i64: 0,
+ },
+ {
+ u64: uint64(1 << 63),
+ overflow: true,
+ },
+ {
+ u64: 1<<64 - 1,
+ overflow: true,
+ },
+ }
+
+ for _, tc := range tests {
+ val, err := toInt64(tc.u64)
+ if err != nil && !tc.overflow {
+ t.Fatalf("error: %+v, expected: %+v", err, tc)
+ }
+
+ if val != tc.i64 {
+ t.Fatalf("Expected: %+v, Got: %+v", tc.i64, val)
+ }
+ }
+}
diff --git a/executor_test.go b/executor_test.go
index bc990dff9..0bbf3bde8 100644
--- a/executor_test.go
+++ b/executor_test.go
@@ -3522,7 +3522,6 @@ func TestExecutor_Execute_Not(t *testing.T) {
func TestExecutor_Execute_FieldValue(t *testing.T) {
c := test.MustRunCluster(t, 2)
defer c.Close()
- //hldr := test.Holder{Holder: c[0].Server.Holder()}
node0 := c[0]
node1 := c[1]
@@ -3535,8 +3534,8 @@ func TestExecutor_Execute_FieldValue(t *testing.T) {
Set(1, f=3)
Set(2, f=-4)
Set(` + strconv.Itoa(ShardWidth+1) + `, f=3)
- Set(1, dec=12.985)
- Set(2, dec=-4.234)
+ Set(1, dec=12.985)
+ Set(2, dec=-4.234)
`}); err != nil {
t.Fatal(err)
}
@@ -3548,8 +3547,8 @@ func TestExecutor_Execute_FieldValue(t *testing.T) {
if _, err := node0.API.Query(context.Background(), &pilosa.QueryRequest{Index: "ik", Query: `
Set("one", f=3)
Set("two", f=-4)
- Set("one", dec=12.985)
- Set("two", dec=-4.234)
+ Set("one", dec=12.985)
+ Set("two", dec=-4.234)
`}); err != nil {
t.Fatal(err)
}
@@ -3577,6 +3576,8 @@ func TestExecutor_Execute_FieldValue(t *testing.T) {
// Errors
{index: "i", qry: "FieldValue()", expErr: pilosa.ErrFieldRequired.Error()},
+ {index: "i", qry: "FieldValue(field=dec)", expErr: pilosa.ErrColumnRequired.Error()},
+ {index: "ik", qry: "FieldValue(field=f)", expErr: pilosa.ErrColumnRequired.Error()},
}
for n, node := range []*test.Command{node0, node1} {
for i, test := range tests {
diff --git a/field.go b/field.go
index b746e1800..f82f03543 100644
--- a/field.go
+++ b/field.go
@@ -31,7 +31,6 @@ import (
"github.com/gogo/protobuf/proto"
"github.com/pilosa/pilosa/v2/internal"
- "github.com/pilosa/pilosa/v2/logger"
"github.com/pilosa/pilosa/v2/pql"
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pilosa/pilosa/v2/stats"
@@ -117,10 +116,6 @@ type Field struct {
// Shards with data on any node in the cluster, according to this node.
remoteAvailableShards *roaring.Bitmap
- logger logger.Logger
-
- snapshotQueue snapshotQueue
-
translateStore TranslateStore
// Instantiates new translation stores
@@ -338,17 +333,17 @@ func OptFieldTypeBool() FieldOption {
// that it's of the type `OptFieldType*`). This means
// this function couldn't be used to set, for example,
// `FieldOptions.Keys`.
-func NewField(path, index, name string, opts FieldOption) (*Field, error) {
+func NewField(holder *Holder, path, index, name string, opts FieldOption) (*Field, error) {
err := validateName(name)
if err != nil {
return nil, errors.Wrap(err, "validating name")
}
- return newField(path, index, name, opts)
+ return newField(holder, path, index, name, opts)
}
// newField returns a new instance of field (without name validation).
-func newField(path, index, name string, opts FieldOption) (*Field, error) {
+func newField(holder *Holder, path, index, name string, opts FieldOption) (*Field, error) {
// Apply functional option.
fo := FieldOptions{}
err := opts(&fo)
@@ -372,7 +367,7 @@ func newField(path, index, name string, opts FieldOption) (*Field, error) {
remoteAvailableShards: roaring.NewBitmap(),
- logger: logger.NopLogger,
+ holder: holder,
OpenTranslateStore: OpenInMemTranslateStore,
}
@@ -448,7 +443,7 @@ func (f *Field) loadAvailableShards() error {
}
// some other problem:
if err != nil {
- f.logger.Printf("available shards file present but unreadable, discarding: %v", err)
+ f.holder.Logger.Printf("available shards file present but unreadable, discarding: %v", err)
err = os.Remove(path)
if err != nil {
return errors.Wrap(err, "deleting corrupt available shards list")
@@ -457,7 +452,7 @@ func (f *Field) loadAvailableShards() error {
}
bm := roaring.NewBitmap()
if err = bm.UnmarshalBinary(buf); err != nil {
- f.logger.Printf("available shards file corrupt, discarding: %v", err)
+ f.holder.Logger.Printf("available shards file corrupt, discarding: %v", err)
err = os.Remove(path)
if err != nil {
return errors.Wrap(err, "deleting corrupt available shards list")
@@ -548,17 +543,17 @@ func (f *Field) Options() FieldOptions {
func (f *Field) Open() error {
if err := func() (err error) {
// Ensure the field's path exists.
- f.logger.Debugf("ensure field path exists: %s", f.path)
+ f.holder.Logger.Debugf("ensure field path exists: %s", f.path)
if err := os.MkdirAll(f.path, 0777); err != nil {
return errors.Wrap(err, "creating field dir")
}
- f.logger.Debugf("load meta file for index/field: %s/%s", f.index, f.name)
+ f.holder.Logger.Debugf("load meta file for index/field: %s/%s", f.index, f.name)
if err := f.loadMeta(); err != nil {
return errors.Wrap(err, "loading meta")
}
- f.logger.Debugf("load available shards for index/field: %s/%s", f.index, f.name)
+ f.holder.Logger.Debugf("load available shards for index/field: %s/%s", f.index, f.name)
if err := f.loadAvailableShards(); err != nil {
return errors.Wrap(err, "loading available shards")
}
@@ -570,17 +565,17 @@ func (f *Field) Open() error {
}
// Apply the field options loaded from meta (or set via setOptions()).
- f.logger.Debugf("apply options for index/field: %s/%s", f.index, f.name)
+ f.holder.Logger.Debugf("apply options for index/field: %s/%s", f.index, f.name)
if err := f.applyOptions(f.options); err != nil {
return errors.Wrap(err, "applying options")
}
- f.logger.Debugf("open views for index/field: %s/%s", f.index, f.name)
+ f.holder.Logger.Debugf("open views for index/field: %s/%s", f.index, f.name)
if err := f.openViews(); err != nil {
return errors.Wrap(err, "opening views")
}
- f.logger.Debugf("open row attribute store for index/field: %s/%s", f.index, f.name)
+ f.holder.Logger.Debugf("open row attribute store for index/field: %s/%s", f.index, f.name)
if err := f.rowAttrStore.Open(); err != nil {
return errors.Wrap(err, "opening attrstore")
}
@@ -607,7 +602,7 @@ func (f *Field) Open() error {
return err
}
- f.logger.Debugf("successfully opened field index/field: %s/%s", f.index, f.name)
+ f.holder.Logger.Debugf("successfully opened field index/field: %s/%s", f.index, f.name)
return nil
}
func blockingWriteAvailableShards(fieldPath string, availableShardBytes []byte) {
@@ -748,7 +743,7 @@ fileLoop:
<-fieldQueue
}()
name := filepath.Base(fi.Name())
- f.logger.Debugf("open index/field/view: %s/%s/%s", f.index, f.name, fi.Name())
+ f.holder.Logger.Debugf("open index/field/view: %s/%s/%s", f.index, f.name, fi.Name())
view := f.newView(f.viewPath(name), name)
if err := view.open(); err != nil {
return fmt.Errorf("opening view: view=%s, err=%s", view.name, err)
@@ -770,7 +765,7 @@ fileLoop:
}
view.rowAttrStore = f.rowAttrStore
- f.logger.Debugf("add index/field/view to field.viewMap: %s/%s/%s", f.index, f.name, view.name)
+ f.holder.Logger.Debugf("add index/field/view to field.viewMap: %s/%s/%s", f.index, f.name, view.name)
mu.Lock()
f.viewMap[view.name] = view
mu.Unlock()
@@ -1183,14 +1178,10 @@ func (f *Field) createViewIfNotExistsBase(name string) (*view, bool, error) {
}
func (f *Field) newView(path, name string) *view {
- view := newView(path, f.index, f.name, name, f.options)
- view.logger = f.logger
+ view := newView(f.holder, path, f.index, f.name, name, f.options)
view.rowAttrStore = f.rowAttrStore
view.stats = f.Stats
view.broadcaster = f.broadcaster
- if f.snapshotQueue != nil {
- view.snapshotQueue = f.snapshotQueue
- }
return view
}
diff --git a/field_internal_test.go b/field_internal_test.go
index db4a3b5e7..296e54be0 100644
--- a/field_internal_test.go
+++ b/field_internal_test.go
@@ -205,7 +205,7 @@ func NewTestField(t *testing.T, opts FieldOption) *TestField {
if err != nil {
t.Fatal(err)
}
- field, err := NewField(path, "i", "f", opts)
+ field, err := NewField(NewHolder(DefaultPartitionN), path, "i", "f", opts)
if err != nil {
t.Fatal(err)
}
@@ -235,7 +235,7 @@ func (f *TestField) Reopen() error {
}
path, index, name := f.Path(), f.Index(), f.Name()
- f.Field, err = NewField(path, index, name, OptFieldTypeDefault())
+ f.Field, err = NewField(NewHolder(DefaultPartitionN), path, index, name, OptFieldTypeDefault())
if err != nil {
return err
}
@@ -730,7 +730,7 @@ func TestDecimalField_MinMaxBoundaries(t *testing.T) {
},
} {
t.Run("minmax"+strconv.Itoa(i), func(t *testing.T) {
- _, err := NewField("no-path", "i", "f", OptFieldTypeDecimal(test.scale, test.min, test.max))
+ _, err := NewField(NewHolder(DefaultPartitionN), "no-path", "i", "f", OptFieldTypeDecimal(test.scale, test.min, test.max))
if err != nil && test.expErr {
if !strings.Contains(err.Error(), "is not supported") {
t.Fatal(err)
diff --git a/field_test.go b/field_test.go
index 40b64728a..ab24b466f 100644
--- a/field_test.go
+++ b/field_test.go
@@ -144,7 +144,7 @@ func TestField_NameRestriction(t *testing.T) {
if err != nil {
panic(err)
}
- field, err := pilosa.NewField(path, "i", ".meta", pilosa.OptFieldTypeDefault())
+ field, err := pilosa.NewField(pilosa.NewHolder(pilosa.DefaultPartitionN), path, "i", ".meta", pilosa.OptFieldTypeDefault())
if field != nil {
t.Fatalf("unexpected field name %s", err)
}
@@ -177,13 +177,13 @@ func TestField_NameValidation(t *testing.T) {
panic(err)
}
for _, name := range validFieldNames {
- _, err := pilosa.NewField(path, "i", name, pilosa.OptFieldTypeDefault())
+ _, err := pilosa.NewField(pilosa.NewHolder(pilosa.DefaultPartitionN), path, "i", name, pilosa.OptFieldTypeDefault())
if err != nil {
t.Fatalf("unexpected field name: %s %s", name, err)
}
}
for _, name := range invalidFieldNames {
- _, err := pilosa.NewField(path, "i", name, pilosa.OptFieldTypeDefault())
+ _, err := pilosa.NewField(pilosa.NewHolder(pilosa.DefaultPartitionN), path, "i", name, pilosa.OptFieldTypeDefault())
if err == nil {
t.Fatalf("expected error on field name: %s", name)
}
diff --git a/fragment.go b/fragment.go
index 3258d6f00..61ac78313 100644
--- a/fragment.go
+++ b/fragment.go
@@ -106,6 +106,10 @@ type fragment struct {
field string
view string
shard uint64
+
+ // parent holder, used to find snapshot queue, etc.
+ holder *Holder
+
// debugging tool: addresses of current and previous maps
prevdata, currdata struct{ from, to uintptr }
@@ -120,6 +124,7 @@ type fragment struct {
snapshotCond sync.Cond
snapshotErr error // error yielded by the last snapshot operation
snapshotStamp time.Time // timestamp of last snapshot
+ open bool // is this fragment actually open?
// Cache for row counts.
CacheType string // passed in by field
@@ -153,11 +158,11 @@ type fragment struct {
stats stats.StatsClient
- snapshotQueue snapshotQueue
+ bitmapInfo *roaring.BitmapInfo
}
// newFragment returns a new instance of Fragment.
-func newFragment(path, index, field, view string, shard uint64, flags byte) *fragment {
+func newFragment(holder *Holder, path, index, field, view string, shard uint64, flags byte) *fragment {
f := &fragment{
path: path,
index: index,
@@ -168,11 +173,10 @@ func newFragment(path, index, field, view string, shard uint64, flags byte) *fra
CacheType: DefaultCacheType,
CacheSize: DefaultCacheSize,
- Logger: logger.NopLogger,
+ holder: holder,
MaxOpN: defaultFragmentMaxOpN,
- stats: stats.NopStatsClient,
- snapshotQueue: defaultSnapshotQueue,
+ stats: stats.NopStatsClient,
}
f.snapshotCond = sync.Cond{L: &f.mu}
return f
@@ -181,6 +185,23 @@ func newFragment(path, index, field, view string, shard uint64, flags byte) *fra
// cachePath returns the path to the fragment's cache data.
func (f *fragment) cachePath() string { return f.path + cacheExt }
+type FragmentInfo struct {
+ BitmapInfo roaring.BitmapInfo
+ BlockChecksums []FragmentBlock `json:"BlockChecksums,omitempty"`
+}
+
+func (f *fragment) inspect(params InspectRequestParams) (fi FragmentInfo) {
+ if f.bitmapInfo == nil {
+ fi.BitmapInfo = f.storage.Info(params.Containers)
+ } else {
+ fi.BitmapInfo = *f.bitmapInfo
+ }
+ if params.Checksum {
+ fi.BlockChecksums = f.Blocks()
+ }
+ return fi
+}
+
// Open opens the underlying storage.
func (f *fragment) Open() error {
f.mu.Lock()
@@ -188,13 +209,13 @@ func (f *fragment) Open() error {
if err := func() error {
// Initialize storage in a function so we can close if anything goes wrong.
- f.Logger.Debugf("open storage for index/field/view/fragment: %s/%s/%s/%d", f.index, f.field, f.view, f.shard)
+ f.holder.Logger.Debugf("open storage for index/field/view/fragment: %s/%s/%s/%d", f.index, f.field, f.view, f.shard)
if err := f.openStorage(true); err != nil {
return errors.Wrap(err, "opening storage")
}
// Fill cache with rows persisted to disk.
- f.Logger.Debugf("open cache for index/field/view/fragment: %s/%s/%s/%d", f.index, f.field, f.view, f.shard)
+ f.holder.Logger.Debugf("open cache for index/field/view/fragment: %s/%s/%s/%d", f.index, f.field, f.view, f.shard)
if err := f.openCache(); err != nil {
e2 := f.closeStorage()
if e2 != nil {
@@ -213,8 +234,9 @@ func (f *fragment) Open() error {
f.close()
return err
}
+ f.open = true
- f.Logger.Debugf("successfully opened index/field/view/fragment: %s/%s/%s/%d", f.index, f.field, f.view, f.shard)
+ f.holder.Logger.Debugf("successfully opened index/field/view/fragment: %s/%s/%s/%d", f.index, f.field, f.view, f.shard)
return nil
}
@@ -222,6 +244,9 @@ func (f *fragment) Open() error {
// get no data. It tries to write the current storage to the provided file,
// which is assumed to be the file they didn't get any data from.
func (f *fragment) emptyStorage(file *os.File) (bool, error) {
+ if f.holder.Opts.ReadOnly {
+ return false, errors.New("can't flush/create storage for read-only holder")
+ }
// No data. We'll mark this for no mapping, clear any existing
// mapped containers, and set the Source to nil. We also have no
// ops.
@@ -273,9 +298,12 @@ func (f *fragment) importStorage(data []byte, file *os.File, newGen generation,
}
return false, fmt.Errorf("unmarshal storage: file=%s, err=%s", file.Name(), err)
}
- f.Logger.Printf("warning: unmarshal storage, file=%s, err=%v", file.Name(), err)
+ f.holder.Logger.Printf("warning: unmarshal storage, file=%s, err=%v", file.Name(), err)
trunc, ok := cause.(roaring.FileShouldBeTruncatedError)
- if ok {
+ if ok && !f.holder.Opts.ReadOnly {
+ // if the holder is ReadOnly, we silently ignore the "advisory"
+ // error. This may be a bad idea.
+
// generation code looks for a FileShouldBeTruncatedError
return false, trunc
}
@@ -299,7 +327,7 @@ func (f *fragment) applyStorage(data []byte, file *os.File, newGen generation, m
if file != nil {
fi, err := file.Stat()
if err != nil {
- f.Logger.Printf("trying to apply new storage to existing bitmap, stat failed: %v", err)
+ f.holder.Logger.Printf("trying to apply new storage to existing bitmap, stat failed: %v", err)
}
if err == nil && fi != nil && fi.Size() == 0 {
return f.emptyStorage(file)
@@ -335,6 +363,12 @@ func (f *fragment) applyStorage(data []byte, file *os.File, newGen generation, m
return mapped, err
}
+func (f *fragment) inspectStorage(data []byte, file *os.File, newGen generation, mapped bool) (didMap bool, err error) {
+ f.bitmapInfo = &roaring.BitmapInfo{}
+ f.storage, didMap, err = roaring.InspectBinary(data, mapped, f.bitmapInfo)
+ return didMap, err
+}
+
// openStorage opens the storage bitmap.
//
// This has been massively reworked recently, and now hands a lot of
@@ -353,13 +387,20 @@ func (f *fragment) openStorage(unmarshalData bool) error {
}
f.rowCache = &simpleCache{make(map[uint64]*Row)}
var storageOp func([]byte, *os.File, generation, bool) (bool, error)
- if unmarshalData {
- storageOp = f.importStorage
+ if f.holder.Opts.Inspect {
+ // note that this will unmarshal even if we already have
+ // storage; when Inspect is on for a holder, we actually want
+ // to be able to report this.
+ storageOp = f.inspectStorage
} else {
- storageOp = f.applyStorage
+ if unmarshalData {
+ storageOp = f.importStorage
+ } else {
+ storageOp = f.applyStorage
+ }
}
var err error
- f.gen, err = newGeneration(f.gen, f.path, unmarshalData, storageOp, f.Logger)
+ f.gen, err = newGeneration(f.gen, f.path, unmarshalData, storageOp, f.holder.Logger)
if f.gen != nil {
scratchData := f.gen.Bytes()
f.prevdata = f.currdata
@@ -407,7 +448,7 @@ func (f *fragment) openCache() error {
// Unmarshal cache data.
var pb internal.Cache
if err := proto.Unmarshal(buf, &pb); err != nil {
- f.Logger.Printf("error unmarshaling cache data, skipping: path=%s, err=%s", path, err)
+ f.holder.Logger.Printf("error unmarshaling cache data, skipping: path=%s, err=%s", path, err)
return nil
}
@@ -429,19 +470,22 @@ func (f *fragment) Close() error {
for f.snapshotPending {
f.snapshotCond.Wait()
}
+ // Note: snapshots won't progress on a closed fragment, so we
+ // wait until after a possible pending snapshot to close.
+ f.open = false
return f.close()
}
func (f *fragment) close() error {
// Flush cache if closing gracefully.
if err := f.flushCache(); err != nil {
- f.Logger.Printf("fragment: error flushing cache on close: err=%s, path=%s", err, f.path)
+ f.holder.Logger.Printf("fragment: error flushing cache on close: err=%s, path=%s", err, f.path)
return errors.Wrap(err, "flushing cache")
}
// Close underlying storage.
if err := f.closeStorage(); err != nil {
- f.Logger.Printf("fragment: error closing storage: err=%s, path=%s", err, f.path)
+ f.holder.Logger.Printf("fragment: error closing storage: err=%s, path=%s", err, f.path)
return errors.Wrap(err, "closing storage")
}
@@ -688,7 +732,8 @@ func (f *fragment) unprotectedSetRow(row *Row, rowID uint64) (changed bool, err
f.rowCache.Add(rowID, nil)
// Snapshot storage.
- f.snapshotQueue.Enqueue(f)
+ f.holder.SnapshotQueue.Enqueue(f)
+ f.stats.Count("setRow", 1, 1.0)
return changed, nil
}
@@ -728,7 +773,7 @@ func (f *fragment) unprotectedClearRow(rowID uint64) (changed bool, err error) {
f.rowCache.Add(rowID, nil)
// Snapshot storage.
- f.snapshotQueue.Enqueue(f)
+ f.holder.SnapshotQueue.Enqueue(f)
return changed, nil
}
@@ -2006,11 +2051,11 @@ func (f *fragment) importPositions(set, clear []uint64, rowSet map[uint64]struct
// we got an error. it's possible that the error indicates that something went wrong.
mappedIn, mappedOut, unmappedIn, errs, e2 := f.storage.SanityCheckMapping(f.currdata.from, f.currdata.to)
if errs != 0 {
- f.Logger.Printf("transaction failed on %s. storage has %d mapped in range, %d mapped out of range, %d unmapped in range, %d errors total, last %v",
+ f.holder.Logger.Printf("transaction failed on %s. storage has %d mapped in range, %d mapped out of range, %d unmapped in range, %d errors total, last %v",
f.path, mappedIn, mappedOut, unmappedIn, errs, e2)
if f.prevdata.from != f.currdata.from {
mappedIn, mappedOut, unmappedIn, errs, e2 = f.storage.SanityCheckMapping(f.prevdata.from, f.prevdata.to)
- f.Logger.Printf("with previous map, storage would have %d mapped in range, %d mapped out of range, %d unmapped in range, %d errors total, last %v",
+ f.holder.Logger.Printf("with previous map, storage would have %d mapped in range, %d mapped out of range, %d unmapped in range, %d errors total, last %v",
mappedIn, mappedOut, unmappedIn, errs, e2)
}
}
@@ -2164,7 +2209,7 @@ func (f *fragment) importValue(columnIDs []uint64, values []int64, bitDepth uint
// in theory, this should probably have been queued anyway, but if enough
// of the bits matched existing bits, we'll be under our opN estimate, and
// we want to ensure that the snapshot happens.
- return f.snapshotQueue.Immediate(f)
+ return f.holder.SnapshotQueue.Immediate(f)
}
// importRoaring imports from the official roaring data format defined at
@@ -2252,7 +2297,7 @@ func (f *fragment) incrementOpN(changed int) {
f.opN += changed
f.ops++
if f.opN > f.MaxOpN {
- f.snapshotQueue.Enqueue(f)
+ f.holder.SnapshotQueue.Enqueue(f)
}
}
@@ -2275,6 +2320,9 @@ func track(start time.Time, message string, stats stats.StatsClient, logger logg
// snapshot does the actual snapshot operation. it does not check or care
// about f.snapshotPending.
func (f *fragment) snapshot() (err error) {
+ if !f.open {
+ return errors.New("snapshot request on closed fragment")
+ }
wouldPanic := debug.SetPanicOnFault(true)
defer func() {
debug.SetPanicOnFault(wouldPanic)
@@ -2286,7 +2334,7 @@ func (f *fragment) snapshot() (err error) {
// we can't see the actual values that were used to generate this, probably.
if e2.Error() == "runtime error: invalid memory address or nil pointer dereference" {
mappedIn, mappedOut, unmappedIn, errs, _ := f.storage.SanityCheckMapping(f.currdata.from, f.currdata.to)
- f.Logger.Printf("transaction failed on %s. storage has %d mapped in range, %d mapped out of range, %d unmapped in range, %d errors total",
+ f.holder.Logger.Printf("transaction failed on %s. storage has %d mapped in range, %d mapped out of range, %d unmapped in range, %d errors total",
f.path, mappedIn, mappedOut, unmappedIn, errs)
}
} else {
@@ -2306,7 +2354,7 @@ func (f *fragment) snapshot() (err error) {
func unprotectedWriteToFragment(f *fragment, bm *roaring.Bitmap) (n int64, err error) { // nolint: interfacer
completeMessage := fmt.Sprintf("fragment: snapshot complete %s/%s/%s/%d", f.index, f.field, f.view, f.shard)
start := time.Now()
- defer track(start, completeMessage, f.stats, f.Logger)
+ defer track(start, completeMessage, f.stats, f.holder.Logger)
// Create a temporary file to snapshot to.
snapshotPath := f.path + snapshotExt
@@ -3096,7 +3144,7 @@ func (s *fragmentSyncer) syncBlockFromPrimary(id int) error {
// the primary node.
nodes := s.Cluster.shardNodes(f.index, f.shard)
if s.Node.ID != nodes[0].ID {
- f.Logger.Debugf("non-primary replica expecting sync from primary: %s, index=%s, field=%s, shard=%d", nodes[0].ID, f.index, f.field, f.shard)
+ f.holder.Logger.Debugf("non-primary replica expecting sync from primary: %s, index=%s, field=%s, shard=%d", nodes[0].ID, f.index, f.field, f.shard)
return nil
}
diff --git a/fragment_internal_test.go b/fragment_internal_test.go
index 5912eb132..fa1ffbeec 100644
--- a/fragment_internal_test.go
+++ b/fragment_internal_test.go
@@ -1477,7 +1477,7 @@ func BenchmarkFragment_Blocks(b *testing.B) {
}
// Open the fragment specified by the path.
- f := newFragment(*FragmentPath, "i", "f", viewStandard, 0, 0)
+ f := newFragment(NewHolder(DefaultPartitionN), *FragmentPath, "i", "f", viewStandard, 0, 0)
if err := f.Open(); err != nil {
b.Fatal(err)
}
@@ -2006,7 +2006,7 @@ func BenchmarkFragment_Snapshot(b *testing.B) {
b.ReportAllocs()
// Open the fragment specified by the path.
- f := newFragment(*FragmentPath, "i", "f", viewStandard, 0, 0)
+ f := newFragment(NewHolder(DefaultPartitionN), *FragmentPath, "i", "f", viewStandard, 0, 0)
if err := f.Open(); err != nil {
b.Fatal(err)
}
@@ -2121,7 +2121,7 @@ func BenchmarkImportRoaring(b *testing.B) {
// care whether this succeeds,
// but if it's happening we want
// it to be done.
- _ = f.snapshotQueue.Await(f)
+ _ = defaultSnapshotQueue.Await(f)
f.Clean(b)
b.Fatalf("import error: %v", err)
}
@@ -2162,7 +2162,7 @@ func BenchmarkImportRoaringConcurrent(b *testing.B) {
err := frags[j].importRoaringT(data[j], false)
// error unimportant if it happened, but we want
// any snapshots to have finished.
- _ = frags[j].snapshotQueue.Await(frags[j])
+ _ = defaultSnapshotQueue.Await(frags[j])
return err
})
}
@@ -2203,7 +2203,7 @@ func BenchmarkImportRoaringUpdateConcurrent(b *testing.B) {
if err != nil {
b.Fatalf("importing roaring: %v", err)
}
- err = frags[j].snapshotQueue.Immediate(frags[j])
+ err = defaultSnapshotQueue.Immediate(frags[j])
if err != nil {
b.Fatalf("snapshot after import: %v", err)
}
@@ -2214,7 +2214,7 @@ func BenchmarkImportRoaringUpdateConcurrent(b *testing.B) {
j := j
eg.Go(func() error {
err := frags[j].importRoaringT(updata, false)
- err2 := frags[j].snapshotQueue.Await(frags[j])
+ err2 := defaultSnapshotQueue.Await(frags[j])
if err == nil {
err = err2
}
@@ -2282,7 +2282,7 @@ func BenchmarkImportRoaringUpdate(b *testing.B) {
if err != nil {
b.Errorf("import error: %v", err)
}
- err = f.snapshotQueue.Immediate(f)
+ err = defaultSnapshotQueue.Immediate(f)
if err != nil {
b.Errorf("snapshot after import error: %v", err)
}
@@ -2292,7 +2292,7 @@ func BenchmarkImportRoaringUpdate(b *testing.B) {
f.Clean(b)
b.Errorf("import error: %v", err)
}
- err = f.snapshotQueue.Await(f)
+ err = defaultSnapshotQueue.Await(f)
if err != nil {
b.Errorf("snapshot after import error: %v", err)
}
@@ -2391,7 +2391,7 @@ func BenchmarkImportIntoLargeFragment(b *testing.B) {
}
origF.Close()
fi.Close()
- nf := newFragment(fi.Name(), "i", "f", viewStandard, 0, 0)
+ nf := newFragment(NewHolder(DefaultPartitionN), fi.Name(), "i", "f", viewStandard, 0, 0)
err = nf.Open()
if err != nil {
b.Fatalf("opening fragment: %v", err)
@@ -2428,7 +2428,7 @@ func BenchmarkImportRoaringIntoLargeFragment(b *testing.B) {
}
origF.Close()
fi.Close()
- nf := newFragment(fi.Name(), "i", "f", viewStandard, 0, 0)
+ nf := newFragment(NewHolder(DefaultPartitionN), fi.Name(), "i", "f", viewStandard, 0, 0)
err = nf.Open()
if err != nil {
b.Fatalf("opening fragment: %v", err)
@@ -2607,17 +2607,23 @@ func (f *fragment) sanityCheck(t testing.TB) {
func (f *fragment) Clean(t testing.TB) {
f.mu.Lock()
- err := f.snapshotQueue.Await(f)
- f.mu.Unlock()
- if err != nil {
- t.Fatalf("snapshot failed before sanity check: %v", err)
- }
- f.sanityCheck(t)
- if f.storage != nil && f.storage.Source != nil {
- if f.storage.Source.Dead() {
- t.Fatalf("cleaning up fragment %s, source %s, source already dead", f.path, f.storage.Source.ID())
+ // we need to ensure that we unlock the mutex before terminating
+ // the clean operation, but we need it held during the sanity
+ // check or else, in some cases, the background snapshot queue
+ // can decide to pick it up.
+ func() {
+ defer f.mu.Unlock()
+ err := defaultSnapshotQueue.Await(f)
+ if err != nil {
+ t.Fatalf("snapshot failed before sanity check: %v", err)
}
- }
+ f.sanityCheck(t)
+ if f.storage != nil && f.storage.Source != nil {
+ if f.storage.Source.Dead() {
+ t.Fatalf("cleaning up fragment %s, source %s, source already dead", f.path, f.storage.Source.ID())
+ }
+ }
+ }()
errc := f.Close()
// prevent double-closes of generation during testing.
f.gen = nil
@@ -2626,10 +2632,6 @@ func (f *fragment) Clean(t testing.TB) {
if errc != nil || errf != nil {
t.Fatal("cleaning up fragment: ", errc, errf, errp)
}
- if f.snapshotQueue != nil {
- f.snapshotQueue.Stop()
- f.snapshotQueue = nil
- }
// not all fragments have cache files
if errp != nil && !os.IsNotExist(errp) {
t.Fatalf("cleaning up fragment cache: %v", errp)
@@ -2649,10 +2651,6 @@ func (f *fragment) CleanKeep(t testing.TB) {
if errc != nil {
t.Fatal("closing fragment: ", errc, errp)
}
- if f.snapshotQueue != nil {
- f.snapshotQueue.Stop()
- f.snapshotQueue = nil
- }
// not all fragments have cache files
if errp != nil && !os.IsNotExist(errp) {
t.Fatalf("cleaning up fragment cache: %v", errp)
@@ -2668,6 +2666,12 @@ func mustOpenBSIFragment(index, field, view string, shard uint64) *fragment {
return mustOpenFragmentFlags(index, field, view, shard, "", 1)
}
+var testHolder = NewHolder(DefaultPartitionN)
+
+func init() {
+ testHolder.SnapshotQueue = newSnapshotQueue(1, 1, nil)
+}
+
// mustOpenFragment returns a new instance of Fragment with a temporary path.
func mustOpenFragmentFlags(index, field, view string, shard uint64, cacheType string, flags byte) *fragment {
file, err := ioutil.TempFile(*TempDir, "pilosa-fragment-")
@@ -2680,12 +2684,12 @@ func mustOpenFragmentFlags(index, field, view string, shard uint64, cacheType st
cacheType = DefaultCacheType
}
- f := newFragment(file.Name(), index, field, view, shard, flags)
+ f := newFragment(testHolder, file.Name(), index, field, view, shard, flags)
+
f.CacheType = cacheType
f.RowAttrStore = &memAttrStore{
store: make(map[uint64]map[string]interface{}),
}
- f.snapshotQueue = newSnapshotQueue(1, 1, nil)
if err := f.Open(); err != nil {
panic(err)
@@ -3179,7 +3183,7 @@ func TestUnionInPlaceMapped(t *testing.T) {
// it's used only in computation of things that usually don't go to
// disk, which is why we handle this specially in testing and not
// generically.
- err = f.snapshotQueue.Immediate(f)
+ err = defaultSnapshotQueue.Immediate(f)
if err != nil {
t.Fatalf("snapshot after union-in-place: %v", err)
}
@@ -3332,7 +3336,6 @@ func TestImportClearRestart(t *testing.T) {
cols: []uint64{1, 1, 1, 1, 1, 1},
},
}
-
for i, test := range tests {
for _, maxOpN := range []int{0, 10000} {
t.Run(fmt.Sprintf("%dMaxOpN%d", i, maxOpN), func(t *testing.T) {
@@ -3385,7 +3388,7 @@ func TestImportClearRestart(t *testing.T) {
check(t, f, exp)
- f2 := newFragment(f.path, "i", "f", viewStandard, 0, 0)
+ f2 := newFragment(NewHolder(DefaultPartitionN), f.path, "i", "f", viewStandard, 0, 0)
f2.MaxOpN = maxOpN
f2.CacheType = f.CacheType
@@ -3419,7 +3422,7 @@ func TestImportClearRestart(t *testing.T) {
check(t, f2, exp)
- f3 := newFragment(f2.path, "i", "f", viewStandard, 0, 0)
+ f3 := newFragment(NewHolder(DefaultPartitionN), f2.path, "i", "f", viewStandard, 0, 0)
f3.MaxOpN = maxOpN
f3.CacheType = f.CacheType
diff --git a/generation.go b/generation.go
index 7c1361b1d..f3cc6ae88 100644
--- a/generation.go
+++ b/generation.go
@@ -263,7 +263,7 @@ func (m *mmapGeneration) openFile() (shouldClose bool, err error) {
}
// do we actually want this in every openFile? I don't know.
if err := syscall.Flock(int(m.file.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil {
- m.file.Close()
+ _ = syswrap.CloseFile(m.file)
m.file = nil
return false, fmt.Errorf("flock: %s", err)
}
@@ -333,6 +333,7 @@ func newGeneration(existing generation, path string, readData bool, setup func([
m := mmapGeneration{path: path, logger: logger}
if existing != nil {
m.generation = existing.Generation() + 1
+ m.retries = existing.(*mmapGeneration).retries
// we might keep a previous generation around just for its generation count.
if !existing.Dead() {
defer existing.Done()
diff --git a/generation_test.go b/generation_test.go
index 75444fcf7..7df71fc98 100644
--- a/generation_test.go
+++ b/generation_test.go
@@ -39,14 +39,14 @@ func TestGenerationPanic(t *testing.T) {
}
prevData = f.gen.(*mmapGeneration).data
f.mu.Lock()
- _ = f.snapshotQueue.Immediate(f)
+ _ = defaultSnapshotQueue.Immediate(f)
f.mu.Unlock()
runtime.GC()
for i := 0; i < (f.MaxOpN / 2); i++ {
_, _ = f.setBit(0, uint64(i*32)+23)
}
f.mu.Lock()
- f.snapshotQueue.Await(f)
+ defaultSnapshotQueue.Await(f)
f.mu.Unlock()
runtime.GC()
newData := f.gen.(*mmapGeneration).data
diff --git a/handler.go b/handler.go
index 17156245b..bb5aa245d 100644
--- a/handler.go
+++ b/handler.go
@@ -287,3 +287,31 @@ type TranslateIDsRequest struct {
type TranslateIDsResponse struct {
Keys []string
}
+
+// InspectRequestParams represents the parts of an InspectRequest that
+// aren't generic holder filtering attributes.
+type InspectRequestParams struct {
+ Containers bool // include container details
+ Checksum bool // perform checksums
+}
+
+// InspectRequest represents a request for a possibly-partial
+// holder inspection, using a provided holder filter and inspect-specific
+// parameters.
+type InspectRequest struct {
+ HolderFilterParams
+ InspectRequestParams
+}
+
+// InspectResponse contains the structured results for an InspectRequest.
+// It may some day be expanded to include metadata about views or indexes.
+type InspectResponse struct {
+ Fragments []struct {
+ Index string
+ Field string
+ View string
+ Shard int64
+ Path string
+ Info *FragmentInfo
+ }
+}
diff --git a/holder.go b/holder.go
index de3cd0701..d1f457371 100644
--- a/holder.go
+++ b/holder.go
@@ -21,7 +21,9 @@ import (
"os"
"path"
"path/filepath"
+ "regexp"
"sort"
+ "strconv"
"strings"
"sync"
"syscall"
@@ -77,9 +79,8 @@ type Holder struct {
// The interval at which the cached row ids are persisted to disk.
cacheFlushInterval time.Duration
- Logger logger.Logger
-
- snapshotQueue snapshotQueue
+ Logger logger.Logger
+ SnapshotQueue SnapshotQueue
// Instantiates new translation stores
OpenTranslateStore OpenTranslateStoreFunc
@@ -102,6 +103,18 @@ type Holder struct {
// needs to be queued and completed after all indexes
// have opened.
opening bool
+
+ Opts HolderOpts
+}
+
+type HolderOpts struct {
+ // ReadOnly indicates that this holder's contents should not produce
+ // disk writes under any circumstances. It must be set before Open
+ // is called, and changing it is not supported.
+ ReadOnly bool
+ // If Inspect is set, we'll try to obtain additional information
+ // about fragments when opening them.
+ Inspect bool
}
func (h *Holder) StartTransaction(ctx context.Context, id string, timeout time.Duration, exclusive bool) (*Transaction, error) {
@@ -169,9 +182,285 @@ func NewHolder(partitionN int) *Holder {
translationSyncer: NopTranslationSyncer,
Logger: logger.NopLogger,
+
+ SnapshotQueue: defaultSnapshotQueue,
}
}
+type HolderInfo struct {
+ FragmentInfo map[string]FragmentInfo
+ FragmentNames []string
+}
+
+type regexpList []*regexp.Regexp
+
+func newRegexpList(regexes string) (results regexpList, err error) {
+ if regexes == "" {
+ return nil, nil
+ }
+ for _, sub := range strings.Split(regexes, ",") {
+ re, err := regexp.Compile(sub)
+ if err != nil {
+ return nil, err
+ }
+ results = append(results, re)
+ }
+ return results, nil
+}
+
+func (rl regexpList) Match(haystack string) bool {
+ if rl == nil {
+ return true
+ }
+ for _, re := range rl {
+ if re.MatchString(haystack) {
+ return true
+ }
+ }
+ return false
+}
+
+// shardRange represents a series of shards
+type shardRange struct {
+ min, max uint64
+}
+
+type shardRangeList []shardRange
+
+func newShardRangeList(shards string) (results shardRangeList, err error) {
+ if shards == "" {
+ return nil, nil
+ }
+ for _, sub := range strings.Split(shards, ",") {
+ var sr shardRange
+ minMax := strings.Split(sub, "-")
+ if len(minMax) > 2 {
+ return nil, fmt.Errorf("invalid range %q", sub)
+ }
+ sr.min, err = strconv.ParseUint(minMax[0], 10, 64)
+ if err != nil {
+ return nil, err
+ }
+ sr.max = sr.min
+ if len(minMax) == 2 {
+ sr.max, err = strconv.ParseUint(minMax[0], 10, 64)
+ if err != nil {
+ return nil, err
+ }
+ }
+ if sr.max < sr.min {
+ return nil, fmt.Errorf("invalid range %q: max < min", sub)
+ }
+ results = append(results, sr)
+ }
+ return results, nil
+}
+
+func (sl shardRangeList) Match(shard uint64) bool {
+ if sl == nil {
+ return true
+ }
+ for _, sr := range sl {
+ if shard >= sr.min && shard <= sr.max {
+ return true
+ }
+ }
+ return false
+}
+
+// HolderFilter represents something that potentially filters out
+// parts of a holder, indicating whether or not to process them,
+// or recurse into them. It is permissible to recurse a thing
+// without processing it, or process it without recursing it.
+// For instance, something looking to accumulate statistics
+// about views might return (true, false) from CheckView,
+// while a fragment scanning operation would return (false, true)
+// from everything above CheckFrag.
+type HolderFilter interface {
+ CheckIndex(iname string) (process bool, recurse bool)
+ CheckField(iname, fname string) (process bool, recurse bool)
+ CheckView(iname, fname, vname string) (process bool, recurse bool)
+ CheckFragment(iname, fname, vname string, shard uint64) (process bool)
+}
+
+// HolderFilterAll is a placeholder type which always returns true for the
+// check functions. You can embed it to make a HolderOperator which processes
+// everything.
+type HolderFilterAll struct{}
+
+func (HolderFilterAll) CheckIndex(string) (bool, bool) {
+ return true, true
+}
+
+func (HolderFilterAll) CheckField(string, string) (bool, bool) {
+ return true, true
+}
+
+func (HolderFilterAll) CheckView(string, string, string) (bool, bool) {
+ return true, true
+}
+
+func (HolderFilterAll) CheckFragment(string, string, string, uint64) bool {
+ return true
+}
+
+// HolderProcessNone is a placeholder type which does nothing for the
+// process functions. You can embed it to make a HolderOperator which
+// does nothing, or embed it and provide your own ProcessFragment to
+// do just that.
+type HolderProcessNone struct{}
+
+func (HolderProcessNone) ProcessIndex(*Index) error {
+ return nil
+}
+
+func (HolderProcessNone) ProcessField(*Field) error {
+ return nil
+}
+
+func (HolderProcessNone) ProcessView(*view) error {
+ return nil
+}
+
+func (HolderProcessNone) ProcessFragment(*fragment) error {
+ return nil
+}
+
+// HolderProcess represents something that has operations which can be
+// performed on indexes, fields, views, and/or fragments.
+type HolderProcess interface {
+ ProcessIndex(*Index) error
+ ProcessField(*Field) error
+ ProcessView(*view) error
+ ProcessFragment(*fragment) error
+}
+
+// HolderOperator is both a filter and a process. This is the general
+// form of "I want to do something to some part of a holder."
+type HolderOperator interface {
+ HolderFilter
+ HolderProcess
+}
+
+var _ HolderOperator = (*holderInspector)(nil)
+
+type HolderFilterParams struct {
+ Indexes string
+ Fields string
+ Views string
+ Shards string
+}
+
+type holderFilterFull struct {
+ HolderFilterParams
+ indexRegexps regexpList
+ fieldRegexps regexpList
+ viewRegexps regexpList
+ shardRanges shardRangeList
+}
+
+type inspectRequestFull struct {
+ HolderFilter
+ params InspectRequestParams
+}
+
+func (i *holderFilterFull) CheckIndex(iname string) (process, recurse bool) {
+ return true, i.indexRegexps.Match(iname)
+}
+
+func (i *holderFilterFull) CheckField(iname, fname string) (process, recurse bool) {
+ return true, i.fieldRegexps.Match(fname)
+}
+
+func (i *holderFilterFull) CheckView(iname, fname, vname string) (process, recurse bool) {
+ return true, i.viewRegexps.Match(vname)
+}
+
+func (i *holderFilterFull) CheckFragment(iname, fname, vname string, shard uint64) (process bool) {
+ return i.shardRanges.Match(shard)
+}
+
+func NewHolderFilter(params HolderFilterParams) (result HolderFilter, err error) {
+ filter := &holderFilterFull{
+ HolderFilterParams: params,
+ }
+ filter.indexRegexps, err = newRegexpList(params.Indexes)
+ if err != nil {
+ return nil, err
+ }
+ filter.fieldRegexps, err = newRegexpList(params.Fields)
+ if err != nil {
+ return nil, err
+ }
+ filter.viewRegexps, err = newRegexpList(params.Views)
+ if err != nil {
+ return nil, err
+ }
+ filter.shardRanges, err = newShardRangeList(params.Shards)
+ if err != nil {
+ return nil, err
+ }
+ return filter, nil
+}
+
+func expandInspectRequest(req *InspectRequest) (*inspectRequestFull, error) {
+ filter, err := NewHolderFilter(req.HolderFilterParams)
+ if err != nil {
+ return nil, err
+ }
+ irf := &inspectRequestFull{
+ HolderFilter: filter,
+ params: req.InspectRequestParams,
+ }
+ return irf, nil
+}
+
+type holderInspector struct {
+ *inspectRequestFull
+ pathParts [3]string
+ path string
+ hi *HolderInfo
+}
+
+func (h *holderInspector) ProcessIndex(i *Index) error {
+ h.pathParts[0] = i.name
+ return nil
+}
+
+func (h *holderInspector) ProcessField(f *Field) error {
+ h.pathParts[1] = f.name
+ return nil
+}
+
+func (h *holderInspector) ProcessView(v *view) error {
+ h.pathParts[2] = v.name
+ h.path = strings.Join(h.pathParts[:], "/")
+ return nil
+}
+
+func (h *holderInspector) ProcessFragment(f *fragment) error {
+ path := h.path + "/" + strconv.FormatUint(f.shard, 10)
+ h.hi.FragmentInfo[path] = f.inspect(h.inspectRequestFull.params)
+ h.hi.FragmentNames = append(h.hi.FragmentNames, path)
+ return nil
+}
+
+func (h *Holder) Inspect(ctx context.Context, req *InspectRequest) (*HolderInfo, error) {
+ fullReq, err := expandInspectRequest(req)
+ if err != nil {
+ return nil, err
+ }
+ inspector := &holderInspector{
+ inspectRequestFull: fullReq,
+ hi: &HolderInfo{
+ FragmentInfo: make(map[string]FragmentInfo),
+ },
+ }
+ err = h.Process(ctx, inspector)
+ sort.Strings(inspector.hi.FragmentNames)
+ return inspector.hi, err
+}
+
// Open initializes the root data directory for the holder.
func (h *Holder) Open() error {
h.opening = true
@@ -213,11 +502,6 @@ func (h *Holder) Open() error {
return errors.Wrap(err, "reading directory")
}
- // Run snapshots asynchronously. The snapshotQueue will have a background
- // task associated with it which flushes it and waits until this channel
- // is closed, so we should always close this channel when done.
- h.snapshotQueue = newSnapshotQueue(10, 2, h.Logger)
-
for _, fi := range fis {
// Skip files or hidden directories.
if !fi.IsDir() || strings.HasPrefix(fi.Name(), ".") {
@@ -261,18 +545,25 @@ func (h *Holder) Open() error {
h.Logger.Printf("open holder: complete")
- // Periodically flush cache.
- h.wg.Add(1)
- go func() { defer h.wg.Done(); h.monitorCacheFlush() }()
-
h.Stats.Open()
- h.snapshotQueue.ScanHolder(h)
h.opened.Close()
return nil
}
+// Activate runs the background tasks relevant to keeping a holder in a stable
+// state, such as scanning it for needed snapshots, or flushing caches. This
+// is separate from opening because, while a server would nearly always want
+// to do this, other use cases (like consistency checks of a data directory)
+// need to avoid it even getting started.
+func (h *Holder) Activate() {
+ // Periodically flush cache.
+ h.wg.Add(2)
+ go func() { defer h.wg.Done(); h.monitorCacheFlush() }()
+ go func() { defer h.wg.Done(); h.SnapshotQueue.ScanHolder(h, h.closing) }()
+}
+
// checkForeignIndex is a check before applying a foreign
// index to a field; if the index is not yet available,
// (because holder is still opening and may not have opened
@@ -313,10 +604,6 @@ func (h *Holder) Close() error {
return errors.Wrap(err, "closing index")
}
}
- if h.snapshotQueue != nil {
- h.snapshotQueue.Stop()
- h.snapshotQueue = nil
- }
// Reset opened in case Holder needs to be reopened.
h.opened.mu.Lock()
@@ -587,19 +874,16 @@ func (h *Holder) createIndex(name string, opt IndexOptions) (*Index, error) {
}
func (h *Holder) newIndex(path, name string) (*Index, error) {
- index, err := NewIndex(path, name, h.partitionN)
+ index, err := NewIndex(h, path, name)
if err != nil {
return nil, err
}
- index.logger = h.Logger
index.Stats = h.Stats.WithTags(fmt.Sprintf("index:%s", index.Name()))
index.broadcaster = h.broadcaster
index.newAttrStore = h.NewAttrStore
index.columnAttrs = h.NewAttrStore(filepath.Join(index.path, ".data"))
- index.snapshotQueue = h.snapshotQueue
index.OpenTranslateStore = h.OpenTranslateStore
index.translationSyncer = h.translationSyncer
- index.holder = h
return index, nil
}
@@ -1368,3 +1652,132 @@ func uint64InSlice(i uint64, s []uint64) bool {
}
return false
}
+
+// Process loops through a holder based on the Check functions in op, calling
+// the Process functions in op when indicated.
+func (h *Holder) Process(ctx context.Context, op HolderOperator) (err error) {
+ var indexNames, fieldNames, viewNames []string
+ var fragNums []uint64
+
+ h.mu.Lock()
+ for indexName := range h.indexes {
+ indexNames = append(indexNames, indexName)
+ }
+ h.mu.Unlock()
+ for _, indexName := range indexNames {
+ if err = ctx.Err(); err != nil {
+ return err
+ }
+ process, recurse := op.CheckIndex(indexName)
+ if !process && !recurse {
+ continue
+ }
+ h.mu.Lock()
+ index := h.indexes[indexName]
+ h.mu.Unlock()
+ if index == nil {
+ continue
+ }
+ if err = ctx.Err(); err != nil {
+ return err
+ }
+ if process {
+ err = op.ProcessIndex(index)
+ if err != nil {
+ return err
+ }
+ }
+ if !recurse {
+ continue
+ }
+ fieldNames = fieldNames[:0]
+ index.mu.Lock()
+ for fieldName := range index.fields {
+ fieldNames = append(fieldNames, fieldName)
+ }
+ index.mu.Unlock()
+ for _, fieldName := range fieldNames {
+ if err = ctx.Err(); err != nil {
+ return err
+ }
+ process, recurse := op.CheckField(indexName, fieldName)
+ if !process && !recurse {
+ continue
+ }
+ index.mu.Lock()
+ field := index.fields[fieldName]
+ index.mu.Unlock()
+ if field == nil {
+ continue
+ }
+ if err = ctx.Err(); err != nil {
+ return err
+ }
+ if process {
+ err = op.ProcessField(field)
+ if err != nil {
+ return err
+ }
+ }
+ if !recurse {
+ continue
+ }
+ viewNames = viewNames[:0]
+ field.mu.Lock()
+ for viewName := range field.viewMap {
+ viewNames = append(viewNames, viewName)
+ }
+ field.mu.Unlock()
+ for _, viewName := range viewNames {
+ if err = ctx.Err(); err != nil {
+ return err
+ }
+ process, recurse := op.CheckView(indexName, fieldName, viewName)
+ if !process && !recurse {
+ continue
+ }
+ field.mu.Lock()
+ view := field.viewMap[viewName]
+ field.mu.Unlock()
+ if view == nil {
+ continue
+ }
+ if err = ctx.Err(); err != nil {
+ return err
+ }
+ if process {
+ err = op.ProcessView(view)
+ if err != nil {
+ return err
+ }
+ }
+ if !recurse {
+ continue
+ }
+ fragNums := fragNums[:0]
+ view.mu.Lock()
+ for fragNum := range view.fragments {
+ fragNums = append(fragNums, fragNum)
+ }
+ view.mu.Unlock()
+ for _, fragNum := range fragNums {
+ if err = ctx.Err(); err != nil {
+ return err
+ }
+ process := op.CheckFragment(indexName, fieldName, viewName, fragNum)
+ if !process {
+ continue
+ }
+ view.mu.Lock()
+ frag := view.fragments[fragNum]
+ view.mu.Unlock()
+ err = op.ProcessFragment(frag)
+ if err != nil {
+ return err
+ }
+ }
+ }
+ }
+ }
+ return nil
+}
diff --git a/holder_internal_test.go b/holder_internal_test.go
index ccc52c5b0..5ae365362 100644
--- a/holder_internal_test.go
+++ b/holder_internal_test.go
@@ -1,4 +1,4 @@
-// Copyright 2017 Pilosa Corp.
+// Copyright 2020 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -15,313 +15,163 @@
package pilosa
import (
+ "context"
"io/ioutil"
"os"
- "path/filepath"
- "reflect"
- "strings"
"testing"
- "time"
-
- "github.com/pilosa/pilosa/v2/roaring"
)
-type tHolder struct {
- *Holder
+type testHolderOperator struct {
+ indexSeen, indexProcessed int
+ fieldSeen, fieldProcessed int
+ viewSeen, viewProcessed int
+ fragmentSeen, fragmentProcessed int
+ waitHere chan struct{}
}
-// Close closes the holder and removes all underlying data.
-func (h *tHolder) Close() error {
- defer os.RemoveAll(h.Path)
- return h.Holder.Close()
+func (t *testHolderOperator) CheckIndex(string) (bool, bool) {
+ t.indexSeen++
+ return true, true
}
-// Reopen instantiates and opens a new holder.
-// Note that the holder must be Closed first.
-func (h *tHolder) Reopen() error {
- path, logger := h.Path, h.Holder.Logger
- h.Holder = NewHolder(DefaultPartitionN)
- h.Holder.Path = path
- h.Holder.Logger = logger
- return h.Holder.Open()
+func (t *testHolderOperator) CheckField(string, string) (bool, bool) {
+ t.fieldSeen++
+ return true, true
}
-func newHolder() *tHolder {
- path, err := ioutil.TempDir(*TempDir, "pilosa-")
+func (t *testHolderOperator) CheckView(string, string, string) (bool, bool) {
+ t.viewSeen++
+ return true, true
+}
+
+func (t *testHolderOperator) CheckFragment(string, string, string, uint64) bool {
+ t.fragmentSeen++
+ return true
+}
+
+func (t *testHolderOperator) ProcessIndex(*Index) error {
+ t.indexProcessed++
+ return nil
+}
+
+func (t *testHolderOperator) ProcessField(*Field) error {
+ t.fieldProcessed++
+ return nil
+}
+
+func (t *testHolderOperator) ProcessView(*view) error {
+ t.viewProcessed++
+ return nil
+}
+
+func (t *testHolderOperator) ProcessFragment(*fragment) error {
+ if t.waitHere != nil {
+ <-t.waitHere
+ }
+ t.fragmentProcessed++
+ return nil
+}
+
+func makeHolder() (*Holder, string, error) {
+ path, err := ioutil.TempDir("", "pilosa-")
if err != nil {
- panic(err)
+ return nil, "", err
}
-
- h := &tHolder{Holder: NewHolder(DefaultPartitionN)}
- h.Path = path
- return h
-}
-
-// MustCreateFieldIfNotExists returns a given field. Panic on error.
-func (h *tHolder) MustCreateFieldIfNotExists(index, field string) *Field {
- f, err := h.MustCreateIndexIfNotExists(index, IndexOptions{}).CreateFieldIfNotExists(field, OptFieldTypeDefault())
- if err != nil {
- panic(err)
- }
- return f
-}
-
-// MustCreateIndexIfNotExists returns a given index. Panic on error.
-func (h *tHolder) MustCreateIndexIfNotExists(index string, opt IndexOptions) *Index {
- idx, err := h.Holder.CreateIndexIfNotExists(index, opt)
- if err != nil {
- panic(err)
- }
- return idx
-}
-
-// SetBit clears a bit on the given field.
-func (h *tHolder) SetBit(index, field string, rowID, columnID uint64) {
- f := h.MustCreateFieldIfNotExists(index, field)
- _, err := f.SetBit(rowID, columnID, nil)
- if err != nil {
- panic(err)
- }
-}
-
-// Row returns a Row for a given field.
-func (h *tHolder) Row(index, field string, rowID uint64) *Row {
- f := h.MustCreateFieldIfNotExists(index, field)
- row, err := f.Row(rowID)
- if err != nil {
- panic(err)
- }
- return row
-}
-
-func TestHolder_Optn(t *testing.T) {
- t.Run("ErrViewPermission", func(t *testing.T) {
- if os.Geteuid() == 0 {
- t.Skip("Skipping permissions test since user is root.")
- }
- availableShardFileFlushDuration.Set(100 * time.Millisecond)
- h := newHolder()
- defer h.Close()
-
- if idx, err := h.CreateIndex("foo", IndexOptions{}); err != nil {
- t.Fatal(err)
- } else if field, err := idx.CreateField("bar", OptFieldTypeDefault()); err != nil {
- t.Fatal(err)
- } else if _, err := field.createViewIfNotExists(viewStandard); err != nil {
- t.Fatal(err)
- } else if err := h.Holder.Close(); err != nil {
- t.Fatal(err)
- } else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard"), 0000); err != nil {
- t.Fatal(err)
- }
- defer func() {
- // we don't care about a failure here
- _ = os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard"), 0755)
- }()
- if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") {
- t.Fatalf("unexpected error: %s", err)
- }
- })
- t.Run("ErrViewFragmentsMkdir", func(t *testing.T) {
- if os.Geteuid() == 0 {
- t.Skip("Skipping permissions test since user is root.")
- }
- h := newHolder()
- defer h.Close()
-
- if idx, err := h.CreateIndex("foo", IndexOptions{}); err != nil {
- t.Fatal(err)
- } else if field, err := idx.CreateField("bar", OptFieldTypeDefault()); err != nil {
- t.Fatal(err)
- } else if _, err := field.createViewIfNotExists(viewStandard); err != nil {
- t.Fatal(err)
- } else if err := h.Holder.Close(); err != nil {
- t.Fatal(err)
- } else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments"), 0000); err != nil {
- t.Fatal(err)
- }
- defer func() {
- // we don't care about a failure here
- _ = os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments"), 0755)
- }()
-
- if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") {
- t.Fatalf("unexpected error: %s", err)
- }
- })
-
- t.Run("ErrFragmentCachePermission", func(t *testing.T) {
- if os.Geteuid() == 0 {
- t.Skip("Skipping permissions test since user is root.")
- }
- h := newHolder()
- defer h.Close()
-
- if idx, err := h.CreateIndex("foo", IndexOptions{}); err != nil {
- t.Fatal(err)
- } else if field, err := idx.CreateField("bar", OptFieldTypeDefault()); err != nil {
- t.Fatal(err)
- } else if view, err := field.createViewIfNotExists(viewStandard); err != nil {
- t.Fatal(err)
- } else if _, err := field.SetBit(0, 0, nil); err != nil {
- t.Fatal(err)
- } else if err := view.Fragment(0).FlushCache(); err != nil {
- t.Fatal(err)
- } else if err := h.Holder.Close(); err != nil {
- t.Fatal(err)
- } else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0.cache"), 0000); err != nil {
- t.Fatal(err)
- }
- defer func() {
- _ = os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0.cache"), 0644)
- }()
- if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") {
- t.Fatalf("unexpected error: %s", err)
- }
- })
-
-}
-
-// Ensure holder can clean up orphaned fragments.
-func TestHolderCleaner_CleanHolder(t *testing.T) {
- availableShardFileFlushDuration.Set(100 * time.Millisecond) //shorten the default time to force a file write
- cluster := NewTestCluster(2)
-
- // Create a local holder.
- hldr0 := newHolder()
- defer hldr0.Close()
-
- // Mock 2-node, fully replicated cluster.
- cluster.ReplicaN = 2
-
- cluster.nodes[0].URI = NewTestURIFromHostPort("localhost", 0)
-
- // Create fields on nodes.
- for _, hldr := range []*tHolder{hldr0} {
- hldr.MustCreateFieldIfNotExists("i", "f")
- hldr.MustCreateFieldIfNotExists("i", "f0")
- hldr.MustCreateFieldIfNotExists("y", "z")
- }
-
- // Set data on the local holder.
- hldr0.SetBit("i", "f", 0, 10)
- hldr0.SetBit("i", "f", 0, 4000)
- hldr0.SetBit("i", "f", 2, 20)
- hldr0.SetBit("i", "f", 3, 10)
- hldr0.SetBit("i", "f", 120, 10)
- hldr0.SetBit("i", "f", 200, 4)
-
- hldr0.SetBit("i", "f0", 9, ShardWidth+5)
-
- hldr0.SetBit("y", "z", 10, (2*ShardWidth)+4)
- hldr0.SetBit("y", "z", 10, (2*ShardWidth)+5)
- hldr0.SetBit("y", "z", 10, (2*ShardWidth)+7)
-
- // Set highest shard.
- err := hldr0.Field("i", "f").AddRemoteAvailableShards(roaring.NewBitmap(0, 1))
- if err != nil {
- t.Fatalf("adding remote shards: %v", err)
- }
- err = hldr0.Field("y", "z").AddRemoteAvailableShards(roaring.NewBitmap(0, 1, 2))
- if err != nil {
- t.Fatalf("adding remote shards: %v", err)
- }
- time.Sleep(2 * availableShardFileFlushDuration.Get())
-
- // Keep replication the same and ensure we get the expected results.
- cluster.ReplicaN = 2
-
- // Set up cleaner for replication 2.
- cleaner2 := holderCleaner{
- Node: cluster.nodes[0],
- Holder: hldr0.Holder,
- Cluster: cluster,
- }
-
- if err := cleaner2.CleanHolder(); err != nil {
- t.Fatal(err)
- }
-
- // Verify data is the same on both nodes.
- for i, hldr := range []*tHolder{hldr0} {
- if a := hldr.Row("i", "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) {
- t.Fatalf("unexpected columns(%d/0): %+v", i, a)
- } else if a := hldr.Row("i", "f", 2).Columns(); !reflect.DeepEqual(a, []uint64{20}) {
- t.Fatalf("unexpected columns(%d/2): %+v", i, a)
- } else if a := hldr.Row("i", "f", 3).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
- t.Fatalf("unexpected columns(%d/3): %+v", i, a)
- } else if a := hldr.Row("i", "f", 120).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
- t.Fatalf("unexpected columns(%d/120): %+v", i, a)
- } else if a := hldr.Row("i", "f", 200).Columns(); !reflect.DeepEqual(a, []uint64{4}) {
- t.Fatalf("unexpected columns(%d/200): %+v", i, a)
- }
-
- if a := hldr.Row("i", "f0", 9).Columns(); !reflect.DeepEqual(a, []uint64{ShardWidth + 5}) {
- t.Fatalf("unexpected columns(%d/d/f0): %+v", i, a)
- }
-
- if a := hldr.Row("y", "z", 10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * ShardWidth) + 4, (2 * ShardWidth) + 5, (2 * ShardWidth) + 7}) {
- t.Fatalf("unexpected columns(%d/y/z): %+v", i, a)
- }
- }
-
- // Change replication factor to ensure we have fragments to remove.
- cluster.ReplicaN = 1
-
- // Set up cleaner for replication 1.
- cleaner1 := holderCleaner{
- Node: cluster.nodes[0],
- Holder: hldr0.Holder,
- Cluster: cluster,
- }
-
- if err := cleaner1.CleanHolder(); err != nil {
- t.Fatal(err)
- }
-
- // Verify data is the same on both nodes.
- for i, hldr := range []*tHolder{hldr0} {
- if a := hldr.Row("i", "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) {
- t.Fatalf("unexpected columns(%d/0): %+v", i, a)
- } else if a := hldr.Row("i", "f", 2).Columns(); !reflect.DeepEqual(a, []uint64{20}) {
- t.Fatalf("unexpected columns(%d/2): %+v", i, a)
- } else if a := hldr.Row("i", "f", 3).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
- t.Fatalf("unexpected columns(%d/3): %+v", i, a)
- } else if a := hldr.Row("i", "f", 120).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
- t.Fatalf("unexpected columns(%d/120): %+v", i, a)
- } else if a := hldr.Row("i", "f", 200).Columns(); !reflect.DeepEqual(a, []uint64{4}) {
- t.Fatalf("unexpected columns(%d/200): %+v", i, a)
- }
-
- f := hldr.fragment("i", "f0", viewStandard, 1)
- if f != nil {
- t.Fatalf("expected fragment to be deleted: (%d/i/f0): %+v", i, f)
- }
-
- if a := hldr.Row("y", "z", 10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * ShardWidth) + 4, (2 * ShardWidth) + 5, (2 * ShardWidth) + 7}) {
- t.Fatalf("unexpected columns(%d/y/z): %+v", i, a)
- }
- }
-}
-
-// Ensure holder can reopen.
-func TestHolderCleaner_Reopen(t *testing.T) {
h := NewHolder(DefaultPartitionN)
- h.Path = "path"
- err := h.Open()
+
+ return h, path, nil
+}
+
+func testSetBit(t *testing.T, h *Holder, index, field string, rowID, columnID uint64) {
+ idx, err := h.CreateIndexIfNotExists(index, IndexOptions{})
if err != nil {
- t.Fatalf("couldn't open holder: %v", err)
+ t.Fatalf("creating index: %v", err)
}
- err = h.Close()
+ f, err := idx.CreateFieldIfNotExists(field, OptFieldTypeDefault())
if err != nil {
- t.Fatalf("couldn't close holder: %v", err)
+ t.Fatalf("setting bit: %v", err)
}
- err = h.Open()
+ _, err = f.SetBit(rowID, columnID, nil)
if err != nil {
- t.Fatalf("couldn't open holder: %v", err)
- }
- err = h.Close()
- if err != nil {
- t.Fatalf("couldn't close holder: %v", err)
+ t.Fatalf("setting bit: %v", err)
+ }
+}
+
+func TestHolderOperatorProcess(t *testing.T) {
+ h, path, err := makeHolder()
+ if err != nil {
+ t.Fatalf("creating holder: %v", err)
+ }
+ defer os.RemoveAll(path)
+ defer h.Close()
+
+ // Write bits to separate indexes.
+ testSetBit(t, h, "i0", "f", 100, 200)
+ testSetBit(t, h, "i1", "f", 100, 200)
+ testSetBit(t, h, "i1", "f", 100, 12345678)
+
+ testOp := testHolderOperator{}
+ ctx := context.Background()
+ err = h.Process(ctx, &testOp)
+ if err != nil {
+ t.Fatalf("processing holder: %v", err)
+ }
+ expected := testHolderOperator{
+ indexSeen: 2, indexProcessed: 2,
+ fieldSeen: 2, fieldProcessed: 2,
+ viewSeen: 2, viewProcessed: 2,
+ fragmentSeen: 3, fragmentProcessed: 3,
+ }
+ if testOp != expected {
+ t.Fatalf("holder processor did not process as expected. expected %#v, got %#v", expected, testOp)
+ }
+}
+
+func TestHolderOperatorCancel(t *testing.T) {
+ h, path, err := makeHolder()
+ if err != nil {
+ t.Fatalf("creating holder: %v", err)
+ }
+ defer os.RemoveAll(path)
+ defer h.Close()
+
+ // Write bits to separate indexes.
+ testSetBit(t, h, "i0", "f", 100, 200)
+ testSetBit(t, h, "i1", "f", 100, 200)
+ testSetBit(t, h, "i1", "f", 100, 12345678)
+
+ // Here, we want to ensure that the operation gets cancelled
+ // successfully. In practice we expect it to process one fragment, then
+ // end up blocked on the waitHere, then get cancelled... But the
+ // waitHere blockage isn't really something holder.Process can do
+ // anything about, so we close the channel, so two fragments are
+ // processed. But in theory you could end up with only one fragment
+ // processed if this goroutine managed to cancel before the processor
+ // gets to the next fragment. Point is, it shouldn't hit all three,
+ // because the checks against the cancellation should fire before it
+ // gets there.
+ testOp := testHolderOperator{waitHere: make(chan struct{})}
+ ctx, cancel := context.WithCancel(context.Background())
+ done := make(chan struct{})
+ go func() {
+ err = h.Process(ctx, &testOp)
+ close(done)
+ }()
+ testOp.waitHere <- struct{}{}
+ cancel()
+ close(testOp.waitHere)
+ <-done
+ if err != context.Canceled {
+ t.Fatalf("processing holder: expected context.Canceled, got %v", err)
+ }
+ testOp.waitHere = nil
+ expected := testHolderOperator{
+ indexSeen: 2, indexProcessed: 2,
+ fieldSeen: 2, fieldProcessed: 2,
+ viewSeen: 2, viewProcessed: 2,
+ fragmentSeen: 3, fragmentProcessed: 3,
+ }
+ if testOp == expected {
+ t.Fatalf("holder processor did not cancel. expected something other than %#v", expected)
}
}
diff --git a/http/handler.go b/http/handler.go
index 67be9e297..06a62ebd7 100644
--- a/http/handler.go
+++ b/http/handler.go
@@ -221,6 +221,8 @@ func (h *Handler) populateValidators() {
h.validators["GetTransaction"] = queryValidationSpecRequired()
h.validators["PostTransaction"] = queryValidationSpecRequired()
h.validators["PostFinishTransaction"] = queryValidationSpecRequired()
+ h.validators["Inspect"] = queryValidationSpecRequired().Optional("indexes", "fields", "views", "shards", "checksum", "containers")
+
}
type contextKeyQuery int
@@ -352,6 +354,7 @@ func newRouter(handler *Handler) *mux.Router {
router.HandleFunc("/index/{index}/field/{field}/import-roaring/{shard}", handler.handlePostImportRoaring).Methods("POST").Name("PostImportRoaring")
router.HandleFunc("/index/{index}/query", handler.handlePostQuery).Methods("POST").Name("PostQuery")
router.HandleFunc("/info", handler.handleGetInfo).Methods("GET").Name("GetInfo")
+ router.HandleFunc("/inspect", handler.handleInspect).Methods("GET").Name("Inspect")
router.HandleFunc("/recalculate-caches", handler.handleRecalculateCaches).Methods("POST").Name("RecalculateCaches")
router.HandleFunc("/schema", handler.handleGetSchema).Methods("GET").Name("GetSchema")
router.HandleFunc("/schema", handler.handlePostSchema).Methods("POST").Name("PostSchema")
@@ -590,6 +593,37 @@ func (h *Handler) handleGetInfo(w http.ResponseWriter, r *http.Request) {
}
}
+func (h *Handler) handleInspect(w http.ResponseWriter, r *http.Request) {
+ if !validHeaderAcceptJSON(r.Header) {
+ http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable)
+ return
+ }
+ q := r.URL.Query()
+ _, checksum := q["checksum"]
+ _, containers := q["containers"]
+ req := pilosa.InspectRequest{
+ HolderFilterParams: pilosa.HolderFilterParams{
+ Indexes: q.Get("indexes"),
+ Fields: q.Get("fields"),
+ Views: q.Get("views"),
+ Shards: q.Get("shards"),
+ },
+ InspectRequestParams: pilosa.InspectRequestParams{
+ Checksum: checksum,
+ Containers: containers,
+ },
+ }
+ info, err := h.api.Inspect(r.Context(), &req)
+ if err != nil {
+ http.Error(w, fmt.Sprintf("inspect request: %v", err), http.StatusBadRequest)
+ return
+ }
+ w.Header().Set("Content-Type", "application/json")
+ if err := json.NewEncoder(w).Encode(info); err != nil {
+ h.logger.Printf("write inspect response error: %s", err)
+ }
+}
+
type getSchemaResponse struct {
Indexes []*pilosa.IndexInfo `json:"indexes"`
}
diff --git a/index.go b/index.go
index 481d22cf4..4a7ee3f5f 100644
--- a/index.go
+++ b/index.go
@@ -27,7 +27,6 @@ import (
"github.com/gogo/protobuf/proto"
"github.com/pilosa/pilosa/v2/internal"
- "github.com/pilosa/pilosa/v2/logger"
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pilosa/pilosa/v2/stats"
"github.com/pkg/errors"
@@ -46,9 +45,6 @@ type Index struct {
trackExistence bool
existenceFld *Field
- // Partitions used by translation.
- partitionN int
-
// Fields by name.
fields map[string]*Field
@@ -60,9 +56,6 @@ type Index struct {
broadcaster broadcaster
Stats stats.StatsClient
- logger logger.Logger
- snapshotQueue snapshotQueue
-
// Passed to field for foreign-index lookup.
holder *Holder
@@ -76,24 +69,23 @@ type Index struct {
}
// NewIndex returns a new instance of Index.
-func NewIndex(path, name string, partitionN int) (*Index, error) {
+func NewIndex(holder *Holder, path, name string) (*Index, error) {
err := validateName(name)
if err != nil {
return nil, errors.Wrap(err, "validating name")
}
return &Index{
- path: path,
- name: name,
- partitionN: partitionN,
- fields: make(map[string]*Field),
+ path: path,
+ name: name,
+ fields: make(map[string]*Field),
newAttrStore: newNopAttrStore,
columnAttrs: nopStore,
broadcaster: NopBroadcaster,
Stats: stats.NopStatsClient,
- logger: logger.NopLogger,
+ holder: holder,
trackExistence: true,
translateStores: make(map[int]TranslateStore),
@@ -155,18 +147,18 @@ func (i *Index) OpenWithTimestamp() error { return i.open(true) }
func (i *Index) open(withTimestamp bool) (err error) {
// Ensure the path exists.
- i.logger.Debugf("ensure index path exists: %s", i.path)
+ i.holder.Logger.Debugf("ensure index path exists: %s", i.path)
if err := os.MkdirAll(i.path, 0777); err != nil {
return errors.Wrap(err, "creating directory")
}
// Read meta file.
- i.logger.Debugf("load meta file for index: %s", i.name)
+ i.holder.Logger.Debugf("load meta file for index: %s", i.name)
if err := i.loadMeta(); err != nil {
return errors.Wrap(err, "loading meta file")
}
- i.logger.Debugf("open fields for index: %s", i.name)
+ i.holder.Logger.Debugf("open fields for index: %s", i.name)
if err := i.openFields(withTimestamp); err != nil {
return errors.Wrap(err, "opening fields")
}
@@ -181,15 +173,15 @@ func (i *Index) open(withTimestamp bool) (err error) {
return errors.Wrap(err, "opening attrstore")
}
- i.logger.Debugf("open translate store for index: %s", i.name)
+ i.holder.Logger.Debugf("open translate store for index: %s", i.name)
var g errgroup.Group
var mu sync.Mutex
- for partitionID := 0; partitionID < i.partitionN; partitionID++ {
+ for partitionID := 0; partitionID < i.holder.partitionN; partitionID++ {
partitionID := partitionID
g.Go(func() error {
- store, err := i.OpenTranslateStore(i.TranslateStorePath(partitionID), i.name, "", partitionID, i.partitionN)
+ store, err := i.OpenTranslateStore(i.TranslateStorePath(partitionID), i.name, "", partitionID, i.holder.partitionN)
if err != nil {
return errors.Wrapf(err, "opening index translate store: partition=%d", partitionID)
}
@@ -239,7 +231,7 @@ fileLoop:
defer func() {
<-indexQueue
}()
- i.logger.Debugf("open field: %s", fi.Name())
+ i.holder.Logger.Debugf("open field: %s", fi.Name())
mu.Lock()
fld, err := i.newField(i.fieldPath(filepath.Base(fi.Name())), filepath.Base(fi.Name()))
if withTimestamp {
@@ -257,7 +249,7 @@ fileLoop:
if err := fld.Open(); err != nil {
return fmt.Errorf("open field: name=%s, err=%s", fld.Name(), err)
}
- i.logger.Debugf("add field to index.fields: %s", fi.Name())
+ i.holder.Logger.Debugf("add field to index.fields: %s", fi.Name())
mu.Lock()
i.fields[fld.Name()] = fld
mu.Unlock()
@@ -512,17 +504,13 @@ func (i *Index) createField(name string, opt *FieldOptions) (*Field, error) {
}
func (i *Index) newField(path, name string) (*Field, error) {
- f, err := newField(path, i.name, name, OptFieldTypeDefault())
+ f, err := newField(i.holder, path, i.name, name, OptFieldTypeDefault())
if err != nil {
return nil, err
}
- f.logger = i.logger
f.Stats = i.Stats
f.broadcaster = i.broadcaster
f.rowAttrStore = i.newAttrStore(filepath.Join(f.path, ".data"))
- if i.snapshotQueue != nil {
- f.snapshotQueue = i.snapshotQueue
- }
f.OpenTranslateStore = i.OpenTranslateStore
return f, nil
}
diff --git a/index_internal_test.go b/index_internal_test.go
index 026607da4..5b5a5b5a3 100644
--- a/index_internal_test.go
+++ b/index_internal_test.go
@@ -25,7 +25,7 @@ func mustOpenIndex(opt IndexOptions) *Index {
if err != nil {
panic(err)
}
- index, err := NewIndex(path, "i", DefaultPartitionN)
+ index, err := NewIndex(NewHolder(1), path, "i")
if err != nil {
panic(err)
}
diff --git a/index_test.go b/index_test.go
index 63e9b6179..bfebee7f7 100644
--- a/index_test.go
+++ b/index_test.go
@@ -242,7 +242,7 @@ func TestIndex_InvalidName(t *testing.T) {
if err != nil {
panic(err)
}
- index, err := pilosa.NewIndex(path, "ABC", pilosa.DefaultPartitionN)
+ index, err := pilosa.NewIndex(pilosa.NewHolder(pilosa.DefaultPartitionN), path, "ABC")
if err == nil {
t.Fatalf("should have gotten an error on index name with caps")
}
diff --git a/license.exceptions b/license.exceptions
index 9f453efc4..ef44dcacc 100644
--- a/license.exceptions
+++ b/license.exceptions
@@ -4,7 +4,6 @@
./internal/private.pb.go
./internal/public.pb.go
./lru/lru.go
-./enterprise/enterprise.go
./roaring/btree.go
./roaring/btree_test.go
./proto/pilosa.pb.go
diff --git a/pilosa.go b/pilosa.go
index 53733d9ce..481c8ff07 100644
--- a/pilosa.go
+++ b/pilosa.go
@@ -33,9 +33,10 @@ var (
ErrForeignIndexNotFound = errors.New("foreign index not found")
// ErrFieldRequired is returned when no field is specified.
- ErrFieldRequired = errors.New("field required")
- ErrFieldExists = errors.New("field already exists")
- ErrFieldNotFound = errors.New("field not found")
+ ErrFieldRequired = errors.New("field required")
+ ErrColumnRequired = errors.New("column required")
+ ErrFieldExists = errors.New("field already exists")
+ ErrFieldNotFound = errors.New("field not found")
ErrBSIGroupNotFound = errors.New("bsigroup not found")
ErrBSIGroupExists = errors.New("bsigroup already exists")
diff --git a/roaring/container_stash.go b/roaring/container_stash.go
index 64a6d217e..a43b3eee8 100644
--- a/roaring/container_stash.go
+++ b/roaring/container_stash.go
@@ -45,35 +45,47 @@ type Container struct {
type containerFlags uint8
+var containerFlagStrings = [...]string{
+ "",
+ "mapped",
+ "frozen",
+ "frozen/mapped",
+ "pristine",
+ "pristine/mapped",
+ "pristine/frozen",
+ "pristine/frozen/mapped",
+}
+
+func (f containerFlags) String() string {
+ return containerFlagStrings[f&7]
+}
+
const (
flagMapped = containerFlags(1 << iota)
flagFrozen
+ flagPristine
)
func (c *Container) String() string {
if c == nil {
return ""
}
- froze := ""
- switch c.flags {
- case flagFrozen:
- froze = "frozen "
- case flagMapped:
- froze = "mapped "
- case flagFrozen | flagMapped:
- froze = "frozen/mapped"
+ var space, froze string
+ if c.flags != 0 {
+ space = " "
+ froze = c.flags.String()
}
switch c.typeID {
case containerArray:
- return fmt.Sprintf("<%sarray container, N=%d>", froze, c.N())
+ return fmt.Sprintf("<%s%sarray container, N=%d>", froze, space, c.N())
case containerBitmap:
- return fmt.Sprintf("<%sbitmap container, N=%d, len %dx uint64>",
- froze, c.N(), len(c.bitmap()))
+ return fmt.Sprintf("<%s%sbitmap container, N=%d, len %dx uint64>",
+ froze, space, c.N(), len(c.bitmap()))
case containerRun:
- return fmt.Sprintf("<%srun container, N=%d, len %dx interval>",
- froze, c.N(), len(c.runs()))
+ return fmt.Sprintf("<%s%srun container, N=%d, len %dx interval>",
+ froze, space, c.N(), len(c.runs()))
default:
- return fmt.Sprintf("", froze, c.typeID, c.N())
+ return fmt.Sprintf("", froze, space, c.typeID, c.N())
}
}
@@ -292,6 +304,7 @@ func (c *Container) unmapOrClone() *Container {
return c.Clone()
}
c.flags &^= flagMapped
+ c.flags &^= flagPristine
// mapped: we want to unmap the storage.
switch c.typeID {
case containerArray:
@@ -368,6 +381,7 @@ func (c *Container) setArrayMaybeCopy(array []uint16, doCopy bool) {
if len(array) > 1<<16 {
panic("impossibly large array")
}
+ c.flags &^= flagPristine
// array we can fit in data store:
if len(array) <= stashedArraySize {
copy(c.data[:stashedArraySize], array)
@@ -497,6 +511,7 @@ func (c *Container) setBitmap(bitmap []uint64) {
panic("illegal bitmap length")
}
c.pointer, c.len, c.cap = (*uint16)(unsafe.Pointer(&bitmap[0])), bitmapN, bitmapN
+ c.flags &^= flagPristine
}
// runs yields the data viewed as a slice of intervals.
@@ -531,6 +546,7 @@ func (c *Container) setRunsMaybeCopy(runs []interval16, doCopy bool) {
if len(runs) > 1<<15 {
panic("impossibly large run set")
}
+ c.flags &^= flagPristine
// array we can fit in data store:
if len(runs) <= stashedRunSize {
newRuns := (*[stashedRunSize]interval16)(unsafe.Pointer(&c.data))[:len(runs)]
diff --git a/roaring/roaring.go b/roaring/roaring.go
index f407b8f69..19dd9ec83 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -1721,11 +1721,14 @@ func (b *Bitmap) writeToUnoptimized(w io.Writer) (n int64, err error) {
// bitmap and yield information about containers, including type, size, and
// the location of their data structures.
type roaringIterator interface {
+ // Len reports the number of containers total.
+ Len() (count int64)
// Next yields the information about the next container
Next() (key uint64, cType byte, n int, length int, pointer *uint16, err error)
// Remaining yields the bytes left over past the end of the roaring data,
- // which is typically an ops log in our case.
- Remaining() []byte
+ // which is typically an ops log in our case, and also its offset in case
+ // we need to talk about truncation.
+ Remaining() ([]byte, int64)
}
// baseRoaringIterator holds values used by both Pilosa and official Roaring
@@ -1884,11 +1887,16 @@ func (r *baseRoaringIterator) Done(err error) {
r.currentDataOffset = 0
}
-func (r *baseRoaringIterator) Remaining() []byte {
+// Len() indicates the total number of containers the iterator expects to have.
+func (r *baseRoaringIterator) Len() int64 {
+ return r.keys
+}
+
+func (r *baseRoaringIterator) Remaining() ([]byte, int64) {
if r.lastDataOffset == 0 {
- return nil
+ return nil, 0
}
- return r.data[r.lastDataOffset:]
+ return r.data[r.lastDataOffset:], r.lastDataOffset
}
func (r *pilosaRoaringIterator) Next() (key uint64, cType byte, n int, length int, pointer *uint16, err error) {
@@ -2435,19 +2443,24 @@ func (b *Bitmap) roaringSize() (int64, int64) {
}
// Info returns stats for the bitmap.
-func (b *Bitmap) Info() bitmapInfo {
- info := bitmapInfo{
- OpN: b.opN,
- Ops: b.ops,
- Containers: make([]containerInfo, 0, b.Containers.Size()),
+func (b *Bitmap) Info(includeContainers bool) BitmapInfo {
+ info := BitmapInfo{
+ OpN: b.opN,
+ Ops: b.ops,
+ ContainerCount: b.Containers.Size(),
+ }
+ if includeContainers {
+ info.Containers = make([]ContainerInfo, 0, info.ContainerCount)
}
-
citer, _ := b.Containers.Iterator(0)
for citer.Next() {
k, c := citer.Value()
ci := c.info()
ci.Key = k
- info.Containers = append(info.Containers, ci)
+ info.BitCount += uint64(c.N())
+ if includeContainers {
+ info.Containers = append(info.Containers, ci)
+ }
}
return info
}
@@ -2504,11 +2517,16 @@ func (b *Bitmap) Flip(start, end uint64) *Bitmap {
return result
}
-// bitmapInfo represents a point-in-time snapshot of bitmap stats.
-type bitmapInfo struct {
- OpN int
- Ops int
- Containers []containerInfo
+// BitmapInfo represents a point-in-time snapshot of bitmap stats.
+type BitmapInfo struct {
+ OpN int
+ Ops int
+ OpDetails []OpInfo `json:"OpDetails,omitempty"`
+ BitCount uint64
+ ContainerCount int
+ Containers []ContainerInfo `json:"Containers,omitempty"` // The containers found in the bitmap originally
+ OpContainers []ContainerInfo `json:"OpContainers,omitempty"` // The containers resulting from ops log changes.
+ From, To uintptr // if set, indicates the address range used when unpacking
}
// Iterator represents an iterator over a Bitmap.
@@ -3704,13 +3722,14 @@ func (c *Container) size() int {
}
// info returns the current stats about the container.
-func (c *Container) info() containerInfo {
- info := containerInfo{N: c.N()}
+func (c *Container) info() ContainerInfo {
+ info := ContainerInfo{N: c.N(), Mapped: c.Mapped()}
if c == nil {
info.Type = "nil"
info.Alloc = 0
return info
}
+ info.Flags = c.flags.String()
if c.isArray() {
info.Type = "array"
@@ -3722,17 +3741,7 @@ func (c *Container) info() containerInfo {
info.Type = "bitmap"
info.Alloc = len(c.bitmap()) * 8 // sizeof(uint64)
}
-
- if c.Mapped() {
- if c.isArray() {
- info.Pointer = unsafe.Pointer(&c.array()[0])
- } else if c.isRun() {
- info.Pointer = unsafe.Pointer(&c.runs()[0])
- } else {
- info.Pointer = unsafe.Pointer(&c.bitmap()[0])
- }
- }
-
+ info.Pointer = uintptr(unsafe.Pointer(c.pointer))
return info
}
@@ -3798,13 +3807,15 @@ func (c *Container) bitmapRepair() {
c.setN(n)
}
-// containerInfo represents a point-in-time snapshot of container stats.
-type containerInfo struct {
- Key uint64 // container key
- Type string // container type (array, bitmap, or run)
- N int32 // number of bits
- Alloc int // memory used
- Pointer unsafe.Pointer // offset within the mmap
+// ContainerInfo represents a point-in-time snapshot of container stats.
+type ContainerInfo struct {
+ Key uint64 // container key
+ Type string // container type (array, bitmap, or run)
+ Flags string // flag state
+ N int32 // number of bits
+ Alloc int // memory used
+ Pointer uintptr // address
+ Mapped bool // whether this container thinks it is mmapped
}
// flip returns a new container containing the inverse of all
@@ -5458,6 +5469,15 @@ const (
opTypeRemoveRoaring = opType(5)
)
+var opTypes = []string{
+ "add",
+ "remove",
+ "addN",
+ "removeN",
+ "addRoaring",
+ "removeRoaring",
+}
+
// op represents an operation on the bitmap.
type op struct {
typ opType
@@ -5467,6 +5487,24 @@ type op struct {
roaring []byte
}
+// OpInfo is a description of an op.
+type OpInfo struct {
+ Type string
+ OpN int
+ Size int
+}
+
+func (op *op) info() (info OpInfo) {
+ if int(op.typ) < len(opTypes) {
+ info.Type = opTypes[op.typ]
+ } else {
+ info.Type = fmt.Sprintf("unknown-type-%d", op.typ)
+ }
+ info.OpN = op.opN
+ info.Size = op.size()
+ return info
+}
+
// apply executes the operation against a bitmap.
func (op *op) apply(b *Bitmap) (changed bool) {
switch op.typ {
diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go
index 55395ea73..7dda0f5c6 100644
--- a/roaring/roaring_test.go
+++ b/roaring/roaring_test.go
@@ -1733,7 +1733,7 @@ type benchmarkSampleData struct {
var sampleData benchmarkSampleData
func isAllType(b *roaring.Bitmap, typ string) bool {
- bi := b.Info()
+ bi := b.Info(true)
for _, c := range bi.Containers {
if c.Type != typ {
return false
diff --git a/roaring/unmarshal_binary.go b/roaring/unmarshal_binary.go
index c9071b1ca..aeab0ef2f 100644
--- a/roaring/unmarshal_binary.go
+++ b/roaring/unmarshal_binary.go
@@ -1,4 +1,4 @@
-// Copyright 2017 Pilosa Corp.
+// Copyright 2019 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -15,232 +15,231 @@
package roaring
import (
- "encoding/binary"
- "fmt"
+ "errors"
+ "io"
"unsafe"
-
- "github.com/pkg/errors"
)
-// UnmarshalBinary decodes b from a binary-encoded byte slice. data can be in
-// either official roaring format or Pilosa's roaring format.
-func (b *Bitmap) UnmarshalBinary(data []byte) error {
+// UnmarshalBinary reads Pilosa's format, or upstream roaring (mostly;
+// it may not handle some edge cases), and decodes them into the given
+// bitmap, replacing the existing contents.
+func (b *Bitmap) UnmarshalBinary(data []byte) (err error) {
if data == nil {
- // Nothing to unmarshal
- return nil
- }
- statsHit("Bitmap/UnmarshalBinary")
- // reset ops/opN since we're reading new data.
- b.ops = 0
- b.opN = 0
- fileMagic := uint32(binary.LittleEndian.Uint16(data[0:2]))
- if fileMagic == MagicNumber { // if pilosa roaring
- return errors.Wrap(b.unmarshalPilosaRoaring(data), "unmarshaling as pilosa roaring")
+ return errors.New("no roaring bitmap provided")
}
+ var itr roaringIterator
+ var itrKey uint64
+ var itrCType byte
+ var itrN int
+ var itrLen int
+ var itrPointer *uint16
+ var itrErr error
- keyN, containerTyper, header, pos, haveRuns, err := readOfficialHeader(data)
+ itr, err = newRoaringIterator(data)
if err != nil {
- return errors.Wrap(err, "reading roaring header")
+ return err
}
- // Only the Pilosa roaring format has flags. The official Roaring format
- // hasn't got space in its header for flags.
- b.Flags = 0
-
- b.Containers.ResetN(int(keyN))
- // Descriptive header section: Read container keys and cardinalities.
- for i, buf := uint(0), data[header:]; i < uint(keyN); i, buf = i+1, buf[4:] {
- card := int(binary.LittleEndian.Uint16(buf[2:4])) + 1
- b.Containers.PutContainerValues(
- uint64(binary.LittleEndian.Uint16(buf[0:2])),
- containerTyper(i, card), /// container type voodo with isRunBitmap
- card,
- true)
+ if itr == nil {
+ return errors.New("failed to create roaring iterator, but don't know why")
}
- // Read container offsets and attach data.
- if haveRuns {
- err := readWithRuns(b, data, pos, keyN)
- if err != nil {
- return errors.Wrap(err, "reading offsets from official roaring format")
+ b.Containers.Reset()
+
+ itrKey, itrCType, itrN, itrLen, itrPointer, itrErr = itr.Next()
+ for itrErr == nil {
+ newC := &Container{
+ typeID: itrCType,
+ n: int32(itrN),
+ len: int32(itrLen),
+ cap: int32(itrLen),
+ pointer: itrPointer,
+ flags: flagMapped,
}
- } else {
- err := readOffsets(b, data, pos, keyN)
- if err != nil {
- return errors.Wrap(err, "reading official roaring format")
+ if !b.preferMapping {
+ newC.unmapOrClone()
}
+ b.Containers.Put(itrKey, newC)
+ itrKey, itrCType, itrN, itrLen, itrPointer, itrErr = itr.Next()
}
- return nil
-}
-
-func readOffsets(b *Bitmap, data []byte, pos int, keyN uint32) error {
-
- citer, _ := b.Containers.Iterator(0)
- for i, buf := 0, data[pos:]; i < int(keyN); i, buf = i+1, buf[4:] {
- // Verify the offset is fully formed
- if len(buf) < 4 {
- return fmt.Errorf("insufficient data for offsets: len=%d", len(buf))
- }
- offset := binary.LittleEndian.Uint32(buf[0:4])
- // Verify the offset is within the bounds of the input data.
- if int(offset) >= len(data) {
- return fmt.Errorf("offset out of bounds: off=%d, len=%d", offset, len(data))
- }
-
- // Map byte slice directly to the container data.
- citer.Next()
- k, c := citer.Value()
- if !c.Mapped() {
- fmt.Printf("inexplicable: container %d (%d/%d) doesn't think it's mapped. fixing that.\n",
- k, i, keyN)
- c.setMapped(true)
- }
- switch c.typ() {
- case containerArray:
- c.setArray((*[0xFFFFFFF]uint16)(unsafe.Pointer(&data[offset]))[:c.N():c.N()])
- case containerBitmap:
- c.setBitmap((*[0xFFFFFFF]uint64)(unsafe.Pointer(&data[offset]))[:bitmapN:bitmapN])
- default:
- return fmt.Errorf("unsupported container type %d", c.typ())
- }
- }
- return nil
-}
-
-func readWithRuns(b *Bitmap, data []byte, pos int, keyN uint32) error {
- if len(data) < pos+runCountHeaderSize {
- return fmt.Errorf("insufficient data for offsets(run): len=%d", len(data))
- }
- citer, _ := b.Containers.Iterator(0)
- for i := 0; i < int(keyN); i++ {
- citer.Next()
- k, c := citer.Value()
- if !c.Mapped() {
- fmt.Printf("inexplicable: container %d (%d/%d) doesn't think it's mapped. fixing that.\n",
- k, i, keyN)
- c.setMapped(true)
- }
- switch c.typ() {
- case containerRun:
- runCount := binary.LittleEndian.Uint16(data[pos : pos+runCountHeaderSize])
- c.setRuns((*[0xFFFFFFF]interval16)(unsafe.Pointer(&data[pos+runCountHeaderSize]))[:runCount:runCount])
- runs := c.runs()
-
- for o := range runs { // must convert from start:length to start:end :(
- runs[o].last = runs[o].start + runs[o].last
- }
- pos += int((runCount * interval16Size) + runCountHeaderSize)
- case containerArray:
- c.setArray((*[0xFFFFFFF]uint16)(unsafe.Pointer(&data[pos]))[:c.N():c.N()])
- pos += int(c.N() * 2)
- case containerBitmap:
- c.setBitmap((*[0xFFFFFFF]uint64)(unsafe.Pointer(&data[pos]))[:bitmapN:bitmapN])
- pos += bitmapN * 8
- }
- }
- return nil
-}
-
-func (b *Bitmap) unmarshalPilosaRoaring(data []byte) error {
- if len(data) < headerBaseSize {
- return errors.New("data too small")
- }
-
- // Verify the first two bytes are a valid MagicNumber, and second two bytes match current storageVersion.
- fileMagic := uint32(binary.LittleEndian.Uint16(data[0:2]))
- fileVersion := uint32(data[2])
- b.Flags = data[3]
- if fileMagic != MagicNumber {
- return fmt.Errorf("invalid roaring file, magic number %v is incorrect", fileMagic)
- }
-
- if fileVersion != storageVersion {
- return fmt.Errorf("wrong roaring version, file is v%d, server requires v%d", fileVersion, storageVersion)
- }
-
- // Read key count in bytes sizeof(cookie)+sizeof(flag):(sizeof(cookie)+sizeof(uint32)).
- keyN := binary.LittleEndian.Uint32(data[3+1 : 8])
- if int64(len(data)) < headerBaseSize+int64(keyN)*12 {
- return fmt.Errorf("insufficient data for header + offsets: key-cardinality not provided for %d containers", keyN)
- }
-
- headerSize := headerBaseSize
- b.Containers.ResetN(int(keyN))
- // Descriptive header section: Read container keys and cardinalities.
- for i, buf := 0, data[headerSize:]; i < int(keyN); i, buf = i+1, buf[12:] {
- b.Containers.PutContainerValues(
- binary.LittleEndian.Uint64(buf[0:8]),
- byte(binary.LittleEndian.Uint16(buf[8:10])),
- int(binary.LittleEndian.Uint16(buf[10:12]))+1,
- true)
- }
- opsOffset := int64(headerSize) + int64(keyN)*12
-
- // Read container offsets and attach data.
- citer, _ := b.Containers.Iterator(0)
- // if you have enough containers that the *headers alone* exceed 4GB, we
- // need to start with a higher cycle offset.
- cycleOffset := opsOffset &^ ((1 << 32) - 1)
- prevOffset32 := uint32(opsOffset)
- for i, buf := 0, data[opsOffset:]; i < int(keyN); i, buf = i+1, buf[4:] {
- offset32 := binary.LittleEndian.Uint32(buf[0:4])
- if offset32 < prevOffset32 {
- cycleOffset += (1 << 32)
- }
- prevOffset32 = offset32
- offset := int64(offset32) + cycleOffset
- // Verify the offset is within the bounds of the input data.
- if offset >= int64(len(data)) {
- return fmt.Errorf("offset out of bounds: off=%d, len=%d", offset, len(data))
- }
-
- // Map byte slice directly to the container data.
- citer.Next()
- k, c := citer.Value()
-
- // this shouldn't happen, since we don't normally store nils.
- if c == nil {
- continue
- }
- if !c.Mapped() {
- fmt.Printf("inexplicable: container %d (%d/%d) doesn't think it's mapped. fixing that.\n",
- k, i, keyN)
- c.setMapped(true)
- }
- switch c.typ() {
- case containerRun:
- runCount := binary.LittleEndian.Uint16(data[offset : offset+runCountHeaderSize])
- c.setRuns((*[0xFFFFFFF]interval16)(unsafe.Pointer(&data[offset+runCountHeaderSize]))[:runCount:runCount])
- opsOffset = offset + runCountHeaderSize + int64(len(c.runs()))*interval16Size
- case containerArray:
- c.setArray((*[0xFFFFFFF]uint16)(unsafe.Pointer(&data[offset]))[:c.N():c.N()])
- opsOffset = offset + int64(len(c.array()))*2 // sizeof(uint32)
- case containerBitmap:
- c.setBitmap((*[0xFFFFFFF]uint64)(unsafe.Pointer(&data[offset]))[:bitmapN:bitmapN])
- opsOffset = offset + int64(len(c.bitmap()))*8 // sizeof(uint64)
- }
+ // note: if we get a non-EOF err, it's possible that we made SOME
+ // changes but didn't log them. I don't have a good solution to this.
+ if itrErr != io.EOF {
+ return itrErr
}
// Read ops log until the end of the file.
- buf := data[opsOffset:]
-
+ b.ops = 0
+ b.opN = 0
+ buf, lastValidOffset := itr.Remaining()
for {
// Exit when there are no more ops to parse.
if len(buf) == 0 {
break
}
+
// Unmarshal the op and apply it.
var opr op
if err := opr.UnmarshalBinary(buf); err != nil {
- return newFileShouldBeTruncatedError(err, int64(opsOffset))
+ return newFileShouldBeTruncatedError(err, int64(lastValidOffset))
}
+
opr.apply(b)
+
// Increase the op count.
b.ops++
b.opN += opr.count()
- opsOffset += int64(opr.size())
- // Move the buffer forward.
- buf = data[opsOffset:]
- }
+ // Move the buffer forward.
+ opSize := opr.size()
+ buf = buf[opSize:]
+ lastValidOffset += int64(opSize)
+ }
return nil
}
+
+// InspectBinary reads a roaring bitmap, plus a possible ops log,
+// and reports back on the contents, including distinguishing between
+// the original ops log and the post-ops-log contents.
+func InspectBinary(data []byte, mapped bool, info *BitmapInfo) (b *Bitmap, mappedAny bool, err error) {
+ b = NewFileBitmap()
+ b.PreferMapping(mapped)
+ if data == nil {
+ return b, mappedAny, errors.New("no roaring bitmap provided")
+ }
+ var itr roaringIterator
+ var itrKey uint64
+ var itrCType byte
+ var itrN int
+ var itrLen int
+ var itrPointer *uint16
+ var itrErr error
+
+ itr, err = newRoaringIterator(data)
+ if err != nil {
+ return b, mappedAny, err
+ }
+ if itr == nil {
+ return b, mappedAny, errors.New("failed to create roaring iterator, but don't know why")
+ }
+ keys := itr.Len()
+ info.Containers = make([]ContainerInfo, 0, keys)
+
+ itrKey, itrCType, itrN, itrLen, itrPointer, itrErr = itr.Next()
+ for itrErr == nil {
+ newC := &Container{
+ typeID: itrCType,
+ n: int32(itrN),
+ len: int32(itrLen),
+ cap: int32(itrLen),
+ pointer: itrPointer,
+ flags: flagMapped,
+ }
+ if !mapped {
+ newC.unmapOrClone()
+ }
+ newC.flags |= flagPristine
+ if newC.flags&flagMapped != 0 {
+ mappedAny = true
+ }
+ var size int
+ b.Containers.Put(itrKey, newC)
+ switch itrCType {
+ case containerArray:
+ size = int(newC.n) * 2
+ case containerBitmap:
+ size = 8192
+ case containerRun:
+ size = itrLen*interval16Size + runCountHeaderSize
+ }
+ info.Containers = append(info.Containers, ContainerInfo{
+ N: newC.n,
+ Mapped: newC.flags&flagMapped != 0,
+ Type: containerTypeNames[itrCType],
+ Alloc: size,
+ Pointer: uintptr(unsafe.Pointer(newC.pointer)),
+ Key: itrKey,
+ Flags: newC.flags.String(),
+ })
+ info.ContainerCount++
+ info.BitCount += uint64(newC.n)
+ itrKey, itrCType, itrN, itrLen, itrPointer, itrErr = itr.Next()
+ }
+ // note: if we get a non-EOF err, it's possible that we made SOME
+ // changes but didn't log them. I don't have a good solution to this.
+ if itrErr != io.EOF {
+ return b, mappedAny, itrErr
+ }
+ // stash pointer ranges
+ info.From = uintptr(unsafe.Pointer(&data[0]))
+ info.To = info.From + uintptr(len(data))
+
+ // Read ops log until the end of the file.
+ b.ops = 0
+ b.opN = 0
+ buf, lastValidOffset := itr.Remaining()
+ // if there's no ops log, we're done and can just return the
+ // info so far.
+ if len(buf) == 0 {
+ return b, mappedAny, err
+ }
+ for {
+ // Exit when there are no more ops to parse.
+ if len(buf) == 0 {
+ break
+ }
+
+ // Unmarshal the op and apply it.
+ var opr op
+ if err = opr.UnmarshalBinary(buf); err != nil {
+ // we break out here, but we continue on to
+ // return the bitmap as-is, along with data about
+ // it, and the error. this lets us share the
+ // "is anything mapped" check with that code.
+ break
+ }
+ opr.apply(b)
+
+ // Increase the op count.
+ if info != nil {
+ info.Ops++
+ info.OpN += opr.count()
+ info.OpDetails = append(info.OpDetails, opr.info())
+ }
+ // Move the buffer forward.
+ opSize := opr.size()
+ buf = buf[opSize:]
+ lastValidOffset += int64(opSize)
+ }
+ citer, _ := b.Containers.Iterator(0)
+ // it's possible the ops log unmapped every mapped container, so we recheck.
+ mappedAny = false
+ if info == nil {
+ for citer.Next() {
+ _, c := citer.Value()
+ if c.Mapped() {
+ mappedAny = true
+ break
+ }
+ }
+ return b, mappedAny, err
+ }
+ // now we want to compute the actual container and bit counts after
+ // ops, and create a report of just the containers which got changed.
+ info.ContainerCount = 0
+ info.BitCount = 0
+ for citer.Next() {
+ k, c := citer.Value()
+ if c.Mapped() {
+ mappedAny = true
+ }
+ info.ContainerCount++
+ info.BitCount += uint64(c.N())
+ if c.flags&flagPristine != 0 {
+ continue
+ }
+ ci := c.info()
+ ci.Key = k
+ info.OpContainers = append(info.OpContainers, ci)
+ }
+ return b, mappedAny, err
+}
diff --git a/server.go b/server.go
index 131a6971c..0afd2fa43 100644
--- a/server.go
+++ b/server.go
@@ -66,9 +66,10 @@ type Server struct { // nolint: maligned
extensions []*ext.ExtensionInfo
// External
- systemInfo SystemInfo
- gcNotifier GCNotifier
- logger logger.Logger
+ systemInfo SystemInfo
+ gcNotifier GCNotifier
+ logger logger.Logger
+ snapshotQueue SnapshotQueue
nodeID string
uri URI
@@ -533,6 +534,9 @@ func (s *Server) UpAndDown() error {
func (s *Server) Open() error {
s.logger.Printf("open server")
+ // Start background monitoring.
+ s.snapshotQueue = newSnapshotQueue(10, 2, s.logger)
+
// Log startup
err := s.holder.logStartup()
if err != nil {
@@ -560,6 +564,9 @@ func (s *Server) Open() error {
if err := s.holder.Open(); err != nil {
return errors.Wrap(err, "opening Holder")
}
+ // bring up the background tasks for the holder.
+ s.holder.SnapshotQueue = s.snapshotQueue
+ s.holder.Activate()
if err := s.cluster.setNodeState(nodeStateReady); err != nil {
return errors.Wrap(err, "setting nodeState")
}
@@ -571,7 +578,6 @@ func (s *Server) Open() error {
// buffered channel.
s.cluster.listenForJoins()
- // Start background monitoring.
s.wg.Add(3)
go func() { defer s.wg.Done(); s.monitorAntiEntropy() }()
go func() { defer s.wg.Done(); s.monitorRuntime() }()
@@ -596,6 +602,11 @@ func (s *Server) Close() error {
if s.holder != nil {
errh = s.holder.Close()
}
+ if s.snapshotQueue != nil {
+ s.holder.SnapshotQueue = nil
+ s.snapshotQueue.Stop()
+ s.snapshotQueue = nil
+ }
// prefer to return holder error over cluster
// error. This order is somewhat arbitrary. It would be better if we had
// some way to combine all the errors, but probably not important enough to
diff --git a/server/enterprise.go b/server/enterprise.go
deleted file mode 100644
index 3db4a5a8b..000000000
--- a/server/enterprise.go
+++ /dev/null
@@ -1,21 +0,0 @@
-// Copyright 2017 Pilosa Corp.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-// +build enterprise
-
-package server
-
-import (
- _ "github.com/pilosa/pilosa/v2/enterprise"
-)
diff --git a/server/server.go b/server/server.go
index 614cfd1ab..f995b7ac2 100644
--- a/server/server.go
+++ b/server/server.go
@@ -238,11 +238,7 @@ func (m *Command) SetupServer() error {
return errors.Wrap(err, "setting up logger")
}
- productName := "Pilosa"
- if pilosa.EnterpriseEnabled {
- productName += " Enterprise"
- }
- m.logger.Printf("%s %s, build time %s\n", productName, pilosa.Version, pilosa.BuildTime)
+ m.logger.Printf("%s", pilosa.VersionInfo())
// validateAddrs sets the appropriate values for Bind and Advertise
// based on the inputs. It is not responsible for applying defaults, although
diff --git a/snapshotqueue.go b/snapshotqueue.go
index 373e5c538..1bdcdb7a9 100644
--- a/snapshotqueue.go
+++ b/snapshotqueue.go
@@ -15,7 +15,10 @@
package pilosa
import (
+ "context"
"fmt"
+ "io"
+ "math/bits"
"os"
"sync"
"sync/atomic"
@@ -39,12 +42,22 @@ import (
// Await, Enqueue, and Immediate should be called only with the fragment lock
// held.
//
-// ScanHolder spawns a new goroutine. You don't need to use `go` on it.
-type snapshotQueue interface {
+// If you create a queue, it should get stopped at some point. The
+// atomicSnapshotQueue implementation used as defaultSnapshotQueue has
+// a Start function which will tell you whether it actually started a
+// queue. This logic exists because in a normal server case, you probably
+// want the queue to be shut down as part of server shutdown, but if you're
+// running cluster tests, you probably want to start and shop the queue as
+// part of the test, not stop it when any server terminates.
+//
+// It's less likely to be desireable to start/stop individual queues,
+// because fragments use the defaultSnapshotQueue anyway. This design
+// needs revisiting.
+type SnapshotQueue interface {
Immediate(*fragment) error
Enqueue(*fragment)
Await(*fragment) error
- ScanHolder(*Holder)
+ ScanHolder(*Holder, chan struct{})
Stop()
}
@@ -53,7 +66,8 @@ type snapshotQueue interface {
type queuelessSnapshotQueue struct{}
func (q *queuelessSnapshotQueue) Enqueue(f *fragment) {
- _ = f.snapshot()
+ // We don't actually try to enqueue the snapshot; it breaks things
+ // if a snapshot gets caused during a transaction.
}
func (q *queuelessSnapshotQueue) Await(f *fragment) error {
@@ -64,20 +78,27 @@ func (q *queuelessSnapshotQueue) Immediate(f *fragment) error {
return f.snapshot()
}
-func (q *queuelessSnapshotQueue) ScanHolder(h *Holder) {
+func (q *queuelessSnapshotQueue) ScanHolder(h *Holder, done chan struct{}) {
}
func (q *queuelessSnapshotQueue) Stop() {
}
-// defaultSnapshotQueue is the fallback to use if none is available,
-// and currently uses queueless -- it runs all snapshots immediately.
-var defaultSnapshotQueue *queuelessSnapshotQueue
+var defaultSnapshotQueue = &queuelessSnapshotQueue{}
// newSnapshotQueue makes a new snapshot queue, of depth N, with
// w worker threads.
-func newSnapshotQueue(n int, w int, l logger.Logger) snapshotQueue {
- sq := prioritySnapshotQueue{normal: make(chan snapshotRequest, n), urgent: make(chan snapshotRequest), background: make(chan snapshotRequest), done: make(chan struct{}), logger: l}
+func newSnapshotQueue(n int, w int, l logger.Logger) SnapshotQueue {
+ ctx, cancel := context.WithCancel(context.Background())
+ sq := prioritySnapshotQueue{
+ normal: make(chan snapshotRequest, n),
+ urgent: make(chan snapshotRequest),
+ background: make(chan snapshotRequest),
+ ctx: ctx,
+ cancel: cancel,
+ maxOpN: 10000,
+ logger: l,
+ }
if sq.logger == nil {
sq.logger = logger.NewStandardLogger(os.Stderr)
}
@@ -105,50 +126,52 @@ type prioritySnapshotQueue struct {
urgent chan snapshotRequest
normal chan snapshotRequest
background chan snapshotRequest
- done chan struct{}
+ ctx context.Context
+ cancel context.CancelFunc
mu sync.RWMutex
scanWG, workerWG sync.WaitGroup
+ maxOpN int
+ observedOpN [16]uint32
stats struct {
- enqueued uint64
- skipped uint64
+ enqueued uint32
+ skipped uint32
}
}
func (sq *prioritySnapshotQueue) spawnWorkers(w int) {
sq.mu.Lock()
defer sq.mu.Unlock()
- if sq.done == nil {
- sq.logger.Printf("prioritySnapshotQueue worker: no done channel, already done?")
+ if sq.ctx.Err() != nil {
+ sq.logger.Printf("prioritySnapshotQueue worker: already done")
return
}
sq.workerWG.Add(w)
for i := 0; i < w; i++ {
- go sq.worker(sq.urgent, sq.normal, sq.background, sq.done)
+ go sq.worker(sq.ctx, sq.urgent, sq.normal, sq.background)
}
}
-func (sq *prioritySnapshotQueue) worker(urgent, normal, background chan snapshotRequest, done chan struct{}) {
- // We don't want a race condition on these. If they're non-nil when
- // we get them, they should get closed at some point. If done is
- // already nil, we shouldn't do anything.
+func (sq *prioritySnapshotQueue) worker(ctx context.Context, urgent, normal, background chan snapshotRequest) {
defer sq.workerWG.Done()
+ done := ctx.Done()
ok := true
var req snapshotRequest
for ok {
req.frag = nil
-
select {
+ case _, ok = <-done:
case req, ok = <-urgent:
default:
select {
+ case _, ok = <-done:
case req, ok = <-urgent:
case req, ok = <-normal:
default:
select {
+ case _, ok = <-done:
case req, ok = <-urgent:
case req, ok = <-normal:
case req, ok = <-background:
- case _, ok = <-done:
}
}
}
@@ -182,17 +205,18 @@ func (sq *prioritySnapshotQueue) process(req snapshotRequest) {
func (sq *prioritySnapshotQueue) Stop() {
sq.mu.Lock()
defer sq.mu.Unlock()
- close(sq.done)
+ sq.cancel()
// scanners need to be done before we close the other channels.
sq.scanWG.Wait()
- sq.done = nil
close(sq.normal)
sq.normal = nil
close(sq.urgent)
sq.urgent = nil
close(sq.background)
sq.background = nil
- if sq.stats.skipped > 0 || sq.stats.enqueued > 1 {
+ enqueued := atomic.LoadUint32(&sq.stats.enqueued)
+ skipped := atomic.LoadUint32(&sq.stats.skipped)
+ if skipped > 0 || enqueued > 1 {
sq.logger.Printf("snapshot queue: enqueued %d, skipped %d\n", sq.stats.enqueued, sq.stats.skipped)
}
}
@@ -203,6 +227,7 @@ func (sq *prioritySnapshotQueue) Enqueue(f *fragment) {
if f.snapshotPending {
return
}
+ sq.observeOpN(uint32(f.opN))
sq.mu.RLock()
defer sq.mu.RUnlock()
if sq.normal == nil {
@@ -217,10 +242,10 @@ func (sq *prioritySnapshotQueue) Enqueue(f *fragment) {
// try to enqueue snapshot
select {
case sq.normal <- snapshotRequest{frag: f, when: time.Now()}:
- atomic.AddUint64(&sq.stats.enqueued, 1)
+ atomic.AddUint32(&sq.stats.enqueued, 1)
return
default:
- atomic.AddUint64(&sq.stats.skipped, 1)
+ atomic.AddUint32(&sq.stats.skipped, 1)
f.snapshotPending = false
return
}
@@ -230,6 +255,11 @@ func (sq *prioritySnapshotQueue) Enqueue(f *fragment) {
// held. Await waits on a condition variable inside f, associated with the
// fragment's lock, so this does not conflict with the lock being used for
// snapshots.
+//
+// Note that workers don't stop just because the queue's been stopped; only
+// the background scanner is stopped. So an Await shouldn't block forever
+// even if the queue gets shut down. If you're reading this, possibly that
+// analysis is incorrect.
func (sq *prioritySnapshotQueue) Await(f *fragment) (err error) {
for f.snapshotPending {
f.snapshotCond.Wait()
@@ -252,6 +282,7 @@ func (sq *prioritySnapshotQueue) Immediate(f *fragment) error {
return errors.New("requested immediate snapshot after snapshot queue was closed")
}
f.snapshotPending = true
+ sq.observeOpN(uint32(f.opN))
req := snapshotRequest{frag: f, when: time.Now()}
// if the fragment was already in the work queue, it's *possible*
// that the only available worker just picked it off the queue, and
@@ -268,141 +299,202 @@ func (sq *prioritySnapshotQueue) Immediate(f *fragment) error {
return sq.Await(f)
}
-// needsSnapshot determines whether a fragment probably wants snapshotting.
-// Specifically, it looks for fragments not already marked to receive
-// snapshots, but which have a high enough opN to justify a snapshot. This
-// is only used from the background scan.
-func (sq *prioritySnapshotQueue) needsSnapshot(f *fragment) bool {
- if f == nil {
- return false
- }
- f.mu.Lock()
- defer f.mu.Unlock()
- if f.snapshotPending {
- return false
- }
- if f.opN > f.MaxOpN {
- return true
- }
- return false
-}
-
// ScanHolder spawns a goroutine which iterates through the holder's
// indexes/fields/views/fragments, looking for fragments which have OpN
// high enough to justify a snapshot but don't seem to have one pending.
// It then dumps these in the low priority background queue.
-func (sq *prioritySnapshotQueue) ScanHolder(h *Holder) {
+func (sq *prioritySnapshotQueue) ScanHolder(h *Holder, done chan struct{}) {
sq.mu.Lock()
sq.scanWG.Add(1)
- go sq.scanHolderWorker(h, sq.background, sq.done)
+ go sq.scanHolderWorker(h, sq.background, done)
sq.mu.Unlock()
}
+// observeOpN reports that a given value of opN was "observed", meaning,
+// we encountered a fragment which had that value. This happens for every
+// enqueue/immediate, including enqueue attempts which fail to actually
+// enter the queue, and it also happens for fragments noticed by the background
+// scan but which don't have high enough opN to trigger a snapshot.
+func (sq *prioritySnapshotQueue) observeOpN(n uint32) {
+ // aka "log2(n) + 1", or 0 for n==0
+ pow2 := 32 - bits.LeadingZeros32(n)
+ // 15 == 16384. Our usual fragment maxOpN is 10k, so most fragments
+ // should end up in the 8k-16k bucket, rather than the 16k+ bucket,
+ // unless we've got a lot of ingests with large batches going on,
+ // in which case the 16k bucket will win.
+ if pow2 > 15 {
+ pow2 = 15
+ }
+ // store in inverse order so the lowest slot in the array is the
+ // highest cardinality
+ atomic.AddUint32(&sq.observedOpN[15-pow2], 1)
+}
+
+// computeMaxOpN tries to pick a reasonable new maxOpN for the background
+// scan to use. On a quiet system, we want to gradually lower opN, picking
+// the fragments with the highest opN values first, because those offer the
+// largest benefit. So, whenever we check a fragment in the background, if we
+// *don't* snapshot it, we'll "observe" its OpN value, and then we pick a
+// value which picks up at least 1/4 of them.
+//
+// If there's ingest activity, the Immediate and Enqueue operations will
+// "observe" the OpN of fragments submitted to them. This can drive OpN back
+// up, if those fragments frequently have very high opN values, which reflects
+// the fact that we have enough of that activity that we don't need the
+// background scanner adding more.
+//
+// If we have enough ingest activity that the background scanner never actually
+// gets to submit work, we'll rarely get here, because the background scanner
+// will block until there's no snapshots pending for the normal workload.
+// When we do, we'll probably pick a MaxOpN which is dominated by the ingest
+// workload's opN values. So for instance, if everything coming in from the
+// ingest workload has 10k or more items, because that's the default fragment
+// maxOpN, that will probably set the background snapshot queue value to 8k.
+func (sq *prioritySnapshotQueue) computeMaxOpN() {
+ sq.logger.Debugf("observedOpN by power of 2: %d\n", sq.observedOpN[:])
+ total := uint32(0)
+ for i := range sq.observedOpN {
+ total += atomic.LoadUint32(&sq.observedOpN[i])
+ }
+ target := (total / 4) + 1
+ subTotal := uint32(0)
+ for i := range sq.observedOpN {
+ v := atomic.LoadUint32(&sq.observedOpN[i])
+ subTotal += v
+ if subTotal >= target {
+ prevMaxOpN := sq.maxOpN
+ sq.maxOpN = (1 << (15 - uint(i))) / 2
+ if sq.maxOpN > 0 {
+ sq.maxOpN--
+ }
+ if prevMaxOpN != sq.maxOpN {
+ sq.logger.Printf("background scan: %d/%d fragments considered have opN %d or higher\n",
+ subTotal, total, sq.maxOpN)
+ }
+ break
+ }
+ }
+ // It's conceptually possible that we'll miss a couple of observations
+ // here but that's not really important. This is all pretty approximate.
+ for i := range sq.observedOpN {
+ atomic.StoreUint32(&sq.observedOpN[i], 0)
+ }
+}
+
+// prioritySnapshotQueueScanner is the data type that implements HolderOperator
+// and represents a single scan of a holder, with a given maxOpN.
+type prioritySnapshotQueueScanner struct {
+ HolderFilterAll
+ HolderProcessNone
+ sq *prioritySnapshotQueue
+ holder *Holder
+ queue chan snapshotRequest
+ ctx context.Context
+ maxOpN int
+ seen, hits, counter int
+}
+
+func (s *prioritySnapshotQueueScanner) ProcessFragment(f *fragment) error {
+ if f == nil {
+ return nil
+ }
+ s.seen++
+ // we can't defer this reasonably, because otherwise we'll keep
+ // the fragment locked forever if we end up trying to send it
+ // to the queue, but the workers are busy on other fragments.
+ f.mu.Lock()
+ open := f.open
+ snapshotPending, opN := f.snapshotPending, f.opN
+ f.mu.Unlock()
+
+ // a pending snapshot is one that is either in the normal or
+ // immediate queue, or is trying to get into the normal queue
+ // and about to fail, but either way, it already got observed
+ // there, so we don't need to observe it here. A closed fragment
+ // doesn't matter to us -- it should be a transient state that
+ // happens during a shutdown, or shouldn't happen, but we don't
+ // care about it.
+ if snapshotPending || !open {
+ return nil
+ }
+ if opN <= s.maxOpN {
+ // observe the value but don't do a snapshot
+ s.sq.observeOpN(uint32(opN))
+ s.counter++
+ if s.counter == 1000 {
+ select {
+ case <-time.After(1 * time.Second):
+ case <-s.ctx.Done():
+ return io.EOF
+ }
+ s.counter = 0
+ }
+ return nil
+ }
+ // we don't observe values when we decide to trigger a snapshot,
+ // because those values will be changing anyway. we could also
+ // observe them as zero, but that's also sort of wrong.
+ s.hits++
+ select {
+ case s.queue <- snapshotRequest{frag: f, when: time.Now()}:
+ s.sq.logger.Debugf("found fragment needing snapshot: %s\n", f.path)
+ case <-s.ctx.Done():
+ return io.EOF
+ }
+ return nil
+
+}
+
+func contextMergedWithStructChan(ctx context.Context, ch chan struct{}) (context.Context, context.CancelFunc) {
+ canCancel, cancel := context.WithCancel(ctx)
+ go func() {
+ select {
+ case <-ctx.Done():
+ cancel()
+ case <-ch:
+ cancel()
+ case <-canCancel.Done():
+ // don't need to cancel, but do need to exit this
+ // function
+ }
+ }()
+ return canCancel, cancel
+}
+
// scanHolderWorker is a background task that scans a holder looking for
// fragments which need snapshots taken. It's the cleanup task for snapshots
// that would have been requested by Enqueue, but the queue was full.
func (sq *prioritySnapshotQueue) scanHolderWorker(h *Holder, background chan snapshotRequest, done chan struct{}) {
defer sq.scanWG.Done()
- var indexNames, fieldNames, viewNames []string
- var fragNums []uint64
+ ctx, cancel := contextMergedWithStructChan(sq.ctx, done)
+ defer cancel()
+ scanner := &prioritySnapshotQueueScanner{
+ sq: sq,
+ holder: h,
+ queue: background,
+ ctx: sq.ctx,
+ maxOpN: sq.maxOpN,
+ }
for {
- // To avoid abusing things, cap activity rate; every time we finish
- // the holder, or every couple hundred fragments considered, we
- // pause for a bit.
- counter := 0
- hits := 0
- h.mu.Lock()
- indexNames = indexNames[:0]
- for indexName := range h.indexes {
- indexNames = append(indexNames, indexName)
+ err := h.Process(ctx, scanner)
+ if err != nil {
+ return
}
- h.mu.Unlock()
- for _, indexName := range indexNames {
- h.mu.Lock()
- index := h.indexes[indexName]
- h.mu.Unlock()
- if index == nil {
- continue
- }
- fieldNames = fieldNames[:0]
- index.mu.Lock()
- for fieldName := range index.fields {
- fieldNames = append(fieldNames, fieldName)
- }
- index.mu.Unlock()
- for _, fieldName := range fieldNames {
- index.mu.Lock()
- field := index.fields[fieldName]
- index.mu.Unlock()
- if field == nil {
- continue
- }
- viewNames = viewNames[:0]
- field.mu.Lock()
- for viewName := range field.viewMap {
- viewNames = append(viewNames, viewName)
- }
- field.mu.Unlock()
- for _, viewName := range viewNames {
- field.mu.Lock()
- view := field.viewMap[viewName]
- field.mu.Unlock()
- if view == nil {
- continue
- }
- fragNums := fragNums[:0]
- view.mu.Lock()
- for fragNum := range view.fragments {
- fragNums = append(fragNums, fragNum)
- }
- view.mu.Unlock()
- for _, fragNum := range fragNums {
- view.mu.Lock()
- frag := view.fragments[fragNum]
- view.mu.Unlock()
- if sq.needsSnapshot(frag) {
- hits++
- select {
- case background <- snapshotRequest{frag: frag, when: time.Now()}:
- sq.logger.Debugf("found fragment needing snapshot: %s\n", frag.path)
- case <-done:
- return
- }
- } else {
- // Count fragments examined *without* finding anything that
- // needed a snapshot. When we find things that need snapshots,
- // the time it takes the workers to respond to us is enough
- // of a delay to keep us from eating every CPU. So, if a lot
- // of things need snapshots, and the workers aren't doing
- // anything else, ScanHolder will mostly keep them saturated.
- // If they're busy, we'll block forever in the write to the
- // background queue. If there's nothing that needs snapshots,
- // we pause frequently for a second or so at a time.
- counter++
- if counter == 100 {
- select {
- case <-time.After(1 * time.Second):
- case <-done:
- return
- }
- counter = 0
- }
- }
- }
- }
- }
- }
- if hits > 0 {
- sq.logger.Printf("background scan: %d fragments needed snapshots\n", hits)
- hits = 0
+
+ if scanner.hits > 0 {
+ sq.logger.Printf("background scan: %d/%d fragments needed snapshots\n", scanner.hits, scanner.seen)
+ scanner.hits = 0
} else {
sq.logger.Debugf("background scan: no fragments needed snapshots, waiting\n")
// No reason to be active if we're not finding anything.
select {
case <-time.After(60 * time.Second):
- case <-done:
+ case <-ctx.Done():
return
}
}
+ scanner.seen = 0
+ sq.computeMaxOpN()
+ scanner.maxOpN = sq.maxOpN
}
}
diff --git a/test/field.go b/test/field.go
index b95e4f88b..f56672834 100644
--- a/test/field.go
+++ b/test/field.go
@@ -33,7 +33,7 @@ func newField(opts pilosa.FieldOption) *Field {
if err != nil {
panic(err)
}
- field, err := pilosa.NewField(path, "i", "f", opts)
+ field, err := pilosa.NewField(pilosa.NewHolder(pilosa.DefaultPartitionN), path, "i", "f", opts)
if err != nil {
panic(err)
}
@@ -63,7 +63,7 @@ func (f *Field) reopen() error {
}
path, index, name := f.Path(), f.Index(), f.Name()
- f.Field, err = pilosa.NewField(path, index, name, pilosa.OptFieldTypeDefault())
+ f.Field, err = pilosa.NewField(pilosa.NewHolder(pilosa.DefaultPartitionN), path, index, name, pilosa.OptFieldTypeDefault())
if err != nil {
return err
}
diff --git a/test/index.go b/test/index.go
index bf65c6ae2..b90702ae2 100644
--- a/test/index.go
+++ b/test/index.go
@@ -32,7 +32,7 @@ func newIndex() *Index {
if err != nil {
panic(err)
}
- index, err := pilosa.NewIndex(path, "i", pilosa.DefaultPartitionN)
+ index, err := pilosa.NewIndex(pilosa.NewHolder(pilosa.DefaultPartitionN), path, "i")
if err != nil {
panic(err)
}
@@ -62,7 +62,7 @@ func (i *Index) Reopen() error {
}
path, name := i.Path(), i.Name()
- i.Index, err = pilosa.NewIndex(path, name, pilosa.DefaultPartitionN)
+ i.Index, err = pilosa.NewIndex(pilosa.NewHolder(pilosa.DefaultPartitionN), path, name)
if err != nil {
return err
}
diff --git a/transaction_test.go b/transaction_test.go
index 171d1bbab..50998cd0d 100644
--- a/transaction_test.go
+++ b/transaction_test.go
@@ -63,8 +63,8 @@ func TestTransactionManager(t *testing.T) {
test.CompareTransactions(t, trnsMap["b"], trns2)
// can submit an exclusive transaction
- trnsE := mustStart(t, tm, "ce", time.Millisecond*5, true)
- test.CompareTransactions(t, &pilosa.Transaction{ID: "ce", Active: false, Exclusive: true, Timeout: time.Millisecond * 5, Deadline: time.Now().Add(time.Millisecond * 5)}, trnsE)
+ trnsE := mustStart(t, tm, "ce", 100*time.Millisecond, true)
+ test.CompareTransactions(t, &pilosa.Transaction{ID: "ce", Active: false, Exclusive: true, Timeout: 100 * time.Millisecond, Deadline: time.Now().Add(100 * time.Millisecond)}, trnsE)
// can't start new transactions while an exclusive transaction is pending
if _, err := tm.Start(ctx, "d", time.Millisecond, false); err != pilosa.ErrTransactionExclusive {
@@ -78,7 +78,7 @@ func TestTransactionManager(t *testing.T) {
// exclusive transaction becomes active after deadlines expire
for i := 0; true; i++ {
- time.Sleep(time.Microsecond)
+ time.Sleep(time.Millisecond)
trnsE, err := tm.Get(ctx, "ce")
if err != nil {
t.Errorf("error retrieving exclusive transaction: %v", err)
@@ -86,7 +86,7 @@ func TestTransactionManager(t *testing.T) {
if trnsE.Active {
break
}
- if i > 100 {
+ if i > 10000 {
t.Fatalf("exclusive transaction never became active: %+v", trnsE)
}
}
@@ -103,7 +103,7 @@ func TestTransactionManager(t *testing.T) {
// exclusive transaction gets expired after other transactions have attempted to start
for i := 0; true; i++ {
- time.Sleep(time.Millisecond * 2)
+ time.Sleep(time.Millisecond * 20)
trnsE, err := tm.Get(ctx, "ce")
if err == nil {
if i > 10 {
@@ -155,26 +155,26 @@ func TestTransactionManager(t *testing.T) {
mustFinish(t, tm, "le")
// can start normal transaction to test deadline reset
- trnsM := mustStart(t, tm, "m", time.Millisecond*4, false)
- test.CompareTransactions(t, &pilosa.Transaction{ID: "m", Active: true, Timeout: time.Millisecond * 4, Deadline: time.Now().Add(time.Millisecond * 4)}, trnsM)
+ trnsM := mustStart(t, tm, "m", time.Millisecond*400, false)
+ test.CompareTransactions(t, &pilosa.Transaction{ID: "m", Active: true, Timeout: time.Millisecond * 400, Deadline: time.Now().Add(time.Millisecond * 400)}, trnsM)
// start new exclusive transaction to trigger deadline check
trnsNE := mustStart(t, tm, "ne", time.Hour, true)
test.CompareTransactions(t, &pilosa.Transaction{ID: "ne", Exclusive: true, Timeout: time.Hour, Deadline: time.Now().Add(time.Hour)}, trnsNE)
// sleep for most of the deadline
- time.Sleep(time.Millisecond * 3)
+ time.Sleep(time.Millisecond * 300)
// reset deadline
trnsM_reset, err := tm.ResetDeadline(ctx, "m")
if err != nil {
t.Errorf("resetting deadline: %v", err)
}
- trnsM.Deadline = time.Now().Add(time.Millisecond * 4)
+ trnsM.Deadline = time.Now().Add(time.Millisecond * 400)
test.CompareTransactions(t, trnsM, trnsM_reset)
// sleep until past the original deadline
- time.Sleep(time.Millisecond * 2)
+ time.Sleep(time.Millisecond * 200)
// verify that trnsM still exists
trnsM_again := mustGet(t, tm, "m")
diff --git a/version.go b/version.go
index 4164dee32..8671d683f 100644
--- a/version.go
+++ b/version.go
@@ -14,15 +14,39 @@
package pilosa
-var Enterprise = "0"
-var EnterpriseEnabled = false
-var Version = "v0.0.0"
-var BuildTime = "not recorded"
+import "time"
-// init sets the EnterpriseEnabled bool, based on the Enterprise string.
-// This is needed because bools cannot be set with ldflags.
-func init() { // nolint: gochecknoinits
- if Enterprise == "1" {
- EnterpriseEnabled = true
+var Version string
+var Commit string
+var Variant string
+var BuildTime string
+
+func VersionInfo() string {
+ var prefix string
+ if Variant != "" {
+ prefix = Variant + " "
}
+ var suffix string
+ if Version != "" {
+ suffix = " " + Version
+ } else {
+ suffix = " v2.x"
+ }
+ buildTime := BuildTime
+ if buildTime != "" {
+ // Normalize the build time into a friendly format in the user's time zone.
+ if t, err := time.Parse("2006-01-02T15:04:05+0000", BuildTime); err == nil {
+ buildTime = t.Local().Format("Jan _2 2006 3:04PM")
+ }
+ }
+ switch {
+ case Commit != "" && buildTime != "":
+ suffix += " (" + buildTime + ", " + Commit + ")"
+ case Commit != "":
+ suffix += " (" + Commit + ")"
+ case buildTime != "":
+ suffix += " (" + buildTime + ")"
+ }
+
+ return prefix + "Pilosa" + suffix
}
diff --git a/view.go b/view.go
index 478725e1e..af73b7fe2 100644
--- a/view.go
+++ b/view.go
@@ -26,7 +26,6 @@ import (
"sync/atomic"
"time"
- "github.com/pilosa/pilosa/v2/logger"
"github.com/pilosa/pilosa/v2/pql"
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pilosa/pilosa/v2/stats"
@@ -49,6 +48,8 @@ type view struct {
field string
name string
+ holder *Holder
+
fieldType string
cacheType string
cacheSize uint32
@@ -56,24 +57,24 @@ type view struct {
// Fragments by shard.
fragments map[uint64]*fragment
- broadcaster broadcaster
- stats stats.StatsClient
- rowAttrStore AttrStore
- logger logger.Logger
- snapshotQueue snapshotQueue
+ broadcaster broadcaster
+ stats stats.StatsClient
+ rowAttrStore AttrStore
knownShards *roaring.Bitmap
knownShardsCopied uint32
}
// newView returns a new instance of View.
-func newView(path, index, field, name string, fieldOptions FieldOptions) *view {
+func newView(holder *Holder, path, index, field, name string, fieldOptions FieldOptions) *view {
return &view{
path: path,
index: index,
field: field,
name: name,
+ holder: holder,
+
fieldType: fieldOptions.Type,
cacheType: fieldOptions.CacheType,
cacheSize: fieldOptions.CacheSize,
@@ -82,7 +83,6 @@ func newView(path, index, field, name string, fieldOptions FieldOptions) *view {
broadcaster: NopBroadcaster,
stats: stats.NopStatsClient,
- logger: logger.NopLogger,
knownShards: roaring.NewSliceBitmap(),
}
}
@@ -133,14 +133,14 @@ func (v *view) open() error {
if err := func() error {
// Ensure the view's path exists.
- v.logger.Debugf("ensure view path exists: %s", v.path)
+ v.holder.Logger.Debugf("ensure view path exists: %s", v.path)
if err := os.MkdirAll(v.path, 0777); err != nil {
return errors.Wrap(err, "creating view directory")
} else if err := os.MkdirAll(filepath.Join(v.path, "fragments"), 0777); err != nil {
return errors.Wrap(err, "creating fragments directory")
}
- v.logger.Debugf("open fragments for index/field/view: %s/%s/%s", v.index, v.field, v.name)
+ v.holder.Logger.Debugf("open fragments for index/field/view: %s/%s/%s", v.index, v.field, v.name)
if err := v.openFragments(); err != nil {
return errors.Wrap(err, "opening fragments")
}
@@ -151,7 +151,7 @@ func (v *view) open() error {
return err
}
- v.logger.Debugf("successfully opened index/field/view: %s/%s/%s", v.index, v.field, v.name)
+ v.holder.Logger.Debugf("successfully opened index/field/view: %s/%s/%s", v.index, v.field, v.name)
return nil
}
@@ -190,12 +190,12 @@ fileLoop:
// Parse filename into integer.
shard, err := strconv.ParseUint(filepath.Base(fi.Name()), 10, 64)
if err != nil {
- v.logger.Debugf("WARNING: couldn't use non-integer file as shard in index/field/view %s/%s/%s: %s", v.index, v.field, v.name, fi.Name())
+ v.holder.Logger.Debugf("WARNING: couldn't use non-integer file as shard in index/field/view %s/%s/%s: %s", v.index, v.field, v.name, fi.Name())
continue
}
workQueue <- struct{}{}
- v.logger.Debugf("open index/field/view/fragment: %s/%s/%s/%d", v.index, v.field, v.name, shard)
+ v.holder.Logger.Debugf("open index/field/view/fragment: %s/%s/%s/%d", v.index, v.field, v.name, shard)
eg.Go(func() error {
defer func() {
<-workQueue
@@ -205,7 +205,7 @@ fileLoop:
return fmt.Errorf("open fragment: shard=%d, err=%s", frag.shard, err)
}
frag.RowAttrStore = v.rowAttrStore
- v.logger.Debugf("add index/field/view/fragment to view.fragments: %s/%s/%s/%d", v.index, v.field, v.name, shard)
+ v.holder.Logger.Debugf("add index/field/view/fragment to view.fragments: %s/%s/%s/%d", v.index, v.field, v.name, shard)
mu.Lock()
v.fragments[frag.shard] = frag
v.addKnownShard(frag.shard)
@@ -342,7 +342,7 @@ func (v *view) notifyIfNewShard(shard uint64) {
// Broadcast a message that a new max shard was just created.
err := v.broadcaster.SendSync(msg)
if err != nil {
- v.logger.Printf("broadcasting create shard: %v", err)
+ v.holder.Logger.Printf("broadcasting create shard: %v", err)
}
close(broadcastChan)
}()
@@ -352,19 +352,15 @@ func (v *view) notifyIfNewShard(shard uint64) {
select {
case <-broadcastChan:
case <-time.After(50 * time.Millisecond):
- v.logger.Debugf("broadcasting create shard took >50ms")
+ v.holder.Logger.Debugf("broadcasting create shard took >50ms")
}
}
func (v *view) newFragment(path string, shard uint64) *fragment {
- frag := newFragment(path, v.index, v.field, v.name, shard, v.flags())
+ frag := newFragment(v.holder, path, v.index, v.field, v.name, shard, v.flags())
frag.CacheType = v.cacheType
frag.CacheSize = v.cacheSize
- frag.Logger = v.logger
frag.stats = v.stats
- if v.snapshotQueue != nil {
- frag.snapshotQueue = v.snapshotQueue
- }
if v.fieldType == FieldTypeMutex {
frag.mutexVector = newRowsVector(frag)
} else if v.fieldType == FieldTypeBool {
@@ -382,7 +378,7 @@ func (v *view) deleteFragment(shard uint64) error {
return ErrFragmentNotFound
}
- v.logger.Printf("delete fragment: (%s/%s/%s) %d", v.index, v.field, v.name, shard)
+ v.holder.Logger.Printf("delete fragment: (%s/%s/%s) %d", v.index, v.field, v.name, shard)
// Close data files before deletion.
if err := fragment.Close(); err != nil {
@@ -396,7 +392,7 @@ func (v *view) deleteFragment(shard uint64) error {
// Delete fragment cache file.
if err := os.Remove(fragment.cachePath()); err != nil {
- v.logger.Printf("no cache file to delete for shard %d", shard)
+ v.holder.Logger.Printf("no cache file to delete for shard %d", shard)
}
delete(v.fragments, shard)
diff --git a/view_internal_test.go b/view_internal_test.go
index bb50003d2..bbb1a20f4 100644
--- a/view_internal_test.go
+++ b/view_internal_test.go
@@ -34,7 +34,7 @@ func mustOpenView(index, field, name string) *view {
CacheSize: DefaultCacheSize,
}
- v := newView(path, index, field, name, fo)
+ v := newView(NewHolder(DefaultPartitionN), path, index, field, name, fo)
if err := v.open(); err != nil {
panic(err)
}