diff --git a/.travis.yml b/.travis.yml
index 3a9abe355..6d5674662 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -9,7 +9,9 @@ env:
- secure: "U4fpHWDVOG4viqZsiVgUDW7OW1JW60uPOZy0q9pfbs86iHvmZq0PaScsZ+YdlYaN2GETVr7endDf6DCcZs1PWfg0F6VQfkOXcShX8HVS9O58lUZA5tyvbDVql9DQs4PbnkZo+ktz+Z0YaXqq2RdtMDOUz4bgZwspLPMA14if+N6w0tqCFpB7bEtpptTGsdbIQPG1n07yvSeNmK4mvrEEs77tWmhulN5iilpOqhpIvD39bJvtCYVALuJpzLd/OjLTPV9l/fl+hJkMXSj+X5ilO1DHINAcCM648iEX2phXAIWmi0O0Rbg2cI4kV9T5ysOIw8ux+YCm9bZDGTCt+VGBW5Fg+Z5iaXXexyKYCGiHleOJ7kCj9kXxh2u8NiYVNgb19dGJV5/HgQ6pcGWjeVEqr8yY1546zMjpTX+SYGQF+XZe+uggEjeAsk53ueXa0pyZTrlrqSvR7BBtWPx47s/dTg2L19FQYv3XpGMxEXLw92RplExQKi1h7QgihRxFpjGgURHhrt7d9eiNiNqBt3ZsHjmh2AkXZHnaDjlgSnFFWaMqP3UtDBWIuO+2BMbZUJVfP+gpQGBZ4gtpUSmV2JDCHgZgX5OAnLD4usxh+ATQ4rvUXF/tf8nMqEKHlGKd8hxpYSyMX21BoqfSfY4/IA0ejVE9BITqlrvqewqkP1yxe7o="
matrix:
- GOARCH=386
+ - GOARCH=386 ENTERPRISE=1
- GOARCH=amd64
+ - GOARCH=amd64 ENTERPRISE=1
install:
- make install-dep install-statik vendor generate-statik
script:
diff --git a/Dockerfile b/Dockerfile
index 0392dbe87..7cdb93cc0 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -11,6 +11,7 @@ LABEL maintainer "dev@pilosa.com"
COPY --from=builder /go/bin/pilosa /pilosa
+COPY LICENSE /LICENSE
COPY NOTICE /NOTICE
EXPOSE 10101
diff --git a/Makefile b/Makefile
index ea1e38d34..fe663d149 100644
--- a/Makefile
+++ b/Makefile
@@ -2,12 +2,13 @@
CLONE_URL=github.com/pilosa/pilosa
VERSION := $(shell git describe --tags 2> /dev/null || echo unknown)
-VERSION_ID := $(VERSION)-$(GOOS)-$(GOARCH)
+VERSION_ID := $(if $(ENTERPRISE),enterprise-)$(VERSION)-$(GOOS)-$(GOARCH)
BRANCH := $(if $(TRAVIS_BRANCH),$(TRAVIS_BRANCH),$(shell git rev-parse --abbrev-ref HEAD))
BRANCH_ID := $(BRANCH)-$(GOOS)-$(GOARCH)
BUILD_TIME := $(shell date -u +%FT%T%z)
-LDFLAGS="-X github.com/pilosa/pilosa.Version=$(VERSION) -X github.com/pilosa/pilosa.BuildTime=$(BUILD_TIME)"
+LDFLAGS="-X github.com/pilosa/pilosa.Version=$(VERSION) -X github.com/pilosa/pilosa.BuildTime=$(BUILD_TIME) -X github.com/pilosa/pilosa.Enterprise=$(ENTERPRISE)"
GO_VERSION=latest
+ENTERPRISE_TAG := $(if $(ENTERPRISE),-tags=enterprise)
# Run tests and compile Pilosa
default: test build
@@ -24,7 +25,7 @@ vendor: Gopkg.toml
# Run test suite
test: vendor
- go test ./... $(TESTFLAGS)
+ go test ./... $(ENTERPRISE_TAG) $(TESTFLAGS)
# Run test suite with coverage enabled
cover: vendor
@@ -37,23 +38,25 @@ cover-viz: cover
# Compile Pilosa
build: vendor
- go build -tags release -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa
+ go build -tags release $(ENTERPRISE_TAG) -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa
# Create a single release build under the build directory
release-build: vendor
$(MAKE) $(if $(DOCKER_BUILD),docker-)build FLAGS="-o build/pilosa-$(VERSION_ID)/pilosa"
cp NOTICE LICENSE README.md build/pilosa-$(VERSION_ID)
+ %(if $(ENTERPRISE),cp enterprise/COPYING build/pilosa-$(VERSION_ID))
tar -cvz -C build -f build/pilosa-$(VERSION_ID).tar.gz pilosa-$(VERSION_ID)/
@echo Created release build: build/pilosa-$(VERSION_ID).tar.gz
# Error out if there are untracked changes in Git
check-clean:
- $(if $(shell git status --porcelain),$(error Git status is not clean! Please commit or checkout/reset changes.))
# Create release build tarballs for all supported platforms. Linux compilation happens under Docker.
release: check-clean
$(MAKE) release-build GOOS=darwin GOARCH=amd64
+ $(MAKE) release-build GOOS=darwin GOARCH=amd64 ENTERPRISE=1
$(MAKE) release-build GOOS=linux GOARCH=amd64 DOCKER_BUILD=1
+ $(MAKE) release-build GOOS=linux GOARCH=amd64 DOCKER_BUILD=1 ENTERPRISE=1
$(MAKE) release-build GOOS=linux GOARCH=386 DOCKER_BUILD=1
# Create branch-tagged pre-release for client library CI jobs
@@ -70,7 +73,7 @@ prerelease-upload: prerelease
# Install Pilosa
install: vendor
- go install -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa
+ go install $(ENTERPRISE_TAG) -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa
# `go generate` protocol buffers
generate-protoc: require-protoc require-protoc-gen-gofast
@@ -94,11 +97,11 @@ docker:
# Compile Pilosa inside Docker container
docker-build:
- docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) -e GOOS=$(GOOS) -e GOARCH=$(GOARCH) golang:$(GO_VERSION) go build -tags release -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa
+ docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) -e GOOS=$(GOOS) -e GOARCH=$(GOARCH) golang:$(GO_VERSION) go build $(ENTERPRISE_TAG) -tags release -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa
# Run Pilosa tests inside Docker container
docker-test:
- docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) golang:$(GO_VERSION) go test $(TESTFLAGS) ./...
+ docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) golang:$(GO_VERSION) go test $(ENTERPRISE_TAG) $(TESTFLAGS) ./...
######################
# Build dependencies #
diff --git a/NOTICE b/NOTICE
index 7613b11e7..99ea00db4 100644
--- a/NOTICE
+++ b/NOTICE
@@ -1,7 +1,7 @@
Software license
================
-Copyright 2017 Pilosa Corp.
+Copyright (C) 2017-2018 Pilosa Corp. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License").
You may obtain a copy of the License at
@@ -14,6 +14,26 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
+Enterprise Edition software license
+===================================
+
+Files contained under the directory `enterprise` are subject to the following
+license notice (Full license included in the file `COPYING`):
+
+ Copyright (C) 2018 Pilosa Corp. All rights reserved.
+
+ Pilosa Enterprise Edition is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ Pilosa Enterprise Edition is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with Pilosa Enterprise Edition. If not, see .
Third-party software licenses
=============================
@@ -34,3 +54,34 @@ The file /pilosa/lru/lru.go contains a redistribution of lru
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
+
+The file /enterprise/b/btree.go contains a modified redistribution of b
+(https://github.com/cznic/b); the license follows:
+
+ Copyright (c) 2014 The b Authors. All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are
+ met:
+
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above
+ copyright notice, this list of conditions and the following disclaimer
+ in the documentation and/or other materials provided with the
+ distribution.
+ * Neither the names of the authors nor the names of the
+ contributors may be used to endorse or promote products derived from
+ this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/README.md b/README.md
index 5ae041dfd..7c1c7cfda 100644
--- a/README.md
+++ b/README.md
@@ -63,6 +63,14 @@ There are supported libraries for the following languages:
- [Java](https://www.pilosa.com/docs/client-libraries/#java)
- [Python](https://www.pilosa.com/docs/client-libraries/#python)
+## Licenses
+
+The core Pilosa code base and all default builds (referred to as Pilosa Community Edition) are licensed completely under the Apache License, Version 2.0.
+If you build Pilosa with the `enterprise` build tag (Pilosa Enterprise Edition), then that build will include features licensed under the GNU Affero General
+Public License (AGPL). Enterprise code is located entirely in the [github.com/pilosa/pilosa/enterprise](https://github.com/pilosa/pilosa/tree/master/enterprise)
+directory. See [github.com/pilosa/pilosa/NOTICE](https://github.com/pilosa/pilosa/blob/master/NOTICE) and
+[github.com/pilosa/pilosa/LICENSE](https://github.com/pilosa/pilosa/blob/master/LICENSE) for more information about Pilosa licenses.
+
## Get Support
There are [several channels](https://www.pilosa.com/community/#support) available for you to reach out to us for support.
diff --git a/bitmap.go b/bitmap.go
index 117d020f7..0b805aa4a 100644
--- a/bitmap.go
+++ b/bitmap.go
@@ -187,11 +187,12 @@ func (b *Bitmap) createSegmentIfNotExists(slice uint64) *BitmapSegment {
}
// Insert new segment.
- b.segments = append(b.segments, BitmapSegment{})
+ b.segments = append(b.segments, BitmapSegment{data: *roaring.NewBitmap()})
if i < len(b.segments) {
copy(b.segments[i+1:], b.segments[i:])
}
b.segments[i] = BitmapSegment{
+ data: *roaring.NewBitmap(),
slice: slice,
writable: true,
}
diff --git a/cmd/root.go b/cmd/root.go
index 72e59abeb..ec9d3be15 100644
--- a/cmd/root.go
+++ b/cmd/root.go
@@ -30,6 +30,10 @@ import (
var subcommandFns = map[string]func(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command{}
func NewRootCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
+ productName := "Pilosa " + pilosa.Version
+ if pilosa.EnterpriseEnabled {
+ productName = "Pilosa Enterprise " + pilosa.Version
+ }
rc := &cobra.Command{
Use: "pilosa",
Short: "Pilosa - A Distributed In-memory Binary Bitmap Index.",
@@ -41,7 +45,7 @@ tools for administering pilosa, importing/exporting data,
backing up, and more. Complete documentation is available
at https://www.pilosa.com/docs/
-Version: ` + pilosa.Version + `
+` + productName + `
Build Time: ` + pilosa.BuildTime + "\n",
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
v := viper.New()
diff --git a/enterprise/COPYING b/enterprise/COPYING
new file mode 100644
index 000000000..be3f7b28e
--- /dev/null
+++ b/enterprise/COPYING
@@ -0,0 +1,661 @@
+ GNU AFFERO GENERAL PUBLIC LICENSE
+ Version 3, 19 November 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc.
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+ Preamble
+
+ The GNU Affero General Public License is a free, copyleft license for
+software and other kinds of works, specifically designed to ensure
+cooperation with the community in the case of network server software.
+
+ The licenses for most software and other practical works are designed
+to take away your freedom to share and change the works. By contrast,
+our General Public Licenses are intended to guarantee your freedom to
+share and change all versions of a program--to make sure it remains free
+software for all its users.
+
+ When we speak of free software, we are referring to freedom, not
+price. Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+them if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces of it in new
+free programs, and that you know you can do these things.
+
+ Developers that use our General Public Licenses protect your rights
+with two steps: (1) assert copyright on the software, and (2) offer
+you this License which gives you legal permission to copy, distribute
+and/or modify the software.
+
+ A secondary benefit of defending all users' freedom is that
+improvements made in alternate versions of the program, if they
+receive widespread use, become available for other developers to
+incorporate. Many developers of free software are heartened and
+encouraged by the resulting cooperation. However, in the case of
+software used on network servers, this result may fail to come about.
+The GNU General Public License permits making a modified version and
+letting the public access it on a server without ever releasing its
+source code to the public.
+
+ The GNU Affero General Public License is designed specifically to
+ensure that, in such cases, the modified source code becomes available
+to the community. It requires the operator of a network server to
+provide the source code of the modified version running there to the
+users of that server. Therefore, public use of a modified version, on
+a publicly accessible server, gives the public access to the source
+code of the modified version.
+
+ An older license, called the Affero General Public License and
+published by Affero, was designed to accomplish similar goals. This is
+a different license, not a version of the Affero GPL, but Affero has
+released a new version of the Affero GPL which permits relicensing under
+this license.
+
+ The precise terms and conditions for copying, distribution and
+modification follow.
+
+ TERMS AND CONDITIONS
+
+ 0. Definitions.
+
+ "This License" refers to version 3 of the GNU Affero General Public License.
+
+ "Copyright" also means copyright-like laws that apply to other kinds of
+works, such as semiconductor masks.
+
+ "The Program" refers to any copyrightable work licensed under this
+License. Each licensee is addressed as "you". "Licensees" and
+"recipients" may be individuals or organizations.
+
+ To "modify" a work means to copy from or adapt all or part of the work
+in a fashion requiring copyright permission, other than the making of an
+exact copy. The resulting work is called a "modified version" of the
+earlier work or a work "based on" the earlier work.
+
+ A "covered work" means either the unmodified Program or a work based
+on the Program.
+
+ To "propagate" a work means to do anything with it that, without
+permission, would make you directly or secondarily liable for
+infringement under applicable copyright law, except executing it on a
+computer or modifying a private copy. Propagation includes copying,
+distribution (with or without modification), making available to the
+public, and in some countries other activities as well.
+
+ To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies. Mere interaction with a user through
+a computer network, with no transfer of a copy, is not conveying.
+
+ An interactive user interface displays "Appropriate Legal Notices"
+to the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License. If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.
+
+ 1. Source Code.
+
+ The "source code" for a work means the preferred form of the work
+for making modifications to it. "Object code" means any non-source
+form of a work.
+
+ A "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.
+
+ The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form. A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.
+
+ The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities. However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work. For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
+
+ The Corresponding Source need not include anything that users
+can regenerate automatically from other parts of the Corresponding
+Source.
+
+ The Corresponding Source for a work in source code form is that
+same work.
+
+ 2. Basic Permissions.
+
+ All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met. This License explicitly affirms your unlimited
+permission to run the unmodified Program. The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work. This License acknowledges your
+rights of fair use or other equivalent, as provided by copyright law.
+
+ You may make, run and propagate covered works that you do not
+convey, without conditions so long as your license otherwise remains
+in force. You may convey covered works to others for the sole purpose
+of having them make modifications exclusively for you, or provide you
+with facilities for running those works, provided that you comply with
+the terms of this License in conveying all material for which you do
+not control copyright. Those thus making or running the covered works
+for you must do so exclusively on your behalf, under your direction
+and control, on terms that prohibit them from making any copies of
+your copyrighted material outside their relationship with you.
+
+ Conveying under any other circumstances is permitted solely under
+the conditions stated below. Sublicensing is not allowed; section 10
+makes it unnecessary.
+
+ 3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+
+ No covered work shall be deemed part of an effective technological
+measure under any applicable law fulfilling obligations under article
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
+similar laws prohibiting or restricting circumvention of such
+measures.
+
+ When you convey a covered work, you waive any legal power to forbid
+circumvention of technological measures to the extent such circumvention
+is effected by exercising rights under this License with respect to
+the covered work, and you disclaim any intention to limit operation or
+modification of the work as a means of enforcing, against the work's
+users, your or third parties' legal rights to forbid circumvention of
+technological measures.
+
+ 4. Conveying Verbatim Copies.
+
+ You may convey verbatim copies of the Program's source code as you
+receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice;
+keep intact all notices stating that this License and any
+non-permissive terms added in accord with section 7 apply to the code;
+keep intact all notices of the absence of any warranty; and give all
+recipients a copy of this License along with the Program.
+
+ You may charge any price or no price for each copy that you convey,
+and you may offer support or warranty protection for a fee.
+
+ 5. Conveying Modified Source Versions.
+
+ You may convey a work based on the Program, or the modifications to
+produce it from the Program, in the form of source code under the
+terms of section 4, provided that you also meet all of these conditions:
+
+ a) The work must carry prominent notices stating that you modified
+ it, and giving a relevant date.
+
+ b) The work must carry prominent notices stating that it is
+ released under this License and any conditions added under section
+ 7. This requirement modifies the requirement in section 4 to
+ "keep intact all notices".
+
+ c) You must license the entire work, as a whole, under this
+ License to anyone who comes into possession of a copy. This
+ License will therefore apply, along with any applicable section 7
+ additional terms, to the whole of the work, and all its parts,
+ regardless of how they are packaged. This License gives no
+ permission to license the work in any other way, but it does not
+ invalidate such permission if you have separately received it.
+
+ d) If the work has interactive user interfaces, each must display
+ Appropriate Legal Notices; however, if the Program has interactive
+ interfaces that do not display Appropriate Legal Notices, your
+ work need not make them do so.
+
+ A compilation of a covered work with other separate and independent
+works, which are not by their nature extensions of the covered work,
+and which are not combined with it such as to form a larger program,
+in or on a volume of a storage or distribution medium, is called an
+"aggregate" if the compilation and its resulting copyright are not
+used to limit the access or legal rights of the compilation's users
+beyond what the individual works permit. Inclusion of a covered work
+in an aggregate does not cause this License to apply to the other
+parts of the aggregate.
+
+ 6. Conveying Non-Source Forms.
+
+ You may convey a covered work in object code form under the terms
+of sections 4 and 5, provided that you also convey the
+machine-readable Corresponding Source under the terms of this License,
+in one of these ways:
+
+ a) Convey the object code in, or embodied in, a physical product
+ (including a physical distribution medium), accompanied by the
+ Corresponding Source fixed on a durable physical medium
+ customarily used for software interchange.
+
+ b) Convey the object code in, or embodied in, a physical product
+ (including a physical distribution medium), accompanied by a
+ written offer, valid for at least three years and valid for as
+ long as you offer spare parts or customer support for that product
+ model, to give anyone who possesses the object code either (1) a
+ copy of the Corresponding Source for all the software in the
+ product that is covered by this License, on a durable physical
+ medium customarily used for software interchange, for a price no
+ more than your reasonable cost of physically performing this
+ conveying of source, or (2) access to copy the
+ Corresponding Source from a network server at no charge.
+
+ c) Convey individual copies of the object code with a copy of the
+ written offer to provide the Corresponding Source. This
+ alternative is allowed only occasionally and noncommercially, and
+ only if you received the object code with such an offer, in accord
+ with subsection 6b.
+
+ d) Convey the object code by offering access from a designated
+ place (gratis or for a charge), and offer equivalent access to the
+ Corresponding Source in the same way through the same place at no
+ further charge. You need not require recipients to copy the
+ Corresponding Source along with the object code. If the place to
+ copy the object code is a network server, the Corresponding Source
+ may be on a different server (operated by you or a third party)
+ that supports equivalent copying facilities, provided you maintain
+ clear directions next to the object code saying where to find the
+ Corresponding Source. Regardless of what server hosts the
+ Corresponding Source, you remain obligated to ensure that it is
+ available for as long as needed to satisfy these requirements.
+
+ e) Convey the object code using peer-to-peer transmission, provided
+ you inform other peers where the object code and Corresponding
+ Source of the work are being offered to the general public at no
+ charge under subsection 6d.
+
+ A separable portion of the object code, whose source code is excluded
+from the Corresponding Source as a System Library, need not be
+included in conveying the object code work.
+
+ A "User Product" is either (1) a "consumer product", which means any
+tangible personal property which is normally used for personal, family,
+or household purposes, or (2) anything designed or sold for incorporation
+into a dwelling. In determining whether a product is a consumer product,
+doubtful cases shall be resolved in favor of coverage. For a particular
+product received by a particular user, "normally used" refers to a
+typical or common use of that class of product, regardless of the status
+of the particular user or of the way in which the particular user
+actually uses, or expects or is expected to use, the product. A product
+is a consumer product regardless of whether the product has substantial
+commercial, industrial or non-consumer uses, unless such uses represent
+the only significant mode of use of the product.
+
+ "Installation Information" for a User Product means any methods,
+procedures, authorization keys, or other information required to install
+and execute modified versions of a covered work in that User Product from
+a modified version of its Corresponding Source. The information must
+suffice to ensure that the continued functioning of the modified object
+code is in no case prevented or interfered with solely because
+modification has been made.
+
+ If you convey an object code work under this section in, or with, or
+specifically for use in, a User Product, and the conveying occurs as
+part of a transaction in which the right of possession and use of the
+User Product is transferred to the recipient in perpetuity or for a
+fixed term (regardless of how the transaction is characterized), the
+Corresponding Source conveyed under this section must be accompanied
+by the Installation Information. But this requirement does not apply
+if neither you nor any third party retains the ability to install
+modified object code on the User Product (for example, the work has
+been installed in ROM).
+
+ The requirement to provide Installation Information does not include a
+requirement to continue to provide support service, warranty, or updates
+for a work that has been modified or installed by the recipient, or for
+the User Product in which it has been modified or installed. Access to a
+network may be denied when the modification itself materially and
+adversely affects the operation of the network or violates the rules and
+protocols for communication across the network.
+
+ Corresponding Source conveyed, and Installation Information provided,
+in accord with this section must be in a format that is publicly
+documented (and with an implementation available to the public in
+source code form), and must require no special password or key for
+unpacking, reading or copying.
+
+ 7. Additional Terms.
+
+ "Additional permissions" are terms that supplement the terms of this
+License by making exceptions from one or more of its conditions.
+Additional permissions that are applicable to the entire Program shall
+be treated as though they were included in this License, to the extent
+that they are valid under applicable law. If additional permissions
+apply only to part of the Program, that part may be used separately
+under those permissions, but the entire Program remains governed by
+this License without regard to the additional permissions.
+
+ When you convey a copy of a covered work, you may at your option
+remove any additional permissions from that copy, or from any part of
+it. (Additional permissions may be written to require their own
+removal in certain cases when you modify the work.) You may place
+additional permissions on material, added by you to a covered work,
+for which you have or can give appropriate copyright permission.
+
+ Notwithstanding any other provision of this License, for material you
+add to a covered work, you may (if authorized by the copyright holders of
+that material) supplement the terms of this License with terms:
+
+ a) Disclaiming warranty or limiting liability differently from the
+ terms of sections 15 and 16 of this License; or
+
+ b) Requiring preservation of specified reasonable legal notices or
+ author attributions in that material or in the Appropriate Legal
+ Notices displayed by works containing it; or
+
+ c) Prohibiting misrepresentation of the origin of that material, or
+ requiring that modified versions of such material be marked in
+ reasonable ways as different from the original version; or
+
+ d) Limiting the use for publicity purposes of names of licensors or
+ authors of the material; or
+
+ e) Declining to grant rights under trademark law for use of some
+ trade names, trademarks, or service marks; or
+
+ f) Requiring indemnification of licensors and authors of that
+ material by anyone who conveys the material (or modified versions of
+ it) with contractual assumptions of liability to the recipient, for
+ any liability that these contractual assumptions directly impose on
+ those licensors and authors.
+
+ All other non-permissive additional terms are considered "further
+restrictions" within the meaning of section 10. If the Program as you
+received it, or any part of it, contains a notice stating that it is
+governed by this License along with a term that is a further
+restriction, you may remove that term. If a license document contains
+a further restriction but permits relicensing or conveying under this
+License, you may add to a covered work material governed by the terms
+of that license document, provided that the further restriction does
+not survive such relicensing or conveying.
+
+ If you add terms to a covered work in accord with this section, you
+must place, in the relevant source files, a statement of the
+additional terms that apply to those files, or a notice indicating
+where to find the applicable terms.
+
+ Additional terms, permissive or non-permissive, may be stated in the
+form of a separately written license, or stated as exceptions;
+the above requirements apply either way.
+
+ 8. Termination.
+
+ You may not propagate or modify a covered work except as expressly
+provided under this License. Any attempt otherwise to propagate or
+modify it is void, and will automatically terminate your rights under
+this License (including any patent licenses granted under the third
+paragraph of section 11).
+
+ However, if you cease all violation of this License, then your
+license from a particular copyright holder is reinstated (a)
+provisionally, unless and until the copyright holder explicitly and
+finally terminates your license, and (b) permanently, if the copyright
+holder fails to notify you of the violation by some reasonable means
+prior to 60 days after the cessation.
+
+ Moreover, your license from a particular copyright holder is
+reinstated permanently if the copyright holder notifies you of the
+violation by some reasonable means, this is the first time you have
+received notice of violation of this License (for any work) from that
+copyright holder, and you cure the violation prior to 30 days after
+your receipt of the notice.
+
+ Termination of your rights under this section does not terminate the
+licenses of parties who have received copies or rights from you under
+this License. If your rights have been terminated and not permanently
+reinstated, you do not qualify to receive new licenses for the same
+material under section 10.
+
+ 9. Acceptance Not Required for Having Copies.
+
+ You are not required to accept this License in order to receive or
+run a copy of the Program. Ancillary propagation of a covered work
+occurring solely as a consequence of using peer-to-peer transmission
+to receive a copy likewise does not require acceptance. However,
+nothing other than this License grants you permission to propagate or
+modify any covered work. These actions infringe copyright if you do
+not accept this License. Therefore, by modifying or propagating a
+covered work, you indicate your acceptance of this License to do so.
+
+ 10. Automatic Licensing of Downstream Recipients.
+
+ Each time you convey a covered work, the recipient automatically
+receives a license from the original licensors, to run, modify and
+propagate that work, subject to this License. You are not responsible
+for enforcing compliance by third parties with this License.
+
+ An "entity transaction" is a transaction transferring control of an
+organization, or substantially all assets of one, or subdividing an
+organization, or merging organizations. If propagation of a covered
+work results from an entity transaction, each party to that
+transaction who receives a copy of the work also receives whatever
+licenses to the work the party's predecessor in interest had or could
+give under the previous paragraph, plus a right to possession of the
+Corresponding Source of the work from the predecessor in interest, if
+the predecessor has it or can get it with reasonable efforts.
+
+ You may not impose any further restrictions on the exercise of the
+rights granted or affirmed under this License. For example, you may
+not impose a license fee, royalty, or other charge for exercise of
+rights granted under this License, and you may not initiate litigation
+(including a cross-claim or counterclaim in a lawsuit) alleging that
+any patent claim is infringed by making, using, selling, offering for
+sale, or importing the Program or any portion of it.
+
+ 11. Patents.
+
+ A "contributor" is a copyright holder who authorizes use under this
+License of the Program or a work on which the Program is based. The
+work thus licensed is called the contributor's "contributor version".
+
+ A contributor's "essential patent claims" are all patent claims
+owned or controlled by the contributor, whether already acquired or
+hereafter acquired, that would be infringed by some manner, permitted
+by this License, of making, using, or selling its contributor version,
+but do not include claims that would be infringed only as a
+consequence of further modification of the contributor version. For
+purposes of this definition, "control" includes the right to grant
+patent sublicenses in a manner consistent with the requirements of
+this License.
+
+ Each contributor grants you a non-exclusive, worldwide, royalty-free
+patent license under the contributor's essential patent claims, to
+make, use, sell, offer for sale, import and otherwise run, modify and
+propagate the contents of its contributor version.
+
+ In the following three paragraphs, a "patent license" is any express
+agreement or commitment, however denominated, not to enforce a patent
+(such as an express permission to practice a patent or covenant not to
+sue for patent infringement). To "grant" such a patent license to a
+party means to make such an agreement or commitment not to enforce a
+patent against the party.
+
+ If you convey a covered work, knowingly relying on a patent license,
+and the Corresponding Source of the work is not available for anyone
+to copy, free of charge and under the terms of this License, through a
+publicly available network server or other readily accessible means,
+then you must either (1) cause the Corresponding Source to be so
+available, or (2) arrange to deprive yourself of the benefit of the
+patent license for this particular work, or (3) arrange, in a manner
+consistent with the requirements of this License, to extend the patent
+license to downstream recipients. "Knowingly relying" means you have
+actual knowledge that, but for the patent license, your conveying the
+covered work in a country, or your recipient's use of the covered work
+in a country, would infringe one or more identifiable patents in that
+country that you have reason to believe are valid.
+
+ If, pursuant to or in connection with a single transaction or
+arrangement, you convey, or propagate by procuring conveyance of, a
+covered work, and grant a patent license to some of the parties
+receiving the covered work authorizing them to use, propagate, modify
+or convey a specific copy of the covered work, then the patent license
+you grant is automatically extended to all recipients of the covered
+work and works based on it.
+
+ A patent license is "discriminatory" if it does not include within
+the scope of its coverage, prohibits the exercise of, or is
+conditioned on the non-exercise of one or more of the rights that are
+specifically granted under this License. You may not convey a covered
+work if you are a party to an arrangement with a third party that is
+in the business of distributing software, under which you make payment
+to the third party based on the extent of your activity of conveying
+the work, and under which the third party grants, to any of the
+parties who would receive the covered work from you, a discriminatory
+patent license (a) in connection with copies of the covered work
+conveyed by you (or copies made from those copies), or (b) primarily
+for and in connection with specific products or compilations that
+contain the covered work, unless you entered into that arrangement,
+or that patent license was granted, prior to 28 March 2007.
+
+ Nothing in this License shall be construed as excluding or limiting
+any implied license or other defenses to infringement that may
+otherwise be available to you under applicable patent law.
+
+ 12. No Surrender of Others' Freedom.
+
+ If conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License. If you cannot convey a
+covered work so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you may
+not convey it at all. For example, if you agree to terms that obligate you
+to collect a royalty for further conveying from those to whom you convey
+the Program, the only way you could satisfy both those terms and this
+License would be to refrain entirely from conveying the Program.
+
+ 13. Remote Network Interaction; Use with the GNU General Public License.
+
+ Notwithstanding any other provision of this License, if you modify the
+Program, your modified version must prominently offer all users
+interacting with it remotely through a computer network (if your version
+supports such interaction) an opportunity to receive the Corresponding
+Source of your version by providing access to the Corresponding Source
+from a network server at no charge, through some standard or customary
+means of facilitating copying of software. This Corresponding Source
+shall include the Corresponding Source for any work covered by version 3
+of the GNU General Public License that is incorporated pursuant to the
+following paragraph.
+
+ Notwithstanding any other provision of this License, you have
+permission to link or combine any covered work with a work licensed
+under version 3 of the GNU General Public License into a single
+combined work, and to convey the resulting work. The terms of this
+License will continue to apply to the part which is the covered work,
+but the work with which it is combined will remain governed by version
+3 of the GNU General Public License.
+
+ 14. Revised Versions of this License.
+
+ The Free Software Foundation may publish revised and/or new versions of
+the GNU Affero General Public License from time to time. Such new versions
+will be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+ Each version is given a distinguishing version number. If the
+Program specifies that a certain numbered version of the GNU Affero General
+Public License "or any later version" applies to it, you have the
+option of following the terms and conditions either of that numbered
+version or of any later version published by the Free Software
+Foundation. If the Program does not specify a version number of the
+GNU Affero General Public License, you may choose any version ever published
+by the Free Software Foundation.
+
+ If the Program specifies that a proxy can decide which future
+versions of the GNU Affero General Public License can be used, that proxy's
+public statement of acceptance of a version permanently authorizes you
+to choose that version for the Program.
+
+ Later license versions may give you additional or different
+permissions. However, no additional obligations are imposed on any
+author or copyright holder as a result of your choosing to follow a
+later version.
+
+ 15. Disclaimer of Warranty.
+
+ THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
+APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
+OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
+IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
+ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+ 16. Limitation of Liability.
+
+ IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
+USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
+DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
+PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
+EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGES.
+
+ 17. Interpretation of Sections 15 and 16.
+
+ If the disclaimer of warranty and limitation of liability provided
+above cannot be given local legal effect according to their terms,
+reviewing courts shall apply local law that most closely approximates
+an absolute waiver of all civil liability in connection with the
+Program, unless a warranty or assumption of liability accompanies a
+copy of the Program in return for a fee.
+
+ END OF TERMS AND CONDITIONS
+
+ How to Apply These Terms to Your New Programs
+
+ If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+ To do so, attach the following notices to the program. It is safest
+to attach them to the start of each source file to most effectively
+state the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+
+ Copyright (C)
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with this program. If not, see .
+
+Also add information on how to contact you by electronic and paper mail.
+
+ If your software can interact with users remotely through a computer
+network, you should also make sure that it provides a way for users to
+get its source. For example, if your program is a web application, its
+interface could display a "Source" link that leads users to an archive
+of the code. There are many ways you could offer source, and different
+solutions will be better for different programs; see section 13 for the
+specific requirements.
+
+ You should also get your employer (if you work as a programmer) or school,
+if any, to sign a "copyright disclaimer" for the program, if necessary.
+For more information on this, and how to apply and follow the GNU AGPL, see
+.
diff --git a/enterprise/b/btree.go b/enterprise/b/btree.go
new file mode 100644
index 000000000..3d4c09888
--- /dev/null
+++ b/enterprise/b/btree.go
@@ -0,0 +1,962 @@
+// This file is a modified redistribution of b (https://github.com/cznic/b),
+// which is governed by the following license notice:
+//
+// Copyright (c) 2014 The b Authors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+// * Neither the names of the authors nor the names of the
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+package b
+
+import (
+ "fmt"
+ "io"
+ "sync"
+
+ "github.com/pilosa/pilosa/roaring"
+)
+
+const (
+ kx = 128 //TODO benchmark tune this number if using custom key/value type(s).
+ kd = 128 //TODO benchmark tune this number if using custom key/value type(s).
+)
+
+func init() {
+ if kd < 1 {
+ panic(fmt.Errorf("kd %d: out of range", kd))
+ }
+
+ if kx < 2 {
+ panic(fmt.Errorf("kx %d: out of range", kx))
+ }
+}
+
+var (
+ btDPool = sync.Pool{New: func() interface{} { return &d{} }}
+ btEPool = btEpool{sync.Pool{New: func() interface{} { return &Enumerator{} }}}
+ btTPool = btTpool{sync.Pool{New: func() interface{} { return &Tree{} }}}
+ btXPool = sync.Pool{New: func() interface{} { return &x{} }}
+)
+
+type btTpool struct{ sync.Pool }
+
+func (p *btTpool) get(cmp Cmp) *Tree {
+ x := p.Get().(*Tree)
+ x.cmp = cmp
+ return x
+}
+
+type btEpool struct{ sync.Pool }
+
+func (p *btEpool) get(err error, hit bool, i int, k uint64, q *d, t *Tree, ver int64) *Enumerator {
+ x := p.Get().(*Enumerator)
+ x.err, x.hit, x.i, x.k, x.q, x.t, x.ver = err, hit, i, k, q, t, ver
+ return x
+}
+
+type (
+ // Cmp compares a and b. Return value is:
+ //
+ // < 0 if a < b
+ // 0 if a == b
+ // > 0 if a > b
+ //
+ Cmp func(a, b uint64) int
+
+ d struct { // data page
+ c int
+ d [2*kd + 1]de
+ n *d
+ p *d
+ }
+
+ de struct { // d element
+ k uint64
+ v *roaring.Container
+ }
+
+ // Enumerator captures the state of enumerating a tree. It is returned
+ // from the Seek* methods. The enumerator is aware of any mutations
+ // made to the tree in the process of enumerating it and automatically
+ // resumes the enumeration at the proper key, if possible.
+ //
+ // However, once an Enumerator returns io.EOF to signal "no more
+ // items", it does no more attempt to "resync" on tree mutation(s). In
+ // other words, io.EOF from an Enumerator is "sticky" (idempotent).
+ Enumerator struct {
+ err error
+ hit bool
+ i int
+ k uint64
+ q *d
+ t *Tree
+ ver int64
+ }
+
+ // Tree is a B+tree.
+ Tree struct {
+ c int
+ cmp Cmp
+ first *d
+ last *d
+ r interface{}
+ ver int64
+ }
+
+ xe struct { // x element
+ ch interface{}
+ k uint64
+ }
+
+ x struct { // index page
+ c int
+ x [2*kx + 2]xe
+ }
+)
+
+var ( // R/O zero values
+ zd d
+ zde de
+ ze Enumerator
+ zk uint64
+ zt Tree
+ zx x
+ zxe xe
+)
+
+func clr(q interface{}) {
+ switch x := q.(type) {
+ case *x:
+ for i := 0; i <= x.c; i++ { // Ch0 Sep0 ... Chn-1 Sepn-1 Chn
+ clr(x.x[i].ch)
+ }
+ *x = zx
+ btXPool.Put(x)
+ case *d:
+ *x = zd
+ btDPool.Put(x)
+ }
+}
+
+// -------------------------------------------------------------------------- x
+
+func newX(ch0 interface{}) *x {
+ r := btXPool.Get().(*x)
+ r.x[0].ch = ch0
+ return r
+}
+
+func (q *x) extract(i int) {
+ q.c--
+ if i < q.c {
+ copy(q.x[i:], q.x[i+1:q.c+1])
+ q.x[q.c].ch = q.x[q.c+1].ch
+ q.x[q.c].k = zk // GC
+ q.x[q.c+1] = zxe // GC
+ }
+}
+
+func (q *x) insert(i int, k uint64, ch interface{}) *x {
+ c := q.c
+ if i < c {
+ q.x[c+1].ch = q.x[c].ch
+ copy(q.x[i+2:], q.x[i+1:c])
+ q.x[i+1].k = q.x[i].k
+ }
+ c++
+ q.c = c
+ q.x[i].k = k
+ q.x[i+1].ch = ch
+ return q
+}
+
+func (q *x) siblings(i int) (l, r *d) {
+ if i >= 0 {
+ if i > 0 {
+ l = q.x[i-1].ch.(*d)
+ }
+ if i < q.c {
+ r = q.x[i+1].ch.(*d)
+ }
+ }
+ return
+}
+
+// -------------------------------------------------------------------------- d
+
+func (l *d) mvL(r *d, c int) {
+ copy(l.d[l.c:], r.d[:c])
+ copy(r.d[:], r.d[c:r.c])
+ // Zero out the de's here to prevent reading bad data
+ // and to avoid creating non-collectible (GC) references.
+ for i := 1; i < c; i++ {
+ r.d[r.c-i] = zde
+ }
+ l.c += c
+ r.c -= c
+}
+
+func (l *d) mvR(r *d, c int) {
+ copy(r.d[c:], r.d[:r.c])
+ copy(r.d[:c], l.d[l.c-c:])
+ // Zero out the de's here to prevent reading bad data
+ // and to avoid creating non-collectible (GC) references.
+ for i := 1; i < c; i++ {
+ l.d[l.c-c+i] = zde
+ }
+ r.c += c
+ l.c -= c
+}
+
+// ----------------------------------------------------------------------- Tree
+
+// TreeNew returns a newly created, empty Tree. The compare function is used
+// for key collation.
+func TreeNew(cmp Cmp) *Tree {
+ return btTPool.get(cmp)
+}
+
+// Clear removes all K/V pairs from the tree.
+func (t *Tree) Clear() {
+ if t.r == nil {
+ return
+ }
+
+ clr(t.r)
+ t.c, t.first, t.last, t.r = 0, nil, nil, nil
+ t.ver++
+}
+
+// Close performs Clear and recycles t to a pool for possible later reuse. No
+// references to t should exist or such references must not be used afterwards.
+func (t *Tree) Close() {
+ t.Clear()
+ *t = zt
+ btTPool.Put(t)
+}
+
+func (t *Tree) cat(p *x, q, r *d, pi int) {
+ t.ver++
+ q.mvL(r, r.c)
+ if r.n != nil {
+ r.n.p = q
+ } else {
+ t.last = q
+ }
+ q.n = r.n
+ *r = zd
+ btDPool.Put(r)
+ if p.c > 1 {
+ p.extract(pi)
+ p.x[pi].ch = q
+ return
+ }
+
+ switch x := t.r.(type) {
+ case *x:
+ *x = zx
+ btXPool.Put(x)
+ case *d:
+ *x = zd
+ btDPool.Put(x)
+ }
+ t.r = q
+}
+
+func (t *Tree) catX(p, q, r *x, pi int) {
+ t.ver++
+ q.x[q.c].k = p.x[pi].k
+ copy(q.x[q.c+1:], r.x[:r.c])
+ q.c += r.c + 1
+ q.x[q.c].ch = r.x[r.c].ch
+ *r = zx
+ btXPool.Put(r)
+ if p.c > 1 {
+ p.c--
+ pc := p.c
+ if pi < pc {
+ p.x[pi].k = p.x[pi+1].k
+ copy(p.x[pi+1:], p.x[pi+2:pc+1])
+ p.x[pc].ch = p.x[pc+1].ch
+ p.x[pc].k = zk // GC
+ p.x[pc+1].ch = nil // GC
+ }
+ return
+ }
+
+ switch x := t.r.(type) {
+ case *x:
+ *x = zx
+ btXPool.Put(x)
+ case *d:
+ *x = zd
+ btDPool.Put(x)
+ }
+ t.r = q
+}
+
+// Delete removes the k's KV pair, if it exists, in which case Delete returns
+// true.
+func (t *Tree) Delete(k uint64) (ok bool) {
+ pi := -1
+ var p *x
+ q := t.r
+ if q == nil {
+ return false
+ }
+
+ for {
+ var i int
+ i, ok = t.find(q, k)
+ if ok {
+ switch x := q.(type) {
+ case *x:
+ if x.c < kx && q != t.r {
+ x, i = t.underflowX(p, x, pi, i)
+ }
+ pi = i + 1
+ p = x
+ q = x.x[pi].ch
+ continue
+ case *d:
+ t.extract(x, i)
+ if x.c >= kd {
+ return true
+ }
+
+ if q != t.r {
+ t.underflow(p, x, pi)
+ } else if t.c == 0 {
+ t.Clear()
+ }
+ return true
+ }
+ }
+
+ switch x := q.(type) {
+ case *x:
+ if x.c < kx && q != t.r {
+ x, i = t.underflowX(p, x, pi, i)
+ }
+ pi = i
+ p = x
+ q = x.x[i].ch
+ case *d:
+ return false
+ }
+ }
+}
+
+func (t *Tree) extract(q *d, i int) { // (r *container) {
+ t.ver++
+ //r = q.d[i].v // prepared for Extract
+ q.c--
+ if i < q.c {
+ copy(q.d[i:], q.d[i+1:q.c+1])
+ }
+ q.d[q.c] = zde // GC
+ t.c--
+}
+
+func (t *Tree) find(q interface{}, k uint64) (i int, ok bool) {
+ var mk uint64
+ l := 0
+ switch x := q.(type) {
+ case *x:
+ h := x.c - 1
+ for l <= h {
+ m := (l + h) >> 1
+ mk = x.x[m].k
+ switch cmp := t.cmp(k, mk); {
+ case cmp > 0:
+ l = m + 1
+ case cmp == 0:
+ return m, true
+ default:
+ h = m - 1
+ }
+ }
+ case *d:
+ h := x.c - 1
+ for l <= h {
+ m := (l + h) >> 1
+ mk = x.d[m].k
+ switch cmp := t.cmp(k, mk); {
+ case cmp > 0:
+ l = m + 1
+ case cmp == 0:
+ return m, true
+ default:
+ h = m - 1
+ }
+ }
+ }
+ return l, false
+}
+
+// First returns the first item of the tree in the key collating order, or
+// (zero-value, zero-value) if the tree is empty.
+func (t *Tree) First() (k uint64, v *roaring.Container) {
+ if q := t.first; q != nil {
+ q := &q.d[0]
+ k, v = q.k, q.v
+ }
+ return
+}
+
+// Get returns the value associated with k and true if it exists. Otherwise Get
+// returns (zero-value, false).
+func (t *Tree) Get(k uint64) (v *roaring.Container, ok bool) {
+ q := t.r
+ if q == nil {
+ return
+ }
+
+ for {
+ var i int
+ if i, ok = t.find(q, k); ok {
+ switch x := q.(type) {
+ case *x:
+ q = x.x[i+1].ch
+ continue
+ case *d:
+ return x.d[i].v, true
+ }
+ }
+ switch x := q.(type) {
+ case *x:
+ q = x.x[i].ch
+ default:
+ return
+ }
+ }
+}
+
+func (t *Tree) insert(q *d, i int, k uint64, v *roaring.Container) *d {
+ t.ver++
+ c := q.c
+ if i < c {
+ copy(q.d[i+1:], q.d[i:c])
+ }
+ c++
+ q.c = c
+ q.d[i].k, q.d[i].v = k, v
+ t.c++
+ return q
+}
+
+// Last returns the last item of the tree in the key collating order, or
+// (zero-value, zero-value) if the tree is empty.
+func (t *Tree) Last() (k uint64, v *roaring.Container) {
+ if q := t.last; q != nil {
+ q := &q.d[q.c-1]
+ k, v = q.k, q.v
+ }
+ return
+}
+
+// Len returns the number of items in the tree.
+func (t *Tree) Len() int {
+ return t.c
+}
+
+func (t *Tree) overflow(p *x, q *d, pi, i int, k uint64, v *roaring.Container) {
+ t.ver++
+ l, r := p.siblings(pi)
+
+ // s is the number of items to shift out of the full data container to
+ // allow for the new data item. This logic shifts by half the available
+ // space plus one. In the case where the new item is to be inserted within
+ // the calculated shift space, then s is reduced to include only the
+ // data items up to the index of the new data item.
+ if l != nil && l.c < 2*kd && i != 0 {
+ s := (2*kd-l.c)/2 + 1 // half plus one
+ //s := 2*kd - l.c // all avaiable
+ if i < s {
+ s = i
+ }
+ l.mvL(q, s)
+ t.insert(q, i-s, k, v)
+ p.x[pi-1].k = q.d[0].k
+ return
+ }
+
+ if r != nil && r.c < 2*kd {
+ if i < 2*kd {
+ s := (2*kd-r.c)/2 + 1 // half plus one
+ //s := 2*kd - r.c // all available
+ if 2*kd-i < s {
+ s = 2*kd - i
+ }
+ q.mvR(r, s)
+ t.insert(q, i, k, v)
+ p.x[pi].k = r.d[0].k
+ return
+ }
+
+ t.insert(r, 0, k, v)
+ p.x[pi].k = k
+ return
+ }
+
+ t.split(p, q, pi, i, k, v)
+}
+
+// Seek returns an Enumerator positioned on an item such that k >= item's key.
+// ok reports if k == item.key The Enumerator's position is possibly after the
+// last item in the tree.
+func (t *Tree) Seek(k uint64) (e *Enumerator, ok bool) {
+ q := t.r
+ if q == nil {
+ e = btEPool.get(nil, false, 0, k, nil, t, t.ver)
+ return
+ }
+
+ for {
+ var i int
+ if i, ok = t.find(q, k); ok {
+ switch x := q.(type) {
+ case *x:
+ q = x.x[i+1].ch
+ continue
+ case *d:
+ return btEPool.get(nil, ok, i, k, x, t, t.ver), true
+ }
+ }
+
+ switch x := q.(type) {
+ case *x:
+ q = x.x[i].ch
+ case *d:
+ return btEPool.get(nil, ok, i, k, x, t, t.ver), false
+ }
+ }
+}
+
+// SeekFirst returns an enumerator positioned on the first KV pair in the tree,
+// if any. For an empty tree, err == io.EOF is returned and e will be nil.
+func (t *Tree) SeekFirst() (e *Enumerator, err error) {
+ q := t.first
+ if q == nil {
+ return nil, io.EOF
+ }
+
+ return btEPool.get(nil, true, 0, q.d[0].k, q, t, t.ver), nil
+}
+
+// SeekLast returns an enumerator positioned on the last KV pair in the tree,
+// if any. For an empty tree, err == io.EOF is returned and e will be nil.
+func (t *Tree) SeekLast() (e *Enumerator, err error) {
+ q := t.last
+ if q == nil {
+ return nil, io.EOF
+ }
+
+ return btEPool.get(nil, true, q.c-1, q.d[q.c-1].k, q, t, t.ver), nil
+}
+
+// Set sets the value associated with k.
+func (t *Tree) Set(k uint64, v *roaring.Container) {
+ //dbg("--- PRE Set(%v, %v)\n%s", k, v, t.dump())
+ //defer func() {
+ // dbg("--- POST\n%s\n====\n", t.dump())
+ //}()
+
+ pi := -1
+ var p *x
+ q := t.r
+ if q == nil {
+ z := t.insert(btDPool.Get().(*d), 0, k, v)
+ t.r, t.first, t.last = z, z, z
+ return
+ }
+
+ for {
+ i, ok := t.find(q, k)
+ if ok {
+ switch x := q.(type) {
+ case *x:
+ i++
+ if x.c > 2*kx {
+ x, i = t.splitX(p, x, pi, i)
+ }
+ pi = i
+ p = x
+ q = x.x[i].ch
+ continue
+ case *d:
+ x.d[i].v = v
+ }
+ return
+ }
+
+ switch x := q.(type) {
+ case *x:
+ if x.c > 2*kx {
+ x, i = t.splitX(p, x, pi, i)
+ }
+ pi = i
+ p = x
+ q = x.x[i].ch
+ case *d:
+ switch {
+ case x.c < 2*kd:
+ t.insert(x, i, k, v)
+ default:
+ t.overflow(p, x, pi, i, k, v)
+ }
+ return
+ }
+ }
+}
+
+// Put combines Get and Set in a more efficient way where the tree is walked
+// only once. The upd(ater) receives (old-value, true) if a KV pair for k
+// exists or (zero-value, false) otherwise. It can then return a (new-value,
+// true) to create or overwrite the existing value in the KV pair, or
+// (whatever, false) if it decides not to create or not to update the value of
+// the KV pair.
+//
+// tree.Set(k, v) call conceptually equals calling
+//
+// tree.Put(k, func(uint64, bool){ return v, true })
+//
+// modulo the differing return values.
+func (t *Tree) Put(k uint64, upd func(oldV *roaring.Container, exists bool) (newV *roaring.Container, write bool)) (oldV *roaring.Container, written bool) {
+ pi := -1
+ var p *x
+ q := t.r
+ var newV *roaring.Container
+ if q == nil {
+ // new KV pair in empty tree
+ newV, written = upd(newV, false)
+ if !written {
+ return
+ }
+
+ z := t.insert(btDPool.Get().(*d), 0, k, newV)
+ t.r, t.first, t.last = z, z, z
+ return
+ }
+
+ for {
+ i, ok := t.find(q, k)
+ if ok {
+ switch x := q.(type) {
+ case *x:
+ i++
+ if x.c > 2*kx {
+ x, i = t.splitX(p, x, pi, i)
+ }
+ pi = i
+ p = x
+ q = x.x[i].ch
+ continue
+ case *d:
+ oldV = x.d[i].v
+ newV, written = upd(oldV, true)
+ if !written {
+ return
+ }
+
+ x.d[i].v = newV
+ }
+ return
+ }
+
+ switch x := q.(type) {
+ case *x:
+ if x.c > 2*kx {
+ x, i = t.splitX(p, x, pi, i)
+ }
+ pi = i
+ p = x
+ q = x.x[i].ch
+ case *d: // new KV pair
+ newV, written = upd(newV, false)
+ if !written {
+ return
+ }
+
+ switch {
+ case x.c < 2*kd:
+ t.insert(x, i, k, newV)
+ default:
+ t.overflow(p, x, pi, i, k, newV)
+ }
+ return
+ }
+ }
+}
+
+func (t *Tree) split(p *x, q *d, pi, i int, k uint64, v *roaring.Container) {
+ t.ver++
+ r := btDPool.Get().(*d)
+ if q.n != nil {
+ r.n = q.n
+ r.n.p = r
+ } else {
+ t.last = r
+ }
+ q.n = r
+ r.p = q
+
+ copy(r.d[:], q.d[kd:2*kd])
+ for i := range q.d[kd:] {
+ q.d[kd+i] = zde
+ }
+ q.c = kd
+ r.c = kd
+ var done bool
+ if i > kd {
+ done = true
+ t.insert(r, i-kd, k, v)
+ }
+ if pi >= 0 {
+ p.insert(pi, r.d[0].k, r)
+ } else {
+ t.r = newX(q).insert(0, r.d[0].k, r)
+ }
+ if done {
+ return
+ }
+
+ t.insert(q, i, k, v)
+}
+
+func (t *Tree) splitX(p *x, q *x, pi int, i int) (*x, int) {
+ t.ver++
+ r := btXPool.Get().(*x)
+ copy(r.x[:], q.x[kx+1:])
+ q.c = kx
+ r.c = kx
+ if pi >= 0 {
+ p.insert(pi, q.x[kx].k, r)
+ } else {
+ t.r = newX(q).insert(0, q.x[kx].k, r)
+ }
+
+ q.x[kx].k = zk
+ for i := range q.x[kx+1:] {
+ q.x[kx+i+1] = zxe
+ }
+ if i > kx {
+ q = r
+ i -= kx + 1
+ }
+
+ return q, i
+}
+
+func (t *Tree) underflow(p *x, q *d, pi int) {
+ t.ver++
+ l, r := p.siblings(pi)
+
+ if l != nil && l.c+q.c >= 2*kd {
+ l.mvR(q, 1)
+ p.x[pi-1].k = q.d[0].k
+ return
+ }
+
+ if r != nil && q.c+r.c >= 2*kd {
+ q.mvL(r, 1)
+ p.x[pi].k = r.d[0].k
+ r.d[r.c] = zde // GC
+ return
+ }
+
+ if l != nil {
+ t.cat(p, l, q, pi-1)
+ return
+ }
+
+ t.cat(p, q, r, pi)
+}
+
+func (t *Tree) underflowX(p *x, q *x, pi int, i int) (*x, int) {
+ t.ver++
+ var l, r *x
+
+ if pi >= 0 {
+ if pi > 0 {
+ l = p.x[pi-1].ch.(*x)
+ }
+ if pi < p.c {
+ r = p.x[pi+1].ch.(*x)
+ }
+ }
+
+ if l != nil && l.c > kx {
+ q.x[q.c+1].ch = q.x[q.c].ch
+ copy(q.x[1:], q.x[:q.c])
+ q.x[0].ch = l.x[l.c].ch
+ q.x[0].k = p.x[pi-1].k
+ q.c++
+ i++
+ l.c--
+ p.x[pi-1].k = l.x[l.c].k
+ return q, i
+ }
+
+ if r != nil && r.c > kx {
+ q.x[q.c].k = p.x[pi].k
+ q.c++
+ q.x[q.c].ch = r.x[0].ch
+ p.x[pi].k = r.x[0].k
+ copy(r.x[:], r.x[1:r.c])
+ r.c--
+ rc := r.c
+ r.x[rc].ch = r.x[rc+1].ch
+ r.x[rc].k = zk
+ r.x[rc+1].ch = nil
+ return q, i
+ }
+
+ if l != nil {
+ i += l.c + 1
+ t.catX(p, l, q, pi-1)
+ q = l
+ return q, i
+ }
+
+ t.catX(p, q, r, pi)
+ return q, i
+}
+
+// ----------------------------------------------------------------- Enumerator
+
+// Close recycles e to a pool for possible later reuse. No references to e
+// should exist or such references must not be used afterwards.
+func (e *Enumerator) Close() {
+ *e = ze
+ btEPool.Put(e)
+}
+
+// Next returns the currently enumerated item, if it exists and moves to the
+// next item in the key collation order. If there is no item to return, err ==
+// io.EOF is returned.
+func (e *Enumerator) Next() (k uint64, v *roaring.Container, err error) {
+ if err = e.err; err != nil {
+ return
+ }
+
+ if e.ver != e.t.ver {
+ f, _ := e.t.Seek(e.k)
+ *e = *f
+ f.Close()
+ }
+ if e.q == nil {
+ e.err, err = io.EOF, io.EOF
+ return
+ }
+
+ if e.i >= e.q.c {
+ if err = e.next(); err != nil {
+ return
+ }
+ }
+
+ i := e.q.d[e.i]
+ k, v = i.k, i.v
+ e.k, e.hit = k, true
+ e.next()
+ return
+}
+
+func (e *Enumerator) next() error {
+ if e.q == nil {
+ e.err = io.EOF
+ return io.EOF
+ }
+
+ switch {
+ case e.i < e.q.c-1:
+ e.i++
+ default:
+ if e.q, e.i = e.q.n, 0; e.q == nil {
+ e.err = io.EOF
+ }
+ }
+ return e.err
+}
+
+// Prev returns the currently enumerated item, if it exists and moves to the
+// previous item in the key collation order. If there is no item to return, err
+// == io.EOF is returned.
+func (e *Enumerator) Prev() (k uint64, v *roaring.Container, err error) {
+ if err = e.err; err != nil {
+ return
+ }
+
+ if e.ver != e.t.ver {
+ f, _ := e.t.Seek(e.k)
+ *e = *f
+ f.Close()
+ }
+ if e.q == nil {
+ e.err, err = io.EOF, io.EOF
+ return
+ }
+
+ if !e.hit {
+ // move to previous because Seek overshoots if there's no hit
+ if err = e.prev(); err != nil {
+ return
+ }
+ }
+
+ if e.i >= e.q.c {
+ if err = e.prev(); err != nil {
+ return
+ }
+ }
+
+ i := e.q.d[e.i]
+ k, v = i.k, i.v
+ e.k, e.hit = k, true
+ e.prev()
+ return
+}
+
+func (e *Enumerator) prev() error {
+ if e.q == nil {
+ e.err = io.EOF
+ return io.EOF
+ }
+
+ switch {
+ case e.i > 0:
+ e.i--
+ default:
+ if e.q = e.q.p; e.q == nil {
+ e.err = io.EOF
+ break
+ }
+
+ e.i = e.q.c - 1
+ }
+ return e.err
+}
diff --git a/enterprise/b/containers_btree.go b/enterprise/b/containers_btree.go
new file mode 100644
index 000000000..257134a82
--- /dev/null
+++ b/enterprise/b/containers_btree.go
@@ -0,0 +1,186 @@
+// Copyright (c) 2018 Pilosa Corp. All rights reserved.
+//
+// This file is part of Pilosa Enterprise Edition.
+//
+// Pilosa Enterprise Edition is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Pilosa Enterprise Edition is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with Pilosa Enterprise Edition. If not, see .
+
+package b
+
+import (
+ "io"
+
+ "github.com/pilosa/pilosa/roaring"
+)
+
+func cmp(a, b uint64) int {
+ return int(a - b)
+}
+
+type BTreeContainers struct {
+ tree *Tree
+
+ lastKey uint64
+ lastContainer *roaring.Container
+}
+
+func NewBTreeContainers() *BTreeContainers {
+ return &BTreeContainers{
+ tree: TreeNew(cmp),
+ }
+}
+
+func NewBTreeBitmap(a ...uint64) *roaring.Bitmap {
+ b := &roaring.Bitmap{
+ Containers: NewBTreeContainers(),
+ }
+ b.Add(a...)
+ return b
+}
+
+func (btc *BTreeContainers) Get(key uint64) *roaring.Container {
+ // Check the last* cache for same container.
+ if key == btc.lastKey && btc.lastContainer != nil {
+ return btc.lastContainer
+ }
+
+ var c *roaring.Container
+ el, ok := btc.tree.Get(key)
+ if ok {
+ c = el
+ btc.lastKey = key
+ btc.lastContainer = c
+ }
+ return c
+}
+
+func (btc *BTreeContainers) Put(key uint64, c *roaring.Container) {
+ // If a mapped container is added to the tree, reset the
+ // lastContainer cache so that the cache is not pointing
+ // at a read-only mmap.
+ if c.Mapped() {
+ btc.lastContainer = nil
+ }
+ btc.tree.Set(key, c)
+}
+
+func (u updater) update(oldV *roaring.Container, exists bool) (*roaring.Container, bool) {
+ // update the existing container
+ if exists {
+ oldV.Update(u.containerType, u.n, u.mapped)
+ return oldV, false
+ }
+ cont := roaring.NewContainer()
+ cont.Update(u.containerType, u.n, u.mapped)
+ return cont, true
+}
+
+// this struct is added to prevent the closure locals from being escaped out to the heap
+type updater struct {
+ key uint64
+ containerType byte
+ n int
+ mapped bool
+}
+
+func (btc *BTreeContainers) PutContainerValues(key uint64, containerType byte, n int, mapped bool) {
+ a := updater{key, containerType, n, mapped}
+ btc.tree.Put(key, a.update)
+}
+
+func (btc *BTreeContainers) Remove(key uint64) {
+ btc.tree.Delete(key)
+}
+
+func (btc *BTreeContainers) GetOrCreate(key uint64) *roaring.Container {
+ // Check the last* cache for same container.
+ if key == btc.lastKey && btc.lastContainer != nil {
+ return btc.lastContainer
+ }
+
+ btc.lastKey = key
+ v, ok := btc.tree.Get(key)
+ if !ok {
+ cont := roaring.NewContainer()
+ btc.tree.Set(key, cont)
+ btc.lastContainer = cont
+ return cont
+ }
+
+ btc.lastContainer = v
+ return btc.lastContainer
+}
+
+func (btc *BTreeContainers) Clone() roaring.Containers {
+ nbtc := NewBTreeContainers()
+
+ itr, err := btc.tree.SeekFirst()
+ if err == io.EOF {
+ return nbtc
+ }
+ for {
+ k, v, err := itr.Next()
+ if err == io.EOF {
+ break
+ }
+ nbtc.tree.Set(k, v.Clone())
+ }
+ return nbtc
+}
+
+func (btc *BTreeContainers) Last() (key uint64, c *roaring.Container) {
+ if btc.tree.Len() == 0 {
+ return 0, nil
+ }
+ k, v := btc.tree.Last()
+ return k, v
+}
+
+func (btc *BTreeContainers) Size() int {
+ return btc.tree.Len()
+}
+
+func (btc *BTreeContainers) Iterator(key uint64) (citer roaring.ContainerIterator, found bool) {
+ e, ok := btc.tree.Seek(key)
+ if ok {
+ found = true
+ }
+
+ return &BTCIterator{
+ e: e,
+ }, found
+}
+
+type BTCIterator struct {
+ e *Enumerator
+ key uint64
+ val *roaring.Container
+}
+
+func (i *BTCIterator) Next() bool {
+
+ k, v, err := i.e.Next()
+ if err == io.EOF {
+ return false
+ }
+ i.key = k
+ i.val = v
+ return true
+}
+
+func (i *BTCIterator) Value() (uint64, *roaring.Container) {
+ if i.val == nil {
+ return 0, nil
+ }
+ return i.key, i.val
+}
diff --git a/enterprise/enterprise.go b/enterprise/enterprise.go
new file mode 100644
index 000000000..db9d6e2dd
--- /dev/null
+++ b/enterprise/enterprise.go
@@ -0,0 +1,32 @@
+// Copyright (c) 2018 Pilosa Corp. All rights reserved.
+//
+// This file is part of Pilosa Enterprise Edition.
+//
+// Pilosa Enterprise Edition is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Pilosa Enterprise Edition is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with Pilosa Enterprise Edition. If not, see .
+
+// Package enterprise injects enterprise implementations of various Pilosa
+// features when Pilosa is built with "ENTERPRISE=1 make install". These
+// features are dual-licensed separately from Pilosa community edition under the
+// AGPL and Pilosa's commercial license.
+package enterprise
+
+import (
+ "github.com/pilosa/pilosa/enterprise/b"
+ "github.com/pilosa/pilosa/roaring"
+)
+
+func init() {
+ // Replace Bitmap constructor with B+Tree implementation
+ roaring.NewFileBitmap = b.NewBTreeBitmap
+}
diff --git a/fragment.go b/fragment.go
index 2ce8eaca4..aab5d0524 100644
--- a/fragment.go
+++ b/fragment.go
@@ -189,7 +189,7 @@ func (f *Fragment) Open() error {
func (f *Fragment) openStorage() error {
// Create a roaring bitmap to serve as storage for the slice.
if f.storage == nil {
- f.storage = roaring.NewBitmap()
+ f.storage = roaring.NewFileBitmap()
}
// Open the data file to be mmap'd and used as an ops log.
file, err := os.OpenFile(f.path, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
diff --git a/fragment_test.go b/fragment_test.go
index 2f8c6e622..f72f03ee1 100644
--- a/fragment_test.go
+++ b/fragment_test.go
@@ -1156,12 +1156,14 @@ func BenchmarkFragment_Snapshot(b *testing.B) {
b.Skip("no fragment specified")
}
+ b.ReportAllocs()
// Open the fragment specified by the path.
f := pilosa.NewFragment(*FragmentPath, "i", "f", pilosa.ViewStandard, 0)
if err := f.Open(); err != nil {
b.Fatal(err)
}
defer f.Close()
+ b.ResetTimer()
// Reset timer and execute benchmark.
b.ResetTimer()
@@ -1173,6 +1175,7 @@ func BenchmarkFragment_Snapshot(b *testing.B) {
}
}
}
+
func BenchmarkFragment_FullSnapshot(b *testing.B) {
f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
defer f.Close()
diff --git a/roaring/containers.go b/roaring/containers.go
new file mode 100644
index 000000000..08c8d25eb
--- /dev/null
+++ b/roaring/containers.go
@@ -0,0 +1,161 @@
+// Copyright (C) 2017-2018 Pilosa Corp. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package roaring
+
+type SliceContainers struct {
+ keys []uint64
+ containers []*Container
+ lastKey uint64
+ lastContainer *Container
+}
+
+func NewSliceContainers() *SliceContainers {
+ return &SliceContainers{}
+}
+
+func (sc *SliceContainers) Get(key uint64) *Container {
+ i := search64(sc.keys, key)
+ if i < 0 {
+ return nil
+ }
+ return sc.containers[i]
+}
+
+func (sc *SliceContainers) Put(key uint64, c *Container) {
+ i := search64(sc.keys, key)
+
+ // If index is negative then there's not an exact match
+ // and a container needs to be added.
+ if i < 0 {
+ sc.insertAt(key, c, -i-1)
+ } else {
+ sc.containers[i] = c
+ }
+
+}
+
+func (sc *SliceContainers) PutContainerValues(key uint64, containerType byte, n int, mapped bool) {
+ i := search64(sc.keys, key)
+ if i < 0 {
+ c := NewContainer()
+ c.containerType = containerType
+ c.n = n
+ c.mapped = mapped
+ sc.insertAt(key, c, -i-1)
+ } else {
+ c := sc.containers[i]
+ c.containerType = containerType
+ c.n = n
+ c.mapped = mapped
+ }
+
+}
+
+func (sc *SliceContainers) Remove(key uint64) {
+ i := search64(sc.keys, key)
+ if i < 0 {
+ return
+ }
+ sc.keys = append(sc.keys[:i], sc.keys[i+1:]...)
+ sc.containers = append(sc.containers[:i], sc.containers[i+1:]...)
+
+}
+func (sc *SliceContainers) insertAt(key uint64, c *Container, i int) {
+ sc.keys = append(sc.keys, 0)
+ copy(sc.keys[i+1:], sc.keys[i:])
+ sc.keys[i] = key
+
+ sc.containers = append(sc.containers, nil)
+ copy(sc.containers[i+1:], sc.containers[i:])
+ sc.containers[i] = c
+}
+
+func (sc *SliceContainers) GetOrCreate(key uint64) *Container {
+ // Check the last* cache for same container.
+ if key == sc.lastKey && sc.lastContainer != nil {
+ return sc.lastContainer
+ }
+
+ sc.lastKey = key
+ i := search64(sc.keys, key)
+ if i < 0 {
+ c := NewContainer()
+ sc.insertAt(key, c, -i-1)
+ sc.lastContainer = c
+ return c
+ }
+
+ sc.lastContainer = sc.containers[i]
+ return sc.lastContainer
+}
+
+func (sc *SliceContainers) Clone() Containers {
+ other := NewSliceContainers()
+ other.keys = make([]uint64, len(sc.keys))
+ other.containers = make([]*Container, len(sc.containers))
+ copy(other.keys, sc.keys)
+ for i, c := range sc.containers {
+ other.containers[i] = c.Clone()
+ }
+ return other
+}
+
+func (sc *SliceContainers) Last() (key uint64, c *Container) {
+ if len(sc.keys) == 0 {
+ return 0, nil
+ }
+ return sc.keys[len(sc.keys)-1], sc.containers[len(sc.keys)-1]
+}
+
+func (sc *SliceContainers) Size() int {
+ return len(sc.keys)
+
+}
+
+func (sc *SliceContainers) seek(key uint64) (int, bool) {
+ i := search64(sc.keys, key)
+ found := true
+ if i < 0 {
+ found = false
+ i = -i - 1
+ }
+ return i, found
+}
+
+func (sc *SliceContainers) Iterator(key uint64) (citer ContainerIterator, found bool) {
+ i, found := sc.seek(key)
+ return &SliceIterator{e: sc, i: i}, found
+}
+
+type SliceIterator struct {
+ e *SliceContainers
+ i int
+ key uint64
+ value *Container
+}
+
+func (si *SliceIterator) Next() bool {
+ if si.e == nil || si.i > len(si.e.keys)-1 {
+ return false
+ }
+ si.key = si.e.keys[si.i]
+ si.value = si.e.containers[si.i]
+ si.i++
+ return true
+}
+
+func (si *SliceIterator) Value() (uint64, *Container) {
+ return si.key, si.value
+}
diff --git a/roaring/containers_test.go b/roaring/containers_test.go
new file mode 100644
index 000000000..ea3106a96
--- /dev/null
+++ b/roaring/containers_test.go
@@ -0,0 +1,102 @@
+// Copyright (C) 2017-2018 Pilosa Corp. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package roaring
+
+import (
+ "testing"
+)
+
+func TestContainersIterator(t *testing.T) {
+ slc := NewFileBitmap().Containers
+ testContainersIterator(slc, t)
+}
+
+func testContainersIterator(cs Containers, t *testing.T) {
+ itr, found := cs.Iterator(0)
+ if found {
+ t.Fatalf("shouldn't have found 0 in empty btc")
+ }
+ if itr.Next() {
+ t.Fatal("Next() should be false for empty btc")
+ }
+
+ cs.Put(1, &Container{n: 1})
+ cs.Put(2, &Container{n: 2})
+
+ itr, found = cs.Iterator(0)
+ if found {
+ t.Fatalf("shouldn't have found 0")
+ }
+
+ if !itr.Next() {
+ t.Fatalf("one should be next, but got false")
+ }
+ if key, val := itr.Value(); key != 1 || val.n != 1 {
+ t.Fatalf("Wrong k/v, exp: 1,1 got: %v,%v", key, val.n)
+ }
+ if !itr.Next() {
+ t.Fatalf("two should be next, but got false")
+ }
+ if key, val := itr.Value(); key != 2 || val.n != 2 {
+ t.Fatalf("Wrong k/v, exp: 2,2 got: %v,%v", key, val.n)
+ }
+
+ if itr.Next() {
+ t.Fatalf("itr should be done, but got true")
+ }
+
+ cs.Put(3, &Container{n: 3})
+ cs.Put(5, &Container{n: 5})
+ cs.Put(6, &Container{n: 6})
+
+ itr, found = cs.Iterator(3)
+ if !itr.Next() {
+ t.Fatalf("3 should be next, but got false")
+ }
+ if !found {
+ t.Fatalf("should have found 3")
+ }
+ if key, val := itr.Value(); key != 3 || val.n != 3 {
+ t.Fatalf("Wrong k/v, exp: 3,3 got: %v,%v", key, val.n)
+ }
+ if !itr.Next() {
+ t.Fatalf("5 should be next, but got false")
+ }
+ if key, val := itr.Value(); key != 5 || val.n != 5 {
+ t.Fatalf("Wrong k/v, exp: 5,5 got: %v,%v", key, val.n)
+ }
+
+ itr, found = cs.Iterator(4)
+ if found {
+ t.Fatalf("shouldn't have found 4")
+ }
+ if !itr.Next() {
+ t.Fatalf("5 should be next, but got false")
+ }
+ if key, val := itr.Value(); key != 5 || val.n != 5 {
+ t.Fatalf("Wrong k/v, exp: 5,5 got: %v,%v", key, val.n)
+ }
+ if !itr.Next() {
+ t.Fatalf("6 should be next, but got false")
+ }
+ if key, val := itr.Value(); key != 6 || val.n != 6 {
+ t.Fatalf("Wrong k/v, exp: 6,6 got: %v,%v", key, val.n)
+ }
+
+ if itr.Next() {
+ t.Fatalf("itr should be done, but got true")
+ }
+
+}
diff --git a/roaring/roaring.go b/roaring/roaring.go
index 2a8b6645c..cc8466a5f 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -63,10 +63,46 @@ const (
maxContainerVal = 0xffff
)
+type Containers interface {
+ // Get returns nil if the key does not exist.
+ Get(key uint64) *Container
+
+ // Put adds the container at key.
+ Put(key uint64, c *Container)
+
+ // PutContainerValues updates an existing container at key.
+ // If a container does not exist for key, a new one is allocated.
+ PutContainerValues(key uint64, containerType byte, n int, mapped bool)
+
+ // Remove takes the container at key out.
+ Remove(key uint64)
+
+ // GetOrCreate returns the container at key, creating a new empty container if necessary.
+ GetOrCreate(key uint64) *Container
+
+ // Clone does a deep copy of Containers, including cloning all containers contained.
+ Clone() Containers
+
+ // Last returns the highest key and associated container.
+ Last() (key uint64, c *Container)
+
+ // Size returns the number of containers stored.
+ Size() int
+
+ // Iterator returns a Contiterator which after a call to Next(), a call to Value() will
+ // return the first container at or after key. found will be true if a
+ // container is found at key.
+ Iterator(key uint64) (citer ContainerIterator, found bool)
+}
+
+type ContainerIterator interface {
+ Next() bool
+ Value() (uint64, *Container)
+}
+
// Bitmap represents a roaring bitmap.
type Bitmap struct {
- keys []uint64 // keys for containers
- containers []*container // array, bitmap and RLE containers
+ Containers Containers
// Number of operations written to the writer.
opN int
@@ -77,11 +113,17 @@ type Bitmap struct {
// NewBitmap returns a Bitmap with an initial set of values.
func NewBitmap(a ...uint64) *Bitmap {
- b := &Bitmap{}
+ b := &Bitmap{
+ Containers: NewSliceContainers(),
+ }
b.Add(a...)
return b
}
+// NewFileBitmap returns a Bitmap with an initial set of values, used for file storage.
+// By default, this is a copy of NewBitmap, but is replaced with B+Tree in server/enterprise.go
+var NewFileBitmap func(a ...uint64) *Bitmap = NewBitmap
+
// Clone returns a heap allocated copy of the bitmap.
// Note: The OpWriter IS NOT copied to the new bitmap.
func (b *Bitmap) Clone() *Bitmap {
@@ -91,14 +133,7 @@ func (b *Bitmap) Clone() *Bitmap {
// Create a copy of the bitmap structure.
other := &Bitmap{
- keys: make([]uint64, len(b.keys)),
- containers: make([]*container, len(b.containers)),
- }
-
- // Copy keys & clone containers.
- copy(other.keys, b.keys)
- for i, c := range b.containers {
- other.containers[i] = c.clone()
+ Containers: b.Containers.Clone(),
}
return other
@@ -127,21 +162,13 @@ func (b *Bitmap) Add(a ...uint64) (changed bool, err error) {
}
func (b *Bitmap) add(v uint64) bool {
- hb := highbits(v)
- i := search64(b.keys, hb)
-
- // If index is negative then there's not an exact match
- // and a container needs to be added.
- if i < 0 {
- b.insertAt(hb, newContainer(), -i-1)
- i = -i - 1
- }
- return b.containers[i].add(lowbits(v))
+ cont := b.Containers.GetOrCreate(highbits(v))
+ return cont.add(lowbits(v))
}
// Contains returns true if v is in the bitmap.
func (b *Bitmap) Contains(v uint64) bool {
- c := b.container(highbits(v))
+ c := b.Containers.Get(highbits(v))
if c == nil {
return false
}
@@ -169,76 +196,75 @@ func (b *Bitmap) Remove(a ...uint64) (changed bool, err error) {
}
func (b *Bitmap) remove(v uint64) bool {
- hb := highbits(v)
- i := search64(b.keys, hb)
- if i < 0 {
+ c := b.Containers.Get(highbits(v))
+ if c == nil {
return false
}
- return b.containers[i].remove(lowbits(v))
+ // TODO - do nil check inside c.remove?
+ return c.remove(lowbits(v))
}
// Max returns the highest value in the bitmap.
// Returns zero if the bitmap is empty.
func (b *Bitmap) Max() uint64 {
- if len(b.keys) == 0 {
+ if b.Containers.Size() == 0 {
return 0
}
- hb := b.keys[len(b.keys)-1]
- lb := b.containers[len(b.containers)-1].max()
+ hb, c := b.Containers.Last()
+ lb := c.max()
return hb<<16 | uint64(lb)
}
// Count returns the number of bits set in the bitmap.
func (b *Bitmap) Count() (n uint64) {
- for _, container := range b.containers {
- n += uint64(container.n)
+ citer, _ := b.Containers.Iterator(0)
+ for citer.Next() {
+ _, c := citer.Value()
+ n += uint64(c.n)
}
return n
}
// CountRange returns the number of bits set between [start, end).
func (b *Bitmap) CountRange(start, end uint64) (n uint64) {
- if len(b.keys) == 0 {
+ if b.Containers.Size() == 0 {
return
}
skey := highbits(start)
ekey := highbits(end)
- i := search64(b.keys, skey)
- j := search64(b.keys, ekey)
-
+ citer, found := b.Containers.Iterator(highbits(start))
// If range is entirely in one container then just count that range.
- if i >= 0 && i == j {
- return uint64(b.containers[i].countRange(int(lowbits(start)), int(lowbits(end))))
+ if found && skey == ekey {
+ citer.Next()
+ _, c := citer.Value()
+ return uint64(c.countRange(int(lowbits(start)), int(lowbits(end))))
}
- if i < 0 {
- // start's container did not exist
- // set i to the index of the first container we have with values higher than start
- i = -i - 1
- } else {
- // Count first partial container and advance i so we don't recount it
- n += uint64(b.containers[i].countRange(int(lowbits(start)), maxContainerVal+1))
- i++
+ for citer.Next() {
+ k, c := citer.Value()
+ if k < skey {
+ // TODO remove once we've validated this stuff works
+ panic("should be impossible for k to be less than skey")
+ }
+ if k == skey {
+ n += uint64(c.countRange(int(lowbits(start)), maxContainerVal+1))
+ continue
+ }
+ if k < ekey {
+ n += uint64(c.n)
+ continue
+ }
+ if k == ekey {
+ n += uint64(c.countRange(0, int(lowbits(end))))
+ break
+ }
+ if k > ekey {
+ break
+ }
}
-
- // Count last container.
- if j < 0 {
- // end's container did not exist
- // set j to the index of the first container with values higher than end (or len(containers))
- j = -j - 1
- } else {
- // end's container exists, count it up to end
- n += uint64(b.containers[j].countRange(0, int(lowbits(end))))
- }
-
- // Count containers in between.
- for x := i; x < j; x++ {
- n += uint64(b.containers[x].n)
- }
-
return n
}
@@ -247,6 +273,7 @@ func (b *Bitmap) Slice() []uint64 {
var a []uint64
itr := b.Iterator()
itr.Seek(0)
+
for v, eof := itr.Next(); !eof; v, eof = itr.Next() {
a = append(a, v)
}
@@ -296,44 +323,21 @@ func (b *Bitmap) OffsetRange(offset, start, end uint64) *Bitmap {
off := highbits(offset)
hi0, hi1 := highbits(start), highbits(end)
-
- // Find starting container.
- n := len(b.containers)
- i := sort.Search(n, func(i int) bool { return b.keys[i] >= hi0 })
-
- var other Bitmap
- for ; i < n; i++ {
- key := b.keys[i]
-
- // If we've exceeded the upper bound then exit.
- if key >= hi1 {
+ citer, _ := b.Containers.Iterator(hi0)
+ other := NewBitmap()
+ for citer.Next() {
+ k, c := citer.Value()
+ if k >= hi1 {
break
}
-
- // Otherwise append container with offset key.
- other.keys = append(other.keys, off+(key-hi0))
- other.containers = append(other.containers, b.containers[i])
+ other.Containers.Put(off+(k-hi0), c)
}
- return &other
+ return other
}
// container returns the container with the given key.
-func (b *Bitmap) container(key uint64) *container {
- i := search64(b.keys, key)
- if i < 0 {
- return nil
- }
- return b.containers[i]
-}
-
-func (b *Bitmap) insertAt(key uint64, c *container, i int) {
- b.keys = append(b.keys, 0)
- copy(b.keys[i+1:], b.keys[i:])
- b.keys[i] = key
-
- b.containers = append(b.containers, nil)
- copy(b.containers[i+1:], b.containers[i:])
- b.containers[i] = c
+func (b *Bitmap) container(key uint64) *Container {
+ return b.Containers.Get(key)
}
// IntersectionCount returns the number of set bits that would result in an
@@ -341,15 +345,23 @@ func (b *Bitmap) insertAt(key uint64, c *container, i int) {
// intersecting the two and counting the result.
func (b *Bitmap) IntersectionCount(other *Bitmap) uint64 {
var n uint64
- for i, j := 0, 0; i < len(b.containers) && j < len(other.containers); {
- ki, kj := b.keys[i], other.keys[j]
+ iiter, _ := b.Containers.Iterator(0)
+ jiter, _ := other.Containers.Iterator(0)
+ i, j := iiter.Next(), jiter.Next()
+ ki, ci := iiter.Value()
+ kj, cj := jiter.Value()
+ for i && j {
if ki < kj {
- i++
+ i = iiter.Next()
+ ki, ci = iiter.Value()
} else if ki > kj {
- j++
+ j = jiter.Next()
+ kj, cj = jiter.Value()
} else {
- n += uint64(intersectionCount(b.containers[i], other.containers[j]))
- i, j = i+1, j+1
+ n += uint64(intersectionCount(ci, cj))
+ i, j = iiter.Next(), jiter.Next()
+ ki, ci = iiter.Value()
+ kj, cj = jiter.Value()
}
}
return n
@@ -357,95 +369,79 @@ func (b *Bitmap) IntersectionCount(other *Bitmap) uint64 {
// Intersect returns the intersection of b and other.
func (b *Bitmap) Intersect(other *Bitmap) *Bitmap {
- output := &Bitmap{}
-
- ki, ci := b.keys, b.containers
- kj, cj := other.keys, other.containers
- for {
- var key uint64
- var container *container
-
- ni, nj := len(ki), len(kj)
- if ni == 0 && nj == 0 { // eof(i,j)
- break
- } else if ni == 0 || (nj != 0 && ki[0] > kj[0]) { // eof(i) or i > j
- kj, cj = kj[1:], cj[1:]
- } else if nj == 0 || (ki[0] < kj[0]) { // eof(j) or i < j
- ki, ci = ki[1:], ci[1:]
- } else { // i == j
- key, container = ki[0], intersect(ci[0], cj[0])
- ki, ci = ki[1:], ci[1:]
- kj, cj = kj[1:], cj[1:]
- output.keys = append(output.keys, key)
- output.containers = append(output.containers, container)
+ output := NewBitmap()
+ iiter, _ := b.Containers.Iterator(0)
+ jiter, _ := other.Containers.Iterator(0)
+ i, j := iiter.Next(), jiter.Next()
+ ki, ci := iiter.Value()
+ kj, cj := jiter.Value()
+ for i && j {
+ if ki < kj {
+ i = iiter.Next()
+ ki, ci = iiter.Value()
+ } else if ki > kj {
+ j = jiter.Next()
+ kj, cj = jiter.Value()
+ } else { // ki == kj
+ output.Containers.Put(ki, intersect(ci, cj))
+ i, j = iiter.Next(), jiter.Next()
+ ki, ci = iiter.Value()
+ kj, cj = jiter.Value()
}
-
}
-
return output
}
// Union returns the bitwise union of b and other.
func (b *Bitmap) Union(other *Bitmap) *Bitmap {
- output := &Bitmap{}
+ output := NewBitmap()
- ki, ci := b.keys, b.containers
- kj, cj := other.keys, other.containers
-
- for {
- var key uint64
- var container *container
-
- ni, nj := len(ki), len(kj)
- if ni == 0 && nj == 0 { // eof(i,j)
- break
- } else if ni == 0 || (nj != 0 && ki[0] > kj[0]) { // eof(i) or i > j
- key, container = kj[0], cj[0].clone()
- kj, cj = kj[1:], cj[1:]
- } else if nj == 0 || (ki[0] < kj[0]) { // eof(j) or i < j
- key, container = ki[0], ci[0].clone()
- ki, ci = ki[1:], ci[1:]
- } else { // i == j
- key, container = ki[0], union(ci[0], cj[0])
- ki, ci = ki[1:], ci[1:]
- kj, cj = kj[1:], cj[1:]
+ iiter, _ := b.Containers.Iterator(0)
+ jiter, _ := other.Containers.Iterator(0)
+ i, j := iiter.Next(), jiter.Next()
+ ki, ci := iiter.Value()
+ kj, cj := jiter.Value()
+ for i || j {
+ if i && (!j || ki < kj) {
+ output.Containers.Put(ki, ci.Clone())
+ i = iiter.Next()
+ ki, ci = iiter.Value()
+ } else if j && (!i || ki > kj) {
+ output.Containers.Put(kj, cj.Clone())
+ j = jiter.Next()
+ kj, cj = jiter.Value()
+ } else { // ki == kj
+ output.Containers.Put(ki, union(ci, cj))
+ i, j = iiter.Next(), jiter.Next()
+ ki, ci = iiter.Value()
+ kj, cj = jiter.Value()
}
-
- output.keys = append(output.keys, key)
- output.containers = append(output.containers, container)
}
-
return output
}
// Difference returns the difference of b and other.
func (b *Bitmap) Difference(other *Bitmap) *Bitmap {
- output := &Bitmap{}
+ output := NewBitmap()
- ki, ci := b.keys, b.containers
- kj, cj := other.keys, other.containers
-
- ni, nj := len(ki), len(kj)
- i, j := 0, 0
- for {
- var key uint64
- var container *container
-
- if ni == i { // eof(i)
- break
- } else if nj == j || ki[i] < kj[j] { // eof(j) or i < j
- key, container = ki[i], ci[i].clone()
- i++
- output.keys = append(output.keys, key)
- output.containers = append(output.containers, container)
- } else if nj > j && ki[i] > kj[j] { // i > j
- j++
- } else { // i == j
- key, container = ki[i], difference(ci[i], cj[j])
- i++
- j++
- output.keys = append(output.keys, key)
- output.containers = append(output.containers, container)
+ iiter, _ := b.Containers.Iterator(0)
+ jiter, _ := other.Containers.Iterator(0)
+ i, j := iiter.Next(), jiter.Next()
+ ki, ci := iiter.Value()
+ kj, cj := jiter.Value()
+ for i || j {
+ if i && (!j || ki < kj) {
+ output.Containers.Put(ki, ci.Clone())
+ i = iiter.Next()
+ ki, ci = iiter.Value()
+ } else if j && (!i || ki > kj) {
+ j = jiter.Next()
+ kj, cj = jiter.Value()
+ } else { // ki == kj
+ output.Containers.Put(ki, difference(ci, cj))
+ i, j = iiter.Next(), jiter.Next()
+ ki, ci = iiter.Value()
+ kj, cj = jiter.Value()
}
}
return output
@@ -453,70 +449,59 @@ func (b *Bitmap) Difference(other *Bitmap) *Bitmap {
// Xor returns the bitwise exclusive or of b and other.
func (b *Bitmap) Xor(other *Bitmap) *Bitmap {
- output := &Bitmap{}
+ output := NewBitmap()
- ki, ci := b.keys, b.containers
- kj, cj := other.keys, other.containers
-
- for {
- var key uint64
- var container *container
-
- ni, nj := len(ki), len(kj)
- if ni == 0 && nj == 0 { // eof(i,j)
- break
- } else if ni == 0 || (nj != 0 && ki[0] > kj[0]) { // eof(i) or i > j
- key, container = kj[0], cj[0].clone()
- kj, cj = kj[1:], cj[1:]
- } else if nj == 0 || (ki[0] < kj[0]) { // eof(j) or i < j
- key, container = ki[0], ci[0].clone()
- ki, ci = ki[1:], ci[1:]
- } else { // i == j
- key, container = ki[0], xor(ci[0], cj[0])
- ki, ci = ki[1:], ci[1:]
- kj, cj = kj[1:], cj[1:]
+ iiter, _ := b.Containers.Iterator(0)
+ jiter, _ := other.Containers.Iterator(0)
+ i, j := iiter.Next(), jiter.Next()
+ ki, ci := iiter.Value()
+ kj, cj := jiter.Value()
+ for i || j {
+ if i && (!j || ki < kj) {
+ output.Containers.Put(ki, ci.Clone())
+ i = iiter.Next()
+ ki, ci = iiter.Value()
+ } else if j && (!i || ki > kj) {
+ output.Containers.Put(kj, cj.Clone())
+ j = jiter.Next()
+ kj, cj = jiter.Value()
+ } else { // ki == kj
+ output.Containers.Put(ki, xor(ci, cj))
+ i, j = iiter.Next(), jiter.Next()
+ ki, ci = iiter.Value()
+ kj, cj = jiter.Value()
}
-
- output.keys = append(output.keys, key)
- output.containers = append(output.containers, container)
}
-
return output
}
// removeEmptyContainers deletes all containers that have a count of zero.
func (b *Bitmap) removeEmptyContainers() {
- for i := 0; i < len(b.containers); {
- c := b.containers[i]
-
+ citer, _ := b.Containers.Iterator(0)
+ for citer.Next() {
+ k, c := citer.Value()
if c.n == 0 {
- b.keys = append(b.keys[:i], b.keys[i+1:]...)
-
- copy(b.containers[i:], b.containers[i+1:])
- b.containers[len(b.containers)-1] = nil
- b.containers = b.containers[:len(b.containers)-1]
- continue
+ b.Containers.Remove(k)
}
-
- i++
}
}
func (b *Bitmap) countEmptyContainers() int {
result := 0
- for i := 0; i < len(b.containers); {
- c := b.containers[i]
-
+ citer, _ := b.Containers.Iterator(0)
+ for citer.Next() {
+ _, c := citer.Value()
if c.n == 0 {
result++
}
- i++
}
return result
}
// Optimize converts array and bitmap containers to run containers as necessary.
func (b *Bitmap) Optimize() {
- for _, c := range b.containers {
+ citer, _ := b.Containers.Iterator(0)
+ for citer.Next() {
+ _, c := citer.Value()
c.Optimize()
}
}
@@ -562,7 +547,7 @@ func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) {
// Remove empty containers before persisting.
//b.removeEmptyContainers()
- containerCount := len(b.keys) - b.countEmptyContainers()
+ containerCount := b.Containers.Size() - b.countEmptyContainers()
headerSize := headerBaseSize
byte2 := make([]byte, 2)
byte4 := make([]byte, 4)
@@ -581,9 +566,9 @@ func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) {
// Descriptive header section: encode keys and cardinality.
// Key and cardinality are stored interleaved here, 12 bytes per container.
- for i, key := range b.keys {
- c := b.containers[i]
-
+ citer, _ := b.Containers.Iterator(0)
+ for citer.Next() {
+ key, c := citer.Value()
// Verify container count before writing.
// TODO: instead of commenting this out, we need to make it a configuration option
//count := c.count()
@@ -593,17 +578,20 @@ func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) {
ew.WriteUint16(byte2, uint16(c.containerType))
ew.WriteUint16(byte2, uint16(c.n-1))
}
+
}
// Offset header section: write the offset for each container block.
// 4 bytes per container.
offset := uint32(headerSize + (containerCount * (8 + 2 + 2 + 4)))
- for _, c := range b.containers {
-
+ citer, _ = b.Containers.Iterator(0)
+ for citer.Next() {
+ _, c := citer.Value()
if c.n > 0 {
ew.WriteUint32(byte4, offset)
offset += uint32(c.size())
}
+
}
if ew.err != nil {
return int64(ew.n), ew.err
@@ -612,7 +600,9 @@ func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) {
n = int64(headerSize + (containerCount * (8 + 2 + 2 + 4)))
// Container storage section: write each container block.
- for _, c := range b.containers {
+ citer, _ = b.Containers.Iterator(0)
+ for citer.Next() {
+ _, c := citer.Value()
if c.n > 0 {
nn, err := c.WriteTo(w)
n += nn
@@ -621,7 +611,6 @@ func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) {
}
}
}
-
return n, nil
}
@@ -645,42 +634,20 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error {
// Read key count in bytes sizeof(cookie):(sizeof(cookie)+sizeof(uint32)).
keyN := binary.LittleEndian.Uint32(data[4:8])
- if len(b.keys) == 0 {
- b.keys = make([]uint64, 0, keyN)
- b.containers = make([]*container, 0, keyN)
- } else if int(keyN) < len(b.keys) { //shrink
- // nil out to allow to be GCed
- for i := range b.containers[keyN:] {
- b.containers[int(keyN)+i] = nil
- }
- b.keys = b.keys[:keyN]
- b.containers = b.containers[:keyN]
- }
-
headerSize := headerBaseSize
// Descriptive header section: Read container keys and cardinalities.
for i, buf := 0, data[headerSize:]; i < int(keyN); i, buf = i+1, buf[12:] {
- // Reuse memory if possible
- if i >= len(b.keys) {
- b.keys = append(b.keys, binary.LittleEndian.Uint64(buf[0:8]))
- b.containers = append(b.containers, &container{
- containerType: byte(binary.LittleEndian.Uint16(buf[8:10])),
- n: int(binary.LittleEndian.Uint16(buf[10:12])) + 1,
- mapped: true,
- })
- } else {
- b.keys[i] = binary.LittleEndian.Uint64(buf[0:8])
- c := b.containers[i]
- c.containerType = byte(binary.LittleEndian.Uint16(buf[8:10]))
- c.n = int(binary.LittleEndian.Uint16(buf[10:12])) + 1
- c.mapped = true
-
- }
+ b.Containers.PutContainerValues(
+ binary.LittleEndian.Uint64(buf[0:8]),
+ byte(binary.LittleEndian.Uint16(buf[8:10])),
+ int(binary.LittleEndian.Uint16(buf[10:12]))+1,
+ true)
}
opsOffset := headerSize + int(keyN)*12
// Read container offsets and attach data.
+ citer, _ := b.Containers.Iterator(0)
for i, buf := 0, data[opsOffset:]; i < int(keyN); i, buf = i+1, buf[4:] {
offset := binary.LittleEndian.Uint32(buf[0:4])
// Verify the offset is within the bounds of the input data.
@@ -689,7 +656,8 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error {
}
// Map byte slice directly to the container data.
- c := b.containers[i]
+ citer.Next()
+ _, c := citer.Value()
switch c.containerType {
case ContainerRun:
c.array = nil
@@ -761,15 +729,16 @@ func (b *Bitmap) Iterator() *Iterator {
func (b *Bitmap) Info() BitmapInfo {
info := BitmapInfo{
OpN: b.opN,
- Containers: make([]ContainerInfo, len(b.containers)),
+ Containers: make([]ContainerInfo, 0, b.Containers.Size()),
}
- for i, c := range b.containers {
+ citer, _ := b.Containers.Iterator(0)
+ for citer.Next() {
+ k, c := citer.Value()
ci := c.info()
- ci.Key = b.keys[i]
- info.Containers[i] = ci
+ ci.Key = k
+ info.Containers = append(info.Containers, ci)
}
-
return info
}
@@ -777,16 +746,12 @@ func (b *Bitmap) Info() BitmapInfo {
func (b *Bitmap) Check() error {
var a ErrorList
- // Check keys/containers match. Return immediately if this happens.
- if len(b.keys) != len(b.containers) {
- a.Append(fmt.Errorf("key/container count mismatch: %d != %d", len(b.keys), len(b.containers)))
- return a
- }
-
// Check each container.
- for i, c := range b.containers {
+ citer, _ := b.Containers.Iterator(0)
+ for citer.Next() {
+ k, c := citer.Value()
if err := c.check(); err != nil {
- a.AppendWithPrefix(err, fmt.Sprintf("%d/", b.keys[i]))
+ a.AppendWithPrefix(err, fmt.Sprintf("%d/", k))
}
}
@@ -832,13 +797,13 @@ type BitmapInfo struct {
// Iterator represents an iterator over a Bitmap.
type Iterator struct {
- bitmap *Bitmap
- i, j, k int // i: container; j: array index, bit index, or run index; k: offset within the run
+ bitmap *Bitmap
+ citer ContainerIterator
+ key uint64
+ c *Container
+ j, k int // i: container; j: array index, bit index, or run index; k: offset within the run
}
-// eof returns true if the iterator is at the end of the bitmap.
-func (itr *Iterator) eof() bool { return itr.i >= len(itr.bitmap.containers) }
-
// Seek moves to the first value equal to or greater than `seek`.
func (itr *Iterator) Seek(seek uint64) {
// k should always be -1 unless we're seeking into a run container. Then the
@@ -846,45 +811,45 @@ func (itr *Iterator) Seek(seek uint64) {
itr.k = -1
// Move to the correct container.
- itr.i = search64(itr.bitmap.keys, highbits(seek))
- if itr.i < 0 {
- itr.i = -itr.i - 1
- }
- if itr.eof() {
- return
+ itr.citer, _ = itr.bitmap.Containers.Iterator(highbits(seek))
+ if !itr.citer.Next() {
+ itr.c = nil
+ return // eof
}
+ itr.key, itr.c = itr.citer.Value()
// Move to the correct value index inside the container.
lb := lowbits(seek)
- if itr.i >= len(itr.bitmap.containers) {
- panic(fmt.Sprintf("data Corruption %d %d %d", itr.i, len(itr.bitmap.containers), seek))
- }
- c := itr.bitmap.containers[itr.i]
- if c.isArray() {
+ if itr.c.isArray() {
// Find index in the container.
- itr.j = search32(c.array, lb)
+ itr.j = search32(itr.c.array, lb)
if itr.j < 0 {
itr.j = -itr.j - 1
}
- if itr.j < len(c.array) {
+ if itr.j < len(itr.c.array) {
itr.j--
return
}
// If it's at the end of the container then move to the next one.
- itr.i, itr.j = itr.i+1, -1
+ if !itr.citer.Next() {
+ itr.c = nil
+ return
+ }
+ itr.key, itr.c = itr.citer.Value()
+ itr.j = -1
return
}
- if c.isRun() {
+ if itr.c.isRun() {
if seek == 0 {
- itr.i, itr.j, itr.k = 0, 0, -1
+ itr.j, itr.k = 0, -1
}
- j, contains := binSearchRuns(lb, c.runs)
+ j, contains := binSearchRuns(lb, itr.c.runs)
if contains {
itr.j = j
- itr.k = int(lb) - int(c.runs[j].start) - 1
+ itr.k = int(lb) - int(itr.c.runs[j].start) - 1
} else {
// Set iterator to next value in the Bitmap.
itr.j = j
@@ -901,24 +866,27 @@ func (itr *Iterator) Seek(seek uint64) {
// Next returns the next value in the bitmap.
// Returns eof as true if there are no values left in the iterator.
func (itr *Iterator) Next() (v uint64, eof bool) {
+ if itr.c == nil {
+ return 0, true
+ }
// Iterate over containers until we find the next value or EOF.
for {
- if itr.eof() {
- return 0, true
- }
-
- c := itr.bitmap.containers[itr.i]
- if c.isArray() {
- if itr.j >= c.n-1 {
+ if itr.c.isArray() {
+ if itr.j >= itr.c.n-1 {
// Reached end of array, move to the next container.
- itr.i, itr.j = itr.i+1, -1
+ if !itr.citer.Next() {
+ itr.c = nil
+ return 0, true
+ }
+ itr.key, itr.c = itr.citer.Value()
+ itr.j = -1
continue
}
itr.j++
return itr.peek(), false
}
- if c.isRun() {
+ if itr.c.isRun() {
// Because itr.j for an array container defaults to -1
// but defaults to 0 for a run container, we need to
// standardize on treating -1 as our default value for itr.j.
@@ -931,12 +899,17 @@ func (itr *Iterator) Next() (v uint64, eof bool) {
}
// If the container is empty, move to the next container.
- if len(c.runs) == 0 {
- itr.i, itr.j = itr.i+1, -1
+ if len(itr.c.runs) == 0 {
+ if !itr.citer.Next() {
+ itr.c = nil
+ return 0, true
+ }
+ itr.key, itr.c = itr.citer.Value()
+ itr.j = -1
continue
}
- r := c.runs[itr.j]
+ r := itr.c.runs[itr.j]
runLength := int(r.last - r.start)
if itr.k >= runLength {
@@ -944,9 +917,14 @@ func (itr *Iterator) Next() (v uint64, eof bool) {
itr.j, itr.k = itr.j+1, -1
}
- if itr.j >= len(c.runs) {
+ if itr.j >= len(itr.c.runs) {
// Reached end of runs, move to the next container.
- itr.i, itr.j = itr.i+1, -1
+ if !itr.citer.Next() {
+ itr.c = nil
+ return 0, true
+ }
+ itr.key, itr.c = itr.citer.Value()
+ itr.j = -1
continue
}
@@ -960,40 +938,51 @@ func (itr *Iterator) Next() (v uint64, eof bool) {
// Find first non-zero bit in current bitmap, if possible.
hb := itr.j >> 6
- if hb >= len(c.bitmap) {
- itr.i, itr.j = itr.i+1, -1
+ if hb >= len(itr.c.bitmap) {
+ if !itr.citer.Next() {
+ itr.c = nil
+ return 0, true
+ }
+ itr.key, itr.c = itr.citer.Value()
+ itr.j = -1
continue
}
- lb := c.bitmap[hb] >> (uint(itr.j) % 64)
+ lb := itr.c.bitmap[hb] >> (uint(itr.j) % 64)
if lb != 0 {
itr.j = itr.j + trailingZeroN(lb)
return itr.peek(), false
}
// Otherwise iterate through remaining bitmaps to find next bit.
- for hb++; hb < len(c.bitmap); hb++ {
- if c.bitmap[hb] != 0 {
- itr.j = hb<<6 + trailingZeroN(c.bitmap[hb])
+ for hb++; hb < len(itr.c.bitmap); hb++ {
+ if itr.c.bitmap[hb] != 0 {
+ itr.j = hb<<6 + trailingZeroN(itr.c.bitmap[hb])
return itr.peek(), false
}
}
// If no bits found then move to the next container.
- itr.i, itr.j = itr.i+1, -1
+ if !itr.citer.Next() {
+ itr.c = nil
+ return 0, true
+ }
+ itr.key, itr.c = itr.citer.Value()
+ itr.j = -1
}
}
// peek returns the current value.
func (itr *Iterator) peek() uint64 {
- key := itr.bitmap.keys[itr.i]
- c := itr.bitmap.containers[itr.i]
- if c.isArray() {
- return key<<16 | uint64(c.array[itr.j])
+ if itr.c == nil {
+ return 0
}
- if c.isRun() {
- return key<<16 | uint64(c.runs[itr.j].start+uint16(itr.k))
+ if itr.c.isArray() {
+ return itr.key<<16 | uint64(itr.c.array[itr.j])
}
- return key<<16 | uint64(itr.j)
+ if itr.c.isRun() {
+ return itr.key<<16 | uint64(itr.c.runs[itr.j].start+uint16(itr.k))
+ }
+ return itr.key<<16 | uint64(itr.j)
}
// ArrayMaxSize represents the maximum size of array containers.
@@ -1002,17 +991,17 @@ const ArrayMaxSize = 4096
// RunMaxSize represents the maximum size of run length encoded containers.
const RunMaxSize = 2048
-// container represents a container for uint16 integers.
+// Container represents a Container for uint16 integers.
//
// These are used for storing the low bits of numbers in larger sets of uint64.
-// The high bits are stored in a container's key which is tracked by a separate
-// data structure. Integers in a container can be encoded in one of three ways -
-// the encoding used is usually whichever is most compact, though any container
+// The high bits are stored in a Container's key which is tracked by a separate
+// data structure. Integers in a Container can be encoded in one of three ways -
+// the encoding used is usually whichever is most compact, though any Container
// type should be able to encode any set of integers safely. For containers with
// less than 4,096 values, an array is often used. Containers with long runs of
// integers would use run length encoding, and more random data usually uses
// bitmap encoding.
-type container struct {
+type Container struct {
mapped bool // mapped directly to a byte slice when true
containerType byte // array, bitmap, or run
n int // number of integers in container
@@ -1032,22 +1021,34 @@ func (iv interval16) runlen() int {
}
// newContainer returns a new instance of container.
-func newContainer() *container {
- return &container{containerType: ContainerArray}
+func NewContainer() *Container {
+ return &Container{containerType: ContainerArray}
+}
+
+// Mapped returns true if the container is mapped directly to a byte slice
+func (c *Container) Mapped() bool {
+ return c.mapped
+}
+
+// Update updates the container
+func (c *Container) Update(containerType byte, n int, mapped bool) {
+ c.containerType = containerType
+ c.n = n
+ c.mapped = mapped
}
// isArray returns true if the container is an array container.
-func (c *container) isArray() bool {
+func (c *Container) isArray() bool {
return c.containerType == ContainerArray
}
// isBitmap returns true if the container is a bitmap container.
-func (c *container) isBitmap() bool {
+func (c *Container) isBitmap() bool {
return c.containerType == ContainerBitmap
}
// isRun returns true if the container is a run-length-encoded container.
-func (c *container) isRun() bool {
+func (c *Container) isRun() bool {
return c.containerType == ContainerRun
}
@@ -1055,7 +1056,7 @@ func (c *container) isRun() bool {
//
// This is performed when altering the container since its contents could be
// pointing at a read-only mmap.
-func (c *container) unmap() {
+func (c *Container) unmap() {
if !c.mapped {
return
}
@@ -1078,12 +1079,12 @@ func (c *container) unmap() {
}
// count counts all bits in the container.
-func (c *container) count() (n int) {
+func (c *Container) count() (n int) {
return c.countRange(0, maxContainerVal+1)
}
// countRange counts the number of bits set between [start, end).
-func (c *container) countRange(start, end int) (n int) {
+func (c *Container) countRange(start, end int) (n int) {
if c.isArray() {
return c.arrayCountRange(start, end)
} else if c.isRun() {
@@ -1092,7 +1093,7 @@ func (c *container) countRange(start, end int) (n int) {
return c.bitmapCountRange(start, end)
}
-func (c *container) arrayCountRange(start, end int) (n int) {
+func (c *Container) arrayCountRange(start, end int) (n int) {
i := sort.Search(len(c.array), func(i int) bool { return int(c.array[i]) >= start })
for ; i < len(c.array); i++ {
v := int(c.array[i])
@@ -1104,7 +1105,7 @@ func (c *container) arrayCountRange(start, end int) (n int) {
return n
}
-func (c *container) bitmapCountRange(start, end int) int {
+func (c *Container) bitmapCountRange(start, end int) int {
var n uint64
i, j := start/64, end/64
// Special case when start and end fall in the same word.
@@ -1134,7 +1135,7 @@ func (c *container) bitmapCountRange(start, end int) int {
return int(n)
}
-func (c *container) runCountRange(start, end int) (n int) {
+func (c *Container) runCountRange(start, end int) (n int) {
for _, iv := range c.runs {
// iv is before range
if int(iv.last) < start {
@@ -1165,7 +1166,7 @@ func (c *container) runCountRange(start, end int) (n int) {
}
// add adds a value to the container.
-func (c *container) add(v uint16) (added bool) {
+func (c *Container) add(v uint16) (added bool) {
if c.isArray() {
added = c.arrayAdd(v)
@@ -1180,7 +1181,7 @@ func (c *container) add(v uint16) (added bool) {
return added
}
-func (c *container) arrayAdd(v uint16) bool {
+func (c *Container) arrayAdd(v uint16) bool {
// Optimize appending to the end of an array container.
if c.n > 0 && c.n < ArrayMaxSize && c.isArray() && c.array[c.n-1] < v {
c.unmap()
@@ -1210,7 +1211,7 @@ func (c *container) arrayAdd(v uint16) bool {
}
-func (c *container) bitmapAdd(v uint16) bool {
+func (c *Container) bitmapAdd(v uint16) bool {
if c.bitmapContains(v) {
return false
}
@@ -1219,7 +1220,7 @@ func (c *container) bitmapAdd(v uint16) bool {
return true
}
-func (c *container) runAdd(v uint16) bool {
+func (c *Container) runAdd(v uint16) bool {
if len(c.runs) == 0 {
c.unmap()
c.runs = []interval16{{start: v, last: v}}
@@ -1266,7 +1267,7 @@ func (c *container) runAdd(v uint16) bool {
}
// contains returns true if v is in the container.
-func (c *container) contains(v uint16) bool {
+func (c *Container) contains(v uint16) bool {
if c.isArray() {
return c.arrayContains(v)
} else if c.isRun() {
@@ -1276,7 +1277,7 @@ func (c *container) contains(v uint16) bool {
}
}
-func (c *container) bitmapCountRuns() (r int) {
+func (c *Container) bitmapCountRuns() (r int) {
for i := 0; i < 1023; i++ {
v, v1 := c.bitmap[i], c.bitmap[i+1]
r = r + int(popcount((v<<1)&^v)+((v>>63)&^v1))
@@ -1286,7 +1287,7 @@ func (c *container) bitmapCountRuns() (r int) {
return r
}
-func (c *container) arrayCountRuns() (r int) {
+func (c *Container) arrayCountRuns() (r int) {
prev := -2
for _, v := range c.array {
if prev+1 != int(v) {
@@ -1297,7 +1298,7 @@ func (c *container) arrayCountRuns() (r int) {
return r
}
-func (c *container) countRuns() (r int) {
+func (c *Container) countRuns() (r int) {
if c.isArray() {
return c.arrayCountRuns()
} else if c.isBitmap() {
@@ -1312,7 +1313,7 @@ func (c *container) countRuns() (r int) {
// Optimize converts the container to the type which will take up the least
// amount of space.
-func (c *container) Optimize() {
+func (c *Container) Optimize() {
if c.n == 0 {
return
}
@@ -1349,11 +1350,11 @@ func (c *container) Optimize() {
}
}
-func (c *container) arrayContains(v uint16) bool {
+func (c *Container) arrayContains(v uint16) bool {
return search32(c.array, v) >= 0
}
-func (c *container) bitmapContains(v uint16) bool {
+func (c *Container) bitmapContains(v uint16) bool {
return (c.bitmap[v/64] & (1 << uint64(v%64))) != 0
}
@@ -1371,13 +1372,13 @@ func binSearchRuns(v uint16, a []interval16) (int, bool) {
// runContains determines if v is in the container assuming c is a run
// container.
-func (c *container) runContains(v uint16) bool {
+func (c *Container) runContains(v uint16) bool {
_, found := binSearchRuns(v, c.runs)
return found
}
// remove removes a value from the container.
-func (c *container) remove(v uint16) (removed bool) {
+func (c *Container) remove(v uint16) (removed bool) {
if c.isArray() {
removed = c.arrayRemove(v)
} else if c.isRun() {
@@ -1391,7 +1392,7 @@ func (c *container) remove(v uint16) (removed bool) {
return removed
}
-func (c *container) arrayRemove(v uint16) bool {
+func (c *Container) arrayRemove(v uint16) bool {
i := search32(c.array, v)
if i < 0 {
return false
@@ -1402,7 +1403,7 @@ func (c *container) arrayRemove(v uint16) bool {
return true
}
-func (c *container) bitmapRemove(v uint16) bool {
+func (c *Container) bitmapRemove(v uint16) bool {
if !c.bitmapContains(v) {
return false
}
@@ -1420,7 +1421,7 @@ func (c *container) bitmapRemove(v uint16) bool {
}
// runRemove removes v from a run container, and returns true if v was removed.
-func (c *container) runRemove(v uint16) bool {
+func (c *Container) runRemove(v uint16) bool {
i, contains := binSearchRuns(v, c.runs)
if !contains {
return false
@@ -1441,7 +1442,7 @@ func (c *container) runRemove(v uint16) bool {
}
// max returns the maximum value in the container.
-func (c *container) max() uint16 {
+func (c *Container) max() uint16 {
if c.isArray() {
return c.arrayMax()
} else if c.isRun() {
@@ -1451,14 +1452,14 @@ func (c *container) max() uint16 {
}
}
-func (c *container) arrayMax() uint16 {
+func (c *Container) arrayMax() uint16 {
if len(c.array) == 0 {
return 0 // probably hiding some ugly bug but it prevents a crash
}
return c.array[len(c.array)-1]
}
-func (c *container) bitmapMax() uint16 {
+func (c *Container) bitmapMax() uint16 {
// Search bitmap in reverse order.
for i := len(c.bitmap) - 1; i >= 0; i-- {
// If value is zero then skip.
@@ -1480,7 +1481,7 @@ func (c *container) bitmapMax() uint16 {
return 0
}
-func (c *container) runMax() uint16 {
+func (c *Container) runMax() uint16 {
if len(c.runs) == 0 {
return 0
}
@@ -1488,7 +1489,7 @@ func (c *container) runMax() uint16 {
}
// bitmapToArray converts from bitmap format to array format.
-func (c *container) bitmapToArray() {
+func (c *Container) bitmapToArray() {
c.array = make([]uint16, 0, c.n)
c.containerType = ContainerArray
@@ -1511,7 +1512,7 @@ func (c *container) bitmapToArray() {
}
// arrayToBitmap converts from array format to bitmap format.
-func (c *container) arrayToBitmap() {
+func (c *Container) arrayToBitmap() {
c.bitmap = make([]uint64, bitmapN)
c.containerType = ContainerBitmap
@@ -1530,7 +1531,7 @@ func (c *container) arrayToBitmap() {
}
// runToBitmap converts from RLE format to bitmap format.
-func (c *container) runToBitmap() {
+func (c *Container) runToBitmap() {
c.bitmap = make([]uint64, bitmapN)
c.containerType = ContainerBitmap
@@ -1553,7 +1554,7 @@ func (c *container) runToBitmap() {
}
// bitmapToRun converts from bitmap format to RLE format.
-func (c *container) bitmapToRun() {
+func (c *Container) bitmapToRun() {
c.containerType = ContainerRun
// return early if empty
if c.n == 0 {
@@ -1609,7 +1610,7 @@ func (c *container) bitmapToRun() {
}
// arrayToRun converts from array format to RLE format.
-func (c *container) arrayToRun() {
+func (c *Container) arrayToRun() {
c.containerType = ContainerRun
// return early if empty
if c.n == 0 {
@@ -1636,7 +1637,7 @@ func (c *container) arrayToRun() {
}
// runToArray converts from RLE format to array format.
-func (c *container) runToArray() {
+func (c *Container) runToArray() {
c.containerType = ContainerArray
c.array = make([]uint16, 0, c.n)
@@ -1656,9 +1657,9 @@ func (c *container) runToArray() {
c.mapped = false
}
-// clone returns a copy of c.
-func (c *container) clone() *container {
- other := &container{n: c.n, containerType: c.containerType}
+// Clone returns a copy of c.
+func (c *Container) Clone() *Container {
+ other := &Container{n: c.n, containerType: c.containerType}
switch c.containerType {
case ContainerArray:
@@ -1675,7 +1676,7 @@ func (c *container) clone() *container {
}
// WriteTo writes c to w.
-func (c *container) WriteTo(w io.Writer) (n int64, err error) {
+func (c *Container) WriteTo(w io.Writer) (n int64, err error) {
if c.isArray() {
return c.arrayWriteTo(w)
} else if c.isRun() {
@@ -1685,7 +1686,7 @@ func (c *container) WriteTo(w io.Writer) (n int64, err error) {
}
}
-func (c *container) arrayWriteTo(w io.Writer) (n int64, err error) {
+func (c *Container) arrayWriteTo(w io.Writer) (n int64, err error) {
if len(c.array) == 0 {
return 0, nil
}
@@ -1701,13 +1702,13 @@ func (c *container) arrayWriteTo(w io.Writer) (n int64, err error) {
return int64(nn), err
}
-func (c *container) bitmapWriteTo(w io.Writer) (n int64, err error) {
+func (c *Container) bitmapWriteTo(w io.Writer) (n int64, err error) {
// Write sizeof(uint64) * bitmapN bytes.
nn, err := w.Write((*[0xFFFFFFF]byte)(unsafe.Pointer(&c.bitmap[0]))[:(8 * bitmapN)])
return int64(nn), err
}
-func (c *container) runWriteTo(w io.Writer) (n int64, err error) {
+func (c *Container) runWriteTo(w io.Writer) (n int64, err error) {
if len(c.runs) == 0 {
return 0, nil
}
@@ -1722,7 +1723,7 @@ func (c *container) runWriteTo(w io.Writer) (n int64, err error) {
}
// size returns the encoded size of the container, in bytes.
-func (c *container) size() int {
+func (c *Container) size() int {
if c.isArray() {
return len(c.array) * 2 // sizeof(uint16)
} else if c.isRun() {
@@ -1733,7 +1734,7 @@ func (c *container) size() int {
}
// info returns the current stats about the container.
-func (c *container) info() ContainerInfo {
+func (c *Container) info() ContainerInfo {
info := ContainerInfo{N: c.n}
if c.isArray() {
@@ -1761,7 +1762,7 @@ func (c *container) info() ContainerInfo {
}
// check performs a consistency check on the container.
-func (c *container) check() error {
+func (c *Container) check() error {
var a ErrorList
if c.isArray() {
@@ -1801,7 +1802,7 @@ type ContainerInfo struct {
// flip returns a new container containing the inverse of all
// bits in a.
-func flip(a *container) *container {
+func flip(a *Container) *Container {
if a.isArray() {
return flipArray(a)
} else if a.isRun() {
@@ -1811,15 +1812,15 @@ func flip(a *container) *container {
}
}
-func flipArray(b *container) *container {
+func flipArray(b *Container) *Container {
// TODO: actually implement this
- x := b.clone()
+ x := b.Clone()
x.arrayToBitmap()
return flipBitmap(x)
}
-func flipBitmap(b *container) *container {
- other := &container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap}
+func flipBitmap(b *Container) *Container {
+ other := &Container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap}
for i, bitmap := range b.bitmap {
other.bitmap[i] = ^bitmap
@@ -1829,14 +1830,14 @@ func flipBitmap(b *container) *container {
return other
}
-func flipRun(b *container) *container {
+func flipRun(b *Container) *Container {
// TODO: actually implement this
- x := b.clone()
+ x := b.Clone()
x.runToBitmap()
return flipBitmap(x)
}
-func intersectionCount(a, b *container) int {
+func intersectionCount(a, b *Container) int {
if a.isArray() {
if b.isArray() {
return intersectionCountArrayArray(a, b)
@@ -1864,7 +1865,7 @@ func intersectionCount(a, b *container) int {
}
}
-func intersectionCountArrayArray(a, b *container) (n int) {
+func intersectionCountArrayArray(a, b *Container) (n int) {
na, nb := len(a.array), len(b.array)
for i, j := 0, 0; i < na && j < nb; {
va, vb := a.array[i], b.array[j]
@@ -1880,7 +1881,7 @@ func intersectionCountArrayArray(a, b *container) (n int) {
return n
}
-func intersectionCountArrayRun(a, b *container) (n int) {
+func intersectionCountArrayRun(a, b *Container) (n int) {
na, nb := len(a.array), len(b.runs)
for i, j := 0, 0; i < na && j < nb; {
va, vb := a.array[i], b.runs[j]
@@ -1896,7 +1897,7 @@ func intersectionCountArrayRun(a, b *container) (n int) {
return n
}
-func intersectionCountRunRun(a, b *container) (n int) {
+func intersectionCountRunRun(a, b *Container) (n int) {
na, nb := len(a.runs), len(b.runs)
for i, j := 0, 0; i < na && j < nb; {
va, vb := a.runs[i], b.runs[j]
@@ -1927,14 +1928,14 @@ func intersectionCountRunRun(a, b *container) (n int) {
return
}
-func intersectionCountBitmapRun(a, b *container) (n int) {
+func intersectionCountBitmapRun(a, b *Container) (n int) {
for _, iv := range b.runs {
n += a.bitmapCountRange(int(iv.start), int(iv.last)+1)
}
return n
}
-func intersectionCountArrayBitmap(a, b *container) (n int) {
+func intersectionCountArrayBitmap(a, b *Container) (n int) {
ln := len(b.bitmap)
for _, val := range a.array {
i := int(val >> 6)
@@ -1947,11 +1948,11 @@ func intersectionCountArrayBitmap(a, b *container) (n int) {
return n
}
-func intersectionCountBitmapBitmap(a, b *container) (n int) {
+func intersectionCountBitmapBitmap(a, b *Container) (n int) {
return int(popcountAndSlice(a.bitmap, b.bitmap))
}
-func intersect(a, b *container) *container {
+func intersect(a, b *Container) *Container {
if a.isArray() {
if b.isArray() {
return intersectArrayArray(a, b)
@@ -1979,8 +1980,8 @@ func intersect(a, b *container) *container {
}
}
-func intersectArrayArray(a, b *container) *container {
- output := &container{containerType: ContainerArray}
+func intersectArrayArray(a, b *Container) *Container {
+ output := &Container{containerType: ContainerArray}
na, nb := len(a.array), len(b.array)
for i, j := 0, 0; i < na && j < nb; {
va, vb := a.array[i], b.array[j]
@@ -2000,8 +2001,8 @@ func intersectArrayArray(a, b *container) *container {
// intersectArrayRun computes the intersect of an array container and a run
// container. The return is always an array container (since it's guaranteed to
// be low-cardinality)
-func intersectArrayRun(a, b *container) *container {
- output := &container{containerType: ContainerArray}
+func intersectArrayRun(a, b *Container) *Container {
+ output := &Container{containerType: ContainerArray}
na, nb := len(a.array), len(b.runs)
for i, j := 0, 0; i < na && j < nb; {
va, vb := a.array[i], b.runs[j]
@@ -2019,8 +2020,8 @@ func intersectArrayRun(a, b *container) *container {
}
// intersectRunRun computes the intersect of two run containers.
-func intersectRunRun(a, b *container) *container {
- output := &container{containerType: ContainerRun}
+func intersectRunRun(a, b *Container) *Container {
+ output := &Container{containerType: ContainerRun}
na, nb := len(a.runs), len(b.runs)
for i, j := 0, 0; i < na && j < nb; {
va, vb := a.runs[i], b.runs[j]
@@ -2058,11 +2059,11 @@ func intersectRunRun(a, b *container) *container {
// intersectBitmapRun returns an array container if the run container's
// cardinality is < ArrayMaxSize. Otherwise it returns a bitmap container.
-func intersectBitmapRun(a, b *container) *container {
- var output *container
+func intersectBitmapRun(a, b *Container) *Container {
+ var output *Container
if b.n < ArrayMaxSize {
// output is array container
- output = &container{containerType: ContainerArray}
+ output = &Container{containerType: ContainerArray}
for _, iv := range b.runs {
for i := iv.start; i <= iv.last; i++ {
if a.bitmapContains(i) {
@@ -2079,7 +2080,7 @@ func intersectBitmapRun(a, b *container) *container {
// right now this iterates through the runs and sets integers in the
// bitmap that are in the runs. alternately, we could zero out ranges in
// the bitmap which are between runs.
- output = &container{
+ output = &Container{
bitmap: make([]uint64, bitmapN),
containerType: ContainerBitmap,
}
@@ -2121,8 +2122,8 @@ func intersectBitmapRun(a, b *container) *container {
return output
}
-func intersectArrayBitmap(a, b *container) *container {
- output := &container{containerType: ContainerArray}
+func intersectArrayBitmap(a, b *Container) *Container {
+ output := &Container{containerType: ContainerArray}
for _, va := range a.array {
bmidx := va / 64
bidx := va % 64
@@ -2136,8 +2137,8 @@ func intersectArrayBitmap(a, b *container) *container {
return output
}
-func intersectBitmapBitmap(a, b *container) *container {
- output := &container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap}
+func intersectBitmapBitmap(a, b *Container) *Container {
+ output := &Container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap}
for i := range a.bitmap {
v := a.bitmap[i] & b.bitmap[i]
@@ -2149,7 +2150,7 @@ func intersectBitmapBitmap(a, b *container) *container {
return output
}
-func union(a, b *container) *container {
+func union(a, b *Container) *Container {
if a.isArray() {
if b.isArray() {
return unionArrayArray(a, b)
@@ -2177,8 +2178,8 @@ func union(a, b *container) *container {
}
}
-func unionArrayArray(a, b *container) *container {
- output := &container{containerType: ContainerArray}
+func unionArrayArray(a, b *Container) *Container {
+ output := &Container{containerType: ContainerArray}
na, nb := len(a.array), len(b.array)
for i, j := 0, 0; ; {
if i >= na && j >= nb {
@@ -2210,11 +2211,11 @@ func unionArrayArray(a, b *container) *container {
// unionArrayRun optimistically assumes that the result will be a run container,
// and converts to a bitmap or array container afterwards if necessary.
-func unionArrayRun(a, b *container) *container {
+func unionArrayRun(a, b *Container) *Container {
if b.n == maxContainerVal+1 {
- return b.clone()
+ return b.Clone()
}
- output := &container{containerType: ContainerRun}
+ output := &Container{containerType: ContainerRun}
na, nb := len(a.array), len(b.runs)
var vb interval16
var va uint16
@@ -2247,7 +2248,7 @@ func unionArrayRun(a, b *container) *container {
// interval is earlier than the start of the last interval in the list of runs.
// Its return value is the amount by which the cardinality of the container was
// increased.
-func (c *container) runAppendInterval(v interval16) int {
+func (c *Container) runAppendInterval(v interval16) int {
if len(c.runs) == 0 {
c.runs = append(c.runs, v)
return int(v.last-v.start) + 1
@@ -2267,15 +2268,15 @@ func (c *container) runAppendInterval(v interval16) int {
return 0
}
-func unionRunRun(a, b *container) *container {
+func unionRunRun(a, b *Container) *Container {
if a.n == maxContainerVal+1 {
- return a.clone()
+ return a.Clone()
}
if b.n == maxContainerVal+1 {
- return b.clone()
+ return b.Clone()
}
na, nb := len(a.runs), len(b.runs)
- output := &container{
+ output := &Container{
runs: make([]interval16, 0, na+nb),
containerType: ContainerRun,
}
@@ -2301,14 +2302,14 @@ func unionRunRun(a, b *container) *container {
return output
}
-func unionBitmapRun(a, b *container) *container {
+func unionBitmapRun(a, b *Container) *Container {
if b.n == maxContainerVal+1 {
- return b.clone()
+ return b.Clone()
}
if a.n == maxContainerVal+1 {
- return a.clone()
+ return a.Clone()
}
- output := a.clone()
+ output := a.Clone()
for j := 0; j < len(b.runs); j++ {
output.bitmapSetRange(uint64(b.runs[j].start), uint64(b.runs[j].last)+1)
}
@@ -2318,7 +2319,7 @@ func unionBitmapRun(a, b *container) *container {
const maxBitmap = 0xFFFFFFFFFFFFFFFF
// sets all bits in [i, j) (c must be a bitmap container)
-func (c *container) bitmapSetRange(i, j uint64) {
+func (c *Container) bitmapSetRange(i, j uint64) {
x := i >> 6
y := (j - 1) >> 6
var X uint64 = maxBitmap << (i % 64)
@@ -2341,7 +2342,7 @@ func (c *container) bitmapSetRange(i, j uint64) {
}
// xor's all bits in [i, j) with all true (c must be a bitmap container).
-func (c *container) bitmapXorRange(i, j uint64) {
+func (c *Container) bitmapXorRange(i, j uint64) {
x := i >> 6
y := (j - 1) >> 6
var X uint64 = maxBitmap << (i % 64)
@@ -2366,7 +2367,7 @@ func (c *container) bitmapXorRange(i, j uint64) {
}
// zeroes all bits in [i, j) (c must be a bitmap container)
-func (c *container) bitmapZeroRange(i, j uint64) {
+func (c *Container) bitmapZeroRange(i, j uint64) {
x := i >> 6
y := (j - 1) >> 6
var X uint64 = maxBitmap << (i % 64)
@@ -2386,8 +2387,45 @@ func (c *container) bitmapZeroRange(i, j uint64) {
}
}
-func unionArrayBitmap(a, b *container) *container {
- output := b.clone()
+func (c *Container) equals(c2 *Container) bool {
+ if c.mapped != c2.mapped || c.containerType != c2.containerType || c.n != c2.n {
+ return false
+ }
+ if c.containerType == ContainerArray {
+ if len(c.array) != len(c2.array) {
+ return false
+ }
+ for i := 0; i < len(c.array); i++ {
+ if c.array[i] != c2.array[i] {
+ return false
+ }
+ }
+ } else if c.containerType == ContainerBitmap {
+ if len(c.bitmap) != len(c2.bitmap) {
+ return false
+ }
+ for i := 0; i < len(c.bitmap); i++ {
+ if c.bitmap[i] != c2.bitmap[i] {
+ return false
+ }
+ }
+ } else if c.containerType == ContainerRun {
+ if len(c.runs) != len(c2.runs) {
+ return false
+ }
+ for i := 0; i < len(c.runs); i++ {
+ if c.runs[i] != c2.runs[i] {
+ return false
+ }
+ }
+ } else {
+ panic(fmt.Sprintf("unknown container type: %v", c.containerType))
+ }
+ return true
+}
+
+func unionArrayBitmap(a, b *Container) *Container {
+ output := b.Clone()
for _, v := range a.array {
if !output.bitmapContains(v) {
output.bitmap[v/64] |= (1 << uint64(v%64))
@@ -2397,8 +2435,8 @@ func unionArrayBitmap(a, b *container) *container {
return output
}
-func unionBitmapBitmap(a, b *container) *container {
- output := &container{
+func unionBitmapBitmap(a, b *Container) *Container {
+ output := &Container{
bitmap: make([]uint64, bitmapN),
containerType: ContainerBitmap,
}
@@ -2412,7 +2450,7 @@ func unionBitmapBitmap(a, b *container) *container {
return output
}
-func difference(a, b *container) *container {
+func difference(a, b *Container) *Container {
if a.isArray() {
if b.isArray() {
return differenceArrayArray(a, b)
@@ -2441,8 +2479,8 @@ func difference(a, b *container) *container {
}
// differenceArrayArray computes the difference bween two arrays.
-func differenceArrayArray(a, b *container) *container {
- output := &container{containerType: ContainerArray}
+func differenceArrayArray(a, b *Container) *Container {
+ output := &Container{containerType: ContainerArray}
na, nb := len(a.array), len(b.array)
for i, j := 0, 0; i < na; {
va := a.array[i]
@@ -2466,14 +2504,14 @@ func differenceArrayArray(a, b *container) *container {
}
// differenceArrayRun computes the difference of an array from a run.
-func differenceArrayRun(a, b *container) *container {
+func differenceArrayRun(a, b *Container) *Container {
// func (ac *arrayContainer) iandNotRun16(rc *runContainer16) container {
if a.n == 0 || b.n == 0 {
- return a.clone()
+ return a.Clone()
}
- output := &container{array: make([]uint16, 0, a.n), containerType: ContainerArray}
+ output := &Container{array: make([]uint16, 0, a.n), containerType: ContainerArray}
// cardinality upper bound: card(A)
i := 0 // array index
@@ -2523,12 +2561,12 @@ func differenceArrayRun(a, b *container) *container {
}
// differenceBitmapRun computes the difference of an bitmap from a run.
-func differenceBitmapRun(a, b *container) *container {
+func differenceBitmapRun(a, b *Container) *Container {
if a.n == 0 || b.n == 0 {
- return a.clone()
+ return a.Clone()
}
- output := a.clone()
+ output := a.Clone()
for j := 0; j < len(b.runs); j++ {
output.bitmapZeroRange(uint64(b.runs[j].start), uint64(b.runs[j].last)+1)
}
@@ -2537,11 +2575,11 @@ func differenceBitmapRun(a, b *container) *container {
// differenceRunArray subtracts the bits in an array container from a run
// container.
-func differenceRunArray(a, b *container) *container {
+func differenceRunArray(a, b *Container) *Container {
if a.n == 0 || b.n == 0 {
- return a.clone()
+ return a.Clone()
}
- output := &container{runs: make([]interval16, 0, len(a.runs)), containerType: ContainerRun}
+ output := &Container{runs: make([]interval16, 0, len(a.runs)), containerType: ContainerRun}
bidx := 0
vb := b.array[bidx]
@@ -2592,12 +2630,12 @@ RUNLOOP:
}
// differenceRunBitmap computes the difference of an run from a bitmap.
-func differenceRunBitmap(a, b *container) *container {
+func differenceRunBitmap(a, b *Container) *Container {
// If a is full, difference is the flip of b.
if len(a.runs) > 0 && a.runs[0].start == 0 && a.runs[0].last == 65535 {
return flipBitmap(b)
}
- output := &container{containerType: ContainerRun}
+ output := &Container{containerType: ContainerRun}
output.n = a.n
if len(a.runs) == 0 {
return output
@@ -2649,9 +2687,9 @@ func differenceRunBitmap(a, b *container) *container {
}
// differenceRunRun computes the difference of two runs.
-func differenceRunRun(a, b *container) *container {
+func differenceRunRun(a, b *Container) *Container {
if a.n == 0 || b.n == 0 {
- return a.clone()
+ return a.Clone()
}
apos := 0 // current a-run index
@@ -2663,7 +2701,7 @@ func differenceRunRun(a, b *container) *container {
alen := len(a.runs)
blen := len(b.runs)
- output := &container{runs: make([]interval16, 0, alen+blen), containerType: ContainerRun} // TODO allocate max then truncate? or something else
+ output := &Container{runs: make([]interval16, 0, alen+blen), containerType: ContainerRun} // TODO allocate max then truncate? or something else
// cardinality upper bound: sum of number of runs
// each B-run could split an A-run in two, up to len(b.runs) times
@@ -2712,8 +2750,8 @@ func differenceRunRun(a, b *container) *container {
return output
}
-func differenceArrayBitmap(a, b *container) *container {
- output := &container{containerType: ContainerArray}
+func differenceArrayBitmap(a, b *Container) *Container {
+ output := &Container{containerType: ContainerArray}
for _, va := range a.array {
bmidx := va / 64
bidx := va % 64
@@ -2728,8 +2766,8 @@ func differenceArrayBitmap(a, b *container) *container {
return output
}
-func differenceBitmapArray(a, b *container) *container {
- output := a.clone()
+func differenceBitmapArray(a, b *Container) *Container {
+ output := a.Clone()
for _, v := range b.array {
if output.bitmapContains(v) {
@@ -2743,8 +2781,8 @@ func differenceBitmapArray(a, b *container) *container {
return output
}
-func differenceBitmapBitmap(a, b *container) *container {
- output := &container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap}
+func differenceBitmapBitmap(a, b *Container) *Container {
+ output := &Container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap}
for i := range a.bitmap {
v := a.bitmap[i] & (^b.bitmap[i])
@@ -2758,7 +2796,7 @@ func differenceBitmapBitmap(a, b *container) *container {
return output
}
-func xor(a, b *container) *container {
+func xor(a, b *Container) *Container {
if a.isArray() {
if b.isArray() {
return xorArrayArray(a, b)
@@ -2786,8 +2824,8 @@ func xor(a, b *container) *container {
}
}
-func xorArrayArray(a, b *container) *container {
- output := &container{containerType: ContainerArray}
+func xorArrayArray(a, b *Container) *Container {
+ output := &Container{containerType: ContainerArray}
na, nb := len(a.array), len(b.array)
for i, j := 0, 0; i < na || j < nb; {
if i < na && j >= nb {
@@ -2815,8 +2853,8 @@ func xorArrayArray(a, b *container) *container {
return output
}
-func xorArrayBitmap(a, b *container) *container {
- output := b.clone()
+func xorArrayBitmap(a, b *Container) *Container {
+ output := b.Clone()
for _, v := range a.array {
if b.bitmapContains(v) {
output.remove(v)
@@ -2834,8 +2872,8 @@ func xorArrayBitmap(a, b *container) *container {
return output
}
-func xorBitmapBitmap(a, b *container) *container {
- output := &container{
+func xorBitmapBitmap(a, b *Container) *Container {
+ output := &Container{
bitmap: make([]uint64, bitmapN),
containerType: ContainerBitmap,
}
@@ -3044,8 +3082,8 @@ func (a *ErrorList) AppendWithPrefix(err error, prefix string) {
}
// xorArrayRun computes the exclusive or of an array and a run container.
-func xorArrayRun(a, b *container) *container {
- output := &container{containerType: ContainerRun}
+func xorArrayRun(a, b *Container) *Container {
+ output := &Container{containerType: ContainerRun}
na, nb := len(a.array), len(b.runs)
var vb interval16
var va uint16
@@ -3206,15 +3244,15 @@ type xorstm struct {
}
// xorRunRun computes the exclusive or of two run containers.
-func xorRunRun(a, b *container) *container {
+func xorRunRun(a, b *Container) *Container {
na, nb := len(a.runs), len(b.runs)
if na == 0 {
- return b.clone()
+ return b.Clone()
}
if nb == 0 {
- return a.clone()
+ return a.Clone()
}
- output := &container{containerType: ContainerRun}
+ output := &Container{containerType: ContainerRun}
lastI, lastJ := -1, -1
@@ -3254,8 +3292,8 @@ func xorRunRun(a, b *container) *container {
}
// xorRunRun computes the exclusive or of a bitmap and a run container.
-func xorBitmapRun(a, b *container) *container {
- output := a.clone()
+func xorBitmapRun(a, b *Container) *Container {
+ output := a.Clone()
for j := 0; j < len(b.runs); j++ {
output.bitmapXorRange(uint64(b.runs[j].start), uint64(b.runs[j].last)+1)
}
@@ -3268,6 +3306,34 @@ func xorBitmapRun(a, b *container) *container {
return output
}
+func bitmapsEqual(b, c *Bitmap) error {
+ if b.OpWriter != c.OpWriter {
+ return errors.New("opWriters not equal")
+ }
+ if b.opN != c.opN {
+ return errors.New("opNs not equal")
+ }
+
+ biter, _ := b.Containers.Iterator(0)
+ citer, _ := c.Containers.Iterator(0)
+ bn, cn := biter.Next(), citer.Next()
+ for ; bn && cn; bn, cn = biter.Next(), citer.Next() {
+ bk, bc := biter.Value()
+ ck, cc := citer.Value()
+ if bk != ck {
+ return errors.New("keys not equal")
+ }
+ if !bc.equals(cc) {
+ return errors.New("containers not equal")
+ }
+ }
+ if bn && !cn || cn && !bn {
+ return errors.New("different numbers of containers")
+ }
+
+ return nil
+}
+
func popcount(x uint64) uint64 {
return uint64(bits.OnesCount64(x))
}
diff --git a/roaring/roaring_helpers_test.go b/roaring/roaring_helpers_test.go
index 417ac7db6..24db8e1ca 100644
--- a/roaring/roaring_helpers_test.go
+++ b/roaring/roaring_helpers_test.go
@@ -230,8 +230,8 @@ type testOp struct {
exp string
}
-func doContainer(containerType byte, data interface{}) *container {
- c := &container{
+func doContainer(containerType byte, data interface{}) *Container {
+ c := &Container{
containerType: containerType,
}
@@ -248,12 +248,12 @@ func doContainer(containerType byte, data interface{}) *container {
return c
}
-func setupContainerTests() map[byte]map[string]*container {
+func setupContainerTests() map[byte]map[string]*Container {
- cts := make(map[byte]map[string]*container)
+ cts := make(map[byte]map[string]*Container)
// array containers
- cts[ContainerArray] = map[string]*container{
+ cts[ContainerArray] = map[string]*Container{
"empty": doContainer(ContainerArray, arrayEmpty()),
"full": doContainer(ContainerArray, arrayFull()),
"firstBitSet": doContainer(ContainerArray, arrayFirstBitSet()),
@@ -267,7 +267,7 @@ func setupContainerTests() map[byte]map[string]*container {
}
// bitmap containers
- cts[ContainerBitmap] = map[string]*container{
+ cts[ContainerBitmap] = map[string]*Container{
"empty": doContainer(ContainerBitmap, bitmapEmpty()),
"full": doContainer(ContainerBitmap, bitmapFull()),
"firstBitSet": doContainer(ContainerBitmap, bitmapFirstBitSet()),
@@ -281,7 +281,7 @@ func setupContainerTests() map[byte]map[string]*container {
}
// run containers
- cts[ContainerRun] = map[string]*container{
+ cts[ContainerRun] = map[string]*Container{
"empty": doContainer(ContainerRun, runEmpty()),
"full": doContainer(ContainerRun, runFull()),
"firstBitSet": doContainer(ContainerRun, runFirstBitSet()),
diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go
index 216aa7eb7..6e1f105bf 100644
--- a/roaring/roaring_internal_test.go
+++ b/roaring/roaring_internal_test.go
@@ -28,12 +28,12 @@ func (iv interval16) String() string {
return fmt.Sprintf("[%d, %d]", iv.start, iv.last)
}
-func (c *container) String() string {
+func (c *Container) String() string {
return fmt.Sprintf("<%s container n=%d, array[%d], runs[%d], bitmap[%d]> type:%d", c.info().Type, c.n, len(c.array), len(c.runs), len(c.bitmap), c.containerType)
}
func TestRunAppendInterval(t *testing.T) {
- a := container{containerType: ContainerRun}
+ a := Container{containerType: ContainerRun}
tests := []struct {
base []interval16
app interval16
@@ -82,7 +82,7 @@ func TestInterval16RunLen(t *testing.T) {
}
func TestContainerRunAdd(t *testing.T) {
- c := container{runs: make([]interval16, 0), containerType: ContainerRun}
+ c := Container{runs: make([]interval16, 0), containerType: ContainerRun}
tests := []struct {
op uint16
exp []interval16
@@ -113,7 +113,7 @@ func TestContainerRunAdd(t *testing.T) {
}
func TestContainerRunAdd2(t *testing.T) {
- c := container{runs: make([]interval16, 0), containerType: ContainerRun}
+ c := Container{runs: make([]interval16, 0), containerType: ContainerRun}
ret := c.add(0)
if !ret {
t.Fatalf("result of adding new bit should be true: %v", c.runs)
@@ -128,7 +128,7 @@ func TestContainerRunAdd2(t *testing.T) {
}
func TestRunCountRange(t *testing.T) {
- c := container{runs: make([]interval16, 0), containerType: ContainerRun}
+ c := Container{runs: make([]interval16, 0), containerType: ContainerRun}
cnt := c.runCountRange(2, 9)
if cnt != 0 {
t.Fatalf("should get 0 from empty container, but got: %v", cnt)
@@ -181,7 +181,7 @@ func TestRunCountRange(t *testing.T) {
}
func TestRunContains(t *testing.T) {
- c := container{runs: make([]interval16, 0), containerType: ContainerRun}
+ c := Container{runs: make([]interval16, 0), containerType: ContainerRun}
if c.runContains(5) {
t.Fatalf("empty run container should not contain 5")
}
@@ -203,7 +203,7 @@ func TestRunContains(t *testing.T) {
}
func TestBitmapCountRange(t *testing.T) {
- c := container{containerType: ContainerBitmap}
+ c := Container{containerType: ContainerBitmap}
tests := []struct {
start int
end int
@@ -228,7 +228,7 @@ func TestBitmapCountRange(t *testing.T) {
}
func TestIntersectionCountArrayBitmap3(t *testing.T) {
- a, b := &container{}, &container{}
+ a, b := &Container{}, &Container{}
a.containerType = ContainerBitmap
a.bitmap = getFullBitmap()
a.n = maxContainerVal + 1
@@ -255,7 +255,7 @@ func TestIntersectionCountArrayBitmap3(t *testing.T) {
}
func TestIntersectionCountArrayBitmap2(t *testing.T) {
- a, b := &container{}, &container{}
+ a, b := &Container{}, &Container{}
tests := []struct {
array []uint16
bitmap []uint64
@@ -301,7 +301,7 @@ func TestIntersectionCountArrayBitmap2(t *testing.T) {
}
func TestRunRemove(t *testing.T) {
- c := container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, containerType: ContainerRun}
+ c := Container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, containerType: ContainerRun}
tests := []struct {
op uint16
exp []interval16
@@ -335,13 +335,13 @@ func TestRunRemove(t *testing.T) {
}
func TestRunMax(t *testing.T) {
- c := container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, containerType: ContainerRun}
+ c := Container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, containerType: ContainerRun}
max := c.max()
if max != 16 {
t.Fatalf("max for %v should be 16", c.runs)
}
- c = container{runs: []interval16{}}
+ c = Container{runs: []interval16{}}
max = c.max()
if max != 0 {
t.Fatalf("max for %v should be 0", c.runs)
@@ -349,8 +349,8 @@ func TestRunMax(t *testing.T) {
}
func TestIntersectionCountArrayRun(t *testing.T) {
- a := &container{containerType: ContainerArray, array: []uint16{1, 5, 10, 11, 12}}
- b := &container{containerType: ContainerRun, runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}}
+ a := &Container{containerType: ContainerArray, array: []uint16{1, 5, 10, 11, 12}}
+ b := &Container{containerType: ContainerRun, runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}}
ret := intersectionCountArrayRun(a, b)
if ret != 3 {
@@ -359,16 +359,16 @@ func TestIntersectionCountArrayRun(t *testing.T) {
}
func TestIntersectionCountBitmapRun(t *testing.T) {
- a := &container{containerType: ContainerBitmap, bitmap: []uint64{0x8000000000000000}}
- b := &container{containerType: ContainerRun, runs: []interval16{{start: 63, last: 64}}}
+ a := &Container{containerType: ContainerBitmap, bitmap: []uint64{0x8000000000000000}}
+ b := &Container{containerType: ContainerRun, runs: []interval16{{start: 63, last: 64}}}
ret := intersectionCountBitmapRun(a, b)
if ret != 1 {
t.Fatalf("count of %v with %v should be 1, but got %v", a.bitmap, b.runs, ret)
}
- a = &container{containerType: ContainerBitmap, bitmap: []uint64{0xF0000001, 0xFF00000000000000, 0xFF000000000000F0, 0x0F0000}}
- b = &container{containerType: ContainerRun, runs: []interval16{{start: 29, last: 31}, {start: 125, last: 134}, {start: 191, last: 197}, {start: 200, last: 300}}}
+ a = &Container{containerType: ContainerBitmap, bitmap: []uint64{0xF0000001, 0xFF00000000000000, 0xFF000000000000F0, 0x0F0000}}
+ b = &Container{containerType: ContainerRun, runs: []interval16{{start: 29, last: 31}, {start: 125, last: 134}, {start: 191, last: 197}, {start: 200, last: 300}}}
ret = intersectionCountBitmapRun(a, b)
if ret != 14 {
@@ -377,8 +377,8 @@ func TestIntersectionCountBitmapRun(t *testing.T) {
}
func TestIntersectionCountRunRun(t *testing.T) {
- a := &container{}
- b := &container{}
+ a := &Container{}
+ b := &Container{}
tests := []struct {
aruns []interval16
bruns []interval16
@@ -428,8 +428,8 @@ func TestIntersectionCountRunRun(t *testing.T) {
}
func TestIntersectArrayRun(t *testing.T) {
- a := &container{}
- b := &container{}
+ a := &Container{}
+ b := &Container{}
tests := []struct {
array []uint16
runs []interval16
@@ -470,8 +470,8 @@ func TestIntersectArrayRun(t *testing.T) {
}
func TestIntersectRunRun(t *testing.T) {
- a := &container{}
- b := &container{}
+ a := &Container{}
+ b := &Container{}
tests := []struct {
aruns []interval16
bruns []interval16
@@ -532,8 +532,8 @@ func TestIntersectRunRun(t *testing.T) {
}
func TestIntersectBitmapRunBitmap(t *testing.T) {
- a := &container{bitmap: make([]uint64, bitmapN)}
- b := &container{}
+ a := &Container{bitmap: make([]uint64, bitmapN)}
+ b := &Container{}
tests := []struct {
bitmap []uint64
runs []interval16
@@ -598,8 +598,8 @@ func TestIntersectBitmapRunBitmap(t *testing.T) {
}
func TestIntersectBitmapRunArray(t *testing.T) {
- a := &container{bitmap: make([]uint64, bitmapN)}
- b := &container{}
+ a := &Container{bitmap: make([]uint64, bitmapN)}
+ b := &Container{}
tests := []struct {
bitmap []uint64
runs []interval16
@@ -658,19 +658,19 @@ func TestIntersectBitmapRunArray(t *testing.T) {
func TestUnionMixed(t *testing.T) {
// array container
- a := &container{}
+ a := &Container{}
a.array = []uint16{1, 4, 5, 7, 10, 11, 12}
a.containerType = ContainerArray
a.n = 7
// bitmap container
- b := &container{bitmap: make([]uint64, bitmapN)}
+ b := &Container{bitmap: make([]uint64, bitmapN)}
b.bitmap[0] = uint64(0x3)
b.n = 2
b.containerType = ContainerBitmap
// run container
- r := &container{}
+ r := &Container{}
r.runs = []interval16{{start: 5, last: 10}}
r.containerType = ContainerRun
r.n = 6
@@ -678,8 +678,8 @@ func TestUnionMixed(t *testing.T) {
t.Run("various container Unions", func(t *testing.T) {
tests := []struct {
name string
- c1 *container
- c2 *container
+ c1 *Container
+ c2 *Container
exp []uint16
}{
{name: "run-array", c1: r, c2: a, exp: []uint16{1, 4, 5, 6, 7, 8, 9, 10, 11, 12}},
@@ -707,9 +707,9 @@ func TestUnionMixed(t *testing.T) {
}
func TestIntersectMixed(t *testing.T) {
- a := &container{}
- b := &container{}
- c := &container{}
+ a := &Container{}
+ b := &Container{}
+ c := &Container{}
a.runs = []interval16{{start: 5, last: 10}}
a.n = 6
@@ -755,10 +755,10 @@ func TestIntersectMixed(t *testing.T) {
}
func TestDifferenceMixed(t *testing.T) {
- a := &container{}
- b := &container{}
- c := &container{}
- d := &container{}
+ a := &Container{}
+ b := &Container{}
+ c := &Container{}
+ d := &Container{}
a.runs = []interval16{{start: 5, last: 10}}
a.n = a.runCountRange(0, 100)
@@ -834,8 +834,8 @@ func TestDifferenceMixed(t *testing.T) {
}
func TestUnionRunRun(t *testing.T) {
- a := &container{}
- b := &container{}
+ a := &Container{}
+ b := &Container{}
tests := []struct {
aruns []interval16
bruns []interval16
@@ -895,8 +895,8 @@ func TestUnionRunRun(t *testing.T) {
}
func TestUnionArrayRun(t *testing.T) {
- a := &container{}
- b := &container{}
+ a := &Container{}
+ b := &Container{}
tests := []struct {
array []uint16
runs []interval16
@@ -937,7 +937,7 @@ func TestUnionArrayRun(t *testing.T) {
}
func TestBitmapSetRange(t *testing.T) {
- c := &container{containerType: ContainerBitmap, bitmap: make([]uint64, bitmapN)}
+ c := &Container{containerType: ContainerBitmap, bitmap: make([]uint64, bitmapN)}
tests := []struct {
bitmap []uint64
start uint64
@@ -977,7 +977,7 @@ func TestBitmapSetRange(t *testing.T) {
}
func TestArrayToBitmap(t *testing.T) {
- a := &container{containerType: ContainerArray}
+ a := &Container{containerType: ContainerArray}
tests := []struct {
array []uint16
exp []uint64
@@ -1008,7 +1008,7 @@ func TestArrayToBitmap(t *testing.T) {
}
func TestBitmapToArray(t *testing.T) {
- a := &container{containerType: ContainerBitmap}
+ a := &Container{containerType: ContainerBitmap}
tests := []struct {
bitmap []uint64
exp []uint16
@@ -1039,7 +1039,7 @@ func TestBitmapToArray(t *testing.T) {
}
func TestRunToBitmap(t *testing.T) {
- a := &container{containerType: ContainerRun}
+ a := &Container{containerType: ContainerRun}
tests := []struct {
runs []interval16
exp []uint64
@@ -1093,7 +1093,7 @@ func getFullBitmap() []uint64 {
}
func TestBitmapToRun(t *testing.T) {
- a := &container{containerType: ContainerBitmap}
+ a := &Container{containerType: ContainerBitmap}
tests := []struct {
bitmap []uint64
exp []interval16
@@ -1171,7 +1171,7 @@ func TestBitmapToRun(t *testing.T) {
}
func TestArrayToRun(t *testing.T) {
- a := &container{containerType: ContainerArray}
+ a := &Container{containerType: ContainerArray}
tests := []struct {
array []uint16
exp []interval16
@@ -1205,7 +1205,7 @@ func TestArrayToRun(t *testing.T) {
}
func TestRunToArray(t *testing.T) {
- a := &container{containerType: ContainerRun}
+ a := &Container{containerType: ContainerRun}
tests := []struct {
runs []interval16
exp []uint16
@@ -1239,7 +1239,7 @@ func TestRunToArray(t *testing.T) {
}
func TestBitmapZeroRange(t *testing.T) {
- c := &container{containerType: ContainerBitmap, bitmap: make([]uint64, bitmapN)}
+ c := &Container{containerType: ContainerBitmap, bitmap: make([]uint64, bitmapN)}
tests := []struct {
bitmap []uint64
start uint64
@@ -1283,8 +1283,8 @@ func TestBitmapZeroRange(t *testing.T) {
}
func TestUnionBitmapRun(t *testing.T) {
- a := &container{containerType: ContainerBitmap, bitmap: make([]uint64, bitmapN)}
- b := &container{containerType: ContainerRun}
+ a := &Container{containerType: ContainerBitmap, bitmap: make([]uint64, bitmapN)}
+ b := &Container{containerType: ContainerRun}
tests := []struct {
bitmap []uint64
runs []interval16
@@ -1322,7 +1322,7 @@ func TestUnionBitmapRun(t *testing.T) {
}
func TestBitmapCountRuns(t *testing.T) {
- c := &container{containerType: ContainerBitmap, bitmap: make([]uint64, bitmapN)}
+ c := &Container{containerType: ContainerBitmap, bitmap: make([]uint64, bitmapN)}
tests := []struct {
bitmap []uint64
exp int
@@ -1372,7 +1372,7 @@ func TestBitmapCountRuns(t *testing.T) {
}
func TestArrayCountRuns(t *testing.T) {
- c := &container{containerType: ContainerArray}
+ c := &Container{containerType: ContainerArray}
tests := []struct {
array []uint16
exp int
@@ -1413,8 +1413,8 @@ func TestArrayCountRuns(t *testing.T) {
}
func TestDifferenceArrayRun(t *testing.T) {
- a := &container{containerType: ContainerArray}
- b := &container{containerType: ContainerRun}
+ a := &Container{containerType: ContainerArray}
+ b := &Container{containerType: ContainerRun}
tests := []struct {
array []uint16
runs []interval16
@@ -1439,8 +1439,8 @@ func TestDifferenceArrayRun(t *testing.T) {
}
func TestDifferenceRunArray(t *testing.T) {
- a := &container{containerType: ContainerRun}
- b := &container{containerType: ContainerArray}
+ a := &Container{containerType: ContainerRun}
+ b := &Container{containerType: ContainerArray}
tests := []struct {
runs []interval16
array []uint16
@@ -1513,15 +1513,15 @@ func MakeBitmap(start []uint64) []uint64 {
return b
}
func MakeLastBitSet() []uint64 {
- obj := NewBitmap(65535)
+ obj := NewFileBitmap(65535)
c := obj.container(0)
c.arrayToBitmap()
return c.bitmap
}
func TestDifferenceRunBitmap(t *testing.T) {
- a := &container{containerType: ContainerRun}
- b := &container{containerType: ContainerBitmap, bitmap: make([]uint64, bitmapN)}
+ a := &Container{containerType: ContainerRun}
+ b := &Container{containerType: ContainerBitmap, bitmap: make([]uint64, bitmapN)}
tests := []struct {
runs []interval16
bitmap []uint64
@@ -1583,8 +1583,8 @@ func TestDifferenceRunBitmap(t *testing.T) {
}
func TestDifferenceBitmapRun(t *testing.T) {
- a := &container{containerType: ContainerBitmap, bitmap: make([]uint64, bitmapN)}
- b := &container{containerType: ContainerRun}
+ a := &Container{containerType: ContainerBitmap, bitmap: make([]uint64, bitmapN)}
+ b := &Container{containerType: ContainerRun}
tests := []struct {
bitmap []uint64
runs []interval16
@@ -1666,8 +1666,8 @@ func TestDifferenceBitmapRun(t *testing.T) {
}
func TestDifferenceBitmapArray(t *testing.T) {
- b := &container{containerType: ContainerBitmap, bitmap: make([]uint64, bitmapN)}
- a := &container{containerType: ContainerArray}
+ b := &Container{containerType: ContainerBitmap, bitmap: make([]uint64, bitmapN)}
+ a := &Container{containerType: ContainerArray}
tests := []struct {
bitmap []uint64
array []uint16
@@ -1716,8 +1716,8 @@ func TestDifferenceBitmapArray(t *testing.T) {
}
func TestDifferenceBitmapBitmap(t *testing.T) {
- a := &container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap}
- b := &container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap}
+ a := &Container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap}
+ b := &Container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap}
tests := []struct {
abitmap []uint64
bbitmap []uint64
@@ -1746,8 +1746,8 @@ func TestDifferenceBitmapBitmap(t *testing.T) {
}
func TestDifferenceRunRun(t *testing.T) {
- a := &container{containerType: ContainerRun}
- b := &container{containerType: ContainerRun}
+ a := &Container{containerType: ContainerRun}
+ b := &Container{containerType: ContainerRun}
tests := []struct {
aruns []interval16
bruns []interval16
@@ -1780,9 +1780,10 @@ func TestDifferenceRunRun(t *testing.T) {
}
func TestWriteReadArray(t *testing.T) {
- ca := &container{array: []uint16{1, 10, 100, 1000}, n: 4, containerType: ContainerArray}
- ba := &Bitmap{keys: []uint64{0}, containers: []*container{ca}}
- ba2 := &Bitmap{}
+ ca := &Container{array: []uint16{1, 10, 100, 1000}, n: 4, containerType: ContainerArray}
+ ba := NewFileBitmap()
+ ba.Containers.Put(0, ca)
+ ba2 := NewFileBitmap()
var buf bytes.Buffer
_, err := ba.WriteTo(&buf)
if err != nil {
@@ -1792,19 +1793,20 @@ func TestWriteReadArray(t *testing.T) {
if err != nil {
t.Fatalf("error unmarshaling: %v", err)
}
- if !reflect.DeepEqual(ba2.containers[0].array, ca.array) {
- t.Fatalf("array test expected %x, but got %x", ca.array, ba2.containers[0].array)
+ if !reflect.DeepEqual(ba2.Containers.Get(0).array, ca.array) {
+ t.Fatalf("array test expected %x, but got %x", ca.array, ba2.Containers.Get(0).array)
}
}
func TestWriteReadBitmap(t *testing.T) {
// create bitmap containing > 4096 bits
- cb := &container{bitmap: make([]uint64, bitmapN), n: 129 * 32, containerType: ContainerBitmap}
+ cb := &Container{bitmap: make([]uint64, bitmapN), n: 129 * 32, containerType: ContainerBitmap}
for i := 0; i < 129; i++ {
cb.bitmap[i] = 0x5555555555555555
}
- bb := &Bitmap{keys: []uint64{0}, containers: []*container{cb}}
- bb2 := &Bitmap{}
+ bb := NewFileBitmap()
+ bb.Containers.Put(0, cb)
+ bb2 := NewFileBitmap()
var buf bytes.Buffer
_, err := bb.WriteTo(&buf)
if err != nil {
@@ -1814,19 +1816,20 @@ func TestWriteReadBitmap(t *testing.T) {
if err != nil {
t.Fatalf("error unmarshaling: %v", err)
}
- if !reflect.DeepEqual(bb2.containers[0].bitmap, cb.bitmap) {
- t.Fatalf("bitmap test expected %x, but got %x", cb.bitmap, bb2.containers[0].bitmap)
+ if !reflect.DeepEqual(bb2.Containers.Get(0).bitmap, cb.bitmap) {
+ t.Fatalf("bitmap test expected %x, but got %x", cb.bitmap, bb2.Containers.Get(0).bitmap)
}
}
func TestWriteReadFullBitmap(t *testing.T) {
// create bitmap containing > 4096 bits
- cb := &container{bitmap: make([]uint64, bitmapN), n: 65536, containerType: ContainerBitmap}
+ cb := &Container{bitmap: make([]uint64, bitmapN), n: 65536, containerType: ContainerBitmap}
for i := 0; i < bitmapN; i++ {
cb.bitmap[i] = 0xffffffffffffffff
}
- bb := &Bitmap{keys: []uint64{0}, containers: []*container{cb}}
- bb2 := &Bitmap{}
+ bb := NewFileBitmap()
+ bb.Containers.Put(0, cb)
+ bb2 := NewFileBitmap()
var buf bytes.Buffer
_, err := bb.WriteTo(&buf)
if err != nil {
@@ -1836,22 +1839,23 @@ func TestWriteReadFullBitmap(t *testing.T) {
if err != nil {
t.Fatalf("error unmarshaling: %v", err)
}
- if !reflect.DeepEqual(bb2.containers[0].bitmap, cb.bitmap) {
- t.Fatalf("bitmap test expected %x, but got %x", cb.bitmap, bb2.containers[0].bitmap)
+ if !reflect.DeepEqual(bb2.Containers.Get(0).bitmap, cb.bitmap) {
+ t.Fatalf("bitmap test expected %x, but got %x", cb.bitmap, bb2.Containers.Get(0).bitmap)
}
- if bb2.containers[0].n != cb.n {
- t.Fatalf("bitmap test expected count %x, but got %x", cb.n, bb2.containers[0].n)
+ if bb2.Containers.Get(0).n != cb.n {
+ t.Fatalf("bitmap test expected count %x, but got %x", cb.n, bb2.Containers.Get(0).n)
}
- if bb2.containers[0].count() != cb.count() {
- t.Fatalf("bitmap test expected count %x, but got %x", cb.n, bb2.containers[0].n)
+ if bb2.Containers.Get(0).count() != cb.count() {
+ t.Fatalf("bitmap test expected count %x, but got %x", cb.n, bb2.Containers.Get(0).n)
}
}
func TestWriteReadRun(t *testing.T) {
- cr := &container{runs: []interval16{{start: 3, last: 13}, {start: 100, last: 109}}, n: 21, containerType: ContainerRun}
- br := &Bitmap{keys: []uint64{0}, containers: []*container{cr}}
- br2 := &Bitmap{}
+ cr := &Container{runs: []interval16{{start: 3, last: 13}, {start: 100, last: 109}}, n: 21, containerType: ContainerRun}
+ br := NewFileBitmap()
+ br.Containers.Put(0, cr)
+ br2 := NewFileBitmap()
var buf bytes.Buffer
_, err := br.WriteTo(&buf)
if err != nil {
@@ -1861,33 +1865,33 @@ func TestWriteReadRun(t *testing.T) {
if err != nil {
t.Fatalf("error unmarshaling: %v", err)
}
- if !reflect.DeepEqual(br2.containers[0].runs, cr.runs) {
- t.Fatalf("run test expected %x, but got %x", cr.runs, br2.containers[0].runs)
+ if !reflect.DeepEqual(br2.Containers.Get(0).runs, cr.runs) {
+ t.Fatalf("run test expected %x, but got %x", cr.runs, br2.Containers.Get(0).runs)
}
}
func TestXorArrayRun(t *testing.T) {
tests := []struct {
- a *container
- b *container
- exp *container
+ a *Container
+ b *Container
+ exp *Container
}{
{
- a: &container{array: []uint16{1, 5, 10, 11, 12}, containerType: ContainerArray},
- b: &container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, containerType: ContainerRun},
- exp: &container{array: []uint16{1, 2, 3, 4, 6, 7, 8, 9, 11, 13, 15, 16}, containerType: ContainerArray, n: 12},
+ a: &Container{array: []uint16{1, 5, 10, 11, 12}, containerType: ContainerArray},
+ b: &Container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, containerType: ContainerRun},
+ exp: &Container{array: []uint16{1, 2, 3, 4, 6, 7, 8, 9, 11, 13, 15, 16}, containerType: ContainerArray, n: 12},
}, {
- a: &container{array: []uint16{1, 5, 10, 11, 12, 13, 14}, containerType: ContainerArray},
- b: &container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, containerType: ContainerRun},
- exp: &container{array: []uint16{1, 2, 3, 4, 6, 7, 8, 9, 11, 14, 15, 16}, containerType: ContainerArray, n: 12},
+ a: &Container{array: []uint16{1, 5, 10, 11, 12, 13, 14}, containerType: ContainerArray},
+ b: &Container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, containerType: ContainerRun},
+ exp: &Container{array: []uint16{1, 2, 3, 4, 6, 7, 8, 9, 11, 14, 15, 16}, containerType: ContainerArray, n: 12},
}, {
- a: &container{array: []uint16{65535}, containerType: ContainerArray},
- b: &container{runs: []interval16{{start: 65534, last: 65535}}, containerType: ContainerRun},
- exp: &container{array: []uint16{65534}, containerType: ContainerArray, n: 1},
+ a: &Container{array: []uint16{65535}, containerType: ContainerArray},
+ b: &Container{runs: []interval16{{start: 65534, last: 65535}}, containerType: ContainerRun},
+ exp: &Container{array: []uint16{65534}, containerType: ContainerArray, n: 1},
}, {
- a: &container{array: []uint16{65535}, containerType: ContainerArray},
- b: &container{runs: []interval16{{start: 65535, last: 65535}}, containerType: ContainerRun},
- exp: &container{array: []uint16{}, containerType: ContainerArray, n: 0},
+ a: &Container{array: []uint16{65535}, containerType: ContainerArray},
+ b: &Container{runs: []interval16{{start: 65535, last: 65535}}, containerType: ContainerRun},
+ exp: &Container{array: []uint16{}, containerType: ContainerArray, n: 0},
},
}
@@ -1908,8 +1912,8 @@ func TestXorArrayRun(t *testing.T) {
//special case that didn't fit the xorrunrun table testing below.
func TestXorRunRun1(t *testing.T) {
- a := &container{containerType: ContainerRun}
- b := &container{containerType: ContainerRun}
+ a := &Container{containerType: ContainerRun}
+ b := &Container{containerType: ContainerRun}
a.runs = []interval16{{start: 4, last: 10}}
b.runs = []interval16{{start: 5, last: 10}}
ret := xorRunRun(a, b)
@@ -1923,8 +1927,8 @@ func TestXorRunRun1(t *testing.T) {
}
func TestXorRunRun(t *testing.T) {
- a := &container{containerType: ContainerRun}
- b := &container{containerType: ContainerRun}
+ a := &Container{containerType: ContainerRun}
+ b := &Container{containerType: ContainerRun}
tests := []struct {
aruns []interval16
bruns []interval16
@@ -2021,7 +2025,7 @@ func TestXorRunRun(t *testing.T) {
}
func TestBitmapXorRange(t *testing.T) {
- c := &container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap}
+ c := &Container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap}
tests := []struct {
bitmap []uint64
start uint64
@@ -2089,8 +2093,8 @@ func TestBitmapXorRange(t *testing.T) {
}
func TestXorBitmapRun(t *testing.T) {
- a := &container{containerType: ContainerBitmap}
- b := &container{containerType: ContainerRun}
+ a := &Container{containerType: ContainerBitmap}
+ b := &Container{containerType: ContainerRun}
tests := []struct {
bitmap []uint64
runs []interval16
@@ -2120,35 +2124,35 @@ func TestXorBitmapRun(t *testing.T) {
func TestIteratorArray(t *testing.T) {
// use values that span two containers
- b := NewBitmap(0, 1, 10, 100, 1000, 10000, 90000, 100000)
- if !b.containers[0].isArray() {
+ b := NewFileBitmap(0, 1, 10, 100, 1000, 10000, 90000, 100000)
+ if !b.Containers.Get(0).isArray() {
t.Fatalf("wrong container type")
}
itr := b.Iterator()
- if !(itr.i == 0 && itr.j == -1) {
+ if !(itr.key == 0 && itr.j == -1) {
t.Fatalf("iterator did not zero correctly: %v\n", itr)
}
itr.Seek(1000)
- if !(itr.i == 0 && itr.j == 3) {
- t.Fatalf("iterator did not seek correctly: %v\n", itr)
+ if !(itr.key == 0 && itr.j == 3) {
+ t.Fatalf("iterator did not seek correctly: %#v\n", itr)
}
itr.Seek(10000)
itr.Next()
val, eof := itr.Next()
- if !(itr.i == 1 && itr.j == 0 && val == 90000 && !eof) {
+ if !(itr.key == 1 && itr.j == 0 && val == 90000 && !eof) {
t.Fatalf("iterator did not next correctly across containers: %v\n", itr)
}
itr.Seek(80000)
- if !(itr.i == 1 && itr.j == -1) {
+ if !(itr.key == 1 && itr.j == -1) {
t.Fatalf("iterator did not seek missing value correctly: %v\n", itr)
}
itr.Seek(100000)
- if !(itr.i == 1 && itr.j == 0) {
+ if !(itr.key == 1 && itr.j == 0) {
t.Fatalf("iterator did not seek correctly in multiple containers: %v\n", itr)
}
@@ -2167,41 +2171,41 @@ func TestIteratorBitmap(t *testing.T) {
// use values that span two containers
// this dataset will update to bitmap after enough Adds,
// but won't update to RLE until Optimize() is called
- b := NewBitmap()
+ b := NewFileBitmap()
for i := uint64(61000); i < 71000; i++ {
b.Add(i)
}
for i := uint64(75000); i < 75100; i++ {
b.Add(i)
}
- if !b.containers[0].isBitmap() {
+ if !b.Containers.Get(0).isBitmap() {
t.Fatalf("wrong container type")
}
itr := b.Iterator()
- if !(itr.i == 0 && itr.j == -1) {
+ if !(itr.key == 0 && itr.j == -1) {
t.Fatalf("iterator did not zero correctly: %v\n", itr)
}
itr.Seek(65000)
- if !(itr.i == 0 && itr.j == 64999) {
+ if !(itr.key == 0 && itr.j == 64999) {
t.Fatalf("iterator did not seek correctly: %v\n", itr)
}
itr.Seek(65535)
itr.Next()
val, eof := itr.Next()
- if !(itr.i == 1 && itr.j == 0 && val == 65536 && !eof) {
+ if !(itr.key == 1 && itr.j == 0 && val == 65536 && !eof) {
t.Fatalf("iterator did not next correctly across containers: %v\n", itr)
}
itr.Seek(74000)
- if !(itr.i == 1 && itr.j == 8463) {
+ if !(itr.key == 1 && itr.j == 8463) {
t.Fatalf("iterator did not seek missing value correctly: %v\n", itr)
}
itr.Seek(70999)
- if !(itr.i == 1 && itr.j == 5462) {
+ if !(itr.key == 1 && itr.j == 5462) {
t.Fatalf("iterator did not seek correctly in multiple containers: %v\n", itr)
}
@@ -2218,19 +2222,19 @@ func TestIteratorBitmap(t *testing.T) {
}
func TestIteratorRuns(t *testing.T) {
- b := NewBitmap(0, 1, 2, 3, 4, 5, 1000, 1001, 1002, 1003, 1004, 1005, 100000, 100001, 100002, 100003, 100004, 100005)
+ b := NewFileBitmap(0, 1, 2, 3, 4, 5, 1000, 1001, 1002, 1003, 1004, 1005, 100000, 100001, 100002, 100003, 100004, 100005)
b.Optimize()
- if !b.containers[0].isRun() {
+ if !b.Containers.Get(0).isRun() {
t.Fatalf("wrong container type")
}
itr := b.Iterator()
- if !(itr.i == 0 && itr.j == 0 && itr.k == -1) {
+ if !(itr.key == 0 && itr.j == 0 && itr.k == -1) {
t.Fatalf("iterator did not zero correctly: %v\n", itr)
}
itr.Seek(4)
- if !(itr.i == 0 && itr.j == 0 && itr.k == 3) {
+ if !(itr.key == 0 && itr.j == 0 && itr.k == 3) {
t.Fatalf("iterator did not seek correctly: %v\n", itr)
}
itr.Next()
@@ -2253,22 +2257,22 @@ func TestIteratorRuns(t *testing.T) {
}
itr.Seek(500)
- if !(itr.i == 0 && itr.j == 1 && itr.k == -1) {
+ if !(itr.key == 0 && itr.j == 1 && itr.k == -1) {
t.Fatalf("iterator did not seek missing value correctly: %v\n", itr)
}
itr.Seek(1004)
- if !(itr.i == 0 && itr.j == 1 && itr.k == 3) {
+ if !(itr.key == 0 && itr.j == 1 && itr.k == 3) {
t.Fatalf("iterator did not seek correctly in multiple runs: %v\n", itr)
}
itr.Seek(1005)
- if !(itr.i == 0 && itr.j == 1 && itr.k == 4) {
+ if !(itr.key == 0 && itr.j == 1 && itr.k == 4) {
t.Fatalf("iterator did not seek correctly to end of run: %v\n", itr)
}
itr.Seek(100005)
- if !(itr.i == 1 && itr.j == 0 && itr.k == 4) {
+ if !(itr.key == 1 && itr.j == 0 && itr.k == 4) {
t.Fatalf("iterator did not seek correctly in multiple containers: %v\n", itr)
}
@@ -2285,7 +2289,7 @@ func TestIteratorVarious(t *testing.T) {
exp uint64
}{
{
- bm: NewBitmap(3, 4, 5),
+ bm: NewFileBitmap(3, 4, 5),
exp: 3,
},
{
@@ -2293,7 +2297,7 @@ func TestIteratorVarious(t *testing.T) {
exp: 61221,
},
{
- bm: NewBitmap(2, 66000, 70000, 70001, 70002, 70003, 70004),
+ bm: NewFileBitmap(2, 66000, 70000, 70001, 70002, 70003, 70004),
exp: 7,
},
}
@@ -2434,7 +2438,7 @@ func TestRunBinSearch(t *testing.T) {
}
}
func TestBitmap_RemoveEmptyContainers(t *testing.T) {
- bm1 := NewBitmap(1<<16, 2<<16, 3<<16)
+ bm1 := NewFileBitmap(1<<16, 2<<16, 3<<16)
bm1.Remove(2 << 16)
if bm1.countEmptyContainers() != 1 {
t.Fatalf("Should be 1 empty container ")
@@ -2447,13 +2451,13 @@ func TestBitmap_RemoveEmptyContainers(t *testing.T) {
}
func TestBitmap_BitmapWriteToWithEmpty(t *testing.T) {
- bm1 := NewBitmap(1<<16, 2<<16, 3<<16)
+ bm1 := NewFileBitmap(1<<16, 2<<16, 3<<16)
bm1.Remove(2 << 16)
var buf bytes.Buffer
if _, err := bm1.WriteTo(&buf); err != nil {
t.Fatalf("Failure to write to bitmap buffer. ")
}
- bm0 := NewBitmap()
+ bm0 := NewFileBitmap()
bm0.UnmarshalBinary(buf.Bytes())
if bm0.countEmptyContainers() != 0 {
t.Fatalf("Should be no empty containers ")
@@ -2542,7 +2546,7 @@ func TestSearch64(t *testing.T) {
}
func TestIntersectArrayBitmap(t *testing.T) {
- a, b := &container{containerType: ContainerArray}, &container{
+ a, b := &Container{containerType: ContainerArray}, &Container{
containerType: ContainerBitmap,
bitmap: make([]uint64, bitmapN),
}
@@ -2605,6 +2609,24 @@ func TestIntersectArrayBitmap(t *testing.T) {
}
}
+func TestBitmapClone(t *testing.T) {
+ b := NewFileBitmap()
+ for i := uint64(61000); i < 71000; i++ {
+ b.Add(i)
+ }
+ c := b.Clone()
+ if err := bitmapsEqual(b, c); err != nil {
+ t.Fatalf("Clone Objects not equal: %v\n", err)
+ }
+ d := func() *Bitmap { //anybody know how to declare a nil value?
+ return nil
+ }()
+ e := d.Clone()
+ if e != nil {
+ t.Fatalf("Clone nil Objects not equal\n")
+ }
+}
+
// rleCont returns a slice of numbers all in the range starting from
// container_width*num, and ending at container_width*(num+1)-1. If left is
// true, then the first 100 bits will be set, if mid is true, 100 bits in the
@@ -2682,7 +2704,7 @@ func bitmapVariousContainers() *Bitmap {
bits = append(bits, bitCont(7, true, true, true)...)
bits = append(bits, arrCont(8, true, true, true)...)
bits = append(bits, rleCont(9, true, true, true)...)
- bm := NewBitmap(bits...)
+ bm := NewFileBitmap(bits...)
bm.Optimize()
return bm
}
@@ -3175,7 +3197,7 @@ func TestContainerCombinations(t *testing.T) {
// Convert to all container types and check result.
for _, ct := range containerTypes {
- clone := ret.clone()
+ clone := ret.Clone()
if ct == ContainerArray {
if clone.isBitmap() {
clone.bitmapToArray()
@@ -3222,12 +3244,12 @@ func TestContainerCombinations(t *testing.T) {
}
//func getFunc(func(a, b *container) *container, m, n *container) *container {
-func runContainerFunc(f interface{}, c ...*container) *container {
+func runContainerFunc(f interface{}, c ...*Container) *Container {
switch f.(type) {
- case func(*container) *container:
- return f.(func(*container) *container)(c[0])
- case func(*container, *container) *container:
- return f.(func(a, b *container) *container)(c[0], c[1])
+ case func(*Container) *Container:
+ return f.(func(*Container) *Container)(c[0])
+ case func(*Container, *Container) *Container:
+ return f.(func(a, b *Container) *Container)(c[0], c[1])
}
return nil
}
diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go
index 7c5539b34..c1291d1f7 100644
--- a/roaring/roaring_test.go
+++ b/roaring/roaring_test.go
@@ -29,26 +29,8 @@ import (
_ "github.com/pilosa/pilosa/test"
)
-func TestBitmapClone(t *testing.T) {
- b := roaring.NewBitmap()
- for i := uint64(61000); i < 71000; i++ {
- b.Add(i)
- }
- c := b.Clone()
- if !reflect.DeepEqual(b, c) {
- t.Fatalf("Clone Objects not equal\n")
- }
- d := func() *roaring.Bitmap { //anybody know how to declare a nil value?
- return nil
- }()
- e := d.Clone()
- if e != nil {
- t.Fatalf("Clone nil Objects not equal\n")
- }
-}
-
func TestContainerCount(t *testing.T) {
- b := roaring.NewBitmap(65535)
+ b := roaring.NewFileBitmap(65535)
if b.Count() != b.CountRange(0, 65546) {
t.Fatalf("Count != CountRange\n")
@@ -144,7 +126,7 @@ func TestCountRange(t *testing.T) {
for _, test := range tests {
t.Run(fmt.Sprintf("%s: %d to %d in '%v'", test.name, test.start, test.end, test.bitmap), func(t *testing.T) {
- b := roaring.NewBitmap(test.bitmap...)
+ b := roaring.NewFileBitmap(test.bitmap...)
actual := b.CountRange(test.start, test.end)
if actual != test.exp {
t.Errorf("got: %d, exp: %d", actual, test.exp)
@@ -154,7 +136,7 @@ func TestCountRange(t *testing.T) {
}
func TestCheckBitmap(t *testing.T) {
- b := roaring.NewBitmap()
+ b := roaring.NewFileBitmap()
x := 0
for i := uint64(61000); i < 71000; i++ {
x++
@@ -171,7 +153,7 @@ func TestCheckBitmap(t *testing.T) {
}
func TestCheckArray(t *testing.T) {
- b := roaring.NewBitmap(0, 1, 10, 100, 1000, 10000, 90000, 100000)
+ b := roaring.NewFileBitmap(0, 1, 10, 100, 1000, 10000, 90000, 100000)
err := b.Check()
if err != nil {
t.Fatalf("%v\n", err)
@@ -179,7 +161,7 @@ func TestCheckArray(t *testing.T) {
}
func TestCheckRun(t *testing.T) {
- b := roaring.NewBitmap(0, 1, 2, 3, 4, 5, 1000, 1001, 1002, 1003, 1004, 1005, 100000, 100001, 100002, 100003, 100004, 100005)
+ b := roaring.NewFileBitmap(0, 1, 2, 3, 4, 5, 1000, 1001, 1002, 1003, 1004, 1005, 100000, 100001, 100002, 100003, 100004, 100005)
b.Optimize() // convert to runs
err := b.Check()
if err != nil {
@@ -187,7 +169,7 @@ func TestCheckRun(t *testing.T) {
}
}
func TestCheckFullRun(t *testing.T) {
- b := roaring.NewBitmap()
+ b := roaring.NewFileBitmap()
for i := uint64(0); i < 2097152; i++ {
if i%16384 == 0 {
b.Optimize() // convert to runs
@@ -208,7 +190,7 @@ func TestCheckFullRun(t *testing.T) {
// Ensure that we can transition between runs and arrays when materializing the bitmap.
func TestContainerTransitions(t *testing.T) {
// [run, run][array][run]
- b := roaring.NewBitmap(0, 1, 2, 3, 4, 5, 1000, 1001, 1002, 1003, 1004, 1005, 100000, 100001, 100002, 132000, 132001, 132002, 132003, 132004, 132005)
+ b := roaring.NewFileBitmap(0, 1, 2, 3, 4, 5, 1000, 1001, 1002, 1003, 1004, 1005, 100000, 100001, 100002, 132000, 132001, 132002, 132003, 132004, 132005)
b.Optimize() // convert to runs
if !reflect.DeepEqual(b.Slice(), []uint64{0, 1, 2, 3, 4, 5, 1000, 1001, 1002, 1003, 1004, 1005, 100000, 100001, 100002, 132000, 132001, 132002, 132003, 132004, 132005}) {
t.Fatalf("unexpected slice: %+v", b.Slice())
@@ -216,7 +198,7 @@ func TestContainerTransitions(t *testing.T) {
// Test the case where last and first bits of adjoining containers are set.
// [run][array][run]
- b2 := roaring.NewBitmap(65531, 65532, 65533, 65534, 65535, 65536, 131071, 131072, 131073, 131074, 131075, 131076)
+ b2 := roaring.NewFileBitmap(65531, 65532, 65533, 65534, 65535, 65536, 131071, 131072, 131073, 131074, 131075, 131076)
b2.Optimize() // convert to runs
if !reflect.DeepEqual(b2.Slice(), []uint64{65531, 65532, 65533, 65534, 65535, 65536, 131071, 131072, 131073, 131074, 131075, 131076}) {
t.Fatalf("unexpected slice: %+v", b2.Slice())
@@ -225,26 +207,26 @@ func TestContainerTransitions(t *testing.T) {
// Ensure an empty bitmap returns false if checking for existence.
func TestBitmap_Contains_Empty(t *testing.T) {
- if roaring.NewBitmap().Contains(1000) {
+ if roaring.NewFileBitmap().Contains(1000) {
t.Fatal("expected false")
}
}
// Ensure an empty bitmap does nothing when removing an element.
func TestBitmap_Remove_Empty(t *testing.T) {
- roaring.NewBitmap().Remove(1000)
+ roaring.NewFileBitmap().Remove(1000)
}
// Ensure a bitmap can return a slice of values.
func TestBitmap_Slice(t *testing.T) {
- if a := roaring.NewBitmap(1, 2, 3).Slice(); !reflect.DeepEqual(a, []uint64{1, 2, 3}) {
+ if a := roaring.NewFileBitmap(1, 2, 3).Slice(); !reflect.DeepEqual(a, []uint64{1, 2, 3}) {
t.Fatalf("unexpected slice: %+v", a)
}
}
// Ensure an empty bitmap returns an empty slice of values.
func TestBitmap_Slice_Empty(t *testing.T) {
- if a := roaring.NewBitmap().Slice(); len(a) != 0 {
+ if a := roaring.NewFileBitmap().Slice(); len(a) != 0 {
t.Fatalf("unexpected slice: %+v", a)
}
}
@@ -252,7 +234,7 @@ func TestBitmap_Slice_Empty(t *testing.T) {
// Ensure a bitmap can return a slice of values within a range.
// TODO duplicate for all container types
func TestBitmap_SliceRange(t *testing.T) {
- if a := roaring.NewBitmap(0, 1000001, 1000002, 1000003).SliceRange(1, 1000003); !reflect.DeepEqual(a, []uint64{1000001, 1000002}) {
+ if a := roaring.NewFileBitmap(0, 1000001, 1000002, 1000003).SliceRange(1, 1000003); !reflect.DeepEqual(a, []uint64{1000001, 1000002}) {
t.Fatalf("unexpected slice: %+v", a)
}
}
@@ -260,7 +242,7 @@ func TestBitmap_SliceRange(t *testing.T) {
// Ensure a bitmap can loop over a set of values.
func TestBitmap_ForEach(t *testing.T) {
var a []uint64
- roaring.NewBitmap(1, 2, 3).ForEach(func(v uint64) {
+ roaring.NewFileBitmap(1, 2, 3).ForEach(func(v uint64) {
a = append(a, v)
})
if !reflect.DeepEqual(a, []uint64{1, 2, 3}) {
@@ -271,7 +253,7 @@ func TestBitmap_ForEach(t *testing.T) {
// Ensure a bitmap can loop over a set of values in a range.
func TestBitmap_ForEachRange(t *testing.T) {
var a []uint64
- roaring.NewBitmap(1, 2, 3, 4).ForEachRange(2, 4, func(v uint64) {
+ roaring.NewFileBitmap(1, 2, 3, 4).ForEachRange(2, 4, func(v uint64) {
a = append(a, v)
})
if !reflect.DeepEqual(a, []uint64{2, 3}) {
@@ -281,7 +263,7 @@ func TestBitmap_ForEachRange(t *testing.T) {
// Ensure bitmap can return the highest value.
func TestBitmap_Max(t *testing.T) {
- bm := roaring.NewBitmap()
+ bm := roaring.NewFileBitmap()
for i := uint64(1000); i <= 100000; i++ {
bm.Add(i)
@@ -297,7 +279,7 @@ func TestBitmap_BitmapCountRangeEdgeCase(t *testing.T) {
e := uint64(2010 * 1048576)
start := s + (39314024 % 1048576)
- bm0 := roaring.NewBitmap()
+ bm0 := roaring.NewFileBitmap()
for i := uint64(0); i < 65536; i++ {
if (i+1)%4096 == 0 {
start += 16384
@@ -315,7 +297,7 @@ func TestBitmap_BitmapCountRangeEdgeCase(t *testing.T) {
}
func TestBitmap_BitmapCountRange(t *testing.T) {
- bm0 := roaring.NewBitmap(0, 2683177)
+ bm0 := roaring.NewFileBitmap(0, 2683177)
for i := uint64(628); i < 2683301; i++ {
bm0.Add(i)
}
@@ -343,20 +325,20 @@ func TestBitmap_BitmapCountRange(t *testing.T) {
}
func TestBitmap_ArrayCountRange(t *testing.T) {
- bm0 := roaring.NewBitmap(0, 2683177, 2683313)
+ bm0 := roaring.NewFileBitmap(0, 2683177, 2683313)
if n := bm0.CountRange(1, 2683313); n != 1 {
t.Fatalf("unexpected n: %d", n)
}
}
func TestBitmap_RunCountRange(t *testing.T) {
- bm0 := roaring.NewBitmap(0, 1, 2, 3, 4, 5, 12, 13, 14, 15, 16, 17, 1000000, 1000002, 1000003, 1000004, 1000005, 1000006, 1000010, 1000011, 1000012, 1000013, 1000014)
+ bm0 := roaring.NewFileBitmap(0, 1, 2, 3, 4, 5, 12, 13, 14, 15, 16, 17, 1000000, 1000002, 1000003, 1000004, 1000005, 1000006, 1000010, 1000011, 1000012, 1000013, 1000014)
bm0.Optimize() // convert to runs
if n := bm0.CountRange(15, 1000003); n != 5 {
t.Fatalf("unexpected n: %d", n)
}
- bm1 := roaring.NewBitmap(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17)
+ bm1 := roaring.NewFileBitmap(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17)
bm1.Optimize() // convert to runs
if n := bm1.CountRange(5, 12); n != 7 {
t.Fatalf("unexpected n: %d", n)
@@ -364,8 +346,8 @@ func TestBitmap_RunCountRange(t *testing.T) {
}
func TestBitmap_Intersection(t *testing.T) {
- bm0 := roaring.NewBitmap(0, 2683177)
- bm1 := roaring.NewBitmap()
+ bm0 := roaring.NewFileBitmap(0, 2683177)
+ bm1 := roaring.NewFileBitmap()
for i := uint64(628); i < 2683301; i++ {
bm1.Add(i)
}
@@ -378,8 +360,8 @@ func TestBitmap_Intersection(t *testing.T) {
}
func TestBitmap_Union1(t *testing.T) {
- bm0 := roaring.NewBitmap(0, 2683177)
- bm1 := roaring.NewBitmap()
+ bm0 := roaring.NewFileBitmap(0, 2683177)
+ bm1 := roaring.NewFileBitmap()
for i := uint64(628); i < 2683301; i++ {
bm1.Add(i)
}
@@ -402,8 +384,8 @@ func TestBitmap_Union1(t *testing.T) {
}
func TestBitmap_Intersection_Empty(t *testing.T) {
- bm0 := roaring.NewBitmap(0, 2683177)
- bm1 := roaring.NewBitmap()
+ bm0 := roaring.NewFileBitmap(0, 2683177)
+ bm1 := roaring.NewFileBitmap()
result := bm0.Intersect(bm1)
if n := result.Count(); n != 0 {
@@ -413,8 +395,8 @@ func TestBitmap_Intersection_Empty(t *testing.T) {
}
func TestBitmap_IntersectArrayArray(t *testing.T) {
- bm0 := roaring.NewBitmap(0, 1, 2683, 5005)
- bm1 := roaring.NewBitmap(0, 2683, 2684, 5000)
+ bm0 := roaring.NewFileBitmap(0, 1, 2683, 5005)
+ bm1 := roaring.NewFileBitmap(0, 2683, 2684, 5000)
result := bm0.Intersect(bm1)
if n := result.Count(); n != 2 {
@@ -423,12 +405,12 @@ func TestBitmap_IntersectArrayArray(t *testing.T) {
}
func TestBitmap_IntersectBitmapBitmap(t *testing.T) {
- bm0 := roaring.NewBitmap()
+ bm0 := roaring.NewFileBitmap()
for i := uint64(0); i < 65536; i += 2 {
bm0.Add(i)
}
- bm1 := roaring.NewBitmap()
+ bm1 := roaring.NewFileBitmap()
for i := uint64(0); i < 65536; i += 3 {
bm1.Add(i)
}
@@ -441,9 +423,9 @@ func TestBitmap_IntersectBitmapBitmap(t *testing.T) {
func TestBitmap_IntersectRunRun(t *testing.T) {
// Intersect two runs that result in an array.
- bm0 := roaring.NewBitmap(0, 1, 2, 3, 4, 5, 10, 11, 12, 13, 14, 15)
+ bm0 := roaring.NewFileBitmap(0, 1, 2, 3, 4, 5, 10, 11, 12, 13, 14, 15)
bm0.Optimize() // convert to runs
- bm1 := roaring.NewBitmap(5, 6, 7, 8, 9, 10, 11)
+ bm1 := roaring.NewFileBitmap(5, 6, 7, 8, 9, 10, 11)
bm1.Optimize() // convert to runs
result := bm0.Intersect(bm1)
if n := result.Count(); n != 3 {
@@ -451,7 +433,7 @@ func TestBitmap_IntersectRunRun(t *testing.T) {
}
// Intersect two runs that result in a bitmap.
- bm2 := roaring.NewBitmap()
+ bm2 := roaring.NewFileBitmap()
runLen := uint64(25)
spaceLen := uint64(8)
offset := (runLen / 2) + spaceLen
@@ -461,7 +443,7 @@ func TestBitmap_IntersectRunRun(t *testing.T) {
}
}
bm2.Optimize() // convert to runs
- bm3 := roaring.NewBitmap()
+ bm3 := roaring.NewFileBitmap()
runLen = uint64(32)
spaceLen = uint64(1)
for i := uint64(0); i < (65536 - runLen); i += (runLen + spaceLen) {
@@ -477,8 +459,8 @@ func TestBitmap_IntersectRunRun(t *testing.T) {
}
func TestBitmap_Difference(t *testing.T) {
- bm0 := roaring.NewBitmap(0, 2683177)
- bm1 := roaring.NewBitmap()
+ bm0 := roaring.NewFileBitmap(0, 2683177)
+ bm1 := roaring.NewFileBitmap()
for i := uint64(628); i < 2683301; i++ {
bm1.Add(i)
}
@@ -489,8 +471,8 @@ func TestBitmap_Difference(t *testing.T) {
}
func TestBitmap_Difference2(t *testing.T) {
- bm0 := roaring.NewBitmap(0, 1, 2, 131072, 262144, pilosa.SliceWidth+5, pilosa.SliceWidth+7)
- bm1 := roaring.NewBitmap(2, 3, 100000, 262144, 2*pilosa.SliceWidth+1)
+ bm0 := roaring.NewFileBitmap(0, 1, 2, 131072, 262144, pilosa.SliceWidth+5, pilosa.SliceWidth+7)
+ bm1 := roaring.NewFileBitmap(2, 3, 100000, 262144, 2*pilosa.SliceWidth+1)
result := bm0.Difference(bm1)
if !reflect.DeepEqual(result.Slice(), []uint64{0, 1, 131072, pilosa.SliceWidth + 5, pilosa.SliceWidth + 7}) {
t.Fatalf("unexpected : %v", result.Slice())
@@ -498,8 +480,8 @@ func TestBitmap_Difference2(t *testing.T) {
}
func TestBitmap_Difference_Empty(t *testing.T) {
- bm0 := roaring.NewBitmap(0, 2683177)
- bm1 := roaring.NewBitmap()
+ bm0 := roaring.NewFileBitmap(0, 2683177)
+ bm1 := roaring.NewFileBitmap()
result := bm0.Difference(bm1)
if n := result.Count(); n != 2 {
t.Fatalf("unexpected n: %d", n)
@@ -507,8 +489,8 @@ func TestBitmap_Difference_Empty(t *testing.T) {
}
func TestBitmap_DifferenceArrayArray(t *testing.T) {
- bm0 := roaring.NewBitmap(0, 4, 8, 12, 16, 20)
- bm1 := roaring.NewBitmap(1, 3, 6, 9, 12, 15, 18)
+ bm0 := roaring.NewFileBitmap(0, 4, 8, 12, 16, 20)
+ bm1 := roaring.NewFileBitmap(1, 3, 6, 9, 12, 15, 18)
result := bm0.Difference(bm1)
if n := result.Count(); n != 5 {
t.Fatalf("unexpected n: %d", n)
@@ -516,9 +498,9 @@ func TestBitmap_DifferenceArrayArray(t *testing.T) {
}
func TestBitmap_DifferenceArrayRun(t *testing.T) {
- bm0 := roaring.NewBitmap(0, 4, 8, 12, 16, 20, 36, 40, 44)
+ bm0 := roaring.NewFileBitmap(0, 4, 8, 12, 16, 20, 36, 40, 44)
- bm1 := roaring.NewBitmap(1, 2, 3, 4, 5, 6, 7, 8, 9, 30, 31, 32, 33, 34, 35, 36)
+ bm1 := roaring.NewFileBitmap(1, 2, 3, 4, 5, 6, 7, 8, 9, 30, 31, 32, 33, 34, 35, 36)
bm1.Optimize() // convert to runs
result := bm0.Difference(bm1)
if n := result.Count(); n != 6 {
@@ -527,8 +509,8 @@ func TestBitmap_DifferenceArrayRun(t *testing.T) {
}
func TestBitmap_Union(t *testing.T) {
- bm0 := roaring.NewBitmap(0, 1000001, 1000002, 1000003)
- bm1 := roaring.NewBitmap(0, 50000, 1000001, 1000002)
+ bm0 := roaring.NewFileBitmap(0, 1000001, 1000002, 1000003)
+ bm1 := roaring.NewFileBitmap(0, 50000, 1000001, 1000002)
result := bm0.Union(bm1)
if n := result.Count(); n != 5 {
t.Fatalf("unexpected n: %d", n)
@@ -537,7 +519,7 @@ func TestBitmap_Union(t *testing.T) {
func TestBitmap_Xor(t *testing.T) {
bm0 := testBM()
- bm1 := roaring.NewBitmap(0, 1, 2, 3)
+ bm1 := roaring.NewFileBitmap(0, 1, 2, 3)
result := bm1.Xor(bm0)
if n := result.Count(); n != 75011 {
t.Fatalf("unexpected n: %d", n)
@@ -555,8 +537,8 @@ func TestBitmap_Xor(t *testing.T) {
}
func TestBitmap_Xor_ArrayArray(t *testing.T) {
- bm0 := roaring.NewBitmap(0, 1000001, 1000002, 1000003)
- bm1 := roaring.NewBitmap(0, 50000, 1000001, 1000002)
+ bm0 := roaring.NewFileBitmap(0, 1000001, 1000002, 1000003)
+ bm1 := roaring.NewFileBitmap(0, 50000, 1000001, 1000002)
result := bm0.Xor(bm1)
if n := result.Count(); n != 2 {
t.Fatalf("unexpected n: %d", n)
@@ -572,8 +554,8 @@ func TestBitmap_Xor_ArrayArray(t *testing.T) {
//empty array test
func TestBitmap_Xor_Empty(t *testing.T) {
- bm1 := roaring.NewBitmap(0, 50000, 1000001, 1000002)
- empty := roaring.NewBitmap()
+ bm1 := roaring.NewFileBitmap(0, 50000, 1000001, 1000002)
+ empty := roaring.NewFileBitmap()
result := bm1.Xor(empty)
if n := result.Count(); n != 4 {
@@ -581,8 +563,8 @@ func TestBitmap_Xor_Empty(t *testing.T) {
}
}
func TestBitmap_Xor_ArrayBitmap(t *testing.T) {
- bm0 := roaring.NewBitmap(1, 70, 200, 4097, 4098)
- bm1 := roaring.NewBitmap()
+ bm0 := roaring.NewFileBitmap(1, 70, 200, 4097, 4098)
+ bm1 := roaring.NewFileBitmap()
for i := uint64(0); i < 10000; i += 2 {
bm1.Add(i)
}
@@ -603,7 +585,7 @@ func TestBitmap_Xor_ArrayBitmap(t *testing.T) {
t.Fatalf("test 3 unexpected n: %d", n)
}
- empty := roaring.NewBitmap()
+ empty := roaring.NewFileBitmap()
result = bm1.Xor(empty)
if n := result.Count(); n != 5000 {
t.Fatalf("unexpected n: %d", n)
@@ -611,8 +593,8 @@ func TestBitmap_Xor_ArrayBitmap(t *testing.T) {
}
func TestBitmap_Xor_BitmapBitmap(t *testing.T) {
- bm0 := roaring.NewBitmap()
- bm1 := roaring.NewBitmap()
+ bm0 := roaring.NewFileBitmap()
+ bm1 := roaring.NewFileBitmap()
for i := uint64(0); i < 10000; i += 2 {
bm1.Add(i)
@@ -630,7 +612,7 @@ func TestBitmap_Xor_BitmapBitmap(t *testing.T) {
// Ensure bitmap contents alternate.
func TestBitmap_Flip_Empty(t *testing.T) {
- bm := roaring.NewBitmap()
+ bm := roaring.NewFileBitmap()
results := bm.Flip(0, 10)
if n := results.Count(); n != 11 {
t.Fatalf("unexpected n: %d", n)
@@ -643,7 +625,7 @@ func TestBitmap_Flip_Empty(t *testing.T) {
// Test Subrange Flip should not affect bits outside of Range
func TestBitmap_Flip_Array(t *testing.T) {
- bm := roaring.NewBitmap(0, 1, 2, 3, 4, 8, 16, 32, 64, 128, 256, 512, 1024)
+ bm := roaring.NewFileBitmap(0, 1, 2, 3, 4, 8, 16, 32, 64, 128, 256, 512, 1024)
results := bm.Flip(0, 4)
if !reflect.DeepEqual(results.Slice(), []uint64{8, 16, 32, 64, 128, 256, 512, 1024}) {
t.Fatalf("unexpected %v ", results.Slice())
@@ -657,7 +639,7 @@ func TestBitmap_Flip_Array(t *testing.T) {
// Ensure Flip works with underlying Bitmap container.
func TestBitmap_Flip_Bitmap(t *testing.T) {
- bm := roaring.NewBitmap()
+ bm := roaring.NewFileBitmap()
size := uint64(10000)
for i := uint64(0); i < size; i += 2 {
bm.Add(i)
@@ -674,7 +656,7 @@ func TestBitmap_Flip_Bitmap(t *testing.T) {
// Verify Flip works correctly with in different regions of bitmap, beginning, middle, and end.
func TestBitmap_Flip_After(t *testing.T) {
- bm := roaring.NewBitmap(0, 2, 4, 8)
+ bm := roaring.NewFileBitmap(0, 2, 4, 8)
results := bm.Flip(9, 10)
if !reflect.DeepEqual(results.Slice(), []uint64{0, 2, 4, 8, 9, 10}) {
@@ -693,8 +675,8 @@ func TestBitmap_Flip_After(t *testing.T) {
// Ensure bitmap can return the number of intersecting bits in two bitmaps.
func TestBitmap_IntersectionCount_ArrayArray(t *testing.T) {
- bm0 := roaring.NewBitmap(0, 1, 1000001, 1000002, 1000003)
- bm1 := roaring.NewBitmap(0, 50000, 1000001, 1000002)
+ bm0 := roaring.NewFileBitmap(0, 1, 1000001, 1000002, 1000003)
+ bm1 := roaring.NewFileBitmap(0, 50000, 1000001, 1000002)
if n := bm0.IntersectionCount(bm1); n != 3 {
t.Fatalf("unexpected n: %d", n)
@@ -705,8 +687,8 @@ func TestBitmap_IntersectionCount_ArrayArray(t *testing.T) {
// Ensure bitmap can return the number of intersecting bits in two bitmaps.
func TestBitmap_IntersectionCount_ArrayRun(t *testing.T) {
- bm0 := roaring.NewBitmap(0, 1000001, 1000002, 1000003)
- bm1 := roaring.NewBitmap(0, 1, 2, 3, 4, 5, 1000000, 1000002, 1000003, 1000004, 1000005, 1000006)
+ bm0 := roaring.NewFileBitmap(0, 1000001, 1000002, 1000003)
+ bm1 := roaring.NewFileBitmap(0, 1, 2, 3, 4, 5, 1000000, 1000002, 1000003, 1000004, 1000005, 1000006)
bm1.Optimize() // convert to runs
if n := bm0.IntersectionCount(bm1); n != 3 {
@@ -718,9 +700,9 @@ func TestBitmap_IntersectionCount_ArrayRun(t *testing.T) {
// Ensure bitmap can return the number of intersecting bits in two bitmaps.
func TestBitmap_IntersectionCount_RunRun(t *testing.T) {
- bm0 := roaring.NewBitmap(3, 4, 5, 6, 7, 8, 1000001, 1000002, 1000003, 1000004)
+ bm0 := roaring.NewFileBitmap(3, 4, 5, 6, 7, 8, 1000001, 1000002, 1000003, 1000004)
bm0.Optimize() // convert to runs
- bm1 := roaring.NewBitmap(0, 1, 2, 3, 4, 5, 1000000, 1000002, 1000003, 1000004, 1000005, 1000006)
+ bm1 := roaring.NewFileBitmap(0, 1, 2, 3, 4, 5, 1000000, 1000002, 1000003, 1000004, 1000005, 1000006)
bm1.Optimize() // convert to runs
if n := bm0.IntersectionCount(bm1); n != 6 {
@@ -732,11 +714,11 @@ func TestBitmap_IntersectionCount_RunRun(t *testing.T) {
// Ensure bitmap can return the number of intersecting bits in two bitmaps.
func TestBitmap_IntersectionCount_BitmapRun(t *testing.T) {
- bm0 := roaring.NewBitmap()
+ bm0 := roaring.NewFileBitmap()
for i := uint64(3); i <= 1000006; i += 2 {
bm0.Add(i)
}
- bm1 := roaring.NewBitmap(0, 1, 2, 3, 4, 5, 1000000, 1000002, 1000003, 1000004, 1000005, 1000006)
+ bm1 := roaring.NewFileBitmap(0, 1, 2, 3, 4, 5, 1000000, 1000002, 1000003, 1000004, 1000005, 1000006)
bm1.Optimize() // convert to runs
if n := bm0.IntersectionCount(bm1); n != 4 {
@@ -748,8 +730,8 @@ func TestBitmap_IntersectionCount_BitmapRun(t *testing.T) {
// Ensure bitmap can return the number of intersecting bits in two bitmaps.
func TestBitmap_IntersectionCount_ArrayBitmap(t *testing.T) {
- bm0 := roaring.NewBitmap(1, 70, 200, 4097, 4098)
- bm1 := roaring.NewBitmap()
+ bm0 := roaring.NewFileBitmap(1, 70, 200, 4097, 4098)
+ bm1 := roaring.NewFileBitmap()
for i := uint64(0); i <= 10000; i += 2 {
bm1.Add(i)
}
@@ -763,8 +745,8 @@ func TestBitmap_IntersectionCount_ArrayBitmap(t *testing.T) {
// Ensure bitmap can return the number of intersecting bits in two bitmaps.
func TestBitmap_IntersectionCount_BitmapBitmap(t *testing.T) {
- bm0 := roaring.NewBitmap()
- bm1 := roaring.NewBitmap()
+ bm0 := roaring.NewFileBitmap()
+ bm1 := roaring.NewFileBitmap()
for i := uint64(0); i <= 10000; i += 2 {
bm0.Add(i)
bm1.Add(i + 1)
@@ -784,8 +766,8 @@ func TestBitmap_IntersectionCount_BitmapBitmap(t *testing.T) {
}
func TestBitmap_IntersectionCount_Mixed(t *testing.T) {
bm0 := testBM()
- bm1 := roaring.NewBitmap(0, 1, 2, 3, 4, 5, 6, 7, 9, 10, 65536)
- bm3 := roaring.NewBitmap(131072)
+ bm1 := roaring.NewFileBitmap(0, 1, 2, 3, 4, 5, 6, 7, 9, 10, 65536)
+ bm3 := roaring.NewFileBitmap(131072)
if n := bm0.IntersectionCount(bm0); n != bm0.Count() {
t.Fatalf("unexpected n: %d", n)
@@ -807,7 +789,7 @@ func TestBitmap_Quick_LargeValue(t *testing.T) { testBitmapQuick(t, 10000, 0, ma
// Ensure a bitmap can perform basic operations on randomly generated values.
func testBitmapQuick(t *testing.T, n int, min, max uint64) {
quick.Check(func(a []uint64) bool {
- bm := roaring.NewBitmap()
+ bm := roaring.NewFileBitmap()
m := make(map[uint64]struct{})
// Add values to the bitmap and set.
@@ -894,7 +876,7 @@ func testBitmapMarshalQuick(t *testing.T, n int, min, max uint64, sorted bool) {
quick.Check(func(a0, a1 []uint64) bool {
// Create bitmap with initial values set.
- bm := roaring.NewBitmap(a0...)
+ bm := roaring.NewFileBitmap(a0...)
set := make(map[uint64]struct{})
for _, v := range a0 {
@@ -923,7 +905,7 @@ func testBitmapMarshalQuick(t *testing.T, n int, min, max uint64, sorted bool) {
data := buf.Bytes()
// Create new bitmap from ops log data.
- bm2 := roaring.NewBitmap()
+ bm2 := roaring.NewFileBitmap()
if err := bm2.UnmarshalBinary(data); err != nil {
t.Fatal(err)
}
@@ -952,7 +934,7 @@ func testBitmapMarshalQuick(t *testing.T, n int, min, max uint64, sorted bool) {
// TODO duplicate for all container types
func TestIterator(t *testing.T) {
t.Run("bitmap", func(t *testing.T) {
- itr := roaring.NewBitmap(1, 2, 3).Iterator()
+ itr := roaring.NewFileBitmap(1, 2, 3).Iterator()
itr.Seek(0)
var a []uint64
@@ -966,13 +948,13 @@ func TestIterator(t *testing.T) {
})
t.Run("run", func(t *testing.T) {
- bm1 := roaring.NewBitmap()
+ bm1 := roaring.NewFileBitmap()
for i := uint64(0); i < 11; i += 1 {
bm1.Add(i)
}
bm1.Optimize()
- bm2 := roaring.NewBitmap()
+ bm2 := roaring.NewFileBitmap()
for i := uint64(0); i < 12; i += 1 {
bm2.Add(i)
}
@@ -1005,7 +987,7 @@ func TestIterator(t *testing.T) {
// testBM creates a bitmap with 3 containers: array, bitmap, and run.
func testBM() *roaring.Bitmap {
- bm := roaring.NewBitmap()
+ bm := roaring.NewFileBitmap()
//the array
for i := uint64(0); i < 1024; i += 4 {
bm.Add((1 << 16) + i)
@@ -1068,19 +1050,19 @@ func getBenchData() *struct{ a, b, r *roaring.Bitmap } {
const max = (1 << 24) / 64
// Build bitmap with array container.
- data.a = roaring.NewBitmap()
+ data.a = roaring.NewFileBitmap()
for i, n := 0, 2*roaring.ArrayMaxSize/3; i < n; i++ {
data.a.Add(uint64(rand.Intn(max)))
}
// Build bitmap with bitmap container.
- data.b = roaring.NewBitmap()
+ data.b = roaring.NewFileBitmap()
for i, n := 0, MaxContainerVal/3; i < n; i++ {
data.b.Add(uint64(i * 3))
}
// build bitmap with run container
- data.r = roaring.NewBitmap()
+ data.r = roaring.NewFileBitmap()
for i, n := 0, MaxContainerVal; i < n; i++ {
data.r.Add(uint64(i))
}
@@ -1176,7 +1158,7 @@ const (
func BenchmarkContainerLinear(b *testing.B) {
for n := 0; n < b.N; n++ {
- bm := roaring.NewBitmap()
+ bm := roaring.NewFileBitmap()
for row := uint64(1); row < NumRows; row++ {
for col := uint64(1); col < NumColums; col++ {
bm.Add(row*pilosa.SliceWidth + (col * MaxContainerVal))
@@ -1187,7 +1169,7 @@ func BenchmarkContainerLinear(b *testing.B) {
func BenchmarkContainerReverse(b *testing.B) {
for n := 0; n < b.N; n++ {
- bm := roaring.NewBitmap()
+ bm := roaring.NewFileBitmap()
for row := NumRows - 1; row >= 1; row-- {
for col := NumColums - 1; col >= 1; col-- {
bm.Add(row*pilosa.SliceWidth + (col * MaxContainerVal))
@@ -1198,7 +1180,7 @@ func BenchmarkContainerReverse(b *testing.B) {
func BenchmarkContainerColumn(b *testing.B) {
for n := 0; n < b.N; n++ {
- bm := roaring.NewBitmap()
+ bm := roaring.NewFileBitmap()
for col := uint64(1); col < NumColums; col++ {
for row := uint64(1); row < NumRows; row++ {
bm.Add(row*pilosa.SliceWidth + (col * MaxContainerVal))
@@ -1210,7 +1192,7 @@ func BenchmarkContainerColumn(b *testing.B) {
func BenchmarkContainerOutsideIn(b *testing.B) {
middle := NumRows / uint64(2)
for n := 0; n < b.N; n++ {
- bm := roaring.NewBitmap()
+ bm := roaring.NewFileBitmap()
for col := uint64(1); col < NumColums; col++ {
for row := uint64(1); row < middle; row++ {
@@ -1224,7 +1206,7 @@ func BenchmarkContainerOutsideIn(b *testing.B) {
func BenchmarkContainerInsideOut(b *testing.B) {
middle := NumRows / uint64(2)
for n := 0; n < b.N; n++ {
- bm := roaring.NewBitmap()
+ bm := roaring.NewFileBitmap()
for col := uint64(1); col < NumColums; col++ {
for row := uint64(1); row <= middle; row++ {
bm.Add((middle+row)*pilosa.SliceWidth + (col * MaxContainerVal))
@@ -1236,7 +1218,7 @@ func BenchmarkContainerInsideOut(b *testing.B) {
func BenchmarkSliceAscending(b *testing.B) {
for n := 0; n < b.N; n++ {
- bm := roaring.NewBitmap()
+ bm := roaring.NewFileBitmap()
for col := uint64(0); col < pilosa.SliceWidth; col++ {
bm.Add(col)
}
@@ -1245,7 +1227,7 @@ func BenchmarkSliceAscending(b *testing.B) {
func BenchmarkSliceDescending(b *testing.B) {
for n := 0; n < b.N; n++ {
- bm := roaring.NewBitmap()
+ bm := roaring.NewFileBitmap()
for col := uint64(pilosa.SliceWidth); col > uint64(0); col-- {
bm.Add(col)
}
diff --git a/server/enterprise.go b/server/enterprise.go
new file mode 100644
index 000000000..6dd3e5540
--- /dev/null
+++ b/server/enterprise.go
@@ -0,0 +1,21 @@
+// Copyright (C) 2017-2018 Pilosa Corp. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// +build enterprise
+
+package server
+
+import (
+ _ "github.com/pilosa/pilosa/enterprise"
+)
diff --git a/server/server.go b/server/server.go
index eb660fc23..27c6c06e5 100644
--- a/server/server.go
+++ b/server/server.go
@@ -159,6 +159,12 @@ func (m *Command) SetupServer() error {
return errors.Wrap(err, "setting up logger")
}
+ productName := "Pilosa"
+ if pilosa.EnterpriseEnabled {
+ productName += " Enterprise"
+ }
+ m.logger.Printf("%s %s, build time %s\n", productName, pilosa.Version, pilosa.BuildTime)
+
handler := pilosa.NewHandler()
handler.Logger = m.logger
handler.FileSystem = &statik.FileSystem{}
diff --git a/utils_test.go b/utils_test.go
index f4212e0ec..1732522d4 100644
--- a/utils_test.go
+++ b/utils_test.go
@@ -1,3 +1,17 @@
+// Copyright (C) 2017-2018 Pilosa Corp. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
package pilosa
import (
diff --git a/version.go b/version.go
index 3e148ea7b..04096f7cc 100644
--- a/version.go
+++ b/version.go
@@ -14,5 +14,13 @@
package pilosa
+var Enterprise = "0"
+var EnterpriseEnabled = false
var Version = "v0.0.0"
var BuildTime = "not recorded"
+
+func init() {
+ if Enterprise == "1" {
+ EnterpriseEnabled = true
+ }
+}