scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-10 23:11:01 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'master' into decreaseTimeForCICD

Browse source
This commit is contained in:
souhailanoor 2021-11-19 13:19:10 -06:00 • committed by GitHub
commit 99e50c25aa
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
26 changed files with 686 additions and 1512 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
.circleci/config.yml
View file

@ -63,13 +63,6 @@ jobs:
- checkout-plus
- run: go mod download
- save-mod-cache
check-license-headers:
executor:
name: golang
steps:
- checkout-plus
- skip-if-root-unchanged
- run: make check-license-headers
linter:
executor:
name: golang
@ -239,10 +232,6 @@ workflows:
context: molecula
requires:
- setup
- check-license-headers:
context: molecula
requires:
- setup
- go-mod-tidy:
context: molecula
requires:

1
.cloudbuild/Dockerfile
View file

@ -5,7 +5,6 @@ LABEL org.opencontainers.image.authors="dev@molecula.com"
RUN apk add --no-cache curl jq
COPY LICENSE /LICENSE
COPY NOTICE /NOTICE
ARG SHORT_SHA

202
.cloudbuild/LICENSE
View file

@ -1,202 +0,0 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

13
.cloudbuild/NOTICE
View file

@ -1,18 +1,7 @@
Software license
================
Copyright (C) 2017-2018 Pilosa Corp. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License").
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
Copyright (C) 2017-2021 Molecula Corp. All rights reserved.
Third-party software licenses
=============================

4
.github/PULL_REQUEST_TEMPLATE.md vendored
View file

@ -6,13 +6,10 @@ Fixes #
## Pull request checklist
- [ ] I have read the [contributing guide](https://github.com/molecula/featurebase/blob/master/CONTRIBUTING.md).
- [ ] I have agreed to the [Contributor License Agreement](https://cla-assistant.io/pilosa/pilosa).
- [ ] I have updated the [documentation](https://github.com/molecula/docs).
- [ ] I have resolved any merge conflicts.
- [ ] I have included tests that cover my changes.
- [ ] All new and existing tests pass.
- [ ] Make sure PR title conforms to convention in CHANGELOG.md.
- [ ] Add appropriate changelog label to PR (if applicable).
## Code review checklist
@ -24,5 +21,4 @@ This is the checklist that the reviewer will follow while reviewing your pull re
- [ ] Check that tests have been written and that they cover the new functionality.
- [ ] Run tests and ensure they pass.
- [ ] Build and run the code, performing any applicable integration testing.
- [ ] Make sure PR title conforms to convention in CHANGELOG.md.
- [ ] Make sure PR is tagged with appropriate changelog label.

1
.gitlab/Dockerfile
View file

@ -7,7 +7,6 @@ WORKDIR /featurebase
RUN apk add --no-cache curl jq
COPY LICENSE .
COPY NOTICE .
COPY featurebase_linux_amd64 .

6
.gitlab/cloud-init.sh
View file

@ -14,8 +14,8 @@ echo 'cat /proc/sys/fs/file-max'
# yum install golang -y # latest verion in ec2 is 1.15.14
# install go 1.16.9 manually
curl -O https://dl.google.com/go/go1.16.9.linux-amd64.tar.gz
tar xvf go1.16.9.linux-amd64.tar.gz
curl -O https://dl.google.com/go/go1.16.10.linux-amd64.tar.gz
tar xvf go1.16.10.linux-amd64.tar.gz
chown -R root:root ./go
mv go /usr/local
echo "export PATH=/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin:/home/ec2-user/bin:/usr/local/go/bin" | tee -a /etc/profile > /dev/null
@ -23,4 +23,4 @@ source /etc/profile
# install aws session manager pluggin
curl "https://s3.amazonaws.com/session-manager-downloads/plugin/latest/linux_64bit/session-manager-plugin.rpm" -o "session-manager-plugin.rpm"
yum install -y session-manager-plugin.rpm
yum install -y session-manager-plugin.rpm

715
CHANGELOG.md
View file

@ -1,715 +0,0 @@
# Changelog
All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](http://keepachangelog.com/)
and this project adheres to [Semantic Versioning](http://semver.org/).
## [1.4.0] - 2019-09-17
This version contains 99 contributions from 11 contributors. There are 94 files changed; 9,453 insertions; and 6,121 deletions.
**Attention**: Pilosa 1.4.0 changes the way that integer fields are stored. The upgrade from old format to new is handled automatically, however you will not be able to downgrade to 1.3 should you wish to do so. We *always* recommend taking a backup of your Pilosa data directory before upgrading Pilosa, but doubly so with this release.
### Added
- Update "Getting Started" documentation ([#2028](https://github.com/pilosa/pilosa/pull/2028))
- Add ability to disable tracing and use nopTracer ([#2029](https://github.com/pilosa/pilosa/pull/2029))
- Add test for no containers ([#2016](https://github.com/pilosa/pilosa/pull/2016))
- Add naive implementations of Roaring and fuzz test ([#2023](https://github.com/pilosa/pilosa/pull/2023))
- Add fuzzing code and readme.md to explain the fuzzer ([#2004](https://github.com/pilosa/pilosa/pull/2004))
- Add MinRow and MaxRow calls ([#1983](https://github.com/pilosa/pilosa/pull/1983))
- Add Prometheus stats backend ([#1992](https://github.com/pilosa/pilosa/pull/1992))
- Add extra tracing spans and metadata ([#1939](https://github.com/pilosa/pilosa/pull/1939))
- Add more Debugf() statements to the holder open process ([#1950](https://github.com/pilosa/pilosa/pull/1950))
- Add ability to post schema using holder.applySchema ([#1956](https://github.com/pilosa/pilosa/pull/1956))
### Changed
- Update CircleCI build with Go 1.13 and run enterprise tests ([#2064](https://github.com/pilosa/pilosa/pull/2064))
- Update Alpine to 3.9.4 in Dockerfile ([#2001](https://github.com/pilosa/pilosa/pull/2001))
- Add Prometheus tests, refactor http stats as middleware, minor fixes ([#1994](https://github.com/pilosa/pilosa/pull/1994))
- Add confirmation logic to catch false nodeLeave events ([#1993](https://github.com/pilosa/pilosa/pull/1993))
- Improve TopN() errors ([#1978](https://github.com/pilosa/pilosa/pull/1978))
- Make integer fields unbounded by using sign+magnitude representation ([#1902](https://github.com/pilosa/pilosa/pull/1902))
- Simplify contributing instructions by removing weird upstream thing ([#1966](https://github.com/pilosa/pilosa/pull/1966))
### Fixed
- Default BSI base value to min, max, or 0 depending on the min/max range ([#2050](https://github.com/pilosa/pilosa/pull/2050))
- Add worker pool for query processing ([#2034](https://github.com/pilosa/pilosa/pull/2034))
- Move Range deprecation message to higher level ([#2033](https://github.com/pilosa/pilosa/pull/2033))
- Use lock in view.deleteFragment while altering fragments ([#2026](https://github.com/pilosa/pilosa/pull/2026))
- Fix malformed offset bug in readOffsets and readWithRuns ([#2021](https://github.com/pilosa/pilosa/pull/2021))
- Fix various container iteration bugs in Roaring ([#2019](https://github.com/pilosa/pilosa/pull/2019))
- Fix malformed bitmap handling ([#2017](https://github.com/pilosa/pilosa/pull/2017))
- Fix fuzzer errors in roaring ([#2012](https://github.com/pilosa/pilosa/pull/2012))
- Save all state files atomically to avoid corruption ([#2000](https://github.com/pilosa/pilosa/pull/2000))
- Fix slice container updates ([#1997](https://github.com/pilosa/pilosa/pull/1997))
- Fix out of bounds panic to show error ([#1975](https://github.com/pilosa/pilosa/pull/1975))
- Fix error message returned by regex on field and index names ([#1973](https://github.com/pilosa/pilosa/pull/1973))
- Fix filter calls in GroupBy not being translated ([#1970](https://github.com/pilosa/pilosa/pull/1970))
- Fix TranslateFile behavior when reopened ([#1954](https://github.com/pilosa/pilosa/pull/1954))
- Remove buggy shard validation code ([#1951](https://github.com/pilosa/pilosa/pull/1951))
- Fix some lint warnings raised in VS-Code ([#1947](https://github.com/pilosa/pilosa/pull/1947))
### Performance
- Address some startup speed and performance issues ([#1988](https://github.com/pilosa/pilosa/pull/1988))
- Add a worker pool for importRoaring jobs ([#2048](https://github.com/pilosa/pilosa/pull/2048))
- Use UnionInPlace for computing time rows which involve multiple views ([#2041](https://github.com/pilosa/pilosa/pull/2041))
- Improve ingest performance with snapshot queue and unmarshaling improvements ([#2024](https://github.com/pilosa/pilosa/pull/2024))
- Improve row cache ([#1974](https://github.com/pilosa/pilosa/pull/1974))
### Removed
- Remove extraneous stat tags to improve prometheus performance ([#1996](https://github.com/pilosa/pilosa/pull/1996))
## [1.3.1] - 2019-05-01
This version contains 1 contribution from 1 contributor. There are 6 files changed; 10 insertions; and 95 deletions.
### Fixed
- Remove shard validation to fix bug where some nodes weren't loading their fragments. #1951 ([#1964](https://github.com/pilosa/pilosa/pull/1964))
## [1.3.0] - 2019-04-16
This version contains 98 contributions from 10 contributors. There are 144 files changed; 12,635 insertions; and 4,341 deletions.
### Added
- Add license headers and CI check ([#1940](https://github.com/pilosa/pilosa/pull/1940))
- Add support to modify shard width at build time ([#1921](https://github.com/pilosa/pilosa/pull/1921))
- Add 'bench' Makefile target and run fewer concurrency level benchmarks ([#1915](https://github.com/pilosa/pilosa/pull/1915))
- Add server stats to /info endpoint ([#1859](https://github.com/pilosa/pilosa/pull/1859))
- Implement config options for block profile rate and mutex fraction ([#1910](https://github.com/pilosa/pilosa/pull/1910))
- Implement global open file counter using syswrap (to scale past system open file limits) ([#1906](https://github.com/pilosa/pilosa/pull/1906))
- Implement global mmap counter with fallback (to scale past system mmap limits) ([#1903](https://github.com/pilosa/pilosa/pull/1903))
- Add shard width to index info in schema (allows client to get shard width at run time) ([#1881](https://github.com/pilosa/pilosa/pull/1881))
- Add shift operator ([#1761](https://github.com/pilosa/pilosa/pull/1761))
- Support advertise address and listen on 0.0.0.0 ([#1832](https://github.com/pilosa/pilosa/pull/1832))
- Added convenience function to efficiently calculate size of a roaring bitmap in bytes ([#1839](https://github.com/pilosa/pilosa/pull/1839))
- Make sure more tests and benchmarks can have their temp dir set by flag ([#1831](https://github.com/pilosa/pilosa/pull/1831))
- Add sliceascending/slicedescending striped benchmarks ([#1763](https://github.com/pilosa/pilosa/pull/1763))
- Add setValue test and benchmarks ([#1820](https://github.com/pilosa/pilosa/pull/1820))
- Add a test for groupby filter with RangeLTLT ([#1818](https://github.com/pilosa/pilosa/pull/1818))
- Add tests for GroupBy with keys; removes unused Bit message from proto ([#1811](https://github.com/pilosa/pilosa/pull/1811))
### Fixed
- Update to latest memberlist fork with race fixes ([#1944](https://github.com/pilosa/pilosa/pull/1944))
- Return original error instead of cause in handler ([#1943](https://github.com/pilosa/pilosa/pull/1943))
- Validate (and panic) on duplicate PQL arguments ([#1938](https://github.com/pilosa/pilosa/pull/1938))
- Add correct content type to query responses Fixes #1873 ([#1936](https://github.com/pilosa/pilosa/pull/1936))
- Address race condition by getting cluster nodes with lock ([#1931](https://github.com/pilosa/pilosa/pull/1931))
- Make sure to unmap containers before modifying ([#1876](https://github.com/pilosa/pilosa/pull/1876))
- Avoid probable race when creating fragments ([#1863](https://github.com/pilosa/pilosa/pull/1863))
- Improve help strings for metrics options ([#1887](https://github.com/pilosa/pilosa/pull/1887))
- Ensure ClearRow() arguments get translated ([#1848](https://github.com/pilosa/pilosa/pull/1848))
- Prevent omitting zero ids on columnattrs ([#1846](https://github.com/pilosa/pilosa/pull/1846))
- Set cache size to 0 if cache type is none ([#1842](https://github.com/pilosa/pilosa/pull/1842))
- Prevent deadlock in replication logic on reopening a store ([#1834](https://github.com/pilosa/pilosa/pull/1834))
- Pass loggers around properly in gossip ([#1835](https://github.com/pilosa/pilosa/pull/1835))
- Include read lock in cluster.Nodes() ([#1836](https://github.com/pilosa/pilosa/pull/1836))
- Raise an error on Rows() query against a time field with noStandardView: true ([#1826](https://github.com/pilosa/pilosa/pull/1826))
- Don't delete test fragment data (part of repo) ([#1827](https://github.com/pilosa/pilosa/pull/1827))
- Fix bug on upper end of bsi range queries ([#1822](https://github.com/pilosa/pilosa/pull/1822))
- Group by fixes ([#1802](https://github.com/pilosa/pilosa/pull/1802))
### Changed
- Switch to GolangCI lint ([#1924](https://github.com/pilosa/pilosa/pull/1924))
- Return empty result set when query empty ([#1937](https://github.com/pilosa/pilosa/pull/1937))
- Add Go 1.12 to CircleCI ([#1909](https://github.com/pilosa/pilosa/pull/1909))
- Ignore fragment files from shards node doesn't own ([#1900](https://github.com/pilosa/pilosa/pull/1900))
- Go module support. Use Modules instead of dep for dependencies ([#1616](https://github.com/pilosa/pilosa/pull/1616))
- Merge Range() into Row() call. ([#1804](https://github.com/pilosa/pilosa/pull/1804))
- Add from/to range arguments to Rows() call ([#1851](https://github.com/pilosa/pilosa/pull/1851))
- Fixes Store call error messages, Rows doesn't need field argument ([#1830](https://github.com/pilosa/pilosa/pull/1830))
### Performance
- BTree performance improvements ([#1916](https://github.com/pilosa/pilosa/pull/1916))
- Make Containers smaller, especially when they have small contents ([#1901](https://github.com/pilosa/pilosa/pull/1901))
- Address UnionInPlace performance regressions ([#1897](https://github.com/pilosa/pilosa/pull/1897))
- Small write path for import-roaring. Makes small imports faster ([#1892](https://github.com/pilosa/pilosa/pull/1892))
- Small write path for imports ([#1871](https://github.com/pilosa/pilosa/pull/1871))
- Remove copy for pilosa roaring files ([#1865](https://github.com/pilosa/pilosa/pull/1865))
- Disable anti-entropy if not using replication [performance] ([#1814](https://github.com/pilosa/pilosa/pull/1814))
- Group By—skip 0 counts as early as possible ([#1803](https://github.com/pilosa/pilosa/pull/1803))
## [1.2.0] - 2018-12-20
This version contains 155 contributions from 11 contributors. There are 113 files changed; 19,085 insertions; and 4,323 deletions.
### Added
- Cancel queries on Context.Done() ([#1773](https://github.com/pilosa/pilosa/pull/1773))
- Union In Place ([#1766](https://github.com/pilosa/pilosa/pull/1766), [#1774](https://github.com/pilosa/pilosa/pull/1774))
- Import benchmarking ([#1771](https://github.com/pilosa/pilosa/pull/1771))
- Add GroupBy() Filter ([#1753](https://github.com/pilosa/pilosa/pull/1753))
- Add /internal/translate/keys endpoint ([#1751](https://github.com/pilosa/pilosa/pull/1751))
- CircleCI: Add race detector to parallel build, default to Go 1.11. ([#1756](https://github.com/pilosa/pilosa/pull/1756))
- Add distributed tracing. ([#1684](https://github.com/pilosa/pilosa/pull/1684))
- Add NoStandardView field option ([#1733](https://github.com/pilosa/pilosa/pull/1733))
- Add some stat tracking to roaring implementation ([#1743](https://github.com/pilosa/pilosa/pull/1743))
- Add cluster fault testing using docker-compose and pumba ([#1717](https://github.com/pilosa/pilosa/pull/1717))
- Allow backslash, carriage return in PQL strings ([#1713](https://github.com/pilosa/pilosa/pull/1713))
- Add base system, curl and jq for debug and checks ([#1707](https://github.com/pilosa/pilosa/pull/1707))
- Add `Rows` and `GroupBy` functionality ([#1647](https://github.com/pilosa/pilosa/pull/1647))
- Add `clear` functional option for imports ([#1699](https://github.com/pilosa/pilosa/pull/1699))
- Implement tracking of available shards to help support sparse datasets ([#1600](https://github.com/pilosa/pilosa/pull/1600), [#1695](https://github.com/pilosa/pilosa/pull/1695), [#1624](https://github.com/pilosa/pilosa/pull/1624), [#1663](https://github.com/pilosa/pilosa/pull/1663))
- Add missing rowID/Key columnID/Key tests ([#1683](https://github.com/pilosa/pilosa/pull/1683))
- Add Store() operation to PQL ([#1666](https://github.com/pilosa/pilosa/pull/1666))
- Add diagnostics CPUArch field ([#1671](https://github.com/pilosa/pilosa/pull/1671))
- Add CircleCI step to generate Docker image and push to Docker hub ([#1673](https://github.com/pilosa/pilosa/pull/1673))
- Implement ClearRow() query ([#1645](https://github.com/pilosa/pilosa/pull/1645))
- Add support for Bool fields ([#1658](https://github.com/pilosa/pilosa/pull/1658))
- Make translate map size configurable ([#1653](https://github.com/pilosa/pilosa/pull/1653))
- Add DirectAdd function to roaring.Bitmap ([#1646](https://github.com/pilosa/pilosa/pull/1646))
- Implement Roaring import ([#1622](https://github.com/pilosa/pilosa/pull/1622), [#1738](https://github.com/pilosa/pilosa/pull/1738))
- Add Not() query ([#1635](https://github.com/pilosa/pilosa/pull/1635))
- Implement Options call and excludeRowAttrs, excludeColumns, columnAttrs and shards args ([#1631](https://github.com/pilosa/pilosa/pull/1631))
- Add field options to pilosa import ([#1625](https://github.com/pilosa/pilosa/pull/1625))
- Implement column existence tracking ([#1788](https://github.com/pilosa/pilosa/pull/1788), [#1672](https://github.com/pilosa/pilosa/pull/1672), [#1628](https://github.com/pilosa/pilosa/pull/1628))
### Changed
- Convert the anti-entropy logic to use `ImportRoaring` instead of `QueryNode` ([#1780](https://github.com/pilosa/pilosa/pull/1780))
- Simplify `require-*` logic in Makefile ([#1755](https://github.com/pilosa/pilosa/pull/1755))
- Cleanup logging ([#1748](https://github.com/pilosa/pilosa/pull/1748))
- Remove TravisCI, add CircleCI shield ([#1740](https://github.com/pilosa/pilosa/pull/1740))
- Upgrade Peg dependency and regenerate grammar ([#1725](https://github.com/pilosa/pilosa/pull/1725))
- Upgrade to protoc 3.6.1 (also updated protoc-gen-gofast) ([#1724](https://github.com/pilosa/pilosa/pull/1724))
- Move column attrs logic to executor ([#1677](https://github.com/pilosa/pilosa/pull/1677))
- Shrink container bit count to int32 ([#1664](https://github.com/pilosa/pilosa/pull/1664))
### Performance
- Remove bounds check ([#1619](https://github.com/pilosa/pilosa/pull/1619))
- Improve benchmarking and performance ([#1741](https://github.com/pilosa/pilosa/pull/1741))
### Fixed
- Ensure internal client closes all response bodies ([#1795](https://github.com/pilosa/pilosa/pull/1795))
- Allow translate log entry buffer to grow ([#1787](https://github.com/pilosa/pilosa/pull/1787))
- Add Gopkg.lock as a dependency for vendor target ([#1790](https://github.com/pilosa/pilosa/pull/1790))
- Cluster resize fix ([#1785](https://github.com/pilosa/pilosa/pull/1785))
- Attempt to fix deadlock by releasing view lock before broadcasting ([#1782](https://github.com/pilosa/pilosa/pull/1782))
- Fix bug where cluster goes into RESIZING instead of NORMAL ([#1777](https://github.com/pilosa/pilosa/pull/1777))
- Propogate updates to node details (not just additions and deletions) ([#1769](https://github.com/pilosa/pilosa/pull/1769))
- Fix arm64 support ([#1764](https://github.com/pilosa/pilosa/pull/1764))
- Fix data races ([#1750](https://github.com/pilosa/pilosa/pull/1750))
- Fix fragment checksums race condition ([#1749](https://github.com/pilosa/pilosa/pull/1749))
- Import cmd field type flag ([#1732](https://github.com/pilosa/pilosa/pull/1732))
- Increase the translate file size for tests/benchmarks ([#1744](https://github.com/pilosa/pilosa/pull/1744))
- Prevent panic in Bitmap.UnmarshalBinary when there is no data ([#1742](https://github.com/pilosa/pilosa/pull/1742))
- Remove unused rule from peg grammar ([#1737](https://github.com/pilosa/pilosa/pull/1737))
- Improve Internal Client errors ([#1729](https://github.com/pilosa/pilosa/pull/1729))
- Forward imports to non-coordinator shards ([#1719](https://github.com/pilosa/pilosa/pull/1719))
- Fix double escapes in PQL grammar ([#1727](https://github.com/pilosa/pilosa/pull/1727))
- Ensure btree comparison doesn't fail for smallish N ([#1712](https://github.com/pilosa/pilosa/pull/1712))
- Drop now-superfluous methodNotAllowedHandler ([#1711](https://github.com/pilosa/pilosa/pull/1711))
- Use pilosa.Logger everywhere ([#1674](https://github.com/pilosa/pilosa/pull/1674))
- Ensure view closes fragment on broadcast error ([#1675](https://github.com/pilosa/pilosa/pull/1675))
- Prevent closing os.Stderr (used in verbose test logging) ([#1696](https://github.com/pilosa/pilosa/pull/1696))
- Allow holder to close/open/close without panic on closing closed channel ([#1686](https://github.com/pilosa/pilosa/pull/1686))
- Fix bug with Range() queries with field keys ([#1679](https://github.com/pilosa/pilosa/pull/1679))
- Sync query validation for handlers ([#1676](https://github.com/pilosa/pilosa/pull/1676))
- Wrap translation store errors, decrease test map size to prevent failure on 32-bit ([#1665](https://github.com/pilosa/pilosa/pull/1665))
- Fix pass-by-value issue in proto decode ([#1662](https://github.com/pilosa/pilosa/pull/1662))
- Do not run prerelease in CI if this is a pull request ([#1655](https://github.com/pilosa/pilosa/pull/1655))
- Ensure mutex imports unset previous columns ([#1656](https://github.com/pilosa/pilosa/pull/1656))
- Treat import timestamps as UTC ([#1651](https://github.com/pilosa/pilosa/pull/1651))
- Remove unused log buffers from test cluster, fixes race ([#1612](https://github.com/pilosa/pilosa/pull/1612))
- Add --field-keys and --index-keys options to pilosa import ([#1621](https://github.com/pilosa/pilosa/pull/1621))
- Use passed stdin, stdout, and stderr in the cmd package ([#1620](https://github.com/pilosa/pilosa/pull/1620))
- Update Go client sample to match latest master ([#1614](https://github.com/pilosa/pilosa/pull/1614))
## [1.1.0] - 2018-08-21
This version contains 32 contributions from 5 contributors. There are 89 files changed; 2,752 insertions; and 1,013 deletions.
### Added
- Add CircleCI ([#1610](https://github.com/pilosa/pilosa/pull/1610))
- Add key translation to exports ([#1608](https://github.com/pilosa/pilosa/pull/1608))
- Support importing key values ([#1599](https://github.com/pilosa/pilosa/pull/1599), [#1601](https://github.com/pilosa/pilosa/pull/1601))
- Treat coordinator as primary translate store ([#1582](https://github.com/pilosa/pilosa/pull/1582))
- Add DEGRADED cluster state and handle gossip NodeLeave events correctly ([#1584](https://github.com/pilosa/pilosa/pull/1584))
- Add linters to gometalinter and fix related issues ([#1544](https://github.com/pilosa/pilosa/pull/1544), [#1543](https://github.com/pilosa/pilosa/pull/1543), [#1540](https://github.com/pilosa/pilosa/pull/1540), [#1539](https://github.com/pilosa/pilosa/pull/1539), [#1537](https://github.com/pilosa/pilosa/pull/1537), [#1536](https://github.com/pilosa/pilosa/pull/1536), [#1535](https://github.com/pilosa/pilosa/pull/1535), [#1534](https://github.com/pilosa/pilosa/pull/1534), [#1530](https://github.com/pilosa/pilosa/pull/1530), [#1529](https://github.com/pilosa/pilosa/pull/1529), [#1528](https://github.com/pilosa/pilosa/pull/1528), [#1526](https://github.com/pilosa/pilosa/pull/1526), [#1527](https://github.com/pilosa/pilosa/pull/1527))
- Add mutex field type ([#1524](https://github.com/pilosa/pilosa/pull/1524))
- Fragment rows() and rowsForColumn() ([#1532](https://github.com/pilosa/pilosa/pull/1532))
### Fixed
- Fix race on replicationClosing channel ([#1607](https://github.com/pilosa/pilosa/pull/1607))
- Prevent anti-entropy and cluster resize from running simultaneously ([#1586](https://github.com/pilosa/pilosa/pull/1586))
- Require a valid port that isn't greater than 65,535 ([#1603](https://github.com/pilosa/pilosa/pull/1603))
- Add view parameter to sync logic for syncing time fields ([#1602](https://github.com/pilosa/pilosa/pull/1602))
- Fix translator in cluster environment ([#1552](https://github.com/pilosa/pilosa/pull/1552))
- Use string prefix instead of equality so json error message will pass on all Go versions ([#1558](https://github.com/pilosa/pilosa/pull/1558))
## [1.0.2] - 2018-08-01
This version contains 11 contributions from 3 contributors. There are 30 files changed; 1,569 insertions; and 1,215 deletions.
### Fixed
- Fix documentation ([#1503](https://github.com/pilosa/pilosa/pull/1503), [#1495](https://github.com/pilosa/pilosa/pull/1495), [#1551](https://github.com/pilosa/pilosa/pull/1551))
- Fix places where empty IndexOptions were being used ([#1547](https://github.com/pilosa/pilosa/pull/1547))
- Fix translator syncing bug in cluster environments ([#1552](https://github.com/pilosa/pilosa/pull/1552))
- Fix race condition in translate_test ([#1541](https://github.com/pilosa/pilosa/pull/1541))
- Add IndexOptions to IndexInfo json response ([#1542](https://github.com/pilosa/pilosa/pull/1542))
- Add proper locking to cluster code to prevent races ([#1533](https://github.com/pilosa/pilosa/pull/1533))
- Re-export erroneously unexported func Row.Intersect ([#1502](https://github.com/pilosa/pilosa/pull/1502))
- Update parser to handle row keys on SetRowAttrs() ([#1555](https://github.com/pilosa/pilosa/pull/1555))
## [1.0.1] - 2018-07-11
This version contains 12 contributions from 4 contributors. There are 11 files changed; 133 insertions; and 39 deletions.
### Fixed
- Use `dep ensure -vendor-only` for build repeatability ([#1491](https://github.com/pilosa/pilosa/pull/1491))
- Make sure time range views are calculated correctly across months ([#1485](https://github.com/pilosa/pilosa/pull/1485))
- Fix up error handling, add a configurable timeout to http handler closing ([#1486](https://github.com/pilosa/pilosa/pull/1486))
- Add gossip Closer ([#1483](https://github.com/pilosa/pilosa/pull/1483))
- Update docs references to WebUI naming (console) and installation ([#1493](https://github.com/pilosa/pilosa/pull/1493))
## [1.0.0] - 2018-07-09
This version contains 218 contributions from 7 contributors. There are 184 files changed; 21,769 insertions; and 20,275 deletions.
### Added
- ID-Key Translation ([#1337](https://github.com/pilosa/pilosa/pull/1337))
- Add CORS support to handler ([#1327](https://github.com/pilosa/pilosa/pull/1327))
### Changed
- HTTP handler updates ([#1408](https://github.com/pilosa/pilosa/pull/1408), [#1399](https://github.com/pilosa/pilosa/pull/1399), [#1441](https://github.com/pilosa/pilosa/pull/1441), [#1375](https://github.com/pilosa/pilosa/pull/1375), [#1433](https://github.com/pilosa/pilosa/pull/1433), [#1444](https://github.com/pilosa/pilosa/pull/1444), [#1388](https://github.com/pilosa/pilosa/pull/1388), [#1309](https://github.com/pilosa/pilosa/pull/1309), [#1302](https://github.com/pilosa/pilosa/pull/1302), [#1304](https://github.com/pilosa/pilosa/pull/1304), [#1465](https://github.com/pilosa/pilosa/pull/1465), [#1466](https://github.com/pilosa/pilosa/pull/1466))
- Refactor/improve tests ([#1437](https://github.com/pilosa/pilosa/pull/1437), [#1434](https://github.com/pilosa/pilosa/pull/1434), [#1435](https://github.com/pilosa/pilosa/pull/1435), [#1425](https://github.com/pilosa/pilosa/pull/1425), [#1418](https://github.com/pilosa/pilosa/pull/1418), [#1419](https://github.com/pilosa/pilosa/pull/1419), [#1413](https://github.com/pilosa/pilosa/pull/1413), [#1394](https://github.com/pilosa/pilosa/pull/1394), [#1387](https://github.com/pilosa/pilosa/pull/1387), [#1386](https://github.com/pilosa/pilosa/pull/1386), [#1378](https://github.com/pilosa/pilosa/pull/1378), [#1364](https://github.com/pilosa/pilosa/pull/1364), [#1348](https://github.com/pilosa/pilosa/pull/1348), [#1340](https://github.com/pilosa/pilosa/pull/1340), [#1297](https://github.com/pilosa/pilosa/pull/1297))
- Simplify inter-node communication ([#1428](https://github.com/pilosa/pilosa/pull/1428), [#1427](https://github.com/pilosa/pilosa/pull/1427), [#1412](https://github.com/pilosa/pilosa/pull/1412), [#1398](https://github.com/pilosa/pilosa/pull/1398), [#1391](https://github.com/pilosa/pilosa/pull/1391), [#1389](https://github.com/pilosa/pilosa/pull/1389))
- Make gossip's interface to Pilosa the API struct ([#1452](https://github.com/pilosa/pilosa/pull/1452))
- Rename slice to shard ([#1426](https://github.com/pilosa/pilosa/pull/1426))
- Clearbit for time fields ([#1424](https://github.com/pilosa/pilosa/pull/1424))
- Update docs ([#1390](https://github.com/pilosa/pilosa/pull/1390), [#1329](https://github.com/pilosa/pilosa/pull/1329), [#1305](https://github.com/pilosa/pilosa/pull/1305), [#1296](https://github.com/pilosa/pilosa/pull/1296), [#1461](https://github.com/pilosa/pilosa/pull/1461))
- Simplify server setup ([#1417](https://github.com/pilosa/pilosa/pull/1417), [#1393](https://github.com/pilosa/pilosa/pull/1393),[#1451](https://github.com/pilosa/pilosa/pull/1451))
- Refactor API ([#1407](https://github.com/pilosa/pilosa/pull/1407))
- Rewrite PQL parser and add various improvements/simplifications ([#1382](https://github.com/pilosa/pilosa/pull/1382), [#1402](https://github.com/pilosa/pilosa/pull/1402), [#1354](https://github.com/pilosa/pilosa/pull/1354), [#1463](https://github.com/pilosa/pilosa/pull/1463))
- Rename "frame" to "field" ([#1395](https://github.com/pilosa/pilosa/pull/1395), [#1362](https://github.com/pilosa/pilosa/pull/1362), [#1360](https://github.com/pilosa/pilosa/pull/1360), [#1358](https://github.com/pilosa/pilosa/pull/1358), [#1357](https://github.com/pilosa/pilosa/pull/1357), [#1355](https://github.com/pilosa/pilosa/pull/1355))
- Optimize count ([#1365](https://github.com/pilosa/pilosa/pull/1365))
- Simplify bitmap max function ([#1333](https://github.com/pilosa/pilosa/pull/1333))
- Rename "bit" to "column" for clarity ([#1326](https://github.com/pilosa/pilosa/pull/1326))
- Rename pilosa.Bitmap to Row ([#1311](https://github.com/pilosa/pilosa/pull/1311))
- Invert encoding/decoding and remove internal references ([#1454](https://github.com/pilosa/pilosa/pull/1454))
### Removed
- Rename (unexport) many items to reduce public API footprint prior to 1.0 release ([#1470](https://github.com/pilosa/pilosa/pull/1470), [#1458](https://github.com/pilosa/pilosa/pull/1458), [#1450](https://github.com/pilosa/pilosa/pull/1450), [#1449](https://github.com/pilosa/pilosa/pull/1449), [#1448](https://github.com/pilosa/pilosa/pull/1448), [#1447](https://github.com/pilosa/pilosa/pull/1447), [#1446](https://github.com/pilosa/pilosa/pull/1446), [#1438](https://github.com/pilosa/pilosa/pull/1438), [#1443](https://github.com/pilosa/pilosa/pull/1443), [#1440](https://github.com/pilosa/pilosa/pull/1440), [#1439](https://github.com/pilosa/pilosa/pull/1439), [#1409](https://github.com/pilosa/pilosa/pull/1409), [#1392](https://github.com/pilosa/pilosa/pull/1392), [#1374](https://github.com/pilosa/pilosa/pull/1374), [#1372](https://github.com/pilosa/pilosa/pull/1372), [#1369](https://github.com/pilosa/pilosa/pull/1369), [#1367](https://github.com/pilosa/pilosa/pull/1367), [#1366](https://github.com/pilosa/pilosa/pull/1366), [#1351](https://github.com/pilosa/pilosa/pull/1351), [#1420](https://github.com/pilosa/pilosa/pull/1420), [#1416](https://github.com/pilosa/pilosa/pull/1416), [#1397](https://github.com/pilosa/pilosa/pull/1397))
- Remove dead code ([#1432](https://github.com/pilosa/pilosa/pull/1432), [#1457](https://github.com/pilosa/pilosa/pull/1457), [#1421](https://github.com/pilosa/pilosa/pull/1421), [#1411](https://github.com/pilosa/pilosa/pull/1411), [#1377](https://github.com/pilosa/pilosa/pull/1377), [#1393](https://github.com/pilosa/pilosa/pull/1393), [#1462](https://github.com/pilosa/pilosa/pull/1462))
- Remove view argument from Field.SetBit and Field.ClearBit ([#1396](https://github.com/pilosa/pilosa/pull/1396))
- Remove WebUI (now contained in a separate package) ([#1363](https://github.com/pilosa/pilosa/pull/1363))
- Remove bench command ([#1347](https://github.com/pilosa/pilosa/pull/1347))
- Remove "view" from API, handler, docs ([#1346](https://github.com/pilosa/pilosa/pull/1346))
- Remove backup/restore stuff ([#1339](https://github.com/pilosa/pilosa/pull/1339), [#1341](https://github.com/pilosa/pilosa/pull/1341))
- Remove inverse frame functionality ([#1335](https://github.com/pilosa/pilosa/pull/1335))
- Remove rangeEnabled option ([#1332](https://github.com/pilosa/pilosa/pull/1332))
- Remove index and field MarshalJSON ([#1468](https://github.com/pilosa/pilosa/pull/1468))
### Fixed
- Fix a few data races ([#1423](https://github.com/pilosa/pilosa/pull/1423))
- Fix for crash while removing containers ([#1401](https://github.com/pilosa/pilosa/pull/1401))
- Allow dashes in frame names ([#1415](https://github.com/pilosa/pilosa/pull/1415))
- Fix generate-config command, use single toml lib ([#1350](https://github.com/pilosa/pilosa/pull/1350))
## [0.10.0] - 2018-05-15
This version contains 93 contributions from 8 contributors. There are 93 files changed; 4,495 insertions; and 5,392 deletions.
### Added
- Add B+Tree containers (Enterprise Edition) ([#1285](https://github.com/pilosa/pilosa/pull/1285))
- Add /info endpoint ([#1236](https://github.com/pilosa/pilosa/pull/1236))
### Changed
- Wrap errors ([#1271](https://github.com/pilosa/pilosa/pull/1271), [#1258](https://github.com/pilosa/pilosa/pull/1258), [#1274](https://github.com/pilosa/pilosa/pull/1274), [#1270](https://github.com/pilosa/pilosa/pull/1270), [#1273](https://github.com/pilosa/pilosa/pull/1273), [#1272](https://github.com/pilosa/pilosa/pull/1272), [#1260](https://github.com/pilosa/pilosa/pull/1260), [#1259](https://github.com/pilosa/pilosa/pull/1259), [#1256](https://github.com/pilosa/pilosa/pull/1256), [#1257](https://github.com/pilosa/pilosa/pull/1257), [#1261](https://github.com/pilosa/pilosa/pull/1261), [#1262](https://github.com/pilosa/pilosa/pull/1262), [#1263](https://github.com/pilosa/pilosa/pull/1263), [#1265](https://github.com/pilosa/pilosa/pull/1265))
### Removed
- Remove unused code ([#1286](https://github.com/pilosa/pilosa/pull/1286))
- Remove input definition, add install-stringer to Makefile ([#1284](https://github.com/pilosa/pilosa/pull/1284))
- Remove /id and /hosts endpoints. Add local ID to /status ([#1238](https://github.com/pilosa/pilosa/pull/1238))
- Remove API.URI ([#1255](https://github.com/pilosa/pilosa/pull/1255))
### Fixed
- Assorted docs fixes ([#1281](https://github.com/pilosa/pilosa/pull/1281), [#1269](https://github.com/pilosa/pilosa/pull/1269))
- Update PQL syntax in bench subcommand ([#1279](https://github.com/pilosa/pilosa/pull/1279))
- Update help menu in WebUI ([#1278](https://github.com/pilosa/pilosa/pull/1278))
- Fix dead lock ([#1268](https://github.com/pilosa/pilosa/pull/1268))
- Make sure gossipMemberSet.Logger is set during server setup ([#1266](https://github.com/pilosa/pilosa/pull/1266))
- Make sure ~ is expanded in NewServer; BroadcastReceiver uses temp path ([#1242](https://github.com/pilosa/pilosa/pull/1242))
- Avoid creating a slice of nil timestamps on Import() ([#1234](https://github.com/pilosa/pilosa/pull/1234))
- Fixup internal client ([#1253](https://github.com/pilosa/pilosa/pull/1253))
## [0.9.0] - 2018-05-04
This version contains 188 contributions from 12 contributors. There are 141 files changed; 17,832 insertions; and 7,503 deletions.
*Please see special [upgrading instructions](https://www.pilosa.com/docs/latest/administration/#version-0-9) for this release.*
### Added
- Add ability to dynamically resize clusters ([#982](https://github.com/pilosa/pilosa/pull/982), [#946](https://github.com/pilosa/pilosa/pull/946), [#929](https://github.com/pilosa/pilosa/pull/929), [#927](https://github.com/pilosa/pilosa/pull/927), [#917](https://github.com/pilosa/pilosa/pull/917), [#913](https://github.com/pilosa/pilosa/pull/913), [#912](https://github.com/pilosa/pilosa/pull/912), [#908](https://github.com/pilosa/pilosa/pull/908))
- Update docs to include cluster-resize config and instructions ([#1088](https://github.com/pilosa/pilosa/pull/1088))
- Add support for lists of gossip seeds for redundancy ([#1133](https://github.com/pilosa/pilosa/pull/1133))
- Add HTTP Handler validation ([#1140](https://github.com/pilosa/pilosa/pull/1140), [#1121](https://github.com/pilosa/pilosa/pull/1121))
- Add validation around node-remove conditions ([#1138](https://github.com/pilosa/pilosa/pull/1138))
- broadcast.SendSync field creation and deletion to all nodes ([#1132](https://github.com/pilosa/pilosa/pull/1132))
- Spread recalculate caches to all nodes. Fixes #1069 ([#1109](https://github.com/pilosa/pilosa/pull/1109))
- Add QueryResult.Type to protobuf message to distiguish results at the client ([#1064](https://github.com/pilosa/pilosa/pull/1064))
- Modify `pilosa import` to support string rows/columns ([#1063](https://github.com/pilosa/pilosa/pull/1063))
- Add some statsd calls to HolderSyncer ([#1048](https://github.com/pilosa/pilosa/pull/1048))
- Add support for memberlist gossip configuration via pilosa.Config ([#1014](https://github.com/pilosa/pilosa/pull/1014))
- Add local and cluster IDs ([#1013](https://github.com/pilosa/pilosa/pull/1013), [#1245](https://github.com/pilosa/pilosa/pull/1245))
- Add HolderCleaner and view.DeleteFragment ([#985](https://github.com/pilosa/pilosa/pull/985))
- Add set-coordinator endpoint ([#963](https://github.com/pilosa/pilosa/pull/963))
- Implement Min/Max BSI queries ([#1191](https://github.com/pilosa/pilosa/pull/1191))
- Log time/version to startup log ([#1246](https://github.com/pilosa/pilosa/pull/1246))
- Documentation improvements ([#1135](https://github.com/pilosa/pilosa/pull/1135), [#1154](https://github.com/pilosa/pilosa/pull/1154), [#1091](https://github.com/pilosa/pilosa/pull/1091), [#1108](https://github.com/pilosa/pilosa/pull/1108), [#1087](https://github.com/pilosa/pilosa/pull/1087), [#1086](https://github.com/pilosa/pilosa/pull/1086), [#1026](https://github.com/pilosa/pilosa/pull/1026), [#1022](https://github.com/pilosa/pilosa/pull/1022), [#1007](https://github.com/pilosa/pilosa/pull/1007), [#981](https://github.com/pilosa/pilosa/pull/981), [#901](https://github.com/pilosa/pilosa/pull/901), [#972](https://github.com/pilosa/pilosa/pull/972), [#1215](https://github.com/pilosa/pilosa/pull/1215), [#1213](https://github.com/pilosa/pilosa/pull/1213), [#1224](https://github.com/pilosa/pilosa/pull/1224), [#1250](https://github.com/pilosa/pilosa/pull/1250))
### Changed
- Put Statik behind an interface ([#1163](https://github.com/pilosa/pilosa/pull/1163))
- Refactor diagnostics, inject gopsutil dependency ([#1166](https://github.com/pilosa/pilosa/pull/1166))
- Use boolean instead of address to configure coordinator ([#1158](https://github.com/pilosa/pilosa/pull/1158))
- Put GCNotify behind an interface ([#1148](https://github.com/pilosa/pilosa/pull/1148))
- Replace custom assembly bit functions with standard go ([#797](https://github.com/pilosa/pilosa/pull/797))
- Improve roaring tests ([#1115](https://github.com/pilosa/pilosa/pull/1115))
- Change configuration cluster.type (string) to cluster.disabled (bool) ([#1099](https://github.com/pilosa/pilosa/pull/1099))
- Use NodeID instead of URI for node identification ([#1077](https://github.com/pilosa/pilosa/pull/1077))
- Change gossip config from DefaultLocalConfig to DefaultWANConfig ([#1032](https://github.com/pilosa/pilosa/pull/1032))
- Use binary search in runAdd ([#1027](https://github.com/pilosa/pilosa/pull/1027))
- Use HTTP handler for gossip SendSync ([#1001](https://github.com/pilosa/pilosa/pull/1001))
- Group the write operations in syncBlock by MaxWritesPerRequest ([#950](https://github.com/pilosa/pilosa/pull/950))
- Refactor HTTPClient handling ([#991](https://github.com/pilosa/pilosa/pull/991))
- Remove FrameSchema. Move Fields to the Frame struct ([#907](https://github.com/pilosa/pilosa/pull/907))
- Refactor pilosa/server ([#1220](https://github.com/pilosa/pilosa/pull/1220))
- Clean up flipBitmap and add tests ([#1223](https://github.com/pilosa/pilosa/pull/1223))
- Move pilosa.Config to pilosa/server.Config ([#1216](https://github.com/pilosa/pilosa/pull/1216))
- Vendor github.com/golang/groupcache/lru ([#1221](https://github.com/pilosa/pilosa/pull/1221))
### Removed
- Remove the Gossip stutter from memberlist-related config options ([#1171](https://github.com/pilosa/pilosa/pull/1171))
- Remove old GossipPort and GossipSeed config options ([#1142](https://github.com/pilosa/pilosa/pull/1142))
- Remove cluster type `http` from docs ([#1130](https://github.com/pilosa/pilosa/pull/1130))
- Remove holder.Peek, combine with HasData, move server logic ([#1226](https://github.com/pilosa/pilosa/pull/1226))
- Remove PATCH frame endpoint ([#1222](https://github.com/pilosa/pilosa/pull/1222))
- Remove Index.MergeSchemas() method ([#1219](https://github.com/pilosa/pilosa/pull/1219))
- Remove references to Input Definition from the docs ([#1212](https://github.com/pilosa/pilosa/pull/1212))
- Remove Index.TimeQuantum ([#1209](https://github.com/pilosa/pilosa/pull/1209))
- Remove SecurityManager. Implement api restrictions in api package. ([#1207](https://github.com/pilosa/pilosa/pull/1207))
### Fixed
- Handle the scheme correctly in config.Bind ([#1143](https://github.com/pilosa/pilosa/pull/1143))
- Prevent excessive sendSync (createView) messages. ([#1139](https://github.com/pilosa/pilosa/pull/1139))
- Fix a shift logic bug in bitmapZeroRange ([#1110](https://github.com/pilosa/pilosa/pull/1110))
- Fix node id validation on set-coordinator ([#1102](https://github.com/pilosa/pilosa/pull/1102))
- Avoid overflow bug in differenceRunArray ([#1105](https://github.com/pilosa/pilosa/pull/1105))
- Fix bug in NewServerCluster where each host was its own coordinator ([#1101](https://github.com/pilosa/pilosa/pull/1101))
- Fix count/bitmap mismatch bug ([#1084](https://github.com/pilosa/pilosa/pull/1084))
- Fix edge case with Range() calls outside field Min/Max. Fixes #876. ([#979](https://github.com/pilosa/pilosa/pull/979))
- Bind the handler to all interfaces (0.0.0.0) in Dockerfile. Fixes #977. ([#980](https://github.com/pilosa/pilosa/pull/980))
- Fix nil client bug in monitorAntiEntropy (and test) ([#1233](https://github.com/pilosa/pilosa/pull/1233))
- Fix crash due to server.diagnostics.server not set ([#1229](https://github.com/pilosa/pilosa/pull/1229))
- Fix some cluster race conditions ([#1228](https://github.com/pilosa/pilosa/pull/1228))
### Deprecated
- Deprecate RangeEnabled option ([#1205](https://github.com/pilosa/pilosa/pull/1205))
### Performance
- Add benchmark for various container usage patterns ([#1017](https://github.com/pilosa/pilosa/pull/1017))
## [0.8.8] - 2018-02-19
This version contains 1 contribution from 2 contributors. There are 4 files changed; 1,153 insertions; and 618 deletions.
### Fixed
- Bug fixes and improved test coverage in roaring ([#1118](https://github.com/pilosa/pilosa/pull/1118))
## [0.8.7] - 2018-02-12
This version contains 1 contribution from 1 contributors. There are 2 files changed; 84 insertions; and 4 deletions.
### Fixed
- Fix a shift logic bug in bitmapZeroRange ([#1111](https://github.com/pilosa/pilosa/pull/1111))
## [0.8.6] - 2018-02-09
This version contains 2 contributions from 2 contributors. There are 3 files changed; 171 insertions; and 6 deletions.
### Fixed
- Fix overflow bug in differenceRunArray [#1106](https://github.com/pilosa/pilosa/pull/1106)
- Fix bug where count and bitmap queries could return different numbers [#1083](https://github.com/pilosa/pilosa/pull/1083)
## [0.8.5] - 2018-01-18
This version contains 1 contribution from 1 contributor. There is 1 file changed; 1 insertion, and 0 deletions.
### Fixed
- Bind Docker container on all interfaces ([#1061](https://github.com/pilosa/pilosa/pull/1061))
## [0.8.4] - 2018-01-10
This version contains 4 contributions from 3 contributors. There are 17 files changed; 974 insertions; and 221 deletions.
### Fixed
- Group the write operations in syncBlock by MaxWritesPerRequest ([#1038](https://github.com/pilosa/pilosa/pull/1038))
- Change gossip config from memberlist.DefaultLocalConfig to memberlist.DefaultWANConfig ([#1033](https://github.com/pilosa/pilosa/pull/1033))
### Performance
- Change AttrBlock handler calls to support protobuf instead of json ([#1046](https://github.com/pilosa/pilosa/pull/1046))
- Use RLock instead of Lock in a few places ([#1042](https://github.com/pilosa/pilosa/pull/1042))
## [0.8.3] - 2017-12-12
This version contains 1 contribution from 1 contributor. There are 2 files changed; 59 insertions; and 42 deletions.
### Fixed
- Protect against accessing pointers to memory which was unmapped ([#1000](https://github.com/pilosa/pilosa/pull/1000))
## [0.8.2] - 2017-12-05
This version contains 1 contribution from 1 contributor. There are 15 files changed; 127 insertions; and 98 deletions.
### Fixed
- Modify initialization of HTTP client so only one instance is created ([#994](https://github.com/pilosa/pilosa/pull/994))
## [0.8.1] - 2017-11-15
This version contains 2 contributions from 2 contributors. There are 4 files changed; 27 insertions; and 14 deletions.
### Fixed
- Fix CountOpenFiles() fatal crash ([#969](https://github.com/pilosa/pilosa/pull/969))
- Fix version check when local is greater than pilosa.com ([#968](https://github.com/pilosa/pilosa/pull/968))
## [0.8.0] - 2017-11-15
This version contains 31 contributions from 8 contributors. There are 84 files changed; 3,732 insertions; and 1,428 deletions.
### Added
- Diagnostics ([#895](https://github.com/pilosa/pilosa/pull/895))
- Add docker-build make target for repeatable Docker-based builds ([#933](https://github.com/pilosa/pilosa/pull/933))
- Add documentation on importing field values; fixes #924 ([#938](https://github.com/pilosa/pilosa/pull/938))
- Add flag documentation and tests, remove "plugins.path" ([#942](https://github.com/pilosa/pilosa/pull/942))
- Add TLS support ([#867](https://github.com/pilosa/pilosa/pull/867))
- Add TLS cluster how to ([#898](https://github.com/pilosa/pilosa/pull/898))
- Add support for gossip encryption ([#889](https://github.com/pilosa/pilosa/pull/889))
- Add Recalculate Caches endpoint ([#881](https://github.com/pilosa/pilosa/pull/881))
- Add search-friendly documentation for BSI range query syntax ([#955](https://github.com/pilosa/pilosa/pull/955))
### Changed
- Remove unneeded Gopkg.toml constraints and update all dependencies ([#943](https://github.com/pilosa/pilosa/pull/943))
- Remove row and column labels in webUI ([#884](https://github.com/pilosa/pilosa/pull/884))
- Internal Client refactoring ([#892](https://github.com/pilosa/pilosa/pull/892))
- Remove column/row labels for input definition ([#945](https://github.com/pilosa/pilosa/pull/945))
- Update dependencies and Go version ([#878](https://github.com/pilosa/pilosa/pull/878))
### Fixed
- Skip permissions test when run as root. Fixes #940 ([#941](https://github.com/pilosa/pilosa/pull/941))
- Address "connection reset" issues in client ([#934](https://github.com/pilosa/pilosa/pull/934))
- Fix field value import: Use signed int and respect field minimum ([#919](https://github.com/pilosa/pilosa/pull/919))
- Constrain BoltDB to version rather than specific revision ([#887](https://github.com/pilosa/pilosa/pull/887))
- Fix bug in environment variable format ([#882](https://github.com/pilosa/pilosa/pull/882))
- Fix overflow in differenceRunBitmap ([#949](https://github.com/pilosa/pilosa/pull/949))
### Performance
- Use FieldNotNull to improve efficiency of BETWEEN queries ([#874](https://github.com/pilosa/pilosa/pull/874))
## [0.7.2] - 2017-11-15
This version contains 1 contribution from 1 contributor. There is 1 file changed; 16 insertions; and 1 deletion.
### Changed
- Bump HTTP client's MaxIdleConns and MaxIdleConnsPerHost ([#920](https://github.com/pilosa/pilosa/pull/920))
## [0.7.1] - 2017-10-09
This version contains 3 contributions from 3 contributors. There are 14 files changed; 221 insertions; and 52 deletions.
### Changed
- Update dependencies and Go version ([#878](https://github.com/pilosa/pilosa/pull/878))
### Performance
- Leverage not-null field to make BETWEEN queries more efficient ([#874](https://github.com/pilosa/pilosa/pull/874))
## [0.7.0] - 2017-10-03
This version contains 59 contributions from 9 contributors. There are 61 files changed; 5207 insertions; and 1054 deletions.
### Added
- Add HTTP API for fields ([#811](https://github.com/pilosa/pilosa/pull/811), [#856](https://github.com/pilosa/pilosa/pull/856))
- Add HTTP API for delete views ([#785](https://github.com/pilosa/pilosa/pull/785))
- Modify import endpoint to handle BSI field values ([#840](https://github.com/pilosa/pilosa/pull/840))
- Add field Range() support to Executor ([#791](https://github.com/pilosa/pilosa/pull/791))
- Support PQL Range() queries for fields ([#755](https://github.com/pilosa/pilosa/pull/755))
- Add Sum() field query ([#778](https://github.com/pilosa/pilosa/pull/778))
- Add documentation for BSI ([#861](https://github.com/pilosa/pilosa/pull/861))
- Add BETWEEN for Range queries ([#847](https://github.com/pilosa/pilosa/pull/847))
- Add Xor support for PQL ([#789](https://github.com/pilosa/pilosa/pull/789))
- Enable auto-creating the schema on imports ([#837](https://github.com/pilosa/pilosa/pull/837))
- Update client library docs ([#831](https://github.com/pilosa/pilosa/pull/831))
- Handle SIGTERM signal ([#830](https://github.com/pilosa/pilosa/pull/830))
- Add cluster config example to docs ([#806](https://github.com/pilosa/pilosa/pull/806))
- Add ability to exclude attributes and bits in Bitmap queries ([#783](https://github.com/pilosa/pilosa/pull/783))
### Fixed
- Fix panic when iterating over an empty run container ([#860](https://github.com/pilosa/pilosa/pull/860))
- Fix row id zero bug ([#814](https://github.com/pilosa/pilosa/pull/814))
- Fix cache invalidation bug ([#795](https://github.com/pilosa/pilosa/pull/795))
- Set container.n in differenceRunRun ([#794](https://github.com/pilosa/pilosa/pull/794))
- Fix infinite loop in bitmap-to-array conversion ([#779](https://github.com/pilosa/pilosa/pull/779))
- Fix CountRange bug ([#773](https://github.com/pilosa/pilosa/pull/773))
### Deprecated
- Remove support for row/column labels ([#839](https://github.com/pilosa/pilosa/pull/839))
### Performance
- Refactor differenceRunArray ([#859](https://github.com/pilosa/pilosa/pull/859))
- Update fragment.FieldSum to use roaring IntersectionCount() ([#841](https://github.com/pilosa/pilosa/pull/841))
- Add roaring optimizations ([#842](https://github.com/pilosa/pilosa/pull/842))
- Convert lock to read lock ([#848](https://github.com/pilosa/pilosa/pull/848))
- Reduce Lock calls in executor ([#846](https://github.com/pilosa/pilosa/pull/846))
- Implement container.flipBitmap() to improve differenceRunBitmap() ([#849](https://github.com/pilosa/pilosa/pull/849))
- Reuse container storage on UnmarshalBinary to improve memory utilization ([#820](https://github.com/pilosa/pilosa/pull/820))
- Improve WriteTo performance ([#812](https://github.com/pilosa/pilosa/pull/812))
## [0.6.0] - 2017-08-11
This version contains 14 contributions from 5 contributors. There are 28 files changed; 4,936 insertions; and 692 deletions.
### Added
- Add Run-length Encoding ([#758](https://github.com/pilosa/pilosa/pull/758))
### Changed
- Make gossip the default broadcast type ([#750](https://github.com/pilosa/pilosa/pull/750))
### Fixed
- Fix CountRange ([#759](https://github.com/pilosa/pilosa/pull/759))
- Fix `differenceArrayRun` logic ([#674](https://github.com/pilosa/pilosa/pull/674))
## [0.5.0] - 2017-08-02
This version contains 65 contributions from 8 contributors (including 1 volunteer contributor). There are 79 files changed; 7,972 insertions; and 2,800 deletions.
### Added
- Set open file limit during Pilosa startup ([#748](https://github.com/pilosa/pilosa/pull/748))
- Add Input Definition ([#646](https://github.com/pilosa/pilosa/pull/646))
- Add cache type: None ([#745](https://github.com/pilosa/pilosa/pull/745))
- Add panic recovery in top level HTTP handler ([#741](https://github.com/pilosa/pilosa/pull/741))
- Count open file handles as a StatsD metric ([#636](https://github.com/pilosa/pilosa/pull/636))
- Add coverage tools to Makefile ([#635](https://github.com/pilosa/pilosa/pull/635))
- Add Holder test coverage ([#629](https://github.com/pilosa/pilosa/pull/629))
- Add runtime memory metrics ([#600](https://github.com/pilosa/pilosa/pull/600))
- Add sorting flag to import command ([#606](https://github.com/pilosa/pilosa/pull/606))
- Add PQL support for field values (WIP) ([#721](https://github.com/pilosa/pilosa/pull/721))
- Set and retrieve field values (WIP) ([#702](https://github.com/pilosa/pilosa/pull/702))
- Add BSI range-encoding schema support (WIP) ([#670](https://github.com/pilosa/pilosa/pull/670))
### Changed
- Move InternalPort config option to top-level ([#747](https://github.com/pilosa/pilosa/pull/747))
- Switch from glide to dep for dependency management ([#744](https://github.com/pilosa/pilosa/pull/744))
- Remove QueryRequest.Quantum since it is no longer used ([#699](https://github.com/pilosa/pilosa/pull/699))
- Refactor test utilities into importable package ([#675](https://github.com/pilosa/pilosa/pull/675))
### Fixed
- Add mutex for attribute cache ([#729](https://github.com/pilosa/pilosa/pull/729))
- Use log-path flag to specify log file ([#678](https://github.com/pilosa/pilosa/pull/678))
## [0.4.0] - 2017-06-08
This version contains 53 contributions from 13 contributors (including 4 volunteer contributors). There are 96 files changed; 6373 insertions; and 770 deletions.
*Note that data files created in Pilosa < 0.4.0 are not compatible with Pilosa 0.4.0 as a result of [#520](https://github.com/pilosa/pilosa/pull/520).*
### Added
- Support metric reporting through StatsD protocol ([#468](https://github.com/pilosa/pilosa/pull/468), [#568](https://github.com/pilosa/pilosa/pull/568), [#580](https://github.com/pilosa/pilosa/pull/580))
- Improve test coverage for ctl package ([#586](https://github.com/pilosa/pilosa/pull/586))
- Add support for bit flip (negate) in roaring ([#592](https://github.com/pilosa/pilosa/pull/592))
- Add xor support to roaring ([#571](https://github.com/pilosa/pilosa/pull/571))
- Improve WebUI autocomplete ([#560](https://github.com/pilosa/pilosa/pull/560))
- Add syntax hints tooltip to WebUI ([#537](https://github.com/pilosa/pilosa/pull/537))
- Implement 'config' CLI command ([#541](https://github.com/pilosa/pilosa/pull/541))
- Move docs into repo ([#563](https://github.com/pilosa/pilosa/pull/563))
- Add inverse TopN() support ([#551](https://github.com/pilosa/pilosa/pull/551))
- Add various Makefile updates ([#540](https://github.com/pilosa/pilosa/pull/540))
- Provide details on Glide checksum mismatch ([#546](https://github.com/pilosa/pilosa/pull/546))
- Add Docker multi-stage build ([#535](https://github.com/pilosa/pilosa/pull/535))
- Support inverse Range() queries ([#533](https://github.com/pilosa/pilosa/pull/533))
- Support colon commands in WebUI ([#529](https://github.com/pilosa/pilosa/pull/529), [#510](https://github.com/pilosa/pilosa/pull/510))
### Changed
- Increase default partition count from 16 to 256 (BREAKING CHANGE) ([#520](https://github.com/pilosa/pilosa/pull/520))
- Validate unknown query params ([#578](https://github.com/pilosa/pilosa/pull/578))
- Validate configuration file ([#573](https://github.com/pilosa/pilosa/pull/573))
- Change default cache type to ranked ([#524](https://github.com/pilosa/pilosa/pull/524))
- Add max-writes-per-requests limit ([#525](https://github.com/pilosa/pilosa/pull/525))
### Fixed
- Add "make test" to PHONY section of Makefile ([#605](https://github.com/pilosa/pilosa/pull/605))
- Fix failing tests when IPv6 is disabled ([#594](https://github.com/pilosa/pilosa/pull/594))
- Add minor docs fix, indent in JSON ([#599](https://github.com/pilosa/pilosa/pull/599))
- Fix BroadcastHandler handle missing index error ([#597](https://github.com/pilosa/pilosa/pull/597))
- Add WebUI fixes ([#589](https://github.com/pilosa/pilosa/pull/589))
- Fix support for 32-bit Linux ([#549](https://github.com/pilosa/pilosa/pull/549), [#565](https://github.com/pilosa/pilosa/pull/565))
- Fix 3 separate bugs in bitmapCountRange ([#559](https://github.com/pilosa/pilosa/pull/559))
- Add client support for MaxInverseSliceByIndex ([#555](https://github.com/pilosa/pilosa/pull/555))
- Fix bug in `handleGetSliceMax` ([#554](https://github.com/pilosa/pilosa/pull/554))
- Default to `standard` view in export command ([#548](https://github.com/pilosa/pilosa/pull/548))
- Fix vet issues with the assembly code in Roaring ([#528](https://github.com/pilosa/pilosa/pull/528))
- Prevent row labels that match the column label ([#503](https://github.com/pilosa/pilosa/pull/503))
- Fix roaring test: TestBitmap_Quick_Array1 ([#507](https://github.com/pilosa/pilosa/pull/507))
- Don't try to create inverse views on Import() when inverseEnabled is false ([#462](https://github.com/pilosa/pilosa/pull/462))
### Performance
- Set n based on array length instead of incrementing repeatedly ([#590](https://github.com/pilosa/pilosa/pull/590))
- Rewrite intersectCountArrayBitmap for perf test ([#577](https://github.com/pilosa/pilosa/pull/577))
- Check for duplicate attributes under read lock on insert ([#562](https://github.com/pilosa/pilosa/pull/562))
[Unreleased]: https://github.com/pilosa/pilosa/compare/v1.2...HEAD
[0.4.0]: https://github.com/pilosa/pilosa/compare/v0.3...v0.4
[0.5.0]: https://github.com/pilosa/pilosa/compare/v0.4...v0.5
[0.6.0]: https://github.com/pilosa/pilosa/compare/v0.5...v0.6
[0.7.0]: https://github.com/pilosa/pilosa/compare/v0.6...v0.7
[0.8.0]: https://github.com/pilosa/pilosa/compare/v0.7...v0.8
[0.9.0]: https://github.com/pilosa/pilosa/compare/v0.8...v0.9
[0.10.0]: https://github.com/pilosa/pilosa/compare/v0.9...v0.10
[1.0.0]: https://github.com/pilosa/pilosa/compare/v0.10...v1.0
[1.1.0]: https://github.com/pilosa/pilosa/compare/v1.0...v1.1
[1.2.0]: https://github.com/pilosa/pilosa/compare/v1.1...v1.2

177
CONTRIBUTING.md
View file

@ -1,177 +0,0 @@
# Contributing to Pilosa
The workflow components of these instructions apply to all Pilosa repositories.
## Reporting a bug
If you have discovered a bug and don't see it in the [github issue tracker][5], [open a new issue][1].
## Submitting a feature request
Feature requests are managed in Github issues, organized with [Zenhub](https://www.zenhub.com/), which is publicly available as a browser extension. New features typically go through a [Proposal Process][4]
which starts by [opening a new issue][1] that describes the new feature proposal.
## Making code contributions
Before you start working on new features, you should [open a new issue][1] to let others know what
you're doing, otherwise you run the risk of duplicating effort. This also
gives others an opportunity to provide input for your feature.
If you want to help but you aren't sure where to start, check out our [github label for low-effort issues][6].
### Development Environment
- Ensure you have a recent version of [Go](https://golang.org/doc/install) installed. Pilosa generally supports the current and previous minor versions; check our [CircleCI config file](../master/.circleci/config.yml) for the most up-to-date information.
- Make sure `$GOPATH` environment variable points to your Go working directory and `$PATH` incudes `$GOPATH/bin`, as described [here](https://golang.org/doc/code.html#GOPATH).
- Fork the [Pilosa repository][2] to your own account.
- It will be easier to follow these instructions if you:
```sh
export GH_USERNAME=<your github username>
```
- Create a directory (note that we use `github.com/pilosa`, NOT `github.com/USER`) and clone Pilosa:
```sh
mkdir -p ${GOPATH}/src/github.com/pilosa && cd $_
git clone https://github.com/pilosa/pilosa.git
```
- `cd` to your pilosa directory:
```sh
cd ${GOPATH}/src/github.com/pilosa/pilosa
```
- Install Pilosa command line tools:
```sh
make install
```
Running `pilosa` should now run a Pilosa instance.
- The official Pilosa repository is your "origin" remote in git. Add your fork as your github username
```sh
cd ${GOPATH}/src/github.com/pilosa/pilosa
git remote add ${GH_USERNAME} git@github.com:${GH_USERNAME}/pilosa.git
```
### Makefile
Pilosa includes a Makefile that automates several tasks:
- Install Pilosa:
```sh
make install
```
- Install build dependencies:
```sh
make install-build-deps
```
- Create the vendor directory:
```sh
make vendor
```
- Run the test suite:
```sh
make test
```
- View the coverage report:
```sh
make cover-viz
```
- Clear the `vendor/` and `build/` directories:
```sh
make clean
```
- Create release tarballs:
```sh
make release
```
- Regenerate protocol buffer files in `internal/`:
```sh
make generate-protoc
```
- Create tagged Docker image:
```sh
make docker
```
- Run tests inside Docker container:
```sh
make docker-test
```
Additional commands are available in the `Makefile`.
### Submitting code changes
- Before starting to work on a task, sync your branch with the upstream:
```sh
git checkout master
git pull
```
- Create a local feature branch:
```sh
git checkout -b something-amazing
```
- Commit your changes locally using `git add` and `git commit`. Please use [appropriate commit messages](https://chris.beams.io/posts/git-commit/).
- Make sure that you've written tests for your new feature, and then run the tests:
```sh
make test
```
- Verify that your pull request is applied to the latest version of code on github:
```sh
git checkout master
git pull
git checkout something-amazing
git rebase master
```
- Push to your fork:
```sh
git push -u $GH_USERNAME something-amazing:something-amazing
```
- Submit a [pull request][3]
[1]: https://github.com/pilosa/pilosa/issues/new
[2]: https://github.com/pilosa/pilosa
[3]: https://github.com/pilosa/pilosa/compare/
[4]: https://github.com/pilosa/general/blob/master/proposal.md
[5]: https://github.com/pilosa/pilosa/issues
[6]: https://github.com/pilosa/pilosa/issues?q=is%3Aopen+is%3Aissue+label%3Anewcomer

1
Dockerfile
View file

@ -42,7 +42,6 @@ RUN apk add --no-cache curl jq
COPY --from=pilosa-builder /pilosa/build/featurebase /
COPY LICENSE /LICENSE
COPY NOTICE /NOTICE
EXPOSE 10101

1
Dockerfile-clustertests
View file

@ -20,7 +20,6 @@ RUN apt install -y docker.io
RUN cp /go/bin/featurebase /featurebase
COPY LICENSE /LICENSE
COPY NOTICE /NOTICE
EXPOSE 10101

1
Dockerfile.pilosa
View file

@ -24,7 +24,6 @@ RUN apk add --no-cache curl jq
COPY --from=pilosa-builder /pilosa/build/featurebase /
COPY LICENSE /LICENSE
COPY NOTICE /NOTICE
EXPOSE 10101

202
LICENSE
View file

@ -1,202 +0,0 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

23
Makefile
View file

@ -18,10 +18,6 @@ GO_VERSION=1.16.10
DOCKER_BUILD= # set to 1 to use `docker-build` instead of `build` when creating a release
BUILD_TAGS += shardwidth$(SHARD_WIDTH)
TEST_TAGS = roaringparanoia
define LICENSE_HASH_CODE
head -13 $1 | sed -e 's/Copyright 20[0-9][0-9]/Copyright 20XX/' -e 's/Pilosa Corp\./Molecula Corp./' | shasum | cut -f 1 -d " "
endef
LICENSE_HASH=$(shell $(call LICENSE_HASH_CODE, pilosa.go))
UNAME := $(shell uname -s)
ifeq ($(UNAME), Darwin)
IS_MACOS:=1
@ -104,10 +100,19 @@ build:
# Create a single release build under the build directory
release-build:
$(MAKE) $(if $(DOCKER_BUILD),docker-)build FLAGS="-o build/featurebase-$(VERSION_ID)/featurebase"
cp NOTICE README.md LICENSE build/featurebase$(VERSION_ID)
cp NOTICE install/featurebase.conf install/featurebase*.service build/featurebase-$(VERSION_ID)
tar -cvz -C build -f build/featurebase-$(VERSION_ID).tar.gz featurebase-$(VERSION_ID)/
@echo Created release build: build/featurebase-$(VERSION_ID).tar.gz
test-release-build: docker-build
mv build/featurebase-$(VERSION_ID).tar.gz install/
cd install && docker build -t featurebase:test_installation \
-f test_installation.Dockerfile \
--build-arg release_tarball=featurebase-$(VERSION_ID).tar.gz .
mv install/featurebase-$(VERSION_ID).tar.gz build/
docker run -it -v /sys/fs/cgroup:/sys/fs/cgroup:ro \
featurebase:test_installation
# Error out if there are untracked changes in Git
check-clean:
ifndef SKIP_CHECK_CLEAN
@ -220,7 +225,7 @@ docker-build: vendor
docker create --name featurebase-build featurebase:build
mkdir -p build/featurebase-$(VERSION_ID)
docker cp featurebase-build:/pilosa/build/. ./build/featurebase-$(VERSION_ID)
cp NOTICE LICENSE ./build/featurebase-$(VERSION_ID)
cp NOTICE install/featurebase.conf install/featurebase*.service ./build/featurebase-$(VERSION_ID)
docker rm featurebase-build
tar -cvz -C build -f build/featurebase-$(VERSION_ID).tar.gz featurebase-$(VERSION_ID)/
@ -304,12 +309,6 @@ gometalinter: require-gometalinter vendor
--exclude "^pql/pql.peg.go" \
./...
# Verify that all Go files have license header
check-license-headers: SHELL:=/bin/bash
check-license-headers:
@! find . -path ./vendor -prune -o -name '*.go' -print | grep -v -F -f license.exceptions | while read fn;\
do [[ `$(call LICENSE_HASH_CODE, $$fn)` == $(LICENSE_HASH) ]] || echo $$fn; done | grep '.'
######################
# Build dependencies #
######################

26
NOTES
View file

@ -1,26 +0,0 @@
Index Column
┌───────────▼────────────────────────────┐
│0000000000000000000000000000000000000000│
│0000000000000000000000000000000000000000│
│0000000000000000000000000000000000000000│
Row──▶0000000000000000000000000000000000000000│
│0000000000000000000000000000000000000000│
│────────────────────────────────────────┤
│0000000000000000000000000000000000000000│
│0000000000000000000000000000000000000000│
│0000000000000000000000000000000000000000│
│0000000000000000000000000000000000000000│
│0000000000000000000000000000000000000000│
│────────────────────────────────────────┤
F ▶│0000000000000000000000000000000000000000│
i ││0000000000000000000000000000000000000000│
e ││0000000000000000000000000000000000000000│
l ││0000000000000000000000000000000000000000│
d ▶│0000000000000000000000000000000000000000│
└────────────────────────────────────────┘
▲───────────▲
Shard
Fragment=intersection of field & shard

15
NOTICE
View file

@ -1,23 +1,12 @@
Software license
================
Copyright (C) 2017-2018 Pilosa Corp. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License").
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
Copyright (C) 2017-2021 Molecula Corp. All rights reserved.
Third-party software licenses
=============================
The file /pilosa/lru/lru.go contains a redistribution of lru
The file /lru/lru.go contains a redistribution of lru
(github.com/golang/groupcache/lru); the license follows:
Copyright 2013 Google Inc.

302
cmd/random-query/main.go
View file

@ -18,34 +18,47 @@ import (
"context"
"flag"
"fmt"
"io/ioutil"
"math"
"math/rand"
nethttp "net/http"
"os"
"strconv"
"strings"
"time"
"github.com/molecula/featurebase/v2"
"github.com/gogo/protobuf/proto"
pilosa "github.com/molecula/featurebase/v2"
"github.com/molecula/featurebase/v2/client"
"github.com/molecula/featurebase/v2/http"
"github.com/molecula/featurebase/v2/pb"
"github.com/molecula/featurebase/v2/pql"
"github.com/molecula/featurebase/v2/vprint"
. "github.com/molecula/featurebase/v2/vprint" // nolint:staticcheck
"github.com/pkg/errors"
vegeta "github.com/tsenart/vegeta/v12/lib"
)
// RandomQueryConfig
type RandomQueryConfig struct {
// user facing flags
HostPort string // -hostport
TreeDepth int // -d
QueryCount int // -n
Verbose bool // -v
VeryVerbose bool // -V
TimeFromArg string // --time.from
TimeToArg string // --time.to
TimeFrom time.Time // parsed time
TimeTo time.Time // parsed time
TimeRange int64 // hours between parsed times
HostPort string
TreeDepth int
QueryCount int
Verbose bool
GenerateOnly bool
NumRuns int
TimeFromArg string
TimeToArg string
TimeFrom time.Time
TimeTo time.Time
TimeRange int64
Index string
QPM int
Seed int
SrcFile string
Duration time.Duration
Target vegeta.Target
IndexMap map[string]*Features
@ -93,12 +106,18 @@ var defaultStartTime = defaultEndTime.Add(-5 * 365 * 24 * time.Hour)
// call DefineFlags before myflags.Parse()
func (cfg *RandomQueryConfig) DefineFlags(fs *flag.FlagSet) {
fs.StringVar(&cfg.HostPort, "hostport", "localhost:10101", "host:port of pilosa to run random queries on.")
fs.IntVar(&cfg.TreeDepth, "d", 4, "depth of random queries to generate.")
fs.IntVar(&cfg.QueryCount, "n", 100, "number of random queries to generate. Set to 0 for inifinite queries.")
fs.IntVar(&cfg.TreeDepth, "max-nesting-depth", 1, "depth of random queries to generate.")
fs.IntVar(&cfg.QueryCount, "queries-per-request", 1, "number of random queries to generate")
fs.IntVar(&cfg.NumRuns, "number-reports", 1, "number of reports generate ")
fs.IntVar(&cfg.Seed, "seed", int(time.Now().Unix()), "RNG seed, defaults to current time")
fs.DurationVar(&cfg.Duration, "metrics-period", 10*time.Second, "size of time window on metrics reporting, default 10s")
fs.StringVar(&cfg.Index, "index", "i", "index to run queries against")
fs.IntVar(&cfg.QPM, "qpm", 10, "number of current requests per minute to simulate, default 10")
fs.BoolVar(&cfg.Verbose, "v", false, "show queries as they are generated")
fs.BoolVar(&cfg.VeryVerbose, "V", false, "show query results")
fs.BoolVar(&cfg.GenerateOnly, "generate-only", false, "only generate do not run package")
fs.StringVar(&cfg.TimeFromArg, "time.from", defaultStartTime.Format(time.RFC3339), "starting time for time fields (format: 2006-01-02T15:04:05Z07:00)")
fs.StringVar(&cfg.TimeToArg, "time.to", defaultEndTime.Format(time.RFC3339), "starting time for time fields (format: 2006-01-02T15:04:05Z07:00)")
fs.StringVar(&cfg.SrcFile, "query-file", "", "use pql contained in this file for query batch instead of generating")
}
// call c.ValidateConfig() after myflags.Parse()
@ -123,6 +142,9 @@ func (c *RandomQueryConfig) ValidateConfig() error {
return fmt.Errorf("time.to (%s) should be at least one hour after time.from (%s)",
c.TimeToArg, c.TimeFromArg)
}
if c.QPM <= 0 {
return fmt.Errorf("-qpm must be positive")
}
return nil
}
@ -144,7 +166,6 @@ func main() {
fmt.Fprintf(os.Stderr, "%s error: %s\n", ProgramName, err)
os.Exit(1)
}
err = cfg.Run()
if err != nil {
@ -159,76 +180,24 @@ func (cfg *RandomQueryConfig) Run() (err error) {
if err != nil {
return err
}
ctx := context.Background()
totalQ := 0
loops := 0
t0 := time.Now()
report := func() {
dur := time.Since(t0)
if dur > 0 {
qps := 1e9 * float64(totalQ) / float64(dur)
AlwaysPrintf("totalQueries run: %v elapsed: %v qps: %0.02f", totalQ, dur, qps)
} else {
AlwaysPrintf("totalQueries run: %v elapsed: %v qps: N/A", totalQ, dur)
}
}
defer report()
NewSetup:
err = cfg.Setup(cli)
if err != nil {
return err
}
rate := vegeta.Rate{Freq: cfg.QPM, Per: time.Minute}
duration := cfg.Duration
targeter := vegeta.NewStaticTargeter(cfg.Target)
attacker := vegeta.NewAttacker()
if len(cfg.IndexMap) == 0 {
return fmt.Errorf("no rows to query")
}
var indexes []string
for index := range cfg.IndexMap {
indexes = append(indexes, index)
}
for j := 0; ; j++ {
if cfg.QueryCount > 0 {
if j >= cfg.QueryCount {
break
}
} else {
// else keep doing queries forever...
if loops > 0 && loops%500 == 0 {
// ...but account for any new data arrived by getting
// the schema and rows again every so often.
loops++
goto NewSetup
}
for i := 0; i < cfg.NumRuns; i++ {
vprint.VV("================")
var metrics vegeta.Metrics
for res := range attacker.Attack(targeter, rate, duration, "Big Bang!") {
metrics.Add(res)
}
if totalQ > 0 && totalQ%100 == 0 {
report()
}
index := indexes[cfg.Rnd.Intn(len(indexes))]
pql, err := cfg.GenQuery(index)
PanicOn(err)
if cfg.Verbose {
fmt.Printf("pql = '%v'\n", pql)
}
// Query node0.
res, err := cli.Query(ctx, index, &pilosa.QueryRequest{Index: index, Query: pql})
if err != nil {
AlwaysPrintf("QUERY FAILED! queries before this=%v; err = '%v', pql='%v'", loops, err, pql)
return err
}
if cfg.VeryVerbose {
fmt.Printf("success on pql = '%v'; res='%v'\n", pql, res.Results[0])
}
totalQ++
loops++
metrics.Close()
rpt := vegeta.NewTextReporter(&metrics)
rpt(os.Stdout)
}
return nil
@ -238,6 +207,7 @@ type Features struct {
Slc []IndexFieldRow
Ranges []IndexFieldRange
Distinctables []IndexFieldRange
Stores []IndexFieldRow
SlcWeight int
RangeWeight int
}
@ -283,7 +253,7 @@ func (fea *IndexFieldRow) Query(cfg *RandomQueryConfig) *Tree {
endTime.Format(pilosaTimeFmt))
}
if fea.IsRowKey {
return &Tree{S: fmt.Sprintf("Row(%v='%v'%s)", fea.Field, fea.RowKey, fromTo)}
return &Tree{S: fmt.Sprintf(`Row(%v="%v"%s)`, fea.Field, fea.RowKey, fromTo)}
}
return &Tree{S: fmt.Sprintf("Row(%v=%v%s)", fea.Field, fea.RowID, fromTo)}
}
@ -341,65 +311,131 @@ func (i *IndexFieldRange) Query(cfg *RandomQueryConfig) *Tree {
// and spits back a PQL query
//
func (cfg *RandomQueryConfig) Setup(api API) (err error) {
if cfg.SrcFile != "" {
return cfg.buildPayload()
}
ctx := context.Background()
cfg.Info, err = api.Schema(ctx)
if err != nil {
return err
}
foundIntField := false
any := false
for i, ii := range cfg.Info {
_ = i
for k, fld := range ii.Fields {
_ = k
switch fld.Options.Type {
case "set", "mutex", "time":
pql := fmt.Sprintf("Rows(%v)", fld.Name)
if ii.Name == cfg.Index {
any = true
_ = i
for k, fld := range ii.Fields {
_ = k
switch fld.Options.Type {
case "set", "mutex", "time":
pql := fmt.Sprintf("Rows(%v)", fld.Name)
res, err := api.Query(ctx, ii.Name, &pilosa.QueryRequest{Index: ii.Name, Query: pql})
PanicOn(err)
if cfg.VeryVerbose {
fmt.Printf("success on pql = '%v'; res='%v'\n", pql, res.Results[0])
res, err := api.Query(ctx, ii.Name, &pilosa.QueryRequest{Index: ii.Name, Query: pql})
PanicOn(err)
switch x := res.Results[0].(type) {
case *pilosa.RowIdentifiers:
cfg.AddResponse(ii.Name, fld.Name, x, fld.Options.Type == "time", fld.Options.Type == "set")
case pilosa.RowIdentifiers:
cfg.AddResponse(ii.Name, fld.Name, &x, fld.Options.Type == "time", fld.Options.Type == "set")
}
case "int":
foundIntField = true
fallthrough
case "decimal":
cfg.AddIntField(ii.Name, fld.Name, fld.Options.Min, fld.Options.Max, fld.Options.Scale, fld.Options.Type == "decimal")
default:
AlwaysPrintf("ignoring field %q: unhandled type %q\n", fld.Name, fld.Options.Type)
}
// if the option is set to use RowKeys, then must get the Keys instead of the Rows from the RowIdentifiers.
// e.g.
// success on pql = 'Rows(aba)'; res='&pilosa.RowIdentifiers{Rows:[]uint64(nil), Keys:[]string{"aba1", "aba2"}
// success on pql = 'Rows(f)'; res='pilosa.RowIdentifiers{Rows:[]uint64{0x1}, Keys:[]string(nil), field:"f"}'
switch x := res.Results[0].(type) {
case *pilosa.RowIdentifiers:
// internalClient gets this
cfg.AddResponse(ii.Name, fld.Name, x, fld.Options.Type == "time")
case pilosa.RowIdentifiers:
// test gets this
cfg.AddResponse(ii.Name, fld.Name, &x, fld.Options.Type == "time")
}
case "int":
foundIntField = true
fallthrough // I bet you thought you'd never see this used
case "decimal":
cfg.AddIntField(ii.Name, fld.Name, fld.Options.Min, fld.Options.Max, fld.Options.Scale, fld.Options.Type == "decimal")
default:
AlwaysPrintf("ignoring field %q: unhandled type %q\n", fld.Name, fld.Options.Type)
}
}
}
if !any {
return errors.New(fmt.Sprintf("index %v not found", cfg.Index))
}
cfg.BitmapFunc = []string{"Union", "Intersect", "Xor", "Not", "Difference"}
if foundIntField {
cfg.BitmapFunc = append(cfg.BitmapFunc, "Distinct")
}
seed := int64(42)
cfg.Rnd = rand.New(rand.NewSource(seed))
return nil
cfg.Rnd = rand.New(rand.NewSource(int64(cfg.Seed)))
return cfg.buildPayload()
}
func (cfg *RandomQueryConfig) AddResponse(index, field string, x *pilosa.RowIdentifiers, hasTime bool) {
func (cfg *RandomQueryConfig) buildPayload() error {
var request strings.Builder
if cfg.SrcFile != "" {
b, err := ioutil.ReadFile(cfg.SrcFile) // just pass the file name
if err != nil {
return err
}
request.WriteString(string(b))
} else {
for i := 0; i < cfg.QueryCount; i++ {
pql, err := cfg.GenQuery(cfg.Index)
if err != nil {
return err
}
request.WriteString(pql)
}
}
//TODO (twg) tls support
path := fmt.Sprintf("http://%s/index/%s/query", cfg.HostPort, cfg.Index)
header := nethttp.Header{
"Content-Type": []string{"application/x-protobuf"},
"Accept": []string{"application/x-protobuf"},
"PQL-Version": []string{client.PQLVersion},
}
req := &pb.QueryRequest{
Query: request.String(),
}
vprint.VV("%v", request.String())
if cfg.GenerateOnly {
// just output the PQL and exit
os.Exit(0)
}
payload, err := proto.Marshal(req)
if err != nil {
return errors.Wrap(err, "marshaling request to protobuf")
}
cfg.Target = vegeta.Target{
Method: "POST",
URL: path,
Body: payload,
Header: header,
}
return nil
}
func (cfg *RandomQueryConfig) AddResponse(index, field string, x *pilosa.RowIdentifiers, hasTime bool, isSet bool) {
storeID := uint64(0)
for _, rowID := range x.Rows {
cfg.AddFeature(index, field, rowID, "", false, hasTime)
storeID = rowID
}
storeKey := ""
for _, rowKey := range x.Keys {
cfg.AddFeature(index, field, 0, rowKey, true, hasTime)
storeKey = rowKey
}
idx := cfg.IndexMap[index]
if isSet {
if storeID > 0 {
idx.Stores = append(idx.Stores, IndexFieldRow{
Index: index,
Field: field,
RowID: storeID + 1,
})
}
if storeKey != "" {
idx.Stores = append(idx.Stores, IndexFieldRow{
Index: index,
Field: field,
RowKey: storeKey + "_1",
IsRowKey: true,
})
}
}
}
@ -411,10 +447,6 @@ func (cfg *RandomQueryConfig) AddIntField(index, field string, min, max pql.Deci
f = &Features{}
cfg.IndexMap[index] = f
}
if min.Scale != scale || max.Scale != scale {
PanicOn(fmt.Sprintf("scale error; min scale %d, max scale %d, field scale %d, assumed they'd be equal",
min.Scale, max.Scale, scale))
}
effectiveRange := uint64(max.Value) - uint64(min.Value) + 1
// if you have INT64_MAX and INT64_MIN, effectiveRange is 1<<64, which
@ -449,11 +481,32 @@ func (cfg *RandomQueryConfig) AddIntField(index, field string, min, max pql.Deci
}
func (cfg *RandomQueryConfig) GenQuery(index string) (pql string, err error) {
tree := cfg.GenTree(index, cfg.TreeDepth)
pql = tree.ToPQL()
// avoid using too much bandwidth, just count the final bitmap.
dice := cfg.Rnd.Intn(9)
if dice == 3 { // 1 in 9 of getting a store
idx := cfg.IndexMap[index]
if len(idx.Stores) > 0 {
i := cfg.Rnd.Intn(len(idx.Stores))
fr := idx.Stores[i]
var key string
if fr.IsRowKey {
key = fmt.Sprintf(`%v="%v"`, fr.Field, fr.RowKey)
c := strings.LastIndex(fr.RowKey, "_")
n, err := strconv.Atoi(fr.RowKey[c+1:])
PanicOn(err)
n += 1
fr.RowKey = fmt.Sprintf("%v%v", fr.RowKey[:c+1], n)
} else {
key = fmt.Sprintf("%v=%v", fr.Field, fr.RowID)
fr.RowID = fr.RowID + 1
}
idx.Stores[i] = fr
pql = fmt.Sprintf("Store(%v,%v)Count(Row(%v))", pql, key, key)
return
}
}
pql = fmt.Sprintf("Count(%v)", pql)
return
}
@ -522,7 +575,6 @@ func (cfg *RandomQueryConfig) GenTree(index string, depth int) (tr *Tree) {
}
func (tr *Tree) ToPQL() (s string) {
if len(tr.Chd) == 0 {
// leaf
return tr.S

8
cmd/random-query/main_test.go
View file

@ -20,7 +20,7 @@ import (
"strconv"
"testing"
"github.com/molecula/featurebase/v2"
pilosa "github.com/molecula/featurebase/v2"
"github.com/molecula/featurebase/v2/boltdb"
"github.com/molecula/featurebase/v2/http"
"github.com/molecula/featurebase/v2/server"
@ -65,8 +65,8 @@ func Test_RandomQuery(t *testing.T) {
ctx := context.Background()
indexes := []string{"rick", "morty"}
fieldName := []string{"f", "flying_car"}
indexes := []string{"rick"}
fieldName := []string{"f"}
idx := make([]*pilosa.Index, len(indexes))
field := make([]*pilosa.Field, len(indexes))
@ -144,7 +144,7 @@ func Test_RandomQuery(t *testing.T) {
//qcx.Reset()
}
// end of setup.
cfg.Index = indexes[0]
PanicOn(cfg.Setup(wrapApiToInternalClient(nodes[0].API)))
for j := 0; j < 4; j++ {

1
go.mod
View file

@ -43,6 +43,7 @@ require (
github.com/spf13/pflag v1.0.5
github.com/spf13/viper v1.7.1
github.com/stretchr/testify v1.7.0
github.com/tsenart/vegeta/v12 v12.8.4
github.com/uber/jaeger-client-go v2.25.0+incompatible
github.com/uber/jaeger-lib v2.4.0+incompatible // indirect
github.com/zeebo/blake3 v0.1.1

41
go.sum
View file

@ -25,6 +25,7 @@ github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAE
github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA=
github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8=
github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw=
github.com/alecthomas/jsonschema v0.0.0-20180308105923-f2c93856175a/go.mod h1:qpebaTNSsyUn5rPSJMsfqEtDw71TTggXM6stUDI16HA=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
@ -42,8 +43,11 @@ github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b h1:AP/Y7sqYicnjGDfD5VcY4CIfh1hRXBUavxrvELjTiOE=
github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b/go.mod h1:ac9efd0D1fsDb3EJvhqgXRbFx7bs2wqZ10HQPeU8U/Q=
github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs=
github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
github.com/c2h5oh/datasize v0.0.0-20171227191756-4eba002a5eae/go.mod h1:S/7n9copUssQ56c7aAgHqftWO4LTf4xY6CGWt8Bc+3M=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
@ -75,6 +79,9 @@ github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f h1:U5y3Y5UE0w7amNe
github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f/go.mod h1:xH/i4TFMt8koVQZ6WFms69WAsDWr2XsYL3Hkl7jkoLE=
github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
github.com/dgryski/go-gk v0.0.0-20140819190930-201884a44051 h1:ByJUvQYyTtNNCVfYNM48q6uYUT4fAlN0wNmd3th4BSo=
github.com/dgryski/go-gk v0.0.0-20140819190930-201884a44051/go.mod h1:qm+vckxRlDt0aOla0RYJJVeqHZlWfOm2UIxHaqPB46E=
github.com/dgryski/go-lttb v0.0.0-20180810165845-318fcdf10a77/go.mod h1:Va5MyIzkU0rAM92tn3hb3Anb7oz7KcnixF49+2wOMe4=
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
@ -120,6 +127,24 @@ github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I=
github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
github.com/gonum/blas v0.0.0-20181208220705-f22b278b28ac h1:Q0Jsdxl5jbxouNs1TQYt0gxesYMU4VXRbsTlgDloZ50=
github.com/gonum/blas v0.0.0-20181208220705-f22b278b28ac/go.mod h1:P32wAyui1PQ58Oce/KYkOqQv8cVw1zAapXOl+dRFGbc=
github.com/gonum/diff v0.0.0-20181124234638-500114f11e71 h1:BE6g8oinc3Ek2elIHq+uDOiZgX3/ODi+EerJ48yrrKc=
github.com/gonum/diff v0.0.0-20181124234638-500114f11e71/go.mod h1:22dM4PLscQl+Nzf64qNBurVJvfyvZELT0iRW2l/NN70=
github.com/gonum/floats v0.0.0-20181209220543-c233463c7e82 h1:EvokxLQsaaQjcWVWSV38221VAK7qc2zhaO17bKys/18=
github.com/gonum/floats v0.0.0-20181209220543-c233463c7e82/go.mod h1:PxC8OnwL11+aosOB5+iEPoV3picfs8tUpkVd0pDo+Kg=
github.com/gonum/integrate v0.0.0-20181209220457-a422b5c0fdf2 h1:GUSkTcIe1SlregbHNUKbYDhBsS8lNgYfIp4S4cToUyU=
github.com/gonum/integrate v0.0.0-20181209220457-a422b5c0fdf2/go.mod h1:pDgmNM6seYpwvPos3q+zxlXMsbve6mOIPucUnUOrI7Y=
github.com/gonum/internal v0.0.0-20181124074243-f884aa714029 h1:8jtTdc+Nfj9AR+0soOeia9UZSvYBvETVHZrugUowJ7M=
github.com/gonum/internal v0.0.0-20181124074243-f884aa714029/go.mod h1:Pu4dmpkhSyOzRwuXkOgAvijx4o+4YMUJJo9OvPYMkks=
github.com/gonum/lapack v0.0.0-20181123203213-e4cdc5a0bff9 h1:7qnwS9+oeSiOIsiUMajT+0R7HR6hw5NegnKPmn/94oI=
github.com/gonum/lapack v0.0.0-20181123203213-e4cdc5a0bff9/go.mod h1:XA3DeT6rxh2EAE789SSiSJNqxPaC0aE9J8NTOI0Jo/A=
github.com/gonum/mathext v0.0.0-20181121095525-8a4bf007ea55 h1:Ajwn2ENgC/pKtVat0LEHEWNa4a4VGyYJ1feGSccOzFU=
github.com/gonum/mathext v0.0.0-20181121095525-8a4bf007ea55/go.mod h1:fmo8aiSEWkJeiGXUJf+sPvuDgEFgqIoZSs843ePKrGg=
github.com/gonum/matrix v0.0.0-20181209220409-c518dec07be9 h1:V2IgdyerlBa/MxaEFRbV5juy/C3MGdj4ePi+g6ePIp4=
github.com/gonum/matrix v0.0.0-20181209220409-c518dec07be9/go.mod h1:0EXg4mc1CNP0HCqCz+K4ts155PXIlUywf0wqN+GfPZw=
github.com/gonum/stat v0.0.0-20181125101827-41a0da705a5b h1:fbskpz/cPqWH8VqkQ7LJghFkl2KPAiIFUHrTJ2O3RGk=
github.com/gonum/stat v0.0.0-20181125101827-41a0da705a5b/go.mod h1:Z4GIJBJO3Wa4gD4vbwQxXXZ+WHmW6E9ixmNrwvs0iZs=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo=
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
@ -184,6 +209,8 @@ github.com/improbable-eng/grpc-web v0.13.0 h1:7XqtaBWaOCH0cVGKHyvhtcuo6fgW32Y10y
github.com/improbable-eng/grpc-web v0.13.0/go.mod h1:6hRR09jOEG81ADP5wCQju1z71g6OL4eEvELdran/3cs=
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/influxdata/tdigest v0.0.0-20180711151920-a7d76c6f093a h1:vMqgISSVkIqWxCIZs8m1L4096temR7IbYyNdMiBxSPA=
github.com/influxdata/tdigest v0.0.0-20180711151920-a7d76c6f093a/go.mod h1:9GkyshztGufsdPQWjH+ifgnIr3xNUL5syI70g2dzU1o=
github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo=
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
@ -210,6 +237,8 @@ github.com/lib/pq v1.8.0 h1:9xohqzkUwzR4Ga4ivdTcawVS89YSDVxXMa3xJX3cGzg=
github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM=
github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
@ -217,6 +246,7 @@ github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzp
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
github.com/miekg/dns v1.1.17/go.mod h1:WgzbA6oji13JREwiNsRDNfl7jYdPnmz+VEuLrA+/48M=
github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
@ -321,6 +351,8 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An
github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk=
github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
github.com/streadway/quantile v0.0.0-20150917103942-b0c588724d25 h1:7z3LSn867ex6VSaahyKadf4WtSsJIgne6A1WLOAGM8A=
github.com/streadway/quantile v0.0.0-20150917103942-b0c588724d25/go.mod h1:lbP8tGiBjZ5YWIc2fzuRpTaz0b/53vT6PEs3QuAWzuU=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@ -338,6 +370,9 @@ github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcy
github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ=
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/tsenart/go-tsz v0.0.0-20180814232043-cdeb9e1e981e/go.mod h1:SWZznP1z5Ki7hDT2ioqiFKEse8K9tU2OUvaRI0NeGQo=
github.com/tsenart/vegeta/v12 v12.8.4 h1:UQ7tG7WkDorKj0wjx78Z4/vsMBP8RJQMGJqRVrkvngg=
github.com/tsenart/vegeta/v12 v12.8.4/go.mod h1:ZiJtwLn/9M4fTPdMY7bdbIeyNeFVE8/AHbWFqCsUuho=
github.com/uber/jaeger-client-go v2.25.0+incompatible h1:IxcNZ7WRY1Y3G4poYlx24szfsn/3LvK9QHCq9oQw8+U=
github.com/uber/jaeger-client-go v2.25.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk=
github.com/uber/jaeger-lib v2.4.0+incompatible h1:fY7QsGQWiCt8pajv4r7JEvmATdCVaWxXbjwyYwsNaLQ=
@ -367,6 +402,7 @@ golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnf
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
@ -416,6 +452,7 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d h1:20cMwl2fHAzkJMEA+8J4JgqBQcQGzbisXo31MIeenXI=
@ -446,6 +483,7 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@ -486,6 +524,7 @@ golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgw
golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
@ -559,6 +598,8 @@ modernc.org/mathutil v1.0.0 h1:93vKjrJopTPrtTNpZ8XIovER7iCIH1QU7wNbOQXC60I=
modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k=
modernc.org/strutil v1.0.0 h1:XVFtQwFVwc02Wk+0L/Z/zDDXO81r5Lhe6iMKmGX3KhE=
modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs=
pgregory.net/rapid v0.3.3 h1:jCjBsY4ln4Atz78QoBWxUEvAHaFyNDQg9+WU62aCn1U=
pgregory.net/rapid v0.3.3/go.mod h1:UYpPVyjFHzYBGHIxLFoupi8vwk6rXNzRY9OMvVxFIOU=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=

373
install/featurebase.conf Normal file
View file

@ -0,0 +1,373 @@
# FEATUREBASE HOST CONFIGURATION
#
# Uncomment when/where appropriate
# ==============================================================================
# Use advertise to specify the address advertised by the server to other nodes
# in the cluster and to clients via /status endpoint. Host defaults to IP
# address represented by bind parameter with network port.
#
# advertise = <IP Address>:10101
# advertise-grpc = <IP Address>:20101
# "long-query-time" represents duration of time that will trigger log and stat
# message for queries longer than X time. Ex. "1m30s" 1 minute 30 seconds
#
# long-query-time = "10s"
# Unique name for node in cluster. This is just a human-readable label for
# convenience and not used by any underlying logic.
#
# name = "featurebase1"
# Host:Port where Featurebase server listens for HTTP requests.
# Default is localhost:10101
#
# bind = "localhost:10101"
# The address and port featurebase will listen to for all GRPC connections
# Ex. python-molecula, grafana for queries, etc.
#
# bind-grpc = "0.0.0.0:20101"
# Directory to store Featurebase data files
data-dir = "/var/lib/molecula"
# ==============================================================================
# CORS (Cross-Origin Resource Sharing) Allowed Origins
# List of allowed origin URIs for CORS
#
# [handler]
# allowed-origins = ["https://myapp.com", "https://myapp.org"]
# Path to the log file
log-path = "/var/log/molecula/featurebase.log"
# Verbose - Enable verbose logging. Valid options are true or false.
# Set to true only when debugging as directed by Molecula engineers.
#
# verbose = true
# Soft limit on max number of files featurebase will keep open simultaneously.
# When past this limit, featurebase will only keep files open for as long as is
# needed to write updates.
#
# max-file-count = 900000
# Maximum number of active memory maps featurebase will use for fragment files.
# Actual total usage may be slightly higher.
# Best practice is to set this to ~10% lower than your system's max map count.
# See sysctl vm.max_map_count in Linux.
#
# max-map-count = 900000
# Max Writes Per Request - Max number of mutating commands allowed per request.
# This includes Set, Clear, ClearRow, and Store
#
# max-writes-per-request = 5000
# The following option sets the maximum number of queries that are maintained
# for the /query-history endpoint.
# This parameter is per-node, and the result combines the history from all nodes.
#
# query-history-length = 100
# External database to connect to for `ExternalLookup` queries.
# lookup-db-dsn = "postgres://localhost:5432/db"
# ==============================================================================
# For cluster stanza, "name" represents name for cluster. Must be same on all
# nodes in cluster. "replicas" represents number of hosts each piece of data
# should be stored on. Must be greater than or equal to 1 & less than or equal
# to number of nodes in cluster.
# [cluster]
# name = "cluster1"
# replicas = 1
# ==============================================================================
# [etcd]
# etcd is the tool Featurebase uses for node-to-node, intra-cluster
# communication. etcd is embedded in the featurebase cluster rather than
# running as a separate instance.
# It's important to configure this correctly for your network and nodes, and
# that it is consistent across all nodes.
#
# The easiest setup can be used when all nodes can reach all other nodes via a
# local subnet:
# listen-peer-address = advertise-peer-address
# = (what's in the initial-cluster-list)
# = the nodes ip address (which can be reached by every
# other node
# (localhost:10401 would not work for this, as each node can't reach that)
#
# If each node is separated by a proxy, or must be reached via url / dns, you
# will need to use a more complicated setup:
# listen-peer-address = the nodes local ip address
# (specific ip, localhost, or 0.0.0.0 for all
# local ip's)
# advertise-peer-address = the nodes ip address, reachable by all other nodes
# (This address should also be included in
# initital-cluster-list)
# in this case, you specify a different url/ip for listen-peer and
# advertise-peer. E.g. you specify 0.0.0.0 for listen, or (like in their case)
# you use a url for advertise. In each of these cases, you should set listen
# to the local ip, and you set advertise = to how each other node connects to
# this node, and you also use this same address in the initial cluster.
# The key here is that initial-cluster has to include the same node name and
# advertise-peer address as the node it's on (edited)
# for additional assistance, and for help with config issues,
# see https://etcd.io/docs/v3.5/faq/ cluster-url - URL of existing cluster
# that a new node should join when adding nodes to cluster.
#
# cluster-url = "http://localhost:10401"
# Address and port to bind to for client communication
# listen-client-address = "http://localhost:10401"
# Address and port to bind to for peer communication
# listen-peer-address = "http://localhost:10301"
# Comma-separated list of node=address pairs that makes up initial cluster when
# first started. In each pair, "node" value (left side of = ) should match
# name of node specified by "name" configuration parameter
#
# initial-cluster = "featurebase1=http://localhost:10301"
# ==============================================================================
# Profile Block Rate - Block Rate is passed directly to Go's
# runtime.SetBlockProfileRate. Goroutine blocking events will be sampled at 1
# per rate nanoseconds. A value of "1" samples every event, and 0 disables
# profiling.
#
# block-rate = 10000000
# Profile Mutex Fraction - Mutex Fraction is passed directly to Go's
# runtime.SetMutexProfileFraction. 1/ fraction of events will be sampled.
#
# mutex-fraction = 100
# ==============================================================================
# PostgreSQL Section
# [postgres]
#
# Endpoint Bind - Address to bind a PostgreSQL wire protocol endpoint.
# No PostgreSQL endpoint will be exposed unless a bind address is specified.
# Requires Molecula v3.0 or newer.
#
# bind = "localhost:55432"
# The PostgreSQL endpoint has support for a connection limit.
# This is generally not necessary, so it is disabled by default.
#
# connection-limit = 10000
# PostgreSQL Max Startup Packet Size - By default, the postgres endpoint
# uses an 8 MiB limit on incoming PostgreSQL startup packets. This should
# typically be sufficient, but may be exceeded if a client sends an unusually
# large amount of configuration data. Oversized startup packets are typically
# caused by connecting with a different protocol, e.g. HTTP.
#
# max-startup-size = 10000000
# PostgreSQL Timeouts
# In order to detect stalled clients, the PostgreSQL endpoint has connection
# read and write timeouts. There is also a startup timeout, which is used for
# connection setup. The read timeout does not impact idle connections. Idle
# connections will only be closed by the server if TCP keepalive reports a
# break in the connection. TCP keepalives use the default configuration
# provided by the host.
# Caution: Due to a limitation of the PostgreSQL wire protocol,
# raising the write timeout may delay the shutdown of a featurebase node.
#
# startup-timeout = "20s"
# read-timeout = "20s"
# write-timeout - "20s"
# Postgres Endpoint TLS - TLS configuration for the PostgreSQL endpoint is
# structured the same as the TLS configuration for Featurebase's other endpoints,
# but placed under [postgres.tls]. If TLS is configured on the postgres endpoint,
# Featurebase will reject unsecured connections.
# [postgres.tls]
# certificate = "/srv/pilosa/certs/server.crt"
# key = "/srv/pilosa/certs/server.key"
# ca-certificate = "/srv/pilosa/certs/ca.crt"
# enable-client-verification = true
# ==============================================================================
# Usage Duty Cycle - Featurebase maintains a disk/memory usage cache that is
# calculated periodically in the background and accessed by the UI/usage
# endpoint. Since this disk scan can take a long and unpredictable amount of
# time, its timing behavior is specified in a relative, rather than absolute
# sense. That is, the duty cycle sets the percentage of time that is spent
# recalculating this cache. This setting affects the results received from
# the "/ui/usage" http endpoint, as well as all data file and memory usage
# values and graphs on the webui "tables" page
# Special considerations:
# * If disk usage can be calculated quickly (less than 5 seconds), fresh
# results will be calculated when accessed
# * When disk usage takes longer to calculate, there is a minimum of one
# hour wait between cache recalculations
# Setting this value to 0 will completely disable the calculation of disk usage
#
# usage-duty-cycle = 20
# ==============================================================================
# Use [metric] stanza to define attributes for monitoring.
# [metric]
# Specify which service to use for collecting metrics. Valid options are:
# "statsd", "expvar", "prometheus", "none"
#
# service = "prometheus"
# Remote host to send statsd metrics to.
# host = "localhost:8125"
# The interval to send statsd metrics.
# poll-interval = "10s"
# Debugging flag to enable to send diagnostic information to Featurebase
# developers.
#
# diagnostics = false
# ==============================================================================
# TLS Certificate Section - Path to TLC certificate used for service HTTPS.
# Suffix should contain .crt or .pem
#
# [tls]
# certificate = "/srv/pilosa/certs/server.crt"
# TLS Certificate Key - Path to TLS certificate key for HTTPS. Suffix should
# be .key
#
# key = "/srv/pilosa/certs/server.key"
# ==============================================================================
# Tracing Section
# [tracing]
#
# Jaeger sampler type. Valid options are: "const, "probabilistic", "ratelimiting",
# or "remote". Set to 'off' to disable tracing completely.
#
# sampler-type = "remote"
# Jaeger sampler parameter (number)
# sampler-param = 0.001
# Tracing Agent Host:Port
# agent-host-port = "localhost:6831"
# ==============================================================================
# Configuration for the RBF storage format.
# [rbf]
# Maximum size for each RBF database file.
# Allocates virtual memory but does not preallocate physical disk space.
# If you get into the range where you have 16000 shards on a single node
# (across all indexes), you will need to lower this in order to not run out of
# virtual address space.
#
# max-db-size = 4294967296
# Maximum size for each RBF WAL file.
# Allocates virtual memory but does not preallocate physical disk space.
# This is the same as max-db-size, but for the write-ahead log. If you set it
# smaller, set max-wal-checkpoint-size to 1/2 of this (we will likely
# condense these options in the future).
#
# max-wal-size = 4294967296
# Minimum WAL size before WAL pages can be copied to the main database file.
# min-wal-checkpoint-size = 1048576
# Maximum WAL size before transactions are halted to copy WAL pages to the
# main database file.
#
# max-wal-checkpoint-size = 2147483648
# ==============================================================================
# [storage]
# Sync all changes to the file system.
# Should not be changed in production systems unless you know what you are
# doing - Should always be on unless testing or possibly while performing a
# bulk import and you are not worried about data loss
#
# fsync = true
# ==============================================================================

13
install/featurebase.debian.service Normal file
View file

@ -0,0 +1,13 @@
[Unit]
Description="Service for FeatureBase"
After=network.target
[Service]
RestartSec=30
Restart=on-failure
EnvironmentFile=
User=molecula
ExecStart=/usr/local/bin/featurebase server -c /etc/featurebase.conf
[Install]
WantedBy=multi-user.target

12
install/featurebase.redhat.service Normal file
View file

@ -0,0 +1,12 @@
[Unit]
Description="Service for FeatureBase"
[Service]
RestartSec=30
Restart=on-failure
EnvironmentFile=
User=molecula
ExecStart=/usr/local/bin/featurebase server -c /etc/featurebase.conf
[Install]
WantedBy=multi-user.target

18
install/test_installation.Dockerfile Normal file
View file

@ -0,0 +1,18 @@
FROM fedora:rawhide
RUN yum -y install systemd procps
ARG release_tarball
COPY $release_tarball .
COPY test_installation.sh test_installation.sh
RUN mkdir install && \
tar -xf $release_tarball -C install --strip-components 1 && \
cd install && \
cp featurebase /usr/local/bin/featurebase && \
cp featurebase.redhat.service /etc/systemd/system/featurebase.service && \
cp featurebase.conf /etc/featurebase.conf
CMD ["/bin/bash", "test_installation.sh"]

29
install/test_installation.sh Normal file
View file

@ -0,0 +1,29 @@
#!/bin/bash
tests_failed=0
# test validity of featurebase.service
systemd-analyze verify /etc/systemd/system/featurebase.service
exit_code=$?
if [ $exit_code -ne 0 ]; then
echo 'featurebase.redhat.service invalid'
tests_failed=1
fi
# test validity of featurebase.conf
mkdir /var/log/molecula
featurebase_bin='/usr/local/bin/featurebase'
config_file='/etc/featurebase.conf'
if ! $featurebase_bin -c $config_file holder /dev/null 2>&1; then
echo 'featurebase.conf is invalid'
tests_failed=1
fi
# print success if both passed
if [ $tests_failed -eq 0 ]; then
echo 'featurebase.redhat.service is valid'
echo 'featurebase.conf is valid'
fi
# tests_failed set to 0 if none of the tests failed
# otherwise set to non-zero
exit $tests_failed

2
roaring/btree_test.go
View file

@ -1,6 +1,6 @@
// Copyright 2014 The b Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// license that can be found in the NOTICE file.
package roaring